]>
Commit | Line | Data |
---|---|---|
316670eb A |
1 | /* |
2 | * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. | |
3 | * | |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ | |
d9a64523 | 5 | * |
316670eb A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
d9a64523 | 14 | * |
316670eb A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
d9a64523 | 17 | * |
316670eb A |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
d9a64523 | 25 | * |
316670eb A |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ | |
28 | ||
29 | /* | |
30 | * Called from a trigger. Actually takes the data from the different | |
31 | * modules and puts them in a buffer | |
32 | */ | |
33 | ||
34 | #include <mach/mach_types.h> | |
35 | #include <machine/machine_routines.h> | |
316670eb A |
36 | #include <kern/kalloc.h> |
37 | #include <kern/debug.h> /* panic */ | |
38 | #include <kern/thread.h> | |
39 | #include <sys/errno.h> | |
39037602 | 40 | #include <sys/vm.h> |
d9a64523 | 41 | #include <vm/vm_object.h> |
5ba3f43e A |
42 | #include <vm/vm_page.h> |
43 | #include <vm/vm_pageout.h> | |
316670eb | 44 | |
39037602 A |
45 | #include <kperf/action.h> |
46 | #include <kperf/ast.h> | |
316670eb | 47 | #include <kperf/buffer.h> |
316670eb | 48 | #include <kperf/callstack.h> |
316670eb | 49 | #include <kperf/context.h> |
39037602 A |
50 | #include <kperf/kdebug_trigger.h> |
51 | #include <kperf/kperf.h> | |
3e170ce0 | 52 | #include <kperf/kperf_kpc.h> |
f427ee49 | 53 | #include <kperf/kptimer.h> |
39037602 A |
54 | #include <kperf/pet.h> |
55 | #include <kperf/sample.h> | |
56 | #include <kperf/thread_samplers.h> | |
316670eb | 57 | |
39037602 | 58 | #define ACTION_MAX (32) |
316670eb | 59 | |
316670eb | 60 | /* the list of different actions to take */ |
d9a64523 | 61 | struct action { |
39236c6e | 62 | uint32_t sample; |
39037602 A |
63 | uint32_t ucallstack_depth; |
64 | uint32_t kcallstack_depth; | |
39236c6e A |
65 | uint32_t userdata; |
66 | int pid_filter; | |
316670eb A |
67 | }; |
68 | ||
69 | /* the list of actions */ | |
d9a64523 | 70 | static unsigned int actionc = 0; |
316670eb A |
71 | static struct action *actionv = NULL; |
72 | ||
3e170ce0 | 73 | /* should emit tracepoint on context switch */ |
39037602 | 74 | int kperf_kdebug_cswitch = 0; |
316670eb | 75 | |
5ba3f43e | 76 | bool |
d9a64523 | 77 | kperf_action_has_non_system(unsigned int actionid) |
5ba3f43e A |
78 | { |
79 | if (actionid > actionc) { | |
80 | return false; | |
81 | } | |
82 | ||
83 | if (actionv[actionid - 1].sample & ~SAMPLER_SYS_MEM) { | |
84 | return true; | |
85 | } else { | |
86 | return false; | |
87 | } | |
88 | } | |
89 | ||
d9a64523 A |
90 | bool |
91 | kperf_action_has_task(unsigned int actionid) | |
92 | { | |
93 | if (actionid > actionc) { | |
94 | return false; | |
95 | } | |
96 | ||
0a7de745 | 97 | return actionv[actionid - 1].sample & SAMPLER_TASK_MASK; |
d9a64523 A |
98 | } |
99 | ||
100 | bool | |
101 | kperf_action_has_thread(unsigned int actionid) | |
102 | { | |
103 | if (actionid > actionc) { | |
104 | return false; | |
105 | } | |
106 | ||
0a7de745 | 107 | return actionv[actionid - 1].sample & SAMPLER_THREAD_MASK; |
d9a64523 A |
108 | } |
109 | ||
5ba3f43e A |
110 | static void |
111 | kperf_system_memory_log(void) | |
112 | { | |
113 | BUF_DATA(PERF_MI_SYS_DATA, (uintptr_t)vm_page_free_count, | |
0a7de745 A |
114 | (uintptr_t)vm_page_wire_count, (uintptr_t)vm_page_external_count, |
115 | (uintptr_t)(vm_page_active_count + vm_page_inactive_count + | |
116 | vm_page_speculative_count)); | |
d9a64523 | 117 | BUF_DATA(PERF_MI_SYS_DATA_2, (uintptr_t)vm_page_anonymous_count, |
0a7de745 A |
118 | (uintptr_t)vm_page_internal_count, |
119 | (uintptr_t)vm_pageout_vminfo.vm_pageout_compressions, | |
120 | (uintptr_t)VM_PAGE_COMPRESSOR_COUNT); | |
5ba3f43e A |
121 | } |
122 | ||
94ff46dc A |
123 | static void |
124 | kperf_sample_user_internal(struct kperf_usample *sbuf, | |
125 | struct kperf_context *context, unsigned int actionid, | |
126 | unsigned int sample_what) | |
127 | { | |
128 | if (sample_what & SAMPLER_USTACK) { | |
129 | kperf_ucallstack_sample(&sbuf->ucallstack, context); | |
130 | } | |
131 | if (sample_what & SAMPLER_TH_DISPATCH) { | |
132 | kperf_thread_dispatch_sample(&sbuf->th_dispatch, context); | |
133 | } | |
134 | if (sample_what & SAMPLER_TH_INFO) { | |
135 | kperf_thread_info_sample(&sbuf->th_info, context); | |
136 | } | |
137 | ||
138 | boolean_t intren = ml_set_interrupts_enabled(FALSE); | |
139 | ||
140 | /* | |
141 | * No userdata or sample_flags for this one. | |
142 | */ | |
143 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, actionid); | |
144 | ||
145 | if (sample_what & SAMPLER_USTACK) { | |
146 | kperf_ucallstack_log(&sbuf->ucallstack); | |
147 | } | |
148 | if (sample_what & SAMPLER_TH_DISPATCH) { | |
149 | kperf_thread_dispatch_log(&sbuf->th_dispatch); | |
150 | } | |
151 | if (sample_what & SAMPLER_TH_INFO) { | |
152 | kperf_thread_info_log(&sbuf->th_info); | |
153 | } | |
154 | ||
155 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what); | |
156 | ||
157 | ml_set_interrupts_enabled(intren); | |
158 | } | |
159 | ||
160 | void | |
161 | kperf_sample_user(struct kperf_usample *sbuf, struct kperf_context *context, | |
162 | unsigned int actionid, unsigned int sample_flags) | |
163 | { | |
164 | if (actionid == 0 || actionid > actionc) { | |
165 | return; | |
166 | } | |
167 | ||
168 | unsigned int sample_what = actionv[actionid - 1].sample; | |
169 | unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth; | |
170 | ||
171 | /* callstacks should be explicitly ignored */ | |
172 | if (sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) { | |
173 | sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK); | |
174 | } | |
175 | if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) { | |
176 | sample_what &= SAMPLER_SYS_MEM; | |
177 | } | |
178 | assert((sample_flags & (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)) | |
179 | != (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)); | |
180 | if (sample_flags & SAMPLE_FLAG_THREAD_ONLY) { | |
181 | sample_what &= SAMPLER_THREAD_MASK; | |
182 | } | |
183 | if (sample_flags & SAMPLE_FLAG_TASK_ONLY) { | |
184 | sample_what &= SAMPLER_TASK_MASK; | |
185 | } | |
186 | ||
187 | if (sample_what == 0) { | |
188 | return; | |
189 | } | |
190 | ||
191 | sbuf->ucallstack.kpuc_nframes = ucallstack_depth ?: | |
192 | MAX_UCALLSTACK_FRAMES; | |
193 | ||
194 | kperf_sample_user_internal(sbuf, context, actionid, sample_what); | |
195 | } | |
196 | ||
316670eb | 197 | static kern_return_t |
3e170ce0 | 198 | kperf_sample_internal(struct kperf_sample *sbuf, |
0a7de745 A |
199 | struct kperf_context *context, |
200 | unsigned sample_what, unsigned sample_flags, | |
cb323159 | 201 | unsigned actionid, unsigned ucallstack_depth) |
316670eb | 202 | { |
39037602 A |
203 | int pended_ucallstack = 0; |
204 | int pended_th_dispatch = 0; | |
5ba3f43e A |
205 | bool on_idle_thread = false; |
206 | uint32_t userdata = actionid; | |
d9a64523 | 207 | bool task_only = false; |
316670eb | 208 | |
3e170ce0 | 209 | if (sample_what == 0) { |
316670eb | 210 | return SAMPLE_CONTINUE; |
3e170ce0 | 211 | } |
316670eb | 212 | |
39037602 A |
213 | /* callstacks should be explicitly ignored */ |
214 | if (sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) { | |
215 | sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK); | |
216 | } | |
217 | ||
5ba3f43e A |
218 | if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) { |
219 | sample_what &= SAMPLER_SYS_MEM; | |
220 | } | |
221 | ||
d9a64523 | 222 | assert((sample_flags & (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)) |
0a7de745 | 223 | != (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)); |
d9a64523 A |
224 | if (sample_flags & SAMPLE_FLAG_THREAD_ONLY) { |
225 | sample_what &= SAMPLER_THREAD_MASK; | |
226 | } | |
227 | if (sample_flags & SAMPLE_FLAG_TASK_ONLY) { | |
228 | task_only = true; | |
229 | sample_what &= SAMPLER_TASK_MASK; | |
230 | } | |
231 | ||
232 | if (!task_only) { | |
f427ee49 A |
233 | context->cur_thread->kperf_pet_gen = |
234 | os_atomic_load(&kppet_gencount, relaxed); | |
d9a64523 A |
235 | } |
236 | bool is_kernel = (context->cur_pid == 0); | |
39037602 A |
237 | |
238 | if (actionid && actionid <= actionc) { | |
cb323159 A |
239 | sbuf->kcallstack.kpkc_nframes = |
240 | actionv[actionid - 1].kcallstack_depth; | |
39037602 | 241 | } else { |
cb323159 | 242 | sbuf->kcallstack.kpkc_nframes = MAX_KCALLSTACK_FRAMES; |
39037602 A |
243 | } |
244 | ||
94ff46dc | 245 | ucallstack_depth = ucallstack_depth ?: MAX_UCALLSTACK_FRAMES; |
cb323159 | 246 | sbuf->kcallstack.kpkc_flags = 0; |
94ff46dc | 247 | sbuf->usample.ucallstack.kpuc_flags = 0; |
39236c6e | 248 | |
39037602 A |
249 | if (sample_what & SAMPLER_TH_INFO) { |
250 | kperf_thread_info_sample(&sbuf->th_info, context); | |
3e170ce0 | 251 | |
3e170ce0 | 252 | if (!(sample_flags & SAMPLE_FLAG_IDLE_THREADS)) { |
39037602 | 253 | if (sbuf->th_info.kpthi_runmode & 0x40) { |
5ba3f43e A |
254 | on_idle_thread = true; |
255 | goto log_sample; | |
3e170ce0 A |
256 | } |
257 | } | |
316670eb A |
258 | } |
259 | ||
39037602 A |
260 | if (sample_what & SAMPLER_TH_SNAPSHOT) { |
261 | kperf_thread_snapshot_sample(&(sbuf->th_snapshot), context); | |
262 | } | |
263 | if (sample_what & SAMPLER_TH_SCHEDULING) { | |
264 | kperf_thread_scheduling_sample(&(sbuf->th_scheduling), context); | |
265 | } | |
266 | if (sample_what & SAMPLER_KSTACK) { | |
267 | if (sample_flags & SAMPLE_FLAG_CONTINUATION) { | |
268 | kperf_continuation_sample(&(sbuf->kcallstack), context); | |
39037602 | 269 | } else if (sample_flags & SAMPLE_FLAG_NON_INTERRUPT) { |
94ff46dc | 270 | /* outside of interrupt context, backtrace the current thread */ |
39037602 A |
271 | kperf_backtrace_sample(&(sbuf->kcallstack), context); |
272 | } else { | |
273 | kperf_kcallstack_sample(&(sbuf->kcallstack), context); | |
274 | } | |
275 | } | |
276 | if (sample_what & SAMPLER_TK_SNAPSHOT) { | |
d9a64523 | 277 | kperf_task_snapshot_sample(context->cur_task, &(sbuf->tk_snapshot)); |
3e170ce0 | 278 | } |
316670eb | 279 | |
3e170ce0 A |
280 | if (!is_kernel) { |
281 | if (sample_what & SAMPLER_MEMINFO) { | |
d9a64523 | 282 | kperf_meminfo_sample(context->cur_task, &(sbuf->meminfo)); |
316670eb | 283 | } |
3e170ce0 A |
284 | |
285 | if (sample_flags & SAMPLE_FLAG_PEND_USER) { | |
39037602 | 286 | if (sample_what & SAMPLER_USTACK) { |
94ff46dc A |
287 | pended_ucallstack = kperf_ucallstack_pend(context, |
288 | ucallstack_depth, actionid); | |
3e170ce0 A |
289 | } |
290 | ||
39037602 | 291 | if (sample_what & SAMPLER_TH_DISPATCH) { |
94ff46dc A |
292 | pended_th_dispatch = |
293 | kperf_thread_dispatch_pend(context, actionid); | |
3e170ce0 | 294 | } |
316670eb A |
295 | } |
296 | } | |
297 | ||
3e170ce0 A |
298 | if (sample_what & SAMPLER_PMC_THREAD) { |
299 | kperf_kpc_thread_sample(&(sbuf->kpcdata), sample_what); | |
300 | } else if (sample_what & SAMPLER_PMC_CPU) { | |
301 | kperf_kpc_cpu_sample(&(sbuf->kpcdata), sample_what); | |
302 | } | |
39236c6e | 303 | |
5ba3f43e | 304 | log_sample: |
39236c6e | 305 | /* lookup the user tag, if any */ |
3e170ce0 A |
306 | if (actionid && (actionid <= actionc)) { |
307 | userdata = actionv[actionid - 1].userdata; | |
3e170ce0 | 308 | } |
39236c6e | 309 | |
39037602 A |
310 | /* avoid logging if this sample only pended samples */ |
311 | if (sample_flags & SAMPLE_FLAG_PEND_USER && | |
0a7de745 | 312 | !(sample_what & ~(SAMPLER_USTACK | SAMPLER_TH_DISPATCH))) { |
39037602 A |
313 | return SAMPLE_CONTINUE; |
314 | } | |
315 | ||
316670eb A |
316 | /* stash the data into the buffer |
317 | * interrupts off to ensure we don't get split | |
318 | */ | |
39037602 | 319 | boolean_t enabled = ml_set_interrupts_enabled(FALSE); |
316670eb | 320 | |
3e170ce0 | 321 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, |
0a7de745 | 322 | actionid, userdata, sample_flags); |
316670eb | 323 | |
5ba3f43e A |
324 | if (sample_flags & SAMPLE_FLAG_SYSTEM) { |
325 | if (sample_what & SAMPLER_SYS_MEM) { | |
326 | kperf_system_memory_log(); | |
327 | } | |
328 | } | |
329 | if (on_idle_thread) { | |
330 | goto log_sample_end; | |
331 | } | |
332 | ||
39037602 A |
333 | if (sample_what & SAMPLER_TH_INFO) { |
334 | kperf_thread_info_log(&sbuf->th_info); | |
335 | } | |
336 | if (sample_what & SAMPLER_TH_SCHEDULING) { | |
337 | kperf_thread_scheduling_log(&(sbuf->th_scheduling)); | |
338 | } | |
339 | if (sample_what & SAMPLER_TH_SNAPSHOT) { | |
340 | kperf_thread_snapshot_log(&(sbuf->th_snapshot)); | |
3e170ce0 | 341 | } |
3e170ce0 | 342 | if (sample_what & SAMPLER_KSTACK) { |
39037602 A |
343 | kperf_kcallstack_log(&sbuf->kcallstack); |
344 | } | |
5ba3f43e A |
345 | if (sample_what & SAMPLER_TH_INSCYC) { |
346 | kperf_thread_inscyc_log(context); | |
347 | } | |
39037602 A |
348 | if (sample_what & SAMPLER_TK_SNAPSHOT) { |
349 | kperf_task_snapshot_log(&(sbuf->tk_snapshot)); | |
3e170ce0 | 350 | } |
d9a64523 A |
351 | if (sample_what & SAMPLER_TK_INFO) { |
352 | kperf_task_info_log(context); | |
353 | } | |
316670eb A |
354 | |
355 | /* dump user stuff */ | |
3e170ce0 A |
356 | if (!is_kernel) { |
357 | /* dump meminfo */ | |
358 | if (sample_what & SAMPLER_MEMINFO) { | |
359 | kperf_meminfo_log(&(sbuf->meminfo)); | |
316670eb | 360 | } |
316670eb | 361 | |
3e170ce0 | 362 | if (sample_flags & SAMPLE_FLAG_PEND_USER) { |
39037602 A |
363 | if (pended_ucallstack) { |
364 | BUF_INFO(PERF_CS_UPEND); | |
3e170ce0 A |
365 | } |
366 | ||
39037602 A |
367 | if (pended_th_dispatch) { |
368 | BUF_INFO(PERF_TI_DISPPEND); | |
3e170ce0 | 369 | } |
316670eb A |
370 | } |
371 | } | |
372 | ||
cb323159 A |
373 | if (sample_what & SAMPLER_PMC_CONFIG) { |
374 | kperf_kpc_config_log(&(sbuf->kpcdata)); | |
375 | } | |
3e170ce0 A |
376 | if (sample_what & SAMPLER_PMC_THREAD) { |
377 | kperf_kpc_thread_log(&(sbuf->kpcdata)); | |
378 | } else if (sample_what & SAMPLER_PMC_CPU) { | |
379 | kperf_kpc_cpu_log(&(sbuf->kpcdata)); | |
380 | } | |
39236c6e | 381 | |
5ba3f43e A |
382 | log_sample_end: |
383 | BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what, on_idle_thread ? 1 : 0); | |
316670eb A |
384 | |
385 | /* intrs back on */ | |
386 | ml_set_interrupts_enabled(enabled); | |
387 | ||
388 | return SAMPLE_CONTINUE; | |
389 | } | |
390 | ||
391 | /* Translate actionid into sample bits and take a sample */ | |
392 | kern_return_t | |
3e170ce0 | 393 | kperf_sample(struct kperf_sample *sbuf, |
0a7de745 A |
394 | struct kperf_context *context, |
395 | unsigned actionid, unsigned sample_flags) | |
316670eb | 396 | { |
316670eb | 397 | /* work out what to sample, if anything */ |
3e170ce0 | 398 | if ((actionid > actionc) || (actionid == 0)) { |
316670eb | 399 | return SAMPLE_SHUTDOWN; |
3e170ce0 | 400 | } |
316670eb | 401 | |
39236c6e A |
402 | /* check the pid filter against the context's current pid. |
403 | * filter pid == -1 means any pid | |
404 | */ | |
39037602 | 405 | int pid_filter = actionv[actionid - 1].pid_filter; |
3e170ce0 | 406 | if ((pid_filter != -1) && (pid_filter != context->cur_pid)) { |
39236c6e | 407 | return SAMPLE_CONTINUE; |
3e170ce0 | 408 | } |
39236c6e A |
409 | |
410 | /* the samplers to run */ | |
39037602 | 411 | unsigned int sample_what = actionv[actionid - 1].sample; |
94ff46dc | 412 | unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth; |
316670eb | 413 | |
39236c6e | 414 | /* do the actual sample operation */ |
3e170ce0 | 415 | return kperf_sample_internal(sbuf, context, sample_what, |
94ff46dc | 416 | sample_flags, actionid, ucallstack_depth); |
316670eb A |
417 | } |
418 | ||
316670eb | 419 | void |
39037602 | 420 | kperf_kdebug_handler(uint32_t debugid, uintptr_t *starting_fp) |
316670eb | 421 | { |
39037602 | 422 | uint32_t sample_flags = SAMPLE_FLAG_PEND_USER; |
39037602 A |
423 | struct kperf_sample *sample = NULL; |
424 | kern_return_t kr = KERN_SUCCESS; | |
425 | int s; | |
316670eb | 426 | |
39037602 A |
427 | if (!kperf_kdebug_should_trigger(debugid)) { |
428 | return; | |
3e170ce0 | 429 | } |
316670eb | 430 | |
39037602 | 431 | BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_START, debugid); |
3e170ce0 | 432 | |
d9a64523 A |
433 | thread_t thread = current_thread(); |
434 | task_t task = get_threadtask(thread); | |
435 | struct kperf_context ctx = { | |
436 | .cur_thread = thread, | |
437 | .cur_task = task, | |
438 | .cur_pid = task_pid(task), | |
439 | .trigger_type = TRIGGER_TYPE_KDEBUG, | |
440 | .trigger_id = 0, | |
441 | }; | |
3e170ce0 | 442 | |
39037602 | 443 | s = ml_set_interrupts_enabled(0); |
316670eb | 444 | |
39037602 | 445 | sample = kperf_intr_sample_buffer(); |
316670eb | 446 | |
39037602 A |
447 | if (!ml_at_interrupt_context()) { |
448 | sample_flags |= SAMPLE_FLAG_NON_INTERRUPT; | |
449 | ctx.starting_fp = starting_fp; | |
3e170ce0 | 450 | } |
316670eb | 451 | |
39037602 | 452 | kr = kperf_sample(sample, &ctx, kperf_kdebug_get_action(), sample_flags); |
316670eb | 453 | |
39037602 A |
454 | ml_set_interrupts_enabled(s); |
455 | BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_END, kr); | |
316670eb A |
456 | } |
457 | ||
39236c6e | 458 | /* |
39037602 A |
459 | * This function allocates >2.3KB of the stack. Prevent the compiler from |
460 | * inlining this function into ast_taken and ensure the stack memory is only | |
461 | * allocated for the kperf AST. | |
39236c6e | 462 | */ |
39037602 | 463 | __attribute__((noinline)) |
39236c6e | 464 | void |
39037602 | 465 | kperf_thread_ast_handler(thread_t thread) |
39236c6e | 466 | { |
94ff46dc | 467 | uint32_t ast = thread->kperf_ast; |
39236c6e | 468 | |
94ff46dc A |
469 | BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_START, thread, ast); |
470 | ||
471 | struct kperf_usample sbuf = {}; | |
39236c6e | 472 | |
39037602 | 473 | task_t task = get_threadtask(thread); |
39236c6e | 474 | |
743345f9 A |
475 | if (task_did_exec(task) || task_is_exec_copy(task)) { |
476 | BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, SAMPLE_CONTINUE); | |
477 | return; | |
478 | } | |
479 | ||
d9a64523 A |
480 | struct kperf_context ctx = { |
481 | .cur_thread = thread, | |
482 | .cur_task = task, | |
483 | .cur_pid = task_pid(task), | |
484 | }; | |
3e170ce0 | 485 | |
39037602 | 486 | unsigned int sample_what = 0; |
94ff46dc | 487 | if (ast & T_KPERF_AST_DISPATCH) { |
39037602 | 488 | sample_what |= SAMPLER_TH_DISPATCH; |
3e170ce0 | 489 | } |
94ff46dc A |
490 | if (ast & T_KPERF_AST_CALLSTACK) { |
491 | /* TH_INFO for backwards compatibility */ | |
492 | sample_what |= SAMPLER_USTACK | SAMPLER_TH_INFO; | |
3e170ce0 | 493 | } |
3e170ce0 | 494 | |
94ff46dc A |
495 | sbuf.ucallstack.kpuc_nframes = |
496 | T_KPERF_GET_CALLSTACK_DEPTH(ast) ?: MAX_UCALLSTACK_FRAMES; | |
497 | unsigned int actionid = T_KPERF_GET_ACTIONID(ast); | |
498 | kperf_sample_user_internal(&sbuf, &ctx, actionid, sample_what); | |
39236c6e | 499 | |
94ff46dc | 500 | BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END); |
39236c6e A |
501 | } |
502 | ||
503 | int | |
94ff46dc | 504 | kperf_ast_pend(thread_t thread, uint32_t set_flags, unsigned int set_actionid) |
39236c6e | 505 | { |
39037602 | 506 | if (thread != current_thread()) { |
94ff46dc | 507 | panic("kperf: pending AST to non-current thread"); |
39037602 | 508 | } |
39236c6e | 509 | |
94ff46dc A |
510 | uint32_t ast = thread->kperf_ast; |
511 | unsigned int actionid = T_KPERF_GET_ACTIONID(ast); | |
512 | uint32_t flags = ast & T_KPERF_AST_ALL; | |
3e170ce0 | 513 | |
94ff46dc A |
514 | if ((flags | set_flags) != flags || actionid != set_actionid) { |
515 | ast &= ~T_KPERF_SET_ACTIONID(actionid); | |
516 | ast |= T_KPERF_SET_ACTIONID(set_actionid); | |
517 | ast |= set_flags; | |
518 | ||
519 | thread->kperf_ast = ast; | |
3e170ce0 | 520 | |
39037602 A |
521 | /* set the actual AST */ |
522 | act_set_kperf(thread); | |
523 | return 1; | |
524 | } | |
39236c6e A |
525 | |
526 | return 0; | |
527 | } | |
528 | ||
39236c6e | 529 | void |
39037602 | 530 | kperf_ast_set_callstack_depth(thread_t thread, uint32_t depth) |
39236c6e | 531 | { |
94ff46dc A |
532 | uint32_t ast = thread->kperf_ast; |
533 | uint32_t existing_depth = T_KPERF_GET_CALLSTACK_DEPTH(ast); | |
534 | if (existing_depth < depth) { | |
535 | ast &= ~T_KPERF_SET_CALLSTACK_DEPTH(existing_depth); | |
536 | ast |= T_KPERF_SET_CALLSTACK_DEPTH(depth); | |
537 | thread->kperf_ast = ast; | |
3e170ce0 A |
538 | } |
539 | } | |
540 | ||
3e170ce0 A |
541 | int |
542 | kperf_kdbg_cswitch_get(void) | |
543 | { | |
39037602 | 544 | return kperf_kdebug_cswitch; |
3e170ce0 A |
545 | } |
546 | ||
547 | int | |
548 | kperf_kdbg_cswitch_set(int newval) | |
549 | { | |
39037602 A |
550 | kperf_kdebug_cswitch = newval; |
551 | kperf_on_cpu_update(); | |
3e170ce0 A |
552 | |
553 | return 0; | |
39236c6e A |
554 | } |
555 | ||
556 | /* | |
557 | * Action configuration | |
558 | */ | |
39037602 | 559 | unsigned int |
316670eb A |
560 | kperf_action_get_count(void) |
561 | { | |
562 | return actionc; | |
563 | } | |
564 | ||
565 | int | |
3e170ce0 | 566 | kperf_action_set_samplers(unsigned actionid, uint32_t samplers) |
316670eb | 567 | { |
3e170ce0 A |
568 | if ((actionid > actionc) || (actionid == 0)) { |
569 | return EINVAL; | |
570 | } | |
571 | ||
572 | /* disallow both CPU and thread counters to be sampled in the same | |
573 | * action */ | |
574 | if ((samplers & SAMPLER_PMC_THREAD) && (samplers & SAMPLER_PMC_CPU)) { | |
316670eb | 575 | return EINVAL; |
3e170ce0 | 576 | } |
316670eb | 577 | |
3e170ce0 | 578 | actionv[actionid - 1].sample = samplers; |
316670eb A |
579 | |
580 | return 0; | |
581 | } | |
582 | ||
583 | int | |
3e170ce0 | 584 | kperf_action_get_samplers(unsigned actionid, uint32_t *samplers_out) |
316670eb | 585 | { |
3e170ce0 | 586 | if ((actionid > actionc)) { |
316670eb | 587 | return EINVAL; |
3e170ce0 | 588 | } |
316670eb | 589 | |
3e170ce0 | 590 | if (actionid == 0) { |
39236c6e | 591 | *samplers_out = 0; /* "NULL" action */ |
3e170ce0 A |
592 | } else { |
593 | *samplers_out = actionv[actionid - 1].sample; | |
594 | } | |
39236c6e A |
595 | |
596 | return 0; | |
597 | } | |
598 | ||
599 | int | |
3e170ce0 | 600 | kperf_action_set_userdata(unsigned actionid, uint32_t userdata) |
39236c6e | 601 | { |
3e170ce0 | 602 | if ((actionid > actionc) || (actionid == 0)) { |
39236c6e | 603 | return EINVAL; |
3e170ce0 | 604 | } |
39236c6e | 605 | |
3e170ce0 | 606 | actionv[actionid - 1].userdata = userdata; |
39236c6e A |
607 | |
608 | return 0; | |
609 | } | |
610 | ||
611 | int | |
3e170ce0 | 612 | kperf_action_get_userdata(unsigned actionid, uint32_t *userdata_out) |
39236c6e | 613 | { |
3e170ce0 | 614 | if ((actionid > actionc)) { |
39236c6e | 615 | return EINVAL; |
3e170ce0 | 616 | } |
39236c6e | 617 | |
3e170ce0 | 618 | if (actionid == 0) { |
39236c6e | 619 | *userdata_out = 0; /* "NULL" action */ |
3e170ce0 A |
620 | } else { |
621 | *userdata_out = actionv[actionid - 1].userdata; | |
622 | } | |
39236c6e A |
623 | |
624 | return 0; | |
625 | } | |
626 | ||
627 | int | |
3e170ce0 | 628 | kperf_action_set_filter(unsigned actionid, int pid) |
39236c6e | 629 | { |
3e170ce0 | 630 | if ((actionid > actionc) || (actionid == 0)) { |
39236c6e | 631 | return EINVAL; |
3e170ce0 | 632 | } |
39236c6e | 633 | |
3e170ce0 | 634 | actionv[actionid - 1].pid_filter = pid; |
39236c6e A |
635 | |
636 | return 0; | |
637 | } | |
638 | ||
639 | int | |
3e170ce0 | 640 | kperf_action_get_filter(unsigned actionid, int *pid_out) |
39236c6e | 641 | { |
3e170ce0 | 642 | if ((actionid > actionc)) { |
39236c6e | 643 | return EINVAL; |
3e170ce0 | 644 | } |
39236c6e | 645 | |
3e170ce0 | 646 | if (actionid == 0) { |
39236c6e | 647 | *pid_out = -1; /* "NULL" action */ |
3e170ce0 A |
648 | } else { |
649 | *pid_out = actionv[actionid - 1].pid_filter; | |
650 | } | |
316670eb A |
651 | |
652 | return 0; | |
653 | } | |
654 | ||
39037602 A |
655 | void |
656 | kperf_action_reset(void) | |
657 | { | |
658 | for (unsigned int i = 0; i < actionc; i++) { | |
659 | kperf_action_set_samplers(i + 1, 0); | |
660 | kperf_action_set_userdata(i + 1, 0); | |
661 | kperf_action_set_filter(i + 1, -1); | |
cb323159 A |
662 | kperf_action_set_ucallstack_depth(i + 1, MAX_UCALLSTACK_FRAMES); |
663 | kperf_action_set_kcallstack_depth(i + 1, MAX_KCALLSTACK_FRAMES); | |
39037602 A |
664 | } |
665 | } | |
666 | ||
316670eb A |
667 | int |
668 | kperf_action_set_count(unsigned count) | |
669 | { | |
670 | struct action *new_actionv = NULL, *old_actionv = NULL; | |
39037602 | 671 | unsigned old_count; |
316670eb A |
672 | |
673 | /* easy no-op */ | |
3e170ce0 | 674 | if (count == actionc) { |
316670eb | 675 | return 0; |
3e170ce0 | 676 | } |
316670eb A |
677 | |
678 | /* TODO: allow shrinking? */ | |
3e170ce0 | 679 | if (count < actionc) { |
316670eb | 680 | return EINVAL; |
3e170ce0 | 681 | } |
316670eb A |
682 | |
683 | /* cap it for good measure */ | |
3e170ce0 | 684 | if (count > ACTION_MAX) { |
316670eb | 685 | return EINVAL; |
3e170ce0 | 686 | } |
316670eb A |
687 | |
688 | /* creating the action arror for the first time. create a few | |
689 | * more things, too. | |
690 | */ | |
3e170ce0 | 691 | if (actionc == 0) { |
f427ee49 | 692 | kperf_setup(); |
316670eb A |
693 | } |
694 | ||
695 | /* create a new array */ | |
39037602 | 696 | new_actionv = kalloc_tag(count * sizeof(*new_actionv), VM_KERN_MEMORY_DIAG); |
3e170ce0 | 697 | if (new_actionv == NULL) { |
316670eb | 698 | return ENOMEM; |
3e170ce0 | 699 | } |
316670eb A |
700 | |
701 | old_actionv = actionv; | |
702 | old_count = actionc; | |
703 | ||
3e170ce0 A |
704 | if (old_actionv != NULL) { |
705 | memcpy(new_actionv, actionv, actionc * sizeof(*actionv)); | |
706 | } | |
316670eb | 707 | |
3e170ce0 | 708 | memset(&(new_actionv[actionc]), 0, (count - old_count) * sizeof(*actionv)); |
316670eb | 709 | |
39037602 | 710 | for (unsigned int i = old_count; i < count; i++) { |
39236c6e | 711 | new_actionv[i].pid_filter = -1; |
cb323159 A |
712 | new_actionv[i].ucallstack_depth = MAX_UCALLSTACK_FRAMES; |
713 | new_actionv[i].kcallstack_depth = MAX_KCALLSTACK_FRAMES; | |
3e170ce0 | 714 | } |
39236c6e | 715 | |
316670eb A |
716 | actionv = new_actionv; |
717 | actionc = count; | |
718 | ||
3e170ce0 A |
719 | if (old_actionv != NULL) { |
720 | kfree(old_actionv, old_count * sizeof(*actionv)); | |
721 | } | |
316670eb | 722 | |
316670eb A |
723 | return 0; |
724 | } | |
39037602 A |
725 | |
726 | int | |
727 | kperf_action_set_ucallstack_depth(unsigned action_id, uint32_t depth) | |
728 | { | |
729 | if ((action_id > actionc) || (action_id == 0)) { | |
730 | return EINVAL; | |
731 | } | |
732 | ||
cb323159 | 733 | if (depth > MAX_UCALLSTACK_FRAMES) { |
39037602 A |
734 | return EINVAL; |
735 | } | |
94ff46dc A |
736 | if (depth < 2) { |
737 | return EINVAL; | |
738 | } | |
39037602 A |
739 | |
740 | actionv[action_id - 1].ucallstack_depth = depth; | |
741 | ||
742 | return 0; | |
743 | } | |
744 | ||
745 | int | |
746 | kperf_action_set_kcallstack_depth(unsigned action_id, uint32_t depth) | |
747 | { | |
748 | if ((action_id > actionc) || (action_id == 0)) { | |
749 | return EINVAL; | |
750 | } | |
751 | ||
cb323159 | 752 | if (depth > MAX_KCALLSTACK_FRAMES) { |
39037602 A |
753 | return EINVAL; |
754 | } | |
94ff46dc A |
755 | if (depth < 1) { |
756 | return EINVAL; | |
757 | } | |
39037602 A |
758 | |
759 | actionv[action_id - 1].kcallstack_depth = depth; | |
760 | ||
761 | return 0; | |
762 | } | |
763 | ||
764 | int | |
765 | kperf_action_get_ucallstack_depth(unsigned action_id, uint32_t * depth_out) | |
766 | { | |
767 | if ((action_id > actionc)) { | |
768 | return EINVAL; | |
769 | } | |
770 | ||
771 | assert(depth_out); | |
772 | ||
773 | if (action_id == 0) { | |
cb323159 | 774 | *depth_out = MAX_UCALLSTACK_FRAMES; |
39037602 A |
775 | } else { |
776 | *depth_out = actionv[action_id - 1].ucallstack_depth; | |
777 | } | |
778 | ||
779 | return 0; | |
780 | } | |
781 | ||
782 | int | |
783 | kperf_action_get_kcallstack_depth(unsigned action_id, uint32_t * depth_out) | |
784 | { | |
785 | if ((action_id > actionc)) { | |
786 | return EINVAL; | |
787 | } | |
788 | ||
789 | assert(depth_out); | |
790 | ||
791 | if (action_id == 0) { | |
cb323159 | 792 | *depth_out = MAX_KCALLSTACK_FRAMES; |
39037602 A |
793 | } else { |
794 | *depth_out = actionv[action_id - 1].kcallstack_depth; | |
795 | } | |
796 | ||
797 | return 0; | |
798 | } |