]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kperf/action.c
xnu-4570.71.2.tar.gz
[apple/xnu.git] / osfmk / kperf / action.c
1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Called from a trigger. Actually takes the data from the different
31 * modules and puts them in a buffer
32 */
33
34 #include <mach/mach_types.h>
35 #include <machine/machine_routines.h>
36 #include <kern/kalloc.h>
37 #include <kern/debug.h> /* panic */
38 #include <kern/thread.h>
39 #include <sys/errno.h>
40 #include <sys/vm.h>
41 #include <vm/vm_page.h>
42 #include <vm/vm_pageout.h>
43
44 #include <kperf/action.h>
45 #include <kperf/ast.h>
46 #include <kperf/buffer.h>
47 #include <kperf/callstack.h>
48 #include <kperf/context.h>
49 #include <kperf/kdebug_trigger.h>
50 #include <kperf/kperf.h>
51 #include <kperf/kperf_kpc.h>
52 #include <kperf/kperf_timer.h>
53 #include <kperf/pet.h>
54 #include <kperf/sample.h>
55 #include <kperf/thread_samplers.h>
56
57 #define ACTION_MAX (32)
58
59 /* the list of different actions to take */
60 struct action
61 {
62 uint32_t sample;
63 uint32_t ucallstack_depth;
64 uint32_t kcallstack_depth;
65 uint32_t userdata;
66 int pid_filter;
67 };
68
69 /* the list of actions */
70 static unsigned actionc = 0;
71 static struct action *actionv = NULL;
72
73 /* should emit tracepoint on context switch */
74 int kperf_kdebug_cswitch = 0;
75
76 bool
77 kperf_sample_has_non_system(unsigned actionid)
78 {
79 if (actionid > actionc) {
80 return false;
81 }
82
83 if (actionv[actionid - 1].sample & ~SAMPLER_SYS_MEM) {
84 return true;
85 } else {
86 return false;
87 }
88 }
89
90 static void
91 kperf_system_memory_log(void)
92 {
93 BUF_DATA(PERF_MI_SYS_DATA, (uintptr_t)vm_page_free_count,
94 (uintptr_t)vm_page_wire_count, (uintptr_t)vm_page_external_count,
95 (uintptr_t)(vm_page_active_count + vm_page_inactive_count +
96 vm_page_speculative_count));
97 }
98
99 static kern_return_t
100 kperf_sample_internal(struct kperf_sample *sbuf,
101 struct kperf_context *context,
102 unsigned sample_what, unsigned sample_flags,
103 unsigned actionid, uint32_t ucallstack_depth)
104 {
105 int pended_ucallstack = 0;
106 int pended_th_dispatch = 0;
107 bool on_idle_thread = false;
108 uint32_t userdata = actionid;
109
110 /* not much point continuing here, but what to do ? return
111 * Shutdown? cut a tracepoint and continue?
112 */
113 if (sample_what == 0) {
114 return SAMPLE_CONTINUE;
115 }
116
117 /* callstacks should be explicitly ignored */
118 if (sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) {
119 sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK);
120 }
121
122 if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) {
123 sample_what &= SAMPLER_SYS_MEM;
124 }
125
126 context->cur_thread->kperf_pet_gen = kperf_pet_gen;
127 boolean_t is_kernel = (context->cur_pid == 0);
128
129 if (actionid && actionid <= actionc) {
130 sbuf->kcallstack.nframes = actionv[actionid - 1].kcallstack_depth;
131 } else {
132 sbuf->kcallstack.nframes = MAX_CALLSTACK_FRAMES;
133 }
134
135 if (ucallstack_depth) {
136 sbuf->ucallstack.nframes = ucallstack_depth;
137 } else {
138 sbuf->ucallstack.nframes = MAX_CALLSTACK_FRAMES;
139 }
140
141 sbuf->kcallstack.flags = CALLSTACK_VALID;
142 sbuf->ucallstack.flags = CALLSTACK_VALID;
143
144 /* an event occurred. Sample everything and dump it in a
145 * buffer.
146 */
147
148 /* collect data from samplers */
149 if (sample_what & SAMPLER_TH_INFO) {
150 kperf_thread_info_sample(&sbuf->th_info, context);
151
152 /* See if we should drop idle thread samples */
153 if (!(sample_flags & SAMPLE_FLAG_IDLE_THREADS)) {
154 if (sbuf->th_info.kpthi_runmode & 0x40) {
155 on_idle_thread = true;
156 goto log_sample;
157 }
158 }
159 }
160
161 if (sample_what & SAMPLER_TH_SNAPSHOT) {
162 kperf_thread_snapshot_sample(&(sbuf->th_snapshot), context);
163 }
164 if (sample_what & SAMPLER_TH_SCHEDULING) {
165 kperf_thread_scheduling_sample(&(sbuf->th_scheduling), context);
166 }
167 if (sample_what & SAMPLER_KSTACK) {
168 if (sample_flags & SAMPLE_FLAG_CONTINUATION) {
169 kperf_continuation_sample(&(sbuf->kcallstack), context);
170 /* outside of interrupt context, backtrace the current thread */
171 } else if (sample_flags & SAMPLE_FLAG_NON_INTERRUPT) {
172 kperf_backtrace_sample(&(sbuf->kcallstack), context);
173 } else {
174 kperf_kcallstack_sample(&(sbuf->kcallstack), context);
175 }
176 }
177 if (sample_what & SAMPLER_TK_SNAPSHOT) {
178 kperf_task_snapshot_sample(&(sbuf->tk_snapshot), context);
179 }
180
181 /* sensitive ones */
182 if (!is_kernel) {
183 if (sample_what & SAMPLER_MEMINFO) {
184 kperf_meminfo_sample(&(sbuf->meminfo), context);
185 }
186
187 if (sample_flags & SAMPLE_FLAG_PEND_USER) {
188 if (sample_what & SAMPLER_USTACK) {
189 pended_ucallstack = kperf_ucallstack_pend(context, sbuf->ucallstack.nframes);
190 }
191
192 if (sample_what & SAMPLER_TH_DISPATCH) {
193 pended_th_dispatch = kperf_thread_dispatch_pend(context);
194 }
195 } else {
196 if (sample_what & SAMPLER_USTACK) {
197 kperf_ucallstack_sample(&(sbuf->ucallstack), context);
198 }
199
200 if (sample_what & SAMPLER_TH_DISPATCH) {
201 kperf_thread_dispatch_sample(&(sbuf->th_dispatch), context);
202 }
203 }
204 }
205
206 if (sample_what & SAMPLER_PMC_THREAD) {
207 kperf_kpc_thread_sample(&(sbuf->kpcdata), sample_what);
208 } else if (sample_what & SAMPLER_PMC_CPU) {
209 kperf_kpc_cpu_sample(&(sbuf->kpcdata), sample_what);
210 }
211
212 log_sample:
213 /* lookup the user tag, if any */
214 if (actionid && (actionid <= actionc)) {
215 userdata = actionv[actionid - 1].userdata;
216 }
217
218 /* avoid logging if this sample only pended samples */
219 if (sample_flags & SAMPLE_FLAG_PEND_USER &&
220 !(sample_what & ~(SAMPLER_USTACK | SAMPLER_TH_DISPATCH)))
221 {
222 return SAMPLE_CONTINUE;
223 }
224
225 /* stash the data into the buffer
226 * interrupts off to ensure we don't get split
227 */
228 boolean_t enabled = ml_set_interrupts_enabled(FALSE);
229
230 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what,
231 actionid, userdata, sample_flags);
232
233 if (sample_flags & SAMPLE_FLAG_SYSTEM) {
234 if (sample_what & SAMPLER_SYS_MEM) {
235 kperf_system_memory_log();
236 }
237 }
238 if (on_idle_thread) {
239 goto log_sample_end;
240 }
241
242 if (sample_what & SAMPLER_TH_INFO) {
243 kperf_thread_info_log(&sbuf->th_info);
244 }
245 if (sample_what & SAMPLER_TH_SCHEDULING) {
246 kperf_thread_scheduling_log(&(sbuf->th_scheduling));
247 }
248 if (sample_what & SAMPLER_TH_SNAPSHOT) {
249 kperf_thread_snapshot_log(&(sbuf->th_snapshot));
250 }
251 if (sample_what & SAMPLER_KSTACK) {
252 kperf_kcallstack_log(&sbuf->kcallstack);
253 }
254 if (sample_what & SAMPLER_TH_INSCYC) {
255 kperf_thread_inscyc_log(context);
256 }
257 if (sample_what & SAMPLER_TK_SNAPSHOT) {
258 kperf_task_snapshot_log(&(sbuf->tk_snapshot));
259 }
260
261 /* dump user stuff */
262 if (!is_kernel) {
263 /* dump meminfo */
264 if (sample_what & SAMPLER_MEMINFO) {
265 kperf_meminfo_log(&(sbuf->meminfo));
266 }
267
268 if (sample_flags & SAMPLE_FLAG_PEND_USER) {
269 if (pended_ucallstack) {
270 BUF_INFO(PERF_CS_UPEND);
271 }
272
273 if (pended_th_dispatch) {
274 BUF_INFO(PERF_TI_DISPPEND);
275 }
276 } else {
277 if (sample_what & SAMPLER_USTACK) {
278 kperf_ucallstack_log(&(sbuf->ucallstack));
279 }
280
281 if (sample_what & SAMPLER_TH_DISPATCH) {
282 kperf_thread_dispatch_log(&(sbuf->th_dispatch));
283 }
284 }
285 }
286
287 if (sample_what & SAMPLER_PMC_THREAD) {
288 kperf_kpc_thread_log(&(sbuf->kpcdata));
289 } else if (sample_what & SAMPLER_PMC_CPU) {
290 kperf_kpc_cpu_log(&(sbuf->kpcdata));
291 }
292
293 log_sample_end:
294 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what, on_idle_thread ? 1 : 0);
295
296 /* intrs back on */
297 ml_set_interrupts_enabled(enabled);
298
299 return SAMPLE_CONTINUE;
300 }
301
302 /* Translate actionid into sample bits and take a sample */
303 kern_return_t
304 kperf_sample(struct kperf_sample *sbuf,
305 struct kperf_context *context,
306 unsigned actionid, unsigned sample_flags)
307 {
308 /* work out what to sample, if anything */
309 if ((actionid > actionc) || (actionid == 0)) {
310 return SAMPLE_SHUTDOWN;
311 }
312
313 /* check the pid filter against the context's current pid.
314 * filter pid == -1 means any pid
315 */
316 int pid_filter = actionv[actionid - 1].pid_filter;
317 if ((pid_filter != -1) && (pid_filter != context->cur_pid)) {
318 return SAMPLE_CONTINUE;
319 }
320
321 /* the samplers to run */
322 unsigned int sample_what = actionv[actionid - 1].sample;
323
324 /* do the actual sample operation */
325 return kperf_sample_internal(sbuf, context, sample_what,
326 sample_flags, actionid,
327 actionv[actionid - 1].ucallstack_depth);
328 }
329
330 void
331 kperf_kdebug_handler(uint32_t debugid, uintptr_t *starting_fp)
332 {
333 uint32_t sample_flags = SAMPLE_FLAG_PEND_USER;
334 struct kperf_context ctx;
335 struct kperf_sample *sample = NULL;
336 kern_return_t kr = KERN_SUCCESS;
337 int s;
338
339 if (!kperf_kdebug_should_trigger(debugid)) {
340 return;
341 }
342
343 BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_START, debugid);
344
345 ctx.cur_thread = current_thread();
346 ctx.cur_pid = task_pid(get_threadtask(ctx.cur_thread));
347 ctx.trigger_type = TRIGGER_TYPE_KDEBUG;
348 ctx.trigger_id = 0;
349
350 s = ml_set_interrupts_enabled(0);
351
352 sample = kperf_intr_sample_buffer();
353
354 if (!ml_at_interrupt_context()) {
355 sample_flags |= SAMPLE_FLAG_NON_INTERRUPT;
356 ctx.starting_fp = starting_fp;
357 }
358
359 kr = kperf_sample(sample, &ctx, kperf_kdebug_get_action(), sample_flags);
360
361 ml_set_interrupts_enabled(s);
362 BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_END, kr);
363 }
364
365 /*
366 * This function allocates >2.3KB of the stack. Prevent the compiler from
367 * inlining this function into ast_taken and ensure the stack memory is only
368 * allocated for the kperf AST.
369 */
370 __attribute__((noinline))
371 void
372 kperf_thread_ast_handler(thread_t thread)
373 {
374 BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_START, thread, kperf_get_thread_flags(thread));
375
376 /* ~2KB of the stack for the sample since this is called from AST */
377 struct kperf_sample sbuf;
378 memset(&sbuf, 0, sizeof(struct kperf_sample));
379
380 task_t task = get_threadtask(thread);
381
382 if (task_did_exec(task) || task_is_exec_copy(task)) {
383 BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, SAMPLE_CONTINUE);
384 return;
385 }
386
387 /* make a context, take a sample */
388 struct kperf_context ctx;
389 ctx.cur_thread = thread;
390 ctx.cur_pid = task_pid(task);
391
392 /* decode the flags to determine what to sample */
393 unsigned int sample_what = 0;
394 uint32_t flags = kperf_get_thread_flags(thread);
395
396 if (flags & T_KPERF_AST_DISPATCH) {
397 sample_what |= SAMPLER_TH_DISPATCH;
398 }
399 if (flags & T_KPERF_AST_CALLSTACK) {
400 sample_what |= SAMPLER_USTACK;
401 sample_what |= SAMPLER_TH_INFO;
402 }
403
404 uint32_t ucallstack_depth = T_KPERF_GET_CALLSTACK_DEPTH(flags);
405
406 int r = kperf_sample_internal(&sbuf, &ctx, sample_what, 0, 0, ucallstack_depth);
407
408 BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, r);
409 }
410
411 /* register AST bits */
412 int
413 kperf_ast_pend(thread_t thread, uint32_t set_flags)
414 {
415 /* can only pend on the current thread */
416 if (thread != current_thread()) {
417 panic("pending to non-current thread");
418 }
419
420 /* get our current bits */
421 uint32_t flags = kperf_get_thread_flags(thread);
422
423 /* see if it's already been done or pended */
424 if (!(flags & set_flags)) {
425 /* set the bit on the thread */
426 flags |= set_flags;
427 kperf_set_thread_flags(thread, flags);
428
429 /* set the actual AST */
430 act_set_kperf(thread);
431 return 1;
432 }
433
434 return 0;
435 }
436
437 void
438 kperf_ast_set_callstack_depth(thread_t thread, uint32_t depth)
439 {
440 uint32_t ast_flags = kperf_get_thread_flags(thread);
441 uint32_t existing_callstack_depth = T_KPERF_GET_CALLSTACK_DEPTH(ast_flags);
442
443 if (existing_callstack_depth != depth) {
444 ast_flags &= ~T_KPERF_SET_CALLSTACK_DEPTH(depth);
445 ast_flags |= T_KPERF_SET_CALLSTACK_DEPTH(depth);
446
447 kperf_set_thread_flags(thread, ast_flags);
448 }
449 }
450
451 int
452 kperf_kdbg_cswitch_get(void)
453 {
454 return kperf_kdebug_cswitch;
455 }
456
457 int
458 kperf_kdbg_cswitch_set(int newval)
459 {
460 kperf_kdebug_cswitch = newval;
461 kperf_on_cpu_update();
462
463 return 0;
464 }
465
466 /*
467 * Action configuration
468 */
469 unsigned int
470 kperf_action_get_count(void)
471 {
472 return actionc;
473 }
474
475 int
476 kperf_action_set_samplers(unsigned actionid, uint32_t samplers)
477 {
478 if ((actionid > actionc) || (actionid == 0)) {
479 return EINVAL;
480 }
481
482 /* disallow both CPU and thread counters to be sampled in the same
483 * action */
484 if ((samplers & SAMPLER_PMC_THREAD) && (samplers & SAMPLER_PMC_CPU)) {
485 return EINVAL;
486 }
487
488 actionv[actionid - 1].sample = samplers;
489
490 return 0;
491 }
492
493 int
494 kperf_action_get_samplers(unsigned actionid, uint32_t *samplers_out)
495 {
496 if ((actionid > actionc)) {
497 return EINVAL;
498 }
499
500 if (actionid == 0) {
501 *samplers_out = 0; /* "NULL" action */
502 } else {
503 *samplers_out = actionv[actionid - 1].sample;
504 }
505
506 return 0;
507 }
508
509 int
510 kperf_action_set_userdata(unsigned actionid, uint32_t userdata)
511 {
512 if ((actionid > actionc) || (actionid == 0)) {
513 return EINVAL;
514 }
515
516 actionv[actionid - 1].userdata = userdata;
517
518 return 0;
519 }
520
521 int
522 kperf_action_get_userdata(unsigned actionid, uint32_t *userdata_out)
523 {
524 if ((actionid > actionc)) {
525 return EINVAL;
526 }
527
528 if (actionid == 0) {
529 *userdata_out = 0; /* "NULL" action */
530 } else {
531 *userdata_out = actionv[actionid - 1].userdata;
532 }
533
534 return 0;
535 }
536
537 int
538 kperf_action_set_filter(unsigned actionid, int pid)
539 {
540 if ((actionid > actionc) || (actionid == 0)) {
541 return EINVAL;
542 }
543
544 actionv[actionid - 1].pid_filter = pid;
545
546 return 0;
547 }
548
549 int
550 kperf_action_get_filter(unsigned actionid, int *pid_out)
551 {
552 if ((actionid > actionc)) {
553 return EINVAL;
554 }
555
556 if (actionid == 0) {
557 *pid_out = -1; /* "NULL" action */
558 } else {
559 *pid_out = actionv[actionid - 1].pid_filter;
560 }
561
562 return 0;
563 }
564
565 void
566 kperf_action_reset(void)
567 {
568 for (unsigned int i = 0; i < actionc; i++) {
569 kperf_action_set_samplers(i + 1, 0);
570 kperf_action_set_userdata(i + 1, 0);
571 kperf_action_set_filter(i + 1, -1);
572 kperf_action_set_ucallstack_depth(i + 1, MAX_CALLSTACK_FRAMES);
573 kperf_action_set_kcallstack_depth(i + 1, MAX_CALLSTACK_FRAMES);
574 }
575 }
576
577 int
578 kperf_action_set_count(unsigned count)
579 {
580 struct action *new_actionv = NULL, *old_actionv = NULL;
581 unsigned old_count;
582
583 /* easy no-op */
584 if (count == actionc) {
585 return 0;
586 }
587
588 /* TODO: allow shrinking? */
589 if (count < actionc) {
590 return EINVAL;
591 }
592
593 /* cap it for good measure */
594 if (count > ACTION_MAX) {
595 return EINVAL;
596 }
597
598 /* creating the action arror for the first time. create a few
599 * more things, too.
600 */
601 if (actionc == 0) {
602 int r;
603 if ((r = kperf_init())) {
604 return r;
605 }
606 }
607
608 /* create a new array */
609 new_actionv = kalloc_tag(count * sizeof(*new_actionv), VM_KERN_MEMORY_DIAG);
610 if (new_actionv == NULL) {
611 return ENOMEM;
612 }
613
614 old_actionv = actionv;
615 old_count = actionc;
616
617 if (old_actionv != NULL) {
618 memcpy(new_actionv, actionv, actionc * sizeof(*actionv));
619 }
620
621 memset(&(new_actionv[actionc]), 0, (count - old_count) * sizeof(*actionv));
622
623 for (unsigned int i = old_count; i < count; i++) {
624 new_actionv[i].pid_filter = -1;
625 new_actionv[i].ucallstack_depth = MAX_CALLSTACK_FRAMES;
626 new_actionv[i].kcallstack_depth = MAX_CALLSTACK_FRAMES;
627 }
628
629 actionv = new_actionv;
630 actionc = count;
631
632 if (old_actionv != NULL) {
633 kfree(old_actionv, old_count * sizeof(*actionv));
634 }
635
636 return 0;
637 }
638
639 int
640 kperf_action_set_ucallstack_depth(unsigned action_id, uint32_t depth)
641 {
642 if ((action_id > actionc) || (action_id == 0)) {
643 return EINVAL;
644 }
645
646 if (depth > MAX_CALLSTACK_FRAMES) {
647 return EINVAL;
648 }
649
650 actionv[action_id - 1].ucallstack_depth = depth;
651
652 return 0;
653 }
654
655 int
656 kperf_action_set_kcallstack_depth(unsigned action_id, uint32_t depth)
657 {
658 if ((action_id > actionc) || (action_id == 0)) {
659 return EINVAL;
660 }
661
662 if (depth > MAX_CALLSTACK_FRAMES) {
663 return EINVAL;
664 }
665
666 actionv[action_id - 1].kcallstack_depth = depth;
667
668 return 0;
669 }
670
671 int
672 kperf_action_get_ucallstack_depth(unsigned action_id, uint32_t * depth_out)
673 {
674 if ((action_id > actionc)) {
675 return EINVAL;
676 }
677
678 assert(depth_out);
679
680 if (action_id == 0) {
681 *depth_out = MAX_CALLSTACK_FRAMES;
682 } else {
683 *depth_out = actionv[action_id - 1].ucallstack_depth;
684 }
685
686 return 0;
687 }
688
689 int
690 kperf_action_get_kcallstack_depth(unsigned action_id, uint32_t * depth_out)
691 {
692 if ((action_id > actionc)) {
693 return EINVAL;
694 }
695
696 assert(depth_out);
697
698 if (action_id == 0) {
699 *depth_out = MAX_CALLSTACK_FRAMES;
700 } else {
701 *depth_out = actionv[action_id - 1].kcallstack_depth;
702 }
703
704 return 0;
705 }