2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
30 * Called from a trigger. Actually takes the data from the different
31 * modules and puts them in a buffer
34 #include <mach/mach_types.h>
35 #include <machine/machine_routines.h>
36 // #include <libkern/libkern.h>
37 #include <kern/kalloc.h>
38 #include <kern/debug.h> /* panic */
39 #include <kern/thread.h>
40 #include <sys/errno.h>
42 #include <chud/chud_xnu.h>
43 #include <kperf/kperf.h>
45 #include <kperf/buffer.h>
46 #include <kperf/timetrigger.h>
47 #include <kperf/threadinfo.h>
48 #include <kperf/callstack.h>
49 #include <kperf/sample.h>
50 #include <kperf/action.h>
51 #include <kperf/context.h>
52 #include <kperf/ast.h>
53 #include <kperf/kperf_kpc.h>
57 /* the list of different actions to take */
65 /* the list of actions */
66 static unsigned actionc
= 0;
67 static struct action
*actionv
= NULL
;
69 /* manage callbacks from system */
71 /* callback set for kdebug */
72 static int kperf_kdbg_callback_set
= 0;
73 /* whether to record callstacks on kdebug events */
74 static int kdebug_callstacks
= 0;
75 /* the action ID to trigger on signposts */
76 static int kperf_signpost_action
= 0;
78 /* callback set for context-switch */
79 int kperf_cswitch_callback_set
= 0;
80 /* should emit tracepoint on context switch */
81 static int kdebug_cswitch
= 0;
82 /* the action ID to trigger on context switches */
83 static int kperf_cswitch_action
= 0;
85 /* indirect hooks to play nice with CHUD for the transition to kperf */
86 kern_return_t
chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t fn
);
87 kern_return_t
chudxnu_kdebug_callback_cancel(void);
89 /* Do the real work! */
90 /* this can be called in any context ... right? */
92 kperf_sample_internal(struct kperf_sample
*sbuf
,
93 struct kperf_context
*context
,
94 unsigned sample_what
, unsigned sample_flags
,
98 int did_ucallstack
= 0, did_tinfo_extra
= 0;
101 /* not much point continuing here, but what to do ? return
102 * Shutdown? cut a tracepoint and continue?
104 if (sample_what
== 0) {
105 return SAMPLE_CONTINUE
;
108 int is_kernel
= (context
->cur_pid
== 0);
110 sbuf
->kcallstack
.nframes
= 0;
111 sbuf
->kcallstack
.flags
= CALLSTACK_VALID
;
112 sbuf
->ucallstack
.nframes
= 0;
113 sbuf
->ucallstack
.flags
= CALLSTACK_VALID
;
115 /* an event occurred. Sample everything and dump it in a
119 /* collect data from samplers */
120 if (sample_what
& SAMPLER_TINFO
) {
121 kperf_threadinfo_sample(&sbuf
->threadinfo
, context
);
123 /* See if we should drop idle thread samples */
124 if (!(sample_flags
& SAMPLE_FLAG_IDLE_THREADS
)) {
125 if (sbuf
->threadinfo
.runmode
& 0x40) {
126 return SAMPLE_CONTINUE
;
131 if ((sample_what
& SAMPLER_KSTACK
) && !(sample_flags
& SAMPLE_FLAG_EMPTY_CALLSTACK
)) {
132 kperf_kcallstack_sample(&(sbuf
->kcallstack
), context
);
137 if (sample_what
& SAMPLER_MEMINFO
) {
138 kperf_meminfo_sample(&(sbuf
->meminfo
), context
);
141 if (sample_flags
& SAMPLE_FLAG_PEND_USER
) {
142 if ((sample_what
& SAMPLER_USTACK
)
143 && !(sample_flags
& SAMPLE_FLAG_EMPTY_CALLSTACK
))
145 did_ucallstack
= kperf_ucallstack_pend(context
);
148 if (sample_what
& SAMPLER_TINFOEX
) {
149 did_tinfo_extra
= kperf_threadinfo_extra_pend(context
);
152 if ((sample_what
& SAMPLER_USTACK
)
153 && !(sample_flags
& SAMPLE_FLAG_EMPTY_CALLSTACK
))
155 kperf_ucallstack_sample(&(sbuf
->ucallstack
), context
);
158 if (sample_what
& SAMPLER_TINFOEX
) {
159 kperf_threadinfo_extra_sample(&(sbuf
->tinfo_ex
),
165 if (sample_what
& SAMPLER_PMC_THREAD
) {
166 kperf_kpc_thread_sample(&(sbuf
->kpcdata
), sample_what
);
167 } else if (sample_what
& SAMPLER_PMC_CPU
) {
168 kperf_kpc_cpu_sample(&(sbuf
->kpcdata
), sample_what
);
171 /* lookup the user tag, if any */
172 if (actionid
&& (actionid
<= actionc
)) {
173 userdata
= actionv
[actionid
- 1].userdata
;
178 /* stash the data into the buffer
179 * interrupts off to ensure we don't get split
181 enabled
= ml_set_interrupts_enabled(FALSE
);
183 BUF_DATA(PERF_GEN_EVENT
| DBG_FUNC_START
, sample_what
,
184 actionid
, userdata
, sample_flags
);
186 /* dump threadinfo */
187 if (sample_what
& SAMPLER_TINFO
) {
188 kperf_threadinfo_log( &sbuf
->threadinfo
);
191 /* dump kcallstack */
192 if (sample_what
& SAMPLER_KSTACK
) {
193 kperf_kcallstack_log( &sbuf
->kcallstack
);
196 /* dump user stuff */
199 if (sample_what
& SAMPLER_MEMINFO
) {
200 kperf_meminfo_log(&(sbuf
->meminfo
));
203 if (sample_flags
& SAMPLE_FLAG_PEND_USER
) {
204 if (did_ucallstack
) {
205 BUF_INFO1(PERF_CS_UPEND
, 0);
208 if (did_tinfo_extra
) {
209 BUF_INFO1(PERF_TI_XPEND
, 0);
212 if (sample_what
& SAMPLER_USTACK
) {
213 kperf_ucallstack_log(&(sbuf
->ucallstack
));
216 if (sample_what
& SAMPLER_TINFOEX
) {
217 kperf_threadinfo_extra_log(&(sbuf
->tinfo_ex
));
222 if (sample_what
& SAMPLER_PMC_THREAD
) {
223 kperf_kpc_thread_log(&(sbuf
->kpcdata
));
224 } else if (sample_what
& SAMPLER_PMC_CPU
) {
225 kperf_kpc_cpu_log(&(sbuf
->kpcdata
));
228 BUF_DATA1(PERF_GEN_EVENT
| DBG_FUNC_END
, sample_what
);
231 ml_set_interrupts_enabled(enabled
);
233 return SAMPLE_CONTINUE
;
236 /* Translate actionid into sample bits and take a sample */
238 kperf_sample(struct kperf_sample
*sbuf
,
239 struct kperf_context
*context
,
240 unsigned actionid
, unsigned sample_flags
)
242 unsigned sample_what
= 0;
245 /* work out what to sample, if anything */
246 if ((actionid
> actionc
) || (actionid
== 0)) {
247 return SAMPLE_SHUTDOWN
;
250 /* check the pid filter against the context's current pid.
251 * filter pid == -1 means any pid
253 pid_filter
= actionv
[actionid
- 1].pid_filter
;
254 if ((pid_filter
!= -1) && (pid_filter
!= context
->cur_pid
)) {
255 return SAMPLE_CONTINUE
;
258 /* the samplers to run */
259 sample_what
= actionv
[actionid
- 1].sample
;
261 /* do the actual sample operation */
262 return kperf_sample_internal(sbuf
, context
, sample_what
,
263 sample_flags
, actionid
);
266 /* ast callback on a thread */
268 kperf_thread_ast_handler(thread_t thread
)
272 unsigned sample_what
= 0;
273 /* we know we're on a thread, so let's do stuff */
276 BUF_INFO1(PERF_AST_HNDLR
| DBG_FUNC_START
, thread
);
278 /* use ~2kb of the stack for the sample, should be ok since we're in the ast */
279 struct kperf_sample sbuf
;
280 memset(&sbuf
, 0, sizeof(struct kperf_sample
));
282 /* make a context, take a sample */
283 struct kperf_context ctx
;
284 ctx
.cur_thread
= thread
;
287 task
= chudxnu_task_for_thread(thread
);
289 ctx
.cur_pid
= chudxnu_pid_for_task(task
);
292 /* decode the chud bits so we know what to sample */
293 t_chud
= kperf_get_thread_bits(thread
);
295 if (t_chud
& T_AST_NAME
) {
296 sample_what
|= SAMPLER_TINFOEX
;
299 if (t_chud
& T_AST_CALLSTACK
) {
300 sample_what
|= SAMPLER_USTACK
;
301 sample_what
|= SAMPLER_TINFO
;
304 /* do the sample, just of the user stuff */
305 r
= kperf_sample_internal(&sbuf
, &ctx
, sample_what
, 0, 0);
307 BUF_INFO1(PERF_AST_HNDLR
| DBG_FUNC_END
, r
);
310 /* register AST bits */
312 kperf_ast_pend(thread_t cur_thread
, uint32_t check_bits
,
315 /* pend on the thread */
316 uint32_t t_chud
, set_done
= 0;
318 /* can only pend on the current thread */
319 if (cur_thread
!= chudxnu_current_thread()) {
320 panic("pending to non-current thread");
323 /* get our current bits */
324 t_chud
= kperf_get_thread_bits(cur_thread
);
326 /* see if it's already been done or pended */
327 if (!(t_chud
& check_bits
)) {
328 /* set the bit on the thread */
330 kperf_set_thread_bits(cur_thread
, t_chud
);
332 /* set the actual AST */
333 kperf_set_thread_ast(cur_thread
);
342 * kdebug callback & stack management
345 #define IS_END(debugid) ((debugid & 3) == DBG_FUNC_END)
346 #define IS_MIG(debugid) (IS_END(debugid) && ((debugid & 0xff000000U) == KDBG_CLASS_ENCODE((unsigned)DBG_MIG, 0U)))
347 #define IS_MACH_SYSCALL(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_EXCP_SC)))
348 #define IS_VM_FAULT(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_VM)))
349 #define IS_BSD_SYSCTLL(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_BSD, DBG_BSD_EXCP_SC)))
350 #define IS_APPS_SIGNPOST(debugid) (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_APPS, DBG_MACH_CHUD))
351 #define IS_MACH_SIGNPOST(debugid) (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_CHUD))
352 #define IS_ENERGYTRACE(debugid) ((debugid & 0xff000000U) == KDBG_CLASS_ENCODE((unsigned)DBG_ENERGYTRACE, 0U))
355 kperf_kdebug_callback(uint32_t debugid
)
360 if (!kdebug_callstacks
&& !kperf_signpost_action
) {
364 /* if we're looking at a kperf tracepoint, don't recurse */
365 if ((debugid
& 0xff000000) == KDBG_CLASS_ENCODE(DBG_PERF
, 0)) {
369 /* ensure interrupts are already off thanks to kdebug */
370 if (ml_get_interrupts_enabled()) {
374 /* make sure we're not being called recursively. */
376 if (kperf_kdbg_recurse(KPERF_RECURSE_IN
)) {
381 /* check the happy list of trace codes */
383 || IS_MACH_SYSCALL(debugid
)
384 || IS_VM_FAULT(debugid
)
385 || IS_BSD_SYSCTLL(debugid
)
386 || IS_MACH_SIGNPOST(debugid
)
387 || IS_ENERGYTRACE(debugid
)
388 || IS_APPS_SIGNPOST(debugid
)))
393 /* check for kernel */
394 thread_t thread
= chudxnu_current_thread();
395 task
= chudxnu_task_for_thread(thread
);
397 cur_pid
= chudxnu_pid_for_task(task
);
403 if (kdebug_callstacks
) {
404 /* dicing with death */
405 BUF_INFO2(PERF_KDBG_HNDLR
, debugid
, cur_pid
);
408 kperf_ast_pend( thread
, T_AST_CALLSTACK
, T_AST_CALLSTACK
);
411 if (kperf_signpost_action
&& (IS_MACH_SIGNPOST(debugid
)
412 || IS_APPS_SIGNPOST(debugid
)))
415 /* make sure we're not being called recursively. */
416 if(kperf_kdbg_recurse(KPERF_RECURSE_IN
)) {
421 /* setup a context */
422 struct kperf_context ctx
;
423 struct kperf_sample
*intbuf
= NULL
;
424 BUF_INFO2(PERF_SIGNPOST_HNDLR
| DBG_FUNC_START
, debugid
, cur_pid
);
426 ctx
.cur_thread
= thread
;
427 ctx
.cur_pid
= cur_pid
;
428 ctx
.trigger_type
= TRIGGER_TYPE_TRACE
;
431 /* CPU sample buffer -- only valid with interrupts off (above)
432 * Technically this isn't true -- tracepoints can, and often
433 * are, cut from interrupt handlers, but none of those tracepoints
434 * should make it this far.
436 intbuf
= kperf_intr_sample_buffer();
439 kperf_sample(intbuf
, &ctx
, kperf_signpost_action
,
440 SAMPLE_FLAG_PEND_USER
);
442 BUF_INFO2(PERF_SIGNPOST_HNDLR
| DBG_FUNC_END
, debugid
, cur_pid
);
444 /* no longer recursive */
445 kperf_kdbg_recurse(KPERF_RECURSE_OUT
);
451 kperf_kdbg_callback_update(void)
453 unsigned old_callback_set
= kperf_kdbg_callback_set
;
455 /* compute new callback state */
456 kperf_kdbg_callback_set
= kdebug_callstacks
|| kperf_signpost_action
;
458 if (old_callback_set
&& !kperf_kdbg_callback_set
) {
459 /* callback should no longer be set */
460 chudxnu_kdebug_callback_cancel();
461 } else if (!old_callback_set
&& kperf_kdbg_callback_set
) {
462 /* callback must now be set */
463 chudxnu_kdebug_callback_enter(NULL
);
468 kperf_kdbg_get_stacks(void)
470 return kdebug_callstacks
;
474 kperf_kdbg_set_stacks(int newval
)
476 kdebug_callstacks
= newval
;
477 kperf_kdbg_callback_update();
483 kperf_signpost_action_get(void)
485 return kperf_signpost_action
;
489 kperf_signpost_action_set(int newval
)
491 kperf_signpost_action
= newval
;
492 kperf_kdbg_callback_update();
501 /* called from context switch handler */
503 kperf_switch_context(__unused thread_t old
, thread_t
new)
505 task_t task
= get_threadtask(new);
506 int pid
= chudxnu_pid_for_task(task
);
508 /* cut a tracepoint to tell us what the new thread's PID is
511 BUF_DATA2(PERF_TI_CSWITCH
, thread_tid(new), pid
);
513 /* trigger action after counters have been updated */
514 if (kperf_cswitch_action
) {
515 struct kperf_sample sbuf
;
516 struct kperf_context ctx
;
519 BUF_DATA1(PERF_CSWITCH_HNDLR
| DBG_FUNC_START
, 0);
522 ctx
.cur_thread
= old
;
524 /* get PID for context */
525 task_t old_task
= chudxnu_task_for_thread(ctx
.cur_thread
);
527 ctx
.cur_pid
= chudxnu_pid_for_task(old_task
);
530 ctx
.trigger_type
= TRIGGER_TYPE_CSWITCH
;
533 r
= kperf_sample(&sbuf
, &ctx
, kperf_cswitch_action
,
534 SAMPLE_FLAG_PEND_USER
);
536 BUF_INFO1(PERF_CSWITCH_HNDLR
| DBG_FUNC_END
, r
);
541 kperf_cswitch_callback_update(void)
543 unsigned old_callback_set
= kperf_cswitch_callback_set
;
545 unsigned new_callback_set
= kdebug_cswitch
|| kperf_cswitch_action
;
547 if (old_callback_set
&& !new_callback_set
) {
548 kperf_cswitch_callback_set
= 0;
549 } else if (!old_callback_set
&& new_callback_set
) {
550 kperf_cswitch_callback_set
= 1;
555 kperf_kpc_cswitch_callback_update();
559 kperf_kdbg_cswitch_get(void)
561 return kdebug_cswitch
;
565 kperf_kdbg_cswitch_set(int newval
)
567 kdebug_cswitch
= newval
;
568 kperf_cswitch_callback_update();
574 kperf_cswitch_action_get(void)
576 return kperf_cswitch_action
;
580 kperf_cswitch_action_set(int newval
)
582 kperf_cswitch_action
= newval
;
583 kperf_cswitch_callback_update();
589 * Action configuration
592 kperf_action_get_count(void)
598 kperf_action_set_samplers(unsigned actionid
, uint32_t samplers
)
600 if ((actionid
> actionc
) || (actionid
== 0)) {
604 /* disallow both CPU and thread counters to be sampled in the same
606 if ((samplers
& SAMPLER_PMC_THREAD
) && (samplers
& SAMPLER_PMC_CPU
)) {
610 actionv
[actionid
- 1].sample
= samplers
;
616 kperf_action_get_samplers(unsigned actionid
, uint32_t *samplers_out
)
618 if ((actionid
> actionc
)) {
623 *samplers_out
= 0; /* "NULL" action */
625 *samplers_out
= actionv
[actionid
- 1].sample
;
632 kperf_action_set_userdata(unsigned actionid
, uint32_t userdata
)
634 if ((actionid
> actionc
) || (actionid
== 0)) {
638 actionv
[actionid
- 1].userdata
= userdata
;
644 kperf_action_get_userdata(unsigned actionid
, uint32_t *userdata_out
)
646 if ((actionid
> actionc
)) {
651 *userdata_out
= 0; /* "NULL" action */
653 *userdata_out
= actionv
[actionid
- 1].userdata
;
660 kperf_action_set_filter(unsigned actionid
, int pid
)
662 if ((actionid
> actionc
) || (actionid
== 0)) {
666 actionv
[actionid
- 1].pid_filter
= pid
;
672 kperf_action_get_filter(unsigned actionid
, int *pid_out
)
674 if ((actionid
> actionc
)) {
679 *pid_out
= -1; /* "NULL" action */
681 *pid_out
= actionv
[actionid
- 1].pid_filter
;
688 kperf_action_set_count(unsigned count
)
690 struct action
*new_actionv
= NULL
, *old_actionv
= NULL
;
691 unsigned old_count
, i
;
694 if (count
== actionc
) {
698 /* TODO: allow shrinking? */
699 if (count
< actionc
) {
703 /* cap it for good measure */
704 if (count
> ACTION_MAX
) {
708 /* creating the action arror for the first time. create a few
720 /* create a new array */
721 new_actionv
= kalloc(count
* sizeof(*new_actionv
));
722 if (new_actionv
== NULL
) {
726 old_actionv
= actionv
;
729 if (old_actionv
!= NULL
) {
730 memcpy(new_actionv
, actionv
, actionc
* sizeof(*actionv
));
733 memset(&(new_actionv
[actionc
]), 0, (count
- old_count
) * sizeof(*actionv
));
735 for (i
= old_count
; i
< count
; i
++) {
736 new_actionv
[i
].pid_filter
= -1;
739 actionv
= new_actionv
;
742 if (old_actionv
!= NULL
) {
743 kfree(old_actionv
, old_count
* sizeof(*actionv
));