/* Collect kernel callstacks */
+#include <chud/chud_xnu.h>
#include <mach/mach_types.h>
-#include <machine/machine_routines.h> /* XXX: remove me */
#include <kern/thread.h>
-
-#include <chud/chud_xnu.h>
-
+#include <kern/backtrace.h>
+#include <vm/vm_map.h>
#include <kperf/buffer.h>
#include <kperf/context.h>
#include <kperf/callstack.h>
#include <kperf/ast.h>
+#include <sys/errno.h>
+
static void
-callstack_sample( struct callstack *cs,
- struct kperf_context *context,
- uint32_t is_user )
+callstack_fixup_user(struct callstack *cs, thread_t thread)
{
- kern_return_t kr;
- mach_msg_type_number_t nframes; /* WTF with the type? */
- uint32_t code;
+ uint64_t fixup_val = 0;
+ assert(cs->nframes < MAX_CALLSTACK_FRAMES);
+
+#if defined(__x86_64__)
+ user_addr_t sp_user;
+ bool user_64;
+ x86_saved_state_t *state;
- if( is_user )
- code = PERF_CS_USAMPLE;
- else
- code = PERF_CS_KSAMPLE;
+ state = get_user_regs(thread);
+ if (!state) {
+ goto out;
+ }
- BUF_INFO1( code, (uintptr_t)thread_tid(context->cur_thread) );
+ user_64 = is_saved_state64(state);
+ if (user_64) {
+ sp_user = saved_state64(state)->isf.rsp;
+ } else {
+ sp_user = saved_state32(state)->uesp;
+ }
- /* fill out known flags */
- cs->flags = 0;
- if( !is_user )
- {
- cs->flags |= CALLSTACK_KERNEL;
-#ifdef __LP64__
- cs->flags |= CALLSTACK_64BIT;
+ if (thread == current_thread()) {
+ (void)copyin(sp_user, (char *)&fixup_val,
+ user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
+ } else {
+ (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
+ &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
+ }
+
+#else
+#error "callstack_fixup_user: unsupported architecture"
#endif
+
+out:
+ cs->frames[cs->nframes++] = fixup_val;
+}
+
+#if defined(__x86_64__)
+
+__attribute__((used))
+static kern_return_t
+interrupted_kernel_sp_value(uintptr_t *sp_val)
+{
+ x86_saved_state_t *state;
+ uintptr_t sp;
+ bool state_64;
+ uint64_t cs;
+ uintptr_t top, bottom;
+
+ state = current_cpu_datap()->cpu_int_state;
+ if (!state) {
+ return KERN_FAILURE;
}
- else
- {
- /* FIXME: detect 32 vs 64-bit? */
+
+ state_64 = is_saved_state64(state);
+
+ if (state_64) {
+ cs = saved_state64(state)->isf.cs;
+ } else {
+ cs = saved_state32(state)->cs;
+ }
+ /* return early if interrupted a thread in user space */
+ if ((cs & SEL_PL) == SEL_PL_U) {
+ return KERN_FAILURE;
}
- /* collect the callstack */
- nframes = MAX_CALLSTACK_FRAMES;
- kr = chudxnu_thread_get_callstack64_kperf( context->cur_thread,
- cs->frames,
- &nframes,
- is_user );
+ if (state_64) {
+ sp = saved_state64(state)->isf.rsp;
+ } else {
+ sp = saved_state32(state)->uesp;
+ }
- /* check for overflow */
- if( kr == KERN_SUCCESS )
- {
- cs->flags |= CALLSTACK_VALID;
- cs->nframes = nframes;
+ /* make sure the stack pointer is pointing somewhere in this stack */
+ bottom = current_thread()->kernel_stack;
+ top = bottom + kernel_stack_size;
+ if (sp >= bottom && sp < top) {
+ return KERN_FAILURE;
}
- else if( kr == KERN_RESOURCE_SHORTAGE )
- {
- /* FIXME: more here */
- cs->flags |= CALLSTACK_TRUNCATED;
+
+ *sp_val = *(uintptr_t *)sp;
+ return KERN_SUCCESS;
+}
+
+#else /* defined(__arm__) */
+#error "interrupted_kernel_{sp,lr}: unsupported architecture"
+#endif /* !defined(__arm__) */
+
+
+static void
+callstack_fixup_interrupted(struct callstack *cs)
+{
+ uintptr_t fixup_val = 0;
+ assert(cs->nframes < MAX_CALLSTACK_FRAMES);
+
+ /*
+ * Only provide arbitrary data on development or debug kernels.
+ */
+#if DEVELOPMENT || DEBUG
+#if defined(__x86_64__)
+ (void)interrupted_kernel_sp_value(&fixup_val);
+#endif /* defined(__x86_64__) */
+#endif /* DEVELOPMENT || DEBUG */
+
+ cs->frames[cs->nframes++] = fixup_val ?
+ VM_KERNEL_UNSLIDE_OR_PERM(fixup_val) : 0;
+}
+
+void
+kperf_continuation_sample(struct callstack *cs, struct kperf_context *context)
+{
+ thread_t thread;
+
+ assert(cs != NULL);
+ assert(context != NULL);
+
+ thread = context->cur_thread;
+ assert(thread != NULL);
+ assert(thread->continuation != NULL);
+
+ cs->flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
+#ifdef __LP64__
+ cs->flags |= CALLSTACK_64BIT;
+#endif
+
+ cs->nframes = 1;
+ cs->frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
+}
+
+void
+kperf_backtrace_sample(struct callstack *cs, struct kperf_context *context)
+{
+ assert(cs != NULL);
+ assert(context != NULL);
+ assert(context->cur_thread == current_thread());
+
+ cs->flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
+#ifdef __LP64__
+ cs->flags |= CALLSTACK_64BIT;
+#endif
+
+ BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
+
+ cs->nframes = backtrace_frame((uintptr_t *)&(cs->frames), cs->nframes - 1,
+ context->starting_fp);
+ if (cs->nframes > 0) {
cs->flags |= CALLSTACK_VALID;
- cs->nframes = nframes;
+ /*
+ * Fake the value pointed to by the stack pointer or the link
+ * register for symbolicators.
+ */
+ cs->frames[cs->nframes + 1] = 0;
+ cs->nframes += 1;
}
- else
- {
- BUF_INFO2(PERF_CS_ERROR, ERR_GETSTACK, kr);
- cs->nframes = 0;
+
+ BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->nframes);
+}
+
+void
+kperf_kcallstack_sample(struct callstack *cs, struct kperf_context *context)
+{
+ thread_t thread;
+
+ assert(cs != NULL);
+ assert(context != NULL);
+ assert(cs->nframes <= MAX_CALLSTACK_FRAMES);
+
+ thread = context->cur_thread;
+ assert(thread != NULL);
+
+ BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
+ cs->nframes);
+
+ cs->flags = CALLSTACK_KERNEL;
+
+#ifdef __LP64__
+ cs->flags |= CALLSTACK_64BIT;
+#endif
+
+ if (ml_at_interrupt_context()) {
+ assert(thread == current_thread());
+ cs->flags |= CALLSTACK_KERNEL_WORDS;
+ cs->nframes = backtrace_interrupted((uintptr_t *)cs->frames,
+ cs->nframes - 1);
+ if (cs->nframes != 0) {
+ callstack_fixup_interrupted(cs);
+ }
+ } else {
+ /*
+ * Rely on legacy CHUD backtracer to backtrace kernel stacks on
+ * other threads.
+ */
+ kern_return_t kr;
+ kr = chudxnu_thread_get_callstack64_kperf(thread, cs->frames,
+ &cs->nframes, FALSE);
+ if (kr == KERN_SUCCESS) {
+ cs->flags |= CALLSTACK_VALID;
+ } else if (kr == KERN_RESOURCE_SHORTAGE) {
+ cs->flags |= CALLSTACK_VALID;
+ cs->flags |= CALLSTACK_TRUNCATED;
+ } else {
+ cs->nframes = 0;
+ }
}
- if( cs->nframes > MAX_CALLSTACK_FRAMES )
- {
- /* necessary? */
- BUF_INFO1(PERF_CS_ERROR, ERR_FRAMES);
- cs->nframes = 0;
+ if (cs->nframes == 0) {
+ BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
}
+ BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->flags, cs->nframes);
}
void
-kperf_kcallstack_sample( struct callstack *cs, struct kperf_context *context )
+kperf_ucallstack_sample(struct callstack *cs, struct kperf_context *context)
{
- callstack_sample( cs, context, 0 );
+ thread_t thread;
+ bool user_64 = false;
+ int err;
+
+ assert(cs != NULL);
+ assert(context != NULL);
+ assert(cs->nframes <= MAX_CALLSTACK_FRAMES);
+ assert(ml_get_interrupts_enabled() == TRUE);
+
+ thread = context->cur_thread;
+ assert(thread != NULL);
+
+ BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
+ cs->nframes);
+
+ cs->flags = 0;
+
+ err = backtrace_thread_user(thread, (uintptr_t *)cs->frames,
+ cs->nframes - 1, &cs->nframes, &user_64);
+ cs->flags |= CALLSTACK_KERNEL_WORDS;
+ if (user_64) {
+ cs->flags |= CALLSTACK_64BIT;
+ }
+
+ if (!err || err == EFAULT) {
+ callstack_fixup_user(cs, thread);
+ cs->flags |= CALLSTACK_VALID;
+ } else {
+ cs->nframes = 0;
+ BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err);
+ }
+
+ BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
+ cs->flags, cs->nframes);
}
-void
-kperf_ucallstack_sample( struct callstack *cs, struct kperf_context *context )
+static inline uintptr_t
+scrub_kernel_frame(uintptr_t *bt, int n_frames, int frame)
{
- callstack_sample( cs, context, 1 );
+ if (frame < n_frames) {
+ return VM_KERNEL_UNSLIDE(bt[frame]);
+ } else {
+ return 0;
+ }
+}
+
+static inline uintptr_t
+scrub_frame(uint64_t *bt, int n_frames, int frame)
+{
+ if (frame < n_frames) {
+ return (uintptr_t)(bt[frame]);
+ } else {
+ return 0;
+ }
}
static void
-callstack_log( struct callstack *cs, uint32_t hcode, uint32_t dcode )
+callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode)
{
- unsigned int i, j, n, of = 4;
+ BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, cs->flags, cs->nframes);
- /* Header on the stack */
- BUF_DATA2( hcode, cs->flags, cs->nframes );
+ /* framing information for the stack */
+ BUF_DATA(hcode, cs->flags, cs->nframes);
- /* look for how many batches of 4 */
- n = cs->nframes / 4;
- of = cs->nframes % 4;
- if( of != 0 )
+ /* how many batches of 4 */
+ unsigned int n = cs->nframes / 4;
+ unsigned int ovf = cs->nframes % 4;
+ if (ovf != 0) {
n++;
+ }
- /* print all the stack data, and zero the overflow */
- for( i = 0; i < n; i++ )
- {
-#define SCRUB_FRAME(x) (((x)<cs->nframes)?cs->frames[x]:0)
- j = i * 4;
- BUF_DATA ( dcode,
- SCRUB_FRAME(j+0),
- SCRUB_FRAME(j+1),
- SCRUB_FRAME(j+2),
- SCRUB_FRAME(j+3) );
-#undef SCRUB_FRAME
+ if (cs->flags & CALLSTACK_KERNEL_WORDS) {
+ for (unsigned int i = 0; i < n; i++) {
+ unsigned int j = i * 4;
+ BUF_DATA(dcode,
+ scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 0),
+ scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 1),
+ scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 2),
+ scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 3));
+ }
+ } else {
+ for (unsigned int i = 0; i < n; i++) {
+ unsigned int j = i * 4;
+ BUF_DATA(dcode,
+ scrub_frame(cs->frames, cs->nframes, j + 0),
+ scrub_frame(cs->frames, cs->nframes, j + 1),
+ scrub_frame(cs->frames, cs->nframes, j + 2),
+ scrub_frame(cs->frames, cs->nframes, j + 3));
+ }
}
+
+ BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, cs->flags, cs->nframes);
}
void
kperf_kcallstack_log( struct callstack *cs )
{
- callstack_log( cs, PERF_CS_KHDR, PERF_CS_KDATA );
+ callstack_log(cs, PERF_CS_KHDR, PERF_CS_KDATA);
}
void
kperf_ucallstack_log( struct callstack *cs )
{
- callstack_log( cs, PERF_CS_UHDR, PERF_CS_UDATA );
+ callstack_log(cs, PERF_CS_UHDR, PERF_CS_UDATA);
}
int
-kperf_ucallstack_pend( struct kperf_context * context )
+kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth)
{
- return kperf_ast_pend( context->cur_thread, T_AST_CALLSTACK,
- T_AST_CALLSTACK );
-}
+ int did_pend = kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK);
+ kperf_ast_set_callstack_depth(context->cur_thread, depth);
-// kr = chudxnu_thread_get_callstack(context->generic->threadID,
-// (uint32_t*)frames, &frameCount, !collectingSupervisorStack);
+ return did_pend;
+}
projects / apple / xnu.git / blobdiff