X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/316670eb35587141e969394ae8537d66b9211e80..008676633c2ad2c325837c2b64915f7ded690a8f:/osfmk/kperf/callstack.c diff --git a/osfmk/kperf/callstack.c b/osfmk/kperf/callstack.c index d0c1e3947..b45c3f0e7 100644 --- a/osfmk/kperf/callstack.c +++ b/osfmk/kperf/callstack.c @@ -28,140 +28,345 @@ /* Collect kernel callstacks */ +#include #include -#include /* XXX: remove me */ #include - -#include - +#include +#include #include #include #include #include +#include + static void -callstack_sample( struct callstack *cs, - struct kperf_context *context, - uint32_t is_user ) +callstack_fixup_user(struct callstack *cs, thread_t thread) { - kern_return_t kr; - mach_msg_type_number_t nframes; /* WTF with the type? */ - uint32_t code; + uint64_t fixup_val = 0; + assert(cs->nframes < MAX_CALLSTACK_FRAMES); + +#if defined(__x86_64__) + user_addr_t sp_user; + bool user_64; + x86_saved_state_t *state; - if( is_user ) - code = PERF_CS_USAMPLE; - else - code = PERF_CS_KSAMPLE; + state = get_user_regs(thread); + if (!state) { + goto out; + } - BUF_INFO1( code, (uintptr_t)context->cur_thread ); + user_64 = is_saved_state64(state); + if (user_64) { + sp_user = saved_state64(state)->isf.rsp; + } else { + sp_user = saved_state32(state)->uesp; + } - /* fill out known flags */ - cs->flags = 0; - if( !is_user ) - { - cs->flags |= CALLSTACK_KERNEL; -#ifdef __LP64__ - cs->flags |= CALLSTACK_64BIT; + if (thread == current_thread()) { + (void)copyin(sp_user, (char *)&fixup_val, + user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); + } else { + (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user, + &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); + } + +#else +#error "callstack_fixup_user: unsupported architecture" #endif + +out: + cs->frames[cs->nframes++] = fixup_val; +} + +#if defined(__x86_64__) + +__attribute__((used)) +static kern_return_t +interrupted_kernel_sp_value(uintptr_t *sp_val) +{ + x86_saved_state_t *state; + uintptr_t sp; + bool state_64; + uint64_t cs; + uintptr_t top, bottom; + + state = current_cpu_datap()->cpu_int_state; + if (!state) { + return KERN_FAILURE; } - else - { - /* FIXME: detect 32 vs 64-bit? */ + + state_64 = is_saved_state64(state); + + if (state_64) { + cs = saved_state64(state)->isf.cs; + } else { + cs = saved_state32(state)->cs; + } + /* return early if interrupted a thread in user space */ + if ((cs & SEL_PL) == SEL_PL_U) { + return KERN_FAILURE; } - /* collect the callstack */ - nframes = MAX_CALLSTACK_FRAMES; - kr = chudxnu_thread_get_callstack64( context->cur_thread, - cs->frames, - &nframes, - is_user ); + if (state_64) { + sp = saved_state64(state)->isf.rsp; + } else { + sp = saved_state32(state)->uesp; + } - /* check for overflow */ - if( kr == KERN_SUCCESS ) - { - cs->flags |= CALLSTACK_VALID; - cs->nframes = nframes; + /* make sure the stack pointer is pointing somewhere in this stack */ + bottom = current_thread()->kernel_stack; + top = bottom + kernel_stack_size; + if (sp >= bottom && sp < top) { + return KERN_FAILURE; } - else if( kr == KERN_RESOURCE_SHORTAGE ) - { - /* FIXME: more here */ - cs->flags |= CALLSTACK_TRUNCATED; + + *sp_val = *(uintptr_t *)sp; + return KERN_SUCCESS; +} + +#else /* defined(__arm__) */ +#error "interrupted_kernel_{sp,lr}: unsupported architecture" +#endif /* !defined(__arm__) */ + + +static void +callstack_fixup_interrupted(struct callstack *cs) +{ + uintptr_t fixup_val = 0; + assert(cs->nframes < MAX_CALLSTACK_FRAMES); + + /* + * Only provide arbitrary data on development or debug kernels. + */ +#if DEVELOPMENT || DEBUG +#if defined(__x86_64__) + (void)interrupted_kernel_sp_value(&fixup_val); +#endif /* defined(__x86_64__) */ +#endif /* DEVELOPMENT || DEBUG */ + + cs->frames[cs->nframes++] = fixup_val ? + VM_KERNEL_UNSLIDE_OR_PERM(fixup_val) : 0; +} + +void +kperf_continuation_sample(struct callstack *cs, struct kperf_context *context) +{ + thread_t thread; + + assert(cs != NULL); + assert(context != NULL); + + thread = context->cur_thread; + assert(thread != NULL); + assert(thread->continuation != NULL); + + cs->flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL; +#ifdef __LP64__ + cs->flags |= CALLSTACK_64BIT; +#endif + + cs->nframes = 1; + cs->frames[0] = VM_KERNEL_UNSLIDE(thread->continuation); +} + +void +kperf_backtrace_sample(struct callstack *cs, struct kperf_context *context) +{ + assert(cs != NULL); + assert(context != NULL); + assert(context->cur_thread == current_thread()); + + cs->flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS; +#ifdef __LP64__ + cs->flags |= CALLSTACK_64BIT; +#endif + + BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1); + + cs->nframes = backtrace_frame((uintptr_t *)&(cs->frames), cs->nframes - 1, + context->starting_fp); + if (cs->nframes > 0) { cs->flags |= CALLSTACK_VALID; - cs->nframes = nframes; + /* + * Fake the value pointed to by the stack pointer or the link + * register for symbolicators. + */ + cs->frames[cs->nframes + 1] = 0; + cs->nframes += 1; } - else - { - BUF_INFO2(PERF_PET_ERROR, ERR_GETSTACK, kr); - cs->nframes = 0; + + BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->nframes); +} + +void +kperf_kcallstack_sample(struct callstack *cs, struct kperf_context *context) +{ + thread_t thread; + + assert(cs != NULL); + assert(context != NULL); + assert(cs->nframes <= MAX_CALLSTACK_FRAMES); + + thread = context->cur_thread; + assert(thread != NULL); + + BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread), + cs->nframes); + + cs->flags = CALLSTACK_KERNEL; + +#ifdef __LP64__ + cs->flags |= CALLSTACK_64BIT; +#endif + + if (ml_at_interrupt_context()) { + assert(thread == current_thread()); + cs->flags |= CALLSTACK_KERNEL_WORDS; + cs->nframes = backtrace_interrupted((uintptr_t *)cs->frames, + cs->nframes - 1); + if (cs->nframes != 0) { + callstack_fixup_interrupted(cs); + } + } else { + /* + * Rely on legacy CHUD backtracer to backtrace kernel stacks on + * other threads. + */ + kern_return_t kr; + kr = chudxnu_thread_get_callstack64_kperf(thread, cs->frames, + &cs->nframes, FALSE); + if (kr == KERN_SUCCESS) { + cs->flags |= CALLSTACK_VALID; + } else if (kr == KERN_RESOURCE_SHORTAGE) { + cs->flags |= CALLSTACK_VALID; + cs->flags |= CALLSTACK_TRUNCATED; + } else { + cs->nframes = 0; + } } - if( cs->nframes >= MAX_CALLSTACK_FRAMES ) - { - /* necessary? */ - BUF_INFO1(PERF_PET_ERROR, ERR_FRAMES); - cs->nframes = 0; + if (cs->nframes == 0) { + BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK); } + BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->flags, cs->nframes); } void -kperf_kcallstack_sample( struct callstack *cs, struct kperf_context *context ) +kperf_ucallstack_sample(struct callstack *cs, struct kperf_context *context) { - callstack_sample( cs, context, 0 ); + thread_t thread; + bool user_64 = false; + int err; + + assert(cs != NULL); + assert(context != NULL); + assert(cs->nframes <= MAX_CALLSTACK_FRAMES); + assert(ml_get_interrupts_enabled() == TRUE); + + thread = context->cur_thread; + assert(thread != NULL); + + BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread), + cs->nframes); + + cs->flags = 0; + + err = backtrace_thread_user(thread, (uintptr_t *)cs->frames, + cs->nframes - 1, &cs->nframes, &user_64); + cs->flags |= CALLSTACK_KERNEL_WORDS; + if (user_64) { + cs->flags |= CALLSTACK_64BIT; + } + + if (!err || err == EFAULT) { + callstack_fixup_user(cs, thread); + cs->flags |= CALLSTACK_VALID; + } else { + cs->nframes = 0; + BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err); + } + + BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), + cs->flags, cs->nframes); } -void -kperf_ucallstack_sample( struct callstack *cs, struct kperf_context *context ) +static inline uintptr_t +scrub_kernel_frame(uintptr_t *bt, int n_frames, int frame) { - callstack_sample( cs, context, 1 ); + if (frame < n_frames) { + return VM_KERNEL_UNSLIDE(bt[frame]); + } else { + return 0; + } +} + +static inline uintptr_t +scrub_frame(uint64_t *bt, int n_frames, int frame) +{ + if (frame < n_frames) { + return (uintptr_t)(bt[frame]); + } else { + return 0; + } } static void -callstack_log( struct callstack *cs, uint32_t code ) +callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode) { - unsigned int i, j, n, of = 4; + BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, cs->flags, cs->nframes); - /* Header on the stack */ - BUF_DATA2( code, cs->flags, cs->nframes ); + /* framing information for the stack */ + BUF_DATA(hcode, cs->flags, cs->nframes); - /* look for how many batches of 4 */ - n = cs->nframes / 4; - of = cs->nframes % 4; - if( of != 0 ) + /* how many batches of 4 */ + unsigned int n = cs->nframes / 4; + unsigned int ovf = cs->nframes % 4; + if (ovf != 0) { n++; + } - /* print all the stack data, and zero the overflow */ - for( i = 0; i < n; i++ ) - { -#define SCRUB_FRAME(x) (((x)nframes)?cs->frames[x]:0) - j = i * 4; - BUF_DATA ( code, - SCRUB_FRAME(j+0), - SCRUB_FRAME(j+1), - SCRUB_FRAME(j+2), - SCRUB_FRAME(j+3) ); -#undef SCRUB_FRAME + if (cs->flags & CALLSTACK_KERNEL_WORDS) { + for (unsigned int i = 0; i < n; i++) { + unsigned int j = i * 4; + BUF_DATA(dcode, + scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 0), + scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 1), + scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 2), + scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 3)); + } + } else { + for (unsigned int i = 0; i < n; i++) { + unsigned int j = i * 4; + BUF_DATA(dcode, + scrub_frame(cs->frames, cs->nframes, j + 0), + scrub_frame(cs->frames, cs->nframes, j + 1), + scrub_frame(cs->frames, cs->nframes, j + 2), + scrub_frame(cs->frames, cs->nframes, j + 3)); + } } + + BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, cs->flags, cs->nframes); } void kperf_kcallstack_log( struct callstack *cs ) { - callstack_log( cs, PERF_CS_KDATA ); + callstack_log(cs, PERF_CS_KHDR, PERF_CS_KDATA); } void kperf_ucallstack_log( struct callstack *cs ) { - callstack_log( cs, PERF_CS_UDATA ); + callstack_log(cs, PERF_CS_UHDR, PERF_CS_UDATA); } int -kperf_ucallstack_pend( struct kperf_context * context ) +kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth) { - return kperf_ast_pend( context->cur_thread, T_AST_CALLSTACK, - T_AST_CALLSTACK ); -} + int did_pend = kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK); + kperf_ast_set_callstack_depth(context->cur_thread, depth); -// kr = chudxnu_thread_get_callstack(context->generic->threadID, -// (uint32_t*)frames, &frameCount, !collectingSupervisorStack); + return did_pend; +}