X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3903760236c30e3b5ace7a4eefac3a269d68957c..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/osfmk/kperf/action.c diff --git a/osfmk/kperf/action.c b/osfmk/kperf/action.c index db437d771..7c1814e51 100644 --- a/osfmk/kperf/action.c +++ b/osfmk/kperf/action.c @@ -2,7 +2,7 @@ * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -38,6 +38,9 @@ #include #include #include +#include +#include +#include #include #include @@ -47,7 +50,7 @@ #include #include #include -#include +#include #include #include #include @@ -55,8 +58,7 @@ #define ACTION_MAX (32) /* the list of different actions to take */ -struct action -{ +struct action { uint32_t sample; uint32_t ucallstack_depth; uint32_t kcallstack_depth; @@ -65,24 +67,145 @@ struct action }; /* the list of actions */ -static unsigned actionc = 0; +static unsigned int actionc = 0; static struct action *actionv = NULL; /* should emit tracepoint on context switch */ int kperf_kdebug_cswitch = 0; +bool +kperf_action_has_non_system(unsigned int actionid) +{ + if (actionid > actionc) { + return false; + } + + if (actionv[actionid - 1].sample & ~SAMPLER_SYS_MEM) { + return true; + } else { + return false; + } +} + +bool +kperf_action_has_task(unsigned int actionid) +{ + if (actionid > actionc) { + return false; + } + + return actionv[actionid - 1].sample & SAMPLER_TASK_MASK; +} + +bool +kperf_action_has_thread(unsigned int actionid) +{ + if (actionid > actionc) { + return false; + } + + return actionv[actionid - 1].sample & SAMPLER_THREAD_MASK; +} + +static void +kperf_system_memory_log(void) +{ + BUF_DATA(PERF_MI_SYS_DATA, (uintptr_t)vm_page_free_count, + (uintptr_t)vm_page_wire_count, (uintptr_t)vm_page_external_count, + (uintptr_t)(vm_page_active_count + vm_page_inactive_count + + vm_page_speculative_count)); + BUF_DATA(PERF_MI_SYS_DATA_2, (uintptr_t)vm_page_anonymous_count, + (uintptr_t)vm_page_internal_count, + (uintptr_t)vm_pageout_vminfo.vm_pageout_compressions, + (uintptr_t)VM_PAGE_COMPRESSOR_COUNT); +} + +static void +kperf_sample_user_internal(struct kperf_usample *sbuf, + struct kperf_context *context, unsigned int actionid, + unsigned int sample_what) +{ + if (sample_what & SAMPLER_USTACK) { + kperf_ucallstack_sample(&sbuf->ucallstack, context); + } + if (sample_what & SAMPLER_TH_DISPATCH) { + kperf_thread_dispatch_sample(&sbuf->th_dispatch, context); + } + if (sample_what & SAMPLER_TH_INFO) { + kperf_thread_info_sample(&sbuf->th_info, context); + } + + boolean_t intren = ml_set_interrupts_enabled(FALSE); + + /* + * No userdata or sample_flags for this one. + */ + BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, actionid); + + if (sample_what & SAMPLER_USTACK) { + kperf_ucallstack_log(&sbuf->ucallstack); + } + if (sample_what & SAMPLER_TH_DISPATCH) { + kperf_thread_dispatch_log(&sbuf->th_dispatch); + } + if (sample_what & SAMPLER_TH_INFO) { + kperf_thread_info_log(&sbuf->th_info); + } + + BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what); + + ml_set_interrupts_enabled(intren); +} + +void +kperf_sample_user(struct kperf_usample *sbuf, struct kperf_context *context, + unsigned int actionid, unsigned int sample_flags) +{ + if (actionid == 0 || actionid > actionc) { + return; + } + + unsigned int sample_what = actionv[actionid - 1].sample; + unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth; + + /* callstacks should be explicitly ignored */ + if (sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) { + sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK); + } + if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) { + sample_what &= SAMPLER_SYS_MEM; + } + assert((sample_flags & (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)) + != (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)); + if (sample_flags & SAMPLE_FLAG_THREAD_ONLY) { + sample_what &= SAMPLER_THREAD_MASK; + } + if (sample_flags & SAMPLE_FLAG_TASK_ONLY) { + sample_what &= SAMPLER_TASK_MASK; + } + + if (sample_what == 0) { + return; + } + + sbuf->ucallstack.kpuc_nframes = ucallstack_depth ?: + MAX_UCALLSTACK_FRAMES; + + kperf_sample_user_internal(sbuf, context, actionid, sample_what); +} + static kern_return_t kperf_sample_internal(struct kperf_sample *sbuf, - struct kperf_context *context, - unsigned sample_what, unsigned sample_flags, - unsigned actionid, uint32_t ucallstack_depth) + struct kperf_context *context, + unsigned sample_what, unsigned sample_flags, + unsigned actionid, unsigned ucallstack_depth) { int pended_ucallstack = 0; int pended_th_dispatch = 0; + bool on_idle_thread = false; + uint32_t userdata = actionid; + bool task_only = false; - /* not much point continuing here, but what to do ? return - * Shutdown? cut a tracepoint and continue? - */ if (sample_what == 0) { return SAMPLE_CONTINUE; } @@ -92,36 +215,44 @@ kperf_sample_internal(struct kperf_sample *sbuf, sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK); } - context->cur_thread->kperf_pet_gen = kperf_pet_gen; - boolean_t is_kernel = (context->cur_pid == 0); + if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) { + sample_what &= SAMPLER_SYS_MEM; + } - if (actionid && actionid <= actionc) { - sbuf->kcallstack.nframes = actionv[actionid - 1].kcallstack_depth; - } else { - sbuf->kcallstack.nframes = MAX_CALLSTACK_FRAMES; + assert((sample_flags & (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)) + != (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY)); + if (sample_flags & SAMPLE_FLAG_THREAD_ONLY) { + sample_what &= SAMPLER_THREAD_MASK; + } + if (sample_flags & SAMPLE_FLAG_TASK_ONLY) { + task_only = true; + sample_what &= SAMPLER_TASK_MASK; } - if (ucallstack_depth) { - sbuf->ucallstack.nframes = ucallstack_depth; + if (!task_only) { + context->cur_thread->kperf_pet_gen = + os_atomic_load(&kppet_gencount, relaxed); + } + bool is_kernel = (context->cur_pid == 0); + + if (actionid && actionid <= actionc) { + sbuf->kcallstack.kpkc_nframes = + actionv[actionid - 1].kcallstack_depth; } else { - sbuf->ucallstack.nframes = MAX_CALLSTACK_FRAMES; + sbuf->kcallstack.kpkc_nframes = MAX_KCALLSTACK_FRAMES; } - sbuf->kcallstack.flags = CALLSTACK_VALID; - sbuf->ucallstack.flags = CALLSTACK_VALID; + ucallstack_depth = ucallstack_depth ?: MAX_UCALLSTACK_FRAMES; + sbuf->kcallstack.kpkc_flags = 0; + sbuf->usample.ucallstack.kpuc_flags = 0; - /* an event occurred. Sample everything and dump it in a - * buffer. - */ - - /* collect data from samplers */ if (sample_what & SAMPLER_TH_INFO) { kperf_thread_info_sample(&sbuf->th_info, context); - /* See if we should drop idle thread samples */ if (!(sample_flags & SAMPLE_FLAG_IDLE_THREADS)) { if (sbuf->th_info.kpthi_runmode & 0x40) { - return SAMPLE_CONTINUE; + on_idle_thread = true; + goto log_sample; } } } @@ -135,38 +266,31 @@ kperf_sample_internal(struct kperf_sample *sbuf, if (sample_what & SAMPLER_KSTACK) { if (sample_flags & SAMPLE_FLAG_CONTINUATION) { kperf_continuation_sample(&(sbuf->kcallstack), context); - /* outside of interrupt context, backtrace the current thread */ } else if (sample_flags & SAMPLE_FLAG_NON_INTERRUPT) { + /* outside of interrupt context, backtrace the current thread */ kperf_backtrace_sample(&(sbuf->kcallstack), context); } else { kperf_kcallstack_sample(&(sbuf->kcallstack), context); } } if (sample_what & SAMPLER_TK_SNAPSHOT) { - kperf_task_snapshot_sample(&(sbuf->tk_snapshot), context); + kperf_task_snapshot_sample(context->cur_task, &(sbuf->tk_snapshot)); } - /* sensitive ones */ if (!is_kernel) { if (sample_what & SAMPLER_MEMINFO) { - kperf_meminfo_sample(&(sbuf->meminfo), context); + kperf_meminfo_sample(context->cur_task, &(sbuf->meminfo)); } if (sample_flags & SAMPLE_FLAG_PEND_USER) { if (sample_what & SAMPLER_USTACK) { - pended_ucallstack = kperf_ucallstack_pend(context, sbuf->ucallstack.nframes); - } - - if (sample_what & SAMPLER_TH_DISPATCH) { - pended_th_dispatch = kperf_thread_dispatch_pend(context); - } - } else { - if (sample_what & SAMPLER_USTACK) { - kperf_ucallstack_sample(&(sbuf->ucallstack), context); + pended_ucallstack = kperf_ucallstack_pend(context, + ucallstack_depth, actionid); } if (sample_what & SAMPLER_TH_DISPATCH) { - kperf_thread_dispatch_sample(&(sbuf->th_dispatch), context); + pended_th_dispatch = + kperf_thread_dispatch_pend(context, actionid); } } } @@ -177,18 +301,15 @@ kperf_sample_internal(struct kperf_sample *sbuf, kperf_kpc_cpu_sample(&(sbuf->kpcdata), sample_what); } +log_sample: /* lookup the user tag, if any */ - uint32_t userdata; if (actionid && (actionid <= actionc)) { userdata = actionv[actionid - 1].userdata; - } else { - userdata = actionid; } /* avoid logging if this sample only pended samples */ if (sample_flags & SAMPLE_FLAG_PEND_USER && - !(sample_what & ~(SAMPLER_USTACK | SAMPLER_TH_DISPATCH))) - { + !(sample_what & ~(SAMPLER_USTACK | SAMPLER_TH_DISPATCH))) { return SAMPLE_CONTINUE; } @@ -198,7 +319,16 @@ kperf_sample_internal(struct kperf_sample *sbuf, boolean_t enabled = ml_set_interrupts_enabled(FALSE); BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, - actionid, userdata, sample_flags); + actionid, userdata, sample_flags); + + if (sample_flags & SAMPLE_FLAG_SYSTEM) { + if (sample_what & SAMPLER_SYS_MEM) { + kperf_system_memory_log(); + } + } + if (on_idle_thread) { + goto log_sample_end; + } if (sample_what & SAMPLER_TH_INFO) { kperf_thread_info_log(&sbuf->th_info); @@ -212,9 +342,15 @@ kperf_sample_internal(struct kperf_sample *sbuf, if (sample_what & SAMPLER_KSTACK) { kperf_kcallstack_log(&sbuf->kcallstack); } + if (sample_what & SAMPLER_TH_INSCYC) { + kperf_thread_inscyc_log(context); + } if (sample_what & SAMPLER_TK_SNAPSHOT) { kperf_task_snapshot_log(&(sbuf->tk_snapshot)); } + if (sample_what & SAMPLER_TK_INFO) { + kperf_task_info_log(context); + } /* dump user stuff */ if (!is_kernel) { @@ -231,24 +367,20 @@ kperf_sample_internal(struct kperf_sample *sbuf, if (pended_th_dispatch) { BUF_INFO(PERF_TI_DISPPEND); } - } else { - if (sample_what & SAMPLER_USTACK) { - kperf_ucallstack_log(&(sbuf->ucallstack)); - } - - if (sample_what & SAMPLER_TH_DISPATCH) { - kperf_thread_dispatch_log(&(sbuf->th_dispatch)); - } } } + if (sample_what & SAMPLER_PMC_CONFIG) { + kperf_kpc_config_log(&(sbuf->kpcdata)); + } if (sample_what & SAMPLER_PMC_THREAD) { kperf_kpc_thread_log(&(sbuf->kpcdata)); } else if (sample_what & SAMPLER_PMC_CPU) { kperf_kpc_cpu_log(&(sbuf->kpcdata)); } - BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what); +log_sample_end: + BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what, on_idle_thread ? 1 : 0); /* intrs back on */ ml_set_interrupts_enabled(enabled); @@ -259,8 +391,8 @@ kperf_sample_internal(struct kperf_sample *sbuf, /* Translate actionid into sample bits and take a sample */ kern_return_t kperf_sample(struct kperf_sample *sbuf, - struct kperf_context *context, - unsigned actionid, unsigned sample_flags) + struct kperf_context *context, + unsigned actionid, unsigned sample_flags) { /* work out what to sample, if anything */ if ((actionid > actionc) || (actionid == 0)) { @@ -277,18 +409,17 @@ kperf_sample(struct kperf_sample *sbuf, /* the samplers to run */ unsigned int sample_what = actionv[actionid - 1].sample; + unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth; /* do the actual sample operation */ return kperf_sample_internal(sbuf, context, sample_what, - sample_flags, actionid, - actionv[actionid - 1].ucallstack_depth); + sample_flags, actionid, ucallstack_depth); } void kperf_kdebug_handler(uint32_t debugid, uintptr_t *starting_fp) { uint32_t sample_flags = SAMPLE_FLAG_PEND_USER; - struct kperf_context ctx; struct kperf_sample *sample = NULL; kern_return_t kr = KERN_SUCCESS; int s; @@ -299,10 +430,15 @@ kperf_kdebug_handler(uint32_t debugid, uintptr_t *starting_fp) BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_START, debugid); - ctx.cur_thread = current_thread(); - ctx.cur_pid = task_pid(get_threadtask(ctx.cur_thread)); - ctx.trigger_type = TRIGGER_TYPE_KDEBUG; - ctx.trigger_id = 0; + thread_t thread = current_thread(); + task_t task = get_threadtask(thread); + struct kperf_context ctx = { + .cur_thread = thread, + .cur_task = task, + .cur_pid = task_pid(task), + .trigger_type = TRIGGER_TYPE_KDEBUG, + .trigger_id = 0, + }; s = ml_set_interrupts_enabled(0); @@ -328,55 +464,59 @@ __attribute__((noinline)) void kperf_thread_ast_handler(thread_t thread) { - BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_START, thread, kperf_get_thread_flags(thread)); + uint32_t ast = thread->kperf_ast; - /* ~2KB of the stack for the sample since this is called from AST */ - struct kperf_sample sbuf; - memset(&sbuf, 0, sizeof(struct kperf_sample)); + BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_START, thread, ast); + + struct kperf_usample sbuf = {}; task_t task = get_threadtask(thread); - /* make a context, take a sample */ - struct kperf_context ctx; - ctx.cur_thread = thread; - ctx.cur_pid = task_pid(task); + if (task_did_exec(task) || task_is_exec_copy(task)) { + BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, SAMPLE_CONTINUE); + return; + } - /* decode the flags to determine what to sample */ - unsigned int sample_what = 0; - uint32_t flags = kperf_get_thread_flags(thread); + struct kperf_context ctx = { + .cur_thread = thread, + .cur_task = task, + .cur_pid = task_pid(task), + }; - if (flags & T_KPERF_AST_DISPATCH) { + unsigned int sample_what = 0; + if (ast & T_KPERF_AST_DISPATCH) { sample_what |= SAMPLER_TH_DISPATCH; } - if (flags & T_KPERF_AST_CALLSTACK) { - sample_what |= SAMPLER_USTACK; - sample_what |= SAMPLER_TH_INFO; + if (ast & T_KPERF_AST_CALLSTACK) { + /* TH_INFO for backwards compatibility */ + sample_what |= SAMPLER_USTACK | SAMPLER_TH_INFO; } - uint32_t ucallstack_depth = T_KPERF_GET_CALLSTACK_DEPTH(flags); - - int r = kperf_sample_internal(&sbuf, &ctx, sample_what, 0, 0, ucallstack_depth); + sbuf.ucallstack.kpuc_nframes = + T_KPERF_GET_CALLSTACK_DEPTH(ast) ?: MAX_UCALLSTACK_FRAMES; + unsigned int actionid = T_KPERF_GET_ACTIONID(ast); + kperf_sample_user_internal(&sbuf, &ctx, actionid, sample_what); - BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, r); + BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END); } -/* register AST bits */ int -kperf_ast_pend(thread_t thread, uint32_t set_flags) +kperf_ast_pend(thread_t thread, uint32_t set_flags, unsigned int set_actionid) { - /* can only pend on the current thread */ if (thread != current_thread()) { - panic("pending to non-current thread"); + panic("kperf: pending AST to non-current thread"); } - /* get our current bits */ - uint32_t flags = kperf_get_thread_flags(thread); + uint32_t ast = thread->kperf_ast; + unsigned int actionid = T_KPERF_GET_ACTIONID(ast); + uint32_t flags = ast & T_KPERF_AST_ALL; + + if ((flags | set_flags) != flags || actionid != set_actionid) { + ast &= ~T_KPERF_SET_ACTIONID(actionid); + ast |= T_KPERF_SET_ACTIONID(set_actionid); + ast |= set_flags; - /* see if it's already been done or pended */ - if (!(flags & set_flags)) { - /* set the bit on the thread */ - flags |= set_flags; - kperf_set_thread_flags(thread, flags); + thread->kperf_ast = ast; /* set the actual AST */ act_set_kperf(thread); @@ -389,14 +529,12 @@ kperf_ast_pend(thread_t thread, uint32_t set_flags) void kperf_ast_set_callstack_depth(thread_t thread, uint32_t depth) { - uint32_t ast_flags = kperf_get_thread_flags(thread); - uint32_t existing_callstack_depth = T_KPERF_GET_CALLSTACK_DEPTH(ast_flags); - - if (existing_callstack_depth != depth) { - ast_flags &= ~T_KPERF_SET_CALLSTACK_DEPTH(depth); - ast_flags |= T_KPERF_SET_CALLSTACK_DEPTH(depth); - - kperf_set_thread_flags(thread, ast_flags); + uint32_t ast = thread->kperf_ast; + uint32_t existing_depth = T_KPERF_GET_CALLSTACK_DEPTH(ast); + if (existing_depth < depth) { + ast &= ~T_KPERF_SET_CALLSTACK_DEPTH(existing_depth); + ast |= T_KPERF_SET_CALLSTACK_DEPTH(depth); + thread->kperf_ast = ast; } } @@ -521,8 +659,8 @@ kperf_action_reset(void) kperf_action_set_samplers(i + 1, 0); kperf_action_set_userdata(i + 1, 0); kperf_action_set_filter(i + 1, -1); - kperf_action_set_ucallstack_depth(i + 1, MAX_CALLSTACK_FRAMES); - kperf_action_set_kcallstack_depth(i + 1, MAX_CALLSTACK_FRAMES); + kperf_action_set_ucallstack_depth(i + 1, MAX_UCALLSTACK_FRAMES); + kperf_action_set_kcallstack_depth(i + 1, MAX_KCALLSTACK_FRAMES); } } @@ -551,10 +689,7 @@ kperf_action_set_count(unsigned count) * more things, too. */ if (actionc == 0) { - int r; - if ((r = kperf_init())) { - return r; - } + kperf_setup(); } /* create a new array */ @@ -574,8 +709,8 @@ kperf_action_set_count(unsigned count) for (unsigned int i = old_count; i < count; i++) { new_actionv[i].pid_filter = -1; - new_actionv[i].ucallstack_depth = MAX_CALLSTACK_FRAMES; - new_actionv[i].kcallstack_depth = MAX_CALLSTACK_FRAMES; + new_actionv[i].ucallstack_depth = MAX_UCALLSTACK_FRAMES; + new_actionv[i].kcallstack_depth = MAX_KCALLSTACK_FRAMES; } actionv = new_actionv; @@ -595,7 +730,10 @@ kperf_action_set_ucallstack_depth(unsigned action_id, uint32_t depth) return EINVAL; } - if (depth > MAX_CALLSTACK_FRAMES) { + if (depth > MAX_UCALLSTACK_FRAMES) { + return EINVAL; + } + if (depth < 2) { return EINVAL; } @@ -611,7 +749,10 @@ kperf_action_set_kcallstack_depth(unsigned action_id, uint32_t depth) return EINVAL; } - if (depth > MAX_CALLSTACK_FRAMES) { + if (depth > MAX_KCALLSTACK_FRAMES) { + return EINVAL; + } + if (depth < 1) { return EINVAL; } @@ -630,7 +771,7 @@ kperf_action_get_ucallstack_depth(unsigned action_id, uint32_t * depth_out) assert(depth_out); if (action_id == 0) { - *depth_out = MAX_CALLSTACK_FRAMES; + *depth_out = MAX_UCALLSTACK_FRAMES; } else { *depth_out = actionv[action_id - 1].ucallstack_depth; } @@ -648,7 +789,7 @@ kperf_action_get_kcallstack_depth(unsigned action_id, uint32_t * depth_out) assert(depth_out); if (action_id == 0) { - *depth_out = MAX_CALLSTACK_FRAMES; + *depth_out = MAX_KCALLSTACK_FRAMES; } else { *depth_out = actionv[action_id - 1].kcallstack_depth; }