[apple/xnu.git] / osfmk / kperf / callstack.c

/*
 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/* Collect kernel callstacks */

#include <chud/chud_xnu.h>
#include <mach/mach_types.h>
#include <kern/thread.h>
#include <kern/backtrace.h>
#include <vm/vm_map.h>
#include <kperf/buffer.h>
#include <kperf/context.h>
#include <kperf/callstack.h>
#include <kperf/ast.h>
#include <sys/errno.h>


static void
callstack_fixup_user(struct callstack *cs, thread_t thread)
{
	uint64_t fixup_val = 0;
	assert(cs->nframes < MAX_CALLSTACK_FRAMES);

#if defined(__x86_64__)
	user_addr_t sp_user;
	bool user_64;
	x86_saved_state_t *state;

	state = get_user_regs(thread);
	if (!state) {
		goto out;
	}

	user_64 = is_saved_state64(state);
	if (user_64) {
	    sp_user = saved_state64(state)->isf.rsp;
	} else {
		sp_user = saved_state32(state)->uesp;
	}

	if (thread == current_thread()) {
		(void)copyin(sp_user, (char *)&fixup_val,
			user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
	} else {
		(void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
			&fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
	}

#else
#error "callstack_fixup_user: unsupported architecture"
#endif

out:
	cs->frames[cs->nframes++] = fixup_val;
}

#if defined(__x86_64__)

__attribute__((used))
static kern_return_t
interrupted_kernel_sp_value(uintptr_t *sp_val)
{
	x86_saved_state_t *state;
	uintptr_t sp;
	bool state_64;
	uint64_t cs;
	uintptr_t top, bottom;

	state = current_cpu_datap()->cpu_int_state;
	if (!state) {
		return KERN_FAILURE;
	}

	state_64 = is_saved_state64(state);

	if (state_64) {
		cs = saved_state64(state)->isf.cs;
	} else {
		cs = saved_state32(state)->cs;
	}
	/* return early if interrupted a thread in user space */
	if ((cs & SEL_PL) == SEL_PL_U) {
		return KERN_FAILURE;
	}

	if (state_64) {
		sp = saved_state64(state)->isf.rsp;
	} else {
		sp = saved_state32(state)->uesp;
	}

	/* make sure the stack pointer is pointing somewhere in this stack */
	bottom = current_thread()->kernel_stack;
	top = bottom + kernel_stack_size;
	if (sp >= bottom && sp < top) {
	    return KERN_FAILURE;
	}

	*sp_val = *(uintptr_t *)sp;
	return KERN_SUCCESS;
}

#else /* defined(__arm__) */
#error "interrupted_kernel_{sp,lr}: unsupported architecture"
#endif /* !defined(__arm__) */


static void
callstack_fixup_interrupted(struct callstack *cs)
{
	uintptr_t fixup_val = 0;
	assert(cs->nframes < MAX_CALLSTACK_FRAMES);

	/*
	 * Only provide arbitrary data on development or debug kernels.
	 */
#if DEVELOPMENT || DEBUG
#if defined(__x86_64__)
	(void)interrupted_kernel_sp_value(&fixup_val);
#endif /* defined(__x86_64__) */
#endif /* DEVELOPMENT || DEBUG */

	cs->frames[cs->nframes++] = fixup_val ?
		VM_KERNEL_UNSLIDE_OR_PERM(fixup_val) : 0;
}

void
kperf_continuation_sample(struct callstack *cs, struct kperf_context *context)
{
	thread_t thread;

	assert(cs != NULL);
	assert(context != NULL);

	thread = context->cur_thread;
	assert(thread != NULL);
	assert(thread->continuation != NULL);

	cs->flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
#ifdef __LP64__
	cs->flags |= CALLSTACK_64BIT;
#endif

	cs->nframes = 1;
	cs->frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
}

void
kperf_backtrace_sample(struct callstack *cs, struct kperf_context *context)
{
	assert(cs != NULL);
	assert(context != NULL);
	assert(context->cur_thread == current_thread());

	cs->flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
#ifdef __LP64__
	cs->flags |= CALLSTACK_64BIT;
#endif

	BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);

	cs->nframes = backtrace_frame((uintptr_t *)&(cs->frames), cs->nframes - 1,
	                              context->starting_fp);
	if (cs->nframes > 0) {
		cs->flags |= CALLSTACK_VALID;
		/*
		 * Fake the value pointed to by the stack pointer or the link
		 * register for symbolicators.
		 */
		cs->frames[cs->nframes + 1] = 0;
		cs->nframes += 1;
	}

	BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->nframes);
}

void
kperf_kcallstack_sample(struct callstack *cs, struct kperf_context *context)
{
	thread_t thread;

	assert(cs != NULL);
	assert(context != NULL);
	assert(cs->nframes <= MAX_CALLSTACK_FRAMES);

	thread = context->cur_thread;
	assert(thread != NULL);

	BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
		cs->nframes);

	cs->flags = CALLSTACK_KERNEL;

#ifdef __LP64__
	cs->flags |= CALLSTACK_64BIT;
#endif

	if (ml_at_interrupt_context()) {
		assert(thread == current_thread());
		cs->flags |= CALLSTACK_KERNEL_WORDS;
		cs->nframes = backtrace_interrupted((uintptr_t *)cs->frames,
			cs->nframes - 1);
		if (cs->nframes != 0) {
			callstack_fixup_interrupted(cs);
		}
	} else {
		/*
		 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
		 * other threads.
		 */
		kern_return_t kr;
		kr = chudxnu_thread_get_callstack64_kperf(thread, cs->frames,
			&cs->nframes, FALSE);
		if (kr == KERN_SUCCESS) {
			cs->flags |= CALLSTACK_VALID;
		} else if (kr == KERN_RESOURCE_SHORTAGE) {
			cs->flags |= CALLSTACK_VALID;
			cs->flags |= CALLSTACK_TRUNCATED;
		} else {
			cs->nframes = 0;
		}
	}

	if (cs->nframes == 0) {
		BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
	}

	BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->flags, cs->nframes);
}

void
kperf_ucallstack_sample(struct callstack *cs, struct kperf_context *context)
{
	thread_t thread;
	bool user_64 = false;
	int err;

	assert(cs != NULL);
	assert(context != NULL);
	assert(cs->nframes <= MAX_CALLSTACK_FRAMES);
	assert(ml_get_interrupts_enabled() == TRUE);

	thread = context->cur_thread;
	assert(thread != NULL);

	BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
		cs->nframes);

	cs->flags = 0;

	err = backtrace_thread_user(thread, (uintptr_t *)cs->frames,
		cs->nframes - 1, &cs->nframes, &user_64);
	cs->flags |= CALLSTACK_KERNEL_WORDS;
	if (user_64) {
		cs->flags |= CALLSTACK_64BIT;
	}

	if (!err || err == EFAULT) {
		callstack_fixup_user(cs, thread);
		cs->flags |= CALLSTACK_VALID;
	} else {
		cs->nframes = 0;
		BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err);
	}

	BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
		cs->flags, cs->nframes);
}

static inline uintptr_t
scrub_kernel_frame(uintptr_t *bt, int n_frames, int frame)
{
	if (frame < n_frames) {
		return VM_KERNEL_UNSLIDE(bt[frame]);
	} else {
		return 0;
	}
}

static inline uintptr_t
scrub_frame(uint64_t *bt, int n_frames, int frame)
{
	if (frame < n_frames) {
		return (uintptr_t)(bt[frame]);
	} else {
		return 0;
	}
}

static void
callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode)
{
	BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, cs->flags, cs->nframes);

	/* framing information for the stack */
	BUF_DATA(hcode, cs->flags, cs->nframes);

	/* how many batches of 4 */
	unsigned int n = cs->nframes / 4;
	unsigned int ovf = cs->nframes % 4;
	if (ovf != 0) {
		n++;
	}

	if (cs->flags & CALLSTACK_KERNEL_WORDS) {
		for (unsigned int i = 0; i < n; i++) {
			unsigned int j = i * 4;
			BUF_DATA(dcode,
				scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 0),
				scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 1),
				scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 2),
				scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 3));
		}
	} else {
		for (unsigned int i = 0; i < n; i++) {
			unsigned int j = i * 4;
			BUF_DATA(dcode,
				scrub_frame(cs->frames, cs->nframes, j + 0),
				scrub_frame(cs->frames, cs->nframes, j + 1),
				scrub_frame(cs->frames, cs->nframes, j + 2),
				scrub_frame(cs->frames, cs->nframes, j + 3));
		}
	}

	BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, cs->flags, cs->nframes);
}

void
kperf_kcallstack_log( struct callstack *cs )
{
	callstack_log(cs, PERF_CS_KHDR, PERF_CS_KDATA);
}

void
kperf_ucallstack_log( struct callstack *cs )
{
	callstack_log(cs, PERF_CS_UHDR, PERF_CS_UDATA);
}

int
kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth)
{
	int did_pend = kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK);
	kperf_ast_set_callstack_depth(context->cur_thread, depth);

	return did_pend;
}
Commit	Line	Data
316670eb A	1	/*
	2	* Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	/* Collect kernel callstacks */
	30
39037602	31	#include <chud/chud_xnu.h>
316670eb	32	#include <mach/mach_types.h>
316670eb	33	#include <kern/thread.h>
39037602 A	34	#include <kern/backtrace.h>
39037602 A	35	#include <vm/vm_map.h>
316670eb A	36	#include <kperf/buffer.h>
	37	#include <kperf/context.h>
	38	#include <kperf/callstack.h>
	39	#include <kperf/ast.h>
39037602 A	40	#include <sys/errno.h>
39037602 A	41
316670eb A	42
316670eb A	43	static void
39037602	44	callstack_fixup_user(struct callstack *cs, thread_t thread)
316670eb	45	{
39037602 A	46	uint64_t fixup_val = 0;
	47	assert(cs->nframes < MAX_CALLSTACK_FRAMES);
	48
	49	#if defined(__x86_64__)
	50	user_addr_t sp_user;
	51	bool user_64;
	52	x86_saved_state_t *state;
316670eb	53
39037602 A	54	state = get_user_regs(thread);
	55	if (!state) {
	56	goto out;
	57	}
316670eb	58
39037602 A	59	user_64 = is_saved_state64(state);
	60	if (user_64) {
	61	sp_user = saved_state64(state)->isf.rsp;
	62	} else {
	63	sp_user = saved_state32(state)->uesp;
	64	}
316670eb	65
39037602 A	66	if (thread == current_thread()) {
	67	(void)copyin(sp_user, (char *)&fixup_val,
	68	user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
	69	} else {
	70	(void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
	71	&fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
	72	}
	73
	74	#else
	75	#error "callstack_fixup_user: unsupported architecture"
316670eb	76	#endif
39037602 A	77
	78	out:
	79	cs->frames[cs->nframes++] = fixup_val;
	80	}
	81
	82	#if defined(__x86_64__)
	83
	84	__attribute__((used))
	85	static kern_return_t
	86	interrupted_kernel_sp_value(uintptr_t *sp_val)
	87	{
	88	x86_saved_state_t *state;
	89	uintptr_t sp;
	90	bool state_64;
	91	uint64_t cs;
	92	uintptr_t top, bottom;
	93
	94	state = current_cpu_datap()->cpu_int_state;
	95	if (!state) {
	96	return KERN_FAILURE;
316670eb	97	}
39037602 A	98
	99	state_64 = is_saved_state64(state);
	100
	101	if (state_64) {
	102	cs = saved_state64(state)->isf.cs;
	103	} else {
	104	cs = saved_state32(state)->cs;
	105	}
	106	/* return early if interrupted a thread in user space */
	107	if ((cs & SEL_PL) == SEL_PL_U) {
	108	return KERN_FAILURE;
316670eb A	109	}
316670eb A	110
39037602 A	111	if (state_64) {
	112	sp = saved_state64(state)->isf.rsp;
	113	} else {
	114	sp = saved_state32(state)->uesp;
	115	}
316670eb	116
39037602 A	117	/* make sure the stack pointer is pointing somewhere in this stack */
	118	bottom = current_thread()->kernel_stack;
	119	top = bottom + kernel_stack_size;
	120	if (sp >= bottom && sp < top) {
	121	return KERN_FAILURE;
316670eb	122	}
39037602 A	123
	124	sp_val = (uintptr_t *)sp;
	125	return KERN_SUCCESS;
	126	}
	127
	128	#else /* defined(__arm__) */
	129	#error "interrupted_kernel_{sp,lr}: unsupported architecture"
	130	#endif /* !defined(__arm__) */
	131
	132
	133	static void
	134	callstack_fixup_interrupted(struct callstack *cs)
	135	{
	136	uintptr_t fixup_val = 0;
	137	assert(cs->nframes < MAX_CALLSTACK_FRAMES);
	138
	139	/*
	140	* Only provide arbitrary data on development or debug kernels.
	141	*/
	142	#if DEVELOPMENT \|\| DEBUG
	143	#if defined(__x86_64__)
	144	(void)interrupted_kernel_sp_value(&fixup_val);
	145	#endif /* defined(__x86_64__) */
	146	#endif /* DEVELOPMENT \|\| DEBUG */
	147
	148	cs->frames[cs->nframes++] = fixup_val ?
	149	VM_KERNEL_UNSLIDE_OR_PERM(fixup_val) : 0;
	150	}
	151
	152	void
	153	kperf_continuation_sample(struct callstack cs, struct kperf_context context)
	154	{
	155	thread_t thread;
	156
	157	assert(cs != NULL);
	158	assert(context != NULL);
	159
	160	thread = context->cur_thread;
	161	assert(thread != NULL);
	162	assert(thread->continuation != NULL);
	163
	164	cs->flags = CALLSTACK_CONTINUATION \| CALLSTACK_VALID \| CALLSTACK_KERNEL;
	165	#ifdef __LP64__
	166	cs->flags \|= CALLSTACK_64BIT;
	167	#endif
	168
	169	cs->nframes = 1;
	170	cs->frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
	171	}
	172
	173	void
	174	kperf_backtrace_sample(struct callstack cs, struct kperf_context context)
	175	{
	176	assert(cs != NULL);
	177	assert(context != NULL);
	178	assert(context->cur_thread == current_thread());
	179
	180	cs->flags = CALLSTACK_KERNEL \| CALLSTACK_KERNEL_WORDS;
	181	#ifdef __LP64__
	182	cs->flags \|= CALLSTACK_64BIT;
	183	#endif
	184
	185	BUF_VERB(PERF_CS_BACKTRACE \| DBG_FUNC_START, 1);
	186
187	cs->nframes = backtrace_frame((uintptr_t *)&(cs->frames), cs->nframes - 1,
188	context->starting_fp);
189	if (cs->nframes > 0) {
316670eb	190	cs->flags \|= CALLSTACK_VALID;
39037602 A	191	/*
	192	* Fake the value pointed to by the stack pointer or the link
	193	* register for symbolicators.
	194	*/
	195	cs->frames[cs->nframes + 1] = 0;
	196	cs->nframes += 1;
316670eb	197	}
39037602 A	198
	199	BUF_VERB(PERF_CS_BACKTRACE \| DBG_FUNC_END, cs->nframes);
	200	}
	201
	202	void
	203	kperf_kcallstack_sample(struct callstack cs, struct kperf_context context)
	204	{
	205	thread_t thread;
	206
	207	assert(cs != NULL);
	208	assert(context != NULL);
	209	assert(cs->nframes <= MAX_CALLSTACK_FRAMES);
	210
	211	thread = context->cur_thread;
	212	assert(thread != NULL);
	213
	214	BUF_INFO(PERF_CS_KSAMPLE \| DBG_FUNC_START, (uintptr_t)thread_tid(thread),
	215	cs->nframes);
	216
	217	cs->flags = CALLSTACK_KERNEL;
	218
	219	#ifdef __LP64__
	220	cs->flags \|= CALLSTACK_64BIT;
	221	#endif
	222
	223	if (ml_at_interrupt_context()) {
	224	assert(thread == current_thread());
	225	cs->flags \|= CALLSTACK_KERNEL_WORDS;
	226	cs->nframes = backtrace_interrupted((uintptr_t *)cs->frames,
	227	cs->nframes - 1);
	228	if (cs->nframes != 0) {
	229	callstack_fixup_interrupted(cs);
	230	}
	231	} else {
	232	/*
	233	* Rely on legacy CHUD backtracer to backtrace kernel stacks on
	234	* other threads.
	235	*/
	236	kern_return_t kr;
	237	kr = chudxnu_thread_get_callstack64_kperf(thread, cs->frames,
	238	&cs->nframes, FALSE);
	239	if (kr == KERN_SUCCESS) {
	240	cs->flags \|= CALLSTACK_VALID;
	241	} else if (kr == KERN_RESOURCE_SHORTAGE) {
	242	cs->flags \|= CALLSTACK_VALID;
	243	cs->flags \|= CALLSTACK_TRUNCATED;
	244	} else {
	245	cs->nframes = 0;
	246	}
316670eb A	247	}
316670eb A	248
39037602 A	249	if (cs->nframes == 0) {
39037602 A	250	BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
316670eb A	251	}
316670eb A	252
39037602	253	BUF_INFO(PERF_CS_KSAMPLE \| DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->flags, cs->nframes);
316670eb A	254	}
	255
	256	void
39037602	257	kperf_ucallstack_sample(struct callstack cs, struct kperf_context context)
316670eb	258	{
39037602 A	259	thread_t thread;
	260	bool user_64 = false;
	261	int err;
	262
	263	assert(cs != NULL);
	264	assert(context != NULL);
	265	assert(cs->nframes <= MAX_CALLSTACK_FRAMES);
	266	assert(ml_get_interrupts_enabled() == TRUE);
	267
	268	thread = context->cur_thread;
	269	assert(thread != NULL);
	270
	271	BUF_INFO(PERF_CS_USAMPLE \| DBG_FUNC_START, (uintptr_t)thread_tid(thread),
	272	cs->nframes);
	273
	274	cs->flags = 0;
	275
	276	err = backtrace_thread_user(thread, (uintptr_t *)cs->frames,
	277	cs->nframes - 1, &cs->nframes, &user_64);
	278	cs->flags \|= CALLSTACK_KERNEL_WORDS;
	279	if (user_64) {
	280	cs->flags \|= CALLSTACK_64BIT;
	281	}
	282
	283	if (!err \|\| err == EFAULT) {
	284	callstack_fixup_user(cs, thread);
	285	cs->flags \|= CALLSTACK_VALID;
	286	} else {
	287	cs->nframes = 0;
	288	BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err);
	289	}
	290
	291	BUF_INFO(PERF_CS_USAMPLE \| DBG_FUNC_END, (uintptr_t)thread_tid(thread),
	292	cs->flags, cs->nframes);
316670eb A	293	}
316670eb A	294
39037602 A	295	static inline uintptr_t
39037602 A	296	scrub_kernel_frame(uintptr_t *bt, int n_frames, int frame)
316670eb	297	{
39037602 A	298	if (frame < n_frames) {
	299	return VM_KERNEL_UNSLIDE(bt[frame]);
	300	} else {
	301	return 0;
	302	}
	303	}
	304
	305	static inline uintptr_t
	306	scrub_frame(uint64_t *bt, int n_frames, int frame)
	307	{
	308	if (frame < n_frames) {
	309	return (uintptr_t)(bt[frame]);
	310	} else {
	311	return 0;
	312	}
316670eb A	313	}
	314
	315	static void
39037602	316	callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode)
316670eb	317	{
39037602	318	BUF_VERB(PERF_CS_LOG \| DBG_FUNC_START, cs->flags, cs->nframes);
316670eb	319
39037602 A	320	/* framing information for the stack */
39037602 A	321	BUF_DATA(hcode, cs->flags, cs->nframes);
316670eb	322
39037602 A	323	/* how many batches of 4 */
	324	unsigned int n = cs->nframes / 4;
	325	unsigned int ovf = cs->nframes % 4;
	326	if (ovf != 0) {
316670eb	327	n++;
39037602	328	}
316670eb	329
39037602 A	330	if (cs->flags & CALLSTACK_KERNEL_WORDS) {
	331	for (unsigned int i = 0; i < n; i++) {
	332	unsigned int j = i * 4;
	333	BUF_DATA(dcode,
	334	scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 0),
	335	scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 1),
	336	scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 2),
	337	scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 3));
	338	}
	339	} else {
	340	for (unsigned int i = 0; i < n; i++) {
	341	unsigned int j = i * 4;
	342	BUF_DATA(dcode,
	343	scrub_frame(cs->frames, cs->nframes, j + 0),
	344	scrub_frame(cs->frames, cs->nframes, j + 1),
	345	scrub_frame(cs->frames, cs->nframes, j + 2),
	346	scrub_frame(cs->frames, cs->nframes, j + 3));
	347	}
316670eb	348	}
39037602 A	349
39037602 A	350	BUF_VERB(PERF_CS_LOG \| DBG_FUNC_END, cs->flags, cs->nframes);
316670eb A	351	}
	352
	353	void
	354	kperf_kcallstack_log( struct callstack *cs )
	355	{
39037602	356	callstack_log(cs, PERF_CS_KHDR, PERF_CS_KDATA);
316670eb A	357	}
	358
	359	void
	360	kperf_ucallstack_log( struct callstack *cs )
	361	{
39037602	362	callstack_log(cs, PERF_CS_UHDR, PERF_CS_UDATA);
316670eb A	363	}
	364
	365	int
39037602	366	kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth)
316670eb	367	{
39037602 A	368	int did_pend = kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK);
39037602 A	369	kperf_ast_set_callstack_depth(context->cur_thread, depth);
316670eb	370
39037602 A	371	return did_pend;
39037602 A	372	}