[apple/xnu.git] / osfmk / kperf / thread_samplers.c

/*
 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/*  Sample thread data */

#include <kern/debug.h> /* panic */
#include <kern/thread.h> /* thread_* */
#include <kern/timer.h> /* timer_data_t */
#include <kern/policy_internal.h> /* TASK_POLICY_* */
#include <mach/mach_types.h>

#include <kperf/kperf.h>
#include <kperf/buffer.h>
#include <kperf/context.h>
#include <kperf/thread_samplers.h>
#include <kperf/ast.h>

#if MONOTONIC
#include <kern/monotonic.h>
#include <machine/monotonic.h>
#endif /* MONOTONIC */

extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);

/*
 * XXX Deprecated, use thread scheduling sampler instead.
 *
 * Taken from AppleProfileGetRunModeOfThread and CHUD.  Still here for
 * backwards compatibility.
 */

#define KPERF_TI_RUNNING   (1U << 0)
#define KPERF_TI_RUNNABLE  (1U << 1)
#define KPERF_TI_WAIT      (1U << 2)
#define KPERF_TI_UNINT     (1U << 3)
#define KPERF_TI_SUSP      (1U << 4)
#define KPERF_TI_TERMINATE (1U << 5)
#define KPERF_TI_IDLE      (1U << 6)

static uint32_t
kperf_thread_info_runmode_legacy(thread_t thread)
{
	uint32_t kperf_state = 0;
	int sched_state = thread->state;
	processor_t last_processor = thread->last_processor;

	if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) {
		kperf_state |= KPERF_TI_RUNNING;
	}
	if (sched_state & TH_RUN) {
		kperf_state |= KPERF_TI_RUNNABLE;
	}
	if (sched_state & TH_WAIT) {
		kperf_state |= KPERF_TI_WAIT;
	}
	if (sched_state & TH_UNINT) {
		kperf_state |= KPERF_TI_UNINT;
	}
	if (sched_state & TH_SUSP) {
		kperf_state |= KPERF_TI_SUSP;
	}
	if (sched_state & TH_TERMINATE) {
		kperf_state |= KPERF_TI_TERMINATE;
	}
	if (sched_state & TH_IDLE) {
		kperf_state |= KPERF_TI_IDLE;
	}

#if defined(XNU_TARGET_OS_OSX)
	/* on desktop, if state is blank, leave not idle set */
	if (kperf_state == 0) {
		return TH_IDLE << 16;
	}
#endif /* defined(XNU_TARGET_OS_OSX) */

	/* high two bytes are inverted mask, low two bytes are normal */
	return ((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff);
}

void
kperf_thread_info_sample(struct kperf_thread_info *ti, struct kperf_context *context)
{
	thread_t cur_thread = context->cur_thread;

	BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread));

	ti->kpthi_pid = context->cur_pid;
	ti->kpthi_tid = thread_tid(cur_thread);
	ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread);
	ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread);

	BUF_VERB(PERF_TI_SAMPLE | DBG_FUNC_END);
}

void
kperf_thread_info_log(struct kperf_thread_info *ti)
{
	BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */,
	    ti->kpthi_dq_addr, ti->kpthi_runmode);
}

/*
 * Scheduling information reports inputs and outputs of the scheduler state for
 * a thread.
 */

void
kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc,
    struct kperf_context *context)
{
	assert(thsc != NULL);
	assert(context != NULL);

	thread_t thread = context->cur_thread;

	BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));

	thsc->kpthsc_user_time = timer_grab(&thread->user_timer);
	uint64_t system_time = timer_grab(&thread->system_timer);

	if (thread->precise_user_kernel_time) {
		thsc->kpthsc_system_time = system_time;
	} else {
		thsc->kpthsc_user_time += system_time;
		thsc->kpthsc_system_time = 0;
	}

	thsc->kpthsc_runnable_time = timer_grab(&thread->runnable_timer);
	thsc->kpthsc_state = thread->state;
	thsc->kpthsc_base_priority = thread->base_pri;
	thsc->kpthsc_sched_priority = thread->sched_pri;
	thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos;
	thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos;
	thsc->kpthsc_requested_qos_override = MAX(thread->requested_policy.thrp_qos_override,
	    thread->requested_policy.thrp_qos_workq_override);
	thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote;
	thsc->kpthsc_requested_qos_kevent_override = MAX(
		thread->requested_policy.thrp_qos_kevent_override,
		thread->requested_policy.thrp_qos_wlsvc_override);
	thsc->kpthsc_requested_qos_sync_ipc_override = THREAD_QOS_UNSPECIFIED;
	thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos;

	BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END);
}


void
kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc)
{
	assert(thsc != NULL);
#if defined(__LP64__)
	BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time,
	    thsc->kpthsc_system_time,
	    (((uint64_t)thsc->kpthsc_base_priority) << 48)
	    | ((uint64_t)thsc->kpthsc_sched_priority << 32)
	    | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24)
	    | (thsc->kpthsc_effective_qos << 6)
	    | (thsc->kpthsc_requested_qos << 3)
	    | thsc->kpthsc_requested_qos_override,
	    ((uint64_t)thsc->kpthsc_effective_latency_qos << 61)
	    | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58)
	    | ((uint64_t)thsc->kpthsc_requested_qos_kevent_override << 55)
	    );
	BUF_DATA(PERF_TI_SCHEDDATA_3, thsc->kpthsc_runnable_time);
#else
	BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time),
	    LOWER_32(thsc->kpthsc_user_time),
	    UPPER_32(thsc->kpthsc_system_time),
	    LOWER_32(thsc->kpthsc_system_time)
	    );
	BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16)
	    | thsc->kpthsc_sched_priority,
	    ((thsc->kpthsc_state & 0xff) << 24)
	    | (thsc->kpthsc_effective_qos << 6)
	    | (thsc->kpthsc_requested_qos << 3)
	    | thsc->kpthsc_requested_qos_override,
	    ((uint32_t)thsc->kpthsc_effective_latency_qos << 29)
	    | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26)
	    | ((uint32_t)thsc->kpthsc_requested_qos_kevent_override << 23)
	    );
	BUF_DATA(PERF_TI_SCHEDDATA3_32, UPPER_32(thsc->kpthsc_runnable_time),
	    LOWER_32(thsc->kpthsc_runnable_time));
#endif /* defined(__LP64__) */
}

/*
 * Snapshot information maintains parity with stackshot information for other,
 * miscellaneous information about threads.
 */

#define KPERF_THREAD_SNAPSHOT_DARWIN_BG  (1U << 0);
#define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1);
#define KPERF_THREAD_SNAPSHOT_GFI        (1U << 2);
#define KPERF_THREAD_SNAPSHOT_IDLE_WQ    (1U << 3);
/* max is 1U << 7 */

void
kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn,
    struct kperf_context *context)
{
	assert(thsn != NULL);
	assert(context != NULL);

	thread_t thread = context->cur_thread;

	BUF_INFO(PERF_TI_SNAPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));

	thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time;

	thsn->kpthsn_flags = 0;
	if (thread->effective_policy.thep_darwinbg) {
		thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_DARWIN_BG;
	}
	if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
		thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_PASSIVE_IO;
	}
	if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
		thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_GFI
	}
	if (stackshot_thread_is_idle_worker_unsafe(thread)) {
		thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_IDLE_WQ;
	}

	thsn->kpthsn_suspend_count = thread->suspend_count;
	/*
	 * Only have room for 8-bits in the trace event, so truncate here.
	 */
	thsn->kpthsn_io_tier = (uint8_t)proc_get_effective_thread_policy(thread, TASK_POLICY_IO);

	BUF_VERB(PERF_TI_SNAPSAMPLE | DBG_FUNC_END);
}

void
kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn)
{
	assert(thsn != NULL);
#if defined(__LP64__)
	BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
	    | (thsn->kpthsn_io_tier << 24),
	    thsn->kpthsn_last_made_runnable_time);
#else
	BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
	    | (thsn->kpthsn_io_tier << 24),
	    UPPER_32(thsn->kpthsn_last_made_runnable_time),
	    LOWER_32(thsn->kpthsn_last_made_runnable_time));
#endif /* defined(__LP64__) */
}

/*
 * Dispatch information only contains the dispatch queue serial number from
 * libdispatch.
 *
 * It's a separate sampler because queue data must be copied in from user space.
 */

void
kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi,
    struct kperf_context *context)
{
	assert(thdi != NULL);
	assert(context != NULL);

	thread_t thread = context->cur_thread;

	BUF_INFO(PERF_TI_DISPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));

	task_t task = thread->task;
	boolean_t task_64 = task_has_64Bit_addr(task);
	size_t user_addr_size = task_64 ? 8 : 4;

	assert(thread->task != kernel_task);
	uint64_t user_dq_key_addr = thread_dispatchqaddr(thread);
	if (user_dq_key_addr == 0) {
		goto error;
	}

	uint64_t user_dq_addr;
	if ((copyin((user_addr_t)user_dq_key_addr,
	    (char *)&user_dq_addr,
	    user_addr_size) != 0) ||
	    (user_dq_addr == 0)) {
		goto error;
	}

	uint64_t user_dq_serialno_addr =
	    user_dq_addr + get_task_dispatchqueue_serialno_offset(task);

	if (copyin((user_addr_t)user_dq_serialno_addr,
	    (char *)&(thdi->kpthdi_dq_serialno),
	    user_addr_size) == 0) {
		goto out;
	}

error:
	thdi->kpthdi_dq_serialno = 0;

out:
	BUF_VERB(PERF_TI_DISPSAMPLE | DBG_FUNC_END);
}

int
kperf_thread_dispatch_pend(struct kperf_context *context,
    unsigned int actionid)
{
	return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH,
	           actionid);
}

void
kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi)
{
	assert(thdi != NULL);
#if defined(__LP64__)
	BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno);
#else
	BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno),
	    LOWER_32(thdi->kpthdi_dq_serialno));
#endif /* defined(__LP64__) */
}

/*
 * A bit different from other samplers -- since logging disables interrupts,
 * it's a fine place to sample the thread counters.
 */
void
kperf_thread_inscyc_log(struct kperf_context *context)
{
#if MONOTONIC
	thread_t cur_thread = current_thread();

	if (context->cur_thread != cur_thread) {
		/* can't safely access another thread's counters */
		return;
	}

	uint64_t counts[MT_CORE_NFIXED] = { 0 };
	mt_cur_thread_fixed_counts(counts);

#if defined(__LP64__)
	BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]);
#else /* defined(__LP64__) */
	/* 32-bit platforms don't count instructions */
	BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]),
	    LOWER_32(counts[MT_CORE_CYCLES]));
#endif /* !defined(__LP64__) */

#else
#pragma unused(context)
#endif /* MONOTONIC */
}
Commit	Line	Data
39037602 A	1	/*
	2	* Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	/* Sample thread data */
	30
	31	#include <kern/debug.h> /* panic */
	32	#include <kern/thread.h> /* thread_* */
	33	#include <kern/timer.h> /* timer_data_t */
	34	#include <kern/policy_internal.h> /* TASK_POLICY_* */
	35	#include <mach/mach_types.h>
	36
	37	#include <kperf/kperf.h>
	38	#include <kperf/buffer.h>
	39	#include <kperf/context.h>
	40	#include <kperf/thread_samplers.h>
	41	#include <kperf/ast.h>
	42
5ba3f43e A	43	#if MONOTONIC
	44	#include <kern/monotonic.h>
	45	#include <machine/monotonic.h>
	46	#endif /* MONOTONIC */
	47
39037602 A	48	extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
	49
	50	/*
	51	* XXX Deprecated, use thread scheduling sampler instead.
	52	*
	53	* Taken from AppleProfileGetRunModeOfThread and CHUD. Still here for
	54	* backwards compatibility.
	55	*/
	56
	57	#define KPERF_TI_RUNNING (1U << 0)
	58	#define KPERF_TI_RUNNABLE (1U << 1)
	59	#define KPERF_TI_WAIT (1U << 2)
	60	#define KPERF_TI_UNINT (1U << 3)
	61	#define KPERF_TI_SUSP (1U << 4)
	62	#define KPERF_TI_TERMINATE (1U << 5)
	63	#define KPERF_TI_IDLE (1U << 6)
	64
	65	static uint32_t
	66	kperf_thread_info_runmode_legacy(thread_t thread)
	67	{
	68	uint32_t kperf_state = 0;
	69	int sched_state = thread->state;
	70	processor_t last_processor = thread->last_processor;
	71
	72	if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) {
	73	kperf_state \|= KPERF_TI_RUNNING;
	74	}
	75	if (sched_state & TH_RUN) {
	76	kperf_state \|= KPERF_TI_RUNNABLE;
	77	}
	78	if (sched_state & TH_WAIT) {
	79	kperf_state \|= KPERF_TI_WAIT;
	80	}
	81	if (sched_state & TH_UNINT) {
	82	kperf_state \|= KPERF_TI_UNINT;
	83	}
	84	if (sched_state & TH_SUSP) {
	85	kperf_state \|= KPERF_TI_SUSP;
	86	}
	87	if (sched_state & TH_TERMINATE) {
	88	kperf_state \|= KPERF_TI_TERMINATE;
	89	}
	90	if (sched_state & TH_IDLE) {
	91	kperf_state \|= KPERF_TI_IDLE;
	92	}
	93
f427ee49	94	#if defined(XNU_TARGET_OS_OSX)
39037602 A	95	/* on desktop, if state is blank, leave not idle set */
39037602 A	96	if (kperf_state == 0) {
0a7de745	97	return TH_IDLE << 16;
39037602	98	}
f427ee49	99	#endif /* defined(XNU_TARGET_OS_OSX) */
39037602 A	100
39037602 A	101	/* high two bytes are inverted mask, low two bytes are normal */
0a7de745	102	return ((~kperf_state & 0xffff) << 16) \| (kperf_state & 0xffff);
39037602 A	103	}
	104
	105	void
	106	kperf_thread_info_sample(struct kperf_thread_info ti, struct kperf_context context)
	107	{
	108	thread_t cur_thread = context->cur_thread;
	109
	110	BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread));
	111
	112	ti->kpthi_pid = context->cur_pid;
	113	ti->kpthi_tid = thread_tid(cur_thread);
	114	ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread);
	115	ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread);
	116
	117	BUF_VERB(PERF_TI_SAMPLE \| DBG_FUNC_END);
	118	}
	119
	120	void
	121	kperf_thread_info_log(struct kperf_thread_info *ti)
	122	{
	123	BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */,
0a7de745	124	ti->kpthi_dq_addr, ti->kpthi_runmode);
39037602 A	125	}
	126
	127	/*
	128	* Scheduling information reports inputs and outputs of the scheduler state for
	129	* a thread.
	130	*/
	131
	132	void
	133	kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc,
0a7de745	134	struct kperf_context *context)
39037602 A	135	{
	136	assert(thsc != NULL);
	137	assert(context != NULL);
	138
	139	thread_t thread = context->cur_thread;
	140
	141	BUF_INFO(PERF_TI_SCHEDSAMPLE \| DBG_FUNC_START, (uintptr_t)thread_tid(thread));
	142
d9a64523 A	143	thsc->kpthsc_user_time = timer_grab(&thread->user_timer);
d9a64523 A	144	uint64_t system_time = timer_grab(&thread->system_timer);
39037602 A	145
	146	if (thread->precise_user_kernel_time) {
	147	thsc->kpthsc_system_time = system_time;
	148	} else {
	149	thsc->kpthsc_user_time += system_time;
	150	thsc->kpthsc_system_time = 0;
	151	}
	152
d9a64523	153	thsc->kpthsc_runnable_time = timer_grab(&thread->runnable_timer);
39037602 A	154	thsc->kpthsc_state = thread->state;
	155	thsc->kpthsc_base_priority = thread->base_pri;
	156	thsc->kpthsc_sched_priority = thread->sched_pri;
	157	thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos;
	158	thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos;
d9a64523	159	thsc->kpthsc_requested_qos_override = MAX(thread->requested_policy.thrp_qos_override,
0a7de745	160	thread->requested_policy.thrp_qos_workq_override);
5ba3f43e	161	thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote;
cb323159 A	162	thsc->kpthsc_requested_qos_kevent_override = MAX(
	163	thread->requested_policy.thrp_qos_kevent_override,
	164	thread->requested_policy.thrp_qos_wlsvc_override);
	165	thsc->kpthsc_requested_qos_sync_ipc_override = THREAD_QOS_UNSPECIFIED;
39037602 A	166	thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos;
	167
	168	BUF_INFO(PERF_TI_SCHEDSAMPLE \| DBG_FUNC_END);
	169	}
	170
	171
	172	void
	173	kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc)
	174	{
	175	assert(thsc != NULL);
	176	#if defined(__LP64__)
5ba3f43e	177	BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time,
0a7de745 A	178	thsc->kpthsc_system_time,
	179	(((uint64_t)thsc->kpthsc_base_priority) << 48)
	180	\| ((uint64_t)thsc->kpthsc_sched_priority << 32)
	181	\| ((uint64_t)(thsc->kpthsc_state & 0xff) << 24)
	182	\| (thsc->kpthsc_effective_qos << 6)
	183	\| (thsc->kpthsc_requested_qos << 3)
	184	\| thsc->kpthsc_requested_qos_override,
	185	((uint64_t)thsc->kpthsc_effective_latency_qos << 61)
	186	\| ((uint64_t)thsc->kpthsc_requested_qos_promote << 58)
cb323159	187	\| ((uint64_t)thsc->kpthsc_requested_qos_kevent_override << 55)
0a7de745	188	);
d9a64523	189	BUF_DATA(PERF_TI_SCHEDDATA_3, thsc->kpthsc_runnable_time);
39037602 A	190	#else
39037602 A	191	BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time),
0a7de745 A	192	LOWER_32(thsc->kpthsc_user_time),
	193	UPPER_32(thsc->kpthsc_system_time),
	194	LOWER_32(thsc->kpthsc_system_time)
	195	);
5ba3f43e	196	BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16)
0a7de745 A	197	\| thsc->kpthsc_sched_priority,
	198	((thsc->kpthsc_state & 0xff) << 24)
	199	\| (thsc->kpthsc_effective_qos << 6)
	200	\| (thsc->kpthsc_requested_qos << 3)
	201	\| thsc->kpthsc_requested_qos_override,
	202	((uint32_t)thsc->kpthsc_effective_latency_qos << 29)
	203	\| ((uint32_t)thsc->kpthsc_requested_qos_promote << 26)
cb323159	204	\| ((uint32_t)thsc->kpthsc_requested_qos_kevent_override << 23)
0a7de745	205	);
d9a64523	206	BUF_DATA(PERF_TI_SCHEDDATA3_32, UPPER_32(thsc->kpthsc_runnable_time),
0a7de745	207	LOWER_32(thsc->kpthsc_runnable_time));
39037602 A	208	#endif /* defined(__LP64__) */
	209	}
	210
	211	/*
	212	* Snapshot information maintains parity with stackshot information for other,
	213	* miscellaneous information about threads.
	214	*/
	215
	216	#define KPERF_THREAD_SNAPSHOT_DARWIN_BG (1U << 0);
	217	#define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1);
	218	#define KPERF_THREAD_SNAPSHOT_GFI (1U << 2);
	219	#define KPERF_THREAD_SNAPSHOT_IDLE_WQ (1U << 3);
	220	/* max is 1U << 7 */
	221
	222	void
	223	kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn,
0a7de745	224	struct kperf_context *context)
39037602 A	225	{
	226	assert(thsn != NULL);
	227	assert(context != NULL);
	228
	229	thread_t thread = context->cur_thread;
	230
	231	BUF_INFO(PERF_TI_SNAPSAMPLE \| DBG_FUNC_START, (uintptr_t)thread_tid(thread));
	232
	233	thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time;
	234
	235	thsn->kpthsn_flags = 0;
	236	if (thread->effective_policy.thep_darwinbg) {
	237	thsn->kpthsn_flags \|= KPERF_THREAD_SNAPSHOT_DARWIN_BG;
	238	}
	239	if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
	240	thsn->kpthsn_flags \|= KPERF_THREAD_SNAPSHOT_PASSIVE_IO;
	241	}
	242	if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
	243	thsn->kpthsn_flags \|= KPERF_THREAD_SNAPSHOT_GFI
	244	}
	245	if (stackshot_thread_is_idle_worker_unsafe(thread)) {
	246	thsn->kpthsn_flags \|= KPERF_THREAD_SNAPSHOT_IDLE_WQ;
	247	}
	248
	249	thsn->kpthsn_suspend_count = thread->suspend_count;
f427ee49 A	250	/*
	251	* Only have room for 8-bits in the trace event, so truncate here.
	252	*/
	253	thsn->kpthsn_io_tier = (uint8_t)proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
39037602 A	254
	255	BUF_VERB(PERF_TI_SNAPSAMPLE \| DBG_FUNC_END);
	256	}
	257
	258	void
	259	kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn)
	260	{
	261	assert(thsn != NULL);
	262	#if defined(__LP64__)
	263	BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags \| ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
0a7de745 A	264	\| (thsn->kpthsn_io_tier << 24),
0a7de745 A	265	thsn->kpthsn_last_made_runnable_time);
39037602 A	266	#else
39037602 A	267	BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags \| ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
0a7de745 A	268	\| (thsn->kpthsn_io_tier << 24),
	269	UPPER_32(thsn->kpthsn_last_made_runnable_time),
	270	LOWER_32(thsn->kpthsn_last_made_runnable_time));
39037602 A	271	#endif /* defined(__LP64__) */
	272	}
	273
	274	/*
	275	* Dispatch information only contains the dispatch queue serial number from
	276	* libdispatch.
	277	*
	278	* It's a separate sampler because queue data must be copied in from user space.
	279	*/
	280
	281	void
	282	kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi,
0a7de745	283	struct kperf_context *context)
39037602 A	284	{
	285	assert(thdi != NULL);
	286	assert(context != NULL);
	287
	288	thread_t thread = context->cur_thread;
	289
	290	BUF_INFO(PERF_TI_DISPSAMPLE \| DBG_FUNC_START, (uintptr_t)thread_tid(thread));
	291
	292	task_t task = thread->task;
d9a64523	293	boolean_t task_64 = task_has_64Bit_addr(task);
39037602 A	294	size_t user_addr_size = task_64 ? 8 : 4;
	295
	296	assert(thread->task != kernel_task);
	297	uint64_t user_dq_key_addr = thread_dispatchqaddr(thread);
	298	if (user_dq_key_addr == 0) {
	299	goto error;
	300	}
	301
	302	uint64_t user_dq_addr;
	303	if ((copyin((user_addr_t)user_dq_key_addr,
0a7de745 A	304	(char *)&user_dq_addr,
	305	user_addr_size) != 0) \|\|
	306	(user_dq_addr == 0)) {
39037602 A	307	goto error;
	308	}
	309
	310	uint64_t user_dq_serialno_addr =
0a7de745	311	user_dq_addr + get_task_dispatchqueue_serialno_offset(task);
39037602 A	312
39037602 A	313	if (copyin((user_addr_t)user_dq_serialno_addr,
0a7de745 A	314	(char *)&(thdi->kpthdi_dq_serialno),
0a7de745 A	315	user_addr_size) == 0) {
39037602 A	316	goto out;
	317	}
	318
	319	error:
	320	thdi->kpthdi_dq_serialno = 0;
	321
	322	out:
	323	BUF_VERB(PERF_TI_DISPSAMPLE \| DBG_FUNC_END);
	324	}
	325
	326	int
94ff46dc A	327	kperf_thread_dispatch_pend(struct kperf_context *context,
94ff46dc A	328	unsigned int actionid)
39037602	329	{
94ff46dc A	330	return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH,
94ff46dc A	331	actionid);
39037602 A	332	}
	333
	334	void
	335	kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi)
	336	{
	337	assert(thdi != NULL);
	338	#if defined(__LP64__)
	339	BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno);
	340	#else
	341	BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno),
0a7de745	342	LOWER_32(thdi->kpthdi_dq_serialno));
39037602 A	343	#endif /* defined(__LP64__) */
39037602 A	344	}
5ba3f43e A	345
	346	/*
	347	* A bit different from other samplers -- since logging disables interrupts,
	348	* it's a fine place to sample the thread counters.
	349	*/
	350	void
	351	kperf_thread_inscyc_log(struct kperf_context *context)
	352	{
	353	#if MONOTONIC
	354	thread_t cur_thread = current_thread();
	355
	356	if (context->cur_thread != cur_thread) {
	357	/* can't safely access another thread's counters */
	358	return;
	359	}
	360
cb323159 A	361	uint64_t counts[MT_CORE_NFIXED] = { 0 };
cb323159 A	362	mt_cur_thread_fixed_counts(counts);
5ba3f43e A	363
	364	#if defined(__LP64__)
	365	BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]);
	366	#else /* defined(__LP64__) */
	367	/* 32-bit platforms don't count instructions */
	368	BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]),
0a7de745	369	LOWER_32(counts[MT_CORE_CYCLES]));
5ba3f43e	370	#endif /* !defined(__LP64__) */
d9a64523 A	371
d9a64523 A	372	#else
5ba3f43e	373	#pragma unused(context)
d9a64523	374	#endif /* MONOTONIC */
5ba3f43e	375	}