osfmk/kperf/thread_samplers.c

   1 /*
   2  * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /*  Sample thread data */
  30
  31 #include <kern/debug.h> /* panic */
  32 #include <kern/thread.h> /* thread_* */
  33 #include <kern/timer.h> /* timer_data_t */
  34 #include <kern/policy_internal.h> /* TASK_POLICY_* */
  35 #include <mach/mach_types.h>
  36
  37 #include <kperf/kperf.h>
  38 #include <kperf/buffer.h>
  39 #include <kperf/context.h>
  40 #include <kperf/thread_samplers.h>
  41 #include <kperf/ast.h>
  42
  43 #if MONOTONIC
  44 #include <kern/monotonic.h>
  45 #include <machine/monotonic.h>
  46 #endif /* MONOTONIC */
  47
  48 extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
  49
  50 /*
  51  * XXX Deprecated, use thread scheduling sampler instead.
  52  *
  53  * Taken from AppleProfileGetRunModeOfThread and CHUD.  Still here for
  54  * backwards compatibility.
  55  */
  56
  57 #define KPERF_TI_RUNNING   (1U << 0)
  58 #define KPERF_TI_RUNNABLE  (1U << 1)
  59 #define KPERF_TI_WAIT      (1U << 2)
  60 #define KPERF_TI_UNINT     (1U << 3)
  61 #define KPERF_TI_SUSP      (1U << 4)
  62 #define KPERF_TI_TERMINATE (1U << 5)
  63 #define KPERF_TI_IDLE      (1U << 6)
  64
  65 static uint32_t
  66 kperf_thread_info_runmode_legacy(thread_t thread)
  67 {
  68         uint32_t kperf_state = 0;
  69         int sched_state = thread->state;
  70         processor_t last_processor = thread->last_processor;
  71
  72         if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) {
  73                 kperf_state |= KPERF_TI_RUNNING;
  74         }
  75         if (sched_state & TH_RUN) {
  76                 kperf_state |= KPERF_TI_RUNNABLE;
  77         }
  78         if (sched_state & TH_WAIT) {
  79                 kperf_state |= KPERF_TI_WAIT;
  80         }
  81         if (sched_state & TH_UNINT) {
  82                 kperf_state |= KPERF_TI_UNINT;
  83         }
  84         if (sched_state & TH_SUSP) {
  85                 kperf_state |= KPERF_TI_SUSP;
  86         }
  87         if (sched_state & TH_TERMINATE) {
  88                 kperf_state |= KPERF_TI_TERMINATE;
  89         }
  90         if (sched_state & TH_IDLE) {
  91                 kperf_state |= KPERF_TI_IDLE;
  92         }
  93
  94 #if defined(XNU_TARGET_OS_OSX)
  95         /* on desktop, if state is blank, leave not idle set */
  96         if (kperf_state == 0) {
  97                 return TH_IDLE << 16;
  98         }
  99 #endif /* defined(XNU_TARGET_OS_OSX) */
 100
 101         /* high two bytes are inverted mask, low two bytes are normal */
 102         return ((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff);
 103 }
 104
 105 void
 106 kperf_thread_info_sample(struct kperf_thread_info *ti, struct kperf_context *context)
 107 {
 108         thread_t cur_thread = context->cur_thread;
 109
 110         BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread));
 111
 112         ti->kpthi_pid = context->cur_pid;
 113         ti->kpthi_tid = thread_tid(cur_thread);
 114         ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread);
 115         ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread);
 116
 117         BUF_VERB(PERF_TI_SAMPLE | DBG_FUNC_END);
 118 }
 119
 120 void
 121 kperf_thread_info_log(struct kperf_thread_info *ti)
 122 {
 123         BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */,
 124             ti->kpthi_dq_addr, ti->kpthi_runmode);
 125 }
 126
 127 /*
 128  * Scheduling information reports inputs and outputs of the scheduler state for
 129  * a thread.
 130  */
 131
 132 void
 133 kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc,
 134     struct kperf_context *context)
 135 {
 136         assert(thsc != NULL);
 137         assert(context != NULL);
 138
 139         thread_t thread = context->cur_thread;
 140
 141         BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
 142
 143         thsc->kpthsc_user_time = timer_grab(&thread->user_timer);
 144         uint64_t system_time = timer_grab(&thread->system_timer);
 145
 146         if (thread->precise_user_kernel_time) {
 147                 thsc->kpthsc_system_time = system_time;
 148         } else {
 149                 thsc->kpthsc_user_time += system_time;
 150                 thsc->kpthsc_system_time = 0;
 151         }
 152
 153         thsc->kpthsc_runnable_time = timer_grab(&thread->runnable_timer);
 154         thsc->kpthsc_state = thread->state;
 155         thsc->kpthsc_base_priority = thread->base_pri;
 156         thsc->kpthsc_sched_priority = thread->sched_pri;
 157         thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos;
 158         thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos;
 159         thsc->kpthsc_requested_qos_override = MAX(thread->requested_policy.thrp_qos_override,
 160             thread->requested_policy.thrp_qos_workq_override);
 161         thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote;
 162         thsc->kpthsc_requested_qos_kevent_override = MAX(
 163                 thread->requested_policy.thrp_qos_kevent_override,
 164                 thread->requested_policy.thrp_qos_wlsvc_override);
 165         thsc->kpthsc_requested_qos_sync_ipc_override = THREAD_QOS_UNSPECIFIED;
 166         thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos;
 167
 168         BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END);
 169 }
 170
 171
 172 void
 173 kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc)
 174 {
 175         assert(thsc != NULL);
 176 #if defined(__LP64__)
 177         BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time,
 178             thsc->kpthsc_system_time,
 179             (((uint64_t)thsc->kpthsc_base_priority) << 48)
 180             | ((uint64_t)thsc->kpthsc_sched_priority << 32)
 181             | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24)
 182             | (thsc->kpthsc_effective_qos << 6)
 183             | (thsc->kpthsc_requested_qos << 3)
 184             | thsc->kpthsc_requested_qos_override,
 185             ((uint64_t)thsc->kpthsc_effective_latency_qos << 61)
 186             | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58)
 187             | ((uint64_t)thsc->kpthsc_requested_qos_kevent_override << 55)
 188             );
 189         BUF_DATA(PERF_TI_SCHEDDATA_3, thsc->kpthsc_runnable_time);
 190 #else
 191         BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time),
 192             LOWER_32(thsc->kpthsc_user_time),
 193             UPPER_32(thsc->kpthsc_system_time),
 194             LOWER_32(thsc->kpthsc_system_time)
 195             );
 196         BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16)
 197             | thsc->kpthsc_sched_priority,
 198             ((thsc->kpthsc_state & 0xff) << 24)
 199             | (thsc->kpthsc_effective_qos << 6)
 200             | (thsc->kpthsc_requested_qos << 3)
 201             | thsc->kpthsc_requested_qos_override,
 202             ((uint32_t)thsc->kpthsc_effective_latency_qos << 29)
 203             | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26)
 204             | ((uint32_t)thsc->kpthsc_requested_qos_kevent_override << 23)
 205             );
 206         BUF_DATA(PERF_TI_SCHEDDATA3_32, UPPER_32(thsc->kpthsc_runnable_time),
 207             LOWER_32(thsc->kpthsc_runnable_time));
 208 #endif /* defined(__LP64__) */
 209 }
 210
 211 /*
 212  * Snapshot information maintains parity with stackshot information for other,
 213  * miscellaneous information about threads.
 214  */
 215
 216 #define KPERF_THREAD_SNAPSHOT_DARWIN_BG  (1U << 0);
 217 #define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1);
 218 #define KPERF_THREAD_SNAPSHOT_GFI        (1U << 2);
 219 #define KPERF_THREAD_SNAPSHOT_IDLE_WQ    (1U << 3);
 220 /* max is 1U << 7 */
 221
 222 void
 223 kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn,
 224     struct kperf_context *context)
 225 {
 226         assert(thsn != NULL);
 227         assert(context != NULL);
 228
 229         thread_t thread = context->cur_thread;
 230
 231         BUF_INFO(PERF_TI_SNAPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
 232
 233         thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time;
 234
 235         thsn->kpthsn_flags = 0;
 236         if (thread->effective_policy.thep_darwinbg) {
 237                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_DARWIN_BG;
 238         }
 239         if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
 240                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_PASSIVE_IO;
 241         }
 242         if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
 243                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_GFI
 244         }
 245         if (stackshot_thread_is_idle_worker_unsafe(thread)) {
 246                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_IDLE_WQ;
 247         }
 248
 249         thsn->kpthsn_suspend_count = thread->suspend_count;
 250         /*
 251          * Only have room for 8-bits in the trace event, so truncate here.
 252          */
 253         thsn->kpthsn_io_tier = (uint8_t)proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
 254
 255         BUF_VERB(PERF_TI_SNAPSAMPLE | DBG_FUNC_END);
 256 }
 257
 258 void
 259 kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn)
 260 {
 261         assert(thsn != NULL);
 262 #if defined(__LP64__)
 263         BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
 264             | (thsn->kpthsn_io_tier << 24),
 265             thsn->kpthsn_last_made_runnable_time);
 266 #else
 267         BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
 268             | (thsn->kpthsn_io_tier << 24),
 269             UPPER_32(thsn->kpthsn_last_made_runnable_time),
 270             LOWER_32(thsn->kpthsn_last_made_runnable_time));
 271 #endif /* defined(__LP64__) */
 272 }
 273
 274 /*
 275  * Dispatch information only contains the dispatch queue serial number from
 276  * libdispatch.
 277  *
 278  * It's a separate sampler because queue data must be copied in from user space.
 279  */
 280
 281 void
 282 kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi,
 283     struct kperf_context *context)
 284 {
 285         assert(thdi != NULL);
 286         assert(context != NULL);
 287
 288         thread_t thread = context->cur_thread;
 289
 290         BUF_INFO(PERF_TI_DISPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
 291
 292         task_t task = thread->task;
 293         boolean_t task_64 = task_has_64Bit_addr(task);
 294         size_t user_addr_size = task_64 ? 8 : 4;
 295
 296         assert(thread->task != kernel_task);
 297         uint64_t user_dq_key_addr = thread_dispatchqaddr(thread);
 298         if (user_dq_key_addr == 0) {
 299                 goto error;
 300         }
 301
 302         uint64_t user_dq_addr;
 303         if ((copyin((user_addr_t)user_dq_key_addr,
 304             (char *)&user_dq_addr,
 305             user_addr_size) != 0) ||
 306             (user_dq_addr == 0)) {
 307                 goto error;
 308         }
 309
 310         uint64_t user_dq_serialno_addr =
 311             user_dq_addr + get_task_dispatchqueue_serialno_offset(task);
 312
 313         if (copyin((user_addr_t)user_dq_serialno_addr,
 314             (char *)&(thdi->kpthdi_dq_serialno),
 315             user_addr_size) == 0) {
 316                 goto out;
 317         }
 318
 319 error:
 320         thdi->kpthdi_dq_serialno = 0;
 321
 322 out:
 323         BUF_VERB(PERF_TI_DISPSAMPLE | DBG_FUNC_END);
 324 }
 325
 326 int
 327 kperf_thread_dispatch_pend(struct kperf_context *context,
 328     unsigned int actionid)
 329 {
 330         return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH,
 331                    actionid);
 332 }
 333
 334 void
 335 kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi)
 336 {
 337         assert(thdi != NULL);
 338 #if defined(__LP64__)
 339         BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno);
 340 #else
 341         BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno),
 342             LOWER_32(thdi->kpthdi_dq_serialno));
 343 #endif /* defined(__LP64__) */
 344 }
 345
 346 /*
 347  * A bit different from other samplers -- since logging disables interrupts,
 348  * it's a fine place to sample the thread counters.
 349  */
 350 void
 351 kperf_thread_inscyc_log(struct kperf_context *context)
 352 {
 353 #if MONOTONIC
 354         thread_t cur_thread = current_thread();
 355
 356         if (context->cur_thread != cur_thread) {
 357                 /* can't safely access another thread's counters */
 358                 return;
 359         }
 360
 361         uint64_t counts[MT_CORE_NFIXED] = { 0 };
 362         mt_cur_thread_fixed_counts(counts);
 363
 364 #if defined(__LP64__)
 365         BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]);
 366 #else /* defined(__LP64__) */
 367         /* 32-bit platforms don't count instructions */
 368         BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]),
 369             LOWER_32(counts[MT_CORE_CYCLES]));
 370 #endif /* !defined(__LP64__) */
 371
 372 #else
 373 #pragma unused(context)
 374 #endif /* MONOTONIC */
 375 }