osfmk/kperf/thread_samplers.c

   1 /*
   2  * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /*  Sample thread data */
  30
  31 #include <kern/debug.h> /* panic */
  32 #include <kern/thread.h> /* thread_* */
  33 #include <kern/timer.h> /* timer_data_t */
  34 #include <kern/policy_internal.h> /* TASK_POLICY_* */
  35 #include <mach/mach_types.h>
  36
  37 #include <kperf/kperf.h>
  38 #include <kperf/buffer.h>
  39 #include <kperf/context.h>
  40 #include <kperf/thread_samplers.h>
  41 #include <kperf/ast.h>
  42
  43 #if MONOTONIC
  44 #include <kern/monotonic.h>
  45 #include <machine/monotonic.h>
  46 #endif /* MONOTONIC */
  47
  48 extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
  49
  50 /*
  51  * XXX Deprecated, use thread scheduling sampler instead.
  52  *
  53  * Taken from AppleProfileGetRunModeOfThread and CHUD.  Still here for
  54  * backwards compatibility.
  55  */
  56
  57 #define KPERF_TI_RUNNING   (1U << 0)
  58 #define KPERF_TI_RUNNABLE  (1U << 1)
  59 #define KPERF_TI_WAIT      (1U << 2)
  60 #define KPERF_TI_UNINT     (1U << 3)
  61 #define KPERF_TI_SUSP      (1U << 4)
  62 #define KPERF_TI_TERMINATE (1U << 5)
  63 #define KPERF_TI_IDLE      (1U << 6)
  64
  65 static uint32_t
  66 kperf_thread_info_runmode_legacy(thread_t thread)
  67 {
  68         uint32_t kperf_state = 0;
  69         int sched_state = thread->state;
  70         processor_t last_processor = thread->last_processor;
  71
  72         if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) {
  73                 kperf_state |= KPERF_TI_RUNNING;
  74         }
  75         if (sched_state & TH_RUN) {
  76                 kperf_state |= KPERF_TI_RUNNABLE;
  77         }
  78         if (sched_state & TH_WAIT) {
  79                 kperf_state |= KPERF_TI_WAIT;
  80         }
  81         if (sched_state & TH_UNINT) {
  82                 kperf_state |= KPERF_TI_UNINT;
  83         }
  84         if (sched_state & TH_SUSP) {
  85                 kperf_state |= KPERF_TI_SUSP;
  86         }
  87         if (sched_state & TH_TERMINATE) {
  88                 kperf_state |= KPERF_TI_TERMINATE;
  89         }
  90         if (sched_state & TH_IDLE) {
  91                 kperf_state |= KPERF_TI_IDLE;
  92         }
  93
  94 #if !CONFIG_EMBEDDED
  95         /* on desktop, if state is blank, leave not idle set */
  96         if (kperf_state == 0) {
  97                 return (TH_IDLE << 16);
  98         }
  99 #endif /* !CONFIG_EMBEDDED */
 100
 101         /* high two bytes are inverted mask, low two bytes are normal */
 102         return (((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff));
 103 }
 104
 105 void
 106 kperf_thread_info_sample(struct kperf_thread_info *ti, struct kperf_context *context)
 107 {
 108         thread_t cur_thread = context->cur_thread;
 109
 110         BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread));
 111
 112         ti->kpthi_pid = context->cur_pid;
 113         ti->kpthi_tid = thread_tid(cur_thread);
 114         ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread);
 115         ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread);
 116
 117         BUF_VERB(PERF_TI_SAMPLE | DBG_FUNC_END);
 118 }
 119
 120 void
 121 kperf_thread_info_log(struct kperf_thread_info *ti)
 122 {
 123         BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */,
 124                                ti->kpthi_dq_addr, ti->kpthi_runmode);
 125 }
 126
 127 /*
 128  * Scheduling information reports inputs and outputs of the scheduler state for
 129  * a thread.
 130  */
 131
 132 void
 133 kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc,
 134                                struct kperf_context *context)
 135 {
 136         assert(thsc != NULL);
 137         assert(context != NULL);
 138
 139         thread_t thread = context->cur_thread;
 140
 141         BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
 142
 143         thsc->kpthsc_user_time = timer_grab(&(thread->user_timer));
 144         uint64_t system_time = timer_grab(&(thread->system_timer));
 145
 146         if (thread->precise_user_kernel_time) {
 147                 thsc->kpthsc_system_time = system_time;
 148         } else {
 149                 thsc->kpthsc_user_time += system_time;
 150                 thsc->kpthsc_system_time = 0;
 151         }
 152
 153         thsc->kpthsc_state = thread->state;
 154         thsc->kpthsc_base_priority = thread->base_pri;
 155         thsc->kpthsc_sched_priority = thread->sched_pri;
 156         thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos;
 157         thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos;
 158         thsc->kpthsc_requested_qos_override = thread->requested_policy.thrp_qos_override;
 159         thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote;
 160         thsc->kpthsc_requested_qos_ipc_override = thread->requested_policy.thrp_qos_ipc_override;
 161         thsc->kpthsc_requested_qos_sync_ipc_override = thread->requested_policy.thrp_qos_sync_ipc_override;
 162         thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos;
 163
 164         BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END);
 165 }
 166
 167
 168 void
 169 kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc)
 170 {
 171         assert(thsc != NULL);
 172 #if defined(__LP64__)
 173         BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time,
 174                         thsc->kpthsc_system_time,
 175                         (((uint64_t)thsc->kpthsc_base_priority) << 48)
 176                         | ((uint64_t)thsc->kpthsc_sched_priority << 32)
 177                         | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24)
 178                         | (thsc->kpthsc_effective_qos << 6)
 179                         | (thsc->kpthsc_requested_qos << 3)
 180                         | thsc->kpthsc_requested_qos_override,
 181                         ((uint64_t)thsc->kpthsc_effective_latency_qos << 61)
 182                         | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58)
 183                         | ((uint64_t)thsc->kpthsc_requested_qos_ipc_override << 55)
 184                         | ((uint64_t)thsc->kpthsc_requested_qos_sync_ipc_override << 52)
 185                         );
 186 #else
 187         BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time),
 188                         LOWER_32(thsc->kpthsc_user_time),
 189                         UPPER_32(thsc->kpthsc_system_time),
 190                         LOWER_32(thsc->kpthsc_system_time)
 191                         );
 192         BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16)
 193                         | thsc->kpthsc_sched_priority,
 194                         ((thsc->kpthsc_state & 0xff) << 24)
 195                         | (thsc->kpthsc_effective_qos << 6)
 196                         | (thsc->kpthsc_requested_qos << 3)
 197                         | thsc->kpthsc_requested_qos_override,
 198                         ((uint32_t)thsc->kpthsc_effective_latency_qos << 29)
 199                         | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26)
 200                         | ((uint32_t)thsc->kpthsc_requested_qos_ipc_override << 23)
 201                         | ((uint32_t)thsc->kpthsc_requested_qos_sync_ipc_override << 20)
 202                         );
 203 #endif /* defined(__LP64__) */
 204 }
 205
 206 /*
 207  * Snapshot information maintains parity with stackshot information for other,
 208  * miscellaneous information about threads.
 209  */
 210
 211 #define KPERF_THREAD_SNAPSHOT_DARWIN_BG  (1U << 0);
 212 #define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1);
 213 #define KPERF_THREAD_SNAPSHOT_GFI        (1U << 2);
 214 #define KPERF_THREAD_SNAPSHOT_IDLE_WQ    (1U << 3);
 215 /* max is 1U << 7 */
 216
 217 void
 218 kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn,
 219                              struct kperf_context *context)
 220 {
 221         assert(thsn != NULL);
 222         assert(context != NULL);
 223
 224         thread_t thread = context->cur_thread;
 225
 226         BUF_INFO(PERF_TI_SNAPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
 227
 228         thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time;
 229
 230         thsn->kpthsn_flags = 0;
 231         if (thread->effective_policy.thep_darwinbg) {
 232                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_DARWIN_BG;
 233         }
 234         if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
 235                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_PASSIVE_IO;
 236         }
 237         if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
 238                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_GFI
 239         }
 240         if (stackshot_thread_is_idle_worker_unsafe(thread)) {
 241                 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_IDLE_WQ;
 242         }
 243
 244         thsn->kpthsn_suspend_count = thread->suspend_count;
 245         thsn->kpthsn_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
 246
 247         BUF_VERB(PERF_TI_SNAPSAMPLE | DBG_FUNC_END);
 248 }
 249
 250 void
 251 kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn)
 252 {
 253         assert(thsn != NULL);
 254 #if defined(__LP64__)
 255         BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
 256                                                       | (thsn->kpthsn_io_tier << 24),
 257                                    thsn->kpthsn_last_made_runnable_time);
 258 #else
 259         BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
 260                                                          | (thsn->kpthsn_io_tier << 24),
 261                                       UPPER_32(thsn->kpthsn_last_made_runnable_time),
 262                                       LOWER_32(thsn->kpthsn_last_made_runnable_time));
 263 #endif /* defined(__LP64__) */
 264 }
 265
 266 /*
 267  * Dispatch information only contains the dispatch queue serial number from
 268  * libdispatch.
 269  *
 270  * It's a separate sampler because queue data must be copied in from user space.
 271  */
 272
 273 void
 274 kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi,
 275                              struct kperf_context *context)
 276 {
 277         assert(thdi != NULL);
 278         assert(context != NULL);
 279
 280         thread_t thread = context->cur_thread;
 281
 282         BUF_INFO(PERF_TI_DISPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
 283
 284         task_t task = thread->task;
 285         boolean_t task_64 = task_has_64BitAddr(task);
 286         size_t user_addr_size = task_64 ? 8 : 4;
 287
 288         assert(thread->task != kernel_task);
 289         uint64_t user_dq_key_addr = thread_dispatchqaddr(thread);
 290         if (user_dq_key_addr == 0) {
 291                 goto error;
 292         }
 293
 294         uint64_t user_dq_addr;
 295         if ((copyin((user_addr_t)user_dq_key_addr,
 296                     (char *)&user_dq_addr,
 297                     user_addr_size) != 0) ||
 298                 (user_dq_addr == 0))
 299         {
 300                 goto error;
 301         }
 302
 303         uint64_t user_dq_serialno_addr =
 304                 user_dq_addr + get_task_dispatchqueue_serialno_offset(task);
 305
 306         if (copyin((user_addr_t)user_dq_serialno_addr,
 307                    (char *)&(thdi->kpthdi_dq_serialno),
 308                    user_addr_size) == 0)
 309         {
 310                 goto out;
 311         }
 312
 313 error:
 314         thdi->kpthdi_dq_serialno = 0;
 315
 316 out:
 317         BUF_VERB(PERF_TI_DISPSAMPLE | DBG_FUNC_END);
 318 }
 319
 320 int
 321 kperf_thread_dispatch_pend(struct kperf_context *context)
 322 {
 323         return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH);
 324 }
 325
 326 void
 327 kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi)
 328 {
 329         assert(thdi != NULL);
 330 #if defined(__LP64__)
 331         BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno);
 332 #else
 333         BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno),
 334                                       LOWER_32(thdi->kpthdi_dq_serialno));
 335 #endif /* defined(__LP64__) */
 336 }
 337
 338 /*
 339  * A bit different from other samplers -- since logging disables interrupts,
 340  * it's a fine place to sample the thread counters.
 341  */
 342 void
 343 kperf_thread_inscyc_log(struct kperf_context *context)
 344 {
 345 #if MONOTONIC
 346         thread_t cur_thread = current_thread();
 347
 348         if (context->cur_thread != cur_thread) {
 349                 /* can't safely access another thread's counters */
 350                 return;
 351         }
 352
 353         uint64_t counts[MT_CORE_NFIXED];
 354
 355         int ret = mt_fixed_thread_counts(cur_thread, counts);
 356         if (ret) {
 357                 return;
 358         }
 359
 360 #if defined(__LP64__)
 361         BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]);
 362 #else /* defined(__LP64__) */
 363         /* 32-bit platforms don't count instructions */
 364         BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]),
 365                         LOWER_32(counts[MT_CORE_CYCLES]));
 366 #endif /* !defined(__LP64__) */
 367 #else /* MONOTONIC */
 368 #pragma unused(context)
 369 #endif /* !MONOTONIC */
 370 }