osfmk/kperf/pet.c

   1 /*
   2  * Copyright (c) 2011-2016 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /* all thread states code */
  30 #include <mach/mach_types.h>
  31 #include <sys/errno.h>
  32
  33 #include <kperf/kperf.h>
  34 #include <kperf/buffer.h>
  35 #include <kperf/sample.h>
  36 #include <kperf/context.h>
  37 #include <kperf/action.h>
  38 #include <kperf/pet.h>
  39 #include <kperf/kperf_timer.h>
  40
  41 #include <kern/task.h>
  42 #include <kern/kalloc.h>
  43
  44 /* action ID to call for each sample
  45  *
  46  * Address is used as the sync point for waiting.
  47  */
  48 static unsigned int pet_action_id = 0;
  49
  50 static lck_mtx_t *pet_lock;
  51 static boolean_t pet_initted = FALSE;
  52 static boolean_t pet_running = FALSE;
  53
  54 /* number of callstack samples to skip for idle threads */
  55 static uint32_t pet_idle_rate = KPERF_PET_DEFAULT_IDLE_RATE;
  56
  57 /*
  58  * Lightweight PET mode samples the system less-intrusively than normal PET
  59  * mode.  Instead of iterating tasks and threads on each sample, it increments
  60  * a global generation count, kperf_pet_gen, which is checked as threads are
  61  * context switched on-core.  If the thread's local generation count is older
  62  * than the global generation, the thread samples itself.
  63  *
  64  *            |  |
  65  * thread A   +--+---------|
  66  *            |  |
  67  * thread B   |--+---------------|
  68  *            |  |
  69  * thread C   |  |         |-------------------------------------
  70  *            |  |         |
  71  * thread D   |  |         |     |-------------------------------
  72  *            |  |         |     |
  73  *            +--+---------+-----+--------------------------------> time
  74  *               |         │     |
  75  *               |         +-----+--- threads sampled when they come on-core in
  76  *               |                    kperf_pet_switch_context
  77  *               |
  78  *               +--- PET timer fire, sample on-core threads A and B,
  79  *                    increment kperf_pet_gen
  80  */
  81 static boolean_t lightweight_pet = FALSE;
  82
  83 /*
  84  * Whether or not lightweight PET and sampling is active.
  85  */
  86 boolean_t kperf_lightweight_pet_active = FALSE;
  87
  88 uint32_t kperf_pet_gen = 0;
  89
  90 static struct kperf_sample *pet_sample;
  91
  92 /* thread lifecycle */
  93
  94 static kern_return_t pet_init(void);
  95 static void pet_start(void);
  96 static void pet_stop(void);
  97
  98 /* PET thread-only */
  99
 100 static void pet_thread_loop(void *param, wait_result_t wr);
 101 static void pet_thread_idle(void);
 102 static void pet_thread_work_unit(void);
 103
 104 /* listing things to sample */
 105
 106 static task_array_t pet_tasks = NULL;
 107 static vm_size_t pet_tasks_size = 0;
 108 static vm_size_t pet_tasks_count = 0;
 109
 110 static thread_array_t pet_threads = NULL;
 111 static vm_size_t pet_threads_size = 0;
 112 static vm_size_t pet_threads_count = 0;
 113
 114 static kern_return_t pet_tasks_prepare(void);
 115 static kern_return_t pet_tasks_prepare_internal(void);
 116
 117 static kern_return_t pet_threads_prepare(task_t task);
 118
 119 /* sampling */
 120
 121 static void pet_sample_all_tasks(uint32_t idle_rate);
 122 static void pet_sample_task(task_t task, uint32_t idle_rate);
 123 static void pet_sample_thread(int pid, thread_t thread, uint32_t idle_rate);
 124
 125 /* functions called by other areas of kperf */
 126
 127 void
 128 kperf_pet_fire_before(void)
 129 {
 130         if (!pet_initted || !pet_running) {
 131                 return;
 132         }
 133
 134         if (lightweight_pet) {
 135                 BUF_INFO(PERF_PET_SAMPLE);
 136                 OSIncrementAtomic(&kperf_pet_gen);
 137         }
 138 }
 139
 140 void
 141 kperf_pet_fire_after(void)
 142 {
 143         if (!pet_initted || !pet_running) {
 144                 return;
 145         }
 146
 147         if (lightweight_pet) {
 148                 kperf_timer_pet_rearm(0);
 149         } else {
 150                 thread_wakeup(&pet_action_id);
 151         }
 152 }
 153
 154 void
 155 kperf_pet_on_cpu(thread_t thread, thread_continue_t continuation,
 156                  uintptr_t *starting_fp)
 157 {
 158         assert(thread != NULL);
 159         assert(ml_get_interrupts_enabled() == FALSE);
 160
 161         if (thread->kperf_pet_gen != kperf_pet_gen) {
 162                 BUF_VERB(PERF_PET_SAMPLE_THREAD | DBG_FUNC_START, kperf_pet_gen, thread->kperf_pet_gen);
 163
 164                 struct kperf_context ctx = {
 165                         .cur_thread = thread,
 166                         .cur_pid = task_pid(get_threadtask(thread)),
 167                         .starting_fp = starting_fp,
 168                 };
 169                 /*
 170                  * Use a per-CPU interrupt buffer, since this is only called
 171                  * while interrupts are disabled, from the scheduler.
 172                  */
 173                 struct kperf_sample *sample = kperf_intr_sample_buffer();
 174                 if (!sample) {
 175                         BUF_VERB(PERF_PET_SAMPLE_THREAD | DBG_FUNC_END, 1);
 176                         return;
 177                 }
 178
 179                 unsigned int flags = SAMPLE_FLAG_NON_INTERRUPT | SAMPLE_FLAG_PEND_USER;
 180                 if (continuation != NULL) {
 181                         flags |= SAMPLE_FLAG_CONTINUATION;
 182                 }
 183                 kperf_sample(sample, &ctx, pet_action_id, flags);
 184
 185                 BUF_VERB(PERF_PET_SAMPLE_THREAD | DBG_FUNC_END);
 186         } else {
 187                 BUF_VERB(PERF_PET_SAMPLE_THREAD, kperf_pet_gen, thread->kperf_pet_gen);
 188         }
 189 }
 190
 191 void
 192 kperf_pet_config(unsigned int action_id)
 193 {
 194         kern_return_t kr = pet_init();
 195         if (kr != KERN_SUCCESS) {
 196                 return;
 197         }
 198
 199         lck_mtx_lock(pet_lock);
 200
 201         BUF_INFO(PERF_PET_THREAD, 3, action_id);
 202
 203         if (action_id == 0) {
 204                 pet_stop();
 205         } else {
 206                 pet_start();
 207         }
 208
 209         pet_action_id = action_id;
 210
 211         lck_mtx_unlock(pet_lock);
 212 }
 213
 214 /* handle resource allocation */
 215
 216 void
 217 pet_start(void)
 218 {
 219         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 220
 221         if (pet_running) {
 222                 return;
 223         }
 224
 225         pet_sample = kalloc(sizeof(struct kperf_sample));
 226         if (!pet_sample) {
 227                 return;
 228         }
 229
 230         pet_running = TRUE;
 231 }
 232
 233 void
 234 pet_stop(void)
 235 {
 236         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 237
 238         if (!pet_initted) {
 239                 return;
 240         }
 241
 242         if (pet_tasks != NULL) {
 243                 assert(pet_tasks_size != 0);
 244                 kfree(pet_tasks, pet_tasks_size);
 245
 246                 pet_tasks = NULL;
 247                 pet_tasks_size = 0;
 248                 pet_tasks_count = 0;
 249         }
 250
 251         if (pet_threads != NULL) {
 252                 assert(pet_threads_size != 0);
 253                 kfree(pet_threads, pet_threads_size);
 254
 255                 pet_threads = NULL;
 256                 pet_threads_size = 0;
 257                 pet_threads_count = 0;
 258         }
 259
 260         if (pet_sample != NULL) {
 261                 kfree(pet_sample, sizeof(struct kperf_sample));
 262                 pet_sample = NULL;
 263         }
 264
 265         pet_running = FALSE;
 266 }
 267
 268 /*
 269  * Lazily initialize PET.  The PET thread never exits once PET has been used
 270  * once.
 271  */
 272 static kern_return_t
 273 pet_init(void)
 274 {
 275         if (pet_initted) {
 276                 return KERN_SUCCESS;
 277         }
 278
 279         /* make the sync point */
 280         pet_lock = lck_mtx_alloc_init(&kperf_lck_grp, NULL);
 281         assert(pet_lock);
 282
 283         /* create the thread */
 284
 285         BUF_INFO(PERF_PET_THREAD, 0);
 286         thread_t t;
 287         kern_return_t kr = kernel_thread_start(pet_thread_loop, NULL, &t);
 288         if (kr != KERN_SUCCESS) {
 289                 lck_mtx_free(pet_lock, &kperf_lck_grp);
 290                 return kr;
 291         }
 292
 293         thread_set_thread_name(t, "kperf sampling");
 294         /* let the thread hold the only reference */
 295         thread_deallocate(t);
 296
 297         pet_initted = TRUE;
 298
 299         return KERN_SUCCESS;
 300 }
 301
 302 /* called by PET thread only */
 303
 304 static void
 305 pet_thread_work_unit(void)
 306 {
 307         pet_sample_all_tasks(pet_idle_rate);
 308 }
 309
 310 static void
 311 pet_thread_idle(void)
 312 {
 313         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 314
 315         (void)lck_mtx_sleep(pet_lock, LCK_SLEEP_DEFAULT, &pet_action_id,
 316                             THREAD_UNINT);
 317 }
 318
 319 __attribute__((noreturn))
 320 static void
 321 pet_thread_loop(void *param, wait_result_t wr)
 322 {
 323 #pragma unused(param, wr)
 324         uint64_t work_unit_ticks;
 325
 326         BUF_INFO(PERF_PET_THREAD, 1);
 327
 328         lck_mtx_lock(pet_lock);
 329         for (;;) {
 330                 BUF_INFO(PERF_PET_IDLE);
 331                 pet_thread_idle();
 332
 333                 BUF_INFO(PERF_PET_RUN);
 334
 335                 /* measure how long the work unit takes */
 336                 work_unit_ticks = mach_absolute_time();
 337                 pet_thread_work_unit();
 338                 work_unit_ticks = mach_absolute_time() - work_unit_ticks;
 339
 340                 /* re-program the timer */
 341                 kperf_timer_pet_rearm(work_unit_ticks);
 342         }
 343 }
 344
 345 /* sampling */
 346
 347 static void
 348 pet_sample_thread(int pid, thread_t thread, uint32_t idle_rate)
 349 {
 350         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 351
 352         uint32_t sample_flags = SAMPLE_FLAG_IDLE_THREADS;
 353
 354         BUF_VERB(PERF_PET_SAMPLE_THREAD | DBG_FUNC_START);
 355
 356         /* work out the context */
 357         struct kperf_context ctx = {
 358                 .cur_thread = thread,
 359                 .cur_pid = pid,
 360         };
 361
 362         boolean_t thread_dirty = kperf_thread_get_dirty(thread);
 363
 364         /*
 365          * Clean a dirty thread and skip callstack sample if the thread was not
 366          * dirty and thread has skipped less than pet_idle_rate samples.
 367          */
 368         if (thread_dirty) {
 369                 kperf_thread_set_dirty(thread, FALSE);
 370         } else if ((thread->kperf_pet_cnt % idle_rate) != 0) {
 371                 sample_flags |= SAMPLE_FLAG_EMPTY_CALLSTACK;
 372         }
 373         thread->kperf_pet_cnt++;
 374
 375         kperf_sample(pet_sample, &ctx, pet_action_id, sample_flags);
 376
 377         BUF_VERB(PERF_PET_SAMPLE_THREAD | DBG_FUNC_END);
 378 }
 379
 380 static kern_return_t
 381 pet_threads_prepare(task_t task)
 382 {
 383         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 384
 385         vm_size_t threads_size_needed;
 386
 387         if (task == TASK_NULL) {
 388                 return KERN_INVALID_ARGUMENT;
 389         }
 390
 391         for (;;) {
 392                 task_lock(task);
 393
 394                 if (!task->active) {
 395                         task_unlock(task);
 396
 397                         return KERN_FAILURE;
 398                 }
 399
 400                 /* do we have the memory we need? */
 401                 threads_size_needed = task->thread_count * sizeof(thread_t);
 402                 if (threads_size_needed <= pet_threads_size) {
 403                         break;
 404                 }
 405
 406                 /* not enough memory, unlock the task and increase allocation */
 407                 task_unlock(task);
 408
 409                 if (pet_threads_size != 0) {
 410                         kfree(pet_threads, pet_threads_size);
 411                 }
 412
 413                 assert(threads_size_needed > 0);
 414                 pet_threads_size = threads_size_needed;
 415
 416                 pet_threads = kalloc(pet_threads_size);
 417                 if (pet_threads == NULL) {
 418                         pet_threads_size = 0;
 419                         return KERN_RESOURCE_SHORTAGE;
 420                 }
 421         }
 422
 423         /* have memory and the task is locked and active */
 424         thread_t thread;
 425         pet_threads_count = 0;
 426         queue_iterate(&(task->threads), thread, thread_t, task_threads) {
 427                 thread_reference_internal(thread);
 428                 pet_threads[pet_threads_count++] = thread;
 429         }
 430
 431         /* can unlock task now that threads are referenced */
 432         task_unlock(task);
 433
 434         return (pet_threads_count == 0) ? KERN_FAILURE : KERN_SUCCESS;
 435 }
 436
 437 static void
 438 pet_sample_task(task_t task, uint32_t idle_rate)
 439 {
 440         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 441
 442         BUF_VERB(PERF_PET_SAMPLE_TASK | DBG_FUNC_START);
 443
 444         kern_return_t kr = pet_threads_prepare(task);
 445         if (kr != KERN_SUCCESS) {
 446                 BUF_INFO(PERF_PET_ERROR, ERR_THREAD, kr);
 447                 BUF_VERB(PERF_PET_SAMPLE_TASK | DBG_FUNC_END, 1);
 448                 return;
 449         }
 450
 451         int pid = task_pid(task);
 452
 453         for (unsigned int i = 0; i < pet_threads_count; i++) {
 454                 thread_t thread = pet_threads[i];
 455                 int cpu;
 456                 assert(thread);
 457
 458                 /* do not sample the thread if it was on a CPU during the IPI. */
 459                 for (cpu = 0; cpu < machine_info.logical_cpu_max; cpu++) {
 460                         if (kperf_tid_on_cpus[cpu] == thread_tid(thread)) {
 461                                 break;
 462                         }
 463                 }
 464
 465                 /* the thread was not on a CPU */
 466                 if (cpu == machine_info.logical_cpu_max) {
 467                         pet_sample_thread(pid, thread, idle_rate);
 468                 }
 469
 470                 thread_deallocate(pet_threads[i]);
 471         }
 472
 473         BUF_VERB(PERF_PET_SAMPLE_TASK | DBG_FUNC_END, pet_threads_count);
 474 }
 475
 476 static kern_return_t
 477 pet_tasks_prepare_internal(void)
 478 {
 479         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 480
 481         vm_size_t tasks_size_needed = 0;
 482
 483         for (;;) {
 484                 lck_mtx_lock(&tasks_threads_lock);
 485
 486                 /* do we have the memory we need? */
 487                 tasks_size_needed = tasks_count * sizeof(task_t);
 488                 if (tasks_size_needed <= pet_tasks_size) {
 489                         break;
 490                 }
 491
 492                 /* unlock and allocate more memory */
 493                 lck_mtx_unlock(&tasks_threads_lock);
 494
 495                 /* grow task array */
 496                 if (tasks_size_needed > pet_tasks_size) {
 497                         if (pet_tasks_size != 0) {
 498                                 kfree(pet_tasks, pet_tasks_size);
 499                         }
 500
 501                         assert(tasks_size_needed > 0);
 502                         pet_tasks_size = tasks_size_needed;
 503
 504                         pet_tasks = (task_array_t)kalloc(pet_tasks_size);
 505                         if (pet_tasks == NULL) {
 506                                 pet_tasks_size = 0;
 507                                 return KERN_RESOURCE_SHORTAGE;
 508                         }
 509                 }
 510         }
 511
 512         return KERN_SUCCESS;
 513 }
 514
 515 static kern_return_t
 516 pet_tasks_prepare(void)
 517 {
 518         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 519
 520         /* allocate space and take the tasks_threads_lock */
 521         kern_return_t kr = pet_tasks_prepare_internal();
 522         if (KERN_SUCCESS != kr) {
 523                 return kr;
 524         }
 525         lck_mtx_assert(&tasks_threads_lock, LCK_MTX_ASSERT_OWNED);
 526
 527         /* make sure the tasks are not deallocated after dropping the lock */
 528         task_t task;
 529         pet_tasks_count = 0;
 530         queue_iterate(&tasks, task, task_t, tasks) {
 531                 if (task != kernel_task) {
 532                         task_reference_internal(task);
 533                         pet_tasks[pet_tasks_count++] = task;
 534                 }
 535         }
 536
 537         lck_mtx_unlock(&tasks_threads_lock);
 538
 539         return KERN_SUCCESS;
 540 }
 541
 542 static void
 543 pet_sample_all_tasks(uint32_t idle_rate)
 544 {
 545         lck_mtx_assert(pet_lock, LCK_MTX_ASSERT_OWNED);
 546
 547         BUF_INFO(PERF_PET_SAMPLE | DBG_FUNC_START);
 548
 549         kern_return_t kr = pet_tasks_prepare();
 550         if (kr != KERN_SUCCESS) {
 551                 BUF_INFO(PERF_PET_ERROR, ERR_TASK, kr);
 552                 BUF_INFO(PERF_PET_SAMPLE | DBG_FUNC_END, 0);
 553                 return;
 554         }
 555
 556         for (unsigned int i = 0; i < pet_tasks_count; i++) {
 557                 task_t task = pet_tasks[i];
 558
 559                 if (task != kernel_task) {
 560                         kr = task_suspend_internal(task);
 561                         if (kr != KERN_SUCCESS) {
 562                                 continue;
 563                         }
 564                 }
 565
 566                 pet_sample_task(task, idle_rate);
 567
 568                 if (task != kernel_task) {
 569                         task_resume_internal(task);
 570                 }
 571         }
 572
 573         for(unsigned int i = 0; i < pet_tasks_count; i++) {
 574                 task_deallocate(pet_tasks[i]);
 575         }
 576
 577         BUF_INFO(PERF_PET_SAMPLE | DBG_FUNC_END, pet_tasks_count);
 578 }
 579
 580 /* support sysctls */
 581
 582 int
 583 kperf_get_pet_idle_rate(void)
 584 {
 585         return pet_idle_rate;
 586 }
 587
 588 int
 589 kperf_set_pet_idle_rate(int val)
 590 {
 591         pet_idle_rate = val;
 592
 593         return 0;
 594 }
 595
 596 int
 597 kperf_get_lightweight_pet(void)
 598 {
 599         return lightweight_pet;
 600 }
 601
 602 int
 603 kperf_set_lightweight_pet(int val)
 604 {
 605         if (kperf_sampling_status() == KPERF_SAMPLING_ON) {
 606                 return EBUSY;
 607         }
 608
 609         lightweight_pet = (val == 1);
 610         kperf_lightweight_pet_active_update();
 611
 612         return 0;
 613 }
 614
 615 void
 616 kperf_lightweight_pet_active_update(void)
 617 {
 618         kperf_lightweight_pet_active = (kperf_sampling_status() && lightweight_pet);
 619         kperf_on_cpu_update();
 620 }