* Copyright (c) 2012-2013 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
+ *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
- *
+ *
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
+ *
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#include <mach/host_priv.h>
#include <kern/debug.h>
#include <kern/host.h>
#include <kern/kalloc.h>
-#include <kern/kern_types.h>
-#include <kern/locks.h>
-#include <kern/misc_protos.h>
+#include <kern/kern_types.h>
+#include <kern/locks.h>
+#include <kern/misc_protos.h>
#include <kern/sched.h>
#include <kern/sched_prim.h>
#include <kern/telemetry.h>
#include <kern/timer_call.h>
+#include <kern/policy_internal.h>
+#include <kern/kcdata.h>
#include <pexpert/pexpert.h>
#include <vm/vm_kern.h>
#include <vm/vm_shared_region.h>
-#include <kperf/kperf.h>
-#include <kperf/context.h>
#include <kperf/callstack.h>
+#include <kern/backtrace.h>
+#include <kern/monotonic.h>
#include <sys/kdebug.h>
#include <uuid/uuid.h>
extern uint64_t proc_uniqueid(void *p);
extern uint64_t proc_was_throttled(void *p);
extern uint64_t proc_did_throttle(void *p);
-extern uint64_t get_dispatchqueue_serialno_offset_from_proc(void *p);
extern int proc_selfpid(void);
+extern boolean_t task_did_exec(task_t task);
+extern boolean_t task_is_exec_copy(task_t task);
struct micro_snapshot_buffer {
vm_offset_t buffer;
int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer);
#define TELEMETRY_DEFAULT_SAMPLE_RATE (1) /* 1 sample every 1 second */
-#define TELEMETRY_DEFAULT_WINDOW_BUFFER_SIZE (512*1024) /* Should hopefully provide 10 seconds worth of samples */
#define TELEMETRY_DEFAULT_BUFFER_SIZE (16*1024)
#define TELEMETRY_MAX_BUFFER_SIZE (64*1024)
uint32_t telemetry_sample_rate = 0;
volatile boolean_t telemetry_needs_record = FALSE;
-volatile boolean_t telemetry_windowed_record = FALSE;
volatile boolean_t telemetry_needs_timer_arming_record = FALSE;
-/*
- * Tells the scheduler that we want it to invoke
- * compute_telemetry_windowed(); it is still our responsibility
- * to ensure that we do not panic if someone disables the window
- * buffer immediately after the scheduler does so.
- */
-volatile boolean_t telemetry_window_enabled = FALSE;
-
/*
* If TRUE, record micro-stackshot samples for all tasks.
* If FALSE, only sample tasks which are marked for telemetry.
*/
-boolean_t telemetry_sample_all_tasks = FALSE;
-uint32_t telemetry_active_tasks = 0; // Number of tasks opted into telemetry
+boolean_t telemetry_sample_all_tasks = FALSE;
+boolean_t telemetry_sample_pmis = FALSE;
+uint32_t telemetry_active_tasks = 0; // Number of tasks opted into telemetry
-uint32_t telemetry_timestamp = 0;
+uint32_t telemetry_timestamp = 0;
/*
- * We have two buffers. The telemetry_buffer is responsible
+ * The telemetry_buffer is responsible
* for timer samples and interrupt samples that are driven by
* compute_averages(). It will notify its client (if one
* exists) when it has enough data to be worth flushing.
- *
- * The window_buffer contains only interrupt_samples that are
- * driven by the scheduler. Its intent is to provide a
- * window of recent activity on the cpu(s).
*/
struct micro_snapshot_buffer telemetry_buffer = {0, 0, 0, 0};
-struct micro_snapshot_buffer window_buffer = {0, 0, 0, 0};
int telemetry_bytes_since_last_mark = -1; // How much data since buf was last marked?
int telemetry_buffer_notify_at = 0;
-lck_grp_t telemetry_lck_grp;
-lck_mtx_t telemetry_mtx;
+lck_grp_t telemetry_lck_grp;
+lck_mtx_t telemetry_mtx;
+lck_mtx_t telemetry_pmi_mtx;
-#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while(0)
+#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while (0)
#define TELEMETRY_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&telemetry_mtx)
-#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while(0)
+#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while (0)
+
+#define TELEMETRY_PMI_LOCK() do { lck_mtx_lock(&telemetry_pmi_mtx); } while (0)
+#define TELEMETRY_PMI_UNLOCK() do { lck_mtx_unlock(&telemetry_pmi_mtx); } while (0)
void telemetry_init(void)
{
lck_grp_init(&telemetry_lck_grp, "telemetry group", LCK_GRP_ATTR_NULL);
lck_mtx_init(&telemetry_mtx, &telemetry_lck_grp, LCK_ATTR_NULL);
+ lck_mtx_init(&telemetry_pmi_mtx, &telemetry_lck_grp, LCK_ATTR_NULL);
if (!PE_parse_boot_argn("telemetry_buffer_size", &telemetry_buffer.size, sizeof(telemetry_buffer.size))) {
telemetry_buffer.size = TELEMETRY_DEFAULT_BUFFER_SIZE;
if (telemetry_buffer.size > TELEMETRY_MAX_BUFFER_SIZE)
telemetry_buffer.size = TELEMETRY_MAX_BUFFER_SIZE;
- ret = kmem_alloc(kernel_map, &telemetry_buffer.buffer, telemetry_buffer.size);
+ ret = kmem_alloc(kernel_map, &telemetry_buffer.buffer, telemetry_buffer.size, VM_KERN_MEMORY_DIAG);
if (ret != KERN_SUCCESS) {
kprintf("Telemetry: Allocation failed: %d\n", ret);
return;
*/
if (!PE_parse_boot_argn("telemetry_sample_all_tasks", &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) {
+#if CONFIG_EMBEDDED && !(DEVELOPMENT || DEBUG)
+ telemetry_sample_all_tasks = FALSE;
+#else
telemetry_sample_all_tasks = TRUE;
+#endif /* CONFIG_EMBEDDED && !(DEVELOPMENT || DEBUG) */
}
* enable_disable == 0: turn it off
*/
void
-telemetry_global_ctl(int enable_disable)
+telemetry_global_ctl(int enable_disable)
{
if (enable_disable == 1) {
telemetry_sample_all_tasks = TRUE;
task->t_flags |= reasons;
if ((origflags & TF_TELEMETRY) == 0) {
OSIncrementAtomic(&telemetry_active_tasks);
-#if TELEMETRY_DEBUG
+#if TELEMETRY_DEBUG
printf("%s: telemetry OFF -> ON (%d active)\n", proc_name_address(task->bsd_info), telemetry_active_tasks);
-#endif
+#endif
}
} else {
task->t_flags &= ~reasons;
}
}
-/*
- * Enable the window_buffer, and do any associated setup.
- */
-kern_return_t
-telemetry_enable_window(void)
-{
- kern_return_t ret = KERN_SUCCESS;
- vm_offset_t kern_buffer = 0;
- vm_size_t kern_buffer_size = TELEMETRY_DEFAULT_WINDOW_BUFFER_SIZE;
-
- /*
- * We have no guarantee we won't allocate the buffer, take
- * the lock, and then discover someone beat us to the punch,
- * but we would prefer to avoid blocking while holding the
- * lock.
- */
- ret = kmem_alloc(kernel_map, &kern_buffer, kern_buffer_size);
-
- TELEMETRY_LOCK();
-
- if (!window_buffer.buffer) {
- if (ret == KERN_SUCCESS) {
- /* No existing buffer was found, so... */
- window_buffer.end_point = 0;
- window_buffer.current_position = 0;
-
- /* Hand off the buffer, and... */
- window_buffer.size = (uint32_t) kern_buffer_size;
- window_buffer.buffer = kern_buffer;
- kern_buffer = 0;
- kern_buffer_size = 0;
- bzero((void *) window_buffer.buffer, window_buffer.size);
-
- /* Let the scheduler know it should drive windowed samples */
- telemetry_window_enabled = TRUE;
- }
- } else {
- /* We already have a buffer, so we have "succeeded" */
- ret = KERN_SUCCESS;
- }
-
- TELEMETRY_UNLOCK();
-
- if (kern_buffer)
- kmem_free(kernel_map, kern_buffer, kern_buffer_size);
-
- return ret;
-}
-
-/*
- * Disable the window_buffer, and do any associated teardown.
- */
-void
-telemetry_disable_window(void)
-{
- vm_offset_t kern_buffer = 0;
- vm_size_t kern_buffer_size = 0;
-
- TELEMETRY_LOCK();
-
- if (window_buffer.buffer) {
- /* We have a window buffer, so tear it down */
- telemetry_window_enabled = FALSE;
- kern_buffer = window_buffer.buffer;
- kern_buffer_size = window_buffer.size;
- window_buffer.buffer = 0;
- window_buffer.size = 0;
- window_buffer.current_position = 0;
- window_buffer.end_point = 0;
- }
-
- TELEMETRY_UNLOCK();
-
- if (kern_buffer)
- kmem_free(kernel_map, kern_buffer, kern_buffer_size);
-}
-
/*
* Determine if the current thread is eligible for telemetry:
*
static boolean_t
telemetry_is_active(thread_t thread)
{
- if (telemetry_sample_all_tasks == TRUE) {
- return (TRUE);
+ task_t task = thread->task;
+
+ if (task == kernel_task) {
+ /* Kernel threads never return to an AST boundary, and are ineligible */
+ return FALSE;
+ }
+
+ if (telemetry_sample_all_tasks || telemetry_sample_pmis) {
+ return TRUE;
}
if ((telemetry_active_tasks > 0) && ((thread->task->t_flags & TF_TELEMETRY) != 0)) {
- return (TRUE);
+ return TRUE;
}
-
- return (FALSE);
+
+ return FALSE;
}
/*
return (0);
}
+#if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES)
+static void
+telemetry_pmi_handler(bool user_mode, __unused void *ctx)
+{
+ telemetry_mark_curthread(user_mode, TRUE);
+}
+#endif /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */
+
+int telemetry_pmi_setup(enum telemetry_pmi pmi_ctr, uint64_t period)
+{
+#if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES)
+ static boolean_t sample_all_tasks_aside = FALSE;
+ static uint32_t active_tasks_aside = FALSE;
+ int error = 0;
+ const char *name = "?";
+
+ unsigned int ctr = 0;
+
+ TELEMETRY_PMI_LOCK();
+
+ switch (pmi_ctr) {
+ case TELEMETRY_PMI_NONE:
+ if (!telemetry_sample_pmis) {
+ error = 1;
+ goto out;
+ }
+
+ telemetry_sample_pmis = FALSE;
+ telemetry_sample_all_tasks = sample_all_tasks_aside;
+ telemetry_active_tasks = active_tasks_aside;
+ error = mt_microstackshot_stop();
+ if (!error) {
+ printf("telemetry: disabling ustackshot on PMI\n");
+ }
+ goto out;
+
+ case TELEMETRY_PMI_INSTRS:
+ ctr = MT_CORE_INSTRS;
+ name = "instructions";
+ break;
+
+ case TELEMETRY_PMI_CYCLES:
+ ctr = MT_CORE_CYCLES;
+ name = "cycles";
+ break;
+
+ default:
+ error = 1;
+ goto out;
+ }
+
+ telemetry_sample_pmis = TRUE;
+ sample_all_tasks_aside = telemetry_sample_all_tasks;
+ active_tasks_aside = telemetry_active_tasks;
+ telemetry_sample_all_tasks = FALSE;
+ telemetry_active_tasks = 0;
+
+ error = mt_microstackshot_start(ctr, period, telemetry_pmi_handler, NULL);
+ if (!error) {
+ printf("telemetry: ustackshot every %llu %s\n", period, name);
+ }
+
+out:
+ TELEMETRY_PMI_UNLOCK();
+ return error;
+#else /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */
+#pragma unused(pmi_ctr, period)
+ return 1;
+#endif /* !defined(MT_CORE_INSTRS) || !defined(MT_CORE_CYCLES) */
+}
+
/*
* Mark the current thread for an interrupt-based
* telemetry record, to be sampled at the next AST boundary.
*/
-void telemetry_mark_curthread(boolean_t interrupted_userspace)
+void telemetry_mark_curthread(boolean_t interrupted_userspace, boolean_t pmi)
{
uint32_t ast_bits = 0;
thread_t thread = current_thread();
}
ast_bits |= (interrupted_userspace ? AST_TELEMETRY_USER : AST_TELEMETRY_KERNEL);
-
- if (telemetry_windowed_record) {
- ast_bits |= AST_TELEMETRY_WINDOWED;
+ if (pmi) {
+ ast_bits |= AST_TELEMETRY_PMI;
}
- telemetry_windowed_record = FALSE;
telemetry_needs_record = FALSE;
thread_ast_set(thread, ast_bits);
- ast_propagate(thread->ast);
+ ast_propagate(thread);
}
void compute_telemetry(void *arg __unused)
}
}
-void compute_telemetry_windowed(void)
-{
- if (telemetry_sample_all_tasks || (telemetry_active_tasks > 0)) {
- /*
- * Due to the relationship between the two fields here,
- * a request for a windowed record will "squash" a
- * request for a regular interrupt record. We hedge
- * against this by doing a quick check for an existing
- * request. compute_telemetry doesn't hedge because
- * a regular request cannot squash a windowed request
- * (due to the implementation).
- *
- * If we really want to do this properly, we could make
- * telemetry_needs_record a bitfield, and process one
- * request per telemetry_mark_curthread... but that
- * would be more expensive (atomics). This should be
- * robust enough for now (although it biases in favor
- * of the regular records).
- */
- if (!telemetry_needs_record) {
- telemetry_needs_record = TRUE;
- telemetry_windowed_record = TRUE;
- }
- }
-}
-
/*
* If userland has registered a port for telemetry notifications, send one now.
*/
static void
telemetry_notify_user(void)
{
- mach_port_t user_port;
- uint32_t flags = 0;
- int error;
+ mach_port_t user_port = MACH_PORT_NULL;
- error = host_get_telemetry_port(host_priv_self(), &user_port);
- if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
+ kern_return_t kr = host_get_telemetry_port(host_priv_self(), &user_port);
+ if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
return;
}
- telemetry_notification(user_port, flags);
+ telemetry_notification(user_port, 0);
+ ipc_port_release_send(user_port);
}
-void telemetry_ast(thread_t thread, boolean_t interrupted_userspace, boolean_t is_windowed)
+void telemetry_ast(thread_t thread, ast_t reasons)
{
- uint8_t microsnapshot_flags = kInterruptRecord;
-
- if (interrupted_userspace)
- microsnapshot_flags |= kUserMode;
+ assert((reasons & AST_TELEMETRY_ALL) != 0);
- if (is_windowed) {
- telemetry_take_sample(thread, microsnapshot_flags, &window_buffer);
- } else {
- telemetry_take_sample(thread, microsnapshot_flags, &telemetry_buffer);
+ uint8_t record_type = 0;
+ if (reasons & AST_TELEMETRY_IO) {
+ record_type |= kIORecord;
+ }
+ if (reasons & (AST_TELEMETRY_USER | AST_TELEMETRY_KERNEL)) {
+ record_type |= (reasons & AST_TELEMETRY_PMI) ? kPMIRecord :
+ kInterruptRecord;
}
+
+ uint8_t user_telemetry = (reasons & AST_TELEMETRY_USER) ? kUserMode : 0;
+
+ uint8_t microsnapshot_flags = record_type | user_telemetry;
+
+ telemetry_take_sample(thread, microsnapshot_flags, &telemetry_buffer);
}
void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer)
{
task_t task;
void *p;
- struct kperf_context ctx;
- struct callstack cs;
- uint32_t btcount, bti;
+ uint32_t btcount = 0, bti;
struct micro_snapshot *msnap;
struct task_snapshot *tsnap;
struct thread_snapshot *thsnap;
return;
task = thread->task;
- if ((task == TASK_NULL) || (task == kernel_task))
+ if ((task == TASK_NULL) || (task == kernel_task) || task_did_exec(task) || task_is_exec_copy(task))
return;
- /*
- * To avoid overloading the system with telemetry requests, make
- * sure we don't add more requests while existing ones are
- * in-flight. Attempt this by checking if we can grab the lock.
- *
- * This concerns me a little; this working as intended is
- * contingent on the workload being done in the context of the
- * telemetry lock being the expensive part of telemetry. This
- * includes populating the buffer and the client gathering it,
- * but excludes the copyin overhead.
- */
- if (!TELEMETRY_TRY_SPIN_LOCK())
- return;
-
- TELEMETRY_UNLOCK();
-
/* telemetry_XXX accessed outside of lock for instrumentation only */
- /* TODO */
- KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START, microsnapshot_flags, telemetry_bytes_since_last_mark, 0, 0, (&telemetry_buffer != current_buffer));
+ KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START,
+ microsnapshot_flags, telemetry_bytes_since_last_mark, 0,
+ (&telemetry_buffer != current_buffer));
p = get_bsdtask_info(task);
- ctx.cur_thread = thread;
- ctx.cur_pid = proc_pid(p);
-
/*
* Gather up the data we'll need for this sample. The sample is written into the kernel
* buffer with the global telemetry lock held -- so we must do our (possibly faulting)
* copies from userland here, before taking the lock.
*/
- kperf_ucallstack_sample(&cs, &ctx);
- if (!(cs.flags & CALLSTACK_VALID))
+ uintptr_t frames[MAX_CALLSTACK_FRAMES] = {};
+ bool user64;
+ int backtrace_error = backtrace_user(frames, MAX_CALLSTACK_FRAMES, &btcount, &user64);
+ if (backtrace_error) {
return;
+ }
/*
* Find the actual [slid] address of the shared cache's UUID, and copy it in from userland.
*/
uint32_t uuid_info_count = 0;
mach_vm_address_t uuid_info_addr = 0;
- if (task_has_64BitAddr(task)) {
+ if (task_has_64Bit_addr(task)) {
struct user64_dyld_all_image_infos task_image_infos;
if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) {
uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
uuid_info_count = TELEMETRY_MAX_UUID_COUNT;
}
- uint32_t uuid_info_size = (uint32_t)(task_has_64BitAddr(thread->task) ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
+ uint32_t uuid_info_size = (uint32_t)(task_has_64Bit_addr(thread->task) ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
char *uuid_info_array = NULL;
uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
if (dqkeyaddr != 0) {
uint64_t dqaddr = 0;
- uint64_t dq_serialno_offset = get_dispatchqueue_serialno_offset_from_proc(task->bsd_info);
- if ((copyin(dqkeyaddr, (char *)&dqaddr, (task_has_64BitAddr(task) ? 8 : 4)) == 0) &&
+ uint64_t dq_serialno_offset = get_task_dispatchqueue_serialno_offset(task);
+ if ((copyin(dqkeyaddr, (char *)&dqaddr, (task_has_64Bit_addr(task) ? 8 : 4)) == 0) &&
(dqaddr != 0) && (dq_serialno_offset != 0)) {
uint64_t dqserialnumaddr = dqaddr + dq_serialno_offset;
- if (copyin(dqserialnumaddr, (char *)&dqserialnum, (task_has_64BitAddr(task) ? 8 : 4)) == 0) {
+ if (copyin(dqserialnumaddr, (char *)&dqserialnum, (task_has_64Bit_addr(task) ? 8 : 4)) == 0) {
dqserialnum_valid = 1;
}
}
TELEMETRY_LOCK();
/*
- * For the benefit of the window buffer; if our buffer is not backed by anything,
+ * If our buffer is not backed by anything,
* then we cannot take the sample. Meant to allow us to deallocate the window
* buffer if it is disabled.
*/
msnap->snapshot_magic = STACKSHOT_MICRO_SNAPSHOT_MAGIC;
msnap->ms_flags = microsnapshot_flags;
msnap->ms_opaque_flags = 0; /* namespace managed by userspace */
- msnap->ms_cpu = 0; /* XXX - does this field make sense for a micro-stackshot? */
+ msnap->ms_cpu = cpu_number();
msnap->ms_time = secs;
msnap->ms_time_microsecs = usecs;
tsnap->user_time_in_terminated_threads = task->total_user_time;
tsnap->system_time_in_terminated_threads = task->total_system_time;
tsnap->suspend_count = task->suspend_count;
- tsnap->task_size = pmap_resident_count(task->map->pmap);
+ tsnap->task_size = (typeof(tsnap->task_size)) (get_task_phys_footprint(task) / PAGE_SIZE);
tsnap->faults = task->faults;
tsnap->pageins = task->pageins;
tsnap->cow_faults = task->cow_faults;
* The throttling counters are maintained as 64-bit counters in the proc
* structure. However, we reserve 32-bits (each) for them in the task_snapshot
* struct to save space and since we do not expect them to overflow 32-bits. If we
- * find these values overflowing in the future, the fix would be to simply
+ * find these values overflowing in the future, the fix would be to simply
* upgrade these counters to 64-bit in the task_snapshot struct
*/
tsnap->was_throttled = (uint32_t) proc_was_throttled(p);
tsnap->did_throttle = (uint32_t) proc_did_throttle(p);
-
+
if (task->t_flags & TF_TELEMETRY) {
tsnap->ss_flags |= kTaskRsrcFlagged;
}
- if (task->effective_policy.darwinbg == 1) {
+ if (proc_get_effective_task_policy(task, TASK_POLICY_DARWIN_BG)) {
tsnap->ss_flags |= kTaskDarwinBG;
}
proc_get_darwinbgstate(task, &tmp);
- if (task->requested_policy.t_role == TASK_FOREGROUND_APPLICATION) {
+ if (proc_get_effective_task_policy(task, TASK_POLICY_ROLE) == TASK_FOREGROUND_APPLICATION) {
tsnap->ss_flags |= kTaskIsForeground;
}
tsnap->latency_qos = task_grab_latency_qos(task);
strlcpy(tsnap->p_comm, proc_name_address(p), sizeof(tsnap->p_comm));
- if (task_has_64BitAddr(thread->task)) {
+ if (task_has_64Bit_addr(thread->task)) {
tsnap->ss_flags |= kUser64_p;
}
if ((current_buffer->size - current_buffer->current_position) < sizeof(struct thread_snapshot)) {
/* wrap and overwrite */
- current_buffer->end_point = current_record_start;
+ current_buffer->end_point = current_record_start;
current_buffer->current_position = 0;
if (current_record_start == 0) {
/* This sample is too large to fit in the buffer even when we started at 0, so skip it */
thsnap->snapshot_magic = STACKSHOT_THREAD_SNAPSHOT_MAGIC;
thsnap->thread_id = thread_tid(thread);
thsnap->state = thread->state;
- thsnap->priority = thread->priority;
+ thsnap->priority = thread->base_pri;
thsnap->sched_pri = thread->sched_pri;
thsnap->sched_flags = thread->sched_flags;
thsnap->ss_flags |= kStacksPCOnly;
thsnap->ts_qos = thread->effective_policy.thep_qos;
+ thsnap->ts_rqos = thread->requested_policy.thrp_qos;
+ thsnap->ts_rqos_override = MAX(thread->requested_policy.thrp_qos_override,
+ thread->requested_policy.thrp_qos_workq_override);
- if (thread->effective_policy.darwinbg) {
+ if (proc_get_effective_thread_policy(thread, TASK_POLICY_DARWIN_BG)) {
thsnap->ss_flags |= kThreadDarwinBG;
}
if (dqserialnum_valid) {
if ((current_buffer->size - current_buffer->current_position) < sizeof(dqserialnum)) {
/* wrap and overwrite */
- current_buffer->end_point = current_record_start;
+ current_buffer->end_point = current_record_start;
current_buffer->current_position = 0;
if (current_record_start == 0) {
/* This sample is too large to fit in the buffer even when we started at 0, so skip it */
current_buffer->current_position += sizeof (dqserialnum);
}
- if (task_has_64BitAddr(task)) {
+ if (user64) {
framesize = 8;
thsnap->ss_flags |= kUser64_p;
} else {
framesize = 4;
}
- btcount = cs.nframes;
-
/*
* If we can't fit this entire stacktrace then cancel this record, wrap to the beginning,
* and start again there so that we always store a full record.
for (bti=0; bti < btcount; bti++, current_buffer->current_position += framesize) {
if (framesize == 8) {
- *(uint64_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = cs.frames[bti];
+ *(uint64_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = frames[bti];
} else {
- *(uint32_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = (uint32_t)cs.frames[bti];
+ *(uint32_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = (uint32_t)frames[bti];
}
}
}
cancel_sample:
-
TELEMETRY_UNLOCK();
- /* TODO */
- KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END, notify, telemetry_bytes_since_last_mark, current_buffer->current_position, current_buffer->end_point, (&telemetry_buffer != current_buffer));
+ KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END,
+ notify, telemetry_bytes_since_last_mark,
+ current_buffer->current_position, current_buffer->end_point);
if (notify) {
telemetry_notify_user();
{
struct micro_snapshot *p;
uint32_t offset;
-
+
printf("Copying out %d bytes of telemetry at offset %d\n", sz, pos);
buf += pos;
return telemetry_buffer_gather(buffer, length, mark, &telemetry_buffer);
}
-int telemetry_gather_windowed(user_addr_t buffer, uint32_t *length)
-{
- return telemetry_buffer_gather(buffer, length, 0, &window_buffer);
-}
-
int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer)
{
int result = 0;
uint32_t oldest_record_offset;
- /* TODO */
- KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START, mark, telemetry_bytes_since_last_mark, 0, 0, (&telemetry_buffer != current_buffer));
+ KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START,
+ mark, telemetry_bytes_since_last_mark, 0,
+ (&telemetry_buffer != current_buffer));
TELEMETRY_LOCK();
if (current_buffer->buffer == 0) {
- *length = 0;
+ *length = 0;
goto out;
}
TELEMETRY_UNLOCK();
- KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END, current_buffer->current_position, *length, current_buffer->end_point, 0, (&telemetry_buffer != current_buffer));
+ KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END,
+ current_buffer->current_position, *length,
+ current_buffer->end_point, (&telemetry_buffer != current_buffer));
return (result);
}
uint32_t bootprofile_buffer_size = 0;
uint32_t bootprofile_buffer_current_position = 0;
uint32_t bootprofile_interval_ms = 0;
+uint32_t bootprofile_stackshot_flags = 0;
uint64_t bootprofile_interval_abs = 0;
uint64_t bootprofile_next_deadline = 0;
uint32_t bootprofile_all_procs = 0;
char bootprofile_proc_name[17];
-
+uint64_t bootprofile_delta_since_timestamp = 0;
lck_grp_t bootprofile_lck_grp;
lck_mtx_t bootprofile_mtx;
+
enum {
kBootProfileDisabled = 0,
kBootProfileStartTimerAtBoot,
timer_call_param_t param0,
timer_call_param_t param1);
-extern int
-stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, unsigned *retbytes);
-
void bootprofile_init(void)
{
kern_return_t ret;
bootprofile_interval_ms = 0;
}
+ if (!PE_parse_boot_argn("bootprofile_stackshot_flags", &bootprofile_stackshot_flags, sizeof(bootprofile_stackshot_flags))) {
+ bootprofile_stackshot_flags = 0;
+ }
+
if (!PE_parse_boot_argn("bootprofile_proc_name", &bootprofile_proc_name, sizeof(bootprofile_proc_name))) {
bootprofile_all_procs = 1;
bootprofile_proc_name[0] = '\0';
if (0 == strcmp(type, "boot")) {
bootprofile_type = kBootProfileStartTimerAtBoot;
} else if (0 == strcmp(type, "wake")) {
- bootprofile_type = kBootProfileStartTimerAtWake;
+ bootprofile_type = kBootProfileStartTimerAtWake;
} else {
bootprofile_type = kBootProfileDisabled;
}
return;
}
- ret = kmem_alloc(kernel_map, &bootprofile_buffer, bootprofile_buffer_size);
+ ret = kmem_alloc(kernel_map, &bootprofile_buffer, bootprofile_buffer_size, VM_KERN_MEMORY_DIAG);
if (ret != KERN_SUCCESS) {
kprintf("Boot profile: Allocation failed: %d\n", ret);
return;
}
-static void bootprofile_timer_call(
+static void
+bootprofile_timer_call(
timer_call_param_t param0 __unused,
timer_call_param_t param1 __unused)
{
/* initiate a stackshot with whatever portion of the buffer is left */
if (bootprofile_buffer_current_position < bootprofile_buffer_size) {
- stack_snapshot_from_kernel(
- pid_to_profile,
- (void *)(bootprofile_buffer + bootprofile_buffer_current_position),
- bootprofile_buffer_size - bootprofile_buffer_current_position,
- STACKSHOT_SAVE_LOADINFO | STACKSHOT_SAVE_KEXT_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS,
- &retbytes
- );
+
+ uint32_t flags = STACKSHOT_KCDATA_FORMAT | STACKSHOT_TRYLOCK | STACKSHOT_SAVE_LOADINFO
+ | STACKSHOT_GET_GLOBAL_MEM_STATS;
+#if __x86_64__
+ flags |= STACKSHOT_SAVE_KEXT_LOADINFO;
+#endif /* __x86_64__ */
+
+
+ /* OR on flags specified in boot-args */
+ flags |= bootprofile_stackshot_flags;
+ if ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) && (bootprofile_delta_since_timestamp == 0)) {
+ /* Can't take deltas until the first one */
+ flags &= ~ STACKSHOT_COLLECT_DELTA_SNAPSHOT;
+ }
+
+ uint64_t timestamp = 0;
+ if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) {
+ timestamp = mach_absolute_time();
+ }
+
+ kern_return_t r = stack_snapshot_from_kernel(
+ pid_to_profile, (void *)(bootprofile_buffer + bootprofile_buffer_current_position),
+ bootprofile_buffer_size - bootprofile_buffer_current_position,
+ flags, bootprofile_delta_since_timestamp, &retbytes);
+
+ /*
+ * We call with STACKSHOT_TRYLOCK because the stackshot lock is coarser
+ * than the bootprofile lock. If someone else has the lock we'll just
+ * try again later.
+ */
+
+ if (r == KERN_LOCK_OWNED) {
+ BOOTPROFILE_UNLOCK();
+ goto reprogram;
+ }
+
+ if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT &&
+ r == KERN_SUCCESS) {
+ bootprofile_delta_since_timestamp = timestamp;
+ }
bootprofile_buffer_current_position += retbytes;
}
FALSE);
}
+void bootprofile_get(void **buffer, uint32_t *length)
+{
+ BOOTPROFILE_LOCK();
+ *buffer = (void*) bootprofile_buffer;
+ *length = bootprofile_buffer_current_position;
+ BOOTPROFILE_UNLOCK();
+}
+
int bootprofile_gather(user_addr_t buffer, uint32_t *length)
{
int result = 0;
BOOTPROFILE_LOCK();
if (bootprofile_buffer == 0) {
- *length = 0;
+ *length = 0;
goto out;
}
projects / apple / xnu.git / blobdiff