X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/osfmk/kern/telemetry.c diff --git a/osfmk/kern/telemetry.c b/osfmk/kern/telemetry.c index cda0bebc6..b777052d4 100644 --- a/osfmk/kern/telemetry.c +++ b/osfmk/kern/telemetry.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include @@ -35,22 +35,24 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include #include +#include +#include #include #include #include -#include -#include #include +#include +#include #include #include @@ -58,91 +60,88 @@ #define TELEMETRY_DEBUG 0 -extern int proc_pid(void *); -extern char *proc_name_address(void *p); +struct proc; +extern int proc_pid(struct proc *); +extern char *proc_name_address(void *p); extern uint64_t proc_uniqueid(void *p); extern uint64_t proc_was_throttled(void *p); extern uint64_t proc_did_throttle(void *p); -extern uint64_t get_dispatchqueue_serialno_offset_from_proc(void *p); -extern int proc_selfpid(void); +extern int proc_selfpid(void); +extern boolean_t task_did_exec(task_t task); +extern boolean_t task_is_exec_copy(task_t task); struct micro_snapshot_buffer { - vm_offset_t buffer; - uint32_t size; - uint32_t current_position; - uint32_t end_point; + vm_offset_t buffer; + uint32_t size; + uint32_t current_position; + uint32_t end_point; }; void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer); int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer); #define TELEMETRY_DEFAULT_SAMPLE_RATE (1) /* 1 sample every 1 second */ -#define TELEMETRY_DEFAULT_WINDOW_BUFFER_SIZE (512*1024) /* Should hopefully provide 10 seconds worth of samples */ #define TELEMETRY_DEFAULT_BUFFER_SIZE (16*1024) #define TELEMETRY_MAX_BUFFER_SIZE (64*1024) -#define TELEMETRY_DEFAULT_NOTIFY_LEEWAY (4*1024) // Userland gets 4k of leeway to collect data after notification -#define TELEMETRY_MAX_UUID_COUNT (128) // Max of 128 non-shared-cache UUIDs to log for symbolication - -uint32_t telemetry_sample_rate = 0; -volatile boolean_t telemetry_needs_record = FALSE; -volatile boolean_t telemetry_windowed_record = FALSE; -volatile boolean_t telemetry_needs_timer_arming_record = FALSE; +#define TELEMETRY_DEFAULT_NOTIFY_LEEWAY (4*1024) // Userland gets 4k of leeway to collect data after notification +#define TELEMETRY_MAX_UUID_COUNT (128) // Max of 128 non-shared-cache UUIDs to log for symbolication -/* - * Tells the scheduler that we want it to invoke - * compute_telemetry_windowed(); it is still our responsibility - * to ensure that we do not panic if someone disables the window - * buffer immediately after the scheduler does so. - */ -volatile boolean_t telemetry_window_enabled = FALSE; +uint32_t telemetry_sample_rate = 0; +volatile boolean_t telemetry_needs_record = FALSE; +volatile boolean_t telemetry_needs_timer_arming_record = FALSE; /* * If TRUE, record micro-stackshot samples for all tasks. * If FALSE, only sample tasks which are marked for telemetry. */ -boolean_t telemetry_sample_all_tasks = FALSE; -uint32_t telemetry_active_tasks = 0; // Number of tasks opted into telemetry +boolean_t telemetry_sample_all_tasks = FALSE; +boolean_t telemetry_sample_pmis = FALSE; +uint32_t telemetry_active_tasks = 0; // Number of tasks opted into telemetry -uint32_t telemetry_timestamp = 0; +uint32_t telemetry_timestamp = 0; /* - * We have two buffers. The telemetry_buffer is responsible + * The telemetry_buffer is responsible * for timer samples and interrupt samples that are driven by * compute_averages(). It will notify its client (if one * exists) when it has enough data to be worth flushing. - * - * The window_buffer contains only interrupt_samples that are - * driven by the scheduler. Its intent is to provide a - * window of recent activity on the cpu(s). */ -struct micro_snapshot_buffer telemetry_buffer = {0, 0, 0, 0}; -struct micro_snapshot_buffer window_buffer = {0, 0, 0, 0}; +struct micro_snapshot_buffer telemetry_buffer = { + .buffer = 0, + .size = 0, + .current_position = 0, + .end_point = 0 +}; -int telemetry_bytes_since_last_mark = -1; // How much data since buf was last marked? -int telemetry_buffer_notify_at = 0; +int telemetry_bytes_since_last_mark = -1; // How much data since buf was last marked? +int telemetry_buffer_notify_at = 0; -lck_grp_t telemetry_lck_grp; -lck_mtx_t telemetry_mtx; +LCK_GRP_DECLARE(telemetry_lck_grp, "telemetry group"); +LCK_MTX_DECLARE(telemetry_mtx, &telemetry_lck_grp); +LCK_MTX_DECLARE(telemetry_pmi_mtx, &telemetry_lck_grp); -#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while(0) +#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while (0) #define TELEMETRY_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&telemetry_mtx) -#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while(0) +#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while (0) + +#define TELEMETRY_PMI_LOCK() do { lck_mtx_lock(&telemetry_pmi_mtx); } while (0) +#define TELEMETRY_PMI_UNLOCK() do { lck_mtx_unlock(&telemetry_pmi_mtx); } while (0) -void telemetry_init(void) +void +telemetry_init(void) { kern_return_t ret; - uint32_t telemetry_notification_leeway; - - lck_grp_init(&telemetry_lck_grp, "telemetry group", LCK_GRP_ATTR_NULL); - lck_mtx_init(&telemetry_mtx, &telemetry_lck_grp, LCK_ATTR_NULL); + uint32_t telemetry_notification_leeway; - if (!PE_parse_boot_argn("telemetry_buffer_size", &telemetry_buffer.size, sizeof(telemetry_buffer.size))) { + if (!PE_parse_boot_argn("telemetry_buffer_size", + &telemetry_buffer.size, sizeof(telemetry_buffer.size))) { telemetry_buffer.size = TELEMETRY_DEFAULT_BUFFER_SIZE; } - if (telemetry_buffer.size > TELEMETRY_MAX_BUFFER_SIZE) + if (telemetry_buffer.size > TELEMETRY_MAX_BUFFER_SIZE) { telemetry_buffer.size = TELEMETRY_MAX_BUFFER_SIZE; + } ret = kmem_alloc(kernel_map, &telemetry_buffer.buffer, telemetry_buffer.size, VM_KERN_MEMORY_DIAG); if (ret != KERN_SUCCESS) { @@ -151,7 +150,8 @@ void telemetry_init(void) } bzero((void *) telemetry_buffer.buffer, telemetry_buffer.size); - if (!PE_parse_boot_argn("telemetry_notification_leeway", &telemetry_notification_leeway, sizeof(telemetry_notification_leeway))) { + if (!PE_parse_boot_argn("telemetry_notification_leeway", + &telemetry_notification_leeway, sizeof(telemetry_notification_leeway))) { /* * By default, notify the user to collect the buffer when there is this much space left in the buffer. */ @@ -159,27 +159,31 @@ void telemetry_init(void) } if (telemetry_notification_leeway >= telemetry_buffer.size) { printf("telemetry: nonsensical telemetry_notification_leeway boot-arg %d changed to %d\n", - telemetry_notification_leeway, TELEMETRY_DEFAULT_NOTIFY_LEEWAY); + telemetry_notification_leeway, TELEMETRY_DEFAULT_NOTIFY_LEEWAY); telemetry_notification_leeway = TELEMETRY_DEFAULT_NOTIFY_LEEWAY; } telemetry_buffer_notify_at = telemetry_buffer.size - telemetry_notification_leeway; - if (!PE_parse_boot_argn("telemetry_sample_rate", &telemetry_sample_rate, sizeof(telemetry_sample_rate))) { + if (!PE_parse_boot_argn("telemetry_sample_rate", + &telemetry_sample_rate, sizeof(telemetry_sample_rate))) { telemetry_sample_rate = TELEMETRY_DEFAULT_SAMPLE_RATE; } /* * To enable telemetry for all tasks, include "telemetry_sample_all_tasks=1" in boot-args. */ - if (!PE_parse_boot_argn("telemetry_sample_all_tasks", &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) { - + if (!PE_parse_boot_argn("telemetry_sample_all_tasks", + &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) { +#if !defined(XNU_TARGET_OS_OSX) && !(DEVELOPMENT || DEBUG) + telemetry_sample_all_tasks = FALSE; +#else telemetry_sample_all_tasks = TRUE; - +#endif /* !defined(XNU_TARGET_OS_OSX) && !(DEVELOPMENT || DEBUG) */ } kprintf("Telemetry: Sampling %stasks once per %u second%s\n", - (telemetry_sample_all_tasks) ? "all " : "", - telemetry_sample_rate, telemetry_sample_rate == 1 ? "" : "s"); + (telemetry_sample_all_tasks) ? "all " : "", + telemetry_sample_rate, telemetry_sample_rate == 1 ? "" : "s"); } /* @@ -189,7 +193,7 @@ void telemetry_init(void) * enable_disable == 0: turn it off */ void -telemetry_global_ctl(int enable_disable) +telemetry_global_ctl(int enable_disable) { if (enable_disable == 1) { telemetry_sample_all_tasks = TRUE; @@ -231,9 +235,9 @@ telemetry_task_ctl_locked(task_t task, uint32_t reasons, int enable_disable) task->t_flags |= reasons; if ((origflags & TF_TELEMETRY) == 0) { OSIncrementAtomic(&telemetry_active_tasks); -#if TELEMETRY_DEBUG +#if TELEMETRY_DEBUG printf("%s: telemetry OFF -> ON (%d active)\n", proc_name_address(task->bsd_info), telemetry_active_tasks); -#endif +#endif } } else { task->t_flags &= ~reasons; @@ -250,83 +254,6 @@ telemetry_task_ctl_locked(task_t task, uint32_t reasons, int enable_disable) } } -/* - * Enable the window_buffer, and do any associated setup. - */ -kern_return_t -telemetry_enable_window(void) -{ - kern_return_t ret = KERN_SUCCESS; - vm_offset_t kern_buffer = 0; - vm_size_t kern_buffer_size = TELEMETRY_DEFAULT_WINDOW_BUFFER_SIZE; - - /* - * We have no guarantee we won't allocate the buffer, take - * the lock, and then discover someone beat us to the punch, - * but we would prefer to avoid blocking while holding the - * lock. - */ - ret = kmem_alloc(kernel_map, &kern_buffer, kern_buffer_size, VM_KERN_MEMORY_DIAG); - - TELEMETRY_LOCK(); - - if (!window_buffer.buffer) { - if (ret == KERN_SUCCESS) { - /* No existing buffer was found, so... */ - window_buffer.end_point = 0; - window_buffer.current_position = 0; - - /* Hand off the buffer, and... */ - window_buffer.size = (uint32_t) kern_buffer_size; - window_buffer.buffer = kern_buffer; - kern_buffer = 0; - kern_buffer_size = 0; - bzero((void *) window_buffer.buffer, window_buffer.size); - - /* Let the scheduler know it should drive windowed samples */ - telemetry_window_enabled = TRUE; - } - } else { - /* We already have a buffer, so we have "succeeded" */ - ret = KERN_SUCCESS; - } - - TELEMETRY_UNLOCK(); - - if (kern_buffer) - kmem_free(kernel_map, kern_buffer, kern_buffer_size); - - return ret; -} - -/* - * Disable the window_buffer, and do any associated teardown. - */ -void -telemetry_disable_window(void) -{ - vm_offset_t kern_buffer = 0; - vm_size_t kern_buffer_size = 0; - - TELEMETRY_LOCK(); - - if (window_buffer.buffer) { - /* We have a window buffer, so tear it down */ - telemetry_window_enabled = FALSE; - kern_buffer = window_buffer.buffer; - kern_buffer_size = window_buffer.size; - window_buffer.buffer = 0; - window_buffer.size = 0; - window_buffer.current_position = 0; - window_buffer.end_point = 0; - } - - TELEMETRY_UNLOCK(); - - if (kern_buffer) - kmem_free(kernel_map, kern_buffer, kern_buffer_size); -} - /* * Determine if the current thread is eligible for telemetry: * @@ -344,15 +271,15 @@ telemetry_is_active(thread_t thread) return FALSE; } - if (telemetry_sample_all_tasks == TRUE) { - return (TRUE); + if (telemetry_sample_all_tasks || telemetry_sample_pmis) { + return TRUE; } if ((telemetry_active_tasks > 0) && ((thread->task->t_flags & TF_TELEMETRY) != 0)) { - return (TRUE); + return TRUE; } - - return (FALSE); + + return FALSE; } /* @@ -360,21 +287,95 @@ telemetry_is_active(thread_t thread) * sample now. No need to do this one at the AST because we're already at * a safe place in this system call. */ -int telemetry_timer_event(__unused uint64_t deadline, __unused uint64_t interval, __unused uint64_t leeway) +int +telemetry_timer_event(__unused uint64_t deadline, __unused uint64_t interval, __unused uint64_t leeway) { if (telemetry_needs_timer_arming_record == TRUE) { telemetry_needs_timer_arming_record = FALSE; telemetry_take_sample(current_thread(), kTimerArmingRecord | kUserMode, &telemetry_buffer); } - return (0); + return 0; +} + +#if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) +static void +telemetry_pmi_handler(bool user_mode, __unused void *ctx) +{ + telemetry_mark_curthread(user_mode, TRUE); +} +#endif /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */ + +int +telemetry_pmi_setup(enum telemetry_pmi pmi_ctr, uint64_t period) +{ +#if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) + static boolean_t sample_all_tasks_aside = FALSE; + static uint32_t active_tasks_aside = FALSE; + int error = 0; + const char *name = "?"; + + unsigned int ctr = 0; + + TELEMETRY_PMI_LOCK(); + + switch (pmi_ctr) { + case TELEMETRY_PMI_NONE: + if (!telemetry_sample_pmis) { + error = 1; + goto out; + } + + telemetry_sample_pmis = FALSE; + telemetry_sample_all_tasks = sample_all_tasks_aside; + telemetry_active_tasks = active_tasks_aside; + error = mt_microstackshot_stop(); + if (!error) { + printf("telemetry: disabling ustackshot on PMI\n"); + } + goto out; + + case TELEMETRY_PMI_INSTRS: + ctr = MT_CORE_INSTRS; + name = "instructions"; + break; + + case TELEMETRY_PMI_CYCLES: + ctr = MT_CORE_CYCLES; + name = "cycles"; + break; + + default: + error = 1; + goto out; + } + + telemetry_sample_pmis = TRUE; + sample_all_tasks_aside = telemetry_sample_all_tasks; + active_tasks_aside = telemetry_active_tasks; + telemetry_sample_all_tasks = FALSE; + telemetry_active_tasks = 0; + + error = mt_microstackshot_start(ctr, period, telemetry_pmi_handler, NULL); + if (!error) { + printf("telemetry: ustackshot every %llu %s\n", period, name); + } + +out: + TELEMETRY_PMI_UNLOCK(); + return error; +#else /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */ +#pragma unused(pmi_ctr, period) + return 1; +#endif /* !defined(MT_CORE_INSTRS) || !defined(MT_CORE_CYCLES) */ } /* * Mark the current thread for an interrupt-based * telemetry record, to be sampled at the next AST boundary. */ -void telemetry_mark_curthread(boolean_t interrupted_userspace) +void +telemetry_mark_curthread(boolean_t interrupted_userspace, boolean_t pmi) { uint32_t ast_bits = 0; thread_t thread = current_thread(); @@ -388,18 +389,17 @@ void telemetry_mark_curthread(boolean_t interrupted_userspace) } ast_bits |= (interrupted_userspace ? AST_TELEMETRY_USER : AST_TELEMETRY_KERNEL); - - if (telemetry_windowed_record) { - ast_bits |= AST_TELEMETRY_WINDOWED; + if (pmi) { + ast_bits |= AST_TELEMETRY_PMI; } - telemetry_windowed_record = FALSE; telemetry_needs_record = FALSE; thread_ast_set(thread, ast_bits); - ast_propagate(thread->ast); + ast_propagate(thread); } -void compute_telemetry(void *arg __unused) +void +compute_telemetry(void *arg __unused) { if (telemetry_sample_all_tasks || (telemetry_active_tasks > 0)) { if ((++telemetry_timestamp) % telemetry_sample_rate == 0) { @@ -409,71 +409,50 @@ void compute_telemetry(void *arg __unused) } } -void compute_telemetry_windowed(void) -{ - if (telemetry_sample_all_tasks || (telemetry_active_tasks > 0)) { - /* - * Due to the relationship between the two fields here, - * a request for a windowed record will "squash" a - * request for a regular interrupt record. We hedge - * against this by doing a quick check for an existing - * request. compute_telemetry doesn't hedge because - * a regular request cannot squash a windowed request - * (due to the implementation). - * - * If we really want to do this properly, we could make - * telemetry_needs_record a bitfield, and process one - * request per telemetry_mark_curthread... but that - * would be more expensive (atomics). This should be - * robust enough for now (although it biases in favor - * of the regular records). - */ - if (!telemetry_needs_record) { - telemetry_needs_record = TRUE; - telemetry_windowed_record = TRUE; - } - } -} - /* * If userland has registered a port for telemetry notifications, send one now. */ static void telemetry_notify_user(void) { - mach_port_t user_port; - uint32_t flags = 0; - int error; + mach_port_t user_port = MACH_PORT_NULL; - error = host_get_telemetry_port(host_priv_self(), &user_port); - if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { + kern_return_t kr = host_get_telemetry_port(host_priv_self(), &user_port); + if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { return; } - telemetry_notification(user_port, flags); + telemetry_notification(user_port, 0); + ipc_port_release_send(user_port); } -void telemetry_ast(thread_t thread, boolean_t interrupted_userspace, boolean_t is_windowed) +void +telemetry_ast(thread_t thread, ast_t reasons) { - uint8_t microsnapshot_flags = kInterruptRecord; - - if (interrupted_userspace) - microsnapshot_flags |= kUserMode; + assert((reasons & AST_TELEMETRY_ALL) != 0); - if (is_windowed) { - telemetry_take_sample(thread, microsnapshot_flags, &window_buffer); - } else { - telemetry_take_sample(thread, microsnapshot_flags, &telemetry_buffer); + uint8_t record_type = 0; + if (reasons & AST_TELEMETRY_IO) { + record_type |= kIORecord; } + if (reasons & (AST_TELEMETRY_USER | AST_TELEMETRY_KERNEL)) { + record_type |= (reasons & AST_TELEMETRY_PMI) ? kPMIRecord : + kInterruptRecord; + } + + uint8_t user_telemetry = (reasons & AST_TELEMETRY_USER) ? kUserMode : 0; + + uint8_t microsnapshot_flags = record_type | user_telemetry; + + telemetry_take_sample(thread, microsnapshot_flags, &telemetry_buffer); } -void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer) +void +telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer) { task_t task; void *p; - struct kperf_context ctx; - struct callstack cs; - uint32_t btcount, bti; + uint32_t btcount = 0, bti; struct micro_snapshot *msnap; struct task_snapshot *tsnap; struct thread_snapshot *thsnap; @@ -484,54 +463,45 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct uint32_t tmp = 0; boolean_t notify = FALSE; - if (thread == THREAD_NULL) + if (thread == THREAD_NULL) { return; + } task = thread->task; - if ((task == TASK_NULL) || (task == kernel_task)) + if ((task == TASK_NULL) || (task == kernel_task) || task_did_exec(task) || task_is_exec_copy(task)) { return; - - /* - * To avoid overloading the system with telemetry requests, make - * sure we don't add more requests while existing ones are - * in-flight. Attempt this by checking if we can grab the lock. - * - * This concerns me a little; this working as intended is - * contingent on the workload being done in the context of the - * telemetry lock being the expensive part of telemetry. This - * includes populating the buffer and the client gathering it, - * but excludes the copyin overhead. - */ - if (!TELEMETRY_TRY_SPIN_LOCK()) - return; - - TELEMETRY_UNLOCK(); + } /* telemetry_XXX accessed outside of lock for instrumentation only */ - /* TODO */ - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START, microsnapshot_flags, telemetry_bytes_since_last_mark, 0, 0, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START, + microsnapshot_flags, telemetry_bytes_since_last_mark, 0, + (&telemetry_buffer != current_buffer)); p = get_bsdtask_info(task); - ctx.cur_thread = thread; - ctx.cur_pid = proc_pid(p); - /* * Gather up the data we'll need for this sample. The sample is written into the kernel * buffer with the global telemetry lock held -- so we must do our (possibly faulting) * copies from userland here, before taking the lock. */ - kperf_ucallstack_sample(&cs, &ctx); - if (!(cs.flags & CALLSTACK_VALID)) + + uintptr_t frames[128]; + bool user64_regs = false; + int bterror = 0; + btcount = backtrace_user(frames, + sizeof(frames) / sizeof(frames[0]), &bterror, &user64_regs, NULL); + if (bterror != 0) { return; + } + bool user64_va = task_has_64Bit_addr(task); /* * Find the actual [slid] address of the shared cache's UUID, and copy it in from userland. */ - int shared_cache_uuid_valid = 0; - uint64_t shared_cache_base_address; - struct _dyld_cache_header shared_cache_header; - uint64_t shared_cache_slide; + int shared_cache_uuid_valid = 0; + uint64_t shared_cache_base_address = 0; + struct _dyld_cache_header shared_cache_header = {}; + uint64_t shared_cache_slide = 0; /* * Don't copy in the entire shared cache header; we only need the UUID. Calculate the @@ -541,10 +511,10 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct vm_shared_region_t sr = vm_shared_region_get(task); if (sr != NULL) { if ((vm_shared_region_start_address(sr, &shared_cache_base_address) == KERN_SUCCESS) && - (copyin(shared_cache_base_address + sc_header_uuid_offset, (char *)&shared_cache_header.uuid, - sizeof (shared_cache_header.uuid)) == 0)) { + (copyin(shared_cache_base_address + sc_header_uuid_offset, (char *)&shared_cache_header.uuid, + sizeof(shared_cache_header.uuid)) == 0)) { shared_cache_uuid_valid = 1; - shared_cache_slide = vm_shared_region_get_slide(sr); + shared_cache_slide = sr->sr_slide; } // vm_shared_region_get() gave us a reference on the shared region. vm_shared_region_deallocate(sr); @@ -556,15 +526,18 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * * XXX - make this common with kdp? */ - uint32_t uuid_info_count = 0; - mach_vm_address_t uuid_info_addr = 0; - if (task_has_64BitAddr(task)) { + uint32_t uuid_info_count = 0; + mach_vm_address_t uuid_info_addr = 0; + uint32_t uuid_info_size = 0; + if (user64_va) { + uuid_info_size = sizeof(struct user64_dyld_uuid_info); struct user64_dyld_all_image_infos task_image_infos; if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) { uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount; uuid_info_addr = task_image_infos.uuidArray; } } else { + uuid_info_size = sizeof(struct user32_dyld_uuid_info); struct user32_dyld_all_image_infos task_image_infos; if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) { uuid_info_count = task_image_infos.uuidArrayCount; @@ -589,12 +562,13 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct uuid_info_count = TELEMETRY_MAX_UUID_COUNT; } - uint32_t uuid_info_size = (uint32_t)(task_has_64BitAddr(thread->task) ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info)); uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size; - char *uuid_info_array = NULL; + char *uuid_info_array = NULL; if (uuid_info_count > 0) { - if ((uuid_info_array = (char *)kalloc(uuid_info_array_size)) == NULL) { + uuid_info_array = kheap_alloc(KHEAP_TEMP, + uuid_info_array_size, Z_WAITOK); + if (uuid_info_array == NULL) { return; } @@ -603,7 +577,7 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * It may be nonresident, in which case just fix up nloadinfos to 0 in the task snapshot. */ if (copyin(uuid_info_addr, uuid_info_array, uuid_info_array_size) != 0) { - kfree(uuid_info_array, uuid_info_array_size); + kheap_free(KHEAP_TEMP, uuid_info_array, uuid_info_array_size); uuid_info_array = NULL; uuid_info_array_size = 0; } @@ -613,16 +587,16 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * Look for a dispatch queue serial number, and copy it in from userland if present. */ uint64_t dqserialnum = 0; - int dqserialnum_valid = 0; + int dqserialnum_valid = 0; uint64_t dqkeyaddr = thread_dispatchqaddr(thread); if (dqkeyaddr != 0) { uint64_t dqaddr = 0; - uint64_t dq_serialno_offset = get_dispatchqueue_serialno_offset_from_proc(task->bsd_info); - if ((copyin(dqkeyaddr, (char *)&dqaddr, (task_has_64BitAddr(task) ? 8 : 4)) == 0) && + uint64_t dq_serialno_offset = get_task_dispatchqueue_serialno_offset(task); + if ((copyin(dqkeyaddr, (char *)&dqaddr, (user64_va ? 8 : 4)) == 0) && (dqaddr != 0) && (dq_serialno_offset != 0)) { uint64_t dqserialnumaddr = dqaddr + dq_serialno_offset; - if (copyin(dqserialnumaddr, (char *)&dqserialnum, (task_has_64BitAddr(task) ? 8 : 4)) == 0) { + if (copyin(dqserialnumaddr, (char *)&dqserialnum, (user64_va ? 8 : 4)) == 0) { dqserialnum_valid = 1; } } @@ -633,12 +607,13 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct TELEMETRY_LOCK(); /* - * For the benefit of the window buffer; if our buffer is not backed by anything, + * If our buffer is not backed by anything, * then we cannot take the sample. Meant to allow us to deallocate the window * buffer if it is disabled. */ - if (!current_buffer->buffer) + if (!current_buffer->buffer) { goto cancel_sample; + } /* * We do the bulk of the operation under the telemetry lock, on assumption that @@ -670,7 +645,7 @@ copytobuffer: msnap->snapshot_magic = STACKSHOT_MICRO_SNAPSHOT_MAGIC; msnap->ms_flags = microsnapshot_flags; msnap->ms_opaque_flags = 0; /* namespace managed by userspace */ - msnap->ms_cpu = 0; /* XXX - does this field make sense for a micro-stackshot? */ + msnap->ms_cpu = cpu_number(); msnap->ms_time = secs; msnap->ms_time_microsecs = usecs; @@ -694,7 +669,7 @@ copytobuffer: tsnap->user_time_in_terminated_threads = task->total_user_time; tsnap->system_time_in_terminated_threads = task->total_system_time; tsnap->suspend_count = task->suspend_count; - tsnap->task_size = pmap_resident_count(task->map->pmap); + tsnap->task_size = (typeof(tsnap->task_size))(get_task_phys_footprint(task) / PAGE_SIZE); tsnap->faults = task->faults; tsnap->pageins = task->pageins; tsnap->cow_faults = task->cow_faults; @@ -702,23 +677,23 @@ copytobuffer: * The throttling counters are maintained as 64-bit counters in the proc * structure. However, we reserve 32-bits (each) for them in the task_snapshot * struct to save space and since we do not expect them to overflow 32-bits. If we - * find these values overflowing in the future, the fix would be to simply + * find these values overflowing in the future, the fix would be to simply * upgrade these counters to 64-bit in the task_snapshot struct */ tsnap->was_throttled = (uint32_t) proc_was_throttled(p); tsnap->did_throttle = (uint32_t) proc_did_throttle(p); - + if (task->t_flags & TF_TELEMETRY) { tsnap->ss_flags |= kTaskRsrcFlagged; } - if (task->effective_policy.darwinbg == 1) { + if (proc_get_effective_task_policy(task, TASK_POLICY_DARWIN_BG)) { tsnap->ss_flags |= kTaskDarwinBG; } proc_get_darwinbgstate(task, &tmp); - if (task->requested_policy.t_role == TASK_FOREGROUND_APPLICATION) { + if (proc_get_effective_task_policy(task, TASK_POLICY_ROLE) == TASK_FOREGROUND_APPLICATION) { tsnap->ss_flags |= kTaskIsForeground; } @@ -730,16 +705,17 @@ copytobuffer: tsnap->ss_flags |= kTaskIsSuppressed; } + tsnap->latency_qos = task_grab_latency_qos(task); strlcpy(tsnap->p_comm, proc_name_address(p), sizeof(tsnap->p_comm)); - if (task_has_64BitAddr(thread->task)) { + if (user64_va) { tsnap->ss_flags |= kUser64_p; } if (shared_cache_uuid_valid) { tsnap->shared_cache_slide = shared_cache_slide; - bcopy(shared_cache_header.uuid, tsnap->shared_cache_identifier, sizeof (shared_cache_header.uuid)); + bcopy(shared_cache_header.uuid, tsnap->shared_cache_identifier, sizeof(shared_cache_header.uuid)); } current_buffer->current_position += sizeof(struct task_snapshot); @@ -774,7 +750,7 @@ copytobuffer: if ((current_buffer->size - current_buffer->current_position) < sizeof(struct thread_snapshot)) { /* wrap and overwrite */ - current_buffer->end_point = current_record_start; + current_buffer->end_point = current_record_start; current_buffer->current_position = 0; if (current_record_start == 0) { /* This sample is too large to fit in the buffer even when we started at 0, so skip it */ @@ -795,9 +771,10 @@ copytobuffer: thsnap->ss_flags |= kStacksPCOnly; thsnap->ts_qos = thread->effective_policy.thep_qos; thsnap->ts_rqos = thread->requested_policy.thrp_qos; - thsnap->ts_rqos_override = thread->requested_policy.thrp_qos_override; + thsnap->ts_rqos_override = MAX(thread->requested_policy.thrp_qos_override, + thread->requested_policy.thrp_qos_workq_override); - if (thread->effective_policy.darwinbg) { + if (proc_get_effective_thread_policy(thread, TASK_POLICY_DARWIN_BG)) { thsnap->ss_flags |= kThreadDarwinBG; } @@ -820,7 +797,7 @@ copytobuffer: if (dqserialnum_valid) { if ((current_buffer->size - current_buffer->current_position) < sizeof(dqserialnum)) { /* wrap and overwrite */ - current_buffer->end_point = current_record_start; + current_buffer->end_point = current_record_start; current_buffer->current_position = 0; if (current_record_start == 0) { /* This sample is too large to fit in the buffer even when we started at 0, so skip it */ @@ -830,24 +807,22 @@ copytobuffer: } thsnap->ss_flags |= kHasDispatchSerial; - bcopy(&dqserialnum, (char *)current_buffer->buffer + current_buffer->current_position, sizeof (dqserialnum)); - current_buffer->current_position += sizeof (dqserialnum); + bcopy(&dqserialnum, (char *)current_buffer->buffer + current_buffer->current_position, sizeof(dqserialnum)); + current_buffer->current_position += sizeof(dqserialnum); } - if (task_has_64BitAddr(task)) { + if (user64_regs) { framesize = 8; thsnap->ss_flags |= kUser64_p; } else { framesize = 4; } - btcount = cs.nframes; - /* * If we can't fit this entire stacktrace then cancel this record, wrap to the beginning, * and start again there so that we always store a full record. */ - if ((current_buffer->size - current_buffer->current_position)/framesize < btcount) { + if ((current_buffer->size - current_buffer->current_position) / framesize < btcount) { current_buffer->end_point = current_record_start; current_buffer->current_position = 0; if (current_record_start == 0) { @@ -857,11 +832,11 @@ copytobuffer: goto copytobuffer; } - for (bti=0; bti < btcount; bti++, current_buffer->current_position += framesize) { + for (bti = 0; bti < btcount; bti++, current_buffer->current_position += framesize) { if (framesize == 8) { - *(uint64_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = cs.frames[bti]; + *(uint64_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = frames[bti]; } else { - *(uint32_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = (uint32_t)cs.frames[bti]; + *(uint32_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = (uint32_t)frames[bti]; } } @@ -888,18 +863,18 @@ copytobuffer: } cancel_sample: - TELEMETRY_UNLOCK(); - /* TODO */ - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END, notify, telemetry_bytes_since_last_mark, current_buffer->current_position, current_buffer->end_point, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END, + notify, telemetry_bytes_since_last_mark, + current_buffer->current_position, current_buffer->end_point); if (notify) { telemetry_notify_user(); } if (uuid_info_array != NULL) { - kfree(uuid_info_array, uuid_info_array_size); + kheap_free(KHEAP_TEMP, uuid_info_array, uuid_info_array_size); } } @@ -909,7 +884,7 @@ log_telemetry_output(vm_offset_t buf, uint32_t pos, uint32_t sz) { struct micro_snapshot *p; uint32_t offset; - + printf("Copying out %d bytes of telemetry at offset %d\n", sz, pos); buf += pos; @@ -926,28 +901,26 @@ log_telemetry_output(vm_offset_t buf, uint32_t pos, uint32_t sz) } #endif -int telemetry_gather(user_addr_t buffer, uint32_t *length, boolean_t mark) +int +telemetry_gather(user_addr_t buffer, uint32_t *length, boolean_t mark) { return telemetry_buffer_gather(buffer, length, mark, &telemetry_buffer); } -int telemetry_gather_windowed(user_addr_t buffer, uint32_t *length) -{ - return telemetry_buffer_gather(buffer, length, 0, &window_buffer); -} - -int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer) +int +telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer) { int result = 0; uint32_t oldest_record_offset; - /* TODO */ - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START, mark, telemetry_bytes_since_last_mark, 0, 0, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START, + mark, telemetry_bytes_since_last_mark, 0, + (&telemetry_buffer != current_buffer)); TELEMETRY_LOCK(); if (current_buffer->buffer == 0) { - *length = 0; + *length = 0; goto out; } @@ -964,7 +937,6 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark do { if (((oldest_record_offset + sizeof(uint32_t)) > current_buffer->size) || ((oldest_record_offset + sizeof(uint32_t)) > current_buffer->end_point)) { - if (*(uint32_t *)(uintptr_t)(current_buffer->buffer) == 0) { /* * There is no magic number at the start of the buffer, which means @@ -984,8 +956,9 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark break; } - if (*(uint32_t *)(uintptr_t)(current_buffer->buffer + oldest_record_offset) == STACKSHOT_MICRO_SNAPSHOT_MAGIC) + if (*(uint32_t *)(uintptr_t)(current_buffer->buffer + oldest_record_offset) == STACKSHOT_MICRO_SNAPSHOT_MAGIC) { break; + } /* * There are no alignment guarantees for micro-stackshot records, so we must search at each @@ -1001,7 +974,7 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark if (oldest_record_offset != 0) { #if TELEMETRY_DEBUG log_telemetry_output(current_buffer->buffer, oldest_record_offset, - current_buffer->end_point - oldest_record_offset); + current_buffer->end_point - oldest_record_offset); #endif if ((result = copyout((void *)(current_buffer->buffer + oldest_record_offset), buffer, current_buffer->end_point - oldest_record_offset)) != 0) { @@ -1031,9 +1004,11 @@ out: TELEMETRY_UNLOCK(); - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END, current_buffer->current_position, *length, current_buffer->end_point, 0, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END, + current_buffer->current_position, *length, + current_buffer->end_point, (&telemetry_buffer != current_buffer)); - return (result); + return result; } /************************/ @@ -1065,17 +1040,19 @@ out: #define BOOTPROFILE_MAX_BUFFER_SIZE (64*1024*1024) /* see also COPYSIZELIMIT_PANIC */ -vm_offset_t bootprofile_buffer = 0; -uint32_t bootprofile_buffer_size = 0; -uint32_t bootprofile_buffer_current_position = 0; -uint32_t bootprofile_interval_ms = 0; -uint64_t bootprofile_interval_abs = 0; -uint64_t bootprofile_next_deadline = 0; -uint32_t bootprofile_all_procs = 0; -char bootprofile_proc_name[17]; +vm_offset_t bootprofile_buffer = 0; +uint32_t bootprofile_buffer_size = 0; +uint32_t bootprofile_buffer_current_position = 0; +uint32_t bootprofile_interval_ms = 0; +uint64_t bootprofile_stackshot_flags = 0; +uint64_t bootprofile_interval_abs = 0; +uint64_t bootprofile_next_deadline = 0; +uint32_t bootprofile_all_procs = 0; +char bootprofile_proc_name[17]; +uint64_t bootprofile_delta_since_timestamp = 0; +LCK_GRP_DECLARE(bootprofile_lck_grp, "bootprofile_group"); +LCK_MTX_DECLARE(bootprofile_mtx, &bootprofile_lck_grp); -lck_grp_t bootprofile_lck_grp; -lck_mtx_t bootprofile_mtx; enum { kBootProfileDisabled = 0, @@ -1084,7 +1061,7 @@ enum { } bootprofile_type = kBootProfileDisabled; -static timer_call_data_t bootprofile_timer_call_entry; +static timer_call_data_t bootprofile_timer_call_entry; #define BOOTPROFILE_LOCK() do { lck_mtx_lock(&bootprofile_mtx); } while(0) #define BOOTPROFILE_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&bootprofile_mtx) @@ -1094,29 +1071,33 @@ static void bootprofile_timer_call( timer_call_param_t param0, timer_call_param_t param1); -extern int -stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, unsigned *retbytes); - -void bootprofile_init(void) +void +bootprofile_init(void) { kern_return_t ret; char type[32]; - lck_grp_init(&bootprofile_lck_grp, "bootprofile group", LCK_GRP_ATTR_NULL); - lck_mtx_init(&bootprofile_mtx, &bootprofile_lck_grp, LCK_ATTR_NULL); - - if (!PE_parse_boot_argn("bootprofile_buffer_size", &bootprofile_buffer_size, sizeof(bootprofile_buffer_size))) { + if (!PE_parse_boot_argn("bootprofile_buffer_size", + &bootprofile_buffer_size, sizeof(bootprofile_buffer_size))) { bootprofile_buffer_size = 0; } - if (bootprofile_buffer_size > BOOTPROFILE_MAX_BUFFER_SIZE) + if (bootprofile_buffer_size > BOOTPROFILE_MAX_BUFFER_SIZE) { bootprofile_buffer_size = BOOTPROFILE_MAX_BUFFER_SIZE; + } - if (!PE_parse_boot_argn("bootprofile_interval_ms", &bootprofile_interval_ms, sizeof(bootprofile_interval_ms))) { + if (!PE_parse_boot_argn("bootprofile_interval_ms", + &bootprofile_interval_ms, sizeof(bootprofile_interval_ms))) { bootprofile_interval_ms = 0; } - if (!PE_parse_boot_argn("bootprofile_proc_name", &bootprofile_proc_name, sizeof(bootprofile_proc_name))) { + if (!PE_parse_boot_argn("bootprofile_stackshot_flags", + &bootprofile_stackshot_flags, sizeof(bootprofile_stackshot_flags))) { + bootprofile_stackshot_flags = 0; + } + + if (!PE_parse_boot_argn("bootprofile_proc_name", + &bootprofile_proc_name, sizeof(bootprofile_proc_name))) { bootprofile_all_procs = 1; bootprofile_proc_name[0] = '\0'; } @@ -1125,7 +1106,7 @@ void bootprofile_init(void) if (0 == strcmp(type, "boot")) { bootprofile_type = kBootProfileStartTimerAtBoot; } else if (0 == strcmp(type, "wake")) { - bootprofile_type = kBootProfileStartTimerAtWake; + bootprofile_type = kBootProfileStartTimerAtWake; } else { bootprofile_type = kBootProfileDisabled; } @@ -1147,21 +1128,22 @@ void bootprofile_init(void) } bzero((void *) bootprofile_buffer, bootprofile_buffer_size); - kprintf("Boot profile: Sampling %s once per %u ms at %s\n", bootprofile_all_procs ? "all procs" : bootprofile_proc_name, bootprofile_interval_ms, - bootprofile_type == kBootProfileStartTimerAtBoot ? "boot" : (bootprofile_type == kBootProfileStartTimerAtWake ? "wake" : "unknown")); + kprintf("Boot profile: Sampling %s once per %u ms at %s\n", + bootprofile_all_procs ? "all procs" : bootprofile_proc_name, bootprofile_interval_ms, + bootprofile_type == kBootProfileStartTimerAtBoot ? "boot" : (bootprofile_type == kBootProfileStartTimerAtWake ? "wake" : "unknown")); timer_call_setup(&bootprofile_timer_call_entry, - bootprofile_timer_call, - NULL); + bootprofile_timer_call, + NULL); if (bootprofile_type == kBootProfileStartTimerAtBoot) { bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs; timer_call_enter_with_leeway(&bootprofile_timer_call_entry, - NULL, - bootprofile_next_deadline, - 0, - TIMER_CALL_SYS_NORMAL, - FALSE); + NULL, + bootprofile_next_deadline, + 0, + TIMER_CALL_SYS_NORMAL, + FALSE); } } @@ -1171,16 +1153,17 @@ bootprofile_wake_from_sleep(void) if (bootprofile_type == kBootProfileStartTimerAtWake) { bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs; timer_call_enter_with_leeway(&bootprofile_timer_call_entry, - NULL, - bootprofile_next_deadline, - 0, - TIMER_CALL_SYS_NORMAL, - FALSE); + NULL, + bootprofile_next_deadline, + 0, + TIMER_CALL_SYS_NORMAL, + FALSE); } } -static void bootprofile_timer_call( +static void +bootprofile_timer_call( timer_call_param_t param0 __unused, timer_call_param_t param1 __unused) { @@ -1202,8 +1185,7 @@ static void bootprofile_timer_call( if ((current_task() != NULL) && (current_task()->bsd_info != NULL) && (0 == strncmp(bootprofile_proc_name, proc_name_address(current_task()->bsd_info), 17))) { pid_to_profile = proc_selfpid(); - } - else { + } else { /* * Process-specific boot profiling requested but the on-core process is * something else. Nothing to do here. @@ -1215,13 +1197,45 @@ static void bootprofile_timer_call( /* initiate a stackshot with whatever portion of the buffer is left */ if (bootprofile_buffer_current_position < bootprofile_buffer_size) { - stack_snapshot_from_kernel( - pid_to_profile, - (void *)(bootprofile_buffer + bootprofile_buffer_current_position), + uint64_t flags = STACKSHOT_KCDATA_FORMAT | STACKSHOT_TRYLOCK | STACKSHOT_SAVE_LOADINFO + | STACKSHOT_GET_GLOBAL_MEM_STATS; +#if defined(XNU_TARGET_OS_OSX) + flags |= STACKSHOT_SAVE_KEXT_LOADINFO; +#endif + + + /* OR on flags specified in boot-args */ + flags |= bootprofile_stackshot_flags; + if ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) && (bootprofile_delta_since_timestamp == 0)) { + /* Can't take deltas until the first one */ + flags &= ~STACKSHOT_COLLECT_DELTA_SNAPSHOT; + } + + uint64_t timestamp = 0; + if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) { + timestamp = mach_absolute_time(); + } + + kern_return_t r = stack_snapshot_from_kernel( + pid_to_profile, (void *)(bootprofile_buffer + bootprofile_buffer_current_position), bootprofile_buffer_size - bootprofile_buffer_current_position, - STACKSHOT_SAVE_LOADINFO | STACKSHOT_SAVE_KEXT_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS, - &retbytes - ); + flags, bootprofile_delta_since_timestamp, 0, &retbytes); + + /* + * We call with STACKSHOT_TRYLOCK because the stackshot lock is coarser + * than the bootprofile lock. If someone else has the lock we'll just + * try again later. + */ + + if (r == KERN_LOCK_OWNED) { + BOOTPROFILE_UNLOCK(); + goto reprogram; + } + + if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT && + r == KERN_SUCCESS) { + bootprofile_delta_since_timestamp = timestamp; + } bootprofile_buffer_current_position += retbytes; } @@ -1241,24 +1255,34 @@ reprogram: } clock_deadline_for_periodic_event(bootprofile_interval_abs, - mach_absolute_time(), - &bootprofile_next_deadline); + mach_absolute_time(), + &bootprofile_next_deadline); timer_call_enter_with_leeway(&bootprofile_timer_call_entry, - NULL, - bootprofile_next_deadline, - 0, - TIMER_CALL_SYS_NORMAL, - FALSE); + NULL, + bootprofile_next_deadline, + 0, + TIMER_CALL_SYS_NORMAL, + FALSE); } -int bootprofile_gather(user_addr_t buffer, uint32_t *length) +void +bootprofile_get(void **buffer, uint32_t *length) +{ + BOOTPROFILE_LOCK(); + *buffer = (void*) bootprofile_buffer; + *length = bootprofile_buffer_current_position; + BOOTPROFILE_UNLOCK(); +} + +int +bootprofile_gather(user_addr_t buffer, uint32_t *length) { int result = 0; BOOTPROFILE_LOCK(); if (bootprofile_buffer == 0) { - *length = 0; + *length = 0; goto out; } @@ -1281,5 +1305,5 @@ out: BOOTPROFILE_UNLOCK(); - return (result); + return result; }