X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/cc8bc92ae4a8e9f1a1ab61bf83d34ad8150b3405..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/osfmk/kern/telemetry.c?ds=inline diff --git a/osfmk/kern/telemetry.c b/osfmk/kern/telemetry.c index 120885eac..b777052d4 100644 --- a/osfmk/kern/telemetry.c +++ b/osfmk/kern/telemetry.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include @@ -35,9 +35,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include @@ -52,6 +52,7 @@ #include #include +#include #include #include @@ -59,20 +60,21 @@ #define TELEMETRY_DEBUG 0 -extern int proc_pid(void *); -extern char *proc_name_address(void *p); +struct proc; +extern int proc_pid(struct proc *); +extern char *proc_name_address(void *p); extern uint64_t proc_uniqueid(void *p); extern uint64_t proc_was_throttled(void *p); extern uint64_t proc_did_throttle(void *p); -extern int proc_selfpid(void); +extern int proc_selfpid(void); extern boolean_t task_did_exec(task_t task); extern boolean_t task_is_exec_copy(task_t task); struct micro_snapshot_buffer { - vm_offset_t buffer; - uint32_t size; - uint32_t current_position; - uint32_t end_point; + vm_offset_t buffer; + uint32_t size; + uint32_t current_position; + uint32_t end_point; }; void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer); @@ -82,21 +84,22 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark #define TELEMETRY_DEFAULT_BUFFER_SIZE (16*1024) #define TELEMETRY_MAX_BUFFER_SIZE (64*1024) -#define TELEMETRY_DEFAULT_NOTIFY_LEEWAY (4*1024) // Userland gets 4k of leeway to collect data after notification -#define TELEMETRY_MAX_UUID_COUNT (128) // Max of 128 non-shared-cache UUIDs to log for symbolication +#define TELEMETRY_DEFAULT_NOTIFY_LEEWAY (4*1024) // Userland gets 4k of leeway to collect data after notification +#define TELEMETRY_MAX_UUID_COUNT (128) // Max of 128 non-shared-cache UUIDs to log for symbolication -uint32_t telemetry_sample_rate = 0; -volatile boolean_t telemetry_needs_record = FALSE; -volatile boolean_t telemetry_needs_timer_arming_record = FALSE; +uint32_t telemetry_sample_rate = 0; +volatile boolean_t telemetry_needs_record = FALSE; +volatile boolean_t telemetry_needs_timer_arming_record = FALSE; /* * If TRUE, record micro-stackshot samples for all tasks. * If FALSE, only sample tasks which are marked for telemetry. */ -boolean_t telemetry_sample_all_tasks = FALSE; -uint32_t telemetry_active_tasks = 0; // Number of tasks opted into telemetry +boolean_t telemetry_sample_all_tasks = FALSE; +boolean_t telemetry_sample_pmis = FALSE; +uint32_t telemetry_active_tasks = 0; // Number of tasks opted into telemetry -uint32_t telemetry_timestamp = 0; +uint32_t telemetry_timestamp = 0; /* * The telemetry_buffer is responsible @@ -104,32 +107,41 @@ uint32_t telemetry_timestamp = 0; * compute_averages(). It will notify its client (if one * exists) when it has enough data to be worth flushing. */ -struct micro_snapshot_buffer telemetry_buffer = {0, 0, 0, 0}; +struct micro_snapshot_buffer telemetry_buffer = { + .buffer = 0, + .size = 0, + .current_position = 0, + .end_point = 0 +}; -int telemetry_bytes_since_last_mark = -1; // How much data since buf was last marked? -int telemetry_buffer_notify_at = 0; +int telemetry_bytes_since_last_mark = -1; // How much data since buf was last marked? +int telemetry_buffer_notify_at = 0; -lck_grp_t telemetry_lck_grp; -lck_mtx_t telemetry_mtx; +LCK_GRP_DECLARE(telemetry_lck_grp, "telemetry group"); +LCK_MTX_DECLARE(telemetry_mtx, &telemetry_lck_grp); +LCK_MTX_DECLARE(telemetry_pmi_mtx, &telemetry_lck_grp); -#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while(0) +#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while (0) #define TELEMETRY_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&telemetry_mtx) -#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while(0) +#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while (0) -void telemetry_init(void) +#define TELEMETRY_PMI_LOCK() do { lck_mtx_lock(&telemetry_pmi_mtx); } while (0) +#define TELEMETRY_PMI_UNLOCK() do { lck_mtx_unlock(&telemetry_pmi_mtx); } while (0) + +void +telemetry_init(void) { kern_return_t ret; - uint32_t telemetry_notification_leeway; - - lck_grp_init(&telemetry_lck_grp, "telemetry group", LCK_GRP_ATTR_NULL); - lck_mtx_init(&telemetry_mtx, &telemetry_lck_grp, LCK_ATTR_NULL); + uint32_t telemetry_notification_leeway; - if (!PE_parse_boot_argn("telemetry_buffer_size", &telemetry_buffer.size, sizeof(telemetry_buffer.size))) { + if (!PE_parse_boot_argn("telemetry_buffer_size", + &telemetry_buffer.size, sizeof(telemetry_buffer.size))) { telemetry_buffer.size = TELEMETRY_DEFAULT_BUFFER_SIZE; } - if (telemetry_buffer.size > TELEMETRY_MAX_BUFFER_SIZE) + if (telemetry_buffer.size > TELEMETRY_MAX_BUFFER_SIZE) { telemetry_buffer.size = TELEMETRY_MAX_BUFFER_SIZE; + } ret = kmem_alloc(kernel_map, &telemetry_buffer.buffer, telemetry_buffer.size, VM_KERN_MEMORY_DIAG); if (ret != KERN_SUCCESS) { @@ -138,7 +150,8 @@ void telemetry_init(void) } bzero((void *) telemetry_buffer.buffer, telemetry_buffer.size); - if (!PE_parse_boot_argn("telemetry_notification_leeway", &telemetry_notification_leeway, sizeof(telemetry_notification_leeway))) { + if (!PE_parse_boot_argn("telemetry_notification_leeway", + &telemetry_notification_leeway, sizeof(telemetry_notification_leeway))) { /* * By default, notify the user to collect the buffer when there is this much space left in the buffer. */ @@ -146,31 +159,31 @@ void telemetry_init(void) } if (telemetry_notification_leeway >= telemetry_buffer.size) { printf("telemetry: nonsensical telemetry_notification_leeway boot-arg %d changed to %d\n", - telemetry_notification_leeway, TELEMETRY_DEFAULT_NOTIFY_LEEWAY); + telemetry_notification_leeway, TELEMETRY_DEFAULT_NOTIFY_LEEWAY); telemetry_notification_leeway = TELEMETRY_DEFAULT_NOTIFY_LEEWAY; } telemetry_buffer_notify_at = telemetry_buffer.size - telemetry_notification_leeway; - if (!PE_parse_boot_argn("telemetry_sample_rate", &telemetry_sample_rate, sizeof(telemetry_sample_rate))) { + if (!PE_parse_boot_argn("telemetry_sample_rate", + &telemetry_sample_rate, sizeof(telemetry_sample_rate))) { telemetry_sample_rate = TELEMETRY_DEFAULT_SAMPLE_RATE; } /* * To enable telemetry for all tasks, include "telemetry_sample_all_tasks=1" in boot-args. */ - if (!PE_parse_boot_argn("telemetry_sample_all_tasks", &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) { - -#if CONFIG_EMBEDDED && !(DEVELOPMENT || DEBUG) + if (!PE_parse_boot_argn("telemetry_sample_all_tasks", + &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) { +#if !defined(XNU_TARGET_OS_OSX) && !(DEVELOPMENT || DEBUG) telemetry_sample_all_tasks = FALSE; #else telemetry_sample_all_tasks = TRUE; -#endif /* CONFIG_EMBEDDED && !(DEVELOPMENT || DEBUG) */ - +#endif /* !defined(XNU_TARGET_OS_OSX) && !(DEVELOPMENT || DEBUG) */ } kprintf("Telemetry: Sampling %stasks once per %u second%s\n", - (telemetry_sample_all_tasks) ? "all " : "", - telemetry_sample_rate, telemetry_sample_rate == 1 ? "" : "s"); + (telemetry_sample_all_tasks) ? "all " : "", + telemetry_sample_rate, telemetry_sample_rate == 1 ? "" : "s"); } /* @@ -180,7 +193,7 @@ void telemetry_init(void) * enable_disable == 0: turn it off */ void -telemetry_global_ctl(int enable_disable) +telemetry_global_ctl(int enable_disable) { if (enable_disable == 1) { telemetry_sample_all_tasks = TRUE; @@ -222,9 +235,9 @@ telemetry_task_ctl_locked(task_t task, uint32_t reasons, int enable_disable) task->t_flags |= reasons; if ((origflags & TF_TELEMETRY) == 0) { OSIncrementAtomic(&telemetry_active_tasks); -#if TELEMETRY_DEBUG +#if TELEMETRY_DEBUG printf("%s: telemetry OFF -> ON (%d active)\n", proc_name_address(task->bsd_info), telemetry_active_tasks); -#endif +#endif } } else { task->t_flags &= ~reasons; @@ -258,15 +271,15 @@ telemetry_is_active(thread_t thread) return FALSE; } - if (telemetry_sample_all_tasks == TRUE) { - return (TRUE); + if (telemetry_sample_all_tasks || telemetry_sample_pmis) { + return TRUE; } if ((telemetry_active_tasks > 0) && ((thread->task->t_flags & TF_TELEMETRY) != 0)) { - return (TRUE); + return TRUE; } - - return (FALSE); + + return FALSE; } /* @@ -274,21 +287,95 @@ telemetry_is_active(thread_t thread) * sample now. No need to do this one at the AST because we're already at * a safe place in this system call. */ -int telemetry_timer_event(__unused uint64_t deadline, __unused uint64_t interval, __unused uint64_t leeway) +int +telemetry_timer_event(__unused uint64_t deadline, __unused uint64_t interval, __unused uint64_t leeway) { if (telemetry_needs_timer_arming_record == TRUE) { telemetry_needs_timer_arming_record = FALSE; telemetry_take_sample(current_thread(), kTimerArmingRecord | kUserMode, &telemetry_buffer); } - return (0); + return 0; +} + +#if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) +static void +telemetry_pmi_handler(bool user_mode, __unused void *ctx) +{ + telemetry_mark_curthread(user_mode, TRUE); +} +#endif /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */ + +int +telemetry_pmi_setup(enum telemetry_pmi pmi_ctr, uint64_t period) +{ +#if defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) + static boolean_t sample_all_tasks_aside = FALSE; + static uint32_t active_tasks_aside = FALSE; + int error = 0; + const char *name = "?"; + + unsigned int ctr = 0; + + TELEMETRY_PMI_LOCK(); + + switch (pmi_ctr) { + case TELEMETRY_PMI_NONE: + if (!telemetry_sample_pmis) { + error = 1; + goto out; + } + + telemetry_sample_pmis = FALSE; + telemetry_sample_all_tasks = sample_all_tasks_aside; + telemetry_active_tasks = active_tasks_aside; + error = mt_microstackshot_stop(); + if (!error) { + printf("telemetry: disabling ustackshot on PMI\n"); + } + goto out; + + case TELEMETRY_PMI_INSTRS: + ctr = MT_CORE_INSTRS; + name = "instructions"; + break; + + case TELEMETRY_PMI_CYCLES: + ctr = MT_CORE_CYCLES; + name = "cycles"; + break; + + default: + error = 1; + goto out; + } + + telemetry_sample_pmis = TRUE; + sample_all_tasks_aside = telemetry_sample_all_tasks; + active_tasks_aside = telemetry_active_tasks; + telemetry_sample_all_tasks = FALSE; + telemetry_active_tasks = 0; + + error = mt_microstackshot_start(ctr, period, telemetry_pmi_handler, NULL); + if (!error) { + printf("telemetry: ustackshot every %llu %s\n", period, name); + } + +out: + TELEMETRY_PMI_UNLOCK(); + return error; +#else /* defined(MT_CORE_INSTRS) && defined(MT_CORE_CYCLES) */ +#pragma unused(pmi_ctr, period) + return 1; +#endif /* !defined(MT_CORE_INSTRS) || !defined(MT_CORE_CYCLES) */ } /* * Mark the current thread for an interrupt-based * telemetry record, to be sampled at the next AST boundary. */ -void telemetry_mark_curthread(boolean_t interrupted_userspace) +void +telemetry_mark_curthread(boolean_t interrupted_userspace, boolean_t pmi) { uint32_t ast_bits = 0; thread_t thread = current_thread(); @@ -302,13 +389,17 @@ void telemetry_mark_curthread(boolean_t interrupted_userspace) } ast_bits |= (interrupted_userspace ? AST_TELEMETRY_USER : AST_TELEMETRY_KERNEL); + if (pmi) { + ast_bits |= AST_TELEMETRY_PMI; + } telemetry_needs_record = FALSE; thread_ast_set(thread, ast_bits); ast_propagate(thread); } -void compute_telemetry(void *arg __unused) +void +compute_telemetry(void *arg __unused) { if (telemetry_sample_all_tasks || (telemetry_active_tasks > 0)) { if ((++telemetry_timestamp) % telemetry_sample_rate == 0) { @@ -324,38 +415,40 @@ void compute_telemetry(void *arg __unused) static void telemetry_notify_user(void) { - mach_port_t user_port; - uint32_t flags = 0; - int error; + mach_port_t user_port = MACH_PORT_NULL; - error = host_get_telemetry_port(host_priv_self(), &user_port); - if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { + kern_return_t kr = host_get_telemetry_port(host_priv_self(), &user_port); + if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { return; } - telemetry_notification(user_port, flags); + telemetry_notification(user_port, 0); ipc_port_release_send(user_port); } -void telemetry_ast(thread_t thread, ast_t reasons) +void +telemetry_ast(thread_t thread, ast_t reasons) { - assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */ + assert((reasons & AST_TELEMETRY_ALL) != 0); - boolean_t io_telemetry = (reasons & AST_TELEMETRY_IO) ? TRUE : FALSE; - boolean_t interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE; - - uint8_t microsnapshot_flags = kInterruptRecord; + uint8_t record_type = 0; + if (reasons & AST_TELEMETRY_IO) { + record_type |= kIORecord; + } + if (reasons & (AST_TELEMETRY_USER | AST_TELEMETRY_KERNEL)) { + record_type |= (reasons & AST_TELEMETRY_PMI) ? kPMIRecord : + kInterruptRecord; + } - if (io_telemetry == TRUE) - microsnapshot_flags = kIORecord; + uint8_t user_telemetry = (reasons & AST_TELEMETRY_USER) ? kUserMode : 0; - if (interrupted_userspace) - microsnapshot_flags |= kUserMode; + uint8_t microsnapshot_flags = record_type | user_telemetry; telemetry_take_sample(thread, microsnapshot_flags, &telemetry_buffer); } -void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer) +void +telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct micro_snapshot_buffer * current_buffer) { task_t task; void *p; @@ -370,32 +463,19 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct uint32_t tmp = 0; boolean_t notify = FALSE; - if (thread == THREAD_NULL) + if (thread == THREAD_NULL) { return; + } task = thread->task; - if ((task == TASK_NULL) || (task == kernel_task) || task_did_exec(task) || task_is_exec_copy(task)) + if ((task == TASK_NULL) || (task == kernel_task) || task_did_exec(task) || task_is_exec_copy(task)) { return; - - /* - * To avoid overloading the system with telemetry requests, make - * sure we don't add more requests while existing ones are - * in-flight. Attempt this by checking if we can grab the lock. - * - * This concerns me a little; this working as intended is - * contingent on the workload being done in the context of the - * telemetry lock being the expensive part of telemetry. This - * includes populating the buffer and the client gathering it, - * but excludes the copyin overhead. - */ - if (!TELEMETRY_TRY_SPIN_LOCK()) - return; - - TELEMETRY_UNLOCK(); + } /* telemetry_XXX accessed outside of lock for instrumentation only */ - /* TODO */ - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START, microsnapshot_flags, telemetry_bytes_since_last_mark, 0, 0, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START, + microsnapshot_flags, telemetry_bytes_since_last_mark, 0, + (&telemetry_buffer != current_buffer)); p = get_bsdtask_info(task); @@ -404,20 +484,24 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * buffer with the global telemetry lock held -- so we must do our (possibly faulting) * copies from userland here, before taking the lock. */ - uintptr_t frames[MAX_CALLSTACK_FRAMES] = {}; - bool user64; - int backtrace_error = backtrace_user(frames, MAX_CALLSTACK_FRAMES, &btcount, &user64); - if (backtrace_error) { + + uintptr_t frames[128]; + bool user64_regs = false; + int bterror = 0; + btcount = backtrace_user(frames, + sizeof(frames) / sizeof(frames[0]), &bterror, &user64_regs, NULL); + if (bterror != 0) { return; } + bool user64_va = task_has_64Bit_addr(task); /* * Find the actual [slid] address of the shared cache's UUID, and copy it in from userland. */ - int shared_cache_uuid_valid = 0; - uint64_t shared_cache_base_address; - struct _dyld_cache_header shared_cache_header; - uint64_t shared_cache_slide; + int shared_cache_uuid_valid = 0; + uint64_t shared_cache_base_address = 0; + struct _dyld_cache_header shared_cache_header = {}; + uint64_t shared_cache_slide = 0; /* * Don't copy in the entire shared cache header; we only need the UUID. Calculate the @@ -427,10 +511,10 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct vm_shared_region_t sr = vm_shared_region_get(task); if (sr != NULL) { if ((vm_shared_region_start_address(sr, &shared_cache_base_address) == KERN_SUCCESS) && - (copyin(shared_cache_base_address + sc_header_uuid_offset, (char *)&shared_cache_header.uuid, - sizeof (shared_cache_header.uuid)) == 0)) { + (copyin(shared_cache_base_address + sc_header_uuid_offset, (char *)&shared_cache_header.uuid, + sizeof(shared_cache_header.uuid)) == 0)) { shared_cache_uuid_valid = 1; - shared_cache_slide = vm_shared_region_get_slide(sr); + shared_cache_slide = sr->sr_slide; } // vm_shared_region_get() gave us a reference on the shared region. vm_shared_region_deallocate(sr); @@ -442,15 +526,18 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * * XXX - make this common with kdp? */ - uint32_t uuid_info_count = 0; - mach_vm_address_t uuid_info_addr = 0; - if (task_has_64BitAddr(task)) { + uint32_t uuid_info_count = 0; + mach_vm_address_t uuid_info_addr = 0; + uint32_t uuid_info_size = 0; + if (user64_va) { + uuid_info_size = sizeof(struct user64_dyld_uuid_info); struct user64_dyld_all_image_infos task_image_infos; if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) { uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount; uuid_info_addr = task_image_infos.uuidArray; } } else { + uuid_info_size = sizeof(struct user32_dyld_uuid_info); struct user32_dyld_all_image_infos task_image_infos; if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) { uuid_info_count = task_image_infos.uuidArrayCount; @@ -475,12 +562,13 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct uuid_info_count = TELEMETRY_MAX_UUID_COUNT; } - uint32_t uuid_info_size = (uint32_t)(task_has_64BitAddr(thread->task) ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info)); uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size; - char *uuid_info_array = NULL; + char *uuid_info_array = NULL; if (uuid_info_count > 0) { - if ((uuid_info_array = (char *)kalloc(uuid_info_array_size)) == NULL) { + uuid_info_array = kheap_alloc(KHEAP_TEMP, + uuid_info_array_size, Z_WAITOK); + if (uuid_info_array == NULL) { return; } @@ -489,7 +577,7 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * It may be nonresident, in which case just fix up nloadinfos to 0 in the task snapshot. */ if (copyin(uuid_info_addr, uuid_info_array, uuid_info_array_size) != 0) { - kfree(uuid_info_array, uuid_info_array_size); + kheap_free(KHEAP_TEMP, uuid_info_array, uuid_info_array_size); uuid_info_array = NULL; uuid_info_array_size = 0; } @@ -499,16 +587,16 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * Look for a dispatch queue serial number, and copy it in from userland if present. */ uint64_t dqserialnum = 0; - int dqserialnum_valid = 0; + int dqserialnum_valid = 0; uint64_t dqkeyaddr = thread_dispatchqaddr(thread); if (dqkeyaddr != 0) { uint64_t dqaddr = 0; uint64_t dq_serialno_offset = get_task_dispatchqueue_serialno_offset(task); - if ((copyin(dqkeyaddr, (char *)&dqaddr, (task_has_64BitAddr(task) ? 8 : 4)) == 0) && + if ((copyin(dqkeyaddr, (char *)&dqaddr, (user64_va ? 8 : 4)) == 0) && (dqaddr != 0) && (dq_serialno_offset != 0)) { uint64_t dqserialnumaddr = dqaddr + dq_serialno_offset; - if (copyin(dqserialnumaddr, (char *)&dqserialnum, (task_has_64BitAddr(task) ? 8 : 4)) == 0) { + if (copyin(dqserialnumaddr, (char *)&dqserialnum, (user64_va ? 8 : 4)) == 0) { dqserialnum_valid = 1; } } @@ -523,8 +611,9 @@ void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags, struct * then we cannot take the sample. Meant to allow us to deallocate the window * buffer if it is disabled. */ - if (!current_buffer->buffer) + if (!current_buffer->buffer) { goto cancel_sample; + } /* * We do the bulk of the operation under the telemetry lock, on assumption that @@ -556,7 +645,7 @@ copytobuffer: msnap->snapshot_magic = STACKSHOT_MICRO_SNAPSHOT_MAGIC; msnap->ms_flags = microsnapshot_flags; msnap->ms_opaque_flags = 0; /* namespace managed by userspace */ - msnap->ms_cpu = 0; /* XXX - does this field make sense for a micro-stackshot? */ + msnap->ms_cpu = cpu_number(); msnap->ms_time = secs; msnap->ms_time_microsecs = usecs; @@ -580,7 +669,7 @@ copytobuffer: tsnap->user_time_in_terminated_threads = task->total_user_time; tsnap->system_time_in_terminated_threads = task->total_system_time; tsnap->suspend_count = task->suspend_count; - tsnap->task_size = pmap_resident_count(task->map->pmap); + tsnap->task_size = (typeof(tsnap->task_size))(get_task_phys_footprint(task) / PAGE_SIZE); tsnap->faults = task->faults; tsnap->pageins = task->pageins; tsnap->cow_faults = task->cow_faults; @@ -588,12 +677,12 @@ copytobuffer: * The throttling counters are maintained as 64-bit counters in the proc * structure. However, we reserve 32-bits (each) for them in the task_snapshot * struct to save space and since we do not expect them to overflow 32-bits. If we - * find these values overflowing in the future, the fix would be to simply + * find these values overflowing in the future, the fix would be to simply * upgrade these counters to 64-bit in the task_snapshot struct */ tsnap->was_throttled = (uint32_t) proc_was_throttled(p); tsnap->did_throttle = (uint32_t) proc_did_throttle(p); - + if (task->t_flags & TF_TELEMETRY) { tsnap->ss_flags |= kTaskRsrcFlagged; } @@ -616,16 +705,17 @@ copytobuffer: tsnap->ss_flags |= kTaskIsSuppressed; } + tsnap->latency_qos = task_grab_latency_qos(task); strlcpy(tsnap->p_comm, proc_name_address(p), sizeof(tsnap->p_comm)); - if (task_has_64BitAddr(thread->task)) { + if (user64_va) { tsnap->ss_flags |= kUser64_p; } if (shared_cache_uuid_valid) { tsnap->shared_cache_slide = shared_cache_slide; - bcopy(shared_cache_header.uuid, tsnap->shared_cache_identifier, sizeof (shared_cache_header.uuid)); + bcopy(shared_cache_header.uuid, tsnap->shared_cache_identifier, sizeof(shared_cache_header.uuid)); } current_buffer->current_position += sizeof(struct task_snapshot); @@ -660,7 +750,7 @@ copytobuffer: if ((current_buffer->size - current_buffer->current_position) < sizeof(struct thread_snapshot)) { /* wrap and overwrite */ - current_buffer->end_point = current_record_start; + current_buffer->end_point = current_record_start; current_buffer->current_position = 0; if (current_record_start == 0) { /* This sample is too large to fit in the buffer even when we started at 0, so skip it */ @@ -681,7 +771,8 @@ copytobuffer: thsnap->ss_flags |= kStacksPCOnly; thsnap->ts_qos = thread->effective_policy.thep_qos; thsnap->ts_rqos = thread->requested_policy.thrp_qos; - thsnap->ts_rqos_override = thread->requested_policy.thrp_qos_override; + thsnap->ts_rqos_override = MAX(thread->requested_policy.thrp_qos_override, + thread->requested_policy.thrp_qos_workq_override); if (proc_get_effective_thread_policy(thread, TASK_POLICY_DARWIN_BG)) { thsnap->ss_flags |= kThreadDarwinBG; @@ -706,7 +797,7 @@ copytobuffer: if (dqserialnum_valid) { if ((current_buffer->size - current_buffer->current_position) < sizeof(dqserialnum)) { /* wrap and overwrite */ - current_buffer->end_point = current_record_start; + current_buffer->end_point = current_record_start; current_buffer->current_position = 0; if (current_record_start == 0) { /* This sample is too large to fit in the buffer even when we started at 0, so skip it */ @@ -716,11 +807,11 @@ copytobuffer: } thsnap->ss_flags |= kHasDispatchSerial; - bcopy(&dqserialnum, (char *)current_buffer->buffer + current_buffer->current_position, sizeof (dqserialnum)); - current_buffer->current_position += sizeof (dqserialnum); + bcopy(&dqserialnum, (char *)current_buffer->buffer + current_buffer->current_position, sizeof(dqserialnum)); + current_buffer->current_position += sizeof(dqserialnum); } - if (task_has_64BitAddr(task)) { + if (user64_regs) { framesize = 8; thsnap->ss_flags |= kUser64_p; } else { @@ -731,7 +822,7 @@ copytobuffer: * If we can't fit this entire stacktrace then cancel this record, wrap to the beginning, * and start again there so that we always store a full record. */ - if ((current_buffer->size - current_buffer->current_position)/framesize < btcount) { + if ((current_buffer->size - current_buffer->current_position) / framesize < btcount) { current_buffer->end_point = current_record_start; current_buffer->current_position = 0; if (current_record_start == 0) { @@ -741,7 +832,7 @@ copytobuffer: goto copytobuffer; } - for (bti=0; bti < btcount; bti++, current_buffer->current_position += framesize) { + for (bti = 0; bti < btcount; bti++, current_buffer->current_position += framesize) { if (framesize == 8) { *(uint64_t *)(uintptr_t)(current_buffer->buffer + current_buffer->current_position) = frames[bti]; } else { @@ -772,18 +863,18 @@ copytobuffer: } cancel_sample: - TELEMETRY_UNLOCK(); - /* TODO */ - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END, notify, telemetry_bytes_since_last_mark, current_buffer->current_position, current_buffer->end_point, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END, + notify, telemetry_bytes_since_last_mark, + current_buffer->current_position, current_buffer->end_point); if (notify) { telemetry_notify_user(); } if (uuid_info_array != NULL) { - kfree(uuid_info_array, uuid_info_array_size); + kheap_free(KHEAP_TEMP, uuid_info_array, uuid_info_array_size); } } @@ -793,7 +884,7 @@ log_telemetry_output(vm_offset_t buf, uint32_t pos, uint32_t sz) { struct micro_snapshot *p; uint32_t offset; - + printf("Copying out %d bytes of telemetry at offset %d\n", sz, pos); buf += pos; @@ -810,23 +901,26 @@ log_telemetry_output(vm_offset_t buf, uint32_t pos, uint32_t sz) } #endif -int telemetry_gather(user_addr_t buffer, uint32_t *length, boolean_t mark) +int +telemetry_gather(user_addr_t buffer, uint32_t *length, boolean_t mark) { return telemetry_buffer_gather(buffer, length, mark, &telemetry_buffer); } -int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer) +int +telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark, struct micro_snapshot_buffer * current_buffer) { int result = 0; uint32_t oldest_record_offset; - /* TODO */ - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START, mark, telemetry_bytes_since_last_mark, 0, 0, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START, + mark, telemetry_bytes_since_last_mark, 0, + (&telemetry_buffer != current_buffer)); TELEMETRY_LOCK(); if (current_buffer->buffer == 0) { - *length = 0; + *length = 0; goto out; } @@ -843,7 +937,6 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark do { if (((oldest_record_offset + sizeof(uint32_t)) > current_buffer->size) || ((oldest_record_offset + sizeof(uint32_t)) > current_buffer->end_point)) { - if (*(uint32_t *)(uintptr_t)(current_buffer->buffer) == 0) { /* * There is no magic number at the start of the buffer, which means @@ -863,8 +956,9 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark break; } - if (*(uint32_t *)(uintptr_t)(current_buffer->buffer + oldest_record_offset) == STACKSHOT_MICRO_SNAPSHOT_MAGIC) + if (*(uint32_t *)(uintptr_t)(current_buffer->buffer + oldest_record_offset) == STACKSHOT_MICRO_SNAPSHOT_MAGIC) { break; + } /* * There are no alignment guarantees for micro-stackshot records, so we must search at each @@ -880,7 +974,7 @@ int telemetry_buffer_gather(user_addr_t buffer, uint32_t *length, boolean_t mark if (oldest_record_offset != 0) { #if TELEMETRY_DEBUG log_telemetry_output(current_buffer->buffer, oldest_record_offset, - current_buffer->end_point - oldest_record_offset); + current_buffer->end_point - oldest_record_offset); #endif if ((result = copyout((void *)(current_buffer->buffer + oldest_record_offset), buffer, current_buffer->end_point - oldest_record_offset)) != 0) { @@ -910,9 +1004,11 @@ out: TELEMETRY_UNLOCK(); - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END, current_buffer->current_position, *length, current_buffer->end_point, 0, (&telemetry_buffer != current_buffer)); + KDBG(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END, + current_buffer->current_position, *length, + current_buffer->end_point, (&telemetry_buffer != current_buffer)); - return (result); + return result; } /************************/ @@ -944,18 +1040,18 @@ out: #define BOOTPROFILE_MAX_BUFFER_SIZE (64*1024*1024) /* see also COPYSIZELIMIT_PANIC */ -vm_offset_t bootprofile_buffer = 0; -uint32_t bootprofile_buffer_size = 0; -uint32_t bootprofile_buffer_current_position = 0; -uint32_t bootprofile_interval_ms = 0; -uint32_t bootprofile_stackshot_flags = 0; -uint64_t bootprofile_interval_abs = 0; -uint64_t bootprofile_next_deadline = 0; -uint32_t bootprofile_all_procs = 0; -char bootprofile_proc_name[17]; +vm_offset_t bootprofile_buffer = 0; +uint32_t bootprofile_buffer_size = 0; +uint32_t bootprofile_buffer_current_position = 0; +uint32_t bootprofile_interval_ms = 0; +uint64_t bootprofile_stackshot_flags = 0; +uint64_t bootprofile_interval_abs = 0; +uint64_t bootprofile_next_deadline = 0; +uint32_t bootprofile_all_procs = 0; +char bootprofile_proc_name[17]; uint64_t bootprofile_delta_since_timestamp = 0; -lck_grp_t bootprofile_lck_grp; -lck_mtx_t bootprofile_mtx; +LCK_GRP_DECLARE(bootprofile_lck_grp, "bootprofile_group"); +LCK_MTX_DECLARE(bootprofile_mtx, &bootprofile_lck_grp); enum { @@ -965,7 +1061,7 @@ enum { } bootprofile_type = kBootProfileDisabled; -static timer_call_data_t bootprofile_timer_call_entry; +static timer_call_data_t bootprofile_timer_call_entry; #define BOOTPROFILE_LOCK() do { lck_mtx_lock(&bootprofile_mtx); } while(0) #define BOOTPROFILE_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&bootprofile_mtx) @@ -975,30 +1071,33 @@ static void bootprofile_timer_call( timer_call_param_t param0, timer_call_param_t param1); -void bootprofile_init(void) +void +bootprofile_init(void) { kern_return_t ret; char type[32]; - lck_grp_init(&bootprofile_lck_grp, "bootprofile group", LCK_GRP_ATTR_NULL); - lck_mtx_init(&bootprofile_mtx, &bootprofile_lck_grp, LCK_ATTR_NULL); - - if (!PE_parse_boot_argn("bootprofile_buffer_size", &bootprofile_buffer_size, sizeof(bootprofile_buffer_size))) { + if (!PE_parse_boot_argn("bootprofile_buffer_size", + &bootprofile_buffer_size, sizeof(bootprofile_buffer_size))) { bootprofile_buffer_size = 0; } - if (bootprofile_buffer_size > BOOTPROFILE_MAX_BUFFER_SIZE) + if (bootprofile_buffer_size > BOOTPROFILE_MAX_BUFFER_SIZE) { bootprofile_buffer_size = BOOTPROFILE_MAX_BUFFER_SIZE; + } - if (!PE_parse_boot_argn("bootprofile_interval_ms", &bootprofile_interval_ms, sizeof(bootprofile_interval_ms))) { + if (!PE_parse_boot_argn("bootprofile_interval_ms", + &bootprofile_interval_ms, sizeof(bootprofile_interval_ms))) { bootprofile_interval_ms = 0; } - if (!PE_parse_boot_argn("bootprofile_stackshot_flags", &bootprofile_stackshot_flags, sizeof(bootprofile_stackshot_flags))) { + if (!PE_parse_boot_argn("bootprofile_stackshot_flags", + &bootprofile_stackshot_flags, sizeof(bootprofile_stackshot_flags))) { bootprofile_stackshot_flags = 0; } - if (!PE_parse_boot_argn("bootprofile_proc_name", &bootprofile_proc_name, sizeof(bootprofile_proc_name))) { + if (!PE_parse_boot_argn("bootprofile_proc_name", + &bootprofile_proc_name, sizeof(bootprofile_proc_name))) { bootprofile_all_procs = 1; bootprofile_proc_name[0] = '\0'; } @@ -1007,7 +1106,7 @@ void bootprofile_init(void) if (0 == strcmp(type, "boot")) { bootprofile_type = kBootProfileStartTimerAtBoot; } else if (0 == strcmp(type, "wake")) { - bootprofile_type = kBootProfileStartTimerAtWake; + bootprofile_type = kBootProfileStartTimerAtWake; } else { bootprofile_type = kBootProfileDisabled; } @@ -1029,21 +1128,22 @@ void bootprofile_init(void) } bzero((void *) bootprofile_buffer, bootprofile_buffer_size); - kprintf("Boot profile: Sampling %s once per %u ms at %s\n", bootprofile_all_procs ? "all procs" : bootprofile_proc_name, bootprofile_interval_ms, - bootprofile_type == kBootProfileStartTimerAtBoot ? "boot" : (bootprofile_type == kBootProfileStartTimerAtWake ? "wake" : "unknown")); + kprintf("Boot profile: Sampling %s once per %u ms at %s\n", + bootprofile_all_procs ? "all procs" : bootprofile_proc_name, bootprofile_interval_ms, + bootprofile_type == kBootProfileStartTimerAtBoot ? "boot" : (bootprofile_type == kBootProfileStartTimerAtWake ? "wake" : "unknown")); timer_call_setup(&bootprofile_timer_call_entry, - bootprofile_timer_call, - NULL); + bootprofile_timer_call, + NULL); if (bootprofile_type == kBootProfileStartTimerAtBoot) { bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs; timer_call_enter_with_leeway(&bootprofile_timer_call_entry, - NULL, - bootprofile_next_deadline, - 0, - TIMER_CALL_SYS_NORMAL, - FALSE); + NULL, + bootprofile_next_deadline, + 0, + TIMER_CALL_SYS_NORMAL, + FALSE); } } @@ -1053,11 +1153,11 @@ bootprofile_wake_from_sleep(void) if (bootprofile_type == kBootProfileStartTimerAtWake) { bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs; timer_call_enter_with_leeway(&bootprofile_timer_call_entry, - NULL, - bootprofile_next_deadline, - 0, - TIMER_CALL_SYS_NORMAL, - FALSE); + NULL, + bootprofile_next_deadline, + 0, + TIMER_CALL_SYS_NORMAL, + FALSE); } } @@ -1085,8 +1185,7 @@ bootprofile_timer_call( if ((current_task() != NULL) && (current_task()->bsd_info != NULL) && (0 == strncmp(bootprofile_proc_name, proc_name_address(current_task()->bsd_info), 17))) { pid_to_profile = proc_selfpid(); - } - else { + } else { /* * Process-specific boot profiling requested but the on-core process is * something else. Nothing to do here. @@ -1098,19 +1197,18 @@ bootprofile_timer_call( /* initiate a stackshot with whatever portion of the buffer is left */ if (bootprofile_buffer_current_position < bootprofile_buffer_size) { - - uint32_t flags = STACKSHOT_KCDATA_FORMAT | STACKSHOT_TRYLOCK | STACKSHOT_SAVE_LOADINFO - | STACKSHOT_GET_GLOBAL_MEM_STATS; -#if __x86_64__ + uint64_t flags = STACKSHOT_KCDATA_FORMAT | STACKSHOT_TRYLOCK | STACKSHOT_SAVE_LOADINFO + | STACKSHOT_GET_GLOBAL_MEM_STATS; +#if defined(XNU_TARGET_OS_OSX) flags |= STACKSHOT_SAVE_KEXT_LOADINFO; -#endif /* __x86_64__ */ +#endif /* OR on flags specified in boot-args */ flags |= bootprofile_stackshot_flags; if ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) && (bootprofile_delta_since_timestamp == 0)) { /* Can't take deltas until the first one */ - flags &= ~ STACKSHOT_COLLECT_DELTA_SNAPSHOT; + flags &= ~STACKSHOT_COLLECT_DELTA_SNAPSHOT; } uint64_t timestamp = 0; @@ -1119,9 +1217,9 @@ bootprofile_timer_call( } kern_return_t r = stack_snapshot_from_kernel( - pid_to_profile, (void *)(bootprofile_buffer + bootprofile_buffer_current_position), - bootprofile_buffer_size - bootprofile_buffer_current_position, - flags, bootprofile_delta_since_timestamp, &retbytes); + pid_to_profile, (void *)(bootprofile_buffer + bootprofile_buffer_current_position), + bootprofile_buffer_size - bootprofile_buffer_current_position, + flags, bootprofile_delta_since_timestamp, 0, &retbytes); /* * We call with STACKSHOT_TRYLOCK because the stackshot lock is coarser @@ -1134,8 +1232,8 @@ bootprofile_timer_call( goto reprogram; } - if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT && - r == KERN_SUCCESS) { + if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT && + r == KERN_SUCCESS) { bootprofile_delta_since_timestamp = timestamp; } @@ -1157,17 +1255,18 @@ reprogram: } clock_deadline_for_periodic_event(bootprofile_interval_abs, - mach_absolute_time(), - &bootprofile_next_deadline); + mach_absolute_time(), + &bootprofile_next_deadline); timer_call_enter_with_leeway(&bootprofile_timer_call_entry, - NULL, - bootprofile_next_deadline, - 0, - TIMER_CALL_SYS_NORMAL, - FALSE); + NULL, + bootprofile_next_deadline, + 0, + TIMER_CALL_SYS_NORMAL, + FALSE); } -void bootprofile_get(void **buffer, uint32_t *length) +void +bootprofile_get(void **buffer, uint32_t *length) { BOOTPROFILE_LOCK(); *buffer = (void*) bootprofile_buffer; @@ -1175,14 +1274,15 @@ void bootprofile_get(void **buffer, uint32_t *length) BOOTPROFILE_UNLOCK(); } -int bootprofile_gather(user_addr_t buffer, uint32_t *length) +int +bootprofile_gather(user_addr_t buffer, uint32_t *length) { int result = 0; BOOTPROFILE_LOCK(); if (bootprofile_buffer == 0) { - *length = 0; + *length = 0; goto out; } @@ -1205,5 +1305,5 @@ out: BOOTPROFILE_UNLOCK(); - return (result); + return result; }