#include <mach/mach_types.h>
#include <mach/vm_param.h>
+#include <mach/mach_vm.h>
+#include <mach/clock_types.h>
+#include <sys/errno.h>
+#include <sys/stackshot.h>
#ifdef IMPORTANCE_INHERITANCE
#include <ipc/ipc_importance.h>
#endif
#include <sys/appleapiopts.h>
#include <kern/debug.h>
+#include <kern/block_hint.h>
#include <uuid/uuid.h>
#include <kdp/kdp_dyld.h>
#include <kern/processor.h>
#include <kern/thread.h>
+#include <kern/task.h>
+#include <kern/telemetry.h>
#include <kern/clock.h>
+#include <kern/policy_internal.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_pageout.h>
+#include <vm/vm_fault.h>
#include <vm/vm_shared_region.h>
#include <libkern/OSKextLibPrivate.h>
+#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
+#include <pexpert/pexpert.h> /* For gPanicBase/gPanicBase */
+#endif
+
extern unsigned int not_in_kdp;
-/*
- * TODO: Even hackier than the other pieces. This should really
- * be moved off of kdp_pmap, and we should probably separate
- * machine_trace_thread out of the kdp code.
- */
-extern pmap_t kdp_pmap;
+
+/* indicate to the compiler that some accesses are unaligned */
+typedef uint64_t unaligned_u64 __attribute__((aligned(1)));
+
extern addr64_t kdp_vtophys(pmap_t pmap, addr64_t va);
+extern void * proc_get_uthread_uu_threadlist(void * uthread_v);
-int kdp_snapshot = 0;
-static int stack_snapshot_ret = 0;
-static unsigned stack_snapshot_bytes_traced = 0;
+int kdp_snapshot = 0;
+static kern_return_t stack_snapshot_ret = 0;
+static uint32_t stack_snapshot_bytes_traced = 0;
+static kcdata_descriptor_t stackshot_kcdata_p = NULL;
static void *stack_snapshot_buf;
static uint32_t stack_snapshot_bufsize;
int stack_snapshot_pid;
static uint32_t stack_snapshot_flags;
-static uint32_t stack_snapshot_dispatch_offset;
-static unsigned int old_debugger;
-
-void do_stackshot(void);
-void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size,
- uint32_t flags, uint32_t dispatch_offset);
-void kdp_snapshot_postflight(void);
-static int kdp_stackshot(int pid, void *tracebuf, uint32_t tracebuf_size,
- uint32_t flags, uint32_t dispatch_offset, uint32_t *pbytesTraced);
-int kdp_stack_snapshot_geterror(void);
-int kdp_stack_snapshot_bytes_traced(void);
-int kdp_stackshot(int pid, void *tracebuf, uint32_t tracebuf_size, uint32_t trace_flags, uint32_t dispatch_offset, uint32_t *pbytesTraced);
-static int pid_from_task(task_t task);
-static uint64_t proc_uniqueid_from_task(task_t task);
+static uint64_t stack_snapshot_delta_since_timestamp;
+static boolean_t panic_stackshot;
+
+static boolean_t stack_enable_faulting = FALSE;
+static struct stackshot_fault_stats fault_stats;
+
+static unaligned_u64 * stackshot_duration_outer;
+static uint64_t stackshot_microsecs;
+
+void * kernel_stackshot_buf = NULL; /* Pointer to buffer for stackshots triggered from the kernel and retrieved later */
+int kernel_stackshot_buf_size = 0;
+
+void * stackshot_snapbuf = NULL; /* Used by stack_snapshot2 (to be removed) */
+
+__private_extern__ void stackshot_init( void );
+static boolean_t memory_iszero(void *addr, size_t size);
+#if CONFIG_TELEMETRY
+kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
+#endif
+uint32_t get_stackshot_estsize(uint32_t prev_size_hint);
+kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config,
+ size_t stackshot_config_size, boolean_t stackshot_from_user);
+kern_return_t do_stackshot(void *);
+void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp);
+boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
+static int kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t *pBytesTraced);
+kern_return_t kdp_stack_snapshot_geterror(void);
+uint32_t kdp_stack_snapshot_bytes_traced(void);
static void kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap);
-static boolean_t kdp_copyin(pmap_t p, uint64_t uaddr, void *dest, size_t size);
+static boolean_t kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_result);
+static boolean_t kdp_copyin_word(task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results);
static uint64_t proc_was_throttled_from_task(task_t task);
+static void stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t * waitinfo);
+static int stackshot_thread_has_valid_waitinfo(thread_t thread);
+
+extern uint32_t workqueue_get_pwq_state_kdp(void *proc);
extern int proc_pid(void *p);
extern uint64_t proc_uniqueid(void *p);
extern uint64_t proc_was_throttled(void *p);
extern uint64_t proc_did_throttle(void *p);
-static uint64_t proc_did_throttle_from_task(task_t task);
-extern void proc_name_kdp(task_t task, char *buf, int size);
-extern int proc_threadname_kdp(void *uth, char *buf, size_t size);
-extern void proc_starttime_kdp(void *p, uint64_t *tv_sec, uint64_t *tv_usec);
-
-extern int count_busy_buffers(void); /* must track with declaration in bsd/sys/buf_internal.h */
-extern void bcopy_phys(addr64_t, addr64_t, vm_size_t);
-extern int machine_trace_thread(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
-extern int machine_trace_thread64(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
-
-/* Validates that the given address is both a valid page and has
- * default caching attributes for the current kdp_pmap. Returns
+static uint64_t proc_did_throttle_from_task(task_t task);
+extern void proc_name_kdp(task_t task, char * buf, int size);
+extern int proc_threadname_kdp(void * uth, char * buf, size_t size);
+extern void proc_starttime_kdp(void * p, uint64_t * tv_sec, uint64_t * tv_usec, uint64_t * abstime);
+extern int memorystatus_get_pressure_status_kdp(void);
+extern boolean_t memorystatus_proc_is_dirty_unsafe(void * v);
+
+extern int count_busy_buffers(void); /* must track with declaration in bsd/sys/buf_internal.h */
+extern void bcopy_phys(addr64_t, addr64_t, vm_size_t);
+
+#if CONFIG_TELEMETRY
+extern kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
+#endif /* CONFIG_TELEMETRY */
+
+extern kern_return_t kern_stack_snapshot_with_reason(char* reason);
+extern kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user);
+/* Used for stackshot_thread_waitinfo_unsafe */
+extern void kdp_lck_mtx_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
+extern void kdp_sema_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
+extern void kdp_mqueue_send_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
+extern void kdp_mqueue_recv_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
+extern void kdp_ulock_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
+extern void kdp_rwlck_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
+extern void kdp_pthread_find_owner(thread_t thread, thread_waitinfo_t *waitinfo);
+extern void *kdp_pthread_get_thread_kwq(thread_t thread);
+
+/*
+ * Validates that the given address is both a valid page and has
+ * default caching attributes for the current map. Returns
* 0 if the address is invalid, and a kernel virtual address for
* the given address if it is valid.
*/
-vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr);
+vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags);
+
+#define KDP_FAULT_RESULT_PAGED_OUT 0x1 /* some data was unable to be retrieved */
+#define KDP_FAULT_RESULT_TRIED_FAULT 0x2 /* tried to fault in data */
+#define KDP_FAULT_RESULT_FAULTED_IN 0x4 /* successfully faulted in data */
+
+/*
+ * Looks up the physical translation for the given address in the target map, attempting
+ * to fault data in if requested and it is not resident. Populates thread_trace_flags if requested
+ * as well.
+ */
+vm_offset_t kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results);
+
+static size_t stackshot_strlcpy(char *dst, const char *src, size_t maxlen);
+static void stackshot_memcpy(void *dst, const void *src, size_t len);
/* Clears caching information used by the above validation routine
- * (in case the kdp_pmap has been changed or cleared).
+ * (in case the current map has been changed or cleared).
*/
void machine_trace_thread_clear_validation_cache(void);
#define MAX_FRAMES 1000
+#define MAX_LOADINFOS 500
+#define TASK_IMP_WALK_LIMIT 20
typedef struct thread_snapshot *thread_snapshot_t;
typedef struct task_snapshot *task_snapshot_t;
static vm_offset_t prev_target_kva = 0;
static boolean_t validate_next_addr = TRUE;
+/*
+ * Stackshot locking and other defines.
+ */
+static lck_grp_t *stackshot_subsys_lck_grp;
+static lck_grp_attr_t *stackshot_subsys_lck_grp_attr;
+static lck_attr_t *stackshot_subsys_lck_attr;
+static lck_mtx_t stackshot_subsys_mutex;
+
+#define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
+#define STACKSHOT_SUBSYS_TRY_LOCK() lck_mtx_try_lock(&stackshot_subsys_mutex)
+#define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
+
+#define SANE_BOOTPROFILE_TRACEBUF_SIZE (64 * 1024 * 1024)
+#define SANE_TRACEBUF_SIZE (8 * 1024 * 1024)
+
+/*
+ * We currently set a ceiling of 3 milliseconds spent in the kdp fault path
+ * for non-panic stackshots where faulting is requested.
+ */
+#define KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS (3 * NSEC_PER_MSEC)
+
+#define STACKSHOT_SUPP_SIZE (16 * 1024) /* Minimum stackshot size */
+#define TASK_UUID_AVG_SIZE (16 * sizeof(uuid_t)) /* Average space consumed by UUIDs/task */
+
+/*
+ * Initialize the mutex governing access to the stack snapshot subsystem
+ * and other stackshot related bits.
+ */
+__private_extern__ void
+stackshot_init( void )
+{
+ mach_timebase_info_data_t timebase;
+
+ stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
+
+ stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
+
+ stackshot_subsys_lck_attr = lck_attr_alloc_init();
+
+ lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
+
+ clock_timebase_info(&timebase);
+ fault_stats.sfs_system_max_fault_time = ((KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS * timebase.denom)/ timebase.numer);
+}
/*
* Method for grabbing timer values safely, in the sense that no infinite loop will occur
#endif
}
+kern_return_t
+stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, uint64_t delta_since_timestamp, unsigned *bytes_traced)
+{
+ kern_return_t error = KERN_SUCCESS;
+ boolean_t istate;
+
+ if ((buf == NULL) || (size <= 0) || (bytes_traced == NULL)) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ /* cap in individual stackshot to SANE_TRACEBUF_SIZE */
+ if (size > SANE_TRACEBUF_SIZE) {
+ size = SANE_TRACEBUF_SIZE;
+ }
+
+ /* Serialize tracing */
+ if (flags & STACKSHOT_TRYLOCK) {
+ if (!STACKSHOT_SUBSYS_TRY_LOCK()) {
+ return KERN_LOCK_OWNED;
+ }
+ } else {
+ STACKSHOT_SUBSYS_LOCK();
+ }
+
+ istate = ml_set_interrupts_enabled(FALSE);
+
+ struct kcdata_descriptor kcdata;
+ uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ?
+ KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
+
+ error = kcdata_memory_static_init(&kcdata, (mach_vm_address_t)buf, hdr_tag, size,
+ KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
+ if (error) {
+ goto out;
+ }
+
+ /* Preload trace parameters*/
+ kdp_snapshot_preflight(pid, buf, size, flags, &kcdata, delta_since_timestamp);
+
+ /* Trap to the debugger to obtain a coherent stack snapshot; this populates
+ * the trace buffer
+ */
+ stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
+
+ ml_set_interrupts_enabled(istate);
+
+ *bytes_traced = kdp_stack_snapshot_bytes_traced();
+
+ error = kdp_stack_snapshot_geterror();
+
+out:
+ STACKSHOT_SUBSYS_UNLOCK();
+ return error;
+}
+
+#if CONFIG_TELEMETRY
+kern_return_t
+stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval)
+{
+ int error = KERN_SUCCESS;
+ uint32_t bytes_traced = 0;
+
+ *retval = -1;
+
+ /*
+ * Control related operations
+ */
+ if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_ENABLE) {
+ telemetry_global_ctl(1);
+ *retval = 0;
+ goto exit;
+ } else if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_DISABLE) {
+ telemetry_global_ctl(0);
+ *retval = 0;
+ goto exit;
+ }
+
+ /*
+ * Data related operations
+ */
+ *retval = -1;
+
+ if ((((void*)tracebuf) == NULL) || (tracebuf_size == 0)) {
+ error = KERN_INVALID_ARGUMENT;
+ goto exit;
+ }
+
+ STACKSHOT_SUBSYS_LOCK();
+
+ if (flags & STACKSHOT_GET_MICROSTACKSHOT) {
+ if (tracebuf_size > SANE_TRACEBUF_SIZE) {
+ error = KERN_INVALID_ARGUMENT;
+ goto unlock_exit;
+ }
+
+ bytes_traced = tracebuf_size;
+ error = telemetry_gather(tracebuf, &bytes_traced,
+ (flags & STACKSHOT_SET_MICROSTACKSHOT_MARK) ? TRUE : FALSE);
+ *retval = (int)bytes_traced;
+ goto unlock_exit;
+ }
+
+ if (flags & STACKSHOT_GET_BOOT_PROFILE) {
+
+ if (tracebuf_size > SANE_BOOTPROFILE_TRACEBUF_SIZE) {
+ error = KERN_INVALID_ARGUMENT;
+ goto unlock_exit;
+ }
+
+ bytes_traced = tracebuf_size;
+ error = bootprofile_gather(tracebuf, &bytes_traced);
+ *retval = (int)bytes_traced;
+ }
+
+unlock_exit:
+ STACKSHOT_SUBSYS_UNLOCK();
+exit:
+ return error;
+}
+#endif /* CONFIG_TELEMETRY */
+
+/*
+ * Return the estimated size of a stackshot based on the
+ * number of currently running threads and tasks.
+ */
+uint32_t
+get_stackshot_estsize(uint32_t prev_size_hint)
+{
+ vm_size_t thread_total;
+ vm_size_t task_total;
+ uint32_t estimated_size;
+
+ thread_total = (threads_count * sizeof(struct thread_snapshot));
+ task_total = (tasks_count * (sizeof(struct task_snapshot) + TASK_UUID_AVG_SIZE));
+
+ estimated_size = (uint32_t) VM_MAP_ROUND_PAGE((thread_total + task_total + STACKSHOT_SUPP_SIZE), PAGE_MASK);
+ if (estimated_size < prev_size_hint) {
+ estimated_size = (uint32_t) VM_MAP_ROUND_PAGE(prev_size_hint, PAGE_MASK);
+ }
+
+ return estimated_size;
+}
+
+/*
+ * stackshot_remap_buffer: Utility function to remap bytes_traced bytes starting at stackshotbuf
+ * into the current task's user space and subsequently copy out the address
+ * at which the buffer has been mapped in user space to out_buffer_addr.
+ *
+ * Inputs: stackshotbuf - pointer to the original buffer in the kernel's address space
+ * bytes_traced - length of the buffer to remap starting from stackshotbuf
+ * out_buffer_addr - pointer to placeholder where newly mapped buffer will be mapped.
+ * out_size_addr - pointer to be filled in with the size of the buffer
+ *
+ * Outputs: ENOSPC if there is not enough free space in the task's address space to remap the buffer
+ * EINVAL for all other errors returned by task_remap_buffer/mach_vm_remap
+ * an error from copyout
+ */
+static kern_return_t
+stackshot_remap_buffer(void *stackshotbuf, uint32_t bytes_traced, uint64_t out_buffer_addr, uint64_t out_size_addr)
+{
+ int error = 0;
+ mach_vm_offset_t stackshotbuf_user_addr = (mach_vm_offset_t)NULL;
+ vm_prot_t cur_prot, max_prot;
+
+ error = mach_vm_remap(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0,
+ VM_FLAGS_ANYWHERE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT);
+ /*
+ * If the call to mach_vm_remap fails, we return the appropriate converted error
+ */
+ if (error == KERN_SUCCESS) {
+ /*
+ * If we fail to copy out the address or size of the new buffer, we remove the buffer mapping that
+ * we just made in the task's user space.
+ */
+ error = copyout(CAST_DOWN(void *, &stackshotbuf_user_addr), (user_addr_t)out_buffer_addr, sizeof(stackshotbuf_user_addr));
+ if (error != KERN_SUCCESS) {
+ mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
+ return error;
+ }
+ error = copyout(&bytes_traced, (user_addr_t)out_size_addr, sizeof(bytes_traced));
+ if (error != KERN_SUCCESS) {
+ mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
+ return error;
+ }
+ }
+ return error;
+}
+
+kern_return_t
+kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user)
+{
+ int error = 0;
+ boolean_t prev_interrupt_state;
+ uint32_t bytes_traced = 0;
+ uint32_t stackshotbuf_size = 0;
+ void * stackshotbuf = NULL;
+ kcdata_descriptor_t kcdata_p = NULL;
+
+ void * buf_to_free = NULL;
+ int size_to_free = 0;
+
+ /* Parsed arguments */
+ uint64_t out_buffer_addr;
+ uint64_t out_size_addr;
+ int pid = -1;
+ uint32_t flags;
+ uint64_t since_timestamp;
+ uint32_t size_hint = 0;
+
+ if(stackshot_config == NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ switch (stackshot_config_version) {
+ case STACKSHOT_CONFIG_TYPE:
+ if (stackshot_config_size != sizeof(stackshot_config_t)) {
+ return KERN_INVALID_ARGUMENT;
+ }
+ stackshot_config_t *config = (stackshot_config_t *) stackshot_config;
+ out_buffer_addr = config->sc_out_buffer_addr;
+ out_size_addr = config->sc_out_size_addr;
+ pid = config->sc_pid;
+ flags = config->sc_flags;
+ since_timestamp = config->sc_delta_timestamp;
+ if (config->sc_size <= SANE_TRACEBUF_SIZE) {
+ size_hint = config->sc_size;
+ }
+ break;
+ default:
+ return KERN_NOT_SUPPORTED;
+ }
+
+ /*
+ * Currently saving a kernel buffer and trylock are only supported from the
+ * internal/KEXT API.
+ */
+ if (stackshot_from_user) {
+ if (flags & (STACKSHOT_TRYLOCK | STACKSHOT_SAVE_IN_KERNEL_BUFFER | STACKSHOT_FROM_PANIC)) {
+ return KERN_NO_ACCESS;
+ }
+ } else {
+ if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
+ return KERN_NOT_SUPPORTED;
+ }
+ }
+
+ if (!((flags & STACKSHOT_KCDATA_FORMAT) || (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER))) {
+ return KERN_NOT_SUPPORTED;
+ }
+
+ /*
+ * If we're not saving the buffer in the kernel pointer, we need a place to copy into.
+ */
+ if ((!out_buffer_addr || !out_size_addr) && !(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (since_timestamp != 0 && ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) == 0)) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ STACKSHOT_SUBSYS_LOCK();
+
+ if (flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER) {
+ /*
+ * Don't overwrite an existing stackshot
+ */
+ if (kernel_stackshot_buf != NULL) {
+ error = KERN_MEMORY_PRESENT;
+ goto error_exit;
+ }
+ } else if (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER) {
+ if ((kernel_stackshot_buf == NULL) || (kernel_stackshot_buf_size <= 0)) {
+ error = KERN_NOT_IN_SET;
+ goto error_exit;
+ }
+ error = stackshot_remap_buffer(kernel_stackshot_buf, kernel_stackshot_buf_size,
+ out_buffer_addr, out_size_addr);
+ /*
+ * If we successfully remapped the buffer into the user's address space, we
+ * set buf_to_free and size_to_free so the prior kernel mapping will be removed
+ * and then clear the kernel stackshot pointer and associated size.
+ */
+ if (error == KERN_SUCCESS) {
+ buf_to_free = kernel_stackshot_buf;
+ size_to_free = (int) VM_MAP_ROUND_PAGE(kernel_stackshot_buf_size, PAGE_MASK);
+ kernel_stackshot_buf = NULL;
+ kernel_stackshot_buf_size = 0;
+ }
+
+ goto error_exit;
+ }
+
+ if (flags & STACKSHOT_GET_BOOT_PROFILE) {
+ void *bootprofile = NULL;
+ uint32_t len = 0;
+#if CONFIG_TELEMETRY
+ bootprofile_get(&bootprofile, &len);
+#endif
+ if (!bootprofile || !len) {
+ error = KERN_NOT_IN_SET;
+ goto error_exit;
+ }
+ error = stackshot_remap_buffer(bootprofile, len, out_buffer_addr, out_size_addr);
+ goto error_exit;
+ }
+
+ stackshotbuf_size = get_stackshot_estsize(size_hint);
+
+ for (; stackshotbuf_size <= SANE_TRACEBUF_SIZE; stackshotbuf_size <<= 1) {
+ if (kmem_alloc(kernel_map, (vm_offset_t *)&stackshotbuf, stackshotbuf_size, VM_KERN_MEMORY_DIAG) != KERN_SUCCESS) {
+ error = KERN_RESOURCE_SHORTAGE;
+ goto error_exit;
+ }
+
+ /*
+ * If someone has panicked, don't try and enter the debugger
+ */
+ if (panic_active()) {
+ error = KERN_RESOURCE_SHORTAGE;
+ goto error_exit;
+ }
+
+ uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
+ kcdata_p = kcdata_memory_alloc_init((mach_vm_address_t)stackshotbuf, hdr_tag, stackshotbuf_size,
+ KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
+
+ stackshot_duration_outer = NULL;
+ uint64_t time_start = mach_absolute_time();
+
+ /*
+ * Disable interrupts and save the current interrupt state.
+ */
+ prev_interrupt_state = ml_set_interrupts_enabled(FALSE);
+
+ /*
+ * Load stackshot parameters.
+ */
+ kdp_snapshot_preflight(pid, stackshotbuf, stackshotbuf_size, flags, kcdata_p, since_timestamp);
+
+ /*
+ * Trap to the debugger to obtain a stackshot (this will populate the buffer).
+ */
+ stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
+
+ ml_set_interrupts_enabled(prev_interrupt_state);
+
+ /* record the duration that interupts were disabled */
+
+ uint64_t time_end = mach_absolute_time();
+ if (stackshot_duration_outer) {
+ *stackshot_duration_outer = time_end - time_start;
+ }
+
+ error = kdp_stack_snapshot_geterror();
+ if (error != KERN_SUCCESS) {
+ if (kcdata_p != NULL) {
+ kcdata_memory_destroy(kcdata_p);
+ kcdata_p = NULL;
+ stackshot_kcdata_p = NULL;
+ }
+ kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
+ stackshotbuf = NULL;
+ if (error == KERN_INSUFFICIENT_BUFFER_SIZE) {
+ /*
+ * If we didn't allocate a big enough buffer, deallocate and try again.
+ */
+ continue;
+ } else {
+ goto error_exit;
+ }
+ }
+
+ bytes_traced = kdp_stack_snapshot_bytes_traced();
+
+ if (bytes_traced <= 0) {
+ error = KERN_ABORTED;
+ goto error_exit;
+ }
+
+ assert(bytes_traced <= stackshotbuf_size);
+ if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
+ error = stackshot_remap_buffer(stackshotbuf, bytes_traced, out_buffer_addr, out_size_addr);
+ goto error_exit;
+ }
+
+ /*
+ * Save the stackshot in the kernel buffer.
+ */
+ kernel_stackshot_buf = stackshotbuf;
+ kernel_stackshot_buf_size = bytes_traced;
+ /*
+ * Figure out if we didn't use all the pages in the buffer. If so, we set buf_to_free to the beginning of
+ * the next page after the end of the stackshot in the buffer so that the kmem_free clips the buffer and
+ * update size_to_free for kmem_free accordingly.
+ */
+ size_to_free = stackshotbuf_size - (int) VM_MAP_ROUND_PAGE(bytes_traced, PAGE_MASK);
+
+ assert(size_to_free >= 0);
+
+ if (size_to_free != 0) {
+ buf_to_free = (void *)((uint64_t)stackshotbuf + stackshotbuf_size - size_to_free);
+ }
+
+ stackshotbuf = NULL;
+ stackshotbuf_size = 0;
+ goto error_exit;
+ }
+
+ if (stackshotbuf_size > SANE_TRACEBUF_SIZE) {
+ error = KERN_RESOURCE_SHORTAGE;
+ }
+
+error_exit:
+ if (kcdata_p != NULL) {
+ kcdata_memory_destroy(kcdata_p);
+ kcdata_p = NULL;
+ stackshot_kcdata_p = NULL;
+ }
+
+ if (stackshotbuf != NULL) {
+ kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
+ }
+ if (buf_to_free != NULL) {
+ kmem_free(kernel_map, (vm_offset_t)buf_to_free, size_to_free);
+ }
+ STACKSHOT_SUBSYS_UNLOCK();
+ return error;
+}
+
/* Cache stack snapshot parameters in preparation for a trace */
void
-kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, uint32_t dispatch_offset)
+kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags,
+ kcdata_descriptor_t data_p, uint64_t since_timestamp)
{
+ uint64_t microsecs = 0, secs = 0;
+ clock_get_calendar_microtime((clock_sec_t *)&secs, (clock_usec_t *)µsecs);
+
+ stackshot_microsecs = microsecs + (secs * USEC_PER_SEC);
stack_snapshot_pid = pid;
stack_snapshot_buf = tracebuf;
stack_snapshot_bufsize = tracebuf_size;
stack_snapshot_flags = flags;
- stack_snapshot_dispatch_offset = dispatch_offset;
- kdp_snapshot++;
- /* Mark this debugger as active, since the polled mode driver that
- * ordinarily does this may not be enabled (yet), or since KDB may be
- * the primary debugger.
+ stack_snapshot_delta_since_timestamp = since_timestamp;
+
+ panic_stackshot = ((flags & STACKSHOT_FROM_PANIC) != 0);
+
+ if (data_p != NULL) {
+ stackshot_kcdata_p = data_p;
+ }
+}
+
+kern_return_t
+kdp_stack_snapshot_geterror(void)
+{
+ return stack_snapshot_ret;
+}
+
+uint32_t
+kdp_stack_snapshot_bytes_traced(void)
+{
+ return stack_snapshot_bytes_traced;
+}
+
+static boolean_t memory_iszero(void *addr, size_t size)
+{
+ char *data = (char *)addr;
+ for (size_t i = 0; i < size; i++){
+ if (data[i] != 0)
+ return FALSE;
+ }
+ return TRUE;
+}
+
+#define kcd_end_address(kcd) ((void *)((uint64_t)((kcd)->kcd_addr_begin) + kcdata_memory_get_used_bytes((kcd))))
+#define kcd_max_address(kcd) ((void *)((kcd)->kcd_addr_begin + (kcd)->kcd_length))
+/*
+ * Use of the kcd_exit_on_error(action) macro requires a local
+ * 'kern_return_t error' variable and 'error_exit' label.
+ */
+#define kcd_exit_on_error(action) \
+ do { \
+ if (KERN_SUCCESS != (error = (action))) { \
+ if (error == KERN_RESOURCE_SHORTAGE) { \
+ error = KERN_INSUFFICIENT_BUFFER_SIZE; \
+ } \
+ goto error_exit; \
+ } \
+ } while (0); /* end kcd_exit_on_error */
+
+static uint64_t
+kcdata_get_task_ss_flags(task_t task)
+{
+ uint64_t ss_flags = 0;
+ boolean_t task64 = task_has_64BitAddr(task);
+
+ if (task64)
+ ss_flags |= kUser64_p;
+ if (!task->active || task_is_a_corpse(task))
+ ss_flags |= kTerminatedSnapshot;
+ if (task->pidsuspended)
+ ss_flags |= kPidSuspended;
+ if (task->frozen)
+ ss_flags |= kFrozen;
+ if (task->effective_policy.tep_darwinbg == 1)
+ ss_flags |= kTaskDarwinBG;
+ if (task->requested_policy.trp_role == TASK_FOREGROUND_APPLICATION)
+ ss_flags |= kTaskIsForeground;
+ if (task->requested_policy.trp_boosted == 1)
+ ss_flags |= kTaskIsBoosted;
+ if (task->effective_policy.tep_sup_active == 1)
+ ss_flags |= kTaskIsSuppressed;
+#if CONFIG_MEMORYSTATUS
+ if (memorystatus_proc_is_dirty_unsafe(task->bsd_info))
+ ss_flags |= kTaskIsDirty;
+#endif
+
+ ss_flags |= (0x7 & workqueue_get_pwq_state_kdp(task->bsd_info)) << 17;
+
+#if IMPORTANCE_INHERITANCE
+ if (task->task_imp_base) {
+ if (task->task_imp_base->iit_donor)
+ ss_flags |= kTaskIsImpDonor;
+ if (task->task_imp_base->iit_live_donor)
+ ss_flags |= kTaskIsLiveImpDonor;
+ }
+#endif
+
+ return ss_flags;
+}
+
+static kern_return_t
+kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyld_uuid_info_64_v2 *sys_shared_cache_loadinfo, uint32_t trace_flags, unaligned_u64 *task_snap_ss_flags)
+{
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+
+ uint8_t shared_cache_identifier[16];
+ uint64_t shared_cache_slide = 0;
+ uint64_t shared_cache_base_address = 0;
+ int task_pid = pid_from_task(task);
+ boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
+ uint32_t kdp_fault_results = 0;
+
+ assert(task_snap_ss_flags != NULL);
+
+ if (task->shared_region && ml_validate_nofault((vm_offset_t)task->shared_region, sizeof(struct vm_shared_region))) {
+ struct vm_shared_region *sr = task->shared_region;
+ shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping;
+ } else {
+ *task_snap_ss_flags |= kTaskSharedRegionInfoUnavailable;
+ }
+
+ if (!shared_cache_base_address ||
+ !kdp_copyin(task->map, shared_cache_base_address + offsetof(struct _dyld_cache_header, uuid),
+ shared_cache_identifier, sizeof(shared_cache_identifier), should_fault, &kdp_fault_results)) {
+ goto error_exit;
+ }
+
+ if (task->shared_region) {
+ /*
+ * No refcounting here, but we are in debugger
+ * context, so that should be safe.
+ */
+ shared_cache_slide = task->shared_region->sr_slide_info.slide;
+ } else {
+ shared_cache_slide = 0;
+ }
+
+ if (sys_shared_cache_loadinfo) {
+ if (task_pid == 1) {
+ /* save launchd's shared cache info as system level */
+ stackshot_memcpy(sys_shared_cache_loadinfo->imageUUID, shared_cache_identifier, sizeof(sys_shared_cache_loadinfo->imageUUID));
+ sys_shared_cache_loadinfo->imageLoadAddress = shared_cache_slide;
+ sys_shared_cache_loadinfo->imageSlidBaseAddress = shared_cache_slide + task->shared_region->sr_base_address;
+
+ goto error_exit;
+ } else {
+ if (shared_cache_slide == sys_shared_cache_loadinfo->imageLoadAddress &&
+ 0 == memcmp(shared_cache_identifier, sys_shared_cache_loadinfo->imageUUID,
+ sizeof(sys_shared_cache_loadinfo->imageUUID))) {
+ /* skip adding shared cache info. its same as system level one */
+ goto error_exit;
+ }
+ }
+ }
+
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64_v2), &out_addr));
+ struct dyld_uuid_info_64_v2 *shared_cache_data = (struct dyld_uuid_info_64_v2 *)out_addr;
+ shared_cache_data->imageLoadAddress = shared_cache_slide;
+ stackshot_memcpy(shared_cache_data->imageUUID, shared_cache_identifier, sizeof(shared_cache_data->imageUUID));
+ shared_cache_data->imageSlidBaseAddress = shared_cache_base_address;
+
+error_exit:
+ if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
+ *task_snap_ss_flags |= kTaskUUIDInfoMissing;
+ }
+
+ if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
+ *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
+ }
+
+ if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
+ *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
+ }
+
+ return error;
+}
+
+static kern_return_t
+kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 *task_snap_ss_flags)
+{
+ boolean_t save_loadinfo_p = ((trace_flags & STACKSHOT_SAVE_LOADINFO) != 0);
+ boolean_t save_kextloadinfo_p = ((trace_flags & STACKSHOT_SAVE_KEXT_LOADINFO) != 0);
+ boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
+ boolean_t minimize_uuids = collect_delta_stackshot && ((trace_flags & STACKSHOT_TAILSPIN) != 0);
+ boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
+
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+
+ uint32_t uuid_info_count = 0;
+ mach_vm_address_t uuid_info_addr = 0;
+ uint64_t uuid_info_timestamp = 0;
+ uint32_t kdp_fault_results = 0;
+
+ assert(task_snap_ss_flags != NULL);
+
+ int task_pid = pid_from_task(task);
+ boolean_t task64 = task_has_64BitAddr(task);
+
+ if (save_loadinfo_p && have_pmap && task->active && task_pid > 0) {
+ /* Read the dyld_all_image_infos struct from the task memory to get UUID array count and location */
+ if (task64) {
+ struct user64_dyld_all_image_infos task_image_infos;
+ if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
+ sizeof(struct user64_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
+ uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
+ uuid_info_addr = task_image_infos.uuidArray;
+ if (task_image_infos.version >= 15) {
+ uuid_info_timestamp = task_image_infos.timestamp;
+ }
+ }
+ } else {
+ struct user32_dyld_all_image_infos task_image_infos;
+ if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
+ sizeof(struct user32_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
+ uuid_info_count = task_image_infos.uuidArrayCount;
+ uuid_info_addr = task_image_infos.uuidArray;
+ if (task_image_infos.version >= 15) {
+ uuid_info_timestamp = task_image_infos.timestamp;
+ }
+ }
+ }
+
+ /*
+ * If we get a NULL uuid_info_addr (which can happen when we catch dyld in the middle of updating
+ * this data structure), we zero the uuid_info_count so that we won't even try to save load info
+ * for this task.
+ */
+ if (!uuid_info_addr) {
+ uuid_info_count = 0;
+ }
+ }
+
+ if (have_pmap && task_pid == 0) {
+ if (save_kextloadinfo_p && ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader))) {
+ uuid_info_count = gLoadedKextSummaries->numSummaries + 1; /* include main kernel UUID */
+ } else {
+ uuid_info_count = 1; /* include kernelcache UUID (embedded) or kernel UUID (desktop) */
+ }
+ }
+
+ if (task_pid > 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
+ if (minimize_uuids && uuid_info_timestamp != 0 && uuid_info_timestamp < stack_snapshot_delta_since_timestamp)
+ goto error_exit;
+
+ uint32_t uuid_info_size = (uint32_t)(task64 ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
+ uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
+
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, (task64 ? KCDATA_TYPE_LIBRARY_LOADINFO64 : KCDATA_TYPE_LIBRARY_LOADINFO),
+ uuid_info_size, uuid_info_count, &out_addr));
+
+ /* Copy in the UUID info array
+ * It may be nonresident, in which case just fix up nloadinfos to 0 in the task_snap
+ */
+ if (have_pmap && !kdp_copyin(task->map, uuid_info_addr, (void *)out_addr, uuid_info_array_size, should_fault, &kdp_fault_results)) {
+ bzero((void *)out_addr, uuid_info_array_size);
+ }
+
+ } else if (task_pid == 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
+ if (minimize_uuids && gLoadedKextSummaries != 0 && gLoadedKextSummariesTimestamp < stack_snapshot_delta_since_timestamp)
+ goto error_exit;
+
+ uintptr_t image_load_address;
+
+ do {
+
+
+ if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) {
+ /* Kernel UUID not found or inaccessible */
+ break;
+ }
+
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(
+ kcd, (sizeof(kernel_uuid_info) == sizeof(struct user64_dyld_uuid_info)) ? KCDATA_TYPE_LIBRARY_LOADINFO64
+ : KCDATA_TYPE_LIBRARY_LOADINFO,
+ sizeof(kernel_uuid_info), uuid_info_count, &out_addr));
+ kernel_uuid_info *uuid_info_array = (kernel_uuid_info *)out_addr;
+ image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(vm_kernel_stext);
+ uuid_info_array[0].imageLoadAddress = image_load_address;
+ stackshot_memcpy(&uuid_info_array[0].imageUUID, kernel_uuid, sizeof(uuid_t));
+
+ if (save_kextloadinfo_p &&
+ ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader)) &&
+ ml_validate_nofault((vm_offset_t)(&gLoadedKextSummaries->summaries[0]),
+ gLoadedKextSummaries->entry_size * gLoadedKextSummaries->numSummaries)) {
+ uint32_t kexti;
+ for (kexti=0 ; kexti < gLoadedKextSummaries->numSummaries; kexti++) {
+ image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(gLoadedKextSummaries->summaries[kexti].address);
+ uuid_info_array[kexti + 1].imageLoadAddress = image_load_address;
+ stackshot_memcpy(&uuid_info_array[kexti + 1].imageUUID, &gLoadedKextSummaries->summaries[kexti].uuid, sizeof(uuid_t));
+ }
+ }
+ } while(0);
+ }
+
+error_exit:
+ if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
+ *task_snap_ss_flags |= kTaskUUIDInfoMissing;
+ }
+
+ if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
+ *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
+ }
+
+ if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
+ *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
+ }
+
+ return error;
+}
+
+static kern_return_t
+kcdata_record_task_iostats(kcdata_descriptor_t kcd, task_t task)
+{
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+
+ /* I/O Statistics if any counters are non zero */
+ assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
+ if (task->task_io_stats && !memory_iszero(task->task_io_stats, sizeof(struct io_stat_info))) {
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
+ struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
+ _iostat->ss_disk_reads_count = task->task_io_stats->disk_reads.count;
+ _iostat->ss_disk_reads_size = task->task_io_stats->disk_reads.size;
+ _iostat->ss_disk_writes_count = (task->task_io_stats->total_io.count - task->task_io_stats->disk_reads.count);
+ _iostat->ss_disk_writes_size = (task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size);
+ _iostat->ss_paging_count = task->task_io_stats->paging.count;
+ _iostat->ss_paging_size = task->task_io_stats->paging.size;
+ _iostat->ss_non_paging_count = (task->task_io_stats->total_io.count - task->task_io_stats->paging.count);
+ _iostat->ss_non_paging_size = (task->task_io_stats->total_io.size - task->task_io_stats->paging.size);
+ _iostat->ss_metadata_count = task->task_io_stats->metadata.count;
+ _iostat->ss_metadata_size = task->task_io_stats->metadata.size;
+ _iostat->ss_data_count = (task->task_io_stats->total_io.count - task->task_io_stats->metadata.count);
+ _iostat->ss_data_size = (task->task_io_stats->total_io.size - task->task_io_stats->metadata.size);
+ for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
+ _iostat->ss_io_priority_count[i] = task->task_io_stats->io_priority[i].count;
+ _iostat->ss_io_priority_size[i] = task->task_io_stats->io_priority[i].size;
+ }
+ }
+
+error_exit:
+ return error;
+}
+
+static kern_return_t
+kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 **task_snap_ss_flags)
+{
+ boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
+ boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
+
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+ struct task_snapshot_v2 * cur_tsnap = NULL;
+
+ assert(task_snap_ss_flags != NULL);
+
+ int task_pid = pid_from_task(task);
+ uint64_t task_uniqueid = get_task_uniqueid(task);
+ uint64_t proc_starttime_secs = 0;
+
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_SNAPSHOT, sizeof(struct task_snapshot_v2), &out_addr));
+
+ cur_tsnap = (struct task_snapshot_v2 *)out_addr;
+
+ cur_tsnap->ts_unique_pid = task_uniqueid;
+ cur_tsnap->ts_ss_flags = kcdata_get_task_ss_flags(task);
+ *task_snap_ss_flags = (unaligned_u64 *)&cur_tsnap->ts_ss_flags;
+ cur_tsnap->ts_user_time_in_terminated_threads = task->total_user_time;
+ cur_tsnap->ts_system_time_in_terminated_threads = task->total_system_time;
+
+ proc_starttime_kdp(task->bsd_info, &proc_starttime_secs, NULL, NULL);
+ cur_tsnap->ts_p_start_sec = proc_starttime_secs;
+
+ cur_tsnap->ts_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
+ cur_tsnap->ts_max_resident_size = get_task_resident_max(task);
+ cur_tsnap->ts_suspend_count = task->suspend_count;
+ cur_tsnap->ts_faults = task->faults;
+ cur_tsnap->ts_pageins = task->pageins;
+ cur_tsnap->ts_cow_faults = task->cow_faults;
+ cur_tsnap->ts_was_throttled = (uint32_t) proc_was_throttled_from_task(task);
+ cur_tsnap->ts_did_throttle = (uint32_t) proc_did_throttle_from_task(task);
+ cur_tsnap->ts_latency_qos = (task->effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED) ?
+ LATENCY_QOS_TIER_UNSPECIFIED : ((0xFF << 16) | task->effective_policy.tep_latency_qos);
+ cur_tsnap->ts_pid = task_pid;
+
+ /* Add the BSD process identifiers */
+ if (task_pid != -1 && task->bsd_info != NULL)
+ proc_name_kdp(task, cur_tsnap->ts_p_comm, sizeof(cur_tsnap->ts_p_comm));
+ else {
+ cur_tsnap->ts_p_comm[0] = '\0';
+#if IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG)
+ if (task->task_imp_base != NULL) {
+ stackshot_strlcpy(cur_tsnap->ts_p_comm, &task->task_imp_base->iit_procname[0],
+ MIN((int)sizeof(task->task_imp_base->iit_procname), (int)sizeof(cur_tsnap->ts_p_comm)));
+ }
+#endif
+ }
+
+ if (collect_iostats) {
+ kcd_exit_on_error(kcdata_record_task_iostats(kcd, task));
+ }
+
+error_exit:
+ return error;
+}
+
+static kern_return_t
+kcdata_record_task_delta_snapshot(kcdata_descriptor_t kcd, task_t task, boolean_t have_pmap, unaligned_u64 **task_snap_ss_flags)
+{
+ kern_return_t error = KERN_SUCCESS;
+ struct task_delta_snapshot_v2 * cur_tsnap = NULL;
+ mach_vm_address_t out_addr = 0;
+
+ uint64_t task_uniqueid = get_task_uniqueid(task);
+ assert(task_snap_ss_flags != NULL);
+
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT, sizeof(struct task_delta_snapshot_v2), &out_addr));
+
+ cur_tsnap = (struct task_delta_snapshot_v2 *)out_addr;
+
+ cur_tsnap->tds_unique_pid = task_uniqueid;
+ cur_tsnap->tds_ss_flags = kcdata_get_task_ss_flags(task);
+ *task_snap_ss_flags = (unaligned_u64 *)&cur_tsnap->tds_ss_flags;
+
+ cur_tsnap->tds_user_time_in_terminated_threads = task->total_user_time;
+ cur_tsnap->tds_system_time_in_terminated_threads = task->total_system_time;
+
+ cur_tsnap->tds_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
+
+ cur_tsnap->tds_max_resident_size = get_task_resident_max(task);
+ cur_tsnap->tds_suspend_count = task->suspend_count;
+ cur_tsnap->tds_faults = task->faults;
+ cur_tsnap->tds_pageins = task->pageins;
+ cur_tsnap->tds_cow_faults = task->cow_faults;
+ cur_tsnap->tds_was_throttled = (uint32_t)proc_was_throttled_from_task(task);
+ cur_tsnap->tds_did_throttle = (uint32_t)proc_did_throttle_from_task(task);
+ cur_tsnap->tds_latency_qos = (task-> effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED)
+ ? LATENCY_QOS_TIER_UNSPECIFIED
+ : ((0xFF << 16) | task-> effective_policy.tep_latency_qos);
+
+error_exit:
+ return error;
+}
+
+static kern_return_t
+kcdata_record_thread_iostats(kcdata_descriptor_t kcd, thread_t thread)
+{
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+
+ /* I/O Statistics */
+ assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
+ if (thread->thread_io_stats && !memory_iszero(thread->thread_io_stats, sizeof(struct io_stat_info))) {
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
+ struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
+ _iostat->ss_disk_reads_count = thread->thread_io_stats->disk_reads.count;
+ _iostat->ss_disk_reads_size = thread->thread_io_stats->disk_reads.size;
+ _iostat->ss_disk_writes_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->disk_reads.count);
+ _iostat->ss_disk_writes_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->disk_reads.size);
+ _iostat->ss_paging_count = thread->thread_io_stats->paging.count;
+ _iostat->ss_paging_size = thread->thread_io_stats->paging.size;
+ _iostat->ss_non_paging_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->paging.count);
+ _iostat->ss_non_paging_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->paging.size);
+ _iostat->ss_metadata_count = thread->thread_io_stats->metadata.count;
+ _iostat->ss_metadata_size = thread->thread_io_stats->metadata.size;
+ _iostat->ss_data_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->metadata.count);
+ _iostat->ss_data_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->metadata.size);
+ for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
+ _iostat->ss_io_priority_count[i] = thread->thread_io_stats->io_priority[i].count;
+ _iostat->ss_io_priority_size[i] = thread->thread_io_stats->io_priority[i].size;
+ }
+ }
+
+error_exit:
+ return error;
+}
+
+static kern_return_t
+kcdata_record_thread_snapshot(
+ kcdata_descriptor_t kcd, thread_t thread, task_t task, uint32_t trace_flags, boolean_t have_pmap, boolean_t thread_on_core)
+{
+ boolean_t dispatch_p = ((trace_flags & STACKSHOT_GET_DQ) != 0);
+ boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
+ boolean_t trace_fp_p = false;
+ boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
+ boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
+
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+ int saved_count = 0;
+
+ struct thread_snapshot_v3 * cur_thread_snap = NULL;
+ char cur_thread_name[STACKSHOT_MAX_THREAD_NAME_SIZE];
+ uint64_t tval = 0;
+ boolean_t task64 = task_has_64BitAddr(task);
+
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v3), &out_addr));
+ cur_thread_snap = (struct thread_snapshot_v3 *)out_addr;
+
+ /* Populate the thread snapshot header */
+ cur_thread_snap->ths_thread_id = thread_tid(thread);
+ cur_thread_snap->ths_wait_event = VM_KERNEL_UNSLIDE_OR_PERM(thread->wait_event);
+ cur_thread_snap->ths_continuation = VM_KERNEL_UNSLIDE(thread->continuation);
+ cur_thread_snap->ths_total_syscalls = thread->syscalls_mach + thread->syscalls_unix;
+
+ if (IPC_VOUCHER_NULL != thread->ith_voucher)
+ cur_thread_snap->ths_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
+ else
+ cur_thread_snap->ths_voucher_identifier = 0;
+
+ cur_thread_snap->ths_dqserialnum = 0;
+ if (dispatch_p && (task != kernel_task) && (task->active) && have_pmap) {
+ uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
+ if (dqkeyaddr != 0) {
+ uint64_t dqaddr = 0;
+ boolean_t copyin_ok = kdp_copyin_word(task, dqkeyaddr, &dqaddr, FALSE, NULL);
+ if (copyin_ok && dqaddr != 0) {
+ uint64_t dqserialnumaddr = dqaddr + get_task_dispatchqueue_serialno_offset(task);
+ uint64_t dqserialnum = 0;
+ copyin_ok = kdp_copyin_word(task, dqserialnumaddr, &dqserialnum, FALSE, NULL);
+ if (copyin_ok) {
+ cur_thread_snap->ths_ss_flags |= kHasDispatchSerial;
+ cur_thread_snap->ths_dqserialnum = dqserialnum;
+ }
+ }
+ }
+ }
+
+ tval = safe_grab_timer_value(&thread->user_timer);
+ cur_thread_snap->ths_user_time = tval;
+ tval = safe_grab_timer_value(&thread->system_timer);
+
+ if (thread->precise_user_kernel_time) {
+ cur_thread_snap->ths_sys_time = tval;
+ } else {
+ cur_thread_snap->ths_user_time += tval;
+ cur_thread_snap->ths_sys_time = 0;
+ }
+
+ cur_thread_snap->ths_ss_flags = 0;
+ if (thread->effective_policy.thep_darwinbg)
+ cur_thread_snap->ths_ss_flags |= kThreadDarwinBG;
+ if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
+ cur_thread_snap->ths_ss_flags |= kThreadIOPassive;
+ if (thread->suspend_count > 0)
+ cur_thread_snap->ths_ss_flags |= kThreadSuspended;
+ if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
+ cur_thread_snap->ths_ss_flags |= kGlobalForcedIdle;
+ if (thread_on_core)
+ cur_thread_snap->ths_ss_flags |= kThreadOnCore;
+ if (stackshot_thread_is_idle_worker_unsafe(thread))
+ cur_thread_snap->ths_ss_flags |= kThreadIdleWorker;
+
+ /* make sure state flags defined in kcdata.h still match internal flags */
+ static_assert(SS_TH_WAIT == TH_WAIT);
+ static_assert(SS_TH_SUSP == TH_SUSP);
+ static_assert(SS_TH_RUN == TH_RUN);
+ static_assert(SS_TH_UNINT == TH_UNINT);
+ static_assert(SS_TH_TERMINATE == TH_TERMINATE);
+ static_assert(SS_TH_TERMINATE2 == TH_TERMINATE2);
+ static_assert(SS_TH_IDLE == TH_IDLE);
+
+ cur_thread_snap->ths_last_run_time = thread->last_run_time;
+ cur_thread_snap->ths_last_made_runnable_time = thread->last_made_runnable_time;
+ cur_thread_snap->ths_state = thread->state;
+ cur_thread_snap->ths_sched_flags = thread->sched_flags;
+ cur_thread_snap->ths_base_priority = thread->base_pri;
+ cur_thread_snap->ths_sched_priority = thread->sched_pri;
+ cur_thread_snap->ths_eqos = thread->effective_policy.thep_qos;
+ cur_thread_snap->ths_rqos = thread->requested_policy.thrp_qos;
+ cur_thread_snap->ths_rqos_override = thread->requested_policy.thrp_qos_override;
+ cur_thread_snap->ths_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
+ cur_thread_snap->ths_thread_t = VM_KERNEL_ADDRPERM(thread);
+
+ /* if there is thread name then add to buffer */
+ cur_thread_name[0] = '\0';
+ proc_threadname_kdp(thread->uthread, cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE);
+ if (strnlen(cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE) > 0) {
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_NAME, sizeof(cur_thread_name), &out_addr));
+ stackshot_memcpy((void *)out_addr, (void *)cur_thread_name, sizeof(cur_thread_name));
+ }
+
+ /* record system and user cpu times */
+ time_value_t user_time;
+ time_value_t system_time;
+ thread_read_times(thread, &user_time, &system_time);
+ kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_CPU_TIMES, sizeof(struct stackshot_cpu_times), &out_addr));
+ struct stackshot_cpu_times * stackshot_cpu_times = (struct stackshot_cpu_times *)out_addr;
+ stackshot_cpu_times->user_usec = ((uint64_t)user_time.seconds) * USEC_PER_SEC + user_time.microseconds;
+ stackshot_cpu_times->system_usec = ((uint64_t)system_time.seconds) * USEC_PER_SEC + system_time.microseconds;
+
+ /* Trace user stack, if any */
+ if (!active_kthreads_only_p && task->active && thread->task->map != kernel_map) {
+ uint32_t thread_snapshot_flags = 0;
+ /* 64-bit task? */
+ if (task64) {
+ out_addr = (mach_vm_address_t)kcd_end_address(kcd);
+ saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE,
+ trace_fp_p, &thread_snapshot_flags);
+ if (saved_count > 0) {
+ int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME64
+ : STACKSHOT_KCTYPE_USER_STACKLR64,
+ frame_size, saved_count / frame_size, &out_addr));
+ cur_thread_snap->ths_ss_flags |= kUser64_p;
+ }
+ } else {
+ out_addr = (mach_vm_address_t)kcd_end_address(kcd);
+ saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE, trace_fp_p,
+ &thread_snapshot_flags);
+ if (saved_count > 0) {
+ int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME
+ : STACKSHOT_KCTYPE_USER_STACKLR,
+ frame_size, saved_count / frame_size, &out_addr));
+ }
+ }
+
+ if (thread_snapshot_flags != 0) {
+ cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
+ }
+ }
+
+ /* Call through to the machine specific trace routines
+ * Frames are added past the snapshot header.
*/
- old_debugger = current_debugger;
- if (old_debugger != KDP_CUR_DB) {
- current_debugger = KDP_CUR_DB;
+ if (thread->kernel_stack != 0) {
+ uint32_t thread_snapshot_flags = 0;
+#if defined(__LP64__)
+ out_addr = (mach_vm_address_t)kcd_end_address(kcd);
+ saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
+ &thread_snapshot_flags);
+ if (saved_count > 0) {
+ int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
+ cur_thread_snap->ths_ss_flags |= kKernel64_p;
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME64
+ : STACKSHOT_KCTYPE_KERN_STACKLR64,
+ frame_size, saved_count / frame_size, &out_addr));
+ }
+#else
+ out_addr = (mach_vm_address_t)kcd_end_address(kcd);
+ saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
+ &thread_snapshot_flags);
+ if (saved_count > 0) {
+ int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
+ kcd_exit_on_error(
+ kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME : STACKSHOT_KCTYPE_KERN_STACKLR,
+ frame_size, saved_count / frame_size, &out_addr));
+ }
+#endif
+ if (thread_snapshot_flags != 0) {
+ cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
+ }
+ }
+
+ if (collect_iostats) {
+ kcd_exit_on_error(kcdata_record_thread_iostats(kcd, thread));
}
+
+error_exit:
+ return error;
}
-void
-kdp_snapshot_postflight(void)
+static int
+kcdata_record_thread_delta_snapshot(struct thread_delta_snapshot_v2 * cur_thread_snap, thread_t thread, boolean_t thread_on_core)
{
- kdp_snapshot--;
-#if CONFIG_KDP_INTERACTIVE_DEBUGGING
- if (
- (kdp_en_send_pkt == NULL) || (old_debugger == KDB_CUR_DB))
- current_debugger = old_debugger;
-#else
- current_debugger = old_debugger;
-#endif
+ cur_thread_snap->tds_thread_id = thread_tid(thread);
+ if (IPC_VOUCHER_NULL != thread->ith_voucher)
+ cur_thread_snap->tds_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
+ else
+ cur_thread_snap->tds_voucher_identifier = 0;
+
+ cur_thread_snap->tds_ss_flags = 0;
+ if (thread->effective_policy.thep_darwinbg)
+ cur_thread_snap->tds_ss_flags |= kThreadDarwinBG;
+ if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
+ cur_thread_snap->tds_ss_flags |= kThreadIOPassive;
+ if (thread->suspend_count > 0)
+ cur_thread_snap->tds_ss_flags |= kThreadSuspended;
+ if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
+ cur_thread_snap->tds_ss_flags |= kGlobalForcedIdle;
+ if (thread_on_core)
+ cur_thread_snap->tds_ss_flags |= kThreadOnCore;
+ if (stackshot_thread_is_idle_worker_unsafe(thread))
+ cur_thread_snap->tds_ss_flags |= kThreadIdleWorker;
+
+ cur_thread_snap->tds_last_made_runnable_time = thread->last_made_runnable_time;
+ cur_thread_snap->tds_state = thread->state;
+ cur_thread_snap->tds_sched_flags = thread->sched_flags;
+ cur_thread_snap->tds_base_priority = thread->base_pri;
+ cur_thread_snap->tds_sched_priority = thread->sched_pri;
+ cur_thread_snap->tds_eqos = thread->effective_policy.thep_qos;
+ cur_thread_snap->tds_rqos = thread->requested_policy.thrp_qos;
+ cur_thread_snap->tds_rqos_override = thread->requested_policy.thrp_qos_override;
+ cur_thread_snap->tds_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
+
+ return 0;
}
-int
-kdp_stack_snapshot_geterror(void)
+/*
+ * Why 12? 12 strikes a decent balance between allocating a large array on
+ * the stack and having large kcdata item overheads for recording nonrunable
+ * tasks.
+ */
+#define UNIQUEIDSPERFLUSH 12
+
+struct saved_uniqueids {
+ uint64_t ids[UNIQUEIDSPERFLUSH];
+ unsigned count;
+};
+
+static kern_return_t
+flush_nonrunnable_tasks(struct saved_uniqueids * ids)
{
- return stack_snapshot_ret;
+ if (ids->count == 0)
+ return KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+ kern_return_t ret = kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TASKS, sizeof(uint64_t),
+ ids->count, &out_addr);
+ if (ret != KERN_SUCCESS) {
+ return ret;
+ }
+ stackshot_memcpy((void *)out_addr, ids->ids, sizeof(uint64_t) * ids->count);
+ ids->count = 0;
+ return ret;
}
-int
-kdp_stack_snapshot_bytes_traced(void)
+static kern_return_t
+handle_nonrunnable_task(struct saved_uniqueids * ids, uint64_t pid)
{
- return stack_snapshot_bytes_traced;
+ kern_return_t ret = KERN_SUCCESS;
+ ids->ids[ids->count] = pid;
+ ids->count++;
+ assert(ids->count <= UNIQUEIDSPERFLUSH);
+ if (ids->count == UNIQUEIDSPERFLUSH)
+ ret = flush_nonrunnable_tasks(ids);
+ return ret;
}
-static int
-kdp_stackshot(int pid, void *tracebuf, uint32_t tracebuf_size, uint32_t trace_flags, uint32_t dispatch_offset, uint32_t *pbytesTraced)
+enum thread_classification {
+ tc_full_snapshot, /* take a full snapshot */
+ tc_delta_snapshot, /* take a delta snapshot */
+ tc_nonrunnable, /* only report id */
+};
+
+static enum thread_classification
+classify_thread(thread_t thread, boolean_t * thread_on_core_p, uint32_t trace_flags)
{
- char *tracepos = (char *) tracebuf;
- char *tracebound = tracepos + tracebuf_size;
- uint32_t tracebytes = 0;
- int error = 0, i;
+ boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
+ boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
+
+ processor_t last_processor = thread->last_processor;
+
+ boolean_t thread_on_core =
+ (last_processor != PROCESSOR_NULL && last_processor->state == PROCESSOR_RUNNING && last_processor->active_thread == thread);
+
+ *thread_on_core_p = thread_on_core;
+
+ /* Capture the full thread snapshot if this is not a delta stackshot or if the thread has run subsequent to the
+ * previous full stackshot */
+ if (!collect_delta_stackshot || thread_on_core || (thread->last_run_time > stack_snapshot_delta_since_timestamp)) {
+ return tc_full_snapshot;
+ } else {
+ if (minimize_nonrunnables && !(thread->state & TH_RUN)) {
+ return tc_nonrunnable;
+ } else {
+ return tc_delta_snapshot;
+ }
+ }
+}
+static kern_return_t
+kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTraced)
+{
+ kern_return_t error = KERN_SUCCESS;
+ mach_vm_address_t out_addr = 0;
+ uint64_t abs_time = 0, abs_time_end = 0;
+ uint64_t *abs_time_addr = NULL;
+ uint64_t system_state_flags = 0;
+ int saved_count = 0;
task_t task = TASK_NULL;
thread_t thread = THREAD_NULL;
- unsigned framesize = 2 * sizeof(vm_offset_t);
+ mach_timebase_info_data_t timebase = {0, 0};
+ uint32_t length_to_copy = 0, tmp32 = 0;
- queue_head_t *task_list = &tasks;
- boolean_t is_active_list = TRUE;
-
- boolean_t dispatch_p = ((trace_flags & STACKSHOT_GET_DQ) != 0);
- boolean_t save_loadinfo_p = ((trace_flags & STACKSHOT_SAVE_LOADINFO) != 0);
- boolean_t save_kextloadinfo_p = ((trace_flags & STACKSHOT_SAVE_KEXT_LOADINFO) != 0);
- boolean_t save_userframes_p = ((trace_flags & STACKSHOT_SAVE_KERNEL_FRAMES_ONLY) == 0);
- boolean_t save_donating_pids_p = ((trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0);
-
- if(trace_flags & STACKSHOT_GET_GLOBAL_MEM_STATS) {
- if(tracepos + sizeof(struct mem_and_io_snapshot) > tracebound) {
- error = -1;
- goto error_exit;
- }
- kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)tracepos);
- tracepos += sizeof(struct mem_and_io_snapshot);
+ abs_time = mach_absolute_time();
+
+#if !(DEVELOPMENT || DEBUG)
+ trace_flags &= ~STACKSHOT_THREAD_WAITINFO;
+#endif
+
+ /* process the flags */
+ boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
+ boolean_t save_donating_pids_p = ((trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0);
+ boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
+ boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
+ boolean_t use_fault_path = ((trace_flags & (STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_ENABLE_BT_FAULTING)) != 0);
+ boolean_t save_owner_info = ((trace_flags & STACKSHOT_THREAD_WAITINFO) != 0);
+
+ stack_enable_faulting = (trace_flags & (STACKSHOT_ENABLE_BT_FAULTING));
+
+
+ struct saved_uniqueids saved_uniqueids = {.count = 0};
+
+ if (use_fault_path) {
+ fault_stats.sfs_pages_faulted_in = 0;
+ fault_stats.sfs_time_spent_faulting = 0;
+ fault_stats.sfs_stopped_faulting = (uint8_t) FALSE;
}
-
-walk_list:
+ if (sizeof(void *) == 8)
+ system_state_flags |= kKernel64_p;
+
+ if (stackshot_kcdata_p == NULL || pBytesTraced == NULL) {
+ error = KERN_INVALID_ARGUMENT;
+ goto error_exit;
+ }
+
+ /* setup mach_absolute_time and timebase info -- copy out in some cases and needed to convert since_timestamp to seconds for proc start time */
+ clock_timebase_info(&timebase);
+
+ /* begin saving data into the buffer */
+ *pBytesTraced = 0;
+ kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_in_flags"));
+ kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, (uint32_t)pid, "stackshot_in_pid"));
+ kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_kcdata_p, system_state_flags, "system_state_flags"));
+
+#if CONFIG_JETSAM
+ tmp32 = memorystatus_get_pressure_status_kdp();
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_LEVEL, sizeof(uint32_t), &out_addr));
+ stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
+#endif
+
+ if (!collect_delta_stackshot) {
+ tmp32 = PAGE_SIZE;
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_KERN_PAGE_SIZE, sizeof(uint32_t), &out_addr));
+ stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
+
+ /* save boot-args and osversion string */
+ length_to_copy = MIN((uint32_t)(strlen(version) + 1), OSVERSIZE);
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_OSVERSION, length_to_copy, &out_addr));
+ stackshot_strlcpy((char*)out_addr, &version[0], length_to_copy);
+
+ length_to_copy = MIN((uint32_t)(strlen(PE_boot_args()) + 1), OSVERSIZE);
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_BOOTARGS, length_to_copy, &out_addr));
+ stackshot_strlcpy((char*)out_addr, PE_boot_args(), length_to_copy);
+
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_TIMEBASE, sizeof(timebase), &out_addr));
+ stackshot_memcpy((void *)out_addr, &timebase, sizeof(timebase));
+ } else {
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP, sizeof(uint64_t), &out_addr));
+ stackshot_memcpy((void*)out_addr, &stack_snapshot_delta_since_timestamp, sizeof(stack_snapshot_delta_since_timestamp));
+ }
+
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_MACH_ABSOLUTE_TIME, sizeof(uint64_t), &out_addr));
+ abs_time_addr = (uint64_t *)out_addr;
+ stackshot_memcpy((void *)abs_time_addr, &abs_time, sizeof(uint64_t));
+
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &out_addr));
+ stackshot_memcpy((void *)out_addr, &stackshot_microsecs, sizeof(uint64_t));
+
+ /* reserve space of system level shared cache load info */
+ struct dyld_uuid_info_64_v2 * sys_shared_cache_loadinfo = NULL;
+ if (!collect_delta_stackshot) {
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO,
+ sizeof(struct dyld_uuid_info_64_v2), &out_addr));
+ sys_shared_cache_loadinfo = (struct dyld_uuid_info_64_v2 *)out_addr;
+ bzero((void *)sys_shared_cache_loadinfo, sizeof(struct dyld_uuid_info_64_v2));
+ }
+
+ /* Add requested information first */
+ if (trace_flags & STACKSHOT_GET_GLOBAL_MEM_STATS) {
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_GLOBAL_MEM_STATS, sizeof(struct mem_and_io_snapshot), &out_addr));
+ kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)out_addr);
+ }
+
+ /* Iterate over tasks */
+ queue_head_t *task_list = &tasks;
queue_iterate(task_list, task, task_t, tasks) {
- if ((task == NULL) || !ml_validate_nofault((vm_offset_t) task, sizeof(struct task)))
+ int task_pid = 0;
+ uint64_t task_uniqueid = 0;
+ int num_delta_thread_snapshots = 0;
+ int num_nonrunnable_threads = 0;
+ int num_waitinfo_threads = 0;
+ uint64_t task_start_abstime = 0;
+ boolean_t task_delta_stackshot = FALSE;
+ boolean_t task64 = FALSE, have_map = FALSE, have_pmap = FALSE;
+ boolean_t some_thread_ran = FALSE;
+ unaligned_u64 *task_snap_ss_flags = NULL;
+
+ if ((task == NULL) || !ml_validate_nofault((vm_offset_t)task, sizeof(struct task))) {
+ error = KERN_FAILURE;
goto error_exit;
+ }
+
+ have_map = (task->map != NULL) && (ml_validate_nofault((vm_offset_t)(task->map), sizeof(struct _vm_map)));
+ have_pmap = have_map && (task->map->pmap != NULL) && (ml_validate_nofault((vm_offset_t)(task->map->pmap), sizeof(struct pmap)));
- int task_pid = pid_from_task(task);
- uint64_t task_uniqueid = proc_uniqueid_from_task(task);
- boolean_t task64 = task_has_64BitAddr(task);
+ task_pid = pid_from_task(task);
+ task_uniqueid = get_task_uniqueid(task);
+ task64 = task_has_64BitAddr(task);
- if (!task->active) {
- /*
+ if (!task->active || task_is_a_corpse(task)) {
+ /*
* Not interested in terminated tasks without threads, and
* at the moment, stackshot can't handle a task without a name.
*/
}
}
+ if (collect_delta_stackshot) {
+ proc_starttime_kdp(task->bsd_info, NULL, NULL, &task_start_abstime);
+ }
+
/* Trace everything, unless a process was specified */
if ((pid == -1) || (pid == task_pid)) {
- task_snapshot_t task_snap;
- thread_snapshot_t tsnap = NULL;
- uint32_t uuid_info_count = 0;
- mach_vm_address_t uuid_info_addr = 0;
- boolean_t have_map = (task->map != NULL) &&
- (ml_validate_nofault((vm_offset_t)(task->map), sizeof(struct _vm_map)));
- boolean_t have_pmap = have_map && (task->map->pmap != NULL) &&
- (ml_validate_nofault((vm_offset_t)(task->map->pmap), sizeof(struct pmap)));
- uint64_t shared_cache_base_address = 0;
-
- if (have_pmap && task->active && save_loadinfo_p && task_pid > 0) {
- // Read the dyld_all_image_infos struct from the task memory to get UUID array count and location
- if (task64) {
- struct user64_dyld_all_image_infos task_image_infos;
- if (kdp_copyin(task->map->pmap, task->all_image_info_addr, &task_image_infos, sizeof(struct user64_dyld_all_image_infos))) {
- uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
- uuid_info_addr = task_image_infos.uuidArray;
- }
- } else {
- struct user32_dyld_all_image_infos task_image_infos;
- if (kdp_copyin(task->map->pmap, task->all_image_info_addr, &task_image_infos, sizeof(struct user32_dyld_all_image_infos))) {
- uuid_info_count = task_image_infos.uuidArrayCount;
- uuid_info_addr = task_image_infos.uuidArray;
- }
- }
+#if DEBUG || DEVELOPMENT
+ /* we might want to call kcdata_undo_add_container_begin(), which is
+ * only safe if we call it after kcdata_add_container_marker() but
+ * before adding any other kcdata items. In development kernels,
+ * we'll remember where the buffer end was and confirm after calling
+ * kcdata_undo_add_container_begin() that it's in exactly the same
+ * place.*/
+ mach_vm_address_t revert_addr = stackshot_kcdata_p->kcd_addr_end;
+#endif
- // If we get a NULL uuid_info_addr (which can happen when we catch dyld in the middle of updating
- // this data structure), we zero the uuid_info_count so that we won't even try to save load info
- // for this task.
- if (!uuid_info_addr) {
- uuid_info_count = 0;
- }
- }
+ /* add task snapshot marker */
+ kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
+ STACKSHOT_KCCONTAINER_TASK, task_uniqueid));
- if (have_pmap && save_kextloadinfo_p && task_pid == 0) {
- if (ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader))) {
- uuid_info_count = gLoadedKextSummaries->numSummaries + 1; /* include main kernel UUID */
+ if (!collect_delta_stackshot || (task_start_abstime == 0) ||
+ (task_start_abstime > stack_snapshot_delta_since_timestamp)) {
+ kcd_exit_on_error(kcdata_record_task_snapshot(stackshot_kcdata_p, task, trace_flags, have_pmap, &task_snap_ss_flags));
+ } else {
+ task_delta_stackshot = TRUE;
+ if (minimize_nonrunnables) {
+ // delay taking the task snapshot. If there are no runnable threads we'll skip it.
+ } else {
+ kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
}
}
- if (tracepos + sizeof(struct task_snapshot) > tracebound) {
- error = -1;
- goto error_exit;
- }
-
- task_snap = (task_snapshot_t) tracepos;
- task_snap->snapshot_magic = STACKSHOT_TASK_SNAPSHOT_MAGIC;
- task_snap->pid = task_pid;
- task_snap->uniqueid = task_uniqueid;
- task_snap->nloadinfos = uuid_info_count;
- task_snap->donating_pid_count = 0;
-
- /* Add the BSD process identifiers */
- if (task_pid != -1)
- proc_name_kdp(task, task_snap->p_comm, sizeof(task_snap->p_comm));
- else
- task_snap->p_comm[0] = '\0';
- task_snap->ss_flags = 0;
- if (task64)
- task_snap->ss_flags |= kUser64_p;
- if (task64 && task_pid == 0)
- task_snap->ss_flags |= kKernel64_p;
- if (!task->active)
- task_snap->ss_flags |= kTerminatedSnapshot;
- if(task->pidsuspended) task_snap->ss_flags |= kPidSuspended;
- if(task->frozen) task_snap->ss_flags |= kFrozen;
-
- if (task->effective_policy.darwinbg == 1) {
- task_snap->ss_flags |= kTaskDarwinBG;
- }
-
- if (task->requested_policy.t_role == TASK_FOREGROUND_APPLICATION) {
- task_snap->ss_flags |= kTaskIsForeground;
- }
+ /* Iterate over task threads */
+ queue_iterate(&task->threads, thread, thread_t, task_threads)
+ {
+ uint64_t thread_uniqueid;
- if (task->requested_policy.t_boosted == 1) {
- task_snap->ss_flags |= kTaskIsBoosted;
- }
+ if ((thread == NULL) || !ml_validate_nofault((vm_offset_t)thread, sizeof(struct thread))) {
+ error = KERN_FAILURE;
+ goto error_exit;
+ }
- if (task->effective_policy.t_sup_active == 1)
- task_snap->ss_flags |= kTaskIsSuppressed;
-#if IMPORTANCE_INHERITANCE
- if (task->task_imp_base) {
- if (task->task_imp_base->iit_donor) {
- task_snap->ss_flags |= kTaskIsImpDonor;
-}
+ if (active_kthreads_only_p && thread->kernel_stack == 0)
+ continue;
- if (task->task_imp_base->iit_live_donor) {
- task_snap->ss_flags |= kTaskIsLiveImpDonor;
- }
- }
-#endif
+ thread_uniqueid = thread_tid(thread);
- task_snap->latency_qos = (task->effective_policy.t_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED) ?
- LATENCY_QOS_TIER_UNSPECIFIED : ((0xFF << 16) | task->effective_policy.t_latency_qos);
+ boolean_t thread_on_core;
+ enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
- task_snap->suspend_count = task->suspend_count;
- task_snap->task_size = have_pmap ? pmap_resident_count(task->map->pmap) : 0;
- task_snap->faults = task->faults;
- task_snap->pageins = task->pageins;
- task_snap->cow_faults = task->cow_faults;
+ switch (thread_classification) {
+ case tc_full_snapshot:
+ /* add thread marker */
+ kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
+ STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
+ kcd_exit_on_error(
+ kcdata_record_thread_snapshot(stackshot_kcdata_p, thread, task, trace_flags, have_pmap, thread_on_core));
- task_snap->user_time_in_terminated_threads = task->total_user_time;
- task_snap->system_time_in_terminated_threads = task->total_system_time;
- /*
- * The throttling counters are maintained as 64-bit counters in the proc
- * structure. However, we reserve 32-bits (each) for them in the task_snapshot
- * struct to save space and since we do not expect them to overflow 32-bits. If we
- * find these values overflowing in the future, the fix would be to simply
- * upgrade these counters to 64-bit in the task_snapshot struct
- */
- task_snap->was_throttled = (uint32_t) proc_was_throttled_from_task(task);
- task_snap->did_throttle = (uint32_t) proc_did_throttle_from_task(task);
+ /* mark end of thread snapshot data */
+ kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END,
+ STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
- /* fetch some useful BSD info: */
- task_snap->p_start_sec = task_snap->p_start_usec = 0;
- proc_starttime_kdp(task->bsd_info, &task_snap->p_start_sec, &task_snap->p_start_usec);
- if (task->shared_region && ml_validate_nofault((vm_offset_t)task->shared_region,
- sizeof(struct vm_shared_region))) {
- struct vm_shared_region *sr = task->shared_region;
+ some_thread_ran = TRUE;
+ break;
- shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping;
- }
- if (!shared_cache_base_address
- || !kdp_copyin(task->map->pmap, shared_cache_base_address + offsetof(struct _dyld_cache_header, uuid), task_snap->shared_cache_identifier, sizeof(task_snap->shared_cache_identifier))) {
- memset(task_snap->shared_cache_identifier, 0x0, sizeof(task_snap->shared_cache_identifier));
- }
- if (task->shared_region) {
- /*
- * No refcounting here, but we are in debugger
- * context, so that should be safe.
- */
- task_snap->shared_cache_slide = task->shared_region->sr_slide_info.slide;
- } else {
- task_snap->shared_cache_slide = 0;
- }
+ case tc_delta_snapshot:
+ num_delta_thread_snapshots++;
+ break;
- /* I/O Statistics */
- assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
-
- if (task->task_io_stats) {
- task_snap->disk_reads_count = task->task_io_stats->disk_reads.count;
- task_snap->disk_reads_size = task->task_io_stats->disk_reads.size;
- task_snap->disk_writes_count = (task->task_io_stats->total_io.count - task->task_io_stats->disk_reads.count);
- task_snap->disk_writes_size = (task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size);
- for(i = 0; i < IO_NUM_PRIORITIES; i++) {
- task_snap->io_priority_count[i] = task->task_io_stats->io_priority[i].count;
- task_snap->io_priority_size[i] = task->task_io_stats->io_priority[i].size;
+ case tc_nonrunnable:
+ num_nonrunnable_threads++;
+ break;
}
- task_snap->paging_count = task->task_io_stats->paging.count;
- task_snap->paging_size = task->task_io_stats->paging.size;
- task_snap->non_paging_count = (task->task_io_stats->total_io.count - task->task_io_stats->paging.count);
- task_snap->non_paging_size = (task->task_io_stats->total_io.size - task->task_io_stats->paging.size);
- task_snap->metadata_count = task->task_io_stats->metadata.count;
- task_snap->metadata_size = task->task_io_stats->metadata.size;
- task_snap->data_count = (task->task_io_stats->total_io.count - task->task_io_stats->metadata.count);
- task_snap->data_size = (task->task_io_stats->total_io.size - task->task_io_stats->metadata.size);
- } else {
- /* zero from disk_reads_count to end of structure */
- memset(&task_snap->disk_reads_count, 0, offsetof(struct task_snapshot, metadata_size) - offsetof(struct task_snapshot, disk_reads_count));
- }
- tracepos += sizeof(struct task_snapshot);
- if (task_pid > 0 && uuid_info_count > 0) {
- uint32_t uuid_info_size = (uint32_t)(task64 ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
- uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
+ /* We want to report owner information regardless of whether a thread
+ * has changed since the last delta, whether it's a normal stackshot,
+ * or whether it's nonrunnable */
+ if (save_owner_info && stackshot_thread_has_valid_waitinfo(thread))
+ num_waitinfo_threads++;
+ }
- if (tracepos + uuid_info_array_size > tracebound) {
- error = -1;
- goto error_exit;
- }
+ if (task_delta_stackshot && minimize_nonrunnables) {
+ if (some_thread_ran || num_delta_thread_snapshots > 0) {
+ kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
+ } else {
+ kcd_exit_on_error(kcdata_undo_add_container_begin(stackshot_kcdata_p));
- // Copy in the UUID info array
- // It may be nonresident, in which case just fix up nloadinfos to 0 in the task_snap
- if (have_pmap && !kdp_copyin(task->map->pmap, uuid_info_addr, tracepos, uuid_info_array_size))
- task_snap->nloadinfos = 0;
- else
- tracepos += uuid_info_array_size;
- } else if (task_pid == 0 && uuid_info_count > 0) {
- uint32_t uuid_info_size = (uint32_t)sizeof(kernel_uuid_info);
- uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
- kernel_uuid_info *output_uuids;
-
- if (tracepos + uuid_info_array_size > tracebound) {
- error = -1;
- goto error_exit;
+#if DEBUG || DEVELOPMENT
+ mach_vm_address_t undo_addr = stackshot_kcdata_p->kcd_addr_end;
+ if (revert_addr != undo_addr) {
+ panic("tried to revert a container begin but we already moved past it. revert=%p undo=%p",
+ (void *)revert_addr, (void *)undo_addr);
+ }
+#endif
+ kcd_exit_on_error(handle_nonrunnable_task(&saved_uniqueids, task_uniqueid));
+ continue;
}
+ }
- output_uuids = (kernel_uuid_info *)tracepos;
+ struct thread_delta_snapshot_v2 * delta_snapshots = NULL;
+ int current_delta_snapshot_index = 0;
- do {
+ if (num_delta_thread_snapshots > 0) {
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT,
+ sizeof(struct thread_delta_snapshot_v2),
+ num_delta_thread_snapshots, &out_addr));
+ delta_snapshots = (struct thread_delta_snapshot_v2 *)out_addr;
+ }
- if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) {
- /* Kernel UUID not found or inaccessible */
- task_snap->nloadinfos = 0;
- break;
- }
+ uint64_t * nonrunnable_tids = NULL;
+ int current_nonrunnable_index = 0;
- output_uuids[0].imageLoadAddress = (uintptr_t)VM_KERNEL_UNSLIDE(vm_kernel_stext);
- memcpy(&output_uuids[0].imageUUID, kernel_uuid, sizeof(uuid_t));
+ if (num_nonrunnable_threads > 0) {
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TIDS,
+ sizeof(uint64_t), num_nonrunnable_threads, &out_addr));
+ nonrunnable_tids = (uint64_t *)out_addr;
+ }
- if (ml_validate_nofault((vm_offset_t)(&gLoadedKextSummaries->summaries[0]),
- gLoadedKextSummaries->entry_size * gLoadedKextSummaries->numSummaries)) {
- uint32_t kexti;
+ thread_waitinfo_t *thread_waitinfo = NULL;
+ int current_waitinfo_index = 0;
- for (kexti=0 ; kexti < gLoadedKextSummaries->numSummaries; kexti++) {
- output_uuids[1+kexti].imageLoadAddress = (uintptr_t)VM_KERNEL_UNSLIDE(gLoadedKextSummaries->summaries[kexti].address);
- memcpy(&output_uuids[1+kexti].imageUUID, &gLoadedKextSummaries->summaries[kexti].uuid, sizeof(uuid_t));
- }
+ if (num_waitinfo_threads > 0) {
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_WAITINFO,
+ sizeof(thread_waitinfo_t), num_waitinfo_threads, &out_addr));
+ thread_waitinfo = (thread_waitinfo_t *)out_addr;
+ }
- tracepos += uuid_info_array_size;
- } else {
- /* kext summary invalid, but kernel UUID was copied */
- task_snap->nloadinfos = 1;
- tracepos += uuid_info_size;
- break;
+ if (num_delta_thread_snapshots > 0 || num_nonrunnable_threads > 0 || num_waitinfo_threads > 0) {
+ queue_iterate(&task->threads, thread, thread_t, task_threads)
+ {
+ if (active_kthreads_only_p && thread->kernel_stack == 0)
+ continue;
+
+ /* If we want owner info, we should capture it regardless of its classification */
+ if (save_owner_info && stackshot_thread_has_valid_waitinfo(thread)) {
+ stackshot_thread_wait_owner_info(
+ thread,
+ &thread_waitinfo[current_waitinfo_index++]);
}
- } while(0);
- }
-
- if (save_donating_pids_p) {
- task_snap->donating_pid_count = task_importance_list_pids(task, TASK_IMP_LIST_DONATING_PIDS, (int *)tracepos, (unsigned int)((tracebound - tracepos)/sizeof(int)));
- tracepos += sizeof(int) * task_snap->donating_pid_count;
- }
- queue_iterate(&task->threads, thread, thread_t, task_threads){
- uint64_t tval;
+ boolean_t thread_on_core;
+ enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
- if ((thread == NULL) || !ml_validate_nofault((vm_offset_t) thread, sizeof(struct thread)))
- goto error_exit;
+ switch (thread_classification) {
+ case tc_full_snapshot:
+ /* full thread snapshot captured above */
+ continue;
- if (((tracepos + 4 * sizeof(struct thread_snapshot)) > tracebound)) {
- error = -1;
- goto error_exit;
- }
- if (!save_userframes_p && thread->kernel_stack == 0)
- continue;
-
- /* Populate the thread snapshot header */
- tsnap = (thread_snapshot_t) tracepos;
- tsnap->thread_id = thread_tid(thread);
- tsnap->state = thread->state;
- tsnap->priority = thread->priority;
- tsnap->sched_pri = thread->sched_pri;
- tsnap->sched_flags = thread->sched_flags;
- tsnap->wait_event = VM_KERNEL_UNSLIDE_OR_PERM(thread->wait_event);
- tsnap->continuation = VM_KERNEL_UNSLIDE(thread->continuation);
- tval = safe_grab_timer_value(&thread->user_timer);
- tsnap->user_time = tval;
- tval = safe_grab_timer_value(&thread->system_timer);
- if (thread->precise_user_kernel_time) {
- tsnap->system_time = tval;
- } else {
- tsnap->user_time += tval;
- tsnap->system_time = 0;
- }
- tsnap->snapshot_magic = STACKSHOT_THREAD_SNAPSHOT_MAGIC;
- bzero(&tsnap->pth_name, STACKSHOT_MAX_THREAD_NAME_SIZE);
- proc_threadname_kdp(thread->uthread, &tsnap->pth_name[0], STACKSHOT_MAX_THREAD_NAME_SIZE);
- tracepos += sizeof(struct thread_snapshot);
- tsnap->ss_flags = 0;
- /* I/O Statistics */
- assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
- if (thread->thread_io_stats) {
- tsnap->disk_reads_count = thread->thread_io_stats->disk_reads.count;
- tsnap->disk_reads_size = thread->thread_io_stats->disk_reads.size;
- tsnap->disk_writes_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->disk_reads.count);
- tsnap->disk_writes_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->disk_reads.size);
- for(i = 0; i < IO_NUM_PRIORITIES; i++) {
- tsnap->io_priority_count[i] = thread->thread_io_stats->io_priority[i].count;
- tsnap->io_priority_size[i] = thread->thread_io_stats->io_priority[i].size;
+ case tc_delta_snapshot:
+ kcd_exit_on_error(kcdata_record_thread_delta_snapshot(&delta_snapshots[current_delta_snapshot_index++],
+ thread, thread_on_core));
+ break;
+
+ case tc_nonrunnable:
+ nonrunnable_tids[current_nonrunnable_index++] = thread_tid(thread);
+ continue;
}
- tsnap->paging_count = thread->thread_io_stats->paging.count;
- tsnap->paging_size = thread->thread_io_stats->paging.size;
- tsnap->non_paging_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->paging.count);
- tsnap->non_paging_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->paging.size);
- tsnap->metadata_count = thread->thread_io_stats->metadata.count;
- tsnap->metadata_size = thread->thread_io_stats->metadata.size;
- tsnap->data_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->metadata.count);
- tsnap->data_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->metadata.size);
- } else {
- /* zero from disk_reads_count to end of structure */
- memset(&tsnap->disk_reads_count, 0,
- offsetof(struct thread_snapshot, metadata_size) - offsetof(struct thread_snapshot, disk_reads_count));
}
- if (thread->effective_policy.darwinbg) {
- tsnap->ss_flags |= kThreadDarwinBG;
+#if DEBUG || DEVELOPMENT
+ if (current_delta_snapshot_index != num_delta_thread_snapshots) {
+ panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
+ num_delta_thread_snapshots, current_delta_snapshot_index);
}
-
- tsnap->io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
- if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
- tsnap->ss_flags |= kThreadIOPassive;
+ if (current_nonrunnable_index != num_nonrunnable_threads) {
+ panic("nonrunnable thread count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
+ num_nonrunnable_threads, current_nonrunnable_index);
}
-
- if (thread->suspend_count > 0) {
- tsnap->ss_flags |= kThreadSuspended;
+ if (current_waitinfo_index != num_waitinfo_threads) {
+ panic("thread wait info count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
+ num_waitinfo_threads, current_waitinfo_index);
}
- if (IPC_VOUCHER_NULL != thread->ith_voucher) {
- tsnap->voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
- }
-
- tsnap->ts_qos = thread->effective_policy.thep_qos;
- tsnap->total_syscalls = thread->syscalls_mach + thread->syscalls_unix;
-
- if (dispatch_p && (task != kernel_task) && (task->active) && have_pmap) {
- uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
- if (dqkeyaddr != 0) {
- uint64_t dqaddr = 0;
- if (kdp_copyin(task->map->pmap, dqkeyaddr, &dqaddr, (task64 ? 8 : 4)) && (dqaddr != 0)) {
- uint64_t dqserialnumaddr = dqaddr + dispatch_offset;
- uint64_t dqserialnum = 0;
- if (kdp_copyin(task->map->pmap, dqserialnumaddr, &dqserialnum, (task64 ? 8 : 4))) {
- tsnap->ss_flags |= kHasDispatchSerial;
- *(uint64_t *)tracepos = dqserialnum;
- tracepos += 8;
- }
- }
- }
- }
-/* Call through to the machine specific trace routines
- * Frames are added past the snapshot header.
- */
- tracebytes = 0;
- if (thread->kernel_stack != 0) {
-#if defined(__LP64__)
- tracebytes = machine_trace_thread64(thread, tracepos, tracebound, MAX_FRAMES, FALSE);
- tsnap->ss_flags |= kKernel64_p;
- framesize = 16;
-#else
- tracebytes = machine_trace_thread(thread, tracepos, tracebound, MAX_FRAMES, FALSE);
- framesize = 8;
#endif
- }
- tsnap->nkern_frames = tracebytes/framesize;
- tracepos += tracebytes;
- tracebytes = 0;
- /* Trace user stack, if any */
- if (save_userframes_p && task->active && thread->task->map != kernel_map) {
- /* 64-bit task? */
- if (task_has_64BitAddr(thread->task)) {
- tracebytes = machine_trace_thread64(thread, tracepos, tracebound, MAX_FRAMES, TRUE);
- tsnap->ss_flags |= kUser64_p;
- framesize = 16;
- }
- else {
- tracebytes = machine_trace_thread(thread, tracepos, tracebound, MAX_FRAMES, TRUE);
- framesize = 8;
- }
- }
- tsnap->nuser_frames = tracebytes/framesize;
- tracepos += tracebytes;
- tracebytes = 0;
}
- if (!save_userframes_p && tsnap == NULL) {
- /*
- * No thread info is collected due to lack of kernel frames.
- * Remove information about this task also
- */
- tracepos = (char *)task_snap;
- }
+#if IMPORTANCE_INHERITANCE
+ if (save_donating_pids_p) {
+ kcd_exit_on_error(
+ ((((mach_vm_address_t)kcd_end_address(stackshot_kcdata_p) + (TASK_IMP_WALK_LIMIT * sizeof(int32_t))) <
+ (mach_vm_address_t)kcd_max_address(stackshot_kcdata_p))
+ ? KERN_SUCCESS
+ : KERN_RESOURCE_SHORTAGE));
+ saved_count = task_importance_list_pids(task, TASK_IMP_LIST_DONATING_PIDS,
+ (void *)kcd_end_address(stackshot_kcdata_p), TASK_IMP_WALK_LIMIT);
+ if (saved_count > 0)
+ kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_DONATING_PIDS,
+ sizeof(int32_t), saved_count, &out_addr));
+ }
+#endif
+
+ if (!collect_delta_stackshot || (num_delta_thread_snapshots != task->thread_count) || !task_delta_stackshot) {
+ /*
+ * Collect shared cache info and UUID info in these scenarios
+ * 1) a full stackshot
+ * 2) a delta stackshot where the task started after the previous full stackshot OR
+ * any thread from the task has run since the previous full stackshot
+ */
+
+ kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, sys_shared_cache_loadinfo, trace_flags, task_snap_ss_flags));
+ kcd_exit_on_error(kcdata_record_uuid_info(stackshot_kcdata_p, task, trace_flags, have_pmap, task_snap_ss_flags));
+ }
+ /* mark end of task snapshot data */
+ kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK,
+ task_uniqueid));
}
}
- if (is_active_list) {
- is_active_list = FALSE;
- task_list = &terminated_tasks;
- goto walk_list;
+ if (minimize_nonrunnables) {
+ flush_nonrunnable_tasks(&saved_uniqueids);
}
-error_exit:
- /* Release stack snapshot wait indicator */
- kdp_snapshot_postflight();
-
- *pbytesTraced = (uint32_t)(tracepos - (char *) tracebuf);
+ if (use_fault_path) {
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS,
+ sizeof(struct stackshot_fault_stats), &out_addr));
+ stackshot_memcpy((void*)out_addr, &fault_stats, sizeof(struct stackshot_fault_stats));
+ }
- return error;
-}
+ /* update timestamp of the stackshot */
+ abs_time_end = mach_absolute_time();
+#if DEVELOPMENT || DEBUG
+ kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_DURATION,
+ sizeof(struct stackshot_duration), &out_addr));
+ struct stackshot_duration * stackshot_duration = (struct stackshot_duration *)out_addr;
+ stackshot_duration->stackshot_duration = (abs_time_end - abs_time);
+ stackshot_duration->stackshot_duration_outer = 0;
+ stackshot_duration_outer = (unaligned_u64 *)&stackshot_duration->stackshot_duration_outer;
+#endif
+ stackshot_memcpy((void *)abs_time_addr, &abs_time_end, sizeof(uint64_t));
-static int pid_from_task(task_t task)
-{
- int pid = -1;
- if (task->bsd_info)
- pid = proc_pid(task->bsd_info);
+ kcd_exit_on_error(kcdata_write_buffer_end(stackshot_kcdata_p));
- return pid;
-}
+ /* === END of populating stackshot data === */
-static uint64_t
-proc_uniqueid_from_task(task_t task)
-{
- uint64_t uniqueid = ~(0ULL);
+ *pBytesTraced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_kcdata_p);
+error_exit:
- if (task->bsd_info)
- uniqueid = proc_uniqueid(task->bsd_info);
+ stack_enable_faulting = FALSE;
- return uniqueid;
+ return error;
}
static uint64_t
}
}
+void
+stackshot_memcpy(void *dst, const void *src, size_t len)
+{
+ memcpy(dst, src, len);
+}
+
+size_t
+stackshot_strlcpy(char *dst, const char *src, size_t maxlen)
+{
+ const size_t srclen = strlen(src);
+
+ if (srclen < maxlen) {
+ stackshot_memcpy(dst, src, srclen+1);
+ } else if (maxlen != 0) {
+ stackshot_memcpy(dst, src, maxlen-1);
+ dst[maxlen-1] = '\0';
+ }
+
+ return srclen;
+}
+
+
+/*
+ * Returns the physical address of the specified map:target address,
+ * using the kdp fault path if requested and the page is not resident.
+ */
+vm_offset_t
+kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results)
+{
+ vm_offset_t cur_phys_addr;
+ unsigned cur_wimg_bits;
+ uint64_t fault_start_time = 0;
+
+ if (map == VM_MAP_NULL) {
+ return 0;
+ }
+
+ cur_phys_addr = kdp_vtophys(map->pmap, target_addr);
+ if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
+ if (!try_fault || fault_stats.sfs_stopped_faulting) {
+ if (kdp_fault_results)
+ *kdp_fault_results |= KDP_FAULT_RESULT_PAGED_OUT;
+
+ return 0;
+ }
+
+ /*
+ * The pmap doesn't have a valid page so we start at the top level
+ * vm map and try a lightweight fault. Update fault path usage stats.
+ */
+ fault_start_time = mach_absolute_time();
+ cur_phys_addr = kdp_lightweight_fault(map, (target_addr & ~PAGE_MASK));
+ fault_stats.sfs_time_spent_faulting += (mach_absolute_time() - fault_start_time);
+
+ if ((fault_stats.sfs_time_spent_faulting >= fault_stats.sfs_system_max_fault_time) && !panic_stackshot) {
+ fault_stats.sfs_stopped_faulting = (uint8_t) TRUE;
+ }
+
+ cur_phys_addr += (target_addr & PAGE_MASK);
+
+ if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
+ if (kdp_fault_results)
+ *kdp_fault_results |= (KDP_FAULT_RESULT_TRIED_FAULT | KDP_FAULT_RESULT_PAGED_OUT);
+
+ return 0;
+ }
+
+ if (kdp_fault_results)
+ *kdp_fault_results |= KDP_FAULT_RESULT_FAULTED_IN;
+
+ fault_stats.sfs_pages_faulted_in++;
+ } else {
+ /*
+ * This check is done in kdp_lightweight_fault for the fault path.
+ */
+ cur_wimg_bits = pmap_cache_attributes((ppnum_t) atop(cur_phys_addr));
+
+ if ((cur_wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
+ return 0;
+ }
+ }
+
+ return cur_phys_addr;
+}
+
+boolean_t
+kdp_copyin_word(
+ task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results)
+{
+ if (task_has_64BitAddr(task)) {
+ return kdp_copyin(task->map, addr, result, sizeof(uint64_t), try_fault, kdp_fault_results);
+ } else {
+ uint32_t buf;
+ boolean_t r = kdp_copyin(task->map, addr, &buf, sizeof(uint32_t), try_fault, kdp_fault_results);
+ *result = buf;
+ return r;
+ }
+}
+
boolean_t
-kdp_copyin(pmap_t p, uint64_t uaddr, void *dest, size_t size)
+kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_results)
{
size_t rem = size;
char *kvaddr = dest;
+#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
+ /* Identify if destination buffer is in panic storage area */
+ if ((vm_offset_t)dest >= gPanicBase && (vm_offset_t)dest < gPanicBase + gPanicSize) {
+ if (((vm_offset_t)dest + size) >= (gPanicBase + gPanicSize)) {
+ return FALSE;
+ }
+ }
+#endif
+
while (rem) {
- ppnum_t upn = pmap_find_phys(p, uaddr);
- uint64_t phys_src = ptoa_64(upn) | (uaddr & PAGE_MASK);
+ uint64_t phys_src = kdp_find_phys(map, uaddr, try_fault, kdp_fault_results);
uint64_t phys_dest = kvtophys((vm_offset_t)kvaddr);
uint64_t src_rem = PAGE_SIZE - (phys_src & PAGE_MASK);
uint64_t dst_rem = PAGE_SIZE - (phys_dest & PAGE_MASK);
size_t cur_size = (uint32_t) MIN(src_rem, dst_rem);
cur_size = MIN(cur_size, rem);
- if (upn && pmap_valid_page(upn) && phys_dest) {
- bcopy_phys(phys_src, phys_dest, cur_size);
- }
- else
+ if (phys_src && phys_dest) {
+ bcopy_phys(phys_src, phys_dest, cur_size);
+ } else {
break;
+ }
+
uaddr += cur_size;
kvaddr += cur_size;
- rem -= cur_size;
+ rem -= cur_size;
}
+
return (rem == 0);
}
-void
-do_stackshot()
+kern_return_t
+do_stackshot(void *context)
{
- stack_snapshot_ret = kdp_stackshot(stack_snapshot_pid,
- stack_snapshot_buf, stack_snapshot_bufsize,
- stack_snapshot_flags, stack_snapshot_dispatch_offset,
- &stack_snapshot_bytes_traced);
+#pragma unused(context)
+ kdp_snapshot++;
+
+ stack_snapshot_ret = kdp_stackshot_kcdata_format(stack_snapshot_pid,
+ stack_snapshot_flags,
+ &stack_snapshot_bytes_traced);
+ kdp_snapshot--;
+ return stack_snapshot_ret;
}
/*
* machine_trace_thread and its relatives tend to throw at us.
*
* Please zero the nasty global this uses after a bulk lookup;
- * this isn't safe across a switch of the kdp_pmap or changes
+ * this isn't safe across a switch of the map or changes
* to a pmap.
*
* This also means that if zero is a valid KVA, we are
* happen.
*/
vm_offset_t
-machine_trace_thread_get_kva(vm_offset_t cur_target_addr)
+machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags)
{
- unsigned cur_wimg_bits;
vm_offset_t cur_target_page;
vm_offset_t cur_phys_addr;
vm_offset_t kern_virt_target_addr;
+ uint32_t kdp_fault_results = 0;
cur_target_page = atop(cur_target_addr);
if ((cur_target_page != prev_target_page) || validate_next_addr) {
+
/*
* Alright; it wasn't our previous page. So
* we must validate that there is a page
* table entry for this address under the
- * current kdp_pmap, and that it has default
+ * current pmap, and that it has default
* cache attributes (otherwise it may not be
* safe to access it).
*/
- cur_phys_addr = kdp_vtophys(kdp_pmap ? kdp_pmap : kernel_pmap, cur_target_addr);
+ cur_phys_addr = kdp_find_phys(map, cur_target_addr, stack_enable_faulting, &kdp_fault_results);
+ if (thread_trace_flags) {
+ if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
+ *thread_trace_flags |= kThreadTruncatedBT;
+ }
- if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
- return 0;
- }
+ if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
+ *thread_trace_flags |= kThreadTriedFaultBT;
+ }
- cur_wimg_bits = pmap_cache_attributes((ppnum_t) atop(cur_phys_addr));
+ if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
+ *thread_trace_flags |= kThreadFaultedBT;
+ }
+ }
- if ((cur_wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
+ if (cur_phys_addr == 0) {
return 0;
}
-
#if __x86_64__
kern_virt_target_addr = (vm_offset_t) PHYSMAP_PTOV(cur_phys_addr);
#else
validate_next_addr = TRUE;
}
+boolean_t
+stackshot_thread_is_idle_worker_unsafe(thread_t thread)
+{
+ /* When the pthread kext puts a worker thread to sleep, it will
+ * set kThreadWaitParkedWorkQueue in the block_hint of the thread
+ * struct. See parkit() in kern/kern_support.c in libpthread.
+ */
+ return (thread->state & TH_WAIT) &&
+ (thread->block_hint == kThreadWaitParkedWorkQueue);
+}
+
+/* Determine if a thread has waitinfo that stackshot can provide */
+static int
+stackshot_thread_has_valid_waitinfo(thread_t thread)
+{
+ if (!(thread->state & TH_WAIT))
+ return 0;
+
+ switch (thread->block_hint) {
+ // If set to None or is a parked work queue, ignore it
+ case kThreadWaitParkedWorkQueue:
+ case kThreadWaitNone:
+ return 0;
+ // There is a short window where the pthread kext removes a thread
+ // from its ksyn wait queue before waking the thread up
+ case kThreadWaitPThreadMutex:
+ case kThreadWaitPThreadRWLockRead:
+ case kThreadWaitPThreadRWLockWrite:
+ case kThreadWaitPThreadCondVar:
+ return (kdp_pthread_get_thread_kwq(thread) != NULL);
+ // All other cases are valid block hints if in a wait state
+ default:
+ return 1;
+ }
+}
+
+static void
+stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t *waitinfo)
+{
+ waitinfo->waiter = thread_tid(thread);
+ waitinfo->wait_type = thread->block_hint;
+ switch (waitinfo->wait_type) {
+ case kThreadWaitKernelMutex:
+ kdp_lck_mtx_find_owner(thread->waitq, thread->wait_event, waitinfo);
+ break;
+ case kThreadWaitPortReceive:
+ kdp_mqueue_recv_find_owner(thread->waitq, thread->wait_event, waitinfo);
+ break;
+ case kThreadWaitPortSend:
+ kdp_mqueue_send_find_owner(thread->waitq, thread->wait_event, waitinfo);
+ break;
+ case kThreadWaitSemaphore:
+ kdp_sema_find_owner(thread->waitq, thread->wait_event, waitinfo);
+ break;
+ case kThreadWaitUserLock:
+ kdp_ulock_find_owner(thread->waitq, thread->wait_event, waitinfo);
+ break;
+ case kThreadWaitKernelRWLockRead:
+ case kThreadWaitKernelRWLockWrite:
+ case kThreadWaitKernelRWLockUpgrade:
+ kdp_rwlck_find_owner(thread->waitq, thread->wait_event, waitinfo);
+ break;
+ case kThreadWaitPThreadMutex:
+ case kThreadWaitPThreadRWLockRead:
+ case kThreadWaitPThreadRWLockWrite:
+ case kThreadWaitPThreadCondVar:
+ kdp_pthread_find_owner(thread, waitinfo);
+ break;
+ default:
+ waitinfo->owner = 0;
+ waitinfo->context = 0;
+ break;
+ }
+}
projects / apple / xnu.git / blobdiff