]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/kern_stackshot.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.51.2.tar.gz
[apple/xnu.git] / osfmk / kern / kern_stackshot.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/mach_types.h>
30 #include <mach/vm_param.h>
31 #include <mach/mach_vm.h>
32 #include <mach/clock_types.h>
33 #include <sys/errno.h>
34 #include <sys/stackshot.h>
35 #ifdef IMPORTANCE_INHERITANCE
36 #include <ipc/ipc_importance.h>
37 #endif
38 #include <sys/appleapiopts.h>
39 #include <kern/debug.h>
40 #include <kern/block_hint.h>
41 #include <uuid/uuid.h>
42
43 #include <kdp/kdp_dyld.h>
44 #include <kdp/kdp_en_debugger.h>
45
46 #include <libsa/types.h>
47 #include <libkern/version.h>
48
49 #include <string.h> /* bcopy */
50
51 #include <kern/processor.h>
52 #include <kern/thread.h>
53 #include <kern/task.h>
54 #include <kern/telemetry.h>
55 #include <kern/clock.h>
56 #include <kern/policy_internal.h>
57 #include <vm/vm_map.h>
58 #include <vm/vm_kern.h>
59 #include <vm/vm_pageout.h>
60 #include <vm/vm_fault.h>
61 #include <vm/vm_shared_region.h>
62 #include <libkern/OSKextLibPrivate.h>
63
64 #if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
65 #include <pexpert/pexpert.h> /* For gPanicBase/gPanicBase */
66 #endif
67
68 extern unsigned int not_in_kdp;
69
70
71 /* indicate to the compiler that some accesses are unaligned */
72 typedef uint64_t unaligned_u64 __attribute__((aligned(1)));
73
74 extern addr64_t kdp_vtophys(pmap_t pmap, addr64_t va);
75 extern void * proc_get_uthread_uu_threadlist(void * uthread_v);
76
77 int kdp_snapshot = 0;
78 static kern_return_t stack_snapshot_ret = 0;
79 static uint32_t stack_snapshot_bytes_traced = 0;
80
81 static kcdata_descriptor_t stackshot_kcdata_p = NULL;
82 static void *stack_snapshot_buf;
83 static uint32_t stack_snapshot_bufsize;
84 int stack_snapshot_pid;
85 static uint32_t stack_snapshot_flags;
86 static uint64_t stack_snapshot_delta_since_timestamp;
87 static boolean_t panic_stackshot;
88
89 static boolean_t stack_enable_faulting = FALSE;
90 static struct stackshot_fault_stats fault_stats;
91
92 static unaligned_u64 * stackshot_duration_outer;
93 static uint64_t stackshot_microsecs;
94
95 void * kernel_stackshot_buf = NULL; /* Pointer to buffer for stackshots triggered from the kernel and retrieved later */
96 int kernel_stackshot_buf_size = 0;
97
98 void * stackshot_snapbuf = NULL; /* Used by stack_snapshot2 (to be removed) */
99
100 __private_extern__ void stackshot_init( void );
101 static boolean_t memory_iszero(void *addr, size_t size);
102 #if CONFIG_TELEMETRY
103 kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
104 #endif
105 uint32_t get_stackshot_estsize(uint32_t prev_size_hint);
106 kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config,
107 size_t stackshot_config_size, boolean_t stackshot_from_user);
108 kern_return_t do_stackshot(void *);
109 void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp);
110 boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
111 static int kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t *pBytesTraced);
112 kern_return_t kdp_stack_snapshot_geterror(void);
113 uint32_t kdp_stack_snapshot_bytes_traced(void);
114 static void kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap);
115 static boolean_t kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_result);
116 static boolean_t kdp_copyin_word(task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results);
117 static uint64_t proc_was_throttled_from_task(task_t task);
118 static void stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t * waitinfo);
119 static int stackshot_thread_has_valid_waitinfo(thread_t thread);
120
121 extern uint32_t workqueue_get_pwq_state_kdp(void *proc);
122
123 extern int proc_pid(void *p);
124 extern uint64_t proc_uniqueid(void *p);
125 extern uint64_t proc_was_throttled(void *p);
126 extern uint64_t proc_did_throttle(void *p);
127 static uint64_t proc_did_throttle_from_task(task_t task);
128 extern void proc_name_kdp(task_t task, char * buf, int size);
129 extern int proc_threadname_kdp(void * uth, char * buf, size_t size);
130 extern void proc_starttime_kdp(void * p, uint64_t * tv_sec, uint64_t * tv_usec, uint64_t * abstime);
131 extern int memorystatus_get_pressure_status_kdp(void);
132 extern boolean_t memorystatus_proc_is_dirty_unsafe(void * v);
133
134 extern int count_busy_buffers(void); /* must track with declaration in bsd/sys/buf_internal.h */
135 extern void bcopy_phys(addr64_t, addr64_t, vm_size_t);
136
137 #if CONFIG_TELEMETRY
138 extern kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
139 #endif /* CONFIG_TELEMETRY */
140
141 extern kern_return_t kern_stack_snapshot_with_reason(char* reason);
142 extern kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user);
143 /* Used for stackshot_thread_waitinfo_unsafe */
144 extern void kdp_lck_mtx_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
145 extern void kdp_sema_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
146 extern void kdp_mqueue_send_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
147 extern void kdp_mqueue_recv_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
148 extern void kdp_ulock_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
149 extern void kdp_rwlck_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo);
150 extern void kdp_pthread_find_owner(thread_t thread, thread_waitinfo_t *waitinfo);
151 extern void *kdp_pthread_get_thread_kwq(thread_t thread);
152
153 /*
154 * Validates that the given address is both a valid page and has
155 * default caching attributes for the current map. Returns
156 * 0 if the address is invalid, and a kernel virtual address for
157 * the given address if it is valid.
158 */
159 vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags);
160
161 #define KDP_FAULT_RESULT_PAGED_OUT 0x1 /* some data was unable to be retrieved */
162 #define KDP_FAULT_RESULT_TRIED_FAULT 0x2 /* tried to fault in data */
163 #define KDP_FAULT_RESULT_FAULTED_IN 0x4 /* successfully faulted in data */
164
165 /*
166 * Looks up the physical translation for the given address in the target map, attempting
167 * to fault data in if requested and it is not resident. Populates thread_trace_flags if requested
168 * as well.
169 */
170 vm_offset_t kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results);
171
172 static size_t stackshot_strlcpy(char *dst, const char *src, size_t maxlen);
173 static void stackshot_memcpy(void *dst, const void *src, size_t len);
174
175 /* Clears caching information used by the above validation routine
176 * (in case the current map has been changed or cleared).
177 */
178 void machine_trace_thread_clear_validation_cache(void);
179
180 #define MAX_FRAMES 1000
181 #define MAX_LOADINFOS 500
182 #define TASK_IMP_WALK_LIMIT 20
183
184 typedef struct thread_snapshot *thread_snapshot_t;
185 typedef struct task_snapshot *task_snapshot_t;
186
187 #if CONFIG_KDP_INTERACTIVE_DEBUGGING
188 extern kdp_send_t kdp_en_send_pkt;
189 #endif
190
191 /*
192 * Globals to support machine_trace_thread_get_kva.
193 */
194 static vm_offset_t prev_target_page = 0;
195 static vm_offset_t prev_target_kva = 0;
196 static boolean_t validate_next_addr = TRUE;
197
198 /*
199 * Stackshot locking and other defines.
200 */
201 static lck_grp_t *stackshot_subsys_lck_grp;
202 static lck_grp_attr_t *stackshot_subsys_lck_grp_attr;
203 static lck_attr_t *stackshot_subsys_lck_attr;
204 static lck_mtx_t stackshot_subsys_mutex;
205
206 #define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
207 #define STACKSHOT_SUBSYS_TRY_LOCK() lck_mtx_try_lock(&stackshot_subsys_mutex)
208 #define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
209
210 #define SANE_BOOTPROFILE_TRACEBUF_SIZE (64 * 1024 * 1024)
211 #define SANE_TRACEBUF_SIZE (8 * 1024 * 1024)
212
213 /*
214 * We currently set a ceiling of 3 milliseconds spent in the kdp fault path
215 * for non-panic stackshots where faulting is requested.
216 */
217 #define KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS (3 * NSEC_PER_MSEC)
218
219 #define STACKSHOT_SUPP_SIZE (16 * 1024) /* Minimum stackshot size */
220 #define TASK_UUID_AVG_SIZE (16 * sizeof(uuid_t)) /* Average space consumed by UUIDs/task */
221
222 /*
223 * Initialize the mutex governing access to the stack snapshot subsystem
224 * and other stackshot related bits.
225 */
226 __private_extern__ void
227 stackshot_init( void )
228 {
229 mach_timebase_info_data_t timebase;
230
231 stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
232
233 stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
234
235 stackshot_subsys_lck_attr = lck_attr_alloc_init();
236
237 lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
238
239 clock_timebase_info(&timebase);
240 fault_stats.sfs_system_max_fault_time = ((KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS * timebase.denom)/ timebase.numer);
241 }
242
243 /*
244 * Method for grabbing timer values safely, in the sense that no infinite loop will occur
245 * Certain flavors of the timer_grab function, which would seem to be the thing to use,
246 * can loop infinitely if called while the timer is in the process of being updated.
247 * Unfortunately, it is (rarely) possible to get inconsistent top and bottom halves of
248 * the timer using this method. This seems insoluble, since stackshot runs in a context
249 * where the timer might be half-updated, and has no way of yielding control just long
250 * enough to finish the update.
251 */
252
253 static uint64_t safe_grab_timer_value(struct timer *t)
254 {
255 #if defined(__LP64__)
256 return t->all_bits;
257 #else
258 uint64_t time = t->high_bits; /* endian independent grab */
259 time = (time << 32) | t->low_bits;
260 return time;
261 #endif
262 }
263
264 kern_return_t
265 stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, uint64_t delta_since_timestamp, unsigned *bytes_traced)
266 {
267 kern_return_t error = KERN_SUCCESS;
268 boolean_t istate;
269
270 if ((buf == NULL) || (size <= 0) || (bytes_traced == NULL)) {
271 return KERN_INVALID_ARGUMENT;
272 }
273
274 /* cap in individual stackshot to SANE_TRACEBUF_SIZE */
275 if (size > SANE_TRACEBUF_SIZE) {
276 size = SANE_TRACEBUF_SIZE;
277 }
278
279 /* Serialize tracing */
280 if (flags & STACKSHOT_TRYLOCK) {
281 if (!STACKSHOT_SUBSYS_TRY_LOCK()) {
282 return KERN_LOCK_OWNED;
283 }
284 } else {
285 STACKSHOT_SUBSYS_LOCK();
286 }
287
288 istate = ml_set_interrupts_enabled(FALSE);
289
290 struct kcdata_descriptor kcdata;
291 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ?
292 KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
293
294 error = kcdata_memory_static_init(&kcdata, (mach_vm_address_t)buf, hdr_tag, size,
295 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
296 if (error) {
297 goto out;
298 }
299
300 /* Preload trace parameters*/
301 kdp_snapshot_preflight(pid, buf, size, flags, &kcdata, delta_since_timestamp);
302
303 /* Trap to the debugger to obtain a coherent stack snapshot; this populates
304 * the trace buffer
305 */
306 stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
307
308 ml_set_interrupts_enabled(istate);
309
310 *bytes_traced = kdp_stack_snapshot_bytes_traced();
311
312 error = kdp_stack_snapshot_geterror();
313
314 out:
315 STACKSHOT_SUBSYS_UNLOCK();
316 return error;
317 }
318
319 #if CONFIG_TELEMETRY
320 kern_return_t
321 stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval)
322 {
323 int error = KERN_SUCCESS;
324 uint32_t bytes_traced = 0;
325
326 *retval = -1;
327
328 /*
329 * Control related operations
330 */
331 if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_ENABLE) {
332 telemetry_global_ctl(1);
333 *retval = 0;
334 goto exit;
335 } else if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_DISABLE) {
336 telemetry_global_ctl(0);
337 *retval = 0;
338 goto exit;
339 }
340
341 /*
342 * Data related operations
343 */
344 *retval = -1;
345
346 if ((((void*)tracebuf) == NULL) || (tracebuf_size == 0)) {
347 error = KERN_INVALID_ARGUMENT;
348 goto exit;
349 }
350
351 STACKSHOT_SUBSYS_LOCK();
352
353 if (flags & STACKSHOT_GET_MICROSTACKSHOT) {
354 if (tracebuf_size > SANE_TRACEBUF_SIZE) {
355 error = KERN_INVALID_ARGUMENT;
356 goto unlock_exit;
357 }
358
359 bytes_traced = tracebuf_size;
360 error = telemetry_gather(tracebuf, &bytes_traced,
361 (flags & STACKSHOT_SET_MICROSTACKSHOT_MARK) ? TRUE : FALSE);
362 *retval = (int)bytes_traced;
363 goto unlock_exit;
364 }
365
366 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
367
368 if (tracebuf_size > SANE_BOOTPROFILE_TRACEBUF_SIZE) {
369 error = KERN_INVALID_ARGUMENT;
370 goto unlock_exit;
371 }
372
373 bytes_traced = tracebuf_size;
374 error = bootprofile_gather(tracebuf, &bytes_traced);
375 *retval = (int)bytes_traced;
376 }
377
378 unlock_exit:
379 STACKSHOT_SUBSYS_UNLOCK();
380 exit:
381 return error;
382 }
383 #endif /* CONFIG_TELEMETRY */
384
385 /*
386 * Return the estimated size of a stackshot based on the
387 * number of currently running threads and tasks.
388 */
389 uint32_t
390 get_stackshot_estsize(uint32_t prev_size_hint)
391 {
392 vm_size_t thread_total;
393 vm_size_t task_total;
394 uint32_t estimated_size;
395
396 thread_total = (threads_count * sizeof(struct thread_snapshot));
397 task_total = (tasks_count * (sizeof(struct task_snapshot) + TASK_UUID_AVG_SIZE));
398
399 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE((thread_total + task_total + STACKSHOT_SUPP_SIZE), PAGE_MASK);
400 if (estimated_size < prev_size_hint) {
401 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE(prev_size_hint, PAGE_MASK);
402 }
403
404 return estimated_size;
405 }
406
407 /*
408 * stackshot_remap_buffer: Utility function to remap bytes_traced bytes starting at stackshotbuf
409 * into the current task's user space and subsequently copy out the address
410 * at which the buffer has been mapped in user space to out_buffer_addr.
411 *
412 * Inputs: stackshotbuf - pointer to the original buffer in the kernel's address space
413 * bytes_traced - length of the buffer to remap starting from stackshotbuf
414 * out_buffer_addr - pointer to placeholder where newly mapped buffer will be mapped.
415 * out_size_addr - pointer to be filled in with the size of the buffer
416 *
417 * Outputs: ENOSPC if there is not enough free space in the task's address space to remap the buffer
418 * EINVAL for all other errors returned by task_remap_buffer/mach_vm_remap
419 * an error from copyout
420 */
421 static kern_return_t
422 stackshot_remap_buffer(void *stackshotbuf, uint32_t bytes_traced, uint64_t out_buffer_addr, uint64_t out_size_addr)
423 {
424 int error = 0;
425 mach_vm_offset_t stackshotbuf_user_addr = (mach_vm_offset_t)NULL;
426 vm_prot_t cur_prot, max_prot;
427
428 error = mach_vm_remap(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0,
429 VM_FLAGS_ANYWHERE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT);
430 /*
431 * If the call to mach_vm_remap fails, we return the appropriate converted error
432 */
433 if (error == KERN_SUCCESS) {
434 /*
435 * If we fail to copy out the address or size of the new buffer, we remove the buffer mapping that
436 * we just made in the task's user space.
437 */
438 error = copyout(CAST_DOWN(void *, &stackshotbuf_user_addr), (user_addr_t)out_buffer_addr, sizeof(stackshotbuf_user_addr));
439 if (error != KERN_SUCCESS) {
440 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
441 return error;
442 }
443 error = copyout(&bytes_traced, (user_addr_t)out_size_addr, sizeof(bytes_traced));
444 if (error != KERN_SUCCESS) {
445 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
446 return error;
447 }
448 }
449 return error;
450 }
451
452 kern_return_t
453 kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user)
454 {
455 int error = 0;
456 boolean_t prev_interrupt_state;
457 uint32_t bytes_traced = 0;
458 uint32_t stackshotbuf_size = 0;
459 void * stackshotbuf = NULL;
460 kcdata_descriptor_t kcdata_p = NULL;
461
462 void * buf_to_free = NULL;
463 int size_to_free = 0;
464
465 /* Parsed arguments */
466 uint64_t out_buffer_addr;
467 uint64_t out_size_addr;
468 int pid = -1;
469 uint32_t flags;
470 uint64_t since_timestamp;
471 uint32_t size_hint = 0;
472
473 if(stackshot_config == NULL) {
474 return KERN_INVALID_ARGUMENT;
475 }
476
477 switch (stackshot_config_version) {
478 case STACKSHOT_CONFIG_TYPE:
479 if (stackshot_config_size != sizeof(stackshot_config_t)) {
480 return KERN_INVALID_ARGUMENT;
481 }
482 stackshot_config_t *config = (stackshot_config_t *) stackshot_config;
483 out_buffer_addr = config->sc_out_buffer_addr;
484 out_size_addr = config->sc_out_size_addr;
485 pid = config->sc_pid;
486 flags = config->sc_flags;
487 since_timestamp = config->sc_delta_timestamp;
488 if (config->sc_size <= SANE_TRACEBUF_SIZE) {
489 size_hint = config->sc_size;
490 }
491 break;
492 default:
493 return KERN_NOT_SUPPORTED;
494 }
495
496 /*
497 * Currently saving a kernel buffer and trylock are only supported from the
498 * internal/KEXT API.
499 */
500 if (stackshot_from_user) {
501 if (flags & (STACKSHOT_TRYLOCK | STACKSHOT_SAVE_IN_KERNEL_BUFFER | STACKSHOT_FROM_PANIC)) {
502 return KERN_NO_ACCESS;
503 }
504 } else {
505 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
506 return KERN_NOT_SUPPORTED;
507 }
508 }
509
510 if (!((flags & STACKSHOT_KCDATA_FORMAT) || (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER))) {
511 return KERN_NOT_SUPPORTED;
512 }
513
514 /*
515 * If we're not saving the buffer in the kernel pointer, we need a place to copy into.
516 */
517 if ((!out_buffer_addr || !out_size_addr) && !(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
518 return KERN_INVALID_ARGUMENT;
519 }
520
521 if (since_timestamp != 0 && ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) == 0)) {
522 return KERN_INVALID_ARGUMENT;
523 }
524
525 STACKSHOT_SUBSYS_LOCK();
526
527 if (flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER) {
528 /*
529 * Don't overwrite an existing stackshot
530 */
531 if (kernel_stackshot_buf != NULL) {
532 error = KERN_MEMORY_PRESENT;
533 goto error_exit;
534 }
535 } else if (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER) {
536 if ((kernel_stackshot_buf == NULL) || (kernel_stackshot_buf_size <= 0)) {
537 error = KERN_NOT_IN_SET;
538 goto error_exit;
539 }
540 error = stackshot_remap_buffer(kernel_stackshot_buf, kernel_stackshot_buf_size,
541 out_buffer_addr, out_size_addr);
542 /*
543 * If we successfully remapped the buffer into the user's address space, we
544 * set buf_to_free and size_to_free so the prior kernel mapping will be removed
545 * and then clear the kernel stackshot pointer and associated size.
546 */
547 if (error == KERN_SUCCESS) {
548 buf_to_free = kernel_stackshot_buf;
549 size_to_free = (int) VM_MAP_ROUND_PAGE(kernel_stackshot_buf_size, PAGE_MASK);
550 kernel_stackshot_buf = NULL;
551 kernel_stackshot_buf_size = 0;
552 }
553
554 goto error_exit;
555 }
556
557 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
558 void *bootprofile = NULL;
559 uint32_t len = 0;
560 #if CONFIG_TELEMETRY
561 bootprofile_get(&bootprofile, &len);
562 #endif
563 if (!bootprofile || !len) {
564 error = KERN_NOT_IN_SET;
565 goto error_exit;
566 }
567 error = stackshot_remap_buffer(bootprofile, len, out_buffer_addr, out_size_addr);
568 goto error_exit;
569 }
570
571 stackshotbuf_size = get_stackshot_estsize(size_hint);
572
573 for (; stackshotbuf_size <= SANE_TRACEBUF_SIZE; stackshotbuf_size <<= 1) {
574 if (kmem_alloc(kernel_map, (vm_offset_t *)&stackshotbuf, stackshotbuf_size, VM_KERN_MEMORY_DIAG) != KERN_SUCCESS) {
575 error = KERN_RESOURCE_SHORTAGE;
576 goto error_exit;
577 }
578
579 /*
580 * If someone has panicked, don't try and enter the debugger
581 */
582 if (panic_active()) {
583 error = KERN_RESOURCE_SHORTAGE;
584 goto error_exit;
585 }
586
587 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
588 kcdata_p = kcdata_memory_alloc_init((mach_vm_address_t)stackshotbuf, hdr_tag, stackshotbuf_size,
589 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
590
591 stackshot_duration_outer = NULL;
592 uint64_t time_start = mach_absolute_time();
593
594 /*
595 * Disable interrupts and save the current interrupt state.
596 */
597 prev_interrupt_state = ml_set_interrupts_enabled(FALSE);
598
599 /*
600 * Load stackshot parameters.
601 */
602 kdp_snapshot_preflight(pid, stackshotbuf, stackshotbuf_size, flags, kcdata_p, since_timestamp);
603
604 /*
605 * Trap to the debugger to obtain a stackshot (this will populate the buffer).
606 */
607 stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
608
609 ml_set_interrupts_enabled(prev_interrupt_state);
610
611 /* record the duration that interupts were disabled */
612
613 uint64_t time_end = mach_absolute_time();
614 if (stackshot_duration_outer) {
615 *stackshot_duration_outer = time_end - time_start;
616 }
617
618 error = kdp_stack_snapshot_geterror();
619 if (error != KERN_SUCCESS) {
620 if (kcdata_p != NULL) {
621 kcdata_memory_destroy(kcdata_p);
622 kcdata_p = NULL;
623 stackshot_kcdata_p = NULL;
624 }
625 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
626 stackshotbuf = NULL;
627 if (error == KERN_INSUFFICIENT_BUFFER_SIZE) {
628 /*
629 * If we didn't allocate a big enough buffer, deallocate and try again.
630 */
631 continue;
632 } else {
633 goto error_exit;
634 }
635 }
636
637 bytes_traced = kdp_stack_snapshot_bytes_traced();
638
639 if (bytes_traced <= 0) {
640 error = KERN_ABORTED;
641 goto error_exit;
642 }
643
644 assert(bytes_traced <= stackshotbuf_size);
645 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
646 error = stackshot_remap_buffer(stackshotbuf, bytes_traced, out_buffer_addr, out_size_addr);
647 goto error_exit;
648 }
649
650 /*
651 * Save the stackshot in the kernel buffer.
652 */
653 kernel_stackshot_buf = stackshotbuf;
654 kernel_stackshot_buf_size = bytes_traced;
655 /*
656 * Figure out if we didn't use all the pages in the buffer. If so, we set buf_to_free to the beginning of
657 * the next page after the end of the stackshot in the buffer so that the kmem_free clips the buffer and
658 * update size_to_free for kmem_free accordingly.
659 */
660 size_to_free = stackshotbuf_size - (int) VM_MAP_ROUND_PAGE(bytes_traced, PAGE_MASK);
661
662 assert(size_to_free >= 0);
663
664 if (size_to_free != 0) {
665 buf_to_free = (void *)((uint64_t)stackshotbuf + stackshotbuf_size - size_to_free);
666 }
667
668 stackshotbuf = NULL;
669 stackshotbuf_size = 0;
670 goto error_exit;
671 }
672
673 if (stackshotbuf_size > SANE_TRACEBUF_SIZE) {
674 error = KERN_RESOURCE_SHORTAGE;
675 }
676
677 error_exit:
678 if (kcdata_p != NULL) {
679 kcdata_memory_destroy(kcdata_p);
680 kcdata_p = NULL;
681 stackshot_kcdata_p = NULL;
682 }
683
684 if (stackshotbuf != NULL) {
685 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
686 }
687 if (buf_to_free != NULL) {
688 kmem_free(kernel_map, (vm_offset_t)buf_to_free, size_to_free);
689 }
690 STACKSHOT_SUBSYS_UNLOCK();
691 return error;
692 }
693
694 /* Cache stack snapshot parameters in preparation for a trace */
695 void
696 kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags,
697 kcdata_descriptor_t data_p, uint64_t since_timestamp)
698 {
699 uint64_t microsecs = 0, secs = 0;
700 clock_get_calendar_microtime((clock_sec_t *)&secs, (clock_usec_t *)&microsecs);
701
702 stackshot_microsecs = microsecs + (secs * USEC_PER_SEC);
703 stack_snapshot_pid = pid;
704 stack_snapshot_buf = tracebuf;
705 stack_snapshot_bufsize = tracebuf_size;
706 stack_snapshot_flags = flags;
707 stack_snapshot_delta_since_timestamp = since_timestamp;
708
709 panic_stackshot = ((flags & STACKSHOT_FROM_PANIC) != 0);
710
711 if (data_p != NULL) {
712 stackshot_kcdata_p = data_p;
713 }
714 }
715
716 kern_return_t
717 kdp_stack_snapshot_geterror(void)
718 {
719 return stack_snapshot_ret;
720 }
721
722 uint32_t
723 kdp_stack_snapshot_bytes_traced(void)
724 {
725 return stack_snapshot_bytes_traced;
726 }
727
728 static boolean_t memory_iszero(void *addr, size_t size)
729 {
730 char *data = (char *)addr;
731 for (size_t i = 0; i < size; i++){
732 if (data[i] != 0)
733 return FALSE;
734 }
735 return TRUE;
736 }
737
738 #define kcd_end_address(kcd) ((void *)((uint64_t)((kcd)->kcd_addr_begin) + kcdata_memory_get_used_bytes((kcd))))
739 #define kcd_max_address(kcd) ((void *)((kcd)->kcd_addr_begin + (kcd)->kcd_length))
740 /*
741 * Use of the kcd_exit_on_error(action) macro requires a local
742 * 'kern_return_t error' variable and 'error_exit' label.
743 */
744 #define kcd_exit_on_error(action) \
745 do { \
746 if (KERN_SUCCESS != (error = (action))) { \
747 if (error == KERN_RESOURCE_SHORTAGE) { \
748 error = KERN_INSUFFICIENT_BUFFER_SIZE; \
749 } \
750 goto error_exit; \
751 } \
752 } while (0); /* end kcd_exit_on_error */
753
754 static uint64_t
755 kcdata_get_task_ss_flags(task_t task)
756 {
757 uint64_t ss_flags = 0;
758 boolean_t task64 = task_has_64BitAddr(task);
759
760 if (task64)
761 ss_flags |= kUser64_p;
762 if (!task->active || task_is_a_corpse(task))
763 ss_flags |= kTerminatedSnapshot;
764 if (task->pidsuspended)
765 ss_flags |= kPidSuspended;
766 if (task->frozen)
767 ss_flags |= kFrozen;
768 if (task->effective_policy.tep_darwinbg == 1)
769 ss_flags |= kTaskDarwinBG;
770 if (task->requested_policy.trp_role == TASK_FOREGROUND_APPLICATION)
771 ss_flags |= kTaskIsForeground;
772 if (task->requested_policy.trp_boosted == 1)
773 ss_flags |= kTaskIsBoosted;
774 if (task->effective_policy.tep_sup_active == 1)
775 ss_flags |= kTaskIsSuppressed;
776 #if CONFIG_MEMORYSTATUS
777 if (memorystatus_proc_is_dirty_unsafe(task->bsd_info))
778 ss_flags |= kTaskIsDirty;
779 #endif
780
781 ss_flags |= (0x7 & workqueue_get_pwq_state_kdp(task->bsd_info)) << 17;
782
783 #if IMPORTANCE_INHERITANCE
784 if (task->task_imp_base) {
785 if (task->task_imp_base->iit_donor)
786 ss_flags |= kTaskIsImpDonor;
787 if (task->task_imp_base->iit_live_donor)
788 ss_flags |= kTaskIsLiveImpDonor;
789 }
790 #endif
791
792 return ss_flags;
793 }
794
795 static kern_return_t
796 kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyld_uuid_info_64_v2 *sys_shared_cache_loadinfo, uint32_t trace_flags, unaligned_u64 *task_snap_ss_flags)
797 {
798 kern_return_t error = KERN_SUCCESS;
799 mach_vm_address_t out_addr = 0;
800
801 uint8_t shared_cache_identifier[16];
802 uint64_t shared_cache_slide = 0;
803 uint64_t shared_cache_base_address = 0;
804 int task_pid = pid_from_task(task);
805 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
806 uint32_t kdp_fault_results = 0;
807
808 assert(task_snap_ss_flags != NULL);
809
810 if (task->shared_region && ml_validate_nofault((vm_offset_t)task->shared_region, sizeof(struct vm_shared_region))) {
811 struct vm_shared_region *sr = task->shared_region;
812 shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping;
813 } else {
814 *task_snap_ss_flags |= kTaskSharedRegionInfoUnavailable;
815 }
816
817 if (!shared_cache_base_address ||
818 !kdp_copyin(task->map, shared_cache_base_address + offsetof(struct _dyld_cache_header, uuid),
819 shared_cache_identifier, sizeof(shared_cache_identifier), should_fault, &kdp_fault_results)) {
820 goto error_exit;
821 }
822
823 if (task->shared_region) {
824 /*
825 * No refcounting here, but we are in debugger
826 * context, so that should be safe.
827 */
828 shared_cache_slide = task->shared_region->sr_slide_info.slide;
829 } else {
830 shared_cache_slide = 0;
831 }
832
833 if (sys_shared_cache_loadinfo) {
834 if (task_pid == 1) {
835 /* save launchd's shared cache info as system level */
836 stackshot_memcpy(sys_shared_cache_loadinfo->imageUUID, shared_cache_identifier, sizeof(sys_shared_cache_loadinfo->imageUUID));
837 sys_shared_cache_loadinfo->imageLoadAddress = shared_cache_slide;
838 sys_shared_cache_loadinfo->imageSlidBaseAddress = shared_cache_slide + task->shared_region->sr_base_address;
839
840 goto error_exit;
841 } else {
842 if (shared_cache_slide == sys_shared_cache_loadinfo->imageLoadAddress &&
843 0 == memcmp(shared_cache_identifier, sys_shared_cache_loadinfo->imageUUID,
844 sizeof(sys_shared_cache_loadinfo->imageUUID))) {
845 /* skip adding shared cache info. its same as system level one */
846 goto error_exit;
847 }
848 }
849 }
850
851 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64_v2), &out_addr));
852 struct dyld_uuid_info_64_v2 *shared_cache_data = (struct dyld_uuid_info_64_v2 *)out_addr;
853 shared_cache_data->imageLoadAddress = shared_cache_slide;
854 stackshot_memcpy(shared_cache_data->imageUUID, shared_cache_identifier, sizeof(shared_cache_data->imageUUID));
855 shared_cache_data->imageSlidBaseAddress = shared_cache_base_address;
856
857 error_exit:
858 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
859 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
860 }
861
862 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
863 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
864 }
865
866 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
867 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
868 }
869
870 return error;
871 }
872
873 static kern_return_t
874 kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 *task_snap_ss_flags)
875 {
876 boolean_t save_loadinfo_p = ((trace_flags & STACKSHOT_SAVE_LOADINFO) != 0);
877 boolean_t save_kextloadinfo_p = ((trace_flags & STACKSHOT_SAVE_KEXT_LOADINFO) != 0);
878 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
879 boolean_t minimize_uuids = collect_delta_stackshot && ((trace_flags & STACKSHOT_TAILSPIN) != 0);
880 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
881
882 kern_return_t error = KERN_SUCCESS;
883 mach_vm_address_t out_addr = 0;
884
885 uint32_t uuid_info_count = 0;
886 mach_vm_address_t uuid_info_addr = 0;
887 uint64_t uuid_info_timestamp = 0;
888 uint32_t kdp_fault_results = 0;
889
890 assert(task_snap_ss_flags != NULL);
891
892 int task_pid = pid_from_task(task);
893 boolean_t task64 = task_has_64BitAddr(task);
894
895 if (save_loadinfo_p && have_pmap && task->active && task_pid > 0) {
896 /* Read the dyld_all_image_infos struct from the task memory to get UUID array count and location */
897 if (task64) {
898 struct user64_dyld_all_image_infos task_image_infos;
899 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
900 sizeof(struct user64_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
901 uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
902 uuid_info_addr = task_image_infos.uuidArray;
903 if (task_image_infos.version >= 15) {
904 uuid_info_timestamp = task_image_infos.timestamp;
905 }
906 }
907 } else {
908 struct user32_dyld_all_image_infos task_image_infos;
909 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
910 sizeof(struct user32_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
911 uuid_info_count = task_image_infos.uuidArrayCount;
912 uuid_info_addr = task_image_infos.uuidArray;
913 if (task_image_infos.version >= 15) {
914 uuid_info_timestamp = task_image_infos.timestamp;
915 }
916 }
917 }
918
919 /*
920 * If we get a NULL uuid_info_addr (which can happen when we catch dyld in the middle of updating
921 * this data structure), we zero the uuid_info_count so that we won't even try to save load info
922 * for this task.
923 */
924 if (!uuid_info_addr) {
925 uuid_info_count = 0;
926 }
927 }
928
929 if (have_pmap && task_pid == 0) {
930 if (save_kextloadinfo_p && ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader))) {
931 uuid_info_count = gLoadedKextSummaries->numSummaries + 1; /* include main kernel UUID */
932 } else {
933 uuid_info_count = 1; /* include kernelcache UUID (embedded) or kernel UUID (desktop) */
934 }
935 }
936
937 if (task_pid > 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
938 if (minimize_uuids && uuid_info_timestamp != 0 && uuid_info_timestamp < stack_snapshot_delta_since_timestamp)
939 goto error_exit;
940
941 uint32_t uuid_info_size = (uint32_t)(task64 ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
942 uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
943
944 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, (task64 ? KCDATA_TYPE_LIBRARY_LOADINFO64 : KCDATA_TYPE_LIBRARY_LOADINFO),
945 uuid_info_size, uuid_info_count, &out_addr));
946
947 /* Copy in the UUID info array
948 * It may be nonresident, in which case just fix up nloadinfos to 0 in the task_snap
949 */
950 if (have_pmap && !kdp_copyin(task->map, uuid_info_addr, (void *)out_addr, uuid_info_array_size, should_fault, &kdp_fault_results)) {
951 bzero((void *)out_addr, uuid_info_array_size);
952 }
953
954 } else if (task_pid == 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
955 if (minimize_uuids && gLoadedKextSummaries != 0 && gLoadedKextSummariesTimestamp < stack_snapshot_delta_since_timestamp)
956 goto error_exit;
957
958 uintptr_t image_load_address;
959
960 do {
961
962
963 if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) {
964 /* Kernel UUID not found or inaccessible */
965 break;
966 }
967
968 kcd_exit_on_error(kcdata_get_memory_addr_for_array(
969 kcd, (sizeof(kernel_uuid_info) == sizeof(struct user64_dyld_uuid_info)) ? KCDATA_TYPE_LIBRARY_LOADINFO64
970 : KCDATA_TYPE_LIBRARY_LOADINFO,
971 sizeof(kernel_uuid_info), uuid_info_count, &out_addr));
972 kernel_uuid_info *uuid_info_array = (kernel_uuid_info *)out_addr;
973 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(vm_kernel_stext);
974 uuid_info_array[0].imageLoadAddress = image_load_address;
975 stackshot_memcpy(&uuid_info_array[0].imageUUID, kernel_uuid, sizeof(uuid_t));
976
977 if (save_kextloadinfo_p &&
978 ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader)) &&
979 ml_validate_nofault((vm_offset_t)(&gLoadedKextSummaries->summaries[0]),
980 gLoadedKextSummaries->entry_size * gLoadedKextSummaries->numSummaries)) {
981 uint32_t kexti;
982 for (kexti=0 ; kexti < gLoadedKextSummaries->numSummaries; kexti++) {
983 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(gLoadedKextSummaries->summaries[kexti].address);
984 uuid_info_array[kexti + 1].imageLoadAddress = image_load_address;
985 stackshot_memcpy(&uuid_info_array[kexti + 1].imageUUID, &gLoadedKextSummaries->summaries[kexti].uuid, sizeof(uuid_t));
986 }
987 }
988 } while(0);
989 }
990
991 error_exit:
992 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
993 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
994 }
995
996 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
997 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
998 }
999
1000 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
1001 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
1002 }
1003
1004 return error;
1005 }
1006
1007 static kern_return_t
1008 kcdata_record_task_iostats(kcdata_descriptor_t kcd, task_t task)
1009 {
1010 kern_return_t error = KERN_SUCCESS;
1011 mach_vm_address_t out_addr = 0;
1012
1013 /* I/O Statistics if any counters are non zero */
1014 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1015 if (task->task_io_stats && !memory_iszero(task->task_io_stats, sizeof(struct io_stat_info))) {
1016 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1017 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1018 _iostat->ss_disk_reads_count = task->task_io_stats->disk_reads.count;
1019 _iostat->ss_disk_reads_size = task->task_io_stats->disk_reads.size;
1020 _iostat->ss_disk_writes_count = (task->task_io_stats->total_io.count - task->task_io_stats->disk_reads.count);
1021 _iostat->ss_disk_writes_size = (task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size);
1022 _iostat->ss_paging_count = task->task_io_stats->paging.count;
1023 _iostat->ss_paging_size = task->task_io_stats->paging.size;
1024 _iostat->ss_non_paging_count = (task->task_io_stats->total_io.count - task->task_io_stats->paging.count);
1025 _iostat->ss_non_paging_size = (task->task_io_stats->total_io.size - task->task_io_stats->paging.size);
1026 _iostat->ss_metadata_count = task->task_io_stats->metadata.count;
1027 _iostat->ss_metadata_size = task->task_io_stats->metadata.size;
1028 _iostat->ss_data_count = (task->task_io_stats->total_io.count - task->task_io_stats->metadata.count);
1029 _iostat->ss_data_size = (task->task_io_stats->total_io.size - task->task_io_stats->metadata.size);
1030 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1031 _iostat->ss_io_priority_count[i] = task->task_io_stats->io_priority[i].count;
1032 _iostat->ss_io_priority_size[i] = task->task_io_stats->io_priority[i].size;
1033 }
1034 }
1035
1036 error_exit:
1037 return error;
1038 }
1039
1040 static kern_return_t
1041 kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 **task_snap_ss_flags)
1042 {
1043 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1044 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
1045
1046 kern_return_t error = KERN_SUCCESS;
1047 mach_vm_address_t out_addr = 0;
1048 struct task_snapshot_v2 * cur_tsnap = NULL;
1049
1050 assert(task_snap_ss_flags != NULL);
1051
1052 int task_pid = pid_from_task(task);
1053 uint64_t task_uniqueid = get_task_uniqueid(task);
1054 uint64_t proc_starttime_secs = 0;
1055
1056 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_SNAPSHOT, sizeof(struct task_snapshot_v2), &out_addr));
1057
1058 cur_tsnap = (struct task_snapshot_v2 *)out_addr;
1059
1060 cur_tsnap->ts_unique_pid = task_uniqueid;
1061 cur_tsnap->ts_ss_flags = kcdata_get_task_ss_flags(task);
1062 *task_snap_ss_flags = (unaligned_u64 *)&cur_tsnap->ts_ss_flags;
1063 cur_tsnap->ts_user_time_in_terminated_threads = task->total_user_time;
1064 cur_tsnap->ts_system_time_in_terminated_threads = task->total_system_time;
1065
1066 proc_starttime_kdp(task->bsd_info, &proc_starttime_secs, NULL, NULL);
1067 cur_tsnap->ts_p_start_sec = proc_starttime_secs;
1068
1069 cur_tsnap->ts_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
1070 cur_tsnap->ts_max_resident_size = get_task_resident_max(task);
1071 cur_tsnap->ts_suspend_count = task->suspend_count;
1072 cur_tsnap->ts_faults = task->faults;
1073 cur_tsnap->ts_pageins = task->pageins;
1074 cur_tsnap->ts_cow_faults = task->cow_faults;
1075 cur_tsnap->ts_was_throttled = (uint32_t) proc_was_throttled_from_task(task);
1076 cur_tsnap->ts_did_throttle = (uint32_t) proc_did_throttle_from_task(task);
1077 cur_tsnap->ts_latency_qos = (task->effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED) ?
1078 LATENCY_QOS_TIER_UNSPECIFIED : ((0xFF << 16) | task->effective_policy.tep_latency_qos);
1079 cur_tsnap->ts_pid = task_pid;
1080
1081 /* Add the BSD process identifiers */
1082 if (task_pid != -1 && task->bsd_info != NULL)
1083 proc_name_kdp(task, cur_tsnap->ts_p_comm, sizeof(cur_tsnap->ts_p_comm));
1084 else {
1085 cur_tsnap->ts_p_comm[0] = '\0';
1086 #if IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG)
1087 if (task->task_imp_base != NULL) {
1088 stackshot_strlcpy(cur_tsnap->ts_p_comm, &task->task_imp_base->iit_procname[0],
1089 MIN((int)sizeof(task->task_imp_base->iit_procname), (int)sizeof(cur_tsnap->ts_p_comm)));
1090 }
1091 #endif
1092 }
1093
1094 if (collect_iostats) {
1095 kcd_exit_on_error(kcdata_record_task_iostats(kcd, task));
1096 }
1097
1098 error_exit:
1099 return error;
1100 }
1101
1102 static kern_return_t
1103 kcdata_record_task_delta_snapshot(kcdata_descriptor_t kcd, task_t task, boolean_t have_pmap, unaligned_u64 **task_snap_ss_flags)
1104 {
1105 kern_return_t error = KERN_SUCCESS;
1106 struct task_delta_snapshot_v2 * cur_tsnap = NULL;
1107 mach_vm_address_t out_addr = 0;
1108
1109 uint64_t task_uniqueid = get_task_uniqueid(task);
1110 assert(task_snap_ss_flags != NULL);
1111
1112 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT, sizeof(struct task_delta_snapshot_v2), &out_addr));
1113
1114 cur_tsnap = (struct task_delta_snapshot_v2 *)out_addr;
1115
1116 cur_tsnap->tds_unique_pid = task_uniqueid;
1117 cur_tsnap->tds_ss_flags = kcdata_get_task_ss_flags(task);
1118 *task_snap_ss_flags = (unaligned_u64 *)&cur_tsnap->tds_ss_flags;
1119
1120 cur_tsnap->tds_user_time_in_terminated_threads = task->total_user_time;
1121 cur_tsnap->tds_system_time_in_terminated_threads = task->total_system_time;
1122
1123 cur_tsnap->tds_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
1124
1125 cur_tsnap->tds_max_resident_size = get_task_resident_max(task);
1126 cur_tsnap->tds_suspend_count = task->suspend_count;
1127 cur_tsnap->tds_faults = task->faults;
1128 cur_tsnap->tds_pageins = task->pageins;
1129 cur_tsnap->tds_cow_faults = task->cow_faults;
1130 cur_tsnap->tds_was_throttled = (uint32_t)proc_was_throttled_from_task(task);
1131 cur_tsnap->tds_did_throttle = (uint32_t)proc_did_throttle_from_task(task);
1132 cur_tsnap->tds_latency_qos = (task-> effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED)
1133 ? LATENCY_QOS_TIER_UNSPECIFIED
1134 : ((0xFF << 16) | task-> effective_policy.tep_latency_qos);
1135
1136 error_exit:
1137 return error;
1138 }
1139
1140 static kern_return_t
1141 kcdata_record_thread_iostats(kcdata_descriptor_t kcd, thread_t thread)
1142 {
1143 kern_return_t error = KERN_SUCCESS;
1144 mach_vm_address_t out_addr = 0;
1145
1146 /* I/O Statistics */
1147 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1148 if (thread->thread_io_stats && !memory_iszero(thread->thread_io_stats, sizeof(struct io_stat_info))) {
1149 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1150 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1151 _iostat->ss_disk_reads_count = thread->thread_io_stats->disk_reads.count;
1152 _iostat->ss_disk_reads_size = thread->thread_io_stats->disk_reads.size;
1153 _iostat->ss_disk_writes_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->disk_reads.count);
1154 _iostat->ss_disk_writes_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->disk_reads.size);
1155 _iostat->ss_paging_count = thread->thread_io_stats->paging.count;
1156 _iostat->ss_paging_size = thread->thread_io_stats->paging.size;
1157 _iostat->ss_non_paging_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->paging.count);
1158 _iostat->ss_non_paging_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->paging.size);
1159 _iostat->ss_metadata_count = thread->thread_io_stats->metadata.count;
1160 _iostat->ss_metadata_size = thread->thread_io_stats->metadata.size;
1161 _iostat->ss_data_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->metadata.count);
1162 _iostat->ss_data_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->metadata.size);
1163 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1164 _iostat->ss_io_priority_count[i] = thread->thread_io_stats->io_priority[i].count;
1165 _iostat->ss_io_priority_size[i] = thread->thread_io_stats->io_priority[i].size;
1166 }
1167 }
1168
1169 error_exit:
1170 return error;
1171 }
1172
1173 static kern_return_t
1174 kcdata_record_thread_snapshot(
1175 kcdata_descriptor_t kcd, thread_t thread, task_t task, uint32_t trace_flags, boolean_t have_pmap, boolean_t thread_on_core)
1176 {
1177 boolean_t dispatch_p = ((trace_flags & STACKSHOT_GET_DQ) != 0);
1178 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1179 boolean_t trace_fp_p = ((trace_flags & STACKSHOT_TAILSPIN) == 0);
1180 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1181 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
1182
1183 kern_return_t error = KERN_SUCCESS;
1184 mach_vm_address_t out_addr = 0;
1185 int saved_count = 0;
1186
1187 struct thread_snapshot_v3 * cur_thread_snap = NULL;
1188 char cur_thread_name[STACKSHOT_MAX_THREAD_NAME_SIZE];
1189 uint64_t tval = 0;
1190 boolean_t task64 = task_has_64BitAddr(task);
1191
1192 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v3), &out_addr));
1193 cur_thread_snap = (struct thread_snapshot_v3 *)out_addr;
1194
1195 /* Populate the thread snapshot header */
1196 cur_thread_snap->ths_thread_id = thread_tid(thread);
1197 cur_thread_snap->ths_wait_event = VM_KERNEL_UNSLIDE_OR_PERM(thread->wait_event);
1198 cur_thread_snap->ths_continuation = VM_KERNEL_UNSLIDE(thread->continuation);
1199 cur_thread_snap->ths_total_syscalls = thread->syscalls_mach + thread->syscalls_unix;
1200
1201 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1202 cur_thread_snap->ths_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1203 else
1204 cur_thread_snap->ths_voucher_identifier = 0;
1205
1206 cur_thread_snap->ths_dqserialnum = 0;
1207 if (dispatch_p && (task != kernel_task) && (task->active) && have_pmap) {
1208 uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
1209 if (dqkeyaddr != 0) {
1210 uint64_t dqaddr = 0;
1211 boolean_t copyin_ok = kdp_copyin_word(task, dqkeyaddr, &dqaddr, FALSE, NULL);
1212 if (copyin_ok && dqaddr != 0) {
1213 uint64_t dqserialnumaddr = dqaddr + get_task_dispatchqueue_serialno_offset(task);
1214 uint64_t dqserialnum = 0;
1215 copyin_ok = kdp_copyin_word(task, dqserialnumaddr, &dqserialnum, FALSE, NULL);
1216 if (copyin_ok) {
1217 cur_thread_snap->ths_ss_flags |= kHasDispatchSerial;
1218 cur_thread_snap->ths_dqserialnum = dqserialnum;
1219 }
1220 }
1221 }
1222 }
1223
1224 tval = safe_grab_timer_value(&thread->user_timer);
1225 cur_thread_snap->ths_user_time = tval;
1226 tval = safe_grab_timer_value(&thread->system_timer);
1227
1228 if (thread->precise_user_kernel_time) {
1229 cur_thread_snap->ths_sys_time = tval;
1230 } else {
1231 cur_thread_snap->ths_user_time += tval;
1232 cur_thread_snap->ths_sys_time = 0;
1233 }
1234
1235 cur_thread_snap->ths_ss_flags = 0;
1236 if (thread->effective_policy.thep_darwinbg)
1237 cur_thread_snap->ths_ss_flags |= kThreadDarwinBG;
1238 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1239 cur_thread_snap->ths_ss_flags |= kThreadIOPassive;
1240 if (thread->suspend_count > 0)
1241 cur_thread_snap->ths_ss_flags |= kThreadSuspended;
1242 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1243 cur_thread_snap->ths_ss_flags |= kGlobalForcedIdle;
1244 if (thread_on_core)
1245 cur_thread_snap->ths_ss_flags |= kThreadOnCore;
1246 if (stackshot_thread_is_idle_worker_unsafe(thread))
1247 cur_thread_snap->ths_ss_flags |= kThreadIdleWorker;
1248
1249 /* make sure state flags defined in kcdata.h still match internal flags */
1250 static_assert(SS_TH_WAIT == TH_WAIT);
1251 static_assert(SS_TH_SUSP == TH_SUSP);
1252 static_assert(SS_TH_RUN == TH_RUN);
1253 static_assert(SS_TH_UNINT == TH_UNINT);
1254 static_assert(SS_TH_TERMINATE == TH_TERMINATE);
1255 static_assert(SS_TH_TERMINATE2 == TH_TERMINATE2);
1256 static_assert(SS_TH_IDLE == TH_IDLE);
1257
1258 cur_thread_snap->ths_last_run_time = thread->last_run_time;
1259 cur_thread_snap->ths_last_made_runnable_time = thread->last_made_runnable_time;
1260 cur_thread_snap->ths_state = thread->state;
1261 cur_thread_snap->ths_sched_flags = thread->sched_flags;
1262 cur_thread_snap->ths_base_priority = thread->base_pri;
1263 cur_thread_snap->ths_sched_priority = thread->sched_pri;
1264 cur_thread_snap->ths_eqos = thread->effective_policy.thep_qos;
1265 cur_thread_snap->ths_rqos = thread->requested_policy.thrp_qos;
1266 cur_thread_snap->ths_rqos_override = thread->requested_policy.thrp_qos_override;
1267 cur_thread_snap->ths_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1268 cur_thread_snap->ths_thread_t = VM_KERNEL_ADDRPERM(thread);
1269
1270 /* if there is thread name then add to buffer */
1271 cur_thread_name[0] = '\0';
1272 proc_threadname_kdp(thread->uthread, cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE);
1273 if (strnlen(cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE) > 0) {
1274 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_NAME, sizeof(cur_thread_name), &out_addr));
1275 stackshot_memcpy((void *)out_addr, (void *)cur_thread_name, sizeof(cur_thread_name));
1276 }
1277
1278 /* record system and user cpu times */
1279 time_value_t user_time;
1280 time_value_t system_time;
1281 thread_read_times(thread, &user_time, &system_time);
1282 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_CPU_TIMES, sizeof(struct stackshot_cpu_times), &out_addr));
1283 struct stackshot_cpu_times * stackshot_cpu_times = (struct stackshot_cpu_times *)out_addr;
1284 stackshot_cpu_times->user_usec = ((uint64_t)user_time.seconds) * USEC_PER_SEC + user_time.microseconds;
1285 stackshot_cpu_times->system_usec = ((uint64_t)system_time.seconds) * USEC_PER_SEC + system_time.microseconds;
1286
1287 /* Trace user stack, if any */
1288 if (!active_kthreads_only_p && task->active && thread->task->map != kernel_map) {
1289 uint32_t thread_snapshot_flags = 0;
1290 /* 64-bit task? */
1291 if (task64) {
1292 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1293 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE,
1294 trace_fp_p, &thread_snapshot_flags);
1295 if (saved_count > 0) {
1296 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1297 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME64
1298 : STACKSHOT_KCTYPE_USER_STACKLR64,
1299 frame_size, saved_count / frame_size, &out_addr));
1300 cur_thread_snap->ths_ss_flags |= kUser64_p;
1301 }
1302 } else {
1303 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1304 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE, trace_fp_p,
1305 &thread_snapshot_flags);
1306 if (saved_count > 0) {
1307 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1308 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME
1309 : STACKSHOT_KCTYPE_USER_STACKLR,
1310 frame_size, saved_count / frame_size, &out_addr));
1311 }
1312 }
1313
1314 if (thread_snapshot_flags != 0) {
1315 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1316 }
1317 }
1318
1319 /* Call through to the machine specific trace routines
1320 * Frames are added past the snapshot header.
1321 */
1322 if (thread->kernel_stack != 0) {
1323 uint32_t thread_snapshot_flags = 0;
1324 #if defined(__LP64__)
1325 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1326 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1327 &thread_snapshot_flags);
1328 if (saved_count > 0) {
1329 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1330 cur_thread_snap->ths_ss_flags |= kKernel64_p;
1331 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME64
1332 : STACKSHOT_KCTYPE_KERN_STACKLR64,
1333 frame_size, saved_count / frame_size, &out_addr));
1334 }
1335 #else
1336 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1337 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1338 &thread_snapshot_flags);
1339 if (saved_count > 0) {
1340 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1341 kcd_exit_on_error(
1342 kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME : STACKSHOT_KCTYPE_KERN_STACKLR,
1343 frame_size, saved_count / frame_size, &out_addr));
1344 }
1345 #endif
1346 if (thread_snapshot_flags != 0) {
1347 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1348 }
1349 }
1350
1351 if (collect_iostats) {
1352 kcd_exit_on_error(kcdata_record_thread_iostats(kcd, thread));
1353 }
1354
1355 error_exit:
1356 return error;
1357 }
1358
1359 static int
1360 kcdata_record_thread_delta_snapshot(struct thread_delta_snapshot_v2 * cur_thread_snap, thread_t thread, boolean_t thread_on_core)
1361 {
1362 cur_thread_snap->tds_thread_id = thread_tid(thread);
1363 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1364 cur_thread_snap->tds_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1365 else
1366 cur_thread_snap->tds_voucher_identifier = 0;
1367
1368 cur_thread_snap->tds_ss_flags = 0;
1369 if (thread->effective_policy.thep_darwinbg)
1370 cur_thread_snap->tds_ss_flags |= kThreadDarwinBG;
1371 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1372 cur_thread_snap->tds_ss_flags |= kThreadIOPassive;
1373 if (thread->suspend_count > 0)
1374 cur_thread_snap->tds_ss_flags |= kThreadSuspended;
1375 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1376 cur_thread_snap->tds_ss_flags |= kGlobalForcedIdle;
1377 if (thread_on_core)
1378 cur_thread_snap->tds_ss_flags |= kThreadOnCore;
1379 if (stackshot_thread_is_idle_worker_unsafe(thread))
1380 cur_thread_snap->tds_ss_flags |= kThreadIdleWorker;
1381
1382 cur_thread_snap->tds_last_made_runnable_time = thread->last_made_runnable_time;
1383 cur_thread_snap->tds_state = thread->state;
1384 cur_thread_snap->tds_sched_flags = thread->sched_flags;
1385 cur_thread_snap->tds_base_priority = thread->base_pri;
1386 cur_thread_snap->tds_sched_priority = thread->sched_pri;
1387 cur_thread_snap->tds_eqos = thread->effective_policy.thep_qos;
1388 cur_thread_snap->tds_rqos = thread->requested_policy.thrp_qos;
1389 cur_thread_snap->tds_rqos_override = thread->requested_policy.thrp_qos_override;
1390 cur_thread_snap->tds_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1391
1392 return 0;
1393 }
1394
1395 /*
1396 * Why 12? 12 strikes a decent balance between allocating a large array on
1397 * the stack and having large kcdata item overheads for recording nonrunable
1398 * tasks.
1399 */
1400 #define UNIQUEIDSPERFLUSH 12
1401
1402 struct saved_uniqueids {
1403 uint64_t ids[UNIQUEIDSPERFLUSH];
1404 unsigned count;
1405 };
1406
1407 static kern_return_t
1408 flush_nonrunnable_tasks(struct saved_uniqueids * ids)
1409 {
1410 if (ids->count == 0)
1411 return KERN_SUCCESS;
1412 mach_vm_address_t out_addr = 0;
1413 kern_return_t ret = kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TASKS, sizeof(uint64_t),
1414 ids->count, &out_addr);
1415 if (ret != KERN_SUCCESS) {
1416 return ret;
1417 }
1418 stackshot_memcpy((void *)out_addr, ids->ids, sizeof(uint64_t) * ids->count);
1419 ids->count = 0;
1420 return ret;
1421 }
1422
1423 static kern_return_t
1424 handle_nonrunnable_task(struct saved_uniqueids * ids, uint64_t pid)
1425 {
1426 kern_return_t ret = KERN_SUCCESS;
1427 ids->ids[ids->count] = pid;
1428 ids->count++;
1429 assert(ids->count <= UNIQUEIDSPERFLUSH);
1430 if (ids->count == UNIQUEIDSPERFLUSH)
1431 ret = flush_nonrunnable_tasks(ids);
1432 return ret;
1433 }
1434
1435 enum thread_classification {
1436 tc_full_snapshot, /* take a full snapshot */
1437 tc_delta_snapshot, /* take a delta snapshot */
1438 tc_nonrunnable, /* only report id */
1439 };
1440
1441 static enum thread_classification
1442 classify_thread(thread_t thread, boolean_t * thread_on_core_p, uint32_t trace_flags)
1443 {
1444 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1445 boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
1446
1447 processor_t last_processor = thread->last_processor;
1448
1449 boolean_t thread_on_core =
1450 (last_processor != PROCESSOR_NULL && last_processor->state == PROCESSOR_RUNNING && last_processor->active_thread == thread);
1451
1452 *thread_on_core_p = thread_on_core;
1453
1454 /* Capture the full thread snapshot if this is not a delta stackshot or if the thread has run subsequent to the
1455 * previous full stackshot */
1456 if (!collect_delta_stackshot || thread_on_core || (thread->last_run_time > stack_snapshot_delta_since_timestamp)) {
1457 return tc_full_snapshot;
1458 } else {
1459 if (minimize_nonrunnables && !(thread->state & TH_RUN)) {
1460 return tc_nonrunnable;
1461 } else {
1462 return tc_delta_snapshot;
1463 }
1464 }
1465 }
1466
1467 static kern_return_t
1468 kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTraced)
1469 {
1470 kern_return_t error = KERN_SUCCESS;
1471 mach_vm_address_t out_addr = 0;
1472 uint64_t abs_time = 0, abs_time_end = 0;
1473 uint64_t *abs_time_addr = NULL;
1474 uint64_t system_state_flags = 0;
1475 int saved_count = 0;
1476 task_t task = TASK_NULL;
1477 thread_t thread = THREAD_NULL;
1478 mach_timebase_info_data_t timebase = {0, 0};
1479 uint32_t length_to_copy = 0, tmp32 = 0;
1480
1481 abs_time = mach_absolute_time();
1482
1483 #if !(DEVELOPMENT || DEBUG)
1484 trace_flags &= ~STACKSHOT_THREAD_WAITINFO;
1485 #endif
1486
1487 /* process the flags */
1488 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1489 boolean_t save_donating_pids_p = ((trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0);
1490 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1491 boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
1492 boolean_t use_fault_path = ((trace_flags & (STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_ENABLE_BT_FAULTING)) != 0);
1493 boolean_t save_owner_info = ((trace_flags & STACKSHOT_THREAD_WAITINFO) != 0);
1494
1495 stack_enable_faulting = (trace_flags & (STACKSHOT_ENABLE_BT_FAULTING));
1496
1497
1498 struct saved_uniqueids saved_uniqueids = {.count = 0};
1499
1500 if (use_fault_path) {
1501 fault_stats.sfs_pages_faulted_in = 0;
1502 fault_stats.sfs_time_spent_faulting = 0;
1503 fault_stats.sfs_stopped_faulting = (uint8_t) FALSE;
1504 }
1505
1506 if (sizeof(void *) == 8)
1507 system_state_flags |= kKernel64_p;
1508
1509 if (stackshot_kcdata_p == NULL || pBytesTraced == NULL) {
1510 error = KERN_INVALID_ARGUMENT;
1511 goto error_exit;
1512 }
1513
1514 /* setup mach_absolute_time and timebase info -- copy out in some cases and needed to convert since_timestamp to seconds for proc start time */
1515 clock_timebase_info(&timebase);
1516
1517 /* begin saving data into the buffer */
1518 *pBytesTraced = 0;
1519 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_in_flags"));
1520 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, (uint32_t)pid, "stackshot_in_pid"));
1521 kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_kcdata_p, system_state_flags, "system_state_flags"));
1522
1523 #if CONFIG_JETSAM
1524 tmp32 = memorystatus_get_pressure_status_kdp();
1525 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_LEVEL, sizeof(uint32_t), &out_addr));
1526 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1527 #endif
1528
1529 if (!collect_delta_stackshot) {
1530 tmp32 = PAGE_SIZE;
1531 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_KERN_PAGE_SIZE, sizeof(uint32_t), &out_addr));
1532 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1533
1534 /* save boot-args and osversion string */
1535 length_to_copy = MIN((uint32_t)(strlen(version) + 1), OSVERSIZE);
1536 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_OSVERSION, length_to_copy, &out_addr));
1537 stackshot_strlcpy((char*)out_addr, &version[0], length_to_copy);
1538
1539 length_to_copy = MIN((uint32_t)(strlen(PE_boot_args()) + 1), OSVERSIZE);
1540 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_BOOTARGS, length_to_copy, &out_addr));
1541 stackshot_strlcpy((char*)out_addr, PE_boot_args(), length_to_copy);
1542
1543 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_TIMEBASE, sizeof(timebase), &out_addr));
1544 stackshot_memcpy((void *)out_addr, &timebase, sizeof(timebase));
1545 } else {
1546 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP, sizeof(uint64_t), &out_addr));
1547 stackshot_memcpy((void*)out_addr, &stack_snapshot_delta_since_timestamp, sizeof(stack_snapshot_delta_since_timestamp));
1548 }
1549
1550 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_MACH_ABSOLUTE_TIME, sizeof(uint64_t), &out_addr));
1551 abs_time_addr = (uint64_t *)out_addr;
1552 stackshot_memcpy((void *)abs_time_addr, &abs_time, sizeof(uint64_t));
1553
1554 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &out_addr));
1555 stackshot_memcpy((void *)out_addr, &stackshot_microsecs, sizeof(uint64_t));
1556
1557 /* reserve space of system level shared cache load info */
1558 struct dyld_uuid_info_64_v2 * sys_shared_cache_loadinfo = NULL;
1559 if (!collect_delta_stackshot) {
1560 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO,
1561 sizeof(struct dyld_uuid_info_64_v2), &out_addr));
1562 sys_shared_cache_loadinfo = (struct dyld_uuid_info_64_v2 *)out_addr;
1563 bzero((void *)sys_shared_cache_loadinfo, sizeof(struct dyld_uuid_info_64_v2));
1564 }
1565
1566 /* Add requested information first */
1567 if (trace_flags & STACKSHOT_GET_GLOBAL_MEM_STATS) {
1568 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_GLOBAL_MEM_STATS, sizeof(struct mem_and_io_snapshot), &out_addr));
1569 kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)out_addr);
1570 }
1571
1572 /* Iterate over tasks */
1573 queue_head_t *task_list = &tasks;
1574 queue_iterate(task_list, task, task_t, tasks) {
1575 int task_pid = 0;
1576 uint64_t task_uniqueid = 0;
1577 int num_delta_thread_snapshots = 0;
1578 int num_nonrunnable_threads = 0;
1579 int num_waitinfo_threads = 0;
1580 uint64_t task_start_abstime = 0;
1581 boolean_t task_delta_stackshot = FALSE;
1582 boolean_t task64 = FALSE, have_map = FALSE, have_pmap = FALSE;
1583 boolean_t some_thread_ran = FALSE;
1584 unaligned_u64 *task_snap_ss_flags = NULL;
1585
1586 if ((task == NULL) || !ml_validate_nofault((vm_offset_t)task, sizeof(struct task))) {
1587 error = KERN_FAILURE;
1588 goto error_exit;
1589 }
1590
1591 have_map = (task->map != NULL) && (ml_validate_nofault((vm_offset_t)(task->map), sizeof(struct _vm_map)));
1592 have_pmap = have_map && (task->map->pmap != NULL) && (ml_validate_nofault((vm_offset_t)(task->map->pmap), sizeof(struct pmap)));
1593
1594 task_pid = pid_from_task(task);
1595 task_uniqueid = get_task_uniqueid(task);
1596 task64 = task_has_64BitAddr(task);
1597
1598 if (!task->active || task_is_a_corpse(task)) {
1599 /*
1600 * Not interested in terminated tasks without threads, and
1601 * at the moment, stackshot can't handle a task without a name.
1602 */
1603 if (queue_empty(&task->threads) || task_pid == -1) {
1604 continue;
1605 }
1606 }
1607
1608 if (collect_delta_stackshot) {
1609 proc_starttime_kdp(task->bsd_info, NULL, NULL, &task_start_abstime);
1610 }
1611
1612 /* Trace everything, unless a process was specified */
1613 if ((pid == -1) || (pid == task_pid)) {
1614 #if DEBUG || DEVELOPMENT
1615 /* we might want to call kcdata_undo_add_container_begin(), which is
1616 * only safe if we call it after kcdata_add_container_marker() but
1617 * before adding any other kcdata items. In development kernels,
1618 * we'll remember where the buffer end was and confirm after calling
1619 * kcdata_undo_add_container_begin() that it's in exactly the same
1620 * place.*/
1621 mach_vm_address_t revert_addr = stackshot_kcdata_p->kcd_addr_end;
1622 #endif
1623
1624 /* add task snapshot marker */
1625 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1626 STACKSHOT_KCCONTAINER_TASK, task_uniqueid));
1627
1628 if (!collect_delta_stackshot || (task_start_abstime == 0) ||
1629 (task_start_abstime > stack_snapshot_delta_since_timestamp)) {
1630 kcd_exit_on_error(kcdata_record_task_snapshot(stackshot_kcdata_p, task, trace_flags, have_pmap, &task_snap_ss_flags));
1631 } else {
1632 task_delta_stackshot = TRUE;
1633 if (minimize_nonrunnables) {
1634 // delay taking the task snapshot. If there are no runnable threads we'll skip it.
1635 } else {
1636 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
1637 }
1638 }
1639
1640 /* Iterate over task threads */
1641 queue_iterate(&task->threads, thread, thread_t, task_threads)
1642 {
1643 uint64_t thread_uniqueid;
1644
1645 if ((thread == NULL) || !ml_validate_nofault((vm_offset_t)thread, sizeof(struct thread))) {
1646 error = KERN_FAILURE;
1647 goto error_exit;
1648 }
1649
1650 if (active_kthreads_only_p && thread->kernel_stack == 0)
1651 continue;
1652
1653 thread_uniqueid = thread_tid(thread);
1654
1655 boolean_t thread_on_core;
1656 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
1657
1658 switch (thread_classification) {
1659 case tc_full_snapshot:
1660 /* add thread marker */
1661 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1662 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1663 kcd_exit_on_error(
1664 kcdata_record_thread_snapshot(stackshot_kcdata_p, thread, task, trace_flags, have_pmap, thread_on_core));
1665
1666 /* mark end of thread snapshot data */
1667 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END,
1668 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1669
1670 some_thread_ran = TRUE;
1671 break;
1672
1673 case tc_delta_snapshot:
1674 num_delta_thread_snapshots++;
1675 break;
1676
1677 case tc_nonrunnable:
1678 num_nonrunnable_threads++;
1679 break;
1680 }
1681
1682 /* We want to report owner information regardless of whether a thread
1683 * has changed since the last delta, whether it's a normal stackshot,
1684 * or whether it's nonrunnable */
1685 if (save_owner_info && stackshot_thread_has_valid_waitinfo(thread))
1686 num_waitinfo_threads++;
1687 }
1688
1689 if (task_delta_stackshot && minimize_nonrunnables) {
1690 if (some_thread_ran || num_delta_thread_snapshots > 0) {
1691 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
1692 } else {
1693 kcd_exit_on_error(kcdata_undo_add_container_begin(stackshot_kcdata_p));
1694
1695 #if DEBUG || DEVELOPMENT
1696 mach_vm_address_t undo_addr = stackshot_kcdata_p->kcd_addr_end;
1697 if (revert_addr != undo_addr) {
1698 panic("tried to revert a container begin but we already moved past it. revert=%p undo=%p",
1699 (void *)revert_addr, (void *)undo_addr);
1700 }
1701 #endif
1702 kcd_exit_on_error(handle_nonrunnable_task(&saved_uniqueids, task_uniqueid));
1703 continue;
1704 }
1705 }
1706
1707 struct thread_delta_snapshot_v2 * delta_snapshots = NULL;
1708 int current_delta_snapshot_index = 0;
1709
1710 if (num_delta_thread_snapshots > 0) {
1711 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT,
1712 sizeof(struct thread_delta_snapshot_v2),
1713 num_delta_thread_snapshots, &out_addr));
1714 delta_snapshots = (struct thread_delta_snapshot_v2 *)out_addr;
1715 }
1716
1717 uint64_t * nonrunnable_tids = NULL;
1718 int current_nonrunnable_index = 0;
1719
1720 if (num_nonrunnable_threads > 0) {
1721 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TIDS,
1722 sizeof(uint64_t), num_nonrunnable_threads, &out_addr));
1723 nonrunnable_tids = (uint64_t *)out_addr;
1724 }
1725
1726 thread_waitinfo_t *thread_waitinfo = NULL;
1727 int current_waitinfo_index = 0;
1728
1729 if (num_waitinfo_threads > 0) {
1730 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_WAITINFO,
1731 sizeof(thread_waitinfo_t), num_waitinfo_threads, &out_addr));
1732 thread_waitinfo = (thread_waitinfo_t *)out_addr;
1733 }
1734
1735 if (num_delta_thread_snapshots > 0 || num_nonrunnable_threads > 0 || num_waitinfo_threads > 0) {
1736 queue_iterate(&task->threads, thread, thread_t, task_threads)
1737 {
1738 if (active_kthreads_only_p && thread->kernel_stack == 0)
1739 continue;
1740
1741 /* If we want owner info, we should capture it regardless of its classification */
1742 if (save_owner_info && stackshot_thread_has_valid_waitinfo(thread)) {
1743 stackshot_thread_wait_owner_info(
1744 thread,
1745 &thread_waitinfo[current_waitinfo_index++]);
1746 }
1747
1748 boolean_t thread_on_core;
1749 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
1750
1751 switch (thread_classification) {
1752 case tc_full_snapshot:
1753 /* full thread snapshot captured above */
1754 continue;
1755
1756 case tc_delta_snapshot:
1757 kcd_exit_on_error(kcdata_record_thread_delta_snapshot(&delta_snapshots[current_delta_snapshot_index++],
1758 thread, thread_on_core));
1759 break;
1760
1761 case tc_nonrunnable:
1762 nonrunnable_tids[current_nonrunnable_index++] = thread_tid(thread);
1763 continue;
1764 }
1765 }
1766
1767 #if DEBUG || DEVELOPMENT
1768 if (current_delta_snapshot_index != num_delta_thread_snapshots) {
1769 panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1770 num_delta_thread_snapshots, current_delta_snapshot_index);
1771 }
1772 if (current_nonrunnable_index != num_nonrunnable_threads) {
1773 panic("nonrunnable thread count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1774 num_nonrunnable_threads, current_nonrunnable_index);
1775 }
1776 if (current_waitinfo_index != num_waitinfo_threads) {
1777 panic("thread wait info count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1778 num_waitinfo_threads, current_waitinfo_index);
1779 }
1780 #endif
1781 }
1782
1783 #if IMPORTANCE_INHERITANCE
1784 if (save_donating_pids_p) {
1785 kcd_exit_on_error(
1786 ((((mach_vm_address_t)kcd_end_address(stackshot_kcdata_p) + (TASK_IMP_WALK_LIMIT * sizeof(int32_t))) <
1787 (mach_vm_address_t)kcd_max_address(stackshot_kcdata_p))
1788 ? KERN_SUCCESS
1789 : KERN_RESOURCE_SHORTAGE));
1790 saved_count = task_importance_list_pids(task, TASK_IMP_LIST_DONATING_PIDS,
1791 (void *)kcd_end_address(stackshot_kcdata_p), TASK_IMP_WALK_LIMIT);
1792 if (saved_count > 0)
1793 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_DONATING_PIDS,
1794 sizeof(int32_t), saved_count, &out_addr));
1795 }
1796 #endif
1797
1798 if (!collect_delta_stackshot || (num_delta_thread_snapshots != task->thread_count) || !task_delta_stackshot) {
1799 /*
1800 * Collect shared cache info and UUID info in these scenarios
1801 * 1) a full stackshot
1802 * 2) a delta stackshot where the task started after the previous full stackshot OR
1803 * any thread from the task has run since the previous full stackshot
1804 */
1805
1806 kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, sys_shared_cache_loadinfo, trace_flags, task_snap_ss_flags));
1807 kcd_exit_on_error(kcdata_record_uuid_info(stackshot_kcdata_p, task, trace_flags, have_pmap, task_snap_ss_flags));
1808 }
1809 /* mark end of task snapshot data */
1810 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK,
1811 task_uniqueid));
1812 }
1813 }
1814
1815 if (minimize_nonrunnables) {
1816 flush_nonrunnable_tasks(&saved_uniqueids);
1817 }
1818
1819 if (use_fault_path) {
1820 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS,
1821 sizeof(struct stackshot_fault_stats), &out_addr));
1822 stackshot_memcpy((void*)out_addr, &fault_stats, sizeof(struct stackshot_fault_stats));
1823 }
1824
1825 /* update timestamp of the stackshot */
1826 abs_time_end = mach_absolute_time();
1827 #if DEVELOPMENT || DEBUG
1828 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_DURATION,
1829 sizeof(struct stackshot_duration), &out_addr));
1830 struct stackshot_duration * stackshot_duration = (struct stackshot_duration *)out_addr;
1831 stackshot_duration->stackshot_duration = (abs_time_end - abs_time);
1832 stackshot_duration->stackshot_duration_outer = 0;
1833 stackshot_duration_outer = (unaligned_u64 *)&stackshot_duration->stackshot_duration_outer;
1834 #endif
1835 stackshot_memcpy((void *)abs_time_addr, &abs_time_end, sizeof(uint64_t));
1836
1837
1838 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_kcdata_p));
1839
1840 /* === END of populating stackshot data === */
1841
1842 *pBytesTraced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_kcdata_p);
1843 error_exit:
1844
1845 stack_enable_faulting = FALSE;
1846
1847 return error;
1848 }
1849
1850 static uint64_t
1851 proc_was_throttled_from_task(task_t task)
1852 {
1853 uint64_t was_throttled = 0;
1854
1855 if (task->bsd_info)
1856 was_throttled = proc_was_throttled(task->bsd_info);
1857
1858 return was_throttled;
1859 }
1860
1861 static uint64_t
1862 proc_did_throttle_from_task(task_t task)
1863 {
1864 uint64_t did_throttle = 0;
1865
1866 if (task->bsd_info)
1867 did_throttle = proc_did_throttle(task->bsd_info);
1868
1869 return did_throttle;
1870 }
1871
1872 static void
1873 kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap)
1874 {
1875 unsigned int pages_reclaimed;
1876 unsigned int pages_wanted;
1877 kern_return_t kErr;
1878
1879 processor_t processor;
1880 vm_statistics64_t stat;
1881 vm_statistics64_data_t host_vm_stat;
1882
1883 processor = processor_list;
1884 stat = &PROCESSOR_DATA(processor, vm_stat);
1885 host_vm_stat = *stat;
1886
1887 if (processor_count > 1) {
1888 /*
1889 * processor_list may be in the process of changing as we are
1890 * attempting a stackshot. Ordinarily it will be lock protected,
1891 * but it is not safe to lock in the context of the debugger.
1892 * Fortunately we never remove elements from the processor list,
1893 * and only add to to the end of the list, so we SHOULD be able
1894 * to walk it. If we ever want to truly tear down processors,
1895 * this will have to change.
1896 */
1897 while ((processor = processor->processor_list) != NULL) {
1898 stat = &PROCESSOR_DATA(processor, vm_stat);
1899 host_vm_stat.compressions += stat->compressions;
1900 host_vm_stat.decompressions += stat->decompressions;
1901 }
1902 }
1903
1904 memio_snap->snapshot_magic = STACKSHOT_MEM_AND_IO_SNAPSHOT_MAGIC;
1905 memio_snap->free_pages = vm_page_free_count;
1906 memio_snap->active_pages = vm_page_active_count;
1907 memio_snap->inactive_pages = vm_page_inactive_count;
1908 memio_snap->purgeable_pages = vm_page_purgeable_count;
1909 memio_snap->wired_pages = vm_page_wire_count;
1910 memio_snap->speculative_pages = vm_page_speculative_count;
1911 memio_snap->throttled_pages = vm_page_throttled_count;
1912 memio_snap->busy_buffer_count = count_busy_buffers();
1913 memio_snap->filebacked_pages = vm_page_pageable_external_count;
1914 memio_snap->compressions = (uint32_t)host_vm_stat.compressions;
1915 memio_snap->decompressions = (uint32_t)host_vm_stat.decompressions;
1916 memio_snap->compressor_size = VM_PAGE_COMPRESSOR_COUNT;
1917 kErr = mach_vm_pressure_monitor(FALSE, VM_PRESSURE_TIME_WINDOW, &pages_reclaimed, &pages_wanted);
1918
1919 if ( ! kErr ) {
1920 memio_snap->pages_wanted = (uint32_t)pages_wanted;
1921 memio_snap->pages_reclaimed = (uint32_t)pages_reclaimed;
1922 memio_snap->pages_wanted_reclaimed_valid = 1;
1923 } else {
1924 memio_snap->pages_wanted = 0;
1925 memio_snap->pages_reclaimed = 0;
1926 memio_snap->pages_wanted_reclaimed_valid = 0;
1927 }
1928 }
1929
1930 void
1931 stackshot_memcpy(void *dst, const void *src, size_t len)
1932 {
1933 memcpy(dst, src, len);
1934 }
1935
1936 size_t
1937 stackshot_strlcpy(char *dst, const char *src, size_t maxlen)
1938 {
1939 const size_t srclen = strlen(src);
1940
1941 if (srclen < maxlen) {
1942 stackshot_memcpy(dst, src, srclen+1);
1943 } else if (maxlen != 0) {
1944 stackshot_memcpy(dst, src, maxlen-1);
1945 dst[maxlen-1] = '\0';
1946 }
1947
1948 return srclen;
1949 }
1950
1951
1952 /*
1953 * Returns the physical address of the specified map:target address,
1954 * using the kdp fault path if requested and the page is not resident.
1955 */
1956 vm_offset_t
1957 kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results)
1958 {
1959 vm_offset_t cur_phys_addr;
1960 unsigned cur_wimg_bits;
1961 uint64_t fault_start_time = 0;
1962
1963 if (map == VM_MAP_NULL) {
1964 return 0;
1965 }
1966
1967 cur_phys_addr = kdp_vtophys(map->pmap, target_addr);
1968 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
1969 if (!try_fault || fault_stats.sfs_stopped_faulting) {
1970 if (kdp_fault_results)
1971 *kdp_fault_results |= KDP_FAULT_RESULT_PAGED_OUT;
1972
1973 return 0;
1974 }
1975
1976 /*
1977 * The pmap doesn't have a valid page so we start at the top level
1978 * vm map and try a lightweight fault. Update fault path usage stats.
1979 */
1980 fault_start_time = mach_absolute_time();
1981 cur_phys_addr = kdp_lightweight_fault(map, (target_addr & ~PAGE_MASK));
1982 fault_stats.sfs_time_spent_faulting += (mach_absolute_time() - fault_start_time);
1983
1984 if ((fault_stats.sfs_time_spent_faulting >= fault_stats.sfs_system_max_fault_time) && !panic_stackshot) {
1985 fault_stats.sfs_stopped_faulting = (uint8_t) TRUE;
1986 }
1987
1988 cur_phys_addr += (target_addr & PAGE_MASK);
1989
1990 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
1991 if (kdp_fault_results)
1992 *kdp_fault_results |= (KDP_FAULT_RESULT_TRIED_FAULT | KDP_FAULT_RESULT_PAGED_OUT);
1993
1994 return 0;
1995 }
1996
1997 if (kdp_fault_results)
1998 *kdp_fault_results |= KDP_FAULT_RESULT_FAULTED_IN;
1999
2000 fault_stats.sfs_pages_faulted_in++;
2001 } else {
2002 /*
2003 * This check is done in kdp_lightweight_fault for the fault path.
2004 */
2005 cur_wimg_bits = pmap_cache_attributes((ppnum_t) atop(cur_phys_addr));
2006
2007 if ((cur_wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
2008 return 0;
2009 }
2010 }
2011
2012 return cur_phys_addr;
2013 }
2014
2015 boolean_t
2016 kdp_copyin_word(
2017 task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results)
2018 {
2019 if (task_has_64BitAddr(task)) {
2020 return kdp_copyin(task->map, addr, result, sizeof(uint64_t), try_fault, kdp_fault_results);
2021 } else {
2022 uint32_t buf;
2023 boolean_t r = kdp_copyin(task->map, addr, &buf, sizeof(uint32_t), try_fault, kdp_fault_results);
2024 *result = buf;
2025 return r;
2026 }
2027 }
2028
2029 boolean_t
2030 kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_results)
2031 {
2032 size_t rem = size;
2033 char *kvaddr = dest;
2034
2035 #if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
2036 /* Identify if destination buffer is in panic storage area */
2037 if ((vm_offset_t)dest >= gPanicBase && (vm_offset_t)dest < gPanicBase + gPanicSize) {
2038 if (((vm_offset_t)dest + size) >= (gPanicBase + gPanicSize)) {
2039 return FALSE;
2040 }
2041 }
2042 #endif
2043
2044 while (rem) {
2045 uint64_t phys_src = kdp_find_phys(map, uaddr, try_fault, kdp_fault_results);
2046 uint64_t phys_dest = kvtophys((vm_offset_t)kvaddr);
2047 uint64_t src_rem = PAGE_SIZE - (phys_src & PAGE_MASK);
2048 uint64_t dst_rem = PAGE_SIZE - (phys_dest & PAGE_MASK);
2049 size_t cur_size = (uint32_t) MIN(src_rem, dst_rem);
2050 cur_size = MIN(cur_size, rem);
2051
2052 if (phys_src && phys_dest) {
2053 bcopy_phys(phys_src, phys_dest, cur_size);
2054 } else {
2055 break;
2056 }
2057
2058 uaddr += cur_size;
2059 kvaddr += cur_size;
2060 rem -= cur_size;
2061 }
2062
2063 return (rem == 0);
2064 }
2065
2066 kern_return_t
2067 do_stackshot(void *context)
2068 {
2069 #pragma unused(context)
2070 kdp_snapshot++;
2071
2072 stack_snapshot_ret = kdp_stackshot_kcdata_format(stack_snapshot_pid,
2073 stack_snapshot_flags,
2074 &stack_snapshot_bytes_traced);
2075
2076 kdp_snapshot--;
2077 return stack_snapshot_ret;
2078 }
2079
2080 /*
2081 * A fantastical routine that tries to be fast about returning
2082 * translations. Caches the last page we found a translation
2083 * for, so that we can be quick about multiple queries to the
2084 * same page. It turns out this is exactly the workflow
2085 * machine_trace_thread and its relatives tend to throw at us.
2086 *
2087 * Please zero the nasty global this uses after a bulk lookup;
2088 * this isn't safe across a switch of the map or changes
2089 * to a pmap.
2090 *
2091 * This also means that if zero is a valid KVA, we are
2092 * screwed. Sucks to be us. Fortunately, this should never
2093 * happen.
2094 */
2095 vm_offset_t
2096 machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags)
2097 {
2098 vm_offset_t cur_target_page;
2099 vm_offset_t cur_phys_addr;
2100 vm_offset_t kern_virt_target_addr;
2101 uint32_t kdp_fault_results = 0;
2102
2103 cur_target_page = atop(cur_target_addr);
2104
2105 if ((cur_target_page != prev_target_page) || validate_next_addr) {
2106
2107 /*
2108 * Alright; it wasn't our previous page. So
2109 * we must validate that there is a page
2110 * table entry for this address under the
2111 * current pmap, and that it has default
2112 * cache attributes (otherwise it may not be
2113 * safe to access it).
2114 */
2115 cur_phys_addr = kdp_find_phys(map, cur_target_addr, stack_enable_faulting, &kdp_fault_results);
2116 if (thread_trace_flags) {
2117 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
2118 *thread_trace_flags |= kThreadTruncatedBT;
2119 }
2120
2121 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
2122 *thread_trace_flags |= kThreadTriedFaultBT;
2123 }
2124
2125 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
2126 *thread_trace_flags |= kThreadFaultedBT;
2127 }
2128 }
2129
2130 if (cur_phys_addr == 0) {
2131 return 0;
2132 }
2133 #if __x86_64__
2134 kern_virt_target_addr = (vm_offset_t) PHYSMAP_PTOV(cur_phys_addr);
2135 #else
2136 #error Oh come on... we should really unify the physical -> kernel virtual interface
2137 #endif
2138 prev_target_page = cur_target_page;
2139 prev_target_kva = (kern_virt_target_addr & ~PAGE_MASK);
2140 validate_next_addr = FALSE;
2141 return kern_virt_target_addr;
2142 } else {
2143 /* We found a translation, so stash this page */
2144 kern_virt_target_addr = prev_target_kva + (cur_target_addr & PAGE_MASK);
2145 return kern_virt_target_addr;
2146 }
2147 }
2148
2149 void
2150 machine_trace_thread_clear_validation_cache(void)
2151 {
2152 validate_next_addr = TRUE;
2153 }
2154
2155 boolean_t
2156 stackshot_thread_is_idle_worker_unsafe(thread_t thread)
2157 {
2158 /* When the pthread kext puts a worker thread to sleep, it will
2159 * set kThreadWaitParkedWorkQueue in the block_hint of the thread
2160 * struct. See parkit() in kern/kern_support.c in libpthread.
2161 */
2162 return (thread->state & TH_WAIT) &&
2163 (thread->block_hint == kThreadWaitParkedWorkQueue);
2164 }
2165
2166 /* Determine if a thread has waitinfo that stackshot can provide */
2167 static int
2168 stackshot_thread_has_valid_waitinfo(thread_t thread)
2169 {
2170 if (!(thread->state & TH_WAIT))
2171 return 0;
2172
2173 switch (thread->block_hint) {
2174 // If set to None or is a parked work queue, ignore it
2175 case kThreadWaitParkedWorkQueue:
2176 case kThreadWaitNone:
2177 return 0;
2178 // There is a short window where the pthread kext removes a thread
2179 // from its ksyn wait queue before waking the thread up
2180 case kThreadWaitPThreadMutex:
2181 case kThreadWaitPThreadRWLockRead:
2182 case kThreadWaitPThreadRWLockWrite:
2183 case kThreadWaitPThreadCondVar:
2184 return (kdp_pthread_get_thread_kwq(thread) != NULL);
2185 // All other cases are valid block hints if in a wait state
2186 default:
2187 return 1;
2188 }
2189 }
2190
2191 static void
2192 stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t *waitinfo)
2193 {
2194 waitinfo->waiter = thread_tid(thread);
2195 waitinfo->wait_type = thread->block_hint;
2196 switch (waitinfo->wait_type) {
2197 case kThreadWaitKernelMutex:
2198 kdp_lck_mtx_find_owner(thread->waitq, thread->wait_event, waitinfo);
2199 break;
2200 case kThreadWaitPortReceive:
2201 kdp_mqueue_recv_find_owner(thread->waitq, thread->wait_event, waitinfo);
2202 break;
2203 case kThreadWaitPortSend:
2204 kdp_mqueue_send_find_owner(thread->waitq, thread->wait_event, waitinfo);
2205 break;
2206 case kThreadWaitSemaphore:
2207 kdp_sema_find_owner(thread->waitq, thread->wait_event, waitinfo);
2208 break;
2209 case kThreadWaitUserLock:
2210 kdp_ulock_find_owner(thread->waitq, thread->wait_event, waitinfo);
2211 break;
2212 case kThreadWaitKernelRWLockRead:
2213 case kThreadWaitKernelRWLockWrite:
2214 case kThreadWaitKernelRWLockUpgrade:
2215 kdp_rwlck_find_owner(thread->waitq, thread->wait_event, waitinfo);
2216 break;
2217 case kThreadWaitPThreadMutex:
2218 case kThreadWaitPThreadRWLockRead:
2219 case kThreadWaitPThreadRWLockWrite:
2220 case kThreadWaitPThreadCondVar:
2221 kdp_pthread_find_owner(thread, waitinfo);
2222 break;
2223 default:
2224 waitinfo->owner = 0;
2225 waitinfo->context = 0;
2226 break;
2227 }
2228 }
mirror of XNU