]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/kern_stackshot.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.21.4.tar.gz
[apple/xnu.git] / osfmk / kern / kern_stackshot.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/mach_types.h>
30 #include <mach/vm_param.h>
31 #include <mach/mach_vm.h>
32 #include <mach/clock_types.h>
33 #include <sys/errno.h>
34 #include <sys/stackshot.h>
35 #ifdef IMPORTANCE_INHERITANCE
36 #include <ipc/ipc_importance.h>
37 #endif
38 #include <sys/appleapiopts.h>
39 #include <kern/debug.h>
40 #include <uuid/uuid.h>
41
42 #include <kdp/kdp_dyld.h>
43 #include <kdp/kdp_en_debugger.h>
44
45 #include <libsa/types.h>
46 #include <libkern/version.h>
47
48 #include <string.h> /* bcopy */
49
50 #include <kern/processor.h>
51 #include <kern/thread.h>
52 #include <kern/task.h>
53 #include <kern/telemetry.h>
54 #include <kern/clock.h>
55 #include <kern/policy_internal.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_kern.h>
58 #include <vm/vm_pageout.h>
59 #include <vm/vm_fault.h>
60 #include <vm/vm_shared_region.h>
61 #include <libkern/OSKextLibPrivate.h>
62
63 #if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
64 #include <pexpert/pexpert.h> /* For gPanicBase/gPanicBase */
65 #endif
66
67 extern unsigned int not_in_kdp;
68
69
70 extern addr64_t kdp_vtophys(pmap_t pmap, addr64_t va);
71 extern void * proc_get_uthread_uu_threadlist(void * uthread_v);
72
73 int kdp_snapshot = 0;
74 static kern_return_t stack_snapshot_ret = 0;
75 static uint32_t stack_snapshot_bytes_traced = 0;
76
77 static kcdata_descriptor_t stackshot_kcdata_p = NULL;
78 static void *stack_snapshot_buf;
79 static uint32_t stack_snapshot_bufsize;
80 int stack_snapshot_pid;
81 static uint32_t stack_snapshot_flags;
82 static uint64_t stack_snapshot_delta_since_timestamp;
83 static boolean_t panic_stackshot;
84
85 static boolean_t stack_enable_faulting = FALSE;
86 static struct stackshot_fault_stats fault_stats;
87
88 static uint64_t * stackshot_duration_outer;
89 static uint64_t stackshot_microsecs;
90
91 void * kernel_stackshot_buf = NULL; /* Pointer to buffer for stackshots triggered from the kernel and retrieved later */
92 int kernel_stackshot_buf_size = 0;
93
94 void * stackshot_snapbuf = NULL; /* Used by stack_snapshot2 (to be removed) */
95
96 __private_extern__ void stackshot_init( void );
97 static boolean_t memory_iszero(void *addr, size_t size);
98 #if CONFIG_TELEMETRY
99 kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
100 #endif
101 uint32_t get_stackshot_estsize(uint32_t prev_size_hint);
102 kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config,
103 size_t stackshot_config_size, boolean_t stackshot_from_user);
104 kern_return_t do_stackshot(void *);
105 void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp);
106 boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
107 static int kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t *pBytesTraced);
108 kern_return_t kdp_stack_snapshot_geterror(void);
109 uint32_t kdp_stack_snapshot_bytes_traced(void);
110 static int pid_from_task(task_t task);
111 static void kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap);
112 static boolean_t kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_result);
113 static boolean_t kdp_copyin_word(task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results);
114 static uint64_t proc_was_throttled_from_task(task_t task);
115
116 extern uint32_t workqueue_get_pwq_state_kdp(void *proc);
117
118 extern int proc_pid(void *p);
119 extern uint64_t proc_uniqueid(void *p);
120 extern uint64_t proc_was_throttled(void *p);
121 extern uint64_t proc_did_throttle(void *p);
122 static uint64_t proc_did_throttle_from_task(task_t task);
123 extern void proc_name_kdp(task_t task, char * buf, int size);
124 extern int proc_threadname_kdp(void * uth, char * buf, size_t size);
125 extern void proc_starttime_kdp(void * p, uint64_t * tv_sec, uint64_t * tv_usec, uint64_t * abstime);
126 extern int memorystatus_get_pressure_status_kdp(void);
127 extern boolean_t memorystatus_proc_is_dirty_unsafe(void * v);
128
129 extern int count_busy_buffers(void); /* must track with declaration in bsd/sys/buf_internal.h */
130 extern void bcopy_phys(addr64_t, addr64_t, vm_size_t);
131
132 #if CONFIG_TELEMETRY
133 extern kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
134 #endif /* CONFIG_TELEMETRY */
135
136 extern kern_return_t kern_stack_snapshot_with_reason(char* reason);
137 extern kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user);
138
139 /*
140 * Validates that the given address is both a valid page and has
141 * default caching attributes for the current map. Returns
142 * 0 if the address is invalid, and a kernel virtual address for
143 * the given address if it is valid.
144 */
145 vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags);
146
147 #define KDP_FAULT_RESULT_PAGED_OUT 0x1 /* some data was unable to be retrieved */
148 #define KDP_FAULT_RESULT_TRIED_FAULT 0x2 /* tried to fault in data */
149 #define KDP_FAULT_RESULT_FAULTED_IN 0x4 /* successfully faulted in data */
150
151 /*
152 * Looks up the physical translation for the given address in the target map, attempting
153 * to fault data in if requested and it is not resident. Populates thread_trace_flags if requested
154 * as well.
155 */
156 vm_offset_t kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results);
157
158 static size_t stackshot_strlcpy(char *dst, const char *src, size_t maxlen);
159 static void stackshot_memcpy(void *dst, const void *src, size_t len);
160
161 /* Clears caching information used by the above validation routine
162 * (in case the current map has been changed or cleared).
163 */
164 void machine_trace_thread_clear_validation_cache(void);
165
166 #define MAX_FRAMES 1000
167 #define MAX_LOADINFOS 500
168 #define TASK_IMP_WALK_LIMIT 20
169
170 typedef struct thread_snapshot *thread_snapshot_t;
171 typedef struct task_snapshot *task_snapshot_t;
172
173 #if CONFIG_KDP_INTERACTIVE_DEBUGGING
174 extern kdp_send_t kdp_en_send_pkt;
175 #endif
176
177 /*
178 * Globals to support machine_trace_thread_get_kva.
179 */
180 static vm_offset_t prev_target_page = 0;
181 static vm_offset_t prev_target_kva = 0;
182 static boolean_t validate_next_addr = TRUE;
183
184 /*
185 * Stackshot locking and other defines.
186 */
187 static lck_grp_t *stackshot_subsys_lck_grp;
188 static lck_grp_attr_t *stackshot_subsys_lck_grp_attr;
189 static lck_attr_t *stackshot_subsys_lck_attr;
190 static lck_mtx_t stackshot_subsys_mutex;
191
192 #define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
193 #define STACKSHOT_SUBSYS_TRY_LOCK() lck_mtx_try_lock(&stackshot_subsys_mutex)
194 #define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
195
196 #define SANE_BOOTPROFILE_TRACEBUF_SIZE (64 * 1024 * 1024)
197 #define SANE_TRACEBUF_SIZE (8 * 1024 * 1024)
198
199 /*
200 * We currently set a ceiling of 3 milliseconds spent in the kdp fault path
201 * for non-panic stackshots where faulting is requested.
202 */
203 #define KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS (3 * NSEC_PER_MSEC)
204
205 #define STACKSHOT_SUPP_SIZE (16 * 1024) /* Minimum stackshot size */
206 #define TASK_UUID_AVG_SIZE (16 * sizeof(uuid_t)) /* Average space consumed by UUIDs/task */
207
208 /*
209 * Initialize the mutex governing access to the stack snapshot subsystem
210 * and other stackshot related bits.
211 */
212 __private_extern__ void
213 stackshot_init( void )
214 {
215 mach_timebase_info_data_t timebase;
216
217 stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
218
219 stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
220
221 stackshot_subsys_lck_attr = lck_attr_alloc_init();
222
223 lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
224
225 clock_timebase_info(&timebase);
226 fault_stats.sfs_system_max_fault_time = ((KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS * timebase.denom)/ timebase.numer);
227 }
228
229 /*
230 * Method for grabbing timer values safely, in the sense that no infinite loop will occur
231 * Certain flavors of the timer_grab function, which would seem to be the thing to use,
232 * can loop infinitely if called while the timer is in the process of being updated.
233 * Unfortunately, it is (rarely) possible to get inconsistent top and bottom halves of
234 * the timer using this method. This seems insoluble, since stackshot runs in a context
235 * where the timer might be half-updated, and has no way of yielding control just long
236 * enough to finish the update.
237 */
238
239 static uint64_t safe_grab_timer_value(struct timer *t)
240 {
241 #if defined(__LP64__)
242 return t->all_bits;
243 #else
244 uint64_t time = t->high_bits; /* endian independent grab */
245 time = (time << 32) | t->low_bits;
246 return time;
247 #endif
248 }
249
250 kern_return_t
251 stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, uint64_t delta_since_timestamp, unsigned *bytes_traced)
252 {
253 kern_return_t error = KERN_SUCCESS;
254 boolean_t istate;
255
256 if ((buf == NULL) || (size <= 0) || (bytes_traced == NULL)) {
257 return KERN_INVALID_ARGUMENT;
258 }
259
260 /* cap in individual stackshot to SANE_TRACEBUF_SIZE */
261 if (size > SANE_TRACEBUF_SIZE) {
262 size = SANE_TRACEBUF_SIZE;
263 }
264
265 /* Serialize tracing */
266 if (flags & STACKSHOT_TRYLOCK) {
267 if (!STACKSHOT_SUBSYS_TRY_LOCK()) {
268 return KERN_LOCK_OWNED;
269 }
270 } else {
271 STACKSHOT_SUBSYS_LOCK();
272 }
273
274 istate = ml_set_interrupts_enabled(FALSE);
275
276 struct kcdata_descriptor kcdata;
277 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ?
278 KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
279
280 error = kcdata_memory_static_init(&kcdata, (mach_vm_address_t)buf, hdr_tag, size,
281 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
282 if (error) {
283 goto out;
284 }
285
286 /* Preload trace parameters*/
287 kdp_snapshot_preflight(pid, buf, size, flags, &kcdata, delta_since_timestamp);
288
289 /* Trap to the debugger to obtain a coherent stack snapshot; this populates
290 * the trace buffer
291 */
292 stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
293
294 ml_set_interrupts_enabled(istate);
295
296 *bytes_traced = kdp_stack_snapshot_bytes_traced();
297
298 error = kdp_stack_snapshot_geterror();
299
300 out:
301 STACKSHOT_SUBSYS_UNLOCK();
302 return error;
303 }
304
305 #if CONFIG_TELEMETRY
306 kern_return_t
307 stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval)
308 {
309 int error = KERN_SUCCESS;
310 uint32_t bytes_traced = 0;
311
312 *retval = -1;
313
314 /*
315 * Control related operations
316 */
317 if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_ENABLE) {
318 telemetry_global_ctl(1);
319 *retval = 0;
320 goto exit;
321 } else if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_DISABLE) {
322 telemetry_global_ctl(0);
323 *retval = 0;
324 goto exit;
325 }
326
327 /*
328 * Data related operations
329 */
330 *retval = -1;
331
332 if ((((void*)tracebuf) == NULL) || (tracebuf_size == 0)) {
333 error = KERN_INVALID_ARGUMENT;
334 goto exit;
335 }
336
337 STACKSHOT_SUBSYS_LOCK();
338
339 if (flags & STACKSHOT_GET_MICROSTACKSHOT) {
340 if (tracebuf_size > SANE_TRACEBUF_SIZE) {
341 error = KERN_INVALID_ARGUMENT;
342 goto unlock_exit;
343 }
344
345 bytes_traced = tracebuf_size;
346 error = telemetry_gather(tracebuf, &bytes_traced,
347 (flags & STACKSHOT_SET_MICROSTACKSHOT_MARK) ? TRUE : FALSE);
348 *retval = (int)bytes_traced;
349 goto unlock_exit;
350 }
351
352 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
353
354 if (tracebuf_size > SANE_BOOTPROFILE_TRACEBUF_SIZE) {
355 error = KERN_INVALID_ARGUMENT;
356 goto unlock_exit;
357 }
358
359 bytes_traced = tracebuf_size;
360 error = bootprofile_gather(tracebuf, &bytes_traced);
361 *retval = (int)bytes_traced;
362 }
363
364 unlock_exit:
365 STACKSHOT_SUBSYS_UNLOCK();
366 exit:
367 return error;
368 }
369 #endif /* CONFIG_TELEMETRY */
370
371 /*
372 * Return the estimated size of a stackshot based on the
373 * number of currently running threads and tasks.
374 */
375 uint32_t
376 get_stackshot_estsize(uint32_t prev_size_hint)
377 {
378 vm_size_t thread_total;
379 vm_size_t task_total;
380 uint32_t estimated_size;
381
382 thread_total = (threads_count * sizeof(struct thread_snapshot));
383 task_total = (tasks_count * (sizeof(struct task_snapshot) + TASK_UUID_AVG_SIZE));
384
385 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE((thread_total + task_total + STACKSHOT_SUPP_SIZE), PAGE_MASK);
386 if (estimated_size < prev_size_hint) {
387 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE(prev_size_hint, PAGE_MASK);
388 }
389
390 return estimated_size;
391 }
392
393 /*
394 * stackshot_remap_buffer: Utility function to remap bytes_traced bytes starting at stackshotbuf
395 * into the current task's user space and subsequently copy out the address
396 * at which the buffer has been mapped in user space to out_buffer_addr.
397 *
398 * Inputs: stackshotbuf - pointer to the original buffer in the kernel's address space
399 * bytes_traced - length of the buffer to remap starting from stackshotbuf
400 * out_buffer_addr - pointer to placeholder where newly mapped buffer will be mapped.
401 * out_size_addr - pointer to be filled in with the size of the buffer
402 *
403 * Outputs: ENOSPC if there is not enough free space in the task's address space to remap the buffer
404 * EINVAL for all other errors returned by task_remap_buffer/mach_vm_remap
405 * an error from copyout
406 */
407 static kern_return_t
408 stackshot_remap_buffer(void *stackshotbuf, uint32_t bytes_traced, uint64_t out_buffer_addr, uint64_t out_size_addr)
409 {
410 int error = 0;
411 mach_vm_offset_t stackshotbuf_user_addr = (mach_vm_offset_t)NULL;
412 vm_prot_t cur_prot, max_prot;
413
414 error = mach_vm_remap(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0,
415 VM_FLAGS_ANYWHERE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT);
416 /*
417 * If the call to mach_vm_remap fails, we return the appropriate converted error
418 */
419 if (error == KERN_SUCCESS) {
420 /*
421 * If we fail to copy out the address or size of the new buffer, we remove the buffer mapping that
422 * we just made in the task's user space.
423 */
424 error = copyout(CAST_DOWN(void *, &stackshotbuf_user_addr), (user_addr_t)out_buffer_addr, sizeof(stackshotbuf_user_addr));
425 if (error != KERN_SUCCESS) {
426 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
427 return error;
428 }
429 error = copyout(&bytes_traced, (user_addr_t)out_size_addr, sizeof(bytes_traced));
430 if (error != KERN_SUCCESS) {
431 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
432 return error;
433 }
434 }
435 return error;
436 }
437
438 kern_return_t
439 kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user)
440 {
441 int error = 0;
442 boolean_t prev_interrupt_state;
443 uint32_t bytes_traced = 0;
444 uint32_t stackshotbuf_size = 0;
445 void * stackshotbuf = NULL;
446 kcdata_descriptor_t kcdata_p = NULL;
447
448 void * buf_to_free = NULL;
449 int size_to_free = 0;
450
451 /* Parsed arguments */
452 uint64_t out_buffer_addr;
453 uint64_t out_size_addr;
454 int pid = -1;
455 uint32_t flags;
456 uint64_t since_timestamp;
457 uint32_t size_hint = 0;
458
459 if(stackshot_config == NULL) {
460 return KERN_INVALID_ARGUMENT;
461 }
462
463 switch (stackshot_config_version) {
464 case STACKSHOT_CONFIG_TYPE:
465 if (stackshot_config_size != sizeof(stackshot_config_t)) {
466 return KERN_INVALID_ARGUMENT;
467 }
468 stackshot_config_t *config = (stackshot_config_t *) stackshot_config;
469 out_buffer_addr = config->sc_out_buffer_addr;
470 out_size_addr = config->sc_out_size_addr;
471 pid = config->sc_pid;
472 flags = config->sc_flags;
473 since_timestamp = config->sc_delta_timestamp;
474 if (config->sc_size <= SANE_TRACEBUF_SIZE) {
475 size_hint = config->sc_size;
476 }
477 break;
478 default:
479 return KERN_NOT_SUPPORTED;
480 }
481
482 /*
483 * Currently saving a kernel buffer and trylock are only supported from the
484 * internal/KEXT API.
485 */
486 if (stackshot_from_user) {
487 if (flags & (STACKSHOT_TRYLOCK | STACKSHOT_SAVE_IN_KERNEL_BUFFER | STACKSHOT_FROM_PANIC)) {
488 return KERN_NO_ACCESS;
489 }
490 } else {
491 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
492 return KERN_NOT_SUPPORTED;
493 }
494 }
495
496 /*
497 * We only support the KDP fault path and delta snapshots and tailspin mode with the kcdata format
498 */
499 if (!(flags & STACKSHOT_KCDATA_FORMAT)) {
500 return KERN_NOT_SUPPORTED;
501 }
502
503 /*
504 * If we're not saving the buffer in the kernel pointer, we need places to copy into.
505 */
506 if ((!out_buffer_addr || !out_size_addr) && !(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
507 return KERN_INVALID_ARGUMENT;
508 }
509
510 if (since_timestamp != 0 && ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) == 0)) {
511 return KERN_INVALID_ARGUMENT;
512 }
513
514 STACKSHOT_SUBSYS_LOCK();
515
516 if (flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER) {
517 /*
518 * Don't overwrite an existing stackshot
519 */
520 if (kernel_stackshot_buf != NULL) {
521 error = KERN_MEMORY_PRESENT;
522 goto error_exit;
523 }
524 } else if (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER) {
525 if ((kernel_stackshot_buf == NULL) || (kernel_stackshot_buf_size <= 0)) {
526 error = KERN_NOT_IN_SET;
527 goto error_exit;
528 }
529 error = stackshot_remap_buffer(kernel_stackshot_buf, kernel_stackshot_buf_size,
530 out_buffer_addr, out_size_addr);
531 /*
532 * If we successfully remapped the buffer into the user's address space, we
533 * set buf_to_free and size_to_free so the prior kernel mapping will be removed
534 * and then clear the kernel stackshot pointer and associated size.
535 */
536 if (error == KERN_SUCCESS) {
537 buf_to_free = kernel_stackshot_buf;
538 size_to_free = (int) VM_MAP_ROUND_PAGE(kernel_stackshot_buf_size, PAGE_MASK);
539 kernel_stackshot_buf = NULL;
540 kernel_stackshot_buf_size = 0;
541 }
542
543 goto error_exit;
544 }
545
546 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
547 void *bootprofile = NULL;
548 uint32_t len = 0;
549 #if CONFIG_TELEMETRY
550 bootprofile_get(&bootprofile, &len);
551 #endif
552 if (!bootprofile || !len) {
553 error = KERN_NOT_IN_SET;
554 goto error_exit;
555 }
556 error = stackshot_remap_buffer(bootprofile, len, out_buffer_addr, out_size_addr);
557 goto error_exit;
558 }
559
560 stackshotbuf_size = get_stackshot_estsize(size_hint);
561
562 for (; stackshotbuf_size <= SANE_TRACEBUF_SIZE; stackshotbuf_size <<= 1) {
563 if (kmem_alloc(kernel_map, (vm_offset_t *)&stackshotbuf, stackshotbuf_size, VM_KERN_MEMORY_DIAG) != KERN_SUCCESS) {
564 error = KERN_RESOURCE_SHORTAGE;
565 goto error_exit;
566 }
567
568 /*
569 * If someone has panicked, don't try and enter the debugger
570 */
571 if (panic_active()) {
572 error = KERN_RESOURCE_SHORTAGE;
573 goto error_exit;
574 }
575
576 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
577 kcdata_p = kcdata_memory_alloc_init((mach_vm_address_t)stackshotbuf, hdr_tag, stackshotbuf_size,
578 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
579
580 stackshot_duration_outer = NULL;
581 uint64_t time_start = mach_absolute_time();
582
583 /*
584 * Disable interrupts and save the current interrupt state.
585 */
586 prev_interrupt_state = ml_set_interrupts_enabled(FALSE);
587
588 /*
589 * Load stackshot parameters.
590 */
591 kdp_snapshot_preflight(pid, stackshotbuf, stackshotbuf_size, flags, kcdata_p, since_timestamp);
592
593 /*
594 * Trap to the debugger to obtain a stackshot (this will populate the buffer).
595 */
596 stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
597
598 ml_set_interrupts_enabled(prev_interrupt_state);
599
600 /* record the duration that interupts were disabled */
601
602 uint64_t time_end = mach_absolute_time();
603 if (stackshot_duration_outer) {
604 *stackshot_duration_outer = time_end - time_start;
605 }
606
607 error = kdp_stack_snapshot_geterror();
608 if (error != KERN_SUCCESS) {
609 if (kcdata_p != NULL) {
610 kcdata_memory_destroy(kcdata_p);
611 kcdata_p = NULL;
612 stackshot_kcdata_p = NULL;
613 }
614 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
615 stackshotbuf = NULL;
616 if (error == KERN_INSUFFICIENT_BUFFER_SIZE) {
617 /*
618 * If we didn't allocate a big enough buffer, deallocate and try again.
619 */
620 continue;
621 } else {
622 goto error_exit;
623 }
624 }
625
626 bytes_traced = kdp_stack_snapshot_bytes_traced();
627
628 if (bytes_traced <= 0) {
629 error = KERN_ABORTED;
630 goto error_exit;
631 }
632
633 assert(bytes_traced <= stackshotbuf_size);
634 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
635 error = stackshot_remap_buffer(stackshotbuf, bytes_traced, out_buffer_addr, out_size_addr);
636 goto error_exit;
637 }
638
639 /*
640 * Save the stackshot in the kernel buffer.
641 */
642 kernel_stackshot_buf = stackshotbuf;
643 kernel_stackshot_buf_size = bytes_traced;
644 /*
645 * Figure out if we didn't use all the pages in the buffer. If so, we set buf_to_free to the beginning of
646 * the next page after the end of the stackshot in the buffer so that the kmem_free clips the buffer and
647 * update size_to_free for kmem_free accordingly.
648 */
649 size_to_free = stackshotbuf_size - (int) VM_MAP_ROUND_PAGE(bytes_traced, PAGE_MASK);
650
651 assert(size_to_free >= 0);
652
653 if (size_to_free != 0) {
654 buf_to_free = (void *)((uint64_t)stackshotbuf + stackshotbuf_size - size_to_free);
655 }
656
657 stackshotbuf = NULL;
658 stackshotbuf_size = 0;
659 goto error_exit;
660 }
661
662 if (stackshotbuf_size > SANE_TRACEBUF_SIZE) {
663 error = KERN_RESOURCE_SHORTAGE;
664 }
665
666 error_exit:
667 if (kcdata_p != NULL) {
668 kcdata_memory_destroy(kcdata_p);
669 kcdata_p = NULL;
670 stackshot_kcdata_p = NULL;
671 }
672
673 if (stackshotbuf != NULL) {
674 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
675 }
676 if (buf_to_free != NULL) {
677 kmem_free(kernel_map, (vm_offset_t)buf_to_free, size_to_free);
678 }
679 STACKSHOT_SUBSYS_UNLOCK();
680 return error;
681 }
682
683 /* Cache stack snapshot parameters in preparation for a trace */
684 void
685 kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags,
686 kcdata_descriptor_t data_p, uint64_t since_timestamp)
687 {
688 uint64_t microsecs = 0, secs = 0;
689 clock_get_calendar_microtime((clock_sec_t *)&secs, (clock_usec_t *)&microsecs);
690
691 stackshot_microsecs = microsecs + (secs * USEC_PER_SEC);
692 stack_snapshot_pid = pid;
693 stack_snapshot_buf = tracebuf;
694 stack_snapshot_bufsize = tracebuf_size;
695 stack_snapshot_flags = flags;
696 stack_snapshot_delta_since_timestamp = since_timestamp;
697
698 panic_stackshot = ((flags & STACKSHOT_FROM_PANIC) != 0);
699
700 if (data_p != NULL) {
701 stackshot_kcdata_p = data_p;
702 }
703 }
704
705 kern_return_t
706 kdp_stack_snapshot_geterror(void)
707 {
708 return stack_snapshot_ret;
709 }
710
711 uint32_t
712 kdp_stack_snapshot_bytes_traced(void)
713 {
714 return stack_snapshot_bytes_traced;
715 }
716
717 static boolean_t memory_iszero(void *addr, size_t size)
718 {
719 char *data = (char *)addr;
720 for (size_t i = 0; i < size; i++){
721 if (data[i] != 0)
722 return FALSE;
723 }
724 return TRUE;
725 }
726
727 #define kcd_end_address(kcd) ((void *)((uint64_t)((kcd)->kcd_addr_begin) + kcdata_memory_get_used_bytes((kcd))))
728 #define kcd_max_address(kcd) ((void *)((kcd)->kcd_addr_begin + (kcd)->kcd_length))
729 /*
730 * Use of the kcd_exit_on_error(action) macro requires a local
731 * 'kern_return_t error' variable and 'error_exit' label.
732 */
733 #define kcd_exit_on_error(action) \
734 do { \
735 if (KERN_SUCCESS != (error = (action))) { \
736 if (error == KERN_RESOURCE_SHORTAGE) { \
737 error = KERN_INSUFFICIENT_BUFFER_SIZE; \
738 } \
739 goto error_exit; \
740 } \
741 } while (0); /* end kcd_exit_on_error */
742
743 static uint64_t
744 kcdata_get_task_ss_flags(task_t task)
745 {
746 uint64_t ss_flags = 0;
747 boolean_t task64 = task_has_64BitAddr(task);
748
749 if (task64)
750 ss_flags |= kUser64_p;
751 if (!task->active || task_is_a_corpse(task))
752 ss_flags |= kTerminatedSnapshot;
753 if (task->pidsuspended)
754 ss_flags |= kPidSuspended;
755 if (task->frozen)
756 ss_flags |= kFrozen;
757 if (task->effective_policy.tep_darwinbg == 1)
758 ss_flags |= kTaskDarwinBG;
759 if (task->requested_policy.trp_role == TASK_FOREGROUND_APPLICATION)
760 ss_flags |= kTaskIsForeground;
761 if (task->requested_policy.trp_boosted == 1)
762 ss_flags |= kTaskIsBoosted;
763 if (task->effective_policy.tep_sup_active == 1)
764 ss_flags |= kTaskIsSuppressed;
765 #if CONFIG_MEMORYSTATUS
766 if (memorystatus_proc_is_dirty_unsafe(task->bsd_info))
767 ss_flags |= kTaskIsDirty;
768 #endif
769
770 ss_flags |= (0x7 & workqueue_get_pwq_state_kdp(task->bsd_info)) << 17;
771
772 #if IMPORTANCE_INHERITANCE
773 if (task->task_imp_base) {
774 if (task->task_imp_base->iit_donor)
775 ss_flags |= kTaskIsImpDonor;
776 if (task->task_imp_base->iit_live_donor)
777 ss_flags |= kTaskIsLiveImpDonor;
778 }
779 #endif
780
781 return ss_flags;
782 }
783
784 static kern_return_t
785 kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyld_uuid_info_64_v2 *sys_shared_cache_loadinfo, uint32_t trace_flags, uint64_t *task_snap_ss_flags)
786 {
787 kern_return_t error = KERN_SUCCESS;
788 mach_vm_address_t out_addr = 0;
789
790 uint8_t shared_cache_identifier[16];
791 uint64_t shared_cache_slide = 0;
792 uint64_t shared_cache_base_address = 0;
793 int task_pid = pid_from_task(task);
794 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
795 uint32_t kdp_fault_results = 0;
796
797 assert(task_snap_ss_flags != NULL);
798
799 if (task->shared_region && ml_validate_nofault((vm_offset_t)task->shared_region, sizeof(struct vm_shared_region))) {
800 struct vm_shared_region *sr = task->shared_region;
801 shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping;
802 } else {
803 *task_snap_ss_flags |= kTaskSharedRegionInfoUnavailable;
804 }
805
806 if (!shared_cache_base_address ||
807 !kdp_copyin(task->map, shared_cache_base_address + offsetof(struct _dyld_cache_header, uuid),
808 shared_cache_identifier, sizeof(shared_cache_identifier), should_fault, &kdp_fault_results)) {
809 goto error_exit;
810 }
811
812 if (task->shared_region) {
813 /*
814 * No refcounting here, but we are in debugger
815 * context, so that should be safe.
816 */
817 shared_cache_slide = task->shared_region->sr_slide_info.slide;
818 } else {
819 shared_cache_slide = 0;
820 }
821
822 if (sys_shared_cache_loadinfo) {
823 if (task_pid == 1) {
824 /* save launchd's shared cache info as system level */
825 stackshot_memcpy(sys_shared_cache_loadinfo->imageUUID, shared_cache_identifier, sizeof(sys_shared_cache_loadinfo->imageUUID));
826 sys_shared_cache_loadinfo->imageLoadAddress = shared_cache_slide;
827 sys_shared_cache_loadinfo->imageSlidBaseAddress = shared_cache_slide + task->shared_region->sr_base_address;
828
829 goto error_exit;
830 } else {
831 if (shared_cache_slide == sys_shared_cache_loadinfo->imageLoadAddress &&
832 0 == memcmp(shared_cache_identifier, sys_shared_cache_loadinfo->imageUUID,
833 sizeof(sys_shared_cache_loadinfo->imageUUID))) {
834 /* skip adding shared cache info. its same as system level one */
835 goto error_exit;
836 }
837 }
838 }
839
840 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64_v2), &out_addr));
841 struct dyld_uuid_info_64_v2 *shared_cache_data = (struct dyld_uuid_info_64_v2 *)out_addr;
842 shared_cache_data->imageLoadAddress = shared_cache_slide;
843 stackshot_memcpy(shared_cache_data->imageUUID, shared_cache_identifier, sizeof(shared_cache_data->imageUUID));
844 shared_cache_data->imageSlidBaseAddress = shared_cache_base_address;
845
846 error_exit:
847 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
848 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
849 }
850
851 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
852 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
853 }
854
855 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
856 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
857 }
858
859 return error;
860 }
861
862 static kern_return_t
863 kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, uint64_t *task_snap_ss_flags)
864 {
865 boolean_t save_loadinfo_p = ((trace_flags & STACKSHOT_SAVE_LOADINFO) != 0);
866 boolean_t save_kextloadinfo_p = ((trace_flags & STACKSHOT_SAVE_KEXT_LOADINFO) != 0);
867 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
868 boolean_t minimize_uuids = collect_delta_stackshot && ((trace_flags & STACKSHOT_TAILSPIN) != 0);
869 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
870
871 kern_return_t error = KERN_SUCCESS;
872 mach_vm_address_t out_addr = 0;
873
874 uint32_t uuid_info_count = 0;
875 mach_vm_address_t uuid_info_addr = 0;
876 uint64_t uuid_info_timestamp = 0;
877 uint32_t kdp_fault_results = 0;
878
879 assert(task_snap_ss_flags != NULL);
880
881 int task_pid = pid_from_task(task);
882 boolean_t task64 = task_has_64BitAddr(task);
883
884 if (save_loadinfo_p && have_pmap && task->active && task_pid > 0) {
885 /* Read the dyld_all_image_infos struct from the task memory to get UUID array count and location */
886 if (task64) {
887 struct user64_dyld_all_image_infos task_image_infos;
888 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
889 sizeof(struct user64_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
890 uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
891 uuid_info_addr = task_image_infos.uuidArray;
892 if (task_image_infos.version >= 15) {
893 uuid_info_timestamp = task_image_infos.timestamp;
894 }
895 }
896 } else {
897 struct user32_dyld_all_image_infos task_image_infos;
898 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
899 sizeof(struct user32_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
900 uuid_info_count = task_image_infos.uuidArrayCount;
901 uuid_info_addr = task_image_infos.uuidArray;
902 if (task_image_infos.version >= 15) {
903 uuid_info_timestamp = task_image_infos.timestamp;
904 }
905 }
906 }
907
908 /*
909 * If we get a NULL uuid_info_addr (which can happen when we catch dyld in the middle of updating
910 * this data structure), we zero the uuid_info_count so that we won't even try to save load info
911 * for this task.
912 */
913 if (!uuid_info_addr) {
914 uuid_info_count = 0;
915 }
916 }
917
918 if (have_pmap && task_pid == 0) {
919 if (save_kextloadinfo_p && ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader))) {
920 uuid_info_count = gLoadedKextSummaries->numSummaries + 1; /* include main kernel UUID */
921 } else {
922 uuid_info_count = 1; /* include kernelcache UUID (embedded) or kernel UUID (desktop) */
923 }
924 }
925
926 if (task_pid > 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
927 if (minimize_uuids && uuid_info_timestamp != 0 && uuid_info_timestamp < stack_snapshot_delta_since_timestamp)
928 goto error_exit;
929
930 uint32_t uuid_info_size = (uint32_t)(task64 ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
931 uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
932
933 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, (task64 ? KCDATA_TYPE_LIBRARY_LOADINFO64 : KCDATA_TYPE_LIBRARY_LOADINFO),
934 uuid_info_size, uuid_info_count, &out_addr));
935
936 /* Copy in the UUID info array
937 * It may be nonresident, in which case just fix up nloadinfos to 0 in the task_snap
938 */
939 if (have_pmap && !kdp_copyin(task->map, uuid_info_addr, (void *)out_addr, uuid_info_array_size, should_fault, &kdp_fault_results)) {
940 bzero((void *)out_addr, uuid_info_array_size);
941 }
942
943 } else if (task_pid == 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
944 if (minimize_uuids && gLoadedKextSummaries != 0 && gLoadedKextSummariesTimestamp < stack_snapshot_delta_since_timestamp)
945 goto error_exit;
946
947 uintptr_t image_load_address;
948
949 do {
950
951
952 if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) {
953 /* Kernel UUID not found or inaccessible */
954 break;
955 }
956
957 kcd_exit_on_error(kcdata_get_memory_addr_for_array(
958 kcd, (sizeof(kernel_uuid_info) == sizeof(struct user64_dyld_uuid_info)) ? KCDATA_TYPE_LIBRARY_LOADINFO64
959 : KCDATA_TYPE_LIBRARY_LOADINFO,
960 sizeof(kernel_uuid_info), uuid_info_count, &out_addr));
961 kernel_uuid_info *uuid_info_array = (kernel_uuid_info *)out_addr;
962 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(vm_kernel_stext);
963 uuid_info_array[0].imageLoadAddress = image_load_address;
964 stackshot_memcpy(&uuid_info_array[0].imageUUID, kernel_uuid, sizeof(uuid_t));
965
966 if (save_kextloadinfo_p &&
967 ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader)) &&
968 ml_validate_nofault((vm_offset_t)(&gLoadedKextSummaries->summaries[0]),
969 gLoadedKextSummaries->entry_size * gLoadedKextSummaries->numSummaries)) {
970 uint32_t kexti;
971 for (kexti=0 ; kexti < gLoadedKextSummaries->numSummaries; kexti++) {
972 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(gLoadedKextSummaries->summaries[kexti].address);
973 uuid_info_array[kexti + 1].imageLoadAddress = image_load_address;
974 stackshot_memcpy(&uuid_info_array[kexti + 1].imageUUID, &gLoadedKextSummaries->summaries[kexti].uuid, sizeof(uuid_t));
975 }
976 }
977 } while(0);
978 }
979
980 error_exit:
981 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
982 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
983 }
984
985 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
986 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
987 }
988
989 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
990 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
991 }
992
993 return error;
994 }
995
996 static kern_return_t
997 kcdata_record_task_iostats(kcdata_descriptor_t kcd, task_t task)
998 {
999 kern_return_t error = KERN_SUCCESS;
1000 mach_vm_address_t out_addr = 0;
1001
1002 /* I/O Statistics if any counters are non zero */
1003 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1004 if (task->task_io_stats && !memory_iszero(task->task_io_stats, sizeof(struct io_stat_info))) {
1005 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1006 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1007 _iostat->ss_disk_reads_count = task->task_io_stats->disk_reads.count;
1008 _iostat->ss_disk_reads_size = task->task_io_stats->disk_reads.size;
1009 _iostat->ss_disk_writes_count = (task->task_io_stats->total_io.count - task->task_io_stats->disk_reads.count);
1010 _iostat->ss_disk_writes_size = (task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size);
1011 _iostat->ss_paging_count = task->task_io_stats->paging.count;
1012 _iostat->ss_paging_size = task->task_io_stats->paging.size;
1013 _iostat->ss_non_paging_count = (task->task_io_stats->total_io.count - task->task_io_stats->paging.count);
1014 _iostat->ss_non_paging_size = (task->task_io_stats->total_io.size - task->task_io_stats->paging.size);
1015 _iostat->ss_metadata_count = task->task_io_stats->metadata.count;
1016 _iostat->ss_metadata_size = task->task_io_stats->metadata.size;
1017 _iostat->ss_data_count = (task->task_io_stats->total_io.count - task->task_io_stats->metadata.count);
1018 _iostat->ss_data_size = (task->task_io_stats->total_io.size - task->task_io_stats->metadata.size);
1019 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1020 _iostat->ss_io_priority_count[i] = task->task_io_stats->io_priority[i].count;
1021 _iostat->ss_io_priority_size[i] = task->task_io_stats->io_priority[i].size;
1022 }
1023 }
1024
1025 error_exit:
1026 return error;
1027 }
1028
1029 static kern_return_t
1030 kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, uint64_t **task_snap_ss_flags)
1031 {
1032 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1033 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
1034
1035 kern_return_t error = KERN_SUCCESS;
1036 mach_vm_address_t out_addr = 0;
1037 struct task_snapshot_v2 * cur_tsnap = NULL;
1038
1039 assert(task_snap_ss_flags != NULL);
1040
1041 int task_pid = pid_from_task(task);
1042 uint64_t task_uniqueid = get_task_uniqueid(task);
1043
1044 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_SNAPSHOT, sizeof(struct task_snapshot_v2), &out_addr));
1045
1046 cur_tsnap = (struct task_snapshot_v2 *)out_addr;
1047
1048 cur_tsnap->ts_unique_pid = task_uniqueid;
1049 cur_tsnap->ts_ss_flags = kcdata_get_task_ss_flags(task);
1050 *task_snap_ss_flags = &cur_tsnap->ts_ss_flags;
1051 cur_tsnap->ts_user_time_in_terminated_threads = task->total_user_time;
1052 cur_tsnap->ts_system_time_in_terminated_threads = task->total_system_time;
1053
1054 cur_tsnap->ts_p_start_sec = 0;
1055 proc_starttime_kdp(task->bsd_info, &cur_tsnap->ts_p_start_sec, NULL, NULL);
1056
1057 cur_tsnap->ts_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
1058 cur_tsnap->ts_max_resident_size = get_task_resident_max(task);
1059 cur_tsnap->ts_suspend_count = task->suspend_count;
1060 cur_tsnap->ts_faults = task->faults;
1061 cur_tsnap->ts_pageins = task->pageins;
1062 cur_tsnap->ts_cow_faults = task->cow_faults;
1063 cur_tsnap->ts_was_throttled = (uint32_t) proc_was_throttled_from_task(task);
1064 cur_tsnap->ts_did_throttle = (uint32_t) proc_did_throttle_from_task(task);
1065 cur_tsnap->ts_latency_qos = (task->effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED) ?
1066 LATENCY_QOS_TIER_UNSPECIFIED : ((0xFF << 16) | task->effective_policy.tep_latency_qos);
1067 cur_tsnap->ts_pid = task_pid;
1068
1069 /* Add the BSD process identifiers */
1070 if (task_pid != -1 && task->bsd_info != NULL)
1071 proc_name_kdp(task, cur_tsnap->ts_p_comm, sizeof(cur_tsnap->ts_p_comm));
1072 else {
1073 cur_tsnap->ts_p_comm[0] = '\0';
1074 #if IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG)
1075 if (task->task_imp_base != NULL) {
1076 stackshot_strlcpy(cur_tsnap->ts_p_comm, &task->task_imp_base->iit_procname[0],
1077 MIN((int)sizeof(task->task_imp_base->iit_procname), (int)sizeof(cur_tsnap->ts_p_comm)));
1078 }
1079 #endif
1080 }
1081
1082 if (collect_iostats) {
1083 kcd_exit_on_error(kcdata_record_task_iostats(kcd, task));
1084 }
1085
1086 error_exit:
1087 return error;
1088 }
1089
1090 static kern_return_t
1091 kcdata_record_task_delta_snapshot(kcdata_descriptor_t kcd, task_t task, boolean_t have_pmap, uint64_t **task_snap_ss_flags)
1092 {
1093 kern_return_t error = KERN_SUCCESS;
1094 struct task_delta_snapshot_v2 * cur_tsnap = NULL;
1095 mach_vm_address_t out_addr = 0;
1096
1097 uint64_t task_uniqueid = get_task_uniqueid(task);
1098 assert(task_snap_ss_flags != NULL);
1099
1100 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT, sizeof(struct task_delta_snapshot_v2), &out_addr));
1101
1102 cur_tsnap = (struct task_delta_snapshot_v2 *)out_addr;
1103
1104 cur_tsnap->tds_unique_pid = task_uniqueid;
1105 cur_tsnap->tds_ss_flags = kcdata_get_task_ss_flags(task);
1106 *task_snap_ss_flags = &cur_tsnap->tds_ss_flags;
1107
1108 cur_tsnap->tds_user_time_in_terminated_threads = task->total_user_time;
1109 cur_tsnap->tds_system_time_in_terminated_threads = task->total_system_time;
1110
1111 cur_tsnap->tds_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
1112
1113 cur_tsnap->tds_max_resident_size = get_task_resident_max(task);
1114 cur_tsnap->tds_suspend_count = task->suspend_count;
1115 cur_tsnap->tds_faults = task->faults;
1116 cur_tsnap->tds_pageins = task->pageins;
1117 cur_tsnap->tds_cow_faults = task->cow_faults;
1118 cur_tsnap->tds_was_throttled = (uint32_t)proc_was_throttled_from_task(task);
1119 cur_tsnap->tds_did_throttle = (uint32_t)proc_did_throttle_from_task(task);
1120 cur_tsnap->tds_latency_qos = (task-> effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED)
1121 ? LATENCY_QOS_TIER_UNSPECIFIED
1122 : ((0xFF << 16) | task-> effective_policy.tep_latency_qos);
1123
1124 error_exit:
1125 return error;
1126 }
1127
1128 static kern_return_t
1129 kcdata_record_thread_iostats(kcdata_descriptor_t kcd, thread_t thread)
1130 {
1131 kern_return_t error = KERN_SUCCESS;
1132 mach_vm_address_t out_addr = 0;
1133
1134 /* I/O Statistics */
1135 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1136 if (thread->thread_io_stats && !memory_iszero(thread->thread_io_stats, sizeof(struct io_stat_info))) {
1137 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1138 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1139 _iostat->ss_disk_reads_count = thread->thread_io_stats->disk_reads.count;
1140 _iostat->ss_disk_reads_size = thread->thread_io_stats->disk_reads.size;
1141 _iostat->ss_disk_writes_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->disk_reads.count);
1142 _iostat->ss_disk_writes_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->disk_reads.size);
1143 _iostat->ss_paging_count = thread->thread_io_stats->paging.count;
1144 _iostat->ss_paging_size = thread->thread_io_stats->paging.size;
1145 _iostat->ss_non_paging_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->paging.count);
1146 _iostat->ss_non_paging_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->paging.size);
1147 _iostat->ss_metadata_count = thread->thread_io_stats->metadata.count;
1148 _iostat->ss_metadata_size = thread->thread_io_stats->metadata.size;
1149 _iostat->ss_data_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->metadata.count);
1150 _iostat->ss_data_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->metadata.size);
1151 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1152 _iostat->ss_io_priority_count[i] = thread->thread_io_stats->io_priority[i].count;
1153 _iostat->ss_io_priority_size[i] = thread->thread_io_stats->io_priority[i].size;
1154 }
1155 }
1156
1157 error_exit:
1158 return error;
1159 }
1160
1161 static kern_return_t
1162 kcdata_record_thread_snapshot(
1163 kcdata_descriptor_t kcd, thread_t thread, task_t task, uint32_t trace_flags, boolean_t have_pmap, boolean_t thread_on_core)
1164 {
1165 boolean_t dispatch_p = ((trace_flags & STACKSHOT_GET_DQ) != 0);
1166 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1167 boolean_t trace_fp_p = ((trace_flags & STACKSHOT_TAILSPIN) == 0);
1168 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1169 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
1170
1171 kern_return_t error = KERN_SUCCESS;
1172 mach_vm_address_t out_addr = 0;
1173 int saved_count = 0;
1174
1175 struct thread_snapshot_v3 * cur_thread_snap = NULL;
1176 char cur_thread_name[STACKSHOT_MAX_THREAD_NAME_SIZE];
1177 uint64_t tval = 0;
1178 boolean_t task64 = task_has_64BitAddr(task);
1179
1180 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v3), &out_addr));
1181 cur_thread_snap = (struct thread_snapshot_v3 *)out_addr;
1182
1183 /* Populate the thread snapshot header */
1184 cur_thread_snap->ths_thread_id = thread_tid(thread);
1185 cur_thread_snap->ths_wait_event = VM_KERNEL_UNSLIDE_OR_PERM(thread->wait_event);
1186 cur_thread_snap->ths_continuation = VM_KERNEL_UNSLIDE(thread->continuation);
1187 cur_thread_snap->ths_total_syscalls = thread->syscalls_mach + thread->syscalls_unix;
1188
1189 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1190 cur_thread_snap->ths_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1191 else
1192 cur_thread_snap->ths_voucher_identifier = 0;
1193
1194 cur_thread_snap->ths_dqserialnum = 0;
1195 if (dispatch_p && (task != kernel_task) && (task->active) && have_pmap) {
1196 uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
1197 if (dqkeyaddr != 0) {
1198 uint64_t dqaddr = 0;
1199 boolean_t copyin_ok = kdp_copyin_word(task, dqkeyaddr, &dqaddr, FALSE, NULL);
1200 if (copyin_ok && dqaddr != 0) {
1201 uint64_t dqserialnumaddr = dqaddr + get_task_dispatchqueue_serialno_offset(task);
1202 uint64_t dqserialnum = 0;
1203 copyin_ok = kdp_copyin_word(task, dqserialnumaddr, &dqserialnum, FALSE, NULL);
1204 if (copyin_ok) {
1205 cur_thread_snap->ths_ss_flags |= kHasDispatchSerial;
1206 cur_thread_snap->ths_dqserialnum = dqserialnum;
1207 }
1208 }
1209 }
1210 }
1211
1212 tval = safe_grab_timer_value(&thread->user_timer);
1213 cur_thread_snap->ths_user_time = tval;
1214 tval = safe_grab_timer_value(&thread->system_timer);
1215
1216 if (thread->precise_user_kernel_time) {
1217 cur_thread_snap->ths_sys_time = tval;
1218 } else {
1219 cur_thread_snap->ths_user_time += tval;
1220 cur_thread_snap->ths_sys_time = 0;
1221 }
1222
1223 cur_thread_snap->ths_ss_flags = 0;
1224 if (thread->effective_policy.thep_darwinbg)
1225 cur_thread_snap->ths_ss_flags |= kThreadDarwinBG;
1226 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1227 cur_thread_snap->ths_ss_flags |= kThreadIOPassive;
1228 if (thread->suspend_count > 0)
1229 cur_thread_snap->ths_ss_flags |= kThreadSuspended;
1230 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1231 cur_thread_snap->ths_ss_flags |= kGlobalForcedIdle;
1232 if (thread_on_core)
1233 cur_thread_snap->ths_ss_flags |= kThreadOnCore;
1234 if (stackshot_thread_is_idle_worker_unsafe(thread))
1235 cur_thread_snap->ths_ss_flags |= kThreadIdleWorker;
1236
1237 /* make sure state flags defined in kcdata.h still match internal flags */
1238 static_assert(SS_TH_WAIT == TH_WAIT);
1239 static_assert(SS_TH_SUSP == TH_SUSP);
1240 static_assert(SS_TH_RUN == TH_RUN);
1241 static_assert(SS_TH_UNINT == TH_UNINT);
1242 static_assert(SS_TH_TERMINATE == TH_TERMINATE);
1243 static_assert(SS_TH_TERMINATE2 == TH_TERMINATE2);
1244 static_assert(SS_TH_IDLE == TH_IDLE);
1245
1246 cur_thread_snap->ths_last_run_time = thread->last_run_time;
1247 cur_thread_snap->ths_last_made_runnable_time = thread->last_made_runnable_time;
1248 cur_thread_snap->ths_state = thread->state;
1249 cur_thread_snap->ths_sched_flags = thread->sched_flags;
1250 cur_thread_snap->ths_base_priority = thread->base_pri;
1251 cur_thread_snap->ths_sched_priority = thread->sched_pri;
1252 cur_thread_snap->ths_eqos = thread->effective_policy.thep_qos;
1253 cur_thread_snap->ths_rqos = thread->requested_policy.thrp_qos;
1254 cur_thread_snap->ths_rqos_override = thread->requested_policy.thrp_qos_override;
1255 cur_thread_snap->ths_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1256 cur_thread_snap->ths_thread_t = VM_KERNEL_ADDRPERM(thread);
1257
1258 /* if there is thread name then add to buffer */
1259 cur_thread_name[0] = '\0';
1260 proc_threadname_kdp(thread->uthread, cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE);
1261 if (strnlen(cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE) > 0) {
1262 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_NAME, sizeof(cur_thread_name), &out_addr));
1263 stackshot_memcpy((void *)out_addr, (void *)cur_thread_name, sizeof(cur_thread_name));
1264 }
1265
1266 /* record system and user cpu times */
1267 time_value_t user_time;
1268 time_value_t system_time;
1269 thread_read_times(thread, &user_time, &system_time);
1270 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_CPU_TIMES, sizeof(struct stackshot_cpu_times), &out_addr));
1271 struct stackshot_cpu_times * stackshot_cpu_times = (struct stackshot_cpu_times *)out_addr;
1272 stackshot_cpu_times->user_usec = ((uint64_t)user_time.seconds) * USEC_PER_SEC + user_time.microseconds;
1273 stackshot_cpu_times->system_usec = ((uint64_t)system_time.seconds) * USEC_PER_SEC + system_time.microseconds;
1274
1275 /* Trace user stack, if any */
1276 if (!active_kthreads_only_p && task->active && thread->task->map != kernel_map) {
1277 uint32_t thread_snapshot_flags = 0;
1278 /* 64-bit task? */
1279 if (task64) {
1280 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1281 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE,
1282 trace_fp_p, &thread_snapshot_flags);
1283 if (saved_count > 0) {
1284 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1285 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME64
1286 : STACKSHOT_KCTYPE_USER_STACKLR64,
1287 frame_size, saved_count / frame_size, &out_addr));
1288 cur_thread_snap->ths_ss_flags |= kUser64_p;
1289 }
1290 } else {
1291 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1292 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE, trace_fp_p,
1293 &thread_snapshot_flags);
1294 if (saved_count > 0) {
1295 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1296 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME
1297 : STACKSHOT_KCTYPE_USER_STACKLR,
1298 frame_size, saved_count / frame_size, &out_addr));
1299 }
1300 }
1301
1302 if (thread_snapshot_flags != 0) {
1303 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1304 }
1305 }
1306
1307 /* Call through to the machine specific trace routines
1308 * Frames are added past the snapshot header.
1309 */
1310 if (thread->kernel_stack != 0) {
1311 uint32_t thread_snapshot_flags = 0;
1312 #if defined(__LP64__)
1313 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1314 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1315 &thread_snapshot_flags);
1316 if (saved_count > 0) {
1317 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1318 cur_thread_snap->ths_ss_flags |= kKernel64_p;
1319 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME64
1320 : STACKSHOT_KCTYPE_KERN_STACKLR64,
1321 frame_size, saved_count / frame_size, &out_addr));
1322 }
1323 #else
1324 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1325 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1326 &thread_snapshot_flags);
1327 if (saved_count > 0) {
1328 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1329 kcd_exit_on_error(
1330 kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME : STACKSHOT_KCTYPE_KERN_STACKLR,
1331 frame_size, saved_count / frame_size, &out_addr));
1332 }
1333 #endif
1334 if (thread_snapshot_flags != 0) {
1335 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1336 }
1337 }
1338
1339 if (collect_iostats) {
1340 kcd_exit_on_error(kcdata_record_thread_iostats(kcd, thread));
1341 }
1342
1343 error_exit:
1344 return error;
1345 }
1346
1347 static int
1348 kcdata_record_thread_delta_snapshot(struct thread_delta_snapshot_v2 * cur_thread_snap, thread_t thread, boolean_t thread_on_core)
1349 {
1350 cur_thread_snap->tds_thread_id = thread_tid(thread);
1351 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1352 cur_thread_snap->tds_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1353 else
1354 cur_thread_snap->tds_voucher_identifier = 0;
1355
1356 cur_thread_snap->tds_ss_flags = 0;
1357 if (thread->effective_policy.thep_darwinbg)
1358 cur_thread_snap->tds_ss_flags |= kThreadDarwinBG;
1359 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1360 cur_thread_snap->tds_ss_flags |= kThreadIOPassive;
1361 if (thread->suspend_count > 0)
1362 cur_thread_snap->tds_ss_flags |= kThreadSuspended;
1363 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1364 cur_thread_snap->tds_ss_flags |= kGlobalForcedIdle;
1365 if (thread_on_core)
1366 cur_thread_snap->tds_ss_flags |= kThreadOnCore;
1367 if (stackshot_thread_is_idle_worker_unsafe(thread))
1368 cur_thread_snap->tds_ss_flags |= kThreadIdleWorker;
1369
1370 cur_thread_snap->tds_last_made_runnable_time = thread->last_made_runnable_time;
1371 cur_thread_snap->tds_state = thread->state;
1372 cur_thread_snap->tds_sched_flags = thread->sched_flags;
1373 cur_thread_snap->tds_base_priority = thread->base_pri;
1374 cur_thread_snap->tds_sched_priority = thread->sched_pri;
1375 cur_thread_snap->tds_eqos = thread->effective_policy.thep_qos;
1376 cur_thread_snap->tds_rqos = thread->requested_policy.thrp_qos;
1377 cur_thread_snap->tds_rqos_override = thread->requested_policy.thrp_qos_override;
1378 cur_thread_snap->tds_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1379
1380 return 0;
1381 }
1382
1383 /*
1384 * Why 12? 12 strikes a decent balance between allocating a large array on
1385 * the stack and having large kcdata item overheads for recording nonrunable
1386 * tasks.
1387 */
1388 #define UNIQUEIDSPERFLUSH 12
1389
1390 struct saved_uniqueids {
1391 uint64_t ids[UNIQUEIDSPERFLUSH];
1392 unsigned count;
1393 };
1394
1395 static kern_return_t
1396 flush_nonrunnable_tasks(struct saved_uniqueids * ids)
1397 {
1398 if (ids->count == 0)
1399 return KERN_SUCCESS;
1400 mach_vm_address_t out_addr = 0;
1401 kern_return_t ret = kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TASKS, sizeof(uint64_t),
1402 ids->count, &out_addr);
1403 if (ret != KERN_SUCCESS) {
1404 return ret;
1405 }
1406 stackshot_memcpy((void *)out_addr, ids->ids, sizeof(uint64_t) * ids->count);
1407 ids->count = 0;
1408 return ret;
1409 }
1410
1411 static kern_return_t
1412 handle_nonrunnable_task(struct saved_uniqueids * ids, uint64_t pid)
1413 {
1414 kern_return_t ret = KERN_SUCCESS;
1415 ids->ids[ids->count] = pid;
1416 ids->count++;
1417 assert(ids->count <= UNIQUEIDSPERFLUSH);
1418 if (ids->count == UNIQUEIDSPERFLUSH)
1419 ret = flush_nonrunnable_tasks(ids);
1420 return ret;
1421 }
1422
1423 enum thread_classification {
1424 tc_full_snapshot, /* take a full snapshot */
1425 tc_delta_snapshot, /* take a delta snapshot */
1426 tc_nonrunnable, /* only report id */
1427 };
1428
1429 static enum thread_classification
1430 classify_thread(thread_t thread, boolean_t * thread_on_core_p, uint32_t trace_flags)
1431 {
1432 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1433 boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
1434
1435 processor_t last_processor = thread->last_processor;
1436
1437 boolean_t thread_on_core =
1438 (last_processor != PROCESSOR_NULL && last_processor->state == PROCESSOR_RUNNING && last_processor->active_thread == thread);
1439
1440 *thread_on_core_p = thread_on_core;
1441
1442 /* Capture the full thread snapshot if this is not a delta stackshot or if the thread has run subsequent to the
1443 * previous full stackshot */
1444 if (!collect_delta_stackshot || thread_on_core || (thread->last_run_time > stack_snapshot_delta_since_timestamp)) {
1445 return tc_full_snapshot;
1446 } else {
1447 if (minimize_nonrunnables && !(thread->state & TH_RUN)) {
1448 return tc_nonrunnable;
1449 } else {
1450 return tc_delta_snapshot;
1451 }
1452 }
1453 }
1454
1455 static kern_return_t
1456 kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTraced)
1457 {
1458 kern_return_t error = KERN_SUCCESS;
1459 mach_vm_address_t out_addr = 0;
1460 uint64_t abs_time = 0, abs_time_end = 0;
1461 uint64_t *abs_time_addr = NULL;
1462 uint64_t system_state_flags = 0;
1463 int saved_count = 0;
1464 task_t task = TASK_NULL;
1465 thread_t thread = THREAD_NULL;
1466 mach_timebase_info_data_t timebase = {0, 0};
1467 uint32_t length_to_copy = 0, tmp32 = 0;
1468
1469 abs_time = mach_absolute_time();
1470
1471 /* process the flags */
1472 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1473 boolean_t save_donating_pids_p = ((trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0);
1474 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1475 boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
1476 boolean_t use_fault_path = ((trace_flags & (STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_ENABLE_BT_FAULTING)) != 0);
1477
1478 stack_enable_faulting = (trace_flags & (STACKSHOT_ENABLE_BT_FAULTING));
1479
1480
1481 struct saved_uniqueids saved_uniqueids = {.count = 0};
1482
1483 if (use_fault_path) {
1484 fault_stats.sfs_pages_faulted_in = 0;
1485 fault_stats.sfs_time_spent_faulting = 0;
1486 fault_stats.sfs_stopped_faulting = (uint8_t) FALSE;
1487 }
1488
1489 if (sizeof(void *) == 8)
1490 system_state_flags |= kKernel64_p;
1491
1492 if (stackshot_kcdata_p == NULL || pBytesTraced == NULL) {
1493 error = KERN_INVALID_ARGUMENT;
1494 goto error_exit;
1495 }
1496
1497 /* setup mach_absolute_time and timebase info -- copy out in some cases and needed to convert since_timestamp to seconds for proc start time */
1498 clock_timebase_info(&timebase);
1499
1500 /* begin saving data into the buffer */
1501 *pBytesTraced = 0;
1502 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_in_flags"));
1503 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, (uint32_t)pid, "stackshot_in_pid"));
1504 kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_kcdata_p, system_state_flags, "system_state_flags"));
1505
1506 #if CONFIG_JETSAM
1507 tmp32 = memorystatus_get_pressure_status_kdp();
1508 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_LEVEL, sizeof(uint32_t), &out_addr));
1509 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1510 #endif
1511
1512 if (!collect_delta_stackshot) {
1513 tmp32 = PAGE_SIZE;
1514 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_KERN_PAGE_SIZE, sizeof(uint32_t), &out_addr));
1515 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1516
1517 /* save boot-args and osversion string */
1518 length_to_copy = MIN((uint32_t)(strlen(version) + 1), OSVERSIZE);
1519 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_OSVERSION, length_to_copy, &out_addr));
1520 stackshot_strlcpy((char*)out_addr, &version[0], length_to_copy);
1521
1522 length_to_copy = MIN((uint32_t)(strlen(PE_boot_args()) + 1), OSVERSIZE);
1523 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_BOOTARGS, length_to_copy, &out_addr));
1524 stackshot_strlcpy((char*)out_addr, PE_boot_args(), length_to_copy);
1525
1526 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_TIMEBASE, sizeof(timebase), &out_addr));
1527 stackshot_memcpy((void *)out_addr, &timebase, sizeof(timebase));
1528 } else {
1529 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP, sizeof(uint64_t), &out_addr));
1530 stackshot_memcpy((void*)out_addr, &stack_snapshot_delta_since_timestamp, sizeof(stack_snapshot_delta_since_timestamp));
1531 }
1532
1533 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_MACH_ABSOLUTE_TIME, sizeof(uint64_t), &out_addr));
1534 abs_time_addr = (uint64_t *)out_addr;
1535 stackshot_memcpy((void *)abs_time_addr, &abs_time, sizeof(uint64_t));
1536
1537 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &out_addr));
1538 stackshot_memcpy((void *)out_addr, &stackshot_microsecs, sizeof(uint64_t));
1539
1540 /* reserve space of system level shared cache load info */
1541 struct dyld_uuid_info_64_v2 * sys_shared_cache_loadinfo = NULL;
1542 if (!collect_delta_stackshot) {
1543 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO,
1544 sizeof(struct dyld_uuid_info_64_v2), &out_addr));
1545 sys_shared_cache_loadinfo = (struct dyld_uuid_info_64_v2 *)out_addr;
1546 bzero((void *)sys_shared_cache_loadinfo, sizeof(struct dyld_uuid_info_64_v2));
1547 }
1548
1549 /* Add requested information first */
1550 if (trace_flags & STACKSHOT_GET_GLOBAL_MEM_STATS) {
1551 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_GLOBAL_MEM_STATS, sizeof(struct mem_and_io_snapshot), &out_addr));
1552 kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)out_addr);
1553 }
1554
1555 /* Iterate over tasks */
1556 queue_head_t *task_list = &tasks;
1557 queue_iterate(task_list, task, task_t, tasks) {
1558 int task_pid = 0;
1559 uint64_t task_uniqueid = 0;
1560 int num_delta_thread_snapshots = 0;
1561 int num_nonrunnable_threads = 0;
1562 uint64_t task_start_abstime = 0;
1563 boolean_t task_delta_stackshot = FALSE;
1564 boolean_t task64 = FALSE, have_map = FALSE, have_pmap = FALSE;
1565 boolean_t some_thread_ran = FALSE;
1566 uint64_t *task_snap_ss_flags = NULL;
1567
1568 if ((task == NULL) || !ml_validate_nofault((vm_offset_t)task, sizeof(struct task))) {
1569 error = KERN_FAILURE;
1570 goto error_exit;
1571 }
1572
1573 have_map = (task->map != NULL) && (ml_validate_nofault((vm_offset_t)(task->map), sizeof(struct _vm_map)));
1574 have_pmap = have_map && (task->map->pmap != NULL) && (ml_validate_nofault((vm_offset_t)(task->map->pmap), sizeof(struct pmap)));
1575
1576 task_pid = pid_from_task(task);
1577 task_uniqueid = get_task_uniqueid(task);
1578 task64 = task_has_64BitAddr(task);
1579
1580 if (!task->active || task_is_a_corpse(task)) {
1581 /*
1582 * Not interested in terminated tasks without threads, and
1583 * at the moment, stackshot can't handle a task without a name.
1584 */
1585 if (queue_empty(&task->threads) || task_pid == -1) {
1586 continue;
1587 }
1588 }
1589
1590 if (collect_delta_stackshot) {
1591 proc_starttime_kdp(task->bsd_info, NULL, NULL, &task_start_abstime);
1592 }
1593
1594 /* Trace everything, unless a process was specified */
1595 if ((pid == -1) || (pid == task_pid)) {
1596 #if DEBUG || DEVELOPMENT
1597 /* we might want to call kcdata_undo_add_container_begin(), which is
1598 * only safe if we call it after kcdata_add_container_marker() but
1599 * before adding any other kcdata items. In development kernels,
1600 * we'll remember where the buffer end was and confirm after calling
1601 * kcdata_undo_add_container_begin() that it's in exactly the same
1602 * place.*/
1603 mach_vm_address_t revert_addr = stackshot_kcdata_p->kcd_addr_end;
1604 #endif
1605
1606 /* add task snapshot marker */
1607 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1608 STACKSHOT_KCCONTAINER_TASK, task_uniqueid));
1609
1610 if (!collect_delta_stackshot || (task_start_abstime == 0) ||
1611 (task_start_abstime > stack_snapshot_delta_since_timestamp)) {
1612 kcd_exit_on_error(kcdata_record_task_snapshot(stackshot_kcdata_p, task, trace_flags, have_pmap, &task_snap_ss_flags));
1613 } else {
1614 task_delta_stackshot = TRUE;
1615 if (minimize_nonrunnables) {
1616 // delay taking the task snapshot. If there are no runnable threads we'll skip it.
1617 } else {
1618 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
1619 }
1620 }
1621
1622 /* Iterate over task threads */
1623 queue_iterate(&task->threads, thread, thread_t, task_threads)
1624 {
1625 uint64_t thread_uniqueid;
1626
1627 if ((thread == NULL) || !ml_validate_nofault((vm_offset_t)thread, sizeof(struct thread))) {
1628 error = KERN_FAILURE;
1629 goto error_exit;
1630 }
1631
1632 if (active_kthreads_only_p && thread->kernel_stack == 0)
1633 continue;
1634
1635 thread_uniqueid = thread_tid(thread);
1636
1637 boolean_t thread_on_core;
1638 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
1639
1640 switch (thread_classification) {
1641 case tc_full_snapshot:
1642 /* add thread marker */
1643 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1644 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1645 kcd_exit_on_error(
1646 kcdata_record_thread_snapshot(stackshot_kcdata_p, thread, task, trace_flags, have_pmap, thread_on_core));
1647
1648 /* mark end of thread snapshot data */
1649 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END,
1650 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1651
1652 some_thread_ran = TRUE;
1653 break;
1654
1655 case tc_delta_snapshot:
1656 num_delta_thread_snapshots++;
1657 break;
1658
1659 case tc_nonrunnable:
1660 num_nonrunnable_threads++;
1661 break;
1662 }
1663 }
1664
1665 if (task_delta_stackshot && minimize_nonrunnables) {
1666 if (some_thread_ran || num_delta_thread_snapshots > 0) {
1667 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
1668 } else {
1669 kcd_exit_on_error(kcdata_undo_add_container_begin(stackshot_kcdata_p));
1670
1671 #if DEBUG || DEVELOPMENT
1672 mach_vm_address_t undo_addr = stackshot_kcdata_p->kcd_addr_end;
1673 if (revert_addr != undo_addr) {
1674 panic("tried to revert a container begin but we already moved past it. revert=%p undo=%p",
1675 (void *)revert_addr, (void *)undo_addr);
1676 }
1677 #endif
1678 kcd_exit_on_error(handle_nonrunnable_task(&saved_uniqueids, task_uniqueid));
1679 continue;
1680 }
1681 }
1682
1683 struct thread_delta_snapshot_v2 * delta_snapshots = NULL;
1684 int current_delta_snapshot_index = 0;
1685
1686 if (num_delta_thread_snapshots > 0) {
1687 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT,
1688 sizeof(struct thread_delta_snapshot_v2),
1689 num_delta_thread_snapshots, &out_addr));
1690 delta_snapshots = (struct thread_delta_snapshot_v2 *)out_addr;
1691 }
1692
1693 uint64_t * nonrunnable_tids = NULL;
1694 int current_nonrunnable_index = 0;
1695
1696 if (num_nonrunnable_threads > 0) {
1697 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TIDS,
1698 sizeof(uint64_t), num_nonrunnable_threads, &out_addr));
1699 nonrunnable_tids = (uint64_t *)out_addr;
1700 }
1701
1702 if (num_delta_thread_snapshots > 0 || num_nonrunnable_threads > 0) {
1703 queue_iterate(&task->threads, thread, thread_t, task_threads)
1704 {
1705 if (active_kthreads_only_p && thread->kernel_stack == 0)
1706 continue;
1707
1708 boolean_t thread_on_core;
1709 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
1710
1711 switch (thread_classification) {
1712 case tc_full_snapshot:
1713 /* full thread snapshot captured above */
1714 continue;
1715
1716 case tc_delta_snapshot:
1717 kcd_exit_on_error(kcdata_record_thread_delta_snapshot(&delta_snapshots[current_delta_snapshot_index++],
1718 thread, thread_on_core));
1719 break;
1720
1721 case tc_nonrunnable:
1722 nonrunnable_tids[current_nonrunnable_index++] = thread_tid(thread);
1723 continue;
1724 }
1725 }
1726
1727 #if DEBUG || DEVELOPMENT
1728 if (current_delta_snapshot_index != num_delta_thread_snapshots) {
1729 panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1730 num_delta_thread_snapshots, current_delta_snapshot_index);
1731 }
1732 if (current_nonrunnable_index != num_nonrunnable_threads) {
1733 panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1734 num_nonrunnable_threads, current_nonrunnable_index);
1735 }
1736 #endif
1737 }
1738
1739 #if IMPORTANCE_INHERITANCE
1740 if (save_donating_pids_p) {
1741 kcd_exit_on_error(
1742 ((((mach_vm_address_t)kcd_end_address(stackshot_kcdata_p) + (TASK_IMP_WALK_LIMIT * sizeof(int32_t))) <
1743 (mach_vm_address_t)kcd_max_address(stackshot_kcdata_p))
1744 ? KERN_SUCCESS
1745 : KERN_RESOURCE_SHORTAGE));
1746 saved_count = task_importance_list_pids(task, TASK_IMP_LIST_DONATING_PIDS,
1747 (void *)kcd_end_address(stackshot_kcdata_p), TASK_IMP_WALK_LIMIT);
1748 if (saved_count > 0)
1749 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_DONATING_PIDS,
1750 sizeof(int32_t), saved_count, &out_addr));
1751 }
1752 #endif
1753
1754 if (!collect_delta_stackshot || (num_delta_thread_snapshots != task->thread_count) || !task_delta_stackshot) {
1755 /*
1756 * Collect shared cache info and UUID info in these scenarios
1757 * 1) a full stackshot
1758 * 2) a delta stackshot where the task started after the previous full stackshot OR
1759 * any thread from the task has run since the previous full stackshot
1760 */
1761
1762 kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, sys_shared_cache_loadinfo, trace_flags, task_snap_ss_flags));
1763 kcd_exit_on_error(kcdata_record_uuid_info(stackshot_kcdata_p, task, trace_flags, have_pmap, task_snap_ss_flags));
1764 }
1765 /* mark end of task snapshot data */
1766 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK,
1767 task_uniqueid));
1768 }
1769 }
1770
1771 if (minimize_nonrunnables) {
1772 flush_nonrunnable_tasks(&saved_uniqueids);
1773 }
1774
1775 if (use_fault_path) {
1776 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS,
1777 sizeof(struct stackshot_fault_stats), &out_addr));
1778 stackshot_memcpy((void*)out_addr, &fault_stats, sizeof(struct stackshot_fault_stats));
1779 }
1780
1781 /* update timestamp of the stackshot */
1782 abs_time_end = mach_absolute_time();
1783 #if DEVELOPMENT || DEBUG
1784 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_DURATION,
1785 sizeof(struct stackshot_duration), &out_addr));
1786 struct stackshot_duration * stackshot_duration = (struct stackshot_duration *)out_addr;
1787 stackshot_duration->stackshot_duration = (abs_time_end - abs_time);
1788 stackshot_duration->stackshot_duration_outer = 0;
1789 stackshot_duration_outer = &stackshot_duration->stackshot_duration_outer;
1790 #endif
1791 stackshot_memcpy((void *)abs_time_addr, &abs_time_end, sizeof(uint64_t));
1792
1793
1794 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_kcdata_p));
1795
1796 /* === END of populating stackshot data === */
1797
1798 *pBytesTraced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_kcdata_p);
1799 error_exit:
1800
1801 stack_enable_faulting = FALSE;
1802
1803 return error;
1804 }
1805
1806 static int pid_from_task(task_t task)
1807 {
1808 int pid = -1;
1809
1810 if (task->bsd_info) {
1811 pid = proc_pid(task->bsd_info);
1812 } else {
1813 pid = task_pid(task);
1814 }
1815
1816 return pid;
1817 }
1818
1819 static uint64_t
1820 proc_was_throttled_from_task(task_t task)
1821 {
1822 uint64_t was_throttled = 0;
1823
1824 if (task->bsd_info)
1825 was_throttled = proc_was_throttled(task->bsd_info);
1826
1827 return was_throttled;
1828 }
1829
1830 static uint64_t
1831 proc_did_throttle_from_task(task_t task)
1832 {
1833 uint64_t did_throttle = 0;
1834
1835 if (task->bsd_info)
1836 did_throttle = proc_did_throttle(task->bsd_info);
1837
1838 return did_throttle;
1839 }
1840
1841 static void
1842 kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap)
1843 {
1844 unsigned int pages_reclaimed;
1845 unsigned int pages_wanted;
1846 kern_return_t kErr;
1847
1848 processor_t processor;
1849 vm_statistics64_t stat;
1850 vm_statistics64_data_t host_vm_stat;
1851
1852 processor = processor_list;
1853 stat = &PROCESSOR_DATA(processor, vm_stat);
1854 host_vm_stat = *stat;
1855
1856 if (processor_count > 1) {
1857 /*
1858 * processor_list may be in the process of changing as we are
1859 * attempting a stackshot. Ordinarily it will be lock protected,
1860 * but it is not safe to lock in the context of the debugger.
1861 * Fortunately we never remove elements from the processor list,
1862 * and only add to to the end of the list, so we SHOULD be able
1863 * to walk it. If we ever want to truly tear down processors,
1864 * this will have to change.
1865 */
1866 while ((processor = processor->processor_list) != NULL) {
1867 stat = &PROCESSOR_DATA(processor, vm_stat);
1868 host_vm_stat.compressions += stat->compressions;
1869 host_vm_stat.decompressions += stat->decompressions;
1870 }
1871 }
1872
1873 memio_snap->snapshot_magic = STACKSHOT_MEM_AND_IO_SNAPSHOT_MAGIC;
1874 memio_snap->free_pages = vm_page_free_count;
1875 memio_snap->active_pages = vm_page_active_count;
1876 memio_snap->inactive_pages = vm_page_inactive_count;
1877 memio_snap->purgeable_pages = vm_page_purgeable_count;
1878 memio_snap->wired_pages = vm_page_wire_count;
1879 memio_snap->speculative_pages = vm_page_speculative_count;
1880 memio_snap->throttled_pages = vm_page_throttled_count;
1881 memio_snap->busy_buffer_count = count_busy_buffers();
1882 memio_snap->filebacked_pages = vm_page_pageable_external_count;
1883 memio_snap->compressions = (uint32_t)host_vm_stat.compressions;
1884 memio_snap->decompressions = (uint32_t)host_vm_stat.decompressions;
1885 memio_snap->compressor_size = VM_PAGE_COMPRESSOR_COUNT;
1886 kErr = mach_vm_pressure_monitor(FALSE, VM_PRESSURE_TIME_WINDOW, &pages_reclaimed, &pages_wanted);
1887
1888 if ( ! kErr ) {
1889 memio_snap->pages_wanted = (uint32_t)pages_wanted;
1890 memio_snap->pages_reclaimed = (uint32_t)pages_reclaimed;
1891 memio_snap->pages_wanted_reclaimed_valid = 1;
1892 } else {
1893 memio_snap->pages_wanted = 0;
1894 memio_snap->pages_reclaimed = 0;
1895 memio_snap->pages_wanted_reclaimed_valid = 0;
1896 }
1897 }
1898
1899 void
1900 stackshot_memcpy(void *dst, const void *src, size_t len)
1901 {
1902 memcpy(dst, src, len);
1903 }
1904
1905 size_t
1906 stackshot_strlcpy(char *dst, const char *src, size_t maxlen)
1907 {
1908 const size_t srclen = strlen(src);
1909
1910 if (srclen < maxlen) {
1911 stackshot_memcpy(dst, src, srclen+1);
1912 } else if (maxlen != 0) {
1913 stackshot_memcpy(dst, src, maxlen-1);
1914 dst[maxlen-1] = '\0';
1915 }
1916
1917 return srclen;
1918 }
1919
1920
1921 /*
1922 * Returns the physical address of the specified map:target address,
1923 * using the kdp fault path if requested and the page is not resident.
1924 */
1925 vm_offset_t
1926 kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results)
1927 {
1928 vm_offset_t cur_phys_addr;
1929 unsigned cur_wimg_bits;
1930 uint64_t fault_start_time = 0;
1931
1932 if (map == VM_MAP_NULL) {
1933 return 0;
1934 }
1935
1936 cur_phys_addr = kdp_vtophys(map->pmap, target_addr);
1937 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
1938 if (!try_fault || fault_stats.sfs_stopped_faulting) {
1939 if (kdp_fault_results)
1940 *kdp_fault_results |= KDP_FAULT_RESULT_PAGED_OUT;
1941
1942 return 0;
1943 }
1944
1945 /*
1946 * The pmap doesn't have a valid page so we start at the top level
1947 * vm map and try a lightweight fault. Update fault path usage stats.
1948 */
1949 fault_start_time = mach_absolute_time();
1950 cur_phys_addr = kdp_lightweight_fault(map, (target_addr & ~PAGE_MASK));
1951 fault_stats.sfs_time_spent_faulting += (mach_absolute_time() - fault_start_time);
1952
1953 if ((fault_stats.sfs_time_spent_faulting >= fault_stats.sfs_system_max_fault_time) && !panic_stackshot) {
1954 fault_stats.sfs_stopped_faulting = (uint8_t) TRUE;
1955 }
1956
1957 cur_phys_addr += (target_addr & PAGE_MASK);
1958
1959 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
1960 if (kdp_fault_results)
1961 *kdp_fault_results |= (KDP_FAULT_RESULT_TRIED_FAULT | KDP_FAULT_RESULT_PAGED_OUT);
1962
1963 return 0;
1964 }
1965
1966 if (kdp_fault_results)
1967 *kdp_fault_results |= KDP_FAULT_RESULT_FAULTED_IN;
1968
1969 fault_stats.sfs_pages_faulted_in++;
1970 } else {
1971 /*
1972 * This check is done in kdp_lightweight_fault for the fault path.
1973 */
1974 cur_wimg_bits = pmap_cache_attributes((ppnum_t) atop(cur_phys_addr));
1975
1976 if ((cur_wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
1977 return 0;
1978 }
1979 }
1980
1981 return cur_phys_addr;
1982 }
1983
1984 boolean_t
1985 kdp_copyin_word(
1986 task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results)
1987 {
1988 if (task_has_64BitAddr(task)) {
1989 return kdp_copyin(task->map, addr, result, sizeof(uint64_t), try_fault, kdp_fault_results);
1990 } else {
1991 uint32_t buf;
1992 boolean_t r = kdp_copyin(task->map, addr, &buf, sizeof(uint32_t), try_fault, kdp_fault_results);
1993 *result = buf;
1994 return r;
1995 }
1996 }
1997
1998 boolean_t
1999 kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_results)
2000 {
2001 size_t rem = size;
2002 char *kvaddr = dest;
2003
2004 #if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
2005 /* Identify if destination buffer is in panic storage area */
2006 if ((vm_offset_t)dest >= gPanicBase && (vm_offset_t)dest < gPanicBase + gPanicSize) {
2007 if (((vm_offset_t)dest + size) >= (gPanicBase + gPanicSize)) {
2008 return FALSE;
2009 }
2010 }
2011 #endif
2012
2013 while (rem) {
2014 uint64_t phys_src = kdp_find_phys(map, uaddr, try_fault, kdp_fault_results);
2015 uint64_t phys_dest = kvtophys((vm_offset_t)kvaddr);
2016 uint64_t src_rem = PAGE_SIZE - (phys_src & PAGE_MASK);
2017 uint64_t dst_rem = PAGE_SIZE - (phys_dest & PAGE_MASK);
2018 size_t cur_size = (uint32_t) MIN(src_rem, dst_rem);
2019 cur_size = MIN(cur_size, rem);
2020
2021 if (phys_src && phys_dest) {
2022 bcopy_phys(phys_src, phys_dest, cur_size);
2023 } else {
2024 break;
2025 }
2026
2027 uaddr += cur_size;
2028 kvaddr += cur_size;
2029 rem -= cur_size;
2030 }
2031
2032 return (rem == 0);
2033 }
2034
2035 kern_return_t
2036 do_stackshot(void *context)
2037 {
2038 #pragma unused(context)
2039 kdp_snapshot++;
2040
2041 stack_snapshot_ret = kdp_stackshot_kcdata_format(stack_snapshot_pid,
2042 stack_snapshot_flags,
2043 &stack_snapshot_bytes_traced);
2044
2045 kdp_snapshot--;
2046 return stack_snapshot_ret;
2047 }
2048
2049 /*
2050 * A fantastical routine that tries to be fast about returning
2051 * translations. Caches the last page we found a translation
2052 * for, so that we can be quick about multiple queries to the
2053 * same page. It turns out this is exactly the workflow
2054 * machine_trace_thread and its relatives tend to throw at us.
2055 *
2056 * Please zero the nasty global this uses after a bulk lookup;
2057 * this isn't safe across a switch of the map or changes
2058 * to a pmap.
2059 *
2060 * This also means that if zero is a valid KVA, we are
2061 * screwed. Sucks to be us. Fortunately, this should never
2062 * happen.
2063 */
2064 vm_offset_t
2065 machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags)
2066 {
2067 vm_offset_t cur_target_page;
2068 vm_offset_t cur_phys_addr;
2069 vm_offset_t kern_virt_target_addr;
2070 uint32_t kdp_fault_results = 0;
2071
2072 cur_target_page = atop(cur_target_addr);
2073
2074 if ((cur_target_page != prev_target_page) || validate_next_addr) {
2075
2076 /*
2077 * Alright; it wasn't our previous page. So
2078 * we must validate that there is a page
2079 * table entry for this address under the
2080 * current pmap, and that it has default
2081 * cache attributes (otherwise it may not be
2082 * safe to access it).
2083 */
2084 cur_phys_addr = kdp_find_phys(map, cur_target_addr, stack_enable_faulting, &kdp_fault_results);
2085 if (thread_trace_flags) {
2086 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
2087 *thread_trace_flags |= kThreadTruncatedBT;
2088 }
2089
2090 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
2091 *thread_trace_flags |= kThreadTriedFaultBT;
2092 }
2093
2094 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
2095 *thread_trace_flags |= kThreadFaultedBT;
2096 }
2097 }
2098
2099 if (cur_phys_addr == 0) {
2100 return 0;
2101 }
2102 #if __x86_64__
2103 kern_virt_target_addr = (vm_offset_t) PHYSMAP_PTOV(cur_phys_addr);
2104 #else
2105 #error Oh come on... we should really unify the physical -> kernel virtual interface
2106 #endif
2107 prev_target_page = cur_target_page;
2108 prev_target_kva = (kern_virt_target_addr & ~PAGE_MASK);
2109 validate_next_addr = FALSE;
2110 return kern_virt_target_addr;
2111 } else {
2112 /* We found a translation, so stash this page */
2113 kern_virt_target_addr = prev_target_kva + (cur_target_addr & PAGE_MASK);
2114 return kern_virt_target_addr;
2115 }
2116 }
2117
2118 void
2119 machine_trace_thread_clear_validation_cache(void)
2120 {
2121 validate_next_addr = TRUE;
2122 }
2123
2124 boolean_t
2125 stackshot_thread_is_idle_worker_unsafe(thread_t thread)
2126 {
2127 /* When the pthread kext puts a worker thread to sleep, it will call
2128 * assert_wait on the thread's own threadlist. see parkit() in
2129 * kern_support.c.
2130 */
2131 struct uthread * uthread = get_bsdthread_info(thread);
2132 event64_t threadlist = (event64_t)proc_get_uthread_uu_threadlist(uthread);
2133 event64_t wait_event = thread->wait_event;
2134 return uthread &&
2135 (thread->state & TH_WAIT) &&
2136 wait_event &&
2137 threadlist == wait_event;
2138 }
mirror of XNU