]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/kern_stackshot.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.41.3.tar.gz
[apple/xnu.git] / osfmk / kern / kern_stackshot.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/mach_types.h>
30 #include <mach/vm_param.h>
31 #include <mach/mach_vm.h>
32 #include <mach/clock_types.h>
33 #include <sys/errno.h>
34 #include <sys/stackshot.h>
35 #ifdef IMPORTANCE_INHERITANCE
36 #include <ipc/ipc_importance.h>
37 #endif
38 #include <sys/appleapiopts.h>
39 #include <kern/debug.h>
40 #include <uuid/uuid.h>
41
42 #include <kdp/kdp_dyld.h>
43 #include <kdp/kdp_en_debugger.h>
44
45 #include <libsa/types.h>
46 #include <libkern/version.h>
47
48 #include <string.h> /* bcopy */
49
50 #include <kern/processor.h>
51 #include <kern/thread.h>
52 #include <kern/task.h>
53 #include <kern/telemetry.h>
54 #include <kern/clock.h>
55 #include <kern/policy_internal.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_kern.h>
58 #include <vm/vm_pageout.h>
59 #include <vm/vm_fault.h>
60 #include <vm/vm_shared_region.h>
61 #include <libkern/OSKextLibPrivate.h>
62
63 #if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
64 #include <pexpert/pexpert.h> /* For gPanicBase/gPanicBase */
65 #endif
66
67 extern unsigned int not_in_kdp;
68
69
70 extern addr64_t kdp_vtophys(pmap_t pmap, addr64_t va);
71 extern void * proc_get_uthread_uu_threadlist(void * uthread_v);
72
73 int kdp_snapshot = 0;
74 static kern_return_t stack_snapshot_ret = 0;
75 static uint32_t stack_snapshot_bytes_traced = 0;
76
77 static kcdata_descriptor_t stackshot_kcdata_p = NULL;
78 static void *stack_snapshot_buf;
79 static uint32_t stack_snapshot_bufsize;
80 int stack_snapshot_pid;
81 static uint32_t stack_snapshot_flags;
82 static uint64_t stack_snapshot_delta_since_timestamp;
83 static boolean_t panic_stackshot;
84
85 static boolean_t stack_enable_faulting = FALSE;
86 static struct stackshot_fault_stats fault_stats;
87
88 static uint64_t * stackshot_duration_outer;
89 static uint64_t stackshot_microsecs;
90
91 void * kernel_stackshot_buf = NULL; /* Pointer to buffer for stackshots triggered from the kernel and retrieved later */
92 int kernel_stackshot_buf_size = 0;
93
94 void * stackshot_snapbuf = NULL; /* Used by stack_snapshot2 (to be removed) */
95
96 __private_extern__ void stackshot_init( void );
97 static boolean_t memory_iszero(void *addr, size_t size);
98 #if CONFIG_TELEMETRY
99 kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
100 #endif
101 uint32_t get_stackshot_estsize(uint32_t prev_size_hint);
102 kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config,
103 size_t stackshot_config_size, boolean_t stackshot_from_user);
104 kern_return_t do_stackshot(void *);
105 void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp);
106 boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
107 static int kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t *pBytesTraced);
108 kern_return_t kdp_stack_snapshot_geterror(void);
109 uint32_t kdp_stack_snapshot_bytes_traced(void);
110 static int pid_from_task(task_t task);
111 static void kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap);
112 static boolean_t kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_result);
113 static boolean_t kdp_copyin_word(task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results);
114 static uint64_t proc_was_throttled_from_task(task_t task);
115
116 extern uint32_t workqueue_get_pwq_state_kdp(void *proc);
117
118 extern int proc_pid(void *p);
119 extern uint64_t proc_uniqueid(void *p);
120 extern uint64_t proc_was_throttled(void *p);
121 extern uint64_t proc_did_throttle(void *p);
122 static uint64_t proc_did_throttle_from_task(task_t task);
123 extern void proc_name_kdp(task_t task, char * buf, int size);
124 extern int proc_threadname_kdp(void * uth, char * buf, size_t size);
125 extern void proc_starttime_kdp(void * p, uint64_t * tv_sec, uint64_t * tv_usec, uint64_t * abstime);
126 extern int memorystatus_get_pressure_status_kdp(void);
127 extern boolean_t memorystatus_proc_is_dirty_unsafe(void * v);
128
129 extern int count_busy_buffers(void); /* must track with declaration in bsd/sys/buf_internal.h */
130 extern void bcopy_phys(addr64_t, addr64_t, vm_size_t);
131
132 #if CONFIG_TELEMETRY
133 extern kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
134 #endif /* CONFIG_TELEMETRY */
135
136 extern kern_return_t kern_stack_snapshot_with_reason(char* reason);
137 extern kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user);
138
139 /*
140 * Validates that the given address is both a valid page and has
141 * default caching attributes for the current map. Returns
142 * 0 if the address is invalid, and a kernel virtual address for
143 * the given address if it is valid.
144 */
145 vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags);
146
147 #define KDP_FAULT_RESULT_PAGED_OUT 0x1 /* some data was unable to be retrieved */
148 #define KDP_FAULT_RESULT_TRIED_FAULT 0x2 /* tried to fault in data */
149 #define KDP_FAULT_RESULT_FAULTED_IN 0x4 /* successfully faulted in data */
150
151 /*
152 * Looks up the physical translation for the given address in the target map, attempting
153 * to fault data in if requested and it is not resident. Populates thread_trace_flags if requested
154 * as well.
155 */
156 vm_offset_t kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results);
157
158 static size_t stackshot_strlcpy(char *dst, const char *src, size_t maxlen);
159 static void stackshot_memcpy(void *dst, const void *src, size_t len);
160
161 /* Clears caching information used by the above validation routine
162 * (in case the current map has been changed or cleared).
163 */
164 void machine_trace_thread_clear_validation_cache(void);
165
166 #define MAX_FRAMES 1000
167 #define MAX_LOADINFOS 500
168 #define TASK_IMP_WALK_LIMIT 20
169
170 typedef struct thread_snapshot *thread_snapshot_t;
171 typedef struct task_snapshot *task_snapshot_t;
172
173 #if CONFIG_KDP_INTERACTIVE_DEBUGGING
174 extern kdp_send_t kdp_en_send_pkt;
175 #endif
176
177 /*
178 * Globals to support machine_trace_thread_get_kva.
179 */
180 static vm_offset_t prev_target_page = 0;
181 static vm_offset_t prev_target_kva = 0;
182 static boolean_t validate_next_addr = TRUE;
183
184 /*
185 * Stackshot locking and other defines.
186 */
187 static lck_grp_t *stackshot_subsys_lck_grp;
188 static lck_grp_attr_t *stackshot_subsys_lck_grp_attr;
189 static lck_attr_t *stackshot_subsys_lck_attr;
190 static lck_mtx_t stackshot_subsys_mutex;
191
192 #define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
193 #define STACKSHOT_SUBSYS_TRY_LOCK() lck_mtx_try_lock(&stackshot_subsys_mutex)
194 #define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
195
196 #define SANE_BOOTPROFILE_TRACEBUF_SIZE (64 * 1024 * 1024)
197 #define SANE_TRACEBUF_SIZE (8 * 1024 * 1024)
198
199 /*
200 * We currently set a ceiling of 3 milliseconds spent in the kdp fault path
201 * for non-panic stackshots where faulting is requested.
202 */
203 #define KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS (3 * NSEC_PER_MSEC)
204
205 #define STACKSHOT_SUPP_SIZE (16 * 1024) /* Minimum stackshot size */
206 #define TASK_UUID_AVG_SIZE (16 * sizeof(uuid_t)) /* Average space consumed by UUIDs/task */
207
208 /*
209 * Initialize the mutex governing access to the stack snapshot subsystem
210 * and other stackshot related bits.
211 */
212 __private_extern__ void
213 stackshot_init( void )
214 {
215 mach_timebase_info_data_t timebase;
216
217 stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
218
219 stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
220
221 stackshot_subsys_lck_attr = lck_attr_alloc_init();
222
223 lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
224
225 clock_timebase_info(&timebase);
226 fault_stats.sfs_system_max_fault_time = ((KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS * timebase.denom)/ timebase.numer);
227 }
228
229 /*
230 * Method for grabbing timer values safely, in the sense that no infinite loop will occur
231 * Certain flavors of the timer_grab function, which would seem to be the thing to use,
232 * can loop infinitely if called while the timer is in the process of being updated.
233 * Unfortunately, it is (rarely) possible to get inconsistent top and bottom halves of
234 * the timer using this method. This seems insoluble, since stackshot runs in a context
235 * where the timer might be half-updated, and has no way of yielding control just long
236 * enough to finish the update.
237 */
238
239 static uint64_t safe_grab_timer_value(struct timer *t)
240 {
241 #if defined(__LP64__)
242 return t->all_bits;
243 #else
244 uint64_t time = t->high_bits; /* endian independent grab */
245 time = (time << 32) | t->low_bits;
246 return time;
247 #endif
248 }
249
250 kern_return_t
251 stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, uint64_t delta_since_timestamp, unsigned *bytes_traced)
252 {
253 kern_return_t error = KERN_SUCCESS;
254 boolean_t istate;
255
256 if ((buf == NULL) || (size <= 0) || (bytes_traced == NULL)) {
257 return KERN_INVALID_ARGUMENT;
258 }
259
260 /* cap in individual stackshot to SANE_TRACEBUF_SIZE */
261 if (size > SANE_TRACEBUF_SIZE) {
262 size = SANE_TRACEBUF_SIZE;
263 }
264
265 /* Serialize tracing */
266 if (flags & STACKSHOT_TRYLOCK) {
267 if (!STACKSHOT_SUBSYS_TRY_LOCK()) {
268 return KERN_LOCK_OWNED;
269 }
270 } else {
271 STACKSHOT_SUBSYS_LOCK();
272 }
273
274 istate = ml_set_interrupts_enabled(FALSE);
275
276 struct kcdata_descriptor kcdata;
277 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ?
278 KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
279
280 error = kcdata_memory_static_init(&kcdata, (mach_vm_address_t)buf, hdr_tag, size,
281 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
282 if (error) {
283 goto out;
284 }
285
286 /* Preload trace parameters*/
287 kdp_snapshot_preflight(pid, buf, size, flags, &kcdata, delta_since_timestamp);
288
289 /* Trap to the debugger to obtain a coherent stack snapshot; this populates
290 * the trace buffer
291 */
292 stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
293
294 ml_set_interrupts_enabled(istate);
295
296 *bytes_traced = kdp_stack_snapshot_bytes_traced();
297
298 error = kdp_stack_snapshot_geterror();
299
300 out:
301 STACKSHOT_SUBSYS_UNLOCK();
302 return error;
303 }
304
305 #if CONFIG_TELEMETRY
306 kern_return_t
307 stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval)
308 {
309 int error = KERN_SUCCESS;
310 uint32_t bytes_traced = 0;
311
312 *retval = -1;
313
314 /*
315 * Control related operations
316 */
317 if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_ENABLE) {
318 telemetry_global_ctl(1);
319 *retval = 0;
320 goto exit;
321 } else if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_DISABLE) {
322 telemetry_global_ctl(0);
323 *retval = 0;
324 goto exit;
325 }
326
327 /*
328 * Data related operations
329 */
330 *retval = -1;
331
332 if ((((void*)tracebuf) == NULL) || (tracebuf_size == 0)) {
333 error = KERN_INVALID_ARGUMENT;
334 goto exit;
335 }
336
337 STACKSHOT_SUBSYS_LOCK();
338
339 if (flags & STACKSHOT_GET_MICROSTACKSHOT) {
340 if (tracebuf_size > SANE_TRACEBUF_SIZE) {
341 error = KERN_INVALID_ARGUMENT;
342 goto unlock_exit;
343 }
344
345 bytes_traced = tracebuf_size;
346 error = telemetry_gather(tracebuf, &bytes_traced,
347 (flags & STACKSHOT_SET_MICROSTACKSHOT_MARK) ? TRUE : FALSE);
348 *retval = (int)bytes_traced;
349 goto unlock_exit;
350 }
351
352 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
353
354 if (tracebuf_size > SANE_BOOTPROFILE_TRACEBUF_SIZE) {
355 error = KERN_INVALID_ARGUMENT;
356 goto unlock_exit;
357 }
358
359 bytes_traced = tracebuf_size;
360 error = bootprofile_gather(tracebuf, &bytes_traced);
361 *retval = (int)bytes_traced;
362 }
363
364 unlock_exit:
365 STACKSHOT_SUBSYS_UNLOCK();
366 exit:
367 return error;
368 }
369 #endif /* CONFIG_TELEMETRY */
370
371 /*
372 * Return the estimated size of a stackshot based on the
373 * number of currently running threads and tasks.
374 */
375 uint32_t
376 get_stackshot_estsize(uint32_t prev_size_hint)
377 {
378 vm_size_t thread_total;
379 vm_size_t task_total;
380 uint32_t estimated_size;
381
382 thread_total = (threads_count * sizeof(struct thread_snapshot));
383 task_total = (tasks_count * (sizeof(struct task_snapshot) + TASK_UUID_AVG_SIZE));
384
385 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE((thread_total + task_total + STACKSHOT_SUPP_SIZE), PAGE_MASK);
386 if (estimated_size < prev_size_hint) {
387 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE(prev_size_hint, PAGE_MASK);
388 }
389
390 return estimated_size;
391 }
392
393 /*
394 * stackshot_remap_buffer: Utility function to remap bytes_traced bytes starting at stackshotbuf
395 * into the current task's user space and subsequently copy out the address
396 * at which the buffer has been mapped in user space to out_buffer_addr.
397 *
398 * Inputs: stackshotbuf - pointer to the original buffer in the kernel's address space
399 * bytes_traced - length of the buffer to remap starting from stackshotbuf
400 * out_buffer_addr - pointer to placeholder where newly mapped buffer will be mapped.
401 * out_size_addr - pointer to be filled in with the size of the buffer
402 *
403 * Outputs: ENOSPC if there is not enough free space in the task's address space to remap the buffer
404 * EINVAL for all other errors returned by task_remap_buffer/mach_vm_remap
405 * an error from copyout
406 */
407 static kern_return_t
408 stackshot_remap_buffer(void *stackshotbuf, uint32_t bytes_traced, uint64_t out_buffer_addr, uint64_t out_size_addr)
409 {
410 int error = 0;
411 mach_vm_offset_t stackshotbuf_user_addr = (mach_vm_offset_t)NULL;
412 vm_prot_t cur_prot, max_prot;
413
414 error = mach_vm_remap(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0,
415 VM_FLAGS_ANYWHERE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT);
416 /*
417 * If the call to mach_vm_remap fails, we return the appropriate converted error
418 */
419 if (error == KERN_SUCCESS) {
420 /*
421 * If we fail to copy out the address or size of the new buffer, we remove the buffer mapping that
422 * we just made in the task's user space.
423 */
424 error = copyout(CAST_DOWN(void *, &stackshotbuf_user_addr), (user_addr_t)out_buffer_addr, sizeof(stackshotbuf_user_addr));
425 if (error != KERN_SUCCESS) {
426 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
427 return error;
428 }
429 error = copyout(&bytes_traced, (user_addr_t)out_size_addr, sizeof(bytes_traced));
430 if (error != KERN_SUCCESS) {
431 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
432 return error;
433 }
434 }
435 return error;
436 }
437
438 kern_return_t
439 kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user)
440 {
441 int error = 0;
442 boolean_t prev_interrupt_state;
443 uint32_t bytes_traced = 0;
444 uint32_t stackshotbuf_size = 0;
445 void * stackshotbuf = NULL;
446 kcdata_descriptor_t kcdata_p = NULL;
447
448 void * buf_to_free = NULL;
449 int size_to_free = 0;
450
451 /* Parsed arguments */
452 uint64_t out_buffer_addr;
453 uint64_t out_size_addr;
454 int pid = -1;
455 uint32_t flags;
456 uint64_t since_timestamp;
457 uint32_t size_hint = 0;
458
459 if(stackshot_config == NULL) {
460 return KERN_INVALID_ARGUMENT;
461 }
462
463 switch (stackshot_config_version) {
464 case STACKSHOT_CONFIG_TYPE:
465 if (stackshot_config_size != sizeof(stackshot_config_t)) {
466 return KERN_INVALID_ARGUMENT;
467 }
468 stackshot_config_t *config = (stackshot_config_t *) stackshot_config;
469 out_buffer_addr = config->sc_out_buffer_addr;
470 out_size_addr = config->sc_out_size_addr;
471 pid = config->sc_pid;
472 flags = config->sc_flags;
473 since_timestamp = config->sc_delta_timestamp;
474 if (config->sc_size <= SANE_TRACEBUF_SIZE) {
475 size_hint = config->sc_size;
476 }
477 break;
478 default:
479 return KERN_NOT_SUPPORTED;
480 }
481
482 /*
483 * Currently saving a kernel buffer and trylock are only supported from the
484 * internal/KEXT API.
485 */
486 if (stackshot_from_user) {
487 if (flags & (STACKSHOT_TRYLOCK | STACKSHOT_SAVE_IN_KERNEL_BUFFER | STACKSHOT_FROM_PANIC)) {
488 return KERN_NO_ACCESS;
489 }
490 } else {
491 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
492 return KERN_NOT_SUPPORTED;
493 }
494 }
495
496 if (!((flags & STACKSHOT_KCDATA_FORMAT) || (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER))) {
497 return KERN_NOT_SUPPORTED;
498 }
499
500 /*
501 * If we're not saving the buffer in the kernel pointer, we need a place to copy into.
502 */
503 if ((!out_buffer_addr || !out_size_addr) && !(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
504 return KERN_INVALID_ARGUMENT;
505 }
506
507 if (since_timestamp != 0 && ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) == 0)) {
508 return KERN_INVALID_ARGUMENT;
509 }
510
511 STACKSHOT_SUBSYS_LOCK();
512
513 if (flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER) {
514 /*
515 * Don't overwrite an existing stackshot
516 */
517 if (kernel_stackshot_buf != NULL) {
518 error = KERN_MEMORY_PRESENT;
519 goto error_exit;
520 }
521 } else if (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER) {
522 if ((kernel_stackshot_buf == NULL) || (kernel_stackshot_buf_size <= 0)) {
523 error = KERN_NOT_IN_SET;
524 goto error_exit;
525 }
526 error = stackshot_remap_buffer(kernel_stackshot_buf, kernel_stackshot_buf_size,
527 out_buffer_addr, out_size_addr);
528 /*
529 * If we successfully remapped the buffer into the user's address space, we
530 * set buf_to_free and size_to_free so the prior kernel mapping will be removed
531 * and then clear the kernel stackshot pointer and associated size.
532 */
533 if (error == KERN_SUCCESS) {
534 buf_to_free = kernel_stackshot_buf;
535 size_to_free = (int) VM_MAP_ROUND_PAGE(kernel_stackshot_buf_size, PAGE_MASK);
536 kernel_stackshot_buf = NULL;
537 kernel_stackshot_buf_size = 0;
538 }
539
540 goto error_exit;
541 }
542
543 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
544 void *bootprofile = NULL;
545 uint32_t len = 0;
546 #if CONFIG_TELEMETRY
547 bootprofile_get(&bootprofile, &len);
548 #endif
549 if (!bootprofile || !len) {
550 error = KERN_NOT_IN_SET;
551 goto error_exit;
552 }
553 error = stackshot_remap_buffer(bootprofile, len, out_buffer_addr, out_size_addr);
554 goto error_exit;
555 }
556
557 stackshotbuf_size = get_stackshot_estsize(size_hint);
558
559 for (; stackshotbuf_size <= SANE_TRACEBUF_SIZE; stackshotbuf_size <<= 1) {
560 if (kmem_alloc(kernel_map, (vm_offset_t *)&stackshotbuf, stackshotbuf_size, VM_KERN_MEMORY_DIAG) != KERN_SUCCESS) {
561 error = KERN_RESOURCE_SHORTAGE;
562 goto error_exit;
563 }
564
565 /*
566 * If someone has panicked, don't try and enter the debugger
567 */
568 if (panic_active()) {
569 error = KERN_RESOURCE_SHORTAGE;
570 goto error_exit;
571 }
572
573 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
574 kcdata_p = kcdata_memory_alloc_init((mach_vm_address_t)stackshotbuf, hdr_tag, stackshotbuf_size,
575 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
576
577 stackshot_duration_outer = NULL;
578 uint64_t time_start = mach_absolute_time();
579
580 /*
581 * Disable interrupts and save the current interrupt state.
582 */
583 prev_interrupt_state = ml_set_interrupts_enabled(FALSE);
584
585 /*
586 * Load stackshot parameters.
587 */
588 kdp_snapshot_preflight(pid, stackshotbuf, stackshotbuf_size, flags, kcdata_p, since_timestamp);
589
590 /*
591 * Trap to the debugger to obtain a stackshot (this will populate the buffer).
592 */
593 stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE);
594
595 ml_set_interrupts_enabled(prev_interrupt_state);
596
597 /* record the duration that interupts were disabled */
598
599 uint64_t time_end = mach_absolute_time();
600 if (stackshot_duration_outer) {
601 *stackshot_duration_outer = time_end - time_start;
602 }
603
604 error = kdp_stack_snapshot_geterror();
605 if (error != KERN_SUCCESS) {
606 if (kcdata_p != NULL) {
607 kcdata_memory_destroy(kcdata_p);
608 kcdata_p = NULL;
609 stackshot_kcdata_p = NULL;
610 }
611 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
612 stackshotbuf = NULL;
613 if (error == KERN_INSUFFICIENT_BUFFER_SIZE) {
614 /*
615 * If we didn't allocate a big enough buffer, deallocate and try again.
616 */
617 continue;
618 } else {
619 goto error_exit;
620 }
621 }
622
623 bytes_traced = kdp_stack_snapshot_bytes_traced();
624
625 if (bytes_traced <= 0) {
626 error = KERN_ABORTED;
627 goto error_exit;
628 }
629
630 assert(bytes_traced <= stackshotbuf_size);
631 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
632 error = stackshot_remap_buffer(stackshotbuf, bytes_traced, out_buffer_addr, out_size_addr);
633 goto error_exit;
634 }
635
636 /*
637 * Save the stackshot in the kernel buffer.
638 */
639 kernel_stackshot_buf = stackshotbuf;
640 kernel_stackshot_buf_size = bytes_traced;
641 /*
642 * Figure out if we didn't use all the pages in the buffer. If so, we set buf_to_free to the beginning of
643 * the next page after the end of the stackshot in the buffer so that the kmem_free clips the buffer and
644 * update size_to_free for kmem_free accordingly.
645 */
646 size_to_free = stackshotbuf_size - (int) VM_MAP_ROUND_PAGE(bytes_traced, PAGE_MASK);
647
648 assert(size_to_free >= 0);
649
650 if (size_to_free != 0) {
651 buf_to_free = (void *)((uint64_t)stackshotbuf + stackshotbuf_size - size_to_free);
652 }
653
654 stackshotbuf = NULL;
655 stackshotbuf_size = 0;
656 goto error_exit;
657 }
658
659 if (stackshotbuf_size > SANE_TRACEBUF_SIZE) {
660 error = KERN_RESOURCE_SHORTAGE;
661 }
662
663 error_exit:
664 if (kcdata_p != NULL) {
665 kcdata_memory_destroy(kcdata_p);
666 kcdata_p = NULL;
667 stackshot_kcdata_p = NULL;
668 }
669
670 if (stackshotbuf != NULL) {
671 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
672 }
673 if (buf_to_free != NULL) {
674 kmem_free(kernel_map, (vm_offset_t)buf_to_free, size_to_free);
675 }
676 STACKSHOT_SUBSYS_UNLOCK();
677 return error;
678 }
679
680 /* Cache stack snapshot parameters in preparation for a trace */
681 void
682 kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags,
683 kcdata_descriptor_t data_p, uint64_t since_timestamp)
684 {
685 uint64_t microsecs = 0, secs = 0;
686 clock_get_calendar_microtime((clock_sec_t *)&secs, (clock_usec_t *)&microsecs);
687
688 stackshot_microsecs = microsecs + (secs * USEC_PER_SEC);
689 stack_snapshot_pid = pid;
690 stack_snapshot_buf = tracebuf;
691 stack_snapshot_bufsize = tracebuf_size;
692 stack_snapshot_flags = flags;
693 stack_snapshot_delta_since_timestamp = since_timestamp;
694
695 panic_stackshot = ((flags & STACKSHOT_FROM_PANIC) != 0);
696
697 if (data_p != NULL) {
698 stackshot_kcdata_p = data_p;
699 }
700 }
701
702 kern_return_t
703 kdp_stack_snapshot_geterror(void)
704 {
705 return stack_snapshot_ret;
706 }
707
708 uint32_t
709 kdp_stack_snapshot_bytes_traced(void)
710 {
711 return stack_snapshot_bytes_traced;
712 }
713
714 static boolean_t memory_iszero(void *addr, size_t size)
715 {
716 char *data = (char *)addr;
717 for (size_t i = 0; i < size; i++){
718 if (data[i] != 0)
719 return FALSE;
720 }
721 return TRUE;
722 }
723
724 #define kcd_end_address(kcd) ((void *)((uint64_t)((kcd)->kcd_addr_begin) + kcdata_memory_get_used_bytes((kcd))))
725 #define kcd_max_address(kcd) ((void *)((kcd)->kcd_addr_begin + (kcd)->kcd_length))
726 /*
727 * Use of the kcd_exit_on_error(action) macro requires a local
728 * 'kern_return_t error' variable and 'error_exit' label.
729 */
730 #define kcd_exit_on_error(action) \
731 do { \
732 if (KERN_SUCCESS != (error = (action))) { \
733 if (error == KERN_RESOURCE_SHORTAGE) { \
734 error = KERN_INSUFFICIENT_BUFFER_SIZE; \
735 } \
736 goto error_exit; \
737 } \
738 } while (0); /* end kcd_exit_on_error */
739
740 static uint64_t
741 kcdata_get_task_ss_flags(task_t task)
742 {
743 uint64_t ss_flags = 0;
744 boolean_t task64 = task_has_64BitAddr(task);
745
746 if (task64)
747 ss_flags |= kUser64_p;
748 if (!task->active || task_is_a_corpse(task))
749 ss_flags |= kTerminatedSnapshot;
750 if (task->pidsuspended)
751 ss_flags |= kPidSuspended;
752 if (task->frozen)
753 ss_flags |= kFrozen;
754 if (task->effective_policy.tep_darwinbg == 1)
755 ss_flags |= kTaskDarwinBG;
756 if (task->requested_policy.trp_role == TASK_FOREGROUND_APPLICATION)
757 ss_flags |= kTaskIsForeground;
758 if (task->requested_policy.trp_boosted == 1)
759 ss_flags |= kTaskIsBoosted;
760 if (task->effective_policy.tep_sup_active == 1)
761 ss_flags |= kTaskIsSuppressed;
762 #if CONFIG_MEMORYSTATUS
763 if (memorystatus_proc_is_dirty_unsafe(task->bsd_info))
764 ss_flags |= kTaskIsDirty;
765 #endif
766
767 ss_flags |= (0x7 & workqueue_get_pwq_state_kdp(task->bsd_info)) << 17;
768
769 #if IMPORTANCE_INHERITANCE
770 if (task->task_imp_base) {
771 if (task->task_imp_base->iit_donor)
772 ss_flags |= kTaskIsImpDonor;
773 if (task->task_imp_base->iit_live_donor)
774 ss_flags |= kTaskIsLiveImpDonor;
775 }
776 #endif
777
778 return ss_flags;
779 }
780
781 static kern_return_t
782 kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyld_uuid_info_64_v2 *sys_shared_cache_loadinfo, uint32_t trace_flags, uint64_t *task_snap_ss_flags)
783 {
784 kern_return_t error = KERN_SUCCESS;
785 mach_vm_address_t out_addr = 0;
786
787 uint8_t shared_cache_identifier[16];
788 uint64_t shared_cache_slide = 0;
789 uint64_t shared_cache_base_address = 0;
790 int task_pid = pid_from_task(task);
791 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
792 uint32_t kdp_fault_results = 0;
793
794 assert(task_snap_ss_flags != NULL);
795
796 if (task->shared_region && ml_validate_nofault((vm_offset_t)task->shared_region, sizeof(struct vm_shared_region))) {
797 struct vm_shared_region *sr = task->shared_region;
798 shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping;
799 } else {
800 *task_snap_ss_flags |= kTaskSharedRegionInfoUnavailable;
801 }
802
803 if (!shared_cache_base_address ||
804 !kdp_copyin(task->map, shared_cache_base_address + offsetof(struct _dyld_cache_header, uuid),
805 shared_cache_identifier, sizeof(shared_cache_identifier), should_fault, &kdp_fault_results)) {
806 goto error_exit;
807 }
808
809 if (task->shared_region) {
810 /*
811 * No refcounting here, but we are in debugger
812 * context, so that should be safe.
813 */
814 shared_cache_slide = task->shared_region->sr_slide_info.slide;
815 } else {
816 shared_cache_slide = 0;
817 }
818
819 if (sys_shared_cache_loadinfo) {
820 if (task_pid == 1) {
821 /* save launchd's shared cache info as system level */
822 stackshot_memcpy(sys_shared_cache_loadinfo->imageUUID, shared_cache_identifier, sizeof(sys_shared_cache_loadinfo->imageUUID));
823 sys_shared_cache_loadinfo->imageLoadAddress = shared_cache_slide;
824 sys_shared_cache_loadinfo->imageSlidBaseAddress = shared_cache_slide + task->shared_region->sr_base_address;
825
826 goto error_exit;
827 } else {
828 if (shared_cache_slide == sys_shared_cache_loadinfo->imageLoadAddress &&
829 0 == memcmp(shared_cache_identifier, sys_shared_cache_loadinfo->imageUUID,
830 sizeof(sys_shared_cache_loadinfo->imageUUID))) {
831 /* skip adding shared cache info. its same as system level one */
832 goto error_exit;
833 }
834 }
835 }
836
837 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64_v2), &out_addr));
838 struct dyld_uuid_info_64_v2 *shared_cache_data = (struct dyld_uuid_info_64_v2 *)out_addr;
839 shared_cache_data->imageLoadAddress = shared_cache_slide;
840 stackshot_memcpy(shared_cache_data->imageUUID, shared_cache_identifier, sizeof(shared_cache_data->imageUUID));
841 shared_cache_data->imageSlidBaseAddress = shared_cache_base_address;
842
843 error_exit:
844 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
845 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
846 }
847
848 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
849 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
850 }
851
852 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
853 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
854 }
855
856 return error;
857 }
858
859 static kern_return_t
860 kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, uint64_t *task_snap_ss_flags)
861 {
862 boolean_t save_loadinfo_p = ((trace_flags & STACKSHOT_SAVE_LOADINFO) != 0);
863 boolean_t save_kextloadinfo_p = ((trace_flags & STACKSHOT_SAVE_KEXT_LOADINFO) != 0);
864 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
865 boolean_t minimize_uuids = collect_delta_stackshot && ((trace_flags & STACKSHOT_TAILSPIN) != 0);
866 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
867
868 kern_return_t error = KERN_SUCCESS;
869 mach_vm_address_t out_addr = 0;
870
871 uint32_t uuid_info_count = 0;
872 mach_vm_address_t uuid_info_addr = 0;
873 uint64_t uuid_info_timestamp = 0;
874 uint32_t kdp_fault_results = 0;
875
876 assert(task_snap_ss_flags != NULL);
877
878 int task_pid = pid_from_task(task);
879 boolean_t task64 = task_has_64BitAddr(task);
880
881 if (save_loadinfo_p && have_pmap && task->active && task_pid > 0) {
882 /* Read the dyld_all_image_infos struct from the task memory to get UUID array count and location */
883 if (task64) {
884 struct user64_dyld_all_image_infos task_image_infos;
885 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
886 sizeof(struct user64_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
887 uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
888 uuid_info_addr = task_image_infos.uuidArray;
889 if (task_image_infos.version >= 15) {
890 uuid_info_timestamp = task_image_infos.timestamp;
891 }
892 }
893 } else {
894 struct user32_dyld_all_image_infos task_image_infos;
895 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
896 sizeof(struct user32_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
897 uuid_info_count = task_image_infos.uuidArrayCount;
898 uuid_info_addr = task_image_infos.uuidArray;
899 if (task_image_infos.version >= 15) {
900 uuid_info_timestamp = task_image_infos.timestamp;
901 }
902 }
903 }
904
905 /*
906 * If we get a NULL uuid_info_addr (which can happen when we catch dyld in the middle of updating
907 * this data structure), we zero the uuid_info_count so that we won't even try to save load info
908 * for this task.
909 */
910 if (!uuid_info_addr) {
911 uuid_info_count = 0;
912 }
913 }
914
915 if (have_pmap && task_pid == 0) {
916 if (save_kextloadinfo_p && ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader))) {
917 uuid_info_count = gLoadedKextSummaries->numSummaries + 1; /* include main kernel UUID */
918 } else {
919 uuid_info_count = 1; /* include kernelcache UUID (embedded) or kernel UUID (desktop) */
920 }
921 }
922
923 if (task_pid > 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
924 if (minimize_uuids && uuid_info_timestamp != 0 && uuid_info_timestamp < stack_snapshot_delta_since_timestamp)
925 goto error_exit;
926
927 uint32_t uuid_info_size = (uint32_t)(task64 ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
928 uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
929
930 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, (task64 ? KCDATA_TYPE_LIBRARY_LOADINFO64 : KCDATA_TYPE_LIBRARY_LOADINFO),
931 uuid_info_size, uuid_info_count, &out_addr));
932
933 /* Copy in the UUID info array
934 * It may be nonresident, in which case just fix up nloadinfos to 0 in the task_snap
935 */
936 if (have_pmap && !kdp_copyin(task->map, uuid_info_addr, (void *)out_addr, uuid_info_array_size, should_fault, &kdp_fault_results)) {
937 bzero((void *)out_addr, uuid_info_array_size);
938 }
939
940 } else if (task_pid == 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
941 if (minimize_uuids && gLoadedKextSummaries != 0 && gLoadedKextSummariesTimestamp < stack_snapshot_delta_since_timestamp)
942 goto error_exit;
943
944 uintptr_t image_load_address;
945
946 do {
947
948
949 if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) {
950 /* Kernel UUID not found or inaccessible */
951 break;
952 }
953
954 kcd_exit_on_error(kcdata_get_memory_addr_for_array(
955 kcd, (sizeof(kernel_uuid_info) == sizeof(struct user64_dyld_uuid_info)) ? KCDATA_TYPE_LIBRARY_LOADINFO64
956 : KCDATA_TYPE_LIBRARY_LOADINFO,
957 sizeof(kernel_uuid_info), uuid_info_count, &out_addr));
958 kernel_uuid_info *uuid_info_array = (kernel_uuid_info *)out_addr;
959 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(vm_kernel_stext);
960 uuid_info_array[0].imageLoadAddress = image_load_address;
961 stackshot_memcpy(&uuid_info_array[0].imageUUID, kernel_uuid, sizeof(uuid_t));
962
963 if (save_kextloadinfo_p &&
964 ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader)) &&
965 ml_validate_nofault((vm_offset_t)(&gLoadedKextSummaries->summaries[0]),
966 gLoadedKextSummaries->entry_size * gLoadedKextSummaries->numSummaries)) {
967 uint32_t kexti;
968 for (kexti=0 ; kexti < gLoadedKextSummaries->numSummaries; kexti++) {
969 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(gLoadedKextSummaries->summaries[kexti].address);
970 uuid_info_array[kexti + 1].imageLoadAddress = image_load_address;
971 stackshot_memcpy(&uuid_info_array[kexti + 1].imageUUID, &gLoadedKextSummaries->summaries[kexti].uuid, sizeof(uuid_t));
972 }
973 }
974 } while(0);
975 }
976
977 error_exit:
978 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
979 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
980 }
981
982 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
983 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
984 }
985
986 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
987 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
988 }
989
990 return error;
991 }
992
993 static kern_return_t
994 kcdata_record_task_iostats(kcdata_descriptor_t kcd, task_t task)
995 {
996 kern_return_t error = KERN_SUCCESS;
997 mach_vm_address_t out_addr = 0;
998
999 /* I/O Statistics if any counters are non zero */
1000 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1001 if (task->task_io_stats && !memory_iszero(task->task_io_stats, sizeof(struct io_stat_info))) {
1002 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1003 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1004 _iostat->ss_disk_reads_count = task->task_io_stats->disk_reads.count;
1005 _iostat->ss_disk_reads_size = task->task_io_stats->disk_reads.size;
1006 _iostat->ss_disk_writes_count = (task->task_io_stats->total_io.count - task->task_io_stats->disk_reads.count);
1007 _iostat->ss_disk_writes_size = (task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size);
1008 _iostat->ss_paging_count = task->task_io_stats->paging.count;
1009 _iostat->ss_paging_size = task->task_io_stats->paging.size;
1010 _iostat->ss_non_paging_count = (task->task_io_stats->total_io.count - task->task_io_stats->paging.count);
1011 _iostat->ss_non_paging_size = (task->task_io_stats->total_io.size - task->task_io_stats->paging.size);
1012 _iostat->ss_metadata_count = task->task_io_stats->metadata.count;
1013 _iostat->ss_metadata_size = task->task_io_stats->metadata.size;
1014 _iostat->ss_data_count = (task->task_io_stats->total_io.count - task->task_io_stats->metadata.count);
1015 _iostat->ss_data_size = (task->task_io_stats->total_io.size - task->task_io_stats->metadata.size);
1016 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1017 _iostat->ss_io_priority_count[i] = task->task_io_stats->io_priority[i].count;
1018 _iostat->ss_io_priority_size[i] = task->task_io_stats->io_priority[i].size;
1019 }
1020 }
1021
1022 error_exit:
1023 return error;
1024 }
1025
1026 static kern_return_t
1027 kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, uint64_t **task_snap_ss_flags)
1028 {
1029 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1030 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
1031
1032 kern_return_t error = KERN_SUCCESS;
1033 mach_vm_address_t out_addr = 0;
1034 struct task_snapshot_v2 * cur_tsnap = NULL;
1035
1036 assert(task_snap_ss_flags != NULL);
1037
1038 int task_pid = pid_from_task(task);
1039 uint64_t task_uniqueid = get_task_uniqueid(task);
1040
1041 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_SNAPSHOT, sizeof(struct task_snapshot_v2), &out_addr));
1042
1043 cur_tsnap = (struct task_snapshot_v2 *)out_addr;
1044
1045 cur_tsnap->ts_unique_pid = task_uniqueid;
1046 cur_tsnap->ts_ss_flags = kcdata_get_task_ss_flags(task);
1047 *task_snap_ss_flags = &cur_tsnap->ts_ss_flags;
1048 cur_tsnap->ts_user_time_in_terminated_threads = task->total_user_time;
1049 cur_tsnap->ts_system_time_in_terminated_threads = task->total_system_time;
1050
1051 cur_tsnap->ts_p_start_sec = 0;
1052 proc_starttime_kdp(task->bsd_info, &cur_tsnap->ts_p_start_sec, NULL, NULL);
1053
1054 cur_tsnap->ts_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
1055 cur_tsnap->ts_max_resident_size = get_task_resident_max(task);
1056 cur_tsnap->ts_suspend_count = task->suspend_count;
1057 cur_tsnap->ts_faults = task->faults;
1058 cur_tsnap->ts_pageins = task->pageins;
1059 cur_tsnap->ts_cow_faults = task->cow_faults;
1060 cur_tsnap->ts_was_throttled = (uint32_t) proc_was_throttled_from_task(task);
1061 cur_tsnap->ts_did_throttle = (uint32_t) proc_did_throttle_from_task(task);
1062 cur_tsnap->ts_latency_qos = (task->effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED) ?
1063 LATENCY_QOS_TIER_UNSPECIFIED : ((0xFF << 16) | task->effective_policy.tep_latency_qos);
1064 cur_tsnap->ts_pid = task_pid;
1065
1066 /* Add the BSD process identifiers */
1067 if (task_pid != -1 && task->bsd_info != NULL)
1068 proc_name_kdp(task, cur_tsnap->ts_p_comm, sizeof(cur_tsnap->ts_p_comm));
1069 else {
1070 cur_tsnap->ts_p_comm[0] = '\0';
1071 #if IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG)
1072 if (task->task_imp_base != NULL) {
1073 stackshot_strlcpy(cur_tsnap->ts_p_comm, &task->task_imp_base->iit_procname[0],
1074 MIN((int)sizeof(task->task_imp_base->iit_procname), (int)sizeof(cur_tsnap->ts_p_comm)));
1075 }
1076 #endif
1077 }
1078
1079 if (collect_iostats) {
1080 kcd_exit_on_error(kcdata_record_task_iostats(kcd, task));
1081 }
1082
1083 error_exit:
1084 return error;
1085 }
1086
1087 static kern_return_t
1088 kcdata_record_task_delta_snapshot(kcdata_descriptor_t kcd, task_t task, boolean_t have_pmap, uint64_t **task_snap_ss_flags)
1089 {
1090 kern_return_t error = KERN_SUCCESS;
1091 struct task_delta_snapshot_v2 * cur_tsnap = NULL;
1092 mach_vm_address_t out_addr = 0;
1093
1094 uint64_t task_uniqueid = get_task_uniqueid(task);
1095 assert(task_snap_ss_flags != NULL);
1096
1097 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT, sizeof(struct task_delta_snapshot_v2), &out_addr));
1098
1099 cur_tsnap = (struct task_delta_snapshot_v2 *)out_addr;
1100
1101 cur_tsnap->tds_unique_pid = task_uniqueid;
1102 cur_tsnap->tds_ss_flags = kcdata_get_task_ss_flags(task);
1103 *task_snap_ss_flags = &cur_tsnap->tds_ss_flags;
1104
1105 cur_tsnap->tds_user_time_in_terminated_threads = task->total_user_time;
1106 cur_tsnap->tds_system_time_in_terminated_threads = task->total_system_time;
1107
1108 cur_tsnap->tds_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0;
1109
1110 cur_tsnap->tds_max_resident_size = get_task_resident_max(task);
1111 cur_tsnap->tds_suspend_count = task->suspend_count;
1112 cur_tsnap->tds_faults = task->faults;
1113 cur_tsnap->tds_pageins = task->pageins;
1114 cur_tsnap->tds_cow_faults = task->cow_faults;
1115 cur_tsnap->tds_was_throttled = (uint32_t)proc_was_throttled_from_task(task);
1116 cur_tsnap->tds_did_throttle = (uint32_t)proc_did_throttle_from_task(task);
1117 cur_tsnap->tds_latency_qos = (task-> effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED)
1118 ? LATENCY_QOS_TIER_UNSPECIFIED
1119 : ((0xFF << 16) | task-> effective_policy.tep_latency_qos);
1120
1121 error_exit:
1122 return error;
1123 }
1124
1125 static kern_return_t
1126 kcdata_record_thread_iostats(kcdata_descriptor_t kcd, thread_t thread)
1127 {
1128 kern_return_t error = KERN_SUCCESS;
1129 mach_vm_address_t out_addr = 0;
1130
1131 /* I/O Statistics */
1132 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1133 if (thread->thread_io_stats && !memory_iszero(thread->thread_io_stats, sizeof(struct io_stat_info))) {
1134 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1135 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1136 _iostat->ss_disk_reads_count = thread->thread_io_stats->disk_reads.count;
1137 _iostat->ss_disk_reads_size = thread->thread_io_stats->disk_reads.size;
1138 _iostat->ss_disk_writes_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->disk_reads.count);
1139 _iostat->ss_disk_writes_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->disk_reads.size);
1140 _iostat->ss_paging_count = thread->thread_io_stats->paging.count;
1141 _iostat->ss_paging_size = thread->thread_io_stats->paging.size;
1142 _iostat->ss_non_paging_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->paging.count);
1143 _iostat->ss_non_paging_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->paging.size);
1144 _iostat->ss_metadata_count = thread->thread_io_stats->metadata.count;
1145 _iostat->ss_metadata_size = thread->thread_io_stats->metadata.size;
1146 _iostat->ss_data_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->metadata.count);
1147 _iostat->ss_data_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->metadata.size);
1148 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1149 _iostat->ss_io_priority_count[i] = thread->thread_io_stats->io_priority[i].count;
1150 _iostat->ss_io_priority_size[i] = thread->thread_io_stats->io_priority[i].size;
1151 }
1152 }
1153
1154 error_exit:
1155 return error;
1156 }
1157
1158 static kern_return_t
1159 kcdata_record_thread_snapshot(
1160 kcdata_descriptor_t kcd, thread_t thread, task_t task, uint32_t trace_flags, boolean_t have_pmap, boolean_t thread_on_core)
1161 {
1162 boolean_t dispatch_p = ((trace_flags & STACKSHOT_GET_DQ) != 0);
1163 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1164 boolean_t trace_fp_p = ((trace_flags & STACKSHOT_TAILSPIN) == 0);
1165 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1166 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS);
1167
1168 kern_return_t error = KERN_SUCCESS;
1169 mach_vm_address_t out_addr = 0;
1170 int saved_count = 0;
1171
1172 struct thread_snapshot_v3 * cur_thread_snap = NULL;
1173 char cur_thread_name[STACKSHOT_MAX_THREAD_NAME_SIZE];
1174 uint64_t tval = 0;
1175 boolean_t task64 = task_has_64BitAddr(task);
1176
1177 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v3), &out_addr));
1178 cur_thread_snap = (struct thread_snapshot_v3 *)out_addr;
1179
1180 /* Populate the thread snapshot header */
1181 cur_thread_snap->ths_thread_id = thread_tid(thread);
1182 cur_thread_snap->ths_wait_event = VM_KERNEL_UNSLIDE_OR_PERM(thread->wait_event);
1183 cur_thread_snap->ths_continuation = VM_KERNEL_UNSLIDE(thread->continuation);
1184 cur_thread_snap->ths_total_syscalls = thread->syscalls_mach + thread->syscalls_unix;
1185
1186 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1187 cur_thread_snap->ths_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1188 else
1189 cur_thread_snap->ths_voucher_identifier = 0;
1190
1191 cur_thread_snap->ths_dqserialnum = 0;
1192 if (dispatch_p && (task != kernel_task) && (task->active) && have_pmap) {
1193 uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
1194 if (dqkeyaddr != 0) {
1195 uint64_t dqaddr = 0;
1196 boolean_t copyin_ok = kdp_copyin_word(task, dqkeyaddr, &dqaddr, FALSE, NULL);
1197 if (copyin_ok && dqaddr != 0) {
1198 uint64_t dqserialnumaddr = dqaddr + get_task_dispatchqueue_serialno_offset(task);
1199 uint64_t dqserialnum = 0;
1200 copyin_ok = kdp_copyin_word(task, dqserialnumaddr, &dqserialnum, FALSE, NULL);
1201 if (copyin_ok) {
1202 cur_thread_snap->ths_ss_flags |= kHasDispatchSerial;
1203 cur_thread_snap->ths_dqserialnum = dqserialnum;
1204 }
1205 }
1206 }
1207 }
1208
1209 tval = safe_grab_timer_value(&thread->user_timer);
1210 cur_thread_snap->ths_user_time = tval;
1211 tval = safe_grab_timer_value(&thread->system_timer);
1212
1213 if (thread->precise_user_kernel_time) {
1214 cur_thread_snap->ths_sys_time = tval;
1215 } else {
1216 cur_thread_snap->ths_user_time += tval;
1217 cur_thread_snap->ths_sys_time = 0;
1218 }
1219
1220 cur_thread_snap->ths_ss_flags = 0;
1221 if (thread->effective_policy.thep_darwinbg)
1222 cur_thread_snap->ths_ss_flags |= kThreadDarwinBG;
1223 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1224 cur_thread_snap->ths_ss_flags |= kThreadIOPassive;
1225 if (thread->suspend_count > 0)
1226 cur_thread_snap->ths_ss_flags |= kThreadSuspended;
1227 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1228 cur_thread_snap->ths_ss_flags |= kGlobalForcedIdle;
1229 if (thread_on_core)
1230 cur_thread_snap->ths_ss_flags |= kThreadOnCore;
1231 if (stackshot_thread_is_idle_worker_unsafe(thread))
1232 cur_thread_snap->ths_ss_flags |= kThreadIdleWorker;
1233
1234 /* make sure state flags defined in kcdata.h still match internal flags */
1235 static_assert(SS_TH_WAIT == TH_WAIT);
1236 static_assert(SS_TH_SUSP == TH_SUSP);
1237 static_assert(SS_TH_RUN == TH_RUN);
1238 static_assert(SS_TH_UNINT == TH_UNINT);
1239 static_assert(SS_TH_TERMINATE == TH_TERMINATE);
1240 static_assert(SS_TH_TERMINATE2 == TH_TERMINATE2);
1241 static_assert(SS_TH_IDLE == TH_IDLE);
1242
1243 cur_thread_snap->ths_last_run_time = thread->last_run_time;
1244 cur_thread_snap->ths_last_made_runnable_time = thread->last_made_runnable_time;
1245 cur_thread_snap->ths_state = thread->state;
1246 cur_thread_snap->ths_sched_flags = thread->sched_flags;
1247 cur_thread_snap->ths_base_priority = thread->base_pri;
1248 cur_thread_snap->ths_sched_priority = thread->sched_pri;
1249 cur_thread_snap->ths_eqos = thread->effective_policy.thep_qos;
1250 cur_thread_snap->ths_rqos = thread->requested_policy.thrp_qos;
1251 cur_thread_snap->ths_rqos_override = thread->requested_policy.thrp_qos_override;
1252 cur_thread_snap->ths_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1253 cur_thread_snap->ths_thread_t = VM_KERNEL_ADDRPERM(thread);
1254
1255 /* if there is thread name then add to buffer */
1256 cur_thread_name[0] = '\0';
1257 proc_threadname_kdp(thread->uthread, cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE);
1258 if (strnlen(cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE) > 0) {
1259 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_NAME, sizeof(cur_thread_name), &out_addr));
1260 stackshot_memcpy((void *)out_addr, (void *)cur_thread_name, sizeof(cur_thread_name));
1261 }
1262
1263 /* record system and user cpu times */
1264 time_value_t user_time;
1265 time_value_t system_time;
1266 thread_read_times(thread, &user_time, &system_time);
1267 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_CPU_TIMES, sizeof(struct stackshot_cpu_times), &out_addr));
1268 struct stackshot_cpu_times * stackshot_cpu_times = (struct stackshot_cpu_times *)out_addr;
1269 stackshot_cpu_times->user_usec = ((uint64_t)user_time.seconds) * USEC_PER_SEC + user_time.microseconds;
1270 stackshot_cpu_times->system_usec = ((uint64_t)system_time.seconds) * USEC_PER_SEC + system_time.microseconds;
1271
1272 /* Trace user stack, if any */
1273 if (!active_kthreads_only_p && task->active && thread->task->map != kernel_map) {
1274 uint32_t thread_snapshot_flags = 0;
1275 /* 64-bit task? */
1276 if (task64) {
1277 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1278 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE,
1279 trace_fp_p, &thread_snapshot_flags);
1280 if (saved_count > 0) {
1281 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1282 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME64
1283 : STACKSHOT_KCTYPE_USER_STACKLR64,
1284 frame_size, saved_count / frame_size, &out_addr));
1285 cur_thread_snap->ths_ss_flags |= kUser64_p;
1286 }
1287 } else {
1288 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1289 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE, trace_fp_p,
1290 &thread_snapshot_flags);
1291 if (saved_count > 0) {
1292 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1293 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME
1294 : STACKSHOT_KCTYPE_USER_STACKLR,
1295 frame_size, saved_count / frame_size, &out_addr));
1296 }
1297 }
1298
1299 if (thread_snapshot_flags != 0) {
1300 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1301 }
1302 }
1303
1304 /* Call through to the machine specific trace routines
1305 * Frames are added past the snapshot header.
1306 */
1307 if (thread->kernel_stack != 0) {
1308 uint32_t thread_snapshot_flags = 0;
1309 #if defined(__LP64__)
1310 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1311 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1312 &thread_snapshot_flags);
1313 if (saved_count > 0) {
1314 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1315 cur_thread_snap->ths_ss_flags |= kKernel64_p;
1316 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME64
1317 : STACKSHOT_KCTYPE_KERN_STACKLR64,
1318 frame_size, saved_count / frame_size, &out_addr));
1319 }
1320 #else
1321 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1322 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1323 &thread_snapshot_flags);
1324 if (saved_count > 0) {
1325 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1326 kcd_exit_on_error(
1327 kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME : STACKSHOT_KCTYPE_KERN_STACKLR,
1328 frame_size, saved_count / frame_size, &out_addr));
1329 }
1330 #endif
1331 if (thread_snapshot_flags != 0) {
1332 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1333 }
1334 }
1335
1336 if (collect_iostats) {
1337 kcd_exit_on_error(kcdata_record_thread_iostats(kcd, thread));
1338 }
1339
1340 error_exit:
1341 return error;
1342 }
1343
1344 static int
1345 kcdata_record_thread_delta_snapshot(struct thread_delta_snapshot_v2 * cur_thread_snap, thread_t thread, boolean_t thread_on_core)
1346 {
1347 cur_thread_snap->tds_thread_id = thread_tid(thread);
1348 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1349 cur_thread_snap->tds_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1350 else
1351 cur_thread_snap->tds_voucher_identifier = 0;
1352
1353 cur_thread_snap->tds_ss_flags = 0;
1354 if (thread->effective_policy.thep_darwinbg)
1355 cur_thread_snap->tds_ss_flags |= kThreadDarwinBG;
1356 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1357 cur_thread_snap->tds_ss_flags |= kThreadIOPassive;
1358 if (thread->suspend_count > 0)
1359 cur_thread_snap->tds_ss_flags |= kThreadSuspended;
1360 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1361 cur_thread_snap->tds_ss_flags |= kGlobalForcedIdle;
1362 if (thread_on_core)
1363 cur_thread_snap->tds_ss_flags |= kThreadOnCore;
1364 if (stackshot_thread_is_idle_worker_unsafe(thread))
1365 cur_thread_snap->tds_ss_flags |= kThreadIdleWorker;
1366
1367 cur_thread_snap->tds_last_made_runnable_time = thread->last_made_runnable_time;
1368 cur_thread_snap->tds_state = thread->state;
1369 cur_thread_snap->tds_sched_flags = thread->sched_flags;
1370 cur_thread_snap->tds_base_priority = thread->base_pri;
1371 cur_thread_snap->tds_sched_priority = thread->sched_pri;
1372 cur_thread_snap->tds_eqos = thread->effective_policy.thep_qos;
1373 cur_thread_snap->tds_rqos = thread->requested_policy.thrp_qos;
1374 cur_thread_snap->tds_rqos_override = thread->requested_policy.thrp_qos_override;
1375 cur_thread_snap->tds_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1376
1377 return 0;
1378 }
1379
1380 /*
1381 * Why 12? 12 strikes a decent balance between allocating a large array on
1382 * the stack and having large kcdata item overheads for recording nonrunable
1383 * tasks.
1384 */
1385 #define UNIQUEIDSPERFLUSH 12
1386
1387 struct saved_uniqueids {
1388 uint64_t ids[UNIQUEIDSPERFLUSH];
1389 unsigned count;
1390 };
1391
1392 static kern_return_t
1393 flush_nonrunnable_tasks(struct saved_uniqueids * ids)
1394 {
1395 if (ids->count == 0)
1396 return KERN_SUCCESS;
1397 mach_vm_address_t out_addr = 0;
1398 kern_return_t ret = kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TASKS, sizeof(uint64_t),
1399 ids->count, &out_addr);
1400 if (ret != KERN_SUCCESS) {
1401 return ret;
1402 }
1403 stackshot_memcpy((void *)out_addr, ids->ids, sizeof(uint64_t) * ids->count);
1404 ids->count = 0;
1405 return ret;
1406 }
1407
1408 static kern_return_t
1409 handle_nonrunnable_task(struct saved_uniqueids * ids, uint64_t pid)
1410 {
1411 kern_return_t ret = KERN_SUCCESS;
1412 ids->ids[ids->count] = pid;
1413 ids->count++;
1414 assert(ids->count <= UNIQUEIDSPERFLUSH);
1415 if (ids->count == UNIQUEIDSPERFLUSH)
1416 ret = flush_nonrunnable_tasks(ids);
1417 return ret;
1418 }
1419
1420 enum thread_classification {
1421 tc_full_snapshot, /* take a full snapshot */
1422 tc_delta_snapshot, /* take a delta snapshot */
1423 tc_nonrunnable, /* only report id */
1424 };
1425
1426 static enum thread_classification
1427 classify_thread(thread_t thread, boolean_t * thread_on_core_p, uint32_t trace_flags)
1428 {
1429 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1430 boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
1431
1432 processor_t last_processor = thread->last_processor;
1433
1434 boolean_t thread_on_core =
1435 (last_processor != PROCESSOR_NULL && last_processor->state == PROCESSOR_RUNNING && last_processor->active_thread == thread);
1436
1437 *thread_on_core_p = thread_on_core;
1438
1439 /* Capture the full thread snapshot if this is not a delta stackshot or if the thread has run subsequent to the
1440 * previous full stackshot */
1441 if (!collect_delta_stackshot || thread_on_core || (thread->last_run_time > stack_snapshot_delta_since_timestamp)) {
1442 return tc_full_snapshot;
1443 } else {
1444 if (minimize_nonrunnables && !(thread->state & TH_RUN)) {
1445 return tc_nonrunnable;
1446 } else {
1447 return tc_delta_snapshot;
1448 }
1449 }
1450 }
1451
1452 static kern_return_t
1453 kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTraced)
1454 {
1455 kern_return_t error = KERN_SUCCESS;
1456 mach_vm_address_t out_addr = 0;
1457 uint64_t abs_time = 0, abs_time_end = 0;
1458 uint64_t *abs_time_addr = NULL;
1459 uint64_t system_state_flags = 0;
1460 int saved_count = 0;
1461 task_t task = TASK_NULL;
1462 thread_t thread = THREAD_NULL;
1463 mach_timebase_info_data_t timebase = {0, 0};
1464 uint32_t length_to_copy = 0, tmp32 = 0;
1465
1466 abs_time = mach_absolute_time();
1467
1468 /* process the flags */
1469 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1470 boolean_t save_donating_pids_p = ((trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0);
1471 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1472 boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0);
1473 boolean_t use_fault_path = ((trace_flags & (STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_ENABLE_BT_FAULTING)) != 0);
1474
1475 stack_enable_faulting = (trace_flags & (STACKSHOT_ENABLE_BT_FAULTING));
1476
1477
1478 struct saved_uniqueids saved_uniqueids = {.count = 0};
1479
1480 if (use_fault_path) {
1481 fault_stats.sfs_pages_faulted_in = 0;
1482 fault_stats.sfs_time_spent_faulting = 0;
1483 fault_stats.sfs_stopped_faulting = (uint8_t) FALSE;
1484 }
1485
1486 if (sizeof(void *) == 8)
1487 system_state_flags |= kKernel64_p;
1488
1489 if (stackshot_kcdata_p == NULL || pBytesTraced == NULL) {
1490 error = KERN_INVALID_ARGUMENT;
1491 goto error_exit;
1492 }
1493
1494 /* setup mach_absolute_time and timebase info -- copy out in some cases and needed to convert since_timestamp to seconds for proc start time */
1495 clock_timebase_info(&timebase);
1496
1497 /* begin saving data into the buffer */
1498 *pBytesTraced = 0;
1499 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_in_flags"));
1500 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, (uint32_t)pid, "stackshot_in_pid"));
1501 kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_kcdata_p, system_state_flags, "system_state_flags"));
1502
1503 #if CONFIG_JETSAM
1504 tmp32 = memorystatus_get_pressure_status_kdp();
1505 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_LEVEL, sizeof(uint32_t), &out_addr));
1506 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1507 #endif
1508
1509 if (!collect_delta_stackshot) {
1510 tmp32 = PAGE_SIZE;
1511 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_KERN_PAGE_SIZE, sizeof(uint32_t), &out_addr));
1512 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1513
1514 /* save boot-args and osversion string */
1515 length_to_copy = MIN((uint32_t)(strlen(version) + 1), OSVERSIZE);
1516 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_OSVERSION, length_to_copy, &out_addr));
1517 stackshot_strlcpy((char*)out_addr, &version[0], length_to_copy);
1518
1519 length_to_copy = MIN((uint32_t)(strlen(PE_boot_args()) + 1), OSVERSIZE);
1520 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_BOOTARGS, length_to_copy, &out_addr));
1521 stackshot_strlcpy((char*)out_addr, PE_boot_args(), length_to_copy);
1522
1523 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_TIMEBASE, sizeof(timebase), &out_addr));
1524 stackshot_memcpy((void *)out_addr, &timebase, sizeof(timebase));
1525 } else {
1526 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP, sizeof(uint64_t), &out_addr));
1527 stackshot_memcpy((void*)out_addr, &stack_snapshot_delta_since_timestamp, sizeof(stack_snapshot_delta_since_timestamp));
1528 }
1529
1530 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_MACH_ABSOLUTE_TIME, sizeof(uint64_t), &out_addr));
1531 abs_time_addr = (uint64_t *)out_addr;
1532 stackshot_memcpy((void *)abs_time_addr, &abs_time, sizeof(uint64_t));
1533
1534 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &out_addr));
1535 stackshot_memcpy((void *)out_addr, &stackshot_microsecs, sizeof(uint64_t));
1536
1537 /* reserve space of system level shared cache load info */
1538 struct dyld_uuid_info_64_v2 * sys_shared_cache_loadinfo = NULL;
1539 if (!collect_delta_stackshot) {
1540 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO,
1541 sizeof(struct dyld_uuid_info_64_v2), &out_addr));
1542 sys_shared_cache_loadinfo = (struct dyld_uuid_info_64_v2 *)out_addr;
1543 bzero((void *)sys_shared_cache_loadinfo, sizeof(struct dyld_uuid_info_64_v2));
1544 }
1545
1546 /* Add requested information first */
1547 if (trace_flags & STACKSHOT_GET_GLOBAL_MEM_STATS) {
1548 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_GLOBAL_MEM_STATS, sizeof(struct mem_and_io_snapshot), &out_addr));
1549 kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)out_addr);
1550 }
1551
1552 /* Iterate over tasks */
1553 queue_head_t *task_list = &tasks;
1554 queue_iterate(task_list, task, task_t, tasks) {
1555 int task_pid = 0;
1556 uint64_t task_uniqueid = 0;
1557 int num_delta_thread_snapshots = 0;
1558 int num_nonrunnable_threads = 0;
1559 uint64_t task_start_abstime = 0;
1560 boolean_t task_delta_stackshot = FALSE;
1561 boolean_t task64 = FALSE, have_map = FALSE, have_pmap = FALSE;
1562 boolean_t some_thread_ran = FALSE;
1563 uint64_t *task_snap_ss_flags = NULL;
1564
1565 if ((task == NULL) || !ml_validate_nofault((vm_offset_t)task, sizeof(struct task))) {
1566 error = KERN_FAILURE;
1567 goto error_exit;
1568 }
1569
1570 have_map = (task->map != NULL) && (ml_validate_nofault((vm_offset_t)(task->map), sizeof(struct _vm_map)));
1571 have_pmap = have_map && (task->map->pmap != NULL) && (ml_validate_nofault((vm_offset_t)(task->map->pmap), sizeof(struct pmap)));
1572
1573 task_pid = pid_from_task(task);
1574 task_uniqueid = get_task_uniqueid(task);
1575 task64 = task_has_64BitAddr(task);
1576
1577 if (!task->active || task_is_a_corpse(task)) {
1578 /*
1579 * Not interested in terminated tasks without threads, and
1580 * at the moment, stackshot can't handle a task without a name.
1581 */
1582 if (queue_empty(&task->threads) || task_pid == -1) {
1583 continue;
1584 }
1585 }
1586
1587 if (collect_delta_stackshot) {
1588 proc_starttime_kdp(task->bsd_info, NULL, NULL, &task_start_abstime);
1589 }
1590
1591 /* Trace everything, unless a process was specified */
1592 if ((pid == -1) || (pid == task_pid)) {
1593 #if DEBUG || DEVELOPMENT
1594 /* we might want to call kcdata_undo_add_container_begin(), which is
1595 * only safe if we call it after kcdata_add_container_marker() but
1596 * before adding any other kcdata items. In development kernels,
1597 * we'll remember where the buffer end was and confirm after calling
1598 * kcdata_undo_add_container_begin() that it's in exactly the same
1599 * place.*/
1600 mach_vm_address_t revert_addr = stackshot_kcdata_p->kcd_addr_end;
1601 #endif
1602
1603 /* add task snapshot marker */
1604 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1605 STACKSHOT_KCCONTAINER_TASK, task_uniqueid));
1606
1607 if (!collect_delta_stackshot || (task_start_abstime == 0) ||
1608 (task_start_abstime > stack_snapshot_delta_since_timestamp)) {
1609 kcd_exit_on_error(kcdata_record_task_snapshot(stackshot_kcdata_p, task, trace_flags, have_pmap, &task_snap_ss_flags));
1610 } else {
1611 task_delta_stackshot = TRUE;
1612 if (minimize_nonrunnables) {
1613 // delay taking the task snapshot. If there are no runnable threads we'll skip it.
1614 } else {
1615 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
1616 }
1617 }
1618
1619 /* Iterate over task threads */
1620 queue_iterate(&task->threads, thread, thread_t, task_threads)
1621 {
1622 uint64_t thread_uniqueid;
1623
1624 if ((thread == NULL) || !ml_validate_nofault((vm_offset_t)thread, sizeof(struct thread))) {
1625 error = KERN_FAILURE;
1626 goto error_exit;
1627 }
1628
1629 if (active_kthreads_only_p && thread->kernel_stack == 0)
1630 continue;
1631
1632 thread_uniqueid = thread_tid(thread);
1633
1634 boolean_t thread_on_core;
1635 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
1636
1637 switch (thread_classification) {
1638 case tc_full_snapshot:
1639 /* add thread marker */
1640 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1641 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1642 kcd_exit_on_error(
1643 kcdata_record_thread_snapshot(stackshot_kcdata_p, thread, task, trace_flags, have_pmap, thread_on_core));
1644
1645 /* mark end of thread snapshot data */
1646 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END,
1647 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1648
1649 some_thread_ran = TRUE;
1650 break;
1651
1652 case tc_delta_snapshot:
1653 num_delta_thread_snapshots++;
1654 break;
1655
1656 case tc_nonrunnable:
1657 num_nonrunnable_threads++;
1658 break;
1659 }
1660 }
1661
1662 if (task_delta_stackshot && minimize_nonrunnables) {
1663 if (some_thread_ran || num_delta_thread_snapshots > 0) {
1664 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, have_pmap, &task_snap_ss_flags));
1665 } else {
1666 kcd_exit_on_error(kcdata_undo_add_container_begin(stackshot_kcdata_p));
1667
1668 #if DEBUG || DEVELOPMENT
1669 mach_vm_address_t undo_addr = stackshot_kcdata_p->kcd_addr_end;
1670 if (revert_addr != undo_addr) {
1671 panic("tried to revert a container begin but we already moved past it. revert=%p undo=%p",
1672 (void *)revert_addr, (void *)undo_addr);
1673 }
1674 #endif
1675 kcd_exit_on_error(handle_nonrunnable_task(&saved_uniqueids, task_uniqueid));
1676 continue;
1677 }
1678 }
1679
1680 struct thread_delta_snapshot_v2 * delta_snapshots = NULL;
1681 int current_delta_snapshot_index = 0;
1682
1683 if (num_delta_thread_snapshots > 0) {
1684 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT,
1685 sizeof(struct thread_delta_snapshot_v2),
1686 num_delta_thread_snapshots, &out_addr));
1687 delta_snapshots = (struct thread_delta_snapshot_v2 *)out_addr;
1688 }
1689
1690 uint64_t * nonrunnable_tids = NULL;
1691 int current_nonrunnable_index = 0;
1692
1693 if (num_nonrunnable_threads > 0) {
1694 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TIDS,
1695 sizeof(uint64_t), num_nonrunnable_threads, &out_addr));
1696 nonrunnable_tids = (uint64_t *)out_addr;
1697 }
1698
1699 if (num_delta_thread_snapshots > 0 || num_nonrunnable_threads > 0) {
1700 queue_iterate(&task->threads, thread, thread_t, task_threads)
1701 {
1702 if (active_kthreads_only_p && thread->kernel_stack == 0)
1703 continue;
1704
1705 boolean_t thread_on_core;
1706 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, trace_flags);
1707
1708 switch (thread_classification) {
1709 case tc_full_snapshot:
1710 /* full thread snapshot captured above */
1711 continue;
1712
1713 case tc_delta_snapshot:
1714 kcd_exit_on_error(kcdata_record_thread_delta_snapshot(&delta_snapshots[current_delta_snapshot_index++],
1715 thread, thread_on_core));
1716 break;
1717
1718 case tc_nonrunnable:
1719 nonrunnable_tids[current_nonrunnable_index++] = thread_tid(thread);
1720 continue;
1721 }
1722 }
1723
1724 #if DEBUG || DEVELOPMENT
1725 if (current_delta_snapshot_index != num_delta_thread_snapshots) {
1726 panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1727 num_delta_thread_snapshots, current_delta_snapshot_index);
1728 }
1729 if (current_nonrunnable_index != num_nonrunnable_threads) {
1730 panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1731 num_nonrunnable_threads, current_nonrunnable_index);
1732 }
1733 #endif
1734 }
1735
1736 #if IMPORTANCE_INHERITANCE
1737 if (save_donating_pids_p) {
1738 kcd_exit_on_error(
1739 ((((mach_vm_address_t)kcd_end_address(stackshot_kcdata_p) + (TASK_IMP_WALK_LIMIT * sizeof(int32_t))) <
1740 (mach_vm_address_t)kcd_max_address(stackshot_kcdata_p))
1741 ? KERN_SUCCESS
1742 : KERN_RESOURCE_SHORTAGE));
1743 saved_count = task_importance_list_pids(task, TASK_IMP_LIST_DONATING_PIDS,
1744 (void *)kcd_end_address(stackshot_kcdata_p), TASK_IMP_WALK_LIMIT);
1745 if (saved_count > 0)
1746 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_DONATING_PIDS,
1747 sizeof(int32_t), saved_count, &out_addr));
1748 }
1749 #endif
1750
1751 if (!collect_delta_stackshot || (num_delta_thread_snapshots != task->thread_count) || !task_delta_stackshot) {
1752 /*
1753 * Collect shared cache info and UUID info in these scenarios
1754 * 1) a full stackshot
1755 * 2) a delta stackshot where the task started after the previous full stackshot OR
1756 * any thread from the task has run since the previous full stackshot
1757 */
1758
1759 kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, sys_shared_cache_loadinfo, trace_flags, task_snap_ss_flags));
1760 kcd_exit_on_error(kcdata_record_uuid_info(stackshot_kcdata_p, task, trace_flags, have_pmap, task_snap_ss_flags));
1761 }
1762 /* mark end of task snapshot data */
1763 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK,
1764 task_uniqueid));
1765 }
1766 }
1767
1768 if (minimize_nonrunnables) {
1769 flush_nonrunnable_tasks(&saved_uniqueids);
1770 }
1771
1772 if (use_fault_path) {
1773 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS,
1774 sizeof(struct stackshot_fault_stats), &out_addr));
1775 stackshot_memcpy((void*)out_addr, &fault_stats, sizeof(struct stackshot_fault_stats));
1776 }
1777
1778 /* update timestamp of the stackshot */
1779 abs_time_end = mach_absolute_time();
1780 #if DEVELOPMENT || DEBUG
1781 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_DURATION,
1782 sizeof(struct stackshot_duration), &out_addr));
1783 struct stackshot_duration * stackshot_duration = (struct stackshot_duration *)out_addr;
1784 stackshot_duration->stackshot_duration = (abs_time_end - abs_time);
1785 stackshot_duration->stackshot_duration_outer = 0;
1786 stackshot_duration_outer = &stackshot_duration->stackshot_duration_outer;
1787 #endif
1788 stackshot_memcpy((void *)abs_time_addr, &abs_time_end, sizeof(uint64_t));
1789
1790
1791 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_kcdata_p));
1792
1793 /* === END of populating stackshot data === */
1794
1795 *pBytesTraced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_kcdata_p);
1796 error_exit:
1797
1798 stack_enable_faulting = FALSE;
1799
1800 return error;
1801 }
1802
1803 static int pid_from_task(task_t task)
1804 {
1805 int pid = -1;
1806
1807 if (task->bsd_info) {
1808 pid = proc_pid(task->bsd_info);
1809 } else {
1810 pid = task_pid(task);
1811 }
1812
1813 return pid;
1814 }
1815
1816 static uint64_t
1817 proc_was_throttled_from_task(task_t task)
1818 {
1819 uint64_t was_throttled = 0;
1820
1821 if (task->bsd_info)
1822 was_throttled = proc_was_throttled(task->bsd_info);
1823
1824 return was_throttled;
1825 }
1826
1827 static uint64_t
1828 proc_did_throttle_from_task(task_t task)
1829 {
1830 uint64_t did_throttle = 0;
1831
1832 if (task->bsd_info)
1833 did_throttle = proc_did_throttle(task->bsd_info);
1834
1835 return did_throttle;
1836 }
1837
1838 static void
1839 kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap)
1840 {
1841 unsigned int pages_reclaimed;
1842 unsigned int pages_wanted;
1843 kern_return_t kErr;
1844
1845 processor_t processor;
1846 vm_statistics64_t stat;
1847 vm_statistics64_data_t host_vm_stat;
1848
1849 processor = processor_list;
1850 stat = &PROCESSOR_DATA(processor, vm_stat);
1851 host_vm_stat = *stat;
1852
1853 if (processor_count > 1) {
1854 /*
1855 * processor_list may be in the process of changing as we are
1856 * attempting a stackshot. Ordinarily it will be lock protected,
1857 * but it is not safe to lock in the context of the debugger.
1858 * Fortunately we never remove elements from the processor list,
1859 * and only add to to the end of the list, so we SHOULD be able
1860 * to walk it. If we ever want to truly tear down processors,
1861 * this will have to change.
1862 */
1863 while ((processor = processor->processor_list) != NULL) {
1864 stat = &PROCESSOR_DATA(processor, vm_stat);
1865 host_vm_stat.compressions += stat->compressions;
1866 host_vm_stat.decompressions += stat->decompressions;
1867 }
1868 }
1869
1870 memio_snap->snapshot_magic = STACKSHOT_MEM_AND_IO_SNAPSHOT_MAGIC;
1871 memio_snap->free_pages = vm_page_free_count;
1872 memio_snap->active_pages = vm_page_active_count;
1873 memio_snap->inactive_pages = vm_page_inactive_count;
1874 memio_snap->purgeable_pages = vm_page_purgeable_count;
1875 memio_snap->wired_pages = vm_page_wire_count;
1876 memio_snap->speculative_pages = vm_page_speculative_count;
1877 memio_snap->throttled_pages = vm_page_throttled_count;
1878 memio_snap->busy_buffer_count = count_busy_buffers();
1879 memio_snap->filebacked_pages = vm_page_pageable_external_count;
1880 memio_snap->compressions = (uint32_t)host_vm_stat.compressions;
1881 memio_snap->decompressions = (uint32_t)host_vm_stat.decompressions;
1882 memio_snap->compressor_size = VM_PAGE_COMPRESSOR_COUNT;
1883 kErr = mach_vm_pressure_monitor(FALSE, VM_PRESSURE_TIME_WINDOW, &pages_reclaimed, &pages_wanted);
1884
1885 if ( ! kErr ) {
1886 memio_snap->pages_wanted = (uint32_t)pages_wanted;
1887 memio_snap->pages_reclaimed = (uint32_t)pages_reclaimed;
1888 memio_snap->pages_wanted_reclaimed_valid = 1;
1889 } else {
1890 memio_snap->pages_wanted = 0;
1891 memio_snap->pages_reclaimed = 0;
1892 memio_snap->pages_wanted_reclaimed_valid = 0;
1893 }
1894 }
1895
1896 void
1897 stackshot_memcpy(void *dst, const void *src, size_t len)
1898 {
1899 memcpy(dst, src, len);
1900 }
1901
1902 size_t
1903 stackshot_strlcpy(char *dst, const char *src, size_t maxlen)
1904 {
1905 const size_t srclen = strlen(src);
1906
1907 if (srclen < maxlen) {
1908 stackshot_memcpy(dst, src, srclen+1);
1909 } else if (maxlen != 0) {
1910 stackshot_memcpy(dst, src, maxlen-1);
1911 dst[maxlen-1] = '\0';
1912 }
1913
1914 return srclen;
1915 }
1916
1917
1918 /*
1919 * Returns the physical address of the specified map:target address,
1920 * using the kdp fault path if requested and the page is not resident.
1921 */
1922 vm_offset_t
1923 kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results)
1924 {
1925 vm_offset_t cur_phys_addr;
1926 unsigned cur_wimg_bits;
1927 uint64_t fault_start_time = 0;
1928
1929 if (map == VM_MAP_NULL) {
1930 return 0;
1931 }
1932
1933 cur_phys_addr = kdp_vtophys(map->pmap, target_addr);
1934 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
1935 if (!try_fault || fault_stats.sfs_stopped_faulting) {
1936 if (kdp_fault_results)
1937 *kdp_fault_results |= KDP_FAULT_RESULT_PAGED_OUT;
1938
1939 return 0;
1940 }
1941
1942 /*
1943 * The pmap doesn't have a valid page so we start at the top level
1944 * vm map and try a lightweight fault. Update fault path usage stats.
1945 */
1946 fault_start_time = mach_absolute_time();
1947 cur_phys_addr = kdp_lightweight_fault(map, (target_addr & ~PAGE_MASK));
1948 fault_stats.sfs_time_spent_faulting += (mach_absolute_time() - fault_start_time);
1949
1950 if ((fault_stats.sfs_time_spent_faulting >= fault_stats.sfs_system_max_fault_time) && !panic_stackshot) {
1951 fault_stats.sfs_stopped_faulting = (uint8_t) TRUE;
1952 }
1953
1954 cur_phys_addr += (target_addr & PAGE_MASK);
1955
1956 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
1957 if (kdp_fault_results)
1958 *kdp_fault_results |= (KDP_FAULT_RESULT_TRIED_FAULT | KDP_FAULT_RESULT_PAGED_OUT);
1959
1960 return 0;
1961 }
1962
1963 if (kdp_fault_results)
1964 *kdp_fault_results |= KDP_FAULT_RESULT_FAULTED_IN;
1965
1966 fault_stats.sfs_pages_faulted_in++;
1967 } else {
1968 /*
1969 * This check is done in kdp_lightweight_fault for the fault path.
1970 */
1971 cur_wimg_bits = pmap_cache_attributes((ppnum_t) atop(cur_phys_addr));
1972
1973 if ((cur_wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
1974 return 0;
1975 }
1976 }
1977
1978 return cur_phys_addr;
1979 }
1980
1981 boolean_t
1982 kdp_copyin_word(
1983 task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results)
1984 {
1985 if (task_has_64BitAddr(task)) {
1986 return kdp_copyin(task->map, addr, result, sizeof(uint64_t), try_fault, kdp_fault_results);
1987 } else {
1988 uint32_t buf;
1989 boolean_t r = kdp_copyin(task->map, addr, &buf, sizeof(uint32_t), try_fault, kdp_fault_results);
1990 *result = buf;
1991 return r;
1992 }
1993 }
1994
1995 boolean_t
1996 kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_results)
1997 {
1998 size_t rem = size;
1999 char *kvaddr = dest;
2000
2001 #if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS)
2002 /* Identify if destination buffer is in panic storage area */
2003 if ((vm_offset_t)dest >= gPanicBase && (vm_offset_t)dest < gPanicBase + gPanicSize) {
2004 if (((vm_offset_t)dest + size) >= (gPanicBase + gPanicSize)) {
2005 return FALSE;
2006 }
2007 }
2008 #endif
2009
2010 while (rem) {
2011 uint64_t phys_src = kdp_find_phys(map, uaddr, try_fault, kdp_fault_results);
2012 uint64_t phys_dest = kvtophys((vm_offset_t)kvaddr);
2013 uint64_t src_rem = PAGE_SIZE - (phys_src & PAGE_MASK);
2014 uint64_t dst_rem = PAGE_SIZE - (phys_dest & PAGE_MASK);
2015 size_t cur_size = (uint32_t) MIN(src_rem, dst_rem);
2016 cur_size = MIN(cur_size, rem);
2017
2018 if (phys_src && phys_dest) {
2019 bcopy_phys(phys_src, phys_dest, cur_size);
2020 } else {
2021 break;
2022 }
2023
2024 uaddr += cur_size;
2025 kvaddr += cur_size;
2026 rem -= cur_size;
2027 }
2028
2029 return (rem == 0);
2030 }
2031
2032 kern_return_t
2033 do_stackshot(void *context)
2034 {
2035 #pragma unused(context)
2036 kdp_snapshot++;
2037
2038 stack_snapshot_ret = kdp_stackshot_kcdata_format(stack_snapshot_pid,
2039 stack_snapshot_flags,
2040 &stack_snapshot_bytes_traced);
2041
2042 kdp_snapshot--;
2043 return stack_snapshot_ret;
2044 }
2045
2046 /*
2047 * A fantastical routine that tries to be fast about returning
2048 * translations. Caches the last page we found a translation
2049 * for, so that we can be quick about multiple queries to the
2050 * same page. It turns out this is exactly the workflow
2051 * machine_trace_thread and its relatives tend to throw at us.
2052 *
2053 * Please zero the nasty global this uses after a bulk lookup;
2054 * this isn't safe across a switch of the map or changes
2055 * to a pmap.
2056 *
2057 * This also means that if zero is a valid KVA, we are
2058 * screwed. Sucks to be us. Fortunately, this should never
2059 * happen.
2060 */
2061 vm_offset_t
2062 machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags)
2063 {
2064 vm_offset_t cur_target_page;
2065 vm_offset_t cur_phys_addr;
2066 vm_offset_t kern_virt_target_addr;
2067 uint32_t kdp_fault_results = 0;
2068
2069 cur_target_page = atop(cur_target_addr);
2070
2071 if ((cur_target_page != prev_target_page) || validate_next_addr) {
2072
2073 /*
2074 * Alright; it wasn't our previous page. So
2075 * we must validate that there is a page
2076 * table entry for this address under the
2077 * current pmap, and that it has default
2078 * cache attributes (otherwise it may not be
2079 * safe to access it).
2080 */
2081 cur_phys_addr = kdp_find_phys(map, cur_target_addr, stack_enable_faulting, &kdp_fault_results);
2082 if (thread_trace_flags) {
2083 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
2084 *thread_trace_flags |= kThreadTruncatedBT;
2085 }
2086
2087 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
2088 *thread_trace_flags |= kThreadTriedFaultBT;
2089 }
2090
2091 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
2092 *thread_trace_flags |= kThreadFaultedBT;
2093 }
2094 }
2095
2096 if (cur_phys_addr == 0) {
2097 return 0;
2098 }
2099 #if __x86_64__
2100 kern_virt_target_addr = (vm_offset_t) PHYSMAP_PTOV(cur_phys_addr);
2101 #else
2102 #error Oh come on... we should really unify the physical -> kernel virtual interface
2103 #endif
2104 prev_target_page = cur_target_page;
2105 prev_target_kva = (kern_virt_target_addr & ~PAGE_MASK);
2106 validate_next_addr = FALSE;
2107 return kern_virt_target_addr;
2108 } else {
2109 /* We found a translation, so stash this page */
2110 kern_virt_target_addr = prev_target_kva + (cur_target_addr & PAGE_MASK);
2111 return kern_virt_target_addr;
2112 }
2113 }
2114
2115 void
2116 machine_trace_thread_clear_validation_cache(void)
2117 {
2118 validate_next_addr = TRUE;
2119 }
2120
2121 boolean_t
2122 stackshot_thread_is_idle_worker_unsafe(thread_t thread)
2123 {
2124 /* When the pthread kext puts a worker thread to sleep, it will call
2125 * assert_wait on the thread's own threadlist. see parkit() in
2126 * kern_support.c.
2127 */
2128 struct uthread * uthread = get_bsdthread_info(thread);
2129 event64_t threadlist = (event64_t)proc_get_uthread_uu_threadlist(uthread);
2130 event64_t wait_event = thread->wait_event;
2131 return uthread &&
2132 (thread->state & TH_WAIT) &&
2133 wait_event &&
2134 threadlist == wait_event;
2135 }
mirror of XNU