]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kperf/callstack.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.11.26.tar.gz
[apple/xnu.git] / osfmk / kperf / callstack.c
1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Collect kernel callstacks */
30
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <vm/vm_map.h>
35 #include <kperf/buffer.h>
36 #include <kperf/context.h>
37 #include <kperf/callstack.h>
38 #include <kperf/ast.h>
39 #include <sys/errno.h>
40
41 #if defined(__arm__) || defined(__arm64__)
42 #include <arm/cpu_data.h>
43 #include <arm/cpu_data_internal.h>
44 #endif
45
46 static void
47 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
48 {
49 uint64_t fixup_val = 0;
50 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
51
52 #if defined(__x86_64__)
53 user_addr_t sp_user;
54 bool user_64;
55 x86_saved_state_t *state;
56
57 state = get_user_regs(thread);
58 if (!state) {
59 goto out;
60 }
61
62 user_64 = is_saved_state64(state);
63 if (user_64) {
64 sp_user = saved_state64(state)->isf.rsp;
65 } else {
66 sp_user = saved_state32(state)->uesp;
67 }
68
69 if (thread == current_thread()) {
70 (void)copyin(sp_user, (char *)&fixup_val,
71 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
72 } else {
73 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
74 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
75 }
76
77 #elif defined(__arm64__) || defined(__arm__)
78
79 struct arm_saved_state *state = get_user_regs(thread);
80 if (!state) {
81 goto out;
82 }
83
84 /* encode thumb mode into low bit of PC */
85 if (get_saved_state_cpsr(state) & PSR_TF) {
86 cs->kpuc_frames[0] |= 1ULL;
87 }
88
89 fixup_val = get_saved_state_lr(state);
90
91 #else
92 #error "callstack_fixup_user: unsupported architecture"
93 #endif
94
95 out:
96 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
97 }
98
99 #if defined(__x86_64__)
100
101 __attribute__((used))
102 static kern_return_t
103 interrupted_kernel_sp_value(uintptr_t *sp_val)
104 {
105 x86_saved_state_t *state;
106 uintptr_t sp;
107 bool state_64;
108 uint64_t cs;
109 uintptr_t top, bottom;
110
111 state = current_cpu_datap()->cpu_int_state;
112 if (!state) {
113 return KERN_FAILURE;
114 }
115
116 state_64 = is_saved_state64(state);
117
118 if (state_64) {
119 cs = saved_state64(state)->isf.cs;
120 } else {
121 cs = saved_state32(state)->cs;
122 }
123 /* return early if interrupted a thread in user space */
124 if ((cs & SEL_PL) == SEL_PL_U) {
125 return KERN_FAILURE;
126 }
127
128 if (state_64) {
129 sp = saved_state64(state)->isf.rsp;
130 } else {
131 sp = saved_state32(state)->uesp;
132 }
133
134 /* make sure the stack pointer is pointing somewhere in this stack */
135 bottom = current_thread()->kernel_stack;
136 top = bottom + kernel_stack_size;
137 if (sp >= bottom && sp < top) {
138 return KERN_FAILURE;
139 }
140
141 *sp_val = *(uintptr_t *)sp;
142 return KERN_SUCCESS;
143 }
144
145 #elif defined(__arm64__)
146
147 __attribute__((used))
148 static kern_return_t
149 interrupted_kernel_lr(uintptr_t *lr)
150 {
151 struct arm_saved_state *state;
152
153 state = getCpuDatap()->cpu_int_state;
154
155 /* return early if interrupted a thread in user space */
156 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
157 return KERN_FAILURE;
158 }
159
160 *lr = get_saved_state_lr(state);
161 return KERN_SUCCESS;
162 }
163
164 #elif defined(__arm__)
165
166 __attribute__((used))
167 static kern_return_t
168 interrupted_kernel_lr(uintptr_t *lr)
169 {
170 struct arm_saved_state *state;
171
172 state = getCpuDatap()->cpu_int_state;
173
174 /* return early if interrupted a thread in user space */
175 if (PSR_IS_USER(get_saved_state_cpsr(state))) {
176 return KERN_FAILURE;
177 }
178
179 *lr = get_saved_state_lr(state);
180 return KERN_SUCCESS;
181 }
182
183 #else /* defined(__arm__) */
184 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
185 #endif /* !defined(__arm__) */
186
187
188 static void
189 callstack_fixup_interrupted(struct kp_kcallstack *cs)
190 {
191 uintptr_t fixup_val = 0;
192 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
193
194 /*
195 * Only provide arbitrary data on development or debug kernels.
196 */
197 #if DEVELOPMENT || DEBUG
198 #if defined(__x86_64__)
199 (void)interrupted_kernel_sp_value(&fixup_val);
200 #elif defined(__arm64__) || defined(__arm__)
201 (void)interrupted_kernel_lr(&fixup_val);
202 #endif /* defined(__x86_64__) */
203 #endif /* DEVELOPMENT || DEBUG */
204
205 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
206 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
207 }
208
209 void
210 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
211 {
212 thread_t thread;
213
214 assert(cs != NULL);
215 assert(context != NULL);
216
217 thread = context->cur_thread;
218 assert(thread != NULL);
219 assert(thread->continuation != NULL);
220
221 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
222 #ifdef __LP64__
223 cs->kpkc_flags |= CALLSTACK_64BIT;
224 #endif
225
226 cs->kpkc_nframes = 1;
227 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
228 }
229
230 void
231 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
232 {
233 assert(cs != NULL);
234 assert(context != NULL);
235 assert(context->cur_thread == current_thread());
236
237 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
238 #ifdef __LP64__
239 cs->kpkc_flags |= CALLSTACK_64BIT;
240 #endif
241
242 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
243
244 bool trunc = false;
245 cs->kpkc_nframes = backtrace_frame(cs->kpkc_word_frames,
246 cs->kpkc_nframes - 1, context->starting_fp, &trunc);
247 if (cs->kpkc_nframes > 0) {
248 cs->kpkc_flags |= CALLSTACK_VALID;
249 /*
250 * Fake the value pointed to by the stack pointer or the link
251 * register for symbolicators.
252 */
253 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
254 cs->kpkc_nframes += 1;
255 }
256 if (trunc) {
257 cs->kpkc_nframes |= CALLSTACK_TRUNCATED;
258 }
259
260 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
261 }
262
263 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
264 uint64_t *callStack, mach_msg_type_number_t *count,
265 boolean_t user_only);
266
267 void
268 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
269 {
270 thread_t thread;
271
272 assert(cs != NULL);
273 assert(context != NULL);
274 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
275
276 thread = context->cur_thread;
277 assert(thread != NULL);
278
279 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
280 cs->kpkc_nframes);
281
282 cs->kpkc_flags = CALLSTACK_KERNEL;
283 #ifdef __LP64__
284 cs->kpkc_flags |= CALLSTACK_64BIT;
285 #endif
286
287 if (ml_at_interrupt_context()) {
288 assert(thread == current_thread());
289 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
290 bool trunc = false;
291 cs->kpkc_nframes = backtrace_interrupted(
292 cs->kpkc_word_frames, cs->kpkc_nframes - 1, &trunc);
293 if (cs->kpkc_nframes != 0) {
294 callstack_fixup_interrupted(cs);
295 }
296 if (trunc) {
297 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
298 }
299 } else {
300 /*
301 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
302 * other threads.
303 */
304 kern_return_t kr;
305 kr = chudxnu_thread_get_callstack64_kperf(thread,
306 cs->kpkc_frames, &cs->kpkc_nframes, FALSE);
307 if (kr == KERN_SUCCESS) {
308 cs->kpkc_flags |= CALLSTACK_VALID;
309 } else if (kr == KERN_RESOURCE_SHORTAGE) {
310 cs->kpkc_flags |= CALLSTACK_VALID;
311 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
312 } else {
313 cs->kpkc_nframes = 0;
314 }
315 }
316
317 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
318 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
319 }
320
321 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
322 cs->kpkc_flags, cs->kpkc_nframes);
323 }
324
325 void
326 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
327 {
328 assert(ml_get_interrupts_enabled() == TRUE);
329
330 thread_t thread = context->cur_thread;
331 assert(thread != NULL);
332
333 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
334 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
335
336 bool user64 = false;
337 bool trunc = false;
338 int err = backtrace_thread_user(thread, cs->kpuc_frames,
339 cs->kpuc_nframes - 1, &cs->kpuc_nframes, &user64, &trunc);
340 cs->kpuc_flags = CALLSTACK_KERNEL_WORDS;
341 if (user64) {
342 cs->kpuc_flags |= CALLSTACK_64BIT;
343 }
344 if (trunc) {
345 cs->kpuc_flags |= CALLSTACK_TRUNCATED;
346 }
347
348 if (!err || err == EFAULT) {
349 callstack_fixup_user(cs, thread);
350 cs->kpuc_flags |= CALLSTACK_VALID;
351 } else {
352 cs->kpuc_nframes = 0;
353 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err);
354 }
355
356 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
357 cs->kpuc_flags, cs->kpuc_nframes);
358 }
359
360 static inline uintptr_t
361 scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
362 {
363 if (frame < n_frames) {
364 if (kern) {
365 return VM_KERNEL_UNSLIDE(bt[frame]);
366 } else {
367 return bt[frame];
368 }
369 } else {
370 return 0;
371 }
372 }
373
374 static inline uintptr_t
375 scrub_frame(uint64_t *bt, int n_frames, int frame)
376 {
377 if (frame < n_frames) {
378 return (uintptr_t)(bt[frame]);
379 } else {
380 return 0;
381 }
382 }
383
384 static void
385 callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
386 unsigned int nframes, unsigned int flags)
387 {
388 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
389
390 BUF_DATA(hdrid, flags, nframes);
391
392 unsigned int nevts = nframes / 4;
393 unsigned int ovf = nframes % 4;
394 if (ovf != 0) {
395 nevts++;
396 }
397
398 bool kern = flags & CALLSTACK_KERNEL;
399
400 if (flags & CALLSTACK_KERNEL_WORDS) {
401 uintptr_t *frames = vframes;
402 for (unsigned int i = 0; i < nevts; i++) {
403 unsigned int j = i * 4;
404 BUF_DATA(dataid,
405 scrub_word(frames, nframes, j + 0, kern),
406 scrub_word(frames, nframes, j + 1, kern),
407 scrub_word(frames, nframes, j + 2, kern),
408 scrub_word(frames, nframes, j + 3, kern));
409 }
410 } else {
411 for (unsigned int i = 0; i < nevts; i++) {
412 uint64_t *frames = vframes;
413 unsigned int j = i * 4;
414 BUF_DATA(dataid,
415 scrub_frame(frames, nframes, j + 0),
416 scrub_frame(frames, nframes, j + 1),
417 scrub_frame(frames, nframes, j + 2),
418 scrub_frame(frames, nframes, j + 3));
419 }
420 }
421
422 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
423 }
424
425 void
426 kperf_kcallstack_log(struct kp_kcallstack *cs)
427 {
428 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
429 cs->kpkc_nframes, cs->kpkc_flags);
430 }
431
432 void
433 kperf_ucallstack_log(struct kp_ucallstack *cs)
434 {
435 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
436 cs->kpuc_nframes, cs->kpuc_flags);
437 }
438
439 int
440 kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth)
441 {
442 int did_pend = kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK);
443 kperf_ast_set_callstack_depth(context->cur_thread, depth);
444
445 return did_pend;
446 }
447
448 static kern_return_t
449 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
450 {
451 return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
452 KERN_SUCCESS : KERN_FAILURE;
453 }
454
455 static kern_return_t
456 chudxnu_task_read(
457 task_t task,
458 void *kernaddr,
459 uint64_t usraddr,
460 vm_size_t size)
461 {
462 //ppc version ported to arm
463 kern_return_t ret = KERN_SUCCESS;
464
465 if (ml_at_interrupt_context()) {
466 return KERN_FAILURE; // can't look at tasks on interrupt stack
467 }
468
469 if (current_task() == task) {
470 if (copyin(usraddr, kernaddr, size)) {
471 ret = KERN_FAILURE;
472 }
473 } else {
474 vm_map_t map = get_task_map(task);
475 ret = vm_map_read_user(map, usraddr, kernaddr, size);
476 }
477
478 return ret;
479 }
480
481 static inline uint64_t
482 chudxnu_vm_unslide( uint64_t ptr, int kaddr )
483 {
484 if (!kaddr) {
485 return ptr;
486 }
487
488 return VM_KERNEL_UNSLIDE(ptr);
489 }
490
491 #if __arm__
492 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
493 #define CS_FLAG_EXTRASP 1 // capture extra sp register
494 static kern_return_t
495 chudxnu_thread_get_callstack64_internal(
496 thread_t thread,
497 uint64_t *callStack,
498 mach_msg_type_number_t *count,
499 boolean_t user_only,
500 int flags)
501 {
502 kern_return_t kr;
503 task_t task;
504 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
505 uint64_t prevPC = 0ULL;
506 uint32_t kernStackMin = thread->kernel_stack;
507 uint32_t kernStackMax = kernStackMin + kernel_stack_size;
508 uint64_t *buffer = callStack;
509 uint32_t frame[2];
510 int bufferIndex = 0;
511 int bufferMaxIndex = 0;
512 boolean_t supervisor = FALSE;
513 struct arm_saved_state *state = NULL;
514 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
515 uint64_t pc = 0ULL;
516
517 task = get_threadtask(thread);
518
519 bufferMaxIndex = *count;
520 //get thread state
521 if (user_only) {
522 state = find_user_regs(thread);
523 } else {
524 state = find_kern_regs(thread);
525 }
526
527 if (!state) {
528 *count = 0;
529 return KERN_FAILURE;
530 }
531
532 /* make sure it is safe to dereference before you do it */
533 supervisor = ARM_SUPERVISOR_MODE(state->cpsr);
534
535 /* can't take a kernel callstack if we've got a user frame */
536 if (!user_only && !supervisor) {
537 return KERN_FAILURE;
538 }
539
540 /*
541 * Reserve space for saving LR (and sometimes SP) at the end of the
542 * backtrace.
543 */
544 if (flags & CS_FLAG_EXTRASP) {
545 bufferMaxIndex -= 2;
546 } else {
547 bufferMaxIndex -= 1;
548 }
549
550 if (bufferMaxIndex < 2) {
551 *count = 0;
552 return KERN_RESOURCE_SHORTAGE;
553 }
554
555 currPC = (uint64_t)state->pc; /* r15 */
556 if (state->cpsr & PSR_TF) {
557 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
558 }
559 currLR = (uint64_t)state->lr; /* r14 */
560 currSP = (uint64_t)state->sp; /* r13 */
561
562 fp = (uint32_t *)state->r[7]; /* frame pointer */
563 topfp = fp;
564
565 bufferIndex = 0; // start with a stack of size zero
566 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, supervisor); // save PC in position 0.
567
568 // Now, fill buffer with stack backtraces.
569 while (bufferIndex < bufferMaxIndex) {
570 pc = 0ULL;
571 /*
572 * Below the frame pointer, the following values are saved:
573 * -> FP
574 */
575
576 /*
577 * Note that we read the pc even for the first stack frame
578 * (which, in theory, is always empty because the callee fills
579 * it in just before it lowers the stack. However, if we
580 * catch the program in between filling in the return address
581 * and lowering the stack, we want to still have a valid
582 * backtrace. FixupStack correctly disregards this value if
583 * necessary.
584 */
585
586 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
587 /* frame pointer is invalid - stop backtracing */
588 pc = 0ULL;
589 break;
590 }
591
592 if (supervisor) {
593 if (((uint32_t)fp > kernStackMax) ||
594 ((uint32_t)fp < kernStackMin)) {
595 kr = KERN_FAILURE;
596 } else {
597 kr = chudxnu_kern_read(&frame,
598 (vm_offset_t)fp,
599 (vm_size_t)sizeof(frame));
600 if (kr == KERN_SUCCESS) {
601 pc = (uint64_t)frame[1];
602 nextFramePointer = (uint32_t *) (frame[0]);
603 } else {
604 pc = 0ULL;
605 nextFramePointer = 0ULL;
606 kr = KERN_FAILURE;
607 }
608 }
609 } else {
610 kr = chudxnu_task_read(task,
611 &frame,
612 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
613 sizeof(frame));
614 if (kr == KERN_SUCCESS) {
615 pc = (uint64_t) frame[1];
616 nextFramePointer = (uint32_t *) (frame[0]);
617 } else {
618 pc = 0ULL;
619 nextFramePointer = 0ULL;
620 kr = KERN_FAILURE;
621 }
622 }
623
624 if (kr != KERN_SUCCESS) {
625 pc = 0ULL;
626 break;
627 }
628
629 if (nextFramePointer) {
630 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, supervisor);
631 prevPC = pc;
632 }
633
634 if (nextFramePointer < fp) {
635 break;
636 } else {
637 fp = nextFramePointer;
638 }
639 }
640
641 if (bufferIndex >= bufferMaxIndex) {
642 bufferIndex = bufferMaxIndex;
643 kr = KERN_RESOURCE_SHORTAGE;
644 } else {
645 kr = KERN_SUCCESS;
646 }
647
648 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
649 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, supervisor);
650 if (flags & CS_FLAG_EXTRASP) {
651 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, supervisor);
652 }
653
654 *count = bufferIndex;
655 return kr;
656 }
657
658 kern_return_t
659 chudxnu_thread_get_callstack64_kperf(
660 thread_t thread,
661 uint64_t *callStack,
662 mach_msg_type_number_t *count,
663 boolean_t user_only)
664 {
665 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
666 }
667 #elif __arm64__
668
669 #if defined(HAS_APPLE_PAC)
670 #include <ptrauth.h>
671 #endif
672
673 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
674 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
675 // after sampling has finished.
676 //
677 // For an N-entry callstack:
678 //
679 // [0] current pc
680 // [1..N-3] stack frames (including current one)
681 // [N-2] current LR (return value if we're in a leaf function)
682 // [N-1] current r0 (in case we've saved LR in r0) (optional)
683 //
684 //
685 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
686
687 #define CS_FLAG_EXTRASP 1 // capture extra sp register
688
689 static kern_return_t
690 chudxnu_thread_get_callstack64_internal(
691 thread_t thread,
692 uint64_t *callStack,
693 mach_msg_type_number_t *count,
694 boolean_t user_only,
695 int flags)
696 {
697 kern_return_t kr = KERN_SUCCESS;
698 task_t task;
699 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
700 uint64_t prevPC = 0ULL;
701 uint64_t kernStackMin = thread->kernel_stack;
702 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
703 uint64_t *buffer = callStack;
704 int bufferIndex = 0;
705 int bufferMaxIndex = 0;
706 boolean_t kernel = FALSE;
707 struct arm_saved_state *sstate = NULL;
708 uint64_t pc = 0ULL;
709
710 task = get_threadtask(thread);
711 bufferMaxIndex = *count;
712 //get thread state
713 if (user_only) {
714 sstate = find_user_regs(thread);
715 } else {
716 sstate = find_kern_regs(thread);
717 }
718
719 if (!sstate) {
720 *count = 0;
721 return KERN_FAILURE;
722 }
723
724 if (is_saved_state64(sstate)) {
725 struct arm_saved_state64 *state = NULL;
726 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
727 uint64_t frame[2];
728
729 state = saved_state64(sstate);
730
731 /* make sure it is safe to dereference before you do it */
732 kernel = PSR64_IS_KERNEL(state->cpsr);
733
734 /* can't take a kernel callstack if we've got a user frame */
735 if (!user_only && !kernel) {
736 return KERN_FAILURE;
737 }
738
739 /*
740 * Reserve space for saving LR (and sometimes SP) at the end of the
741 * backtrace.
742 */
743 if (flags & CS_FLAG_EXTRASP) {
744 bufferMaxIndex -= 2;
745 } else {
746 bufferMaxIndex -= 1;
747 }
748
749 if (bufferMaxIndex < 2) {
750 *count = 0;
751 return KERN_RESOURCE_SHORTAGE;
752 }
753
754 currPC = state->pc;
755 currLR = state->lr;
756 currSP = state->sp;
757
758 fp = (uint64_t *)state->fp; /* frame pointer */
759 topfp = fp;
760
761 bufferIndex = 0; // start with a stack of size zero
762 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
763
764 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
765
766 // Now, fill buffer with stack backtraces.
767 while (bufferIndex < bufferMaxIndex) {
768 pc = 0ULL;
769 /*
770 * Below the frame pointer, the following values are saved:
771 * -> FP
772 */
773
774 /*
775 * Note that we read the pc even for the first stack frame
776 * (which, in theory, is always empty because the callee fills
777 * it in just before it lowers the stack. However, if we
778 * catch the program in between filling in the return address
779 * and lowering the stack, we want to still have a valid
780 * backtrace. FixupStack correctly disregards this value if
781 * necessary.
782 */
783
784 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
785 /* frame pointer is invalid - stop backtracing */
786 pc = 0ULL;
787 break;
788 }
789
790 if (kernel) {
791 if (((uint64_t)fp > kernStackMax) ||
792 ((uint64_t)fp < kernStackMin)) {
793 kr = KERN_FAILURE;
794 } else {
795 kr = chudxnu_kern_read(&frame,
796 (vm_offset_t)fp,
797 (vm_size_t)sizeof(frame));
798 if (kr == KERN_SUCCESS) {
799 #if defined(HAS_APPLE_PAC)
800 /* return addresses on stack will be signed by arm64e ABI */
801 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
802 #else
803 pc = frame[1];
804 #endif
805 nextFramePointer = (uint64_t *)frame[0];
806 } else {
807 pc = 0ULL;
808 nextFramePointer = 0ULL;
809 kr = KERN_FAILURE;
810 }
811 }
812 } else {
813 kr = chudxnu_task_read(task,
814 &frame,
815 (vm_offset_t)fp,
816 (vm_size_t)sizeof(frame));
817 if (kr == KERN_SUCCESS) {
818 #if defined(HAS_APPLE_PAC)
819 /* return addresses on stack will be signed by arm64e ABI */
820 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
821 #else
822 pc = frame[1];
823 #endif
824 nextFramePointer = (uint64_t *)(frame[0]);
825 } else {
826 pc = 0ULL;
827 nextFramePointer = 0ULL;
828 kr = KERN_FAILURE;
829 }
830 }
831
832 if (kr != KERN_SUCCESS) {
833 pc = 0ULL;
834 break;
835 }
836
837 if (nextFramePointer) {
838 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
839 prevPC = pc;
840 }
841
842 if (nextFramePointer < fp) {
843 break;
844 } else {
845 fp = nextFramePointer;
846 }
847 }
848
849 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
850
851 if (bufferIndex >= bufferMaxIndex) {
852 bufferIndex = bufferMaxIndex;
853 kr = KERN_RESOURCE_SHORTAGE;
854 } else {
855 kr = KERN_SUCCESS;
856 }
857
858 // Save link register and SP at bottom of stack (used for later fixup).
859 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
860 if (flags & CS_FLAG_EXTRASP) {
861 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
862 }
863 } else {
864 struct arm_saved_state32 *state = NULL;
865 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
866
867 /* 64-bit kernel stacks, 32-bit user stacks */
868 uint64_t frame[2];
869 uint32_t frame32[2];
870
871 state = saved_state32(sstate);
872
873 /* make sure it is safe to dereference before you do it */
874 kernel = ARM_SUPERVISOR_MODE(state->cpsr);
875
876 /* can't take a kernel callstack if we've got a user frame */
877 if (!user_only && !kernel) {
878 return KERN_FAILURE;
879 }
880
881 /*
882 * Reserve space for saving LR (and sometimes SP) at the end of the
883 * backtrace.
884 */
885 if (flags & CS_FLAG_EXTRASP) {
886 bufferMaxIndex -= 2;
887 } else {
888 bufferMaxIndex -= 1;
889 }
890
891 if (bufferMaxIndex < 2) {
892 *count = 0;
893 return KERN_RESOURCE_SHORTAGE;
894 }
895
896 currPC = (uint64_t)state->pc; /* r15 */
897 if (state->cpsr & PSR_TF) {
898 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
899 }
900 currLR = (uint64_t)state->lr; /* r14 */
901 currSP = (uint64_t)state->sp; /* r13 */
902
903 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
904 topfp = fp;
905
906 bufferIndex = 0; // start with a stack of size zero
907 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
908
909 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
910
911 // Now, fill buffer with stack backtraces.
912 while (bufferIndex < bufferMaxIndex) {
913 pc = 0ULL;
914 /*
915 * Below the frame pointer, the following values are saved:
916 * -> FP
917 */
918
919 /*
920 * Note that we read the pc even for the first stack frame
921 * (which, in theory, is always empty because the callee fills
922 * it in just before it lowers the stack. However, if we
923 * catch the program in between filling in the return address
924 * and lowering the stack, we want to still have a valid
925 * backtrace. FixupStack correctly disregards this value if
926 * necessary.
927 */
928
929 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
930 /* frame pointer is invalid - stop backtracing */
931 pc = 0ULL;
932 break;
933 }
934
935 if (kernel) {
936 if (((uint32_t)fp > kernStackMax) ||
937 ((uint32_t)fp < kernStackMin)) {
938 kr = KERN_FAILURE;
939 } else {
940 kr = chudxnu_kern_read(&frame,
941 (vm_offset_t)fp,
942 (vm_size_t)sizeof(frame));
943 if (kr == KERN_SUCCESS) {
944 pc = (uint64_t)frame[1];
945 nextFramePointer = (uint32_t *) (frame[0]);
946 } else {
947 pc = 0ULL;
948 nextFramePointer = 0ULL;
949 kr = KERN_FAILURE;
950 }
951 }
952 } else {
953 kr = chudxnu_task_read(task,
954 &frame32,
955 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
956 sizeof(frame32));
957 if (kr == KERN_SUCCESS) {
958 pc = (uint64_t)frame32[1];
959 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
960 } else {
961 pc = 0ULL;
962 nextFramePointer = 0ULL;
963 kr = KERN_FAILURE;
964 }
965 }
966
967 if (kr != KERN_SUCCESS) {
968 pc = 0ULL;
969 break;
970 }
971
972 if (nextFramePointer) {
973 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
974 prevPC = pc;
975 }
976
977 if (nextFramePointer < fp) {
978 break;
979 } else {
980 fp = nextFramePointer;
981 }
982 }
983
984 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
985
986 /* clamp callstack size to max */
987 if (bufferIndex >= bufferMaxIndex) {
988 bufferIndex = bufferMaxIndex;
989 kr = KERN_RESOURCE_SHORTAGE;
990 } else {
991 /* ignore all other failures */
992 kr = KERN_SUCCESS;
993 }
994
995 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
996 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
997 if (flags & CS_FLAG_EXTRASP) {
998 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
999 }
1000 }
1001
1002 *count = bufferIndex;
1003 return kr;
1004 }
1005
1006 kern_return_t
1007 chudxnu_thread_get_callstack64_kperf(
1008 thread_t thread,
1009 uint64_t *callStack,
1010 mach_msg_type_number_t *count,
1011 boolean_t user_only)
1012 {
1013 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
1014 }
1015 #elif __x86_64__
1016
1017 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
1018 // don't try to read in the hole
1019 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
1020 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
1021 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
1022
1023 typedef struct _cframe64_t {
1024 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
1025 uint64_t caller;
1026 uint64_t args[0];
1027 }cframe64_t;
1028
1029
1030 typedef struct _cframe_t {
1031 uint32_t prev; // this is really a user32-space pointer to the previous frame
1032 uint32_t caller;
1033 uint32_t args[0];
1034 } cframe_t;
1035
1036 extern void * find_user_regs(thread_t);
1037 extern x86_saved_state32_t *find_kern_regs(thread_t);
1038
1039 static kern_return_t
1040 do_kernel_backtrace(
1041 thread_t thread,
1042 struct x86_kernel_state *regs,
1043 uint64_t *frames,
1044 mach_msg_type_number_t *start_idx,
1045 mach_msg_type_number_t max_idx)
1046 {
1047 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1048 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1049 mach_msg_type_number_t ct = *start_idx;
1050 kern_return_t kr = KERN_FAILURE;
1051
1052 #if __LP64__
1053 uint64_t currPC = 0ULL;
1054 uint64_t currFP = 0ULL;
1055 uint64_t prevPC = 0ULL;
1056 uint64_t prevFP = 0ULL;
1057 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
1058 return KERN_FAILURE;
1059 }
1060 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
1061 return KERN_FAILURE;
1062 }
1063 #else
1064 uint32_t currPC = 0U;
1065 uint32_t currFP = 0U;
1066 uint32_t prevPC = 0U;
1067 uint32_t prevFP = 0U;
1068 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
1069 return KERN_FAILURE;
1070 }
1071 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
1072 return KERN_FAILURE;
1073 }
1074 #endif
1075
1076 if (*start_idx >= max_idx) {
1077 return KERN_RESOURCE_SHORTAGE; // no frames traced
1078 }
1079 if (!currPC) {
1080 return KERN_FAILURE;
1081 }
1082
1083 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1084
1085 // build a backtrace of this kernel state
1086 #if __LP64__
1087 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
1088 // this is the address where caller lives in the user thread
1089 uint64_t caller = currFP + sizeof(uint64_t);
1090 #else
1091 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
1092 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
1093 #endif
1094
1095 if (!currFP || !currPC) {
1096 currPC = 0;
1097 break;
1098 }
1099
1100 if (ct >= max_idx) {
1101 *start_idx = ct;
1102 return KERN_RESOURCE_SHORTAGE;
1103 }
1104
1105 /* read our caller */
1106 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
1107
1108 if (kr != KERN_SUCCESS || !currPC) {
1109 currPC = 0UL;
1110 break;
1111 }
1112
1113 /*
1114 * retrive contents of the frame pointer and advance to the next stack
1115 * frame if it's valid
1116 */
1117 prevFP = 0;
1118 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1119
1120 #if __LP64__
1121 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1122 #else
1123 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1124 #endif
1125 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1126 prevPC = currPC;
1127 }
1128 if (prevFP <= currFP) {
1129 break;
1130 } else {
1131 currFP = prevFP;
1132 }
1133 }
1134
1135 *start_idx = ct;
1136 return KERN_SUCCESS;
1137 }
1138
1139
1140
1141 static kern_return_t
1142 do_backtrace32(
1143 task_t task,
1144 thread_t thread,
1145 x86_saved_state32_t *regs,
1146 uint64_t *frames,
1147 mach_msg_type_number_t *start_idx,
1148 mach_msg_type_number_t max_idx,
1149 boolean_t supervisor)
1150 {
1151 uint32_t tmpWord = 0UL;
1152 uint64_t currPC = (uint64_t) regs->eip;
1153 uint64_t currFP = (uint64_t) regs->ebp;
1154 uint64_t prevPC = 0ULL;
1155 uint64_t prevFP = 0ULL;
1156 uint64_t kernStackMin = thread->kernel_stack;
1157 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1158 mach_msg_type_number_t ct = *start_idx;
1159 kern_return_t kr = KERN_FAILURE;
1160
1161 if (ct >= max_idx) {
1162 return KERN_RESOURCE_SHORTAGE; // no frames traced
1163 }
1164 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1165
1166 // build a backtrace of this 32 bit state.
1167 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1168 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1169
1170 if (!currFP) {
1171 currPC = 0;
1172 break;
1173 }
1174
1175 if (ct >= max_idx) {
1176 *start_idx = ct;
1177 return KERN_RESOURCE_SHORTAGE;
1178 }
1179
1180 /* read our caller */
1181 if (supervisor) {
1182 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1183 } else {
1184 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1185 }
1186
1187 if (kr != KERN_SUCCESS) {
1188 currPC = 0ULL;
1189 break;
1190 }
1191
1192 currPC = (uint64_t) tmpWord; // promote 32 bit address
1193
1194 /*
1195 * retrive contents of the frame pointer and advance to the next stack
1196 * frame if it's valid
1197 */
1198 prevFP = 0;
1199 if (supervisor) {
1200 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1201 } else {
1202 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1203 }
1204 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1205
1206 if (prevFP) {
1207 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1208 prevPC = currPC;
1209 }
1210 if (prevFP < currFP) {
1211 break;
1212 } else {
1213 currFP = prevFP;
1214 }
1215 }
1216
1217 *start_idx = ct;
1218 return KERN_SUCCESS;
1219 }
1220
1221 static kern_return_t
1222 do_backtrace64(
1223 task_t task,
1224 thread_t thread,
1225 x86_saved_state64_t *regs,
1226 uint64_t *frames,
1227 mach_msg_type_number_t *start_idx,
1228 mach_msg_type_number_t max_idx,
1229 boolean_t supervisor)
1230 {
1231 uint64_t currPC = regs->isf.rip;
1232 uint64_t currFP = regs->rbp;
1233 uint64_t prevPC = 0ULL;
1234 uint64_t prevFP = 0ULL;
1235 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1236 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1237 mach_msg_type_number_t ct = *start_idx;
1238 kern_return_t kr = KERN_FAILURE;
1239
1240 if (*start_idx >= max_idx) {
1241 return KERN_RESOURCE_SHORTAGE; // no frames traced
1242 }
1243 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1244
1245 // build a backtrace of this 32 bit state.
1246 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1247 // this is the address where caller lives in the user thread
1248 uint64_t caller = currFP + sizeof(uint64_t);
1249
1250 if (!currFP) {
1251 currPC = 0;
1252 break;
1253 }
1254
1255 if (ct >= max_idx) {
1256 *start_idx = ct;
1257 return KERN_RESOURCE_SHORTAGE;
1258 }
1259
1260 /* read our caller */
1261 if (supervisor) {
1262 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1263 } else {
1264 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1265 }
1266
1267 if (kr != KERN_SUCCESS) {
1268 currPC = 0ULL;
1269 break;
1270 }
1271
1272 /*
1273 * retrive contents of the frame pointer and advance to the next stack
1274 * frame if it's valid
1275 */
1276 prevFP = 0;
1277 if (supervisor) {
1278 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1279 } else {
1280 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1281 }
1282
1283 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1284 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1285 prevPC = currPC;
1286 }
1287 if (prevFP < currFP) {
1288 break;
1289 } else {
1290 currFP = prevFP;
1291 }
1292 }
1293
1294 *start_idx = ct;
1295 return KERN_SUCCESS;
1296 }
1297
1298 static kern_return_t
1299 chudxnu_thread_get_callstack64_internal(
1300 thread_t thread,
1301 uint64_t *callstack,
1302 mach_msg_type_number_t *count,
1303 boolean_t user_only,
1304 boolean_t kern_only)
1305 {
1306 kern_return_t kr = KERN_FAILURE;
1307 task_t task = thread->task;
1308 uint64_t currPC = 0ULL;
1309 boolean_t supervisor = FALSE;
1310 mach_msg_type_number_t bufferIndex = 0;
1311 mach_msg_type_number_t bufferMaxIndex = *count;
1312 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1313 x86_saved_state64_t *regs64 = NULL;
1314 x86_saved_state32_t *regs32 = NULL;
1315 x86_saved_state32_t *u_regs32 = NULL;
1316 x86_saved_state64_t *u_regs64 = NULL;
1317 struct x86_kernel_state *kregs = NULL;
1318
1319 if (ml_at_interrupt_context()) {
1320 if (user_only) {
1321 /* can't backtrace user state on interrupt stack. */
1322 return KERN_FAILURE;
1323 }
1324
1325 /* backtracing at interrupt context? */
1326 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1327 /*
1328 * Locate the registers for the interrupted thread, assuming it is
1329 * current_thread().
1330 */
1331 tagged_regs = current_cpu_datap()->cpu_int_state;
1332
1333 if (is_saved_state64(tagged_regs)) {
1334 /* 64 bit registers */
1335 regs64 = saved_state64(tagged_regs);
1336 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1337 } else {
1338 /* 32 bit registers */
1339 regs32 = saved_state32(tagged_regs);
1340 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1341 }
1342 }
1343 }
1344
1345 if (!ml_at_interrupt_context() && kernel_task == task) {
1346 if (!thread->kernel_stack) {
1347 return KERN_FAILURE;
1348 }
1349
1350 // Kernel thread not at interrupt context
1351 kregs = (struct x86_kernel_state *)NULL;
1352
1353 // nofault read of the thread->kernel_stack pointer
1354 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1355 return KERN_FAILURE;
1356 }
1357
1358 // Adjust to find the saved kernel state
1359 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1360
1361 supervisor = TRUE;
1362 } else if (!tagged_regs) {
1363 /*
1364 * not at interrupt context, or tracing a different thread than
1365 * current_thread() at interrupt context
1366 */
1367 tagged_regs = USER_STATE(thread);
1368 if (is_saved_state64(tagged_regs)) {
1369 /* 64 bit registers */
1370 regs64 = saved_state64(tagged_regs);
1371 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1372 } else {
1373 /* 32 bit registers */
1374 regs32 = saved_state32(tagged_regs);
1375 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1376 }
1377 }
1378
1379 *count = 0;
1380
1381 if (supervisor) {
1382 // the caller only wants a user callstack.
1383 if (user_only) {
1384 // bail - we've only got kernel state
1385 return KERN_FAILURE;
1386 }
1387 } else {
1388 // regs32(64) is not in supervisor mode.
1389 u_regs32 = regs32;
1390 u_regs64 = regs64;
1391 regs32 = NULL;
1392 regs64 = NULL;
1393 }
1394
1395 if (user_only) {
1396 /* we only want to backtrace the user mode */
1397 if (!(u_regs32 || u_regs64)) {
1398 /* no user state to look at */
1399 return KERN_FAILURE;
1400 }
1401 }
1402
1403 /*
1404 * Order of preference for top of stack:
1405 * 64 bit kernel state (not likely)
1406 * 32 bit kernel state
1407 * 64 bit user land state
1408 * 32 bit user land state
1409 */
1410
1411 if (kregs) {
1412 /*
1413 * nofault read of the registers from the kernel stack (as they can
1414 * disappear on the fly).
1415 */
1416
1417 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1418 return KERN_FAILURE;
1419 }
1420 } else if (regs64) {
1421 currPC = regs64->isf.rip;
1422 } else if (regs32) {
1423 currPC = (uint64_t) regs32->eip;
1424 } else if (u_regs64) {
1425 currPC = u_regs64->isf.rip;
1426 } else if (u_regs32) {
1427 currPC = (uint64_t) u_regs32->eip;
1428 }
1429
1430 if (!currPC) {
1431 /* no top of the stack, bail out */
1432 return KERN_FAILURE;
1433 }
1434
1435 bufferIndex = 0;
1436
1437 if (bufferMaxIndex < 1) {
1438 *count = 0;
1439 return KERN_RESOURCE_SHORTAGE;
1440 }
1441
1442 /* backtrace kernel */
1443 if (kregs) {
1444 addr64_t address = 0ULL;
1445 size_t size = 0UL;
1446
1447 // do the backtrace
1448 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1449
1450 // and do a nofault read of (r|e)sp
1451 uint64_t rsp = 0ULL;
1452 size = sizeof(uint64_t);
1453
1454 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1455 address = 0ULL;
1456 }
1457
1458 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1459 callstack[bufferIndex++] = (uint64_t)rsp;
1460 }
1461 } else if (regs64) {
1462 uint64_t rsp = 0ULL;
1463
1464 // backtrace the 64bit side.
1465 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1466 bufferMaxIndex - 1, TRUE);
1467
1468 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1469 bufferIndex < bufferMaxIndex) {
1470 callstack[bufferIndex++] = rsp;
1471 }
1472 } else if (regs32) {
1473 uint32_t esp = 0UL;
1474
1475 // backtrace the 32bit side.
1476 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1477 bufferMaxIndex - 1, TRUE);
1478
1479 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1480 bufferIndex < bufferMaxIndex) {
1481 callstack[bufferIndex++] = (uint64_t) esp;
1482 }
1483 } else if (u_regs64 && !kern_only) {
1484 /* backtrace user land */
1485 uint64_t rsp = 0ULL;
1486
1487 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1488 bufferMaxIndex - 1, FALSE);
1489
1490 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1491 bufferIndex < bufferMaxIndex) {
1492 callstack[bufferIndex++] = rsp;
1493 }
1494 } else if (u_regs32 && !kern_only) {
1495 uint32_t esp = 0UL;
1496
1497 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1498 bufferMaxIndex - 1, FALSE);
1499
1500 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1501 bufferIndex < bufferMaxIndex) {
1502 callstack[bufferIndex++] = (uint64_t) esp;
1503 }
1504 }
1505
1506 *count = bufferIndex;
1507 return kr;
1508 }
1509
1510 __private_extern__
1511 kern_return_t
1512 chudxnu_thread_get_callstack64_kperf(
1513 thread_t thread,
1514 uint64_t *callstack,
1515 mach_msg_type_number_t *count,
1516 boolean_t is_user)
1517 {
1518 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1519 }
1520 #else /* !__arm__ && !__arm64__ && !__x86_64__ */
1521 #error kperf: unsupported architecture
1522 #endif /* !__arm__ && !__arm64__ && !__x86_64__ */
mirror of XNU