2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 /* Collect kernel callstacks */
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <vm/vm_map.h>
35 #include <kperf/buffer.h>
36 #include <kperf/context.h>
37 #include <kperf/callstack.h>
38 #include <kperf/ast.h>
39 #include <sys/errno.h>
41 #if defined(__arm__) || defined(__arm64__)
42 #include <arm/cpu_data.h>
43 #include <arm/cpu_data_internal.h>
47 callstack_fixup_user(struct kp_ucallstack
*cs
, thread_t thread
)
49 uint64_t fixup_val
= 0;
50 assert(cs
->kpuc_nframes
< MAX_UCALLSTACK_FRAMES
);
52 #if defined(__x86_64__)
55 x86_saved_state_t
*state
;
57 state
= get_user_regs(thread
);
62 user_64
= is_saved_state64(state
);
64 sp_user
= saved_state64(state
)->isf
.rsp
;
66 sp_user
= saved_state32(state
)->uesp
;
69 if (thread
== current_thread()) {
70 (void)copyin(sp_user
, (char *)&fixup_val
,
71 user_64
? sizeof(uint64_t) : sizeof(uint32_t));
73 (void)vm_map_read_user(get_task_map(get_threadtask(thread
)), sp_user
,
74 &fixup_val
, user_64
? sizeof(uint64_t) : sizeof(uint32_t));
77 #elif defined(__arm64__) || defined(__arm__)
79 struct arm_saved_state
*state
= get_user_regs(thread
);
84 /* encode thumb mode into low bit of PC */
85 if (get_saved_state_cpsr(state
) & PSR_TF
) {
86 cs
->kpuc_frames
[0] |= 1ULL;
89 fixup_val
= get_saved_state_lr(state
);
92 #error "callstack_fixup_user: unsupported architecture"
96 cs
->kpuc_frames
[cs
->kpuc_nframes
++] = fixup_val
;
99 #if defined(__x86_64__)
101 __attribute__((used
))
103 interrupted_kernel_sp_value(uintptr_t *sp_val
)
105 x86_saved_state_t
*state
;
109 uintptr_t top
, bottom
;
111 state
= current_cpu_datap()->cpu_int_state
;
116 state_64
= is_saved_state64(state
);
119 cs
= saved_state64(state
)->isf
.cs
;
121 cs
= saved_state32(state
)->cs
;
123 /* return early if interrupted a thread in user space */
124 if ((cs
& SEL_PL
) == SEL_PL_U
) {
129 sp
= saved_state64(state
)->isf
.rsp
;
131 sp
= saved_state32(state
)->uesp
;
134 /* make sure the stack pointer is pointing somewhere in this stack */
135 bottom
= current_thread()->kernel_stack
;
136 top
= bottom
+ kernel_stack_size
;
137 if (sp
>= bottom
&& sp
< top
) {
141 *sp_val
= *(uintptr_t *)sp
;
145 #elif defined(__arm64__)
147 __attribute__((used
))
149 interrupted_kernel_lr(uintptr_t *lr
)
151 struct arm_saved_state
*state
;
153 state
= getCpuDatap()->cpu_int_state
;
155 /* return early if interrupted a thread in user space */
156 if (PSR64_IS_USER(get_saved_state_cpsr(state
))) {
160 *lr
= get_saved_state_lr(state
);
164 #elif defined(__arm__)
166 __attribute__((used
))
168 interrupted_kernel_lr(uintptr_t *lr
)
170 struct arm_saved_state
*state
;
172 state
= getCpuDatap()->cpu_int_state
;
174 /* return early if interrupted a thread in user space */
175 if (PSR_IS_USER(get_saved_state_cpsr(state
))) {
179 *lr
= get_saved_state_lr(state
);
183 #else /* defined(__arm__) */
184 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
185 #endif /* !defined(__arm__) */
189 callstack_fixup_interrupted(struct kp_kcallstack
*cs
)
191 uintptr_t fixup_val
= 0;
192 assert(cs
->kpkc_nframes
< MAX_KCALLSTACK_FRAMES
);
195 * Only provide arbitrary data on development or debug kernels.
197 #if DEVELOPMENT || DEBUG
198 #if defined(__x86_64__)
199 (void)interrupted_kernel_sp_value(&fixup_val
);
200 #elif defined(__arm64__) || defined(__arm__)
201 (void)interrupted_kernel_lr(&fixup_val
);
202 #endif /* defined(__x86_64__) */
203 #endif /* DEVELOPMENT || DEBUG */
205 assert(cs
->kpkc_flags
& CALLSTACK_KERNEL
);
206 cs
->kpkc_frames
[cs
->kpkc_nframes
++] = fixup_val
;
210 kperf_continuation_sample(struct kp_kcallstack
*cs
, struct kperf_context
*context
)
215 assert(context
!= NULL
);
217 thread
= context
->cur_thread
;
218 assert(thread
!= NULL
);
219 assert(thread
->continuation
!= NULL
);
221 cs
->kpkc_flags
= CALLSTACK_CONTINUATION
| CALLSTACK_VALID
| CALLSTACK_KERNEL
;
223 cs
->kpkc_flags
|= CALLSTACK_64BIT
;
226 cs
->kpkc_nframes
= 1;
227 cs
->kpkc_frames
[0] = VM_KERNEL_UNSLIDE(thread
->continuation
);
231 kperf_backtrace_sample(struct kp_kcallstack
*cs
, struct kperf_context
*context
)
234 assert(context
!= NULL
);
235 assert(context
->cur_thread
== current_thread());
237 cs
->kpkc_flags
= CALLSTACK_KERNEL
| CALLSTACK_KERNEL_WORDS
;
239 cs
->kpkc_flags
|= CALLSTACK_64BIT
;
242 BUF_VERB(PERF_CS_BACKTRACE
| DBG_FUNC_START
, 1);
245 cs
->kpkc_nframes
= backtrace_frame(cs
->kpkc_word_frames
,
246 cs
->kpkc_nframes
- 1, context
->starting_fp
, &trunc
);
247 if (cs
->kpkc_nframes
> 0) {
248 cs
->kpkc_flags
|= CALLSTACK_VALID
;
250 * Fake the value pointed to by the stack pointer or the link
251 * register for symbolicators.
253 cs
->kpkc_word_frames
[cs
->kpkc_nframes
+ 1] = 0;
254 cs
->kpkc_nframes
+= 1;
257 cs
->kpkc_flags
|= CALLSTACK_TRUNCATED
;
260 BUF_VERB(PERF_CS_BACKTRACE
| DBG_FUNC_END
, cs
->kpkc_nframes
);
263 kern_return_t
chudxnu_thread_get_callstack64_kperf(thread_t thread
,
264 uint64_t *callStack
, mach_msg_type_number_t
*count
,
265 boolean_t user_only
);
268 kperf_kcallstack_sample(struct kp_kcallstack
*cs
, struct kperf_context
*context
)
273 assert(context
!= NULL
);
274 assert(cs
->kpkc_nframes
<= MAX_KCALLSTACK_FRAMES
);
276 thread
= context
->cur_thread
;
277 assert(thread
!= NULL
);
279 BUF_INFO(PERF_CS_KSAMPLE
| DBG_FUNC_START
, (uintptr_t)thread_tid(thread
),
282 cs
->kpkc_flags
= CALLSTACK_KERNEL
;
284 cs
->kpkc_flags
|= CALLSTACK_64BIT
;
287 if (ml_at_interrupt_context()) {
288 assert(thread
== current_thread());
289 cs
->kpkc_flags
|= CALLSTACK_KERNEL_WORDS
;
291 cs
->kpkc_nframes
= backtrace_interrupted(
292 cs
->kpkc_word_frames
, cs
->kpkc_nframes
- 1, &trunc
);
293 if (cs
->kpkc_nframes
!= 0) {
294 callstack_fixup_interrupted(cs
);
297 cs
->kpkc_flags
|= CALLSTACK_TRUNCATED
;
301 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
305 kr
= chudxnu_thread_get_callstack64_kperf(thread
,
306 cs
->kpkc_frames
, &cs
->kpkc_nframes
, FALSE
);
307 if (kr
== KERN_SUCCESS
) {
308 cs
->kpkc_flags
|= CALLSTACK_VALID
;
309 } else if (kr
== KERN_RESOURCE_SHORTAGE
) {
310 cs
->kpkc_flags
|= CALLSTACK_VALID
;
311 cs
->kpkc_flags
|= CALLSTACK_TRUNCATED
;
313 cs
->kpkc_nframes
= 0;
317 if (!(cs
->kpkc_flags
& CALLSTACK_VALID
)) {
318 BUF_INFO(PERF_CS_ERROR
, ERR_GETSTACK
);
321 BUF_INFO(PERF_CS_KSAMPLE
| DBG_FUNC_END
, (uintptr_t)thread_tid(thread
),
322 cs
->kpkc_flags
, cs
->kpkc_nframes
);
326 kperf_ucallstack_sample(struct kp_ucallstack
*cs
, struct kperf_context
*context
)
328 assert(ml_get_interrupts_enabled() == TRUE
);
330 thread_t thread
= context
->cur_thread
;
331 assert(thread
!= NULL
);
333 BUF_INFO(PERF_CS_USAMPLE
| DBG_FUNC_START
,
334 (uintptr_t)thread_tid(thread
), cs
->kpuc_nframes
);
340 * Leave space for the fixup information.
342 unsigned int maxnframes
= cs
->kpuc_nframes
- 1;
343 unsigned int nframes
= backtrace_thread_user(thread
, cs
->kpuc_frames
,
344 maxnframes
, &error
, &user64
, &trunc
);
345 cs
->kpuc_nframes
= MIN(maxnframes
, nframes
);
348 * Ignore EFAULT to get as much of the stack as possible. It will be
349 * marked as truncated, below.
351 if (error
== 0 || error
== EFAULT
) {
352 callstack_fixup_user(cs
, thread
);
353 cs
->kpuc_flags
|= CALLSTACK_VALID
;
355 cs
->kpuc_nframes
= 0;
356 BUF_INFO(PERF_CS_ERROR
, ERR_GETSTACK
, error
);
359 cs
->kpuc_flags
|= CALLSTACK_KERNEL_WORDS
| (user64
? CALLSTACK_64BIT
: 0) |
360 (trunc
? CALLSTACK_TRUNCATED
: 0);
362 BUF_INFO(PERF_CS_USAMPLE
| DBG_FUNC_END
, (uintptr_t)thread_tid(thread
),
363 cs
->kpuc_flags
, cs
->kpuc_nframes
);
366 static inline uintptr_t
367 scrub_word(uintptr_t *bt
, int n_frames
, int frame
, bool kern
)
369 if (frame
< n_frames
) {
371 return VM_KERNEL_UNSLIDE(bt
[frame
]);
380 static inline uintptr_t
381 scrub_frame(uint64_t *bt
, int n_frames
, int frame
)
383 if (frame
< n_frames
) {
384 return (uintptr_t)(bt
[frame
]);
391 callstack_log(uint32_t hdrid
, uint32_t dataid
, void *vframes
,
392 unsigned int nframes
, unsigned int flags
)
394 BUF_VERB(PERF_CS_LOG
| DBG_FUNC_START
, flags
, nframes
);
396 BUF_DATA(hdrid
, flags
, nframes
);
398 unsigned int nevts
= nframes
/ 4;
399 unsigned int ovf
= nframes
% 4;
404 bool kern
= flags
& CALLSTACK_KERNEL
;
406 if (flags
& CALLSTACK_KERNEL_WORDS
) {
407 uintptr_t *frames
= vframes
;
408 for (unsigned int i
= 0; i
< nevts
; i
++) {
409 unsigned int j
= i
* 4;
411 scrub_word(frames
, nframes
, j
+ 0, kern
),
412 scrub_word(frames
, nframes
, j
+ 1, kern
),
413 scrub_word(frames
, nframes
, j
+ 2, kern
),
414 scrub_word(frames
, nframes
, j
+ 3, kern
));
417 for (unsigned int i
= 0; i
< nevts
; i
++) {
418 uint64_t *frames
= vframes
;
419 unsigned int j
= i
* 4;
421 scrub_frame(frames
, nframes
, j
+ 0),
422 scrub_frame(frames
, nframes
, j
+ 1),
423 scrub_frame(frames
, nframes
, j
+ 2),
424 scrub_frame(frames
, nframes
, j
+ 3));
428 BUF_VERB(PERF_CS_LOG
| DBG_FUNC_END
, flags
, nframes
);
432 kperf_kcallstack_log(struct kp_kcallstack
*cs
)
434 callstack_log(PERF_CS_KHDR
, PERF_CS_KDATA
, cs
->kpkc_frames
,
435 cs
->kpkc_nframes
, cs
->kpkc_flags
);
439 kperf_ucallstack_log(struct kp_ucallstack
*cs
)
441 callstack_log(PERF_CS_UHDR
, PERF_CS_UDATA
, cs
->kpuc_frames
,
442 cs
->kpuc_nframes
, cs
->kpuc_flags
);
446 kperf_ucallstack_pend(struct kperf_context
* context
, uint32_t depth
,
447 unsigned int actionid
)
452 kperf_ast_set_callstack_depth(context
->cur_thread
, depth
);
453 return kperf_ast_pend(context
->cur_thread
, T_KPERF_AST_CALLSTACK
,
458 chudxnu_kern_read(void *dstaddr
, vm_offset_t srcaddr
, vm_size_t size
)
460 return (ml_nofault_copy(srcaddr
, (vm_offset_t
)dstaddr
, size
) == size
) ?
461 KERN_SUCCESS
: KERN_FAILURE
;
471 //ppc version ported to arm
472 kern_return_t ret
= KERN_SUCCESS
;
474 if (ml_at_interrupt_context()) {
475 return KERN_FAILURE
; // can't look at tasks on interrupt stack
478 if (current_task() == task
) {
479 if (copyin(usraddr
, kernaddr
, size
)) {
483 vm_map_t map
= get_task_map(task
);
484 ret
= vm_map_read_user(map
, usraddr
, kernaddr
, size
);
490 static inline uint64_t
491 chudxnu_vm_unslide( uint64_t ptr
, int kaddr
)
497 return VM_KERNEL_UNSLIDE(ptr
);
501 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
502 #define CS_FLAG_EXTRASP 1 // capture extra sp register
504 chudxnu_thread_get_callstack64_internal(
507 mach_msg_type_number_t
*count
,
513 uint64_t currPC
= 0ULL, currLR
= 0ULL, currSP
= 0ULL;
514 uint64_t prevPC
= 0ULL;
515 uint32_t kernStackMin
= thread
->kernel_stack
;
516 uint32_t kernStackMax
= kernStackMin
+ kernel_stack_size
;
517 uint64_t *buffer
= callStack
;
520 int bufferMaxIndex
= 0;
521 boolean_t supervisor
= FALSE
;
522 struct arm_saved_state
*state
= NULL
;
523 uint32_t *fp
= NULL
, *nextFramePointer
= NULL
, *topfp
= NULL
;
526 task
= get_threadtask(thread
);
528 bufferMaxIndex
= *count
;
531 state
= find_user_regs(thread
);
533 state
= find_kern_regs(thread
);
541 /* make sure it is safe to dereference before you do it */
542 supervisor
= ARM_SUPERVISOR_MODE(state
->cpsr
);
544 /* can't take a kernel callstack if we've got a user frame */
545 if (!user_only
&& !supervisor
) {
550 * Reserve space for saving LR (and sometimes SP) at the end of the
553 if (flags
& CS_FLAG_EXTRASP
) {
559 if (bufferMaxIndex
< 2) {
561 return KERN_RESOURCE_SHORTAGE
;
564 currPC
= (uint64_t)state
->pc
; /* r15 */
565 if (state
->cpsr
& PSR_TF
) {
566 currPC
|= 1ULL; /* encode thumb mode into low bit of PC */
568 currLR
= (uint64_t)state
->lr
; /* r14 */
569 currSP
= (uint64_t)state
->sp
; /* r13 */
571 fp
= (uint32_t *)state
->r
[7]; /* frame pointer */
574 bufferIndex
= 0; // start with a stack of size zero
575 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currPC
, supervisor
); // save PC in position 0.
577 // Now, fill buffer with stack backtraces.
578 while (bufferIndex
< bufferMaxIndex
) {
581 * Below the frame pointer, the following values are saved:
586 * Note that we read the pc even for the first stack frame
587 * (which, in theory, is always empty because the callee fills
588 * it in just before it lowers the stack. However, if we
589 * catch the program in between filling in the return address
590 * and lowering the stack, we want to still have a valid
591 * backtrace. FixupStack correctly disregards this value if
595 if ((uint32_t)fp
== 0 || ((uint32_t)fp
& 0x3) != 0) {
596 /* frame pointer is invalid - stop backtracing */
602 if (((uint32_t)fp
> kernStackMax
) ||
603 ((uint32_t)fp
< kernStackMin
)) {
606 kr
= chudxnu_kern_read(&frame
,
608 (vm_size_t
)sizeof(frame
));
609 if (kr
== KERN_SUCCESS
) {
610 pc
= (uint64_t)frame
[1];
611 nextFramePointer
= (uint32_t *) (frame
[0]);
614 nextFramePointer
= 0ULL;
619 kr
= chudxnu_task_read(task
,
621 (((uint64_t)(uint32_t)fp
) & 0x00000000FFFFFFFFULL
),
623 if (kr
== KERN_SUCCESS
) {
624 pc
= (uint64_t) frame
[1];
625 nextFramePointer
= (uint32_t *) (frame
[0]);
628 nextFramePointer
= 0ULL;
633 if (kr
!= KERN_SUCCESS
) {
638 if (nextFramePointer
) {
639 buffer
[bufferIndex
++] = chudxnu_vm_unslide(pc
, supervisor
);
643 if (nextFramePointer
< fp
) {
646 fp
= nextFramePointer
;
650 if (bufferIndex
>= bufferMaxIndex
) {
651 bufferIndex
= bufferMaxIndex
;
652 kr
= KERN_RESOURCE_SHORTAGE
;
657 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
658 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currLR
, supervisor
);
659 if (flags
& CS_FLAG_EXTRASP
) {
660 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currSP
, supervisor
);
663 *count
= bufferIndex
;
668 chudxnu_thread_get_callstack64_kperf(
671 mach_msg_type_number_t
*count
,
674 return chudxnu_thread_get_callstack64_internal( thread
, callStack
, count
, user_only
, 0 );
678 #if defined(HAS_APPLE_PAC)
682 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
683 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
684 // after sampling has finished.
686 // For an N-entry callstack:
689 // [1..N-3] stack frames (including current one)
690 // [N-2] current LR (return value if we're in a leaf function)
691 // [N-1] current r0 (in case we've saved LR in r0) (optional)
694 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
696 #define CS_FLAG_EXTRASP 1 // capture extra sp register
699 chudxnu_thread_get_callstack64_internal(
702 mach_msg_type_number_t
*count
,
706 kern_return_t kr
= KERN_SUCCESS
;
708 uint64_t currPC
= 0ULL, currLR
= 0ULL, currSP
= 0ULL;
709 uint64_t prevPC
= 0ULL;
710 uint64_t kernStackMin
= thread
->kernel_stack
;
711 uint64_t kernStackMax
= kernStackMin
+ kernel_stack_size
;
712 uint64_t *buffer
= callStack
;
714 int bufferMaxIndex
= 0;
715 boolean_t kernel
= FALSE
;
716 struct arm_saved_state
*sstate
= NULL
;
719 task
= get_threadtask(thread
);
720 bufferMaxIndex
= *count
;
723 sstate
= find_user_regs(thread
);
725 sstate
= find_kern_regs(thread
);
733 if (is_saved_state64(sstate
)) {
734 struct arm_saved_state64
*state
= NULL
;
735 uint64_t *fp
= NULL
, *nextFramePointer
= NULL
, *topfp
= NULL
;
738 state
= saved_state64(sstate
);
740 /* make sure it is safe to dereference before you do it */
741 kernel
= PSR64_IS_KERNEL(state
->cpsr
);
743 /* can't take a kernel callstack if we've got a user frame */
744 if (!user_only
&& !kernel
) {
749 * Reserve space for saving LR (and sometimes SP) at the end of the
752 if (flags
& CS_FLAG_EXTRASP
) {
758 if (bufferMaxIndex
< 2) {
760 return KERN_RESOURCE_SHORTAGE
;
767 fp
= (uint64_t *)state
->fp
; /* frame pointer */
770 bufferIndex
= 0; // start with a stack of size zero
771 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currPC
, kernel
); // save PC in position 0.
773 BUF_VERB(PERF_CS_BACKTRACE
| DBG_FUNC_START
, kernel
, 0);
775 // Now, fill buffer with stack backtraces.
776 while (bufferIndex
< bufferMaxIndex
) {
779 * Below the frame pointer, the following values are saved:
784 * Note that we read the pc even for the first stack frame
785 * (which, in theory, is always empty because the callee fills
786 * it in just before it lowers the stack. However, if we
787 * catch the program in between filling in the return address
788 * and lowering the stack, we want to still have a valid
789 * backtrace. FixupStack correctly disregards this value if
793 if ((uint64_t)fp
== 0 || ((uint64_t)fp
& 0x3) != 0) {
794 /* frame pointer is invalid - stop backtracing */
800 if (((uint64_t)fp
> kernStackMax
) ||
801 ((uint64_t)fp
< kernStackMin
)) {
804 kr
= chudxnu_kern_read(&frame
,
806 (vm_size_t
)sizeof(frame
));
807 if (kr
== KERN_SUCCESS
) {
808 #if defined(HAS_APPLE_PAC)
809 /* return addresses on stack will be signed by arm64e ABI */
810 pc
= (uint64_t)ptrauth_strip((void *)frame
[1], ptrauth_key_return_address
);
814 nextFramePointer
= (uint64_t *)frame
[0];
817 nextFramePointer
= 0ULL;
822 kr
= chudxnu_task_read(task
,
825 (vm_size_t
)sizeof(frame
));
826 if (kr
== KERN_SUCCESS
) {
827 #if defined(HAS_APPLE_PAC)
828 /* return addresses on stack will be signed by arm64e ABI */
829 pc
= (uint64_t)ptrauth_strip((void *)frame
[1], ptrauth_key_return_address
);
833 nextFramePointer
= (uint64_t *)(frame
[0]);
836 nextFramePointer
= 0ULL;
841 if (kr
!= KERN_SUCCESS
) {
846 if (nextFramePointer
) {
847 buffer
[bufferIndex
++] = chudxnu_vm_unslide(pc
, kernel
);
851 if (nextFramePointer
< fp
) {
854 fp
= nextFramePointer
;
858 BUF_VERB(PERF_CS_BACKTRACE
| DBG_FUNC_END
, bufferIndex
);
860 if (bufferIndex
>= bufferMaxIndex
) {
861 bufferIndex
= bufferMaxIndex
;
862 kr
= KERN_RESOURCE_SHORTAGE
;
867 // Save link register and SP at bottom of stack (used for later fixup).
868 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currLR
, kernel
);
869 if (flags
& CS_FLAG_EXTRASP
) {
870 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currSP
, kernel
);
873 struct arm_saved_state32
*state
= NULL
;
874 uint32_t *fp
= NULL
, *nextFramePointer
= NULL
, *topfp
= NULL
;
876 /* 64-bit kernel stacks, 32-bit user stacks */
880 state
= saved_state32(sstate
);
882 /* make sure it is safe to dereference before you do it */
883 kernel
= ARM_SUPERVISOR_MODE(state
->cpsr
);
885 /* can't take a kernel callstack if we've got a user frame */
886 if (!user_only
&& !kernel
) {
891 * Reserve space for saving LR (and sometimes SP) at the end of the
894 if (flags
& CS_FLAG_EXTRASP
) {
900 if (bufferMaxIndex
< 2) {
902 return KERN_RESOURCE_SHORTAGE
;
905 currPC
= (uint64_t)state
->pc
; /* r15 */
906 if (state
->cpsr
& PSR_TF
) {
907 currPC
|= 1ULL; /* encode thumb mode into low bit of PC */
909 currLR
= (uint64_t)state
->lr
; /* r14 */
910 currSP
= (uint64_t)state
->sp
; /* r13 */
912 fp
= (uint32_t *)(uintptr_t)state
->r
[7]; /* frame pointer */
915 bufferIndex
= 0; // start with a stack of size zero
916 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currPC
, kernel
); // save PC in position 0.
918 BUF_VERB(PERF_CS_BACKTRACE
| DBG_FUNC_START
, kernel
, 1);
920 // Now, fill buffer with stack backtraces.
921 while (bufferIndex
< bufferMaxIndex
) {
924 * Below the frame pointer, the following values are saved:
929 * Note that we read the pc even for the first stack frame
930 * (which, in theory, is always empty because the callee fills
931 * it in just before it lowers the stack. However, if we
932 * catch the program in between filling in the return address
933 * and lowering the stack, we want to still have a valid
934 * backtrace. FixupStack correctly disregards this value if
938 if ((uint32_t)fp
== 0 || ((uint32_t)fp
& 0x3) != 0) {
939 /* frame pointer is invalid - stop backtracing */
945 if (((uint32_t)fp
> kernStackMax
) ||
946 ((uint32_t)fp
< kernStackMin
)) {
949 kr
= chudxnu_kern_read(&frame
,
951 (vm_size_t
)sizeof(frame
));
952 if (kr
== KERN_SUCCESS
) {
953 pc
= (uint64_t)frame
[1];
954 nextFramePointer
= (uint32_t *) (frame
[0]);
957 nextFramePointer
= 0ULL;
962 kr
= chudxnu_task_read(task
,
964 (((uint64_t)(uint32_t)fp
) & 0x00000000FFFFFFFFULL
),
966 if (kr
== KERN_SUCCESS
) {
967 pc
= (uint64_t)frame32
[1];
968 nextFramePointer
= (uint32_t *)(uintptr_t)(frame32
[0]);
971 nextFramePointer
= 0ULL;
976 if (kr
!= KERN_SUCCESS
) {
981 if (nextFramePointer
) {
982 buffer
[bufferIndex
++] = chudxnu_vm_unslide(pc
, kernel
);
986 if (nextFramePointer
< fp
) {
989 fp
= nextFramePointer
;
993 BUF_VERB(PERF_CS_BACKTRACE
| DBG_FUNC_END
, bufferIndex
);
995 /* clamp callstack size to max */
996 if (bufferIndex
>= bufferMaxIndex
) {
997 bufferIndex
= bufferMaxIndex
;
998 kr
= KERN_RESOURCE_SHORTAGE
;
1000 /* ignore all other failures */
1004 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
1005 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currLR
, kernel
);
1006 if (flags
& CS_FLAG_EXTRASP
) {
1007 buffer
[bufferIndex
++] = chudxnu_vm_unslide(currSP
, kernel
);
1011 *count
= bufferIndex
;
1016 chudxnu_thread_get_callstack64_kperf(
1018 uint64_t *callStack
,
1019 mach_msg_type_number_t
*count
,
1020 boolean_t user_only
)
1022 return chudxnu_thread_get_callstack64_internal( thread
, callStack
, count
, user_only
, 0 );
1026 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
1027 // don't try to read in the hole
1028 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
1029 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
1030 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
1032 typedef struct _cframe64_t
{
1033 uint64_t prevFP
; // can't use a real pointer here until we're a 64 bit kernel
1039 typedef struct _cframe_t
{
1040 uint32_t prev
; // this is really a user32-space pointer to the previous frame
1045 extern void * find_user_regs(thread_t
);
1046 extern x86_saved_state32_t
*find_kern_regs(thread_t
);
1048 static kern_return_t
1049 do_kernel_backtrace(
1051 struct x86_kernel_state
*regs
,
1053 mach_msg_type_number_t
*start_idx
,
1054 mach_msg_type_number_t max_idx
)
1056 uint64_t kernStackMin
= (uint64_t)thread
->kernel_stack
;
1057 uint64_t kernStackMax
= (uint64_t)kernStackMin
+ kernel_stack_size
;
1058 mach_msg_type_number_t ct
= *start_idx
;
1059 kern_return_t kr
= KERN_FAILURE
;
1062 uint64_t currPC
= 0ULL;
1063 uint64_t currFP
= 0ULL;
1064 uint64_t prevPC
= 0ULL;
1065 uint64_t prevFP
= 0ULL;
1066 if (KERN_SUCCESS
!= chudxnu_kern_read(&currPC
, (vm_offset_t
)&(regs
->k_rip
), sizeof(uint64_t))) {
1067 return KERN_FAILURE
;
1069 if (KERN_SUCCESS
!= chudxnu_kern_read(&currFP
, (vm_offset_t
)&(regs
->k_rbp
), sizeof(uint64_t))) {
1070 return KERN_FAILURE
;
1073 uint32_t currPC
= 0U;
1074 uint32_t currFP
= 0U;
1075 uint32_t prevPC
= 0U;
1076 uint32_t prevFP
= 0U;
1077 if (KERN_SUCCESS
!= chudxnu_kern_read(&currPC
, (vm_offset_t
)&(regs
->k_eip
), sizeof(uint32_t))) {
1078 return KERN_FAILURE
;
1080 if (KERN_SUCCESS
!= chudxnu_kern_read(&currFP
, (vm_offset_t
)&(regs
->k_ebp
), sizeof(uint32_t))) {
1081 return KERN_FAILURE
;
1085 if (*start_idx
>= max_idx
) {
1086 return KERN_RESOURCE_SHORTAGE
; // no frames traced
1089 return KERN_FAILURE
;
1092 frames
[ct
++] = chudxnu_vm_unslide((uint64_t)currPC
, 1);
1094 // build a backtrace of this kernel state
1096 while (VALID_STACK_ADDRESS64(TRUE
, currFP
, kernStackMin
, kernStackMax
)) {
1097 // this is the address where caller lives in the user thread
1098 uint64_t caller
= currFP
+ sizeof(uint64_t);
1100 while (VALID_STACK_ADDRESS(TRUE
, currFP
, kernStackMin
, kernStackMax
)) {
1101 uint32_t caller
= (uint32_t)currFP
+ sizeof(uint32_t);
1104 if (!currFP
|| !currPC
) {
1109 if (ct
>= max_idx
) {
1111 return KERN_RESOURCE_SHORTAGE
;
1114 /* read our caller */
1115 kr
= chudxnu_kern_read(&currPC
, (vm_offset_t
)caller
, sizeof(currPC
));
1117 if (kr
!= KERN_SUCCESS
|| !currPC
) {
1123 * retrive contents of the frame pointer and advance to the next stack
1124 * frame if it's valid
1127 kr
= chudxnu_kern_read(&prevFP
, (vm_offset_t
)currFP
, sizeof(currPC
));
1130 if (VALID_STACK_ADDRESS64(TRUE
, prevFP
, kernStackMin
, kernStackMax
)) {
1132 if (VALID_STACK_ADDRESS(TRUE
, prevFP
, kernStackMin
, kernStackMax
)) {
1134 frames
[ct
++] = chudxnu_vm_unslide((uint64_t)currPC
, 1);
1137 if (prevFP
<= currFP
) {
1145 return KERN_SUCCESS
;
1150 static kern_return_t
1154 x86_saved_state32_t
*regs
,
1156 mach_msg_type_number_t
*start_idx
,
1157 mach_msg_type_number_t max_idx
,
1158 boolean_t supervisor
)
1160 uint32_t tmpWord
= 0UL;
1161 uint64_t currPC
= (uint64_t) regs
->eip
;
1162 uint64_t currFP
= (uint64_t) regs
->ebp
;
1163 uint64_t prevPC
= 0ULL;
1164 uint64_t prevFP
= 0ULL;
1165 uint64_t kernStackMin
= thread
->kernel_stack
;
1166 uint64_t kernStackMax
= kernStackMin
+ kernel_stack_size
;
1167 mach_msg_type_number_t ct
= *start_idx
;
1168 kern_return_t kr
= KERN_FAILURE
;
1170 if (ct
>= max_idx
) {
1171 return KERN_RESOURCE_SHORTAGE
; // no frames traced
1173 frames
[ct
++] = chudxnu_vm_unslide(currPC
, supervisor
);
1175 // build a backtrace of this 32 bit state.
1176 while (VALID_STACK_ADDRESS(supervisor
, currFP
, kernStackMin
, kernStackMax
)) {
1177 cframe_t
*fp
= (cframe_t
*) (uintptr_t) currFP
;
1184 if (ct
>= max_idx
) {
1186 return KERN_RESOURCE_SHORTAGE
;
1189 /* read our caller */
1191 kr
= chudxnu_kern_read(&tmpWord
, (vm_offset_t
) &fp
->caller
, sizeof(uint32_t));
1193 kr
= chudxnu_task_read(task
, &tmpWord
, (vm_offset_t
) &fp
->caller
, sizeof(uint32_t));
1196 if (kr
!= KERN_SUCCESS
) {
1201 currPC
= (uint64_t) tmpWord
; // promote 32 bit address
1204 * retrive contents of the frame pointer and advance to the next stack
1205 * frame if it's valid
1209 kr
= chudxnu_kern_read(&tmpWord
, (vm_offset_t
)&fp
->prev
, sizeof(uint32_t));
1211 kr
= chudxnu_task_read(task
, &tmpWord
, (vm_offset_t
)&fp
->prev
, sizeof(uint32_t));
1213 prevFP
= (uint64_t) tmpWord
; // promote 32 bit address
1216 frames
[ct
++] = chudxnu_vm_unslide(currPC
, supervisor
);
1219 if (prevFP
< currFP
) {
1227 return KERN_SUCCESS
;
1230 static kern_return_t
1234 x86_saved_state64_t
*regs
,
1236 mach_msg_type_number_t
*start_idx
,
1237 mach_msg_type_number_t max_idx
,
1238 boolean_t supervisor
)
1240 uint64_t currPC
= regs
->isf
.rip
;
1241 uint64_t currFP
= regs
->rbp
;
1242 uint64_t prevPC
= 0ULL;
1243 uint64_t prevFP
= 0ULL;
1244 uint64_t kernStackMin
= (uint64_t)thread
->kernel_stack
;
1245 uint64_t kernStackMax
= (uint64_t)kernStackMin
+ kernel_stack_size
;
1246 mach_msg_type_number_t ct
= *start_idx
;
1247 kern_return_t kr
= KERN_FAILURE
;
1249 if (*start_idx
>= max_idx
) {
1250 return KERN_RESOURCE_SHORTAGE
; // no frames traced
1252 frames
[ct
++] = chudxnu_vm_unslide(currPC
, supervisor
);
1254 // build a backtrace of this 32 bit state.
1255 while (VALID_STACK_ADDRESS64(supervisor
, currFP
, kernStackMin
, kernStackMax
)) {
1256 // this is the address where caller lives in the user thread
1257 uint64_t caller
= currFP
+ sizeof(uint64_t);
1264 if (ct
>= max_idx
) {
1266 return KERN_RESOURCE_SHORTAGE
;
1269 /* read our caller */
1271 kr
= chudxnu_kern_read(&currPC
, (vm_offset_t
)caller
, sizeof(uint64_t));
1273 kr
= chudxnu_task_read(task
, &currPC
, caller
, sizeof(uint64_t));
1276 if (kr
!= KERN_SUCCESS
) {
1282 * retrive contents of the frame pointer and advance to the next stack
1283 * frame if it's valid
1287 kr
= chudxnu_kern_read(&prevFP
, (vm_offset_t
)currFP
, sizeof(uint64_t));
1289 kr
= chudxnu_task_read(task
, &prevFP
, currFP
, sizeof(uint64_t));
1292 if (VALID_STACK_ADDRESS64(supervisor
, prevFP
, kernStackMin
, kernStackMax
)) {
1293 frames
[ct
++] = chudxnu_vm_unslide(currPC
, supervisor
);
1296 if (prevFP
< currFP
) {
1304 return KERN_SUCCESS
;
1307 static kern_return_t
1308 chudxnu_thread_get_callstack64_internal(
1310 uint64_t *callstack
,
1311 mach_msg_type_number_t
*count
,
1312 boolean_t user_only
,
1313 boolean_t kern_only
)
1315 kern_return_t kr
= KERN_FAILURE
;
1316 task_t task
= thread
->task
;
1317 uint64_t currPC
= 0ULL;
1318 boolean_t supervisor
= FALSE
;
1319 mach_msg_type_number_t bufferIndex
= 0;
1320 mach_msg_type_number_t bufferMaxIndex
= *count
;
1321 x86_saved_state_t
*tagged_regs
= NULL
; // kernel register state
1322 x86_saved_state64_t
*regs64
= NULL
;
1323 x86_saved_state32_t
*regs32
= NULL
;
1324 x86_saved_state32_t
*u_regs32
= NULL
;
1325 x86_saved_state64_t
*u_regs64
= NULL
;
1326 struct x86_kernel_state
*kregs
= NULL
;
1328 if (ml_at_interrupt_context()) {
1330 /* can't backtrace user state on interrupt stack. */
1331 return KERN_FAILURE
;
1334 /* backtracing at interrupt context? */
1335 if (thread
== current_thread() && current_cpu_datap()->cpu_int_state
) {
1337 * Locate the registers for the interrupted thread, assuming it is
1340 tagged_regs
= current_cpu_datap()->cpu_int_state
;
1342 if (is_saved_state64(tagged_regs
)) {
1343 /* 64 bit registers */
1344 regs64
= saved_state64(tagged_regs
);
1345 supervisor
= ((regs64
->isf
.cs
& SEL_PL
) != SEL_PL_U
);
1347 /* 32 bit registers */
1348 regs32
= saved_state32(tagged_regs
);
1349 supervisor
= ((regs32
->cs
& SEL_PL
) != SEL_PL_U
);
1354 if (!ml_at_interrupt_context() && kernel_task
== task
) {
1355 if (!thread
->kernel_stack
) {
1356 return KERN_FAILURE
;
1359 // Kernel thread not at interrupt context
1360 kregs
= (struct x86_kernel_state
*)NULL
;
1362 // nofault read of the thread->kernel_stack pointer
1363 if (KERN_SUCCESS
!= chudxnu_kern_read(&kregs
, (vm_offset_t
)&(thread
->kernel_stack
), sizeof(void *))) {
1364 return KERN_FAILURE
;
1367 // Adjust to find the saved kernel state
1368 kregs
= STACK_IKS((vm_offset_t
)(uintptr_t)kregs
);
1371 } else if (!tagged_regs
) {
1373 * not at interrupt context, or tracing a different thread than
1374 * current_thread() at interrupt context
1376 tagged_regs
= USER_STATE(thread
);
1377 if (is_saved_state64(tagged_regs
)) {
1378 /* 64 bit registers */
1379 regs64
= saved_state64(tagged_regs
);
1380 supervisor
= ((regs64
->isf
.cs
& SEL_PL
) != SEL_PL_U
);
1382 /* 32 bit registers */
1383 regs32
= saved_state32(tagged_regs
);
1384 supervisor
= ((regs32
->cs
& SEL_PL
) != SEL_PL_U
);
1391 // the caller only wants a user callstack.
1393 // bail - we've only got kernel state
1394 return KERN_FAILURE
;
1397 // regs32(64) is not in supervisor mode.
1405 /* we only want to backtrace the user mode */
1406 if (!(u_regs32
|| u_regs64
)) {
1407 /* no user state to look at */
1408 return KERN_FAILURE
;
1413 * Order of preference for top of stack:
1414 * 64 bit kernel state (not likely)
1415 * 32 bit kernel state
1416 * 64 bit user land state
1417 * 32 bit user land state
1422 * nofault read of the registers from the kernel stack (as they can
1423 * disappear on the fly).
1426 if (KERN_SUCCESS
!= chudxnu_kern_read(&currPC
, (vm_offset_t
)&(kregs
->k_rip
), sizeof(uint64_t))) {
1427 return KERN_FAILURE
;
1429 } else if (regs64
) {
1430 currPC
= regs64
->isf
.rip
;
1431 } else if (regs32
) {
1432 currPC
= (uint64_t) regs32
->eip
;
1433 } else if (u_regs64
) {
1434 currPC
= u_regs64
->isf
.rip
;
1435 } else if (u_regs32
) {
1436 currPC
= (uint64_t) u_regs32
->eip
;
1440 /* no top of the stack, bail out */
1441 return KERN_FAILURE
;
1446 if (bufferMaxIndex
< 1) {
1448 return KERN_RESOURCE_SHORTAGE
;
1451 /* backtrace kernel */
1453 addr64_t address
= 0ULL;
1457 kr
= do_kernel_backtrace(thread
, kregs
, callstack
, &bufferIndex
, bufferMaxIndex
);
1459 // and do a nofault read of (r|e)sp
1460 uint64_t rsp
= 0ULL;
1461 size
= sizeof(uint64_t);
1463 if (KERN_SUCCESS
!= chudxnu_kern_read(&address
, (vm_offset_t
)&(kregs
->k_rsp
), size
)) {
1467 if (address
&& KERN_SUCCESS
== chudxnu_kern_read(&rsp
, (vm_offset_t
)address
, size
) && bufferIndex
< bufferMaxIndex
) {
1468 callstack
[bufferIndex
++] = (uint64_t)rsp
;
1470 } else if (regs64
) {
1471 uint64_t rsp
= 0ULL;
1473 // backtrace the 64bit side.
1474 kr
= do_backtrace64(task
, thread
, regs64
, callstack
, &bufferIndex
,
1475 bufferMaxIndex
- 1, TRUE
);
1477 if (KERN_SUCCESS
== chudxnu_kern_read(&rsp
, (vm_offset_t
) regs64
->isf
.rsp
, sizeof(uint64_t)) &&
1478 bufferIndex
< bufferMaxIndex
) {
1479 callstack
[bufferIndex
++] = rsp
;
1481 } else if (regs32
) {
1484 // backtrace the 32bit side.
1485 kr
= do_backtrace32(task
, thread
, regs32
, callstack
, &bufferIndex
,
1486 bufferMaxIndex
- 1, TRUE
);
1488 if (KERN_SUCCESS
== chudxnu_kern_read(&esp
, (vm_offset_t
) regs32
->uesp
, sizeof(uint32_t)) &&
1489 bufferIndex
< bufferMaxIndex
) {
1490 callstack
[bufferIndex
++] = (uint64_t) esp
;
1492 } else if (u_regs64
&& !kern_only
) {
1493 /* backtrace user land */
1494 uint64_t rsp
= 0ULL;
1496 kr
= do_backtrace64(task
, thread
, u_regs64
, callstack
, &bufferIndex
,
1497 bufferMaxIndex
- 1, FALSE
);
1499 if (KERN_SUCCESS
== chudxnu_task_read(task
, &rsp
, (addr64_t
) u_regs64
->isf
.rsp
, sizeof(uint64_t)) &&
1500 bufferIndex
< bufferMaxIndex
) {
1501 callstack
[bufferIndex
++] = rsp
;
1503 } else if (u_regs32
&& !kern_only
) {
1506 kr
= do_backtrace32(task
, thread
, u_regs32
, callstack
, &bufferIndex
,
1507 bufferMaxIndex
- 1, FALSE
);
1509 if (KERN_SUCCESS
== chudxnu_task_read(task
, &esp
, (addr64_t
) u_regs32
->uesp
, sizeof(uint32_t)) &&
1510 bufferIndex
< bufferMaxIndex
) {
1511 callstack
[bufferIndex
++] = (uint64_t) esp
;
1515 *count
= bufferIndex
;
1521 chudxnu_thread_get_callstack64_kperf(
1523 uint64_t *callstack
,
1524 mach_msg_type_number_t
*count
,
1527 return chudxnu_thread_get_callstack64_internal(thread
, callstack
, count
, is_user
, !is_user
);
1529 #else /* !__arm__ && !__arm64__ && !__x86_64__ */
1530 #error kperf: unsupported architecture
1531 #endif /* !__arm__ && !__arm64__ && !__x86_64__ */