]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kperf/callstack.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.141.1.tar.gz
[apple/xnu.git] / osfmk / kperf / callstack.c
1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Collect kernel callstacks */
30
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <vm/vm_map.h>
35 #include <kperf/buffer.h>
36 #include <kperf/context.h>
37 #include <kperf/callstack.h>
38 #include <kperf/ast.h>
39 #include <sys/errno.h>
40
41 #if defined(__arm__) || defined(__arm64__)
42 #include <arm/cpu_data.h>
43 #include <arm/cpu_data_internal.h>
44 #endif
45
46 static void
47 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
48 {
49 uint64_t fixup_val = 0;
50 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
51
52 #if defined(__x86_64__)
53 user_addr_t sp_user;
54 bool user_64;
55 x86_saved_state_t *state;
56
57 state = get_user_regs(thread);
58 if (!state) {
59 goto out;
60 }
61
62 user_64 = is_saved_state64(state);
63 if (user_64) {
64 sp_user = saved_state64(state)->isf.rsp;
65 } else {
66 sp_user = saved_state32(state)->uesp;
67 }
68
69 if (thread == current_thread()) {
70 (void)copyin(sp_user, (char *)&fixup_val,
71 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
72 } else {
73 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
74 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
75 }
76
77 #elif defined(__arm64__) || defined(__arm__)
78
79 struct arm_saved_state *state = get_user_regs(thread);
80 if (!state) {
81 goto out;
82 }
83
84 /* encode thumb mode into low bit of PC */
85 if (get_saved_state_cpsr(state) & PSR_TF) {
86 cs->kpuc_frames[0] |= 1ULL;
87 }
88
89 fixup_val = get_saved_state_lr(state);
90
91 #else
92 #error "callstack_fixup_user: unsupported architecture"
93 #endif
94
95 out:
96 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
97 }
98
99 #if defined(__x86_64__)
100
101 __attribute__((used))
102 static kern_return_t
103 interrupted_kernel_sp_value(uintptr_t *sp_val)
104 {
105 x86_saved_state_t *state;
106 uintptr_t sp;
107 bool state_64;
108 uint64_t cs;
109 uintptr_t top, bottom;
110
111 state = current_cpu_datap()->cpu_int_state;
112 if (!state) {
113 return KERN_FAILURE;
114 }
115
116 state_64 = is_saved_state64(state);
117
118 if (state_64) {
119 cs = saved_state64(state)->isf.cs;
120 } else {
121 cs = saved_state32(state)->cs;
122 }
123 /* return early if interrupted a thread in user space */
124 if ((cs & SEL_PL) == SEL_PL_U) {
125 return KERN_FAILURE;
126 }
127
128 if (state_64) {
129 sp = saved_state64(state)->isf.rsp;
130 } else {
131 sp = saved_state32(state)->uesp;
132 }
133
134 /* make sure the stack pointer is pointing somewhere in this stack */
135 bottom = current_thread()->kernel_stack;
136 top = bottom + kernel_stack_size;
137 if (sp >= bottom && sp < top) {
138 return KERN_FAILURE;
139 }
140
141 *sp_val = *(uintptr_t *)sp;
142 return KERN_SUCCESS;
143 }
144
145 #elif defined(__arm64__)
146
147 __attribute__((used))
148 static kern_return_t
149 interrupted_kernel_lr(uintptr_t *lr)
150 {
151 struct arm_saved_state *state;
152
153 state = getCpuDatap()->cpu_int_state;
154
155 /* return early if interrupted a thread in user space */
156 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
157 return KERN_FAILURE;
158 }
159
160 *lr = get_saved_state_lr(state);
161 return KERN_SUCCESS;
162 }
163
164 #elif defined(__arm__)
165
166 __attribute__((used))
167 static kern_return_t
168 interrupted_kernel_lr(uintptr_t *lr)
169 {
170 struct arm_saved_state *state;
171
172 state = getCpuDatap()->cpu_int_state;
173
174 /* return early if interrupted a thread in user space */
175 if (PSR_IS_USER(get_saved_state_cpsr(state))) {
176 return KERN_FAILURE;
177 }
178
179 *lr = get_saved_state_lr(state);
180 return KERN_SUCCESS;
181 }
182
183 #else /* defined(__arm__) */
184 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
185 #endif /* !defined(__arm__) */
186
187
188 static void
189 callstack_fixup_interrupted(struct kp_kcallstack *cs)
190 {
191 uintptr_t fixup_val = 0;
192 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
193
194 /*
195 * Only provide arbitrary data on development or debug kernels.
196 */
197 #if DEVELOPMENT || DEBUG
198 #if defined(__x86_64__)
199 (void)interrupted_kernel_sp_value(&fixup_val);
200 #elif defined(__arm64__) || defined(__arm__)
201 (void)interrupted_kernel_lr(&fixup_val);
202 #endif /* defined(__x86_64__) */
203 #endif /* DEVELOPMENT || DEBUG */
204
205 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
206 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
207 }
208
209 void
210 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
211 {
212 thread_t thread;
213
214 assert(cs != NULL);
215 assert(context != NULL);
216
217 thread = context->cur_thread;
218 assert(thread != NULL);
219 assert(thread->continuation != NULL);
220
221 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
222 #ifdef __LP64__
223 cs->kpkc_flags |= CALLSTACK_64BIT;
224 #endif
225
226 cs->kpkc_nframes = 1;
227 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
228 }
229
230 void
231 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
232 {
233 assert(cs != NULL);
234 assert(context != NULL);
235 assert(context->cur_thread == current_thread());
236
237 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
238 #ifdef __LP64__
239 cs->kpkc_flags |= CALLSTACK_64BIT;
240 #endif
241
242 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
243
244 bool trunc = false;
245 cs->kpkc_nframes = backtrace_frame(cs->kpkc_word_frames,
246 cs->kpkc_nframes - 1, context->starting_fp, &trunc);
247 if (cs->kpkc_nframes > 0) {
248 cs->kpkc_flags |= CALLSTACK_VALID;
249 /*
250 * Fake the value pointed to by the stack pointer or the link
251 * register for symbolicators.
252 */
253 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
254 cs->kpkc_nframes += 1;
255 }
256 if (trunc) {
257 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
258 }
259
260 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
261 }
262
263 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
264 uint64_t *callStack, mach_msg_type_number_t *count,
265 boolean_t user_only);
266
267 void
268 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
269 {
270 thread_t thread;
271
272 assert(cs != NULL);
273 assert(context != NULL);
274 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
275
276 thread = context->cur_thread;
277 assert(thread != NULL);
278
279 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
280 cs->kpkc_nframes);
281
282 cs->kpkc_flags = CALLSTACK_KERNEL;
283 #ifdef __LP64__
284 cs->kpkc_flags |= CALLSTACK_64BIT;
285 #endif
286
287 if (ml_at_interrupt_context()) {
288 assert(thread == current_thread());
289 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
290 bool trunc = false;
291 cs->kpkc_nframes = backtrace_interrupted(
292 cs->kpkc_word_frames, cs->kpkc_nframes - 1, &trunc);
293 if (cs->kpkc_nframes != 0) {
294 callstack_fixup_interrupted(cs);
295 }
296 if (trunc) {
297 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
298 }
299 } else {
300 /*
301 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
302 * other threads.
303 */
304 kern_return_t kr;
305 kr = chudxnu_thread_get_callstack64_kperf(thread,
306 cs->kpkc_frames, &cs->kpkc_nframes, FALSE);
307 if (kr == KERN_SUCCESS) {
308 cs->kpkc_flags |= CALLSTACK_VALID;
309 } else if (kr == KERN_RESOURCE_SHORTAGE) {
310 cs->kpkc_flags |= CALLSTACK_VALID;
311 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
312 } else {
313 cs->kpkc_nframes = 0;
314 }
315 }
316
317 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
318 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
319 }
320
321 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
322 cs->kpkc_flags, cs->kpkc_nframes);
323 }
324
325 void
326 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
327 {
328 assert(ml_get_interrupts_enabled() == TRUE);
329
330 thread_t thread = context->cur_thread;
331 assert(thread != NULL);
332
333 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
334 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
335
336 bool user64 = false;
337 bool trunc = false;
338 int error = 0;
339 /*
340 * Leave space for the fixup information.
341 */
342 unsigned int maxnframes = cs->kpuc_nframes - 1;
343 unsigned int nframes = backtrace_thread_user(thread, cs->kpuc_frames,
344 maxnframes, &error, &user64, &trunc);
345 cs->kpuc_nframes = MIN(maxnframes, nframes);
346
347 /*
348 * Ignore EFAULT to get as much of the stack as possible. It will be
349 * marked as truncated, below.
350 */
351 if (error == 0 || error == EFAULT) {
352 callstack_fixup_user(cs, thread);
353 cs->kpuc_flags |= CALLSTACK_VALID;
354 } else {
355 cs->kpuc_nframes = 0;
356 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, error);
357 }
358
359 cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS | (user64 ? CALLSTACK_64BIT : 0) |
360 (trunc ? CALLSTACK_TRUNCATED : 0);
361
362 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
363 cs->kpuc_flags, cs->kpuc_nframes);
364 }
365
366 static inline uintptr_t
367 scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
368 {
369 if (frame < n_frames) {
370 if (kern) {
371 return VM_KERNEL_UNSLIDE(bt[frame]);
372 } else {
373 return bt[frame];
374 }
375 } else {
376 return 0;
377 }
378 }
379
380 static inline uintptr_t
381 scrub_frame(uint64_t *bt, int n_frames, int frame)
382 {
383 if (frame < n_frames) {
384 return (uintptr_t)(bt[frame]);
385 } else {
386 return 0;
387 }
388 }
389
390 static void
391 callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
392 unsigned int nframes, unsigned int flags)
393 {
394 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
395
396 BUF_DATA(hdrid, flags, nframes);
397
398 unsigned int nevts = nframes / 4;
399 unsigned int ovf = nframes % 4;
400 if (ovf != 0) {
401 nevts++;
402 }
403
404 bool kern = flags & CALLSTACK_KERNEL;
405
406 if (flags & CALLSTACK_KERNEL_WORDS) {
407 uintptr_t *frames = vframes;
408 for (unsigned int i = 0; i < nevts; i++) {
409 unsigned int j = i * 4;
410 BUF_DATA(dataid,
411 scrub_word(frames, nframes, j + 0, kern),
412 scrub_word(frames, nframes, j + 1, kern),
413 scrub_word(frames, nframes, j + 2, kern),
414 scrub_word(frames, nframes, j + 3, kern));
415 }
416 } else {
417 for (unsigned int i = 0; i < nevts; i++) {
418 uint64_t *frames = vframes;
419 unsigned int j = i * 4;
420 BUF_DATA(dataid,
421 scrub_frame(frames, nframes, j + 0),
422 scrub_frame(frames, nframes, j + 1),
423 scrub_frame(frames, nframes, j + 2),
424 scrub_frame(frames, nframes, j + 3));
425 }
426 }
427
428 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
429 }
430
431 void
432 kperf_kcallstack_log(struct kp_kcallstack *cs)
433 {
434 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
435 cs->kpkc_nframes, cs->kpkc_flags);
436 }
437
438 void
439 kperf_ucallstack_log(struct kp_ucallstack *cs)
440 {
441 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
442 cs->kpuc_nframes, cs->kpuc_flags);
443 }
444
445 int
446 kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
447 unsigned int actionid)
448 {
449 if (depth < 2) {
450 panic("HUH");
451 }
452 kperf_ast_set_callstack_depth(context->cur_thread, depth);
453 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
454 actionid);
455 }
456
457 static kern_return_t
458 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
459 {
460 return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
461 KERN_SUCCESS : KERN_FAILURE;
462 }
463
464 static kern_return_t
465 chudxnu_task_read(
466 task_t task,
467 void *kernaddr,
468 uint64_t usraddr,
469 vm_size_t size)
470 {
471 //ppc version ported to arm
472 kern_return_t ret = KERN_SUCCESS;
473
474 if (ml_at_interrupt_context()) {
475 return KERN_FAILURE; // can't look at tasks on interrupt stack
476 }
477
478 if (current_task() == task) {
479 if (copyin(usraddr, kernaddr, size)) {
480 ret = KERN_FAILURE;
481 }
482 } else {
483 vm_map_t map = get_task_map(task);
484 ret = vm_map_read_user(map, usraddr, kernaddr, size);
485 }
486
487 return ret;
488 }
489
490 static inline uint64_t
491 chudxnu_vm_unslide( uint64_t ptr, int kaddr )
492 {
493 if (!kaddr) {
494 return ptr;
495 }
496
497 return VM_KERNEL_UNSLIDE(ptr);
498 }
499
500 #if __arm__
501 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
502 #define CS_FLAG_EXTRASP 1 // capture extra sp register
503 static kern_return_t
504 chudxnu_thread_get_callstack64_internal(
505 thread_t thread,
506 uint64_t *callStack,
507 mach_msg_type_number_t *count,
508 boolean_t user_only,
509 int flags)
510 {
511 kern_return_t kr;
512 task_t task;
513 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
514 uint64_t prevPC = 0ULL;
515 uint32_t kernStackMin = thread->kernel_stack;
516 uint32_t kernStackMax = kernStackMin + kernel_stack_size;
517 uint64_t *buffer = callStack;
518 uint32_t frame[2];
519 int bufferIndex = 0;
520 int bufferMaxIndex = 0;
521 boolean_t supervisor = FALSE;
522 struct arm_saved_state *state = NULL;
523 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
524 uint64_t pc = 0ULL;
525
526 task = get_threadtask(thread);
527
528 bufferMaxIndex = *count;
529 //get thread state
530 if (user_only) {
531 state = find_user_regs(thread);
532 } else {
533 state = find_kern_regs(thread);
534 }
535
536 if (!state) {
537 *count = 0;
538 return KERN_FAILURE;
539 }
540
541 /* make sure it is safe to dereference before you do it */
542 supervisor = ARM_SUPERVISOR_MODE(state->cpsr);
543
544 /* can't take a kernel callstack if we've got a user frame */
545 if (!user_only && !supervisor) {
546 return KERN_FAILURE;
547 }
548
549 /*
550 * Reserve space for saving LR (and sometimes SP) at the end of the
551 * backtrace.
552 */
553 if (flags & CS_FLAG_EXTRASP) {
554 bufferMaxIndex -= 2;
555 } else {
556 bufferMaxIndex -= 1;
557 }
558
559 if (bufferMaxIndex < 2) {
560 *count = 0;
561 return KERN_RESOURCE_SHORTAGE;
562 }
563
564 currPC = (uint64_t)state->pc; /* r15 */
565 if (state->cpsr & PSR_TF) {
566 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
567 }
568 currLR = (uint64_t)state->lr; /* r14 */
569 currSP = (uint64_t)state->sp; /* r13 */
570
571 fp = (uint32_t *)state->r[7]; /* frame pointer */
572 topfp = fp;
573
574 bufferIndex = 0; // start with a stack of size zero
575 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, supervisor); // save PC in position 0.
576
577 // Now, fill buffer with stack backtraces.
578 while (bufferIndex < bufferMaxIndex) {
579 pc = 0ULL;
580 /*
581 * Below the frame pointer, the following values are saved:
582 * -> FP
583 */
584
585 /*
586 * Note that we read the pc even for the first stack frame
587 * (which, in theory, is always empty because the callee fills
588 * it in just before it lowers the stack. However, if we
589 * catch the program in between filling in the return address
590 * and lowering the stack, we want to still have a valid
591 * backtrace. FixupStack correctly disregards this value if
592 * necessary.
593 */
594
595 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
596 /* frame pointer is invalid - stop backtracing */
597 pc = 0ULL;
598 break;
599 }
600
601 if (supervisor) {
602 if (((uint32_t)fp > kernStackMax) ||
603 ((uint32_t)fp < kernStackMin)) {
604 kr = KERN_FAILURE;
605 } else {
606 kr = chudxnu_kern_read(&frame,
607 (vm_offset_t)fp,
608 (vm_size_t)sizeof(frame));
609 if (kr == KERN_SUCCESS) {
610 pc = (uint64_t)frame[1];
611 nextFramePointer = (uint32_t *) (frame[0]);
612 } else {
613 pc = 0ULL;
614 nextFramePointer = 0ULL;
615 kr = KERN_FAILURE;
616 }
617 }
618 } else {
619 kr = chudxnu_task_read(task,
620 &frame,
621 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
622 sizeof(frame));
623 if (kr == KERN_SUCCESS) {
624 pc = (uint64_t) frame[1];
625 nextFramePointer = (uint32_t *) (frame[0]);
626 } else {
627 pc = 0ULL;
628 nextFramePointer = 0ULL;
629 kr = KERN_FAILURE;
630 }
631 }
632
633 if (kr != KERN_SUCCESS) {
634 pc = 0ULL;
635 break;
636 }
637
638 if (nextFramePointer) {
639 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, supervisor);
640 prevPC = pc;
641 }
642
643 if (nextFramePointer < fp) {
644 break;
645 } else {
646 fp = nextFramePointer;
647 }
648 }
649
650 if (bufferIndex >= bufferMaxIndex) {
651 bufferIndex = bufferMaxIndex;
652 kr = KERN_RESOURCE_SHORTAGE;
653 } else {
654 kr = KERN_SUCCESS;
655 }
656
657 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
658 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, supervisor);
659 if (flags & CS_FLAG_EXTRASP) {
660 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, supervisor);
661 }
662
663 *count = bufferIndex;
664 return kr;
665 }
666
667 kern_return_t
668 chudxnu_thread_get_callstack64_kperf(
669 thread_t thread,
670 uint64_t *callStack,
671 mach_msg_type_number_t *count,
672 boolean_t user_only)
673 {
674 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
675 }
676 #elif __arm64__
677
678 #if defined(HAS_APPLE_PAC)
679 #include <ptrauth.h>
680 #endif
681
682 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
683 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
684 // after sampling has finished.
685 //
686 // For an N-entry callstack:
687 //
688 // [0] current pc
689 // [1..N-3] stack frames (including current one)
690 // [N-2] current LR (return value if we're in a leaf function)
691 // [N-1] current r0 (in case we've saved LR in r0) (optional)
692 //
693 //
694 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
695
696 #define CS_FLAG_EXTRASP 1 // capture extra sp register
697
698 static kern_return_t
699 chudxnu_thread_get_callstack64_internal(
700 thread_t thread,
701 uint64_t *callStack,
702 mach_msg_type_number_t *count,
703 boolean_t user_only,
704 int flags)
705 {
706 kern_return_t kr = KERN_SUCCESS;
707 task_t task;
708 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
709 uint64_t prevPC = 0ULL;
710 uint64_t kernStackMin = thread->kernel_stack;
711 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
712 uint64_t *buffer = callStack;
713 int bufferIndex = 0;
714 int bufferMaxIndex = 0;
715 boolean_t kernel = FALSE;
716 struct arm_saved_state *sstate = NULL;
717 uint64_t pc = 0ULL;
718
719 task = get_threadtask(thread);
720 bufferMaxIndex = *count;
721 //get thread state
722 if (user_only) {
723 sstate = find_user_regs(thread);
724 } else {
725 sstate = find_kern_regs(thread);
726 }
727
728 if (!sstate) {
729 *count = 0;
730 return KERN_FAILURE;
731 }
732
733 if (is_saved_state64(sstate)) {
734 struct arm_saved_state64 *state = NULL;
735 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
736 uint64_t frame[2];
737
738 state = saved_state64(sstate);
739
740 /* make sure it is safe to dereference before you do it */
741 kernel = PSR64_IS_KERNEL(state->cpsr);
742
743 /* can't take a kernel callstack if we've got a user frame */
744 if (!user_only && !kernel) {
745 return KERN_FAILURE;
746 }
747
748 /*
749 * Reserve space for saving LR (and sometimes SP) at the end of the
750 * backtrace.
751 */
752 if (flags & CS_FLAG_EXTRASP) {
753 bufferMaxIndex -= 2;
754 } else {
755 bufferMaxIndex -= 1;
756 }
757
758 if (bufferMaxIndex < 2) {
759 *count = 0;
760 return KERN_RESOURCE_SHORTAGE;
761 }
762
763 currPC = state->pc;
764 currLR = state->lr;
765 currSP = state->sp;
766
767 fp = (uint64_t *)state->fp; /* frame pointer */
768 topfp = fp;
769
770 bufferIndex = 0; // start with a stack of size zero
771 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
772
773 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
774
775 // Now, fill buffer with stack backtraces.
776 while (bufferIndex < bufferMaxIndex) {
777 pc = 0ULL;
778 /*
779 * Below the frame pointer, the following values are saved:
780 * -> FP
781 */
782
783 /*
784 * Note that we read the pc even for the first stack frame
785 * (which, in theory, is always empty because the callee fills
786 * it in just before it lowers the stack. However, if we
787 * catch the program in between filling in the return address
788 * and lowering the stack, we want to still have a valid
789 * backtrace. FixupStack correctly disregards this value if
790 * necessary.
791 */
792
793 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
794 /* frame pointer is invalid - stop backtracing */
795 pc = 0ULL;
796 break;
797 }
798
799 if (kernel) {
800 if (((uint64_t)fp > kernStackMax) ||
801 ((uint64_t)fp < kernStackMin)) {
802 kr = KERN_FAILURE;
803 } else {
804 kr = chudxnu_kern_read(&frame,
805 (vm_offset_t)fp,
806 (vm_size_t)sizeof(frame));
807 if (kr == KERN_SUCCESS) {
808 #if defined(HAS_APPLE_PAC)
809 /* return addresses on stack will be signed by arm64e ABI */
810 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
811 #else
812 pc = frame[1];
813 #endif
814 nextFramePointer = (uint64_t *)frame[0];
815 } else {
816 pc = 0ULL;
817 nextFramePointer = 0ULL;
818 kr = KERN_FAILURE;
819 }
820 }
821 } else {
822 kr = chudxnu_task_read(task,
823 &frame,
824 (vm_offset_t)fp,
825 (vm_size_t)sizeof(frame));
826 if (kr == KERN_SUCCESS) {
827 #if defined(HAS_APPLE_PAC)
828 /* return addresses on stack will be signed by arm64e ABI */
829 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
830 #else
831 pc = frame[1];
832 #endif
833 nextFramePointer = (uint64_t *)(frame[0]);
834 } else {
835 pc = 0ULL;
836 nextFramePointer = 0ULL;
837 kr = KERN_FAILURE;
838 }
839 }
840
841 if (kr != KERN_SUCCESS) {
842 pc = 0ULL;
843 break;
844 }
845
846 if (nextFramePointer) {
847 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
848 prevPC = pc;
849 }
850
851 if (nextFramePointer < fp) {
852 break;
853 } else {
854 fp = nextFramePointer;
855 }
856 }
857
858 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
859
860 if (bufferIndex >= bufferMaxIndex) {
861 bufferIndex = bufferMaxIndex;
862 kr = KERN_RESOURCE_SHORTAGE;
863 } else {
864 kr = KERN_SUCCESS;
865 }
866
867 // Save link register and SP at bottom of stack (used for later fixup).
868 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
869 if (flags & CS_FLAG_EXTRASP) {
870 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
871 }
872 } else {
873 struct arm_saved_state32 *state = NULL;
874 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
875
876 /* 64-bit kernel stacks, 32-bit user stacks */
877 uint64_t frame[2];
878 uint32_t frame32[2];
879
880 state = saved_state32(sstate);
881
882 /* make sure it is safe to dereference before you do it */
883 kernel = ARM_SUPERVISOR_MODE(state->cpsr);
884
885 /* can't take a kernel callstack if we've got a user frame */
886 if (!user_only && !kernel) {
887 return KERN_FAILURE;
888 }
889
890 /*
891 * Reserve space for saving LR (and sometimes SP) at the end of the
892 * backtrace.
893 */
894 if (flags & CS_FLAG_EXTRASP) {
895 bufferMaxIndex -= 2;
896 } else {
897 bufferMaxIndex -= 1;
898 }
899
900 if (bufferMaxIndex < 2) {
901 *count = 0;
902 return KERN_RESOURCE_SHORTAGE;
903 }
904
905 currPC = (uint64_t)state->pc; /* r15 */
906 if (state->cpsr & PSR_TF) {
907 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
908 }
909 currLR = (uint64_t)state->lr; /* r14 */
910 currSP = (uint64_t)state->sp; /* r13 */
911
912 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
913 topfp = fp;
914
915 bufferIndex = 0; // start with a stack of size zero
916 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
917
918 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
919
920 // Now, fill buffer with stack backtraces.
921 while (bufferIndex < bufferMaxIndex) {
922 pc = 0ULL;
923 /*
924 * Below the frame pointer, the following values are saved:
925 * -> FP
926 */
927
928 /*
929 * Note that we read the pc even for the first stack frame
930 * (which, in theory, is always empty because the callee fills
931 * it in just before it lowers the stack. However, if we
932 * catch the program in between filling in the return address
933 * and lowering the stack, we want to still have a valid
934 * backtrace. FixupStack correctly disregards this value if
935 * necessary.
936 */
937
938 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
939 /* frame pointer is invalid - stop backtracing */
940 pc = 0ULL;
941 break;
942 }
943
944 if (kernel) {
945 if (((uint32_t)fp > kernStackMax) ||
946 ((uint32_t)fp < kernStackMin)) {
947 kr = KERN_FAILURE;
948 } else {
949 kr = chudxnu_kern_read(&frame,
950 (vm_offset_t)fp,
951 (vm_size_t)sizeof(frame));
952 if (kr == KERN_SUCCESS) {
953 pc = (uint64_t)frame[1];
954 nextFramePointer = (uint32_t *) (frame[0]);
955 } else {
956 pc = 0ULL;
957 nextFramePointer = 0ULL;
958 kr = KERN_FAILURE;
959 }
960 }
961 } else {
962 kr = chudxnu_task_read(task,
963 &frame32,
964 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
965 sizeof(frame32));
966 if (kr == KERN_SUCCESS) {
967 pc = (uint64_t)frame32[1];
968 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
969 } else {
970 pc = 0ULL;
971 nextFramePointer = 0ULL;
972 kr = KERN_FAILURE;
973 }
974 }
975
976 if (kr != KERN_SUCCESS) {
977 pc = 0ULL;
978 break;
979 }
980
981 if (nextFramePointer) {
982 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
983 prevPC = pc;
984 }
985
986 if (nextFramePointer < fp) {
987 break;
988 } else {
989 fp = nextFramePointer;
990 }
991 }
992
993 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
994
995 /* clamp callstack size to max */
996 if (bufferIndex >= bufferMaxIndex) {
997 bufferIndex = bufferMaxIndex;
998 kr = KERN_RESOURCE_SHORTAGE;
999 } else {
1000 /* ignore all other failures */
1001 kr = KERN_SUCCESS;
1002 }
1003
1004 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
1005 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
1006 if (flags & CS_FLAG_EXTRASP) {
1007 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
1008 }
1009 }
1010
1011 *count = bufferIndex;
1012 return kr;
1013 }
1014
1015 kern_return_t
1016 chudxnu_thread_get_callstack64_kperf(
1017 thread_t thread,
1018 uint64_t *callStack,
1019 mach_msg_type_number_t *count,
1020 boolean_t user_only)
1021 {
1022 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
1023 }
1024 #elif __x86_64__
1025
1026 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
1027 // don't try to read in the hole
1028 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
1029 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
1030 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
1031
1032 typedef struct _cframe64_t {
1033 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
1034 uint64_t caller;
1035 uint64_t args[0];
1036 }cframe64_t;
1037
1038
1039 typedef struct _cframe_t {
1040 uint32_t prev; // this is really a user32-space pointer to the previous frame
1041 uint32_t caller;
1042 uint32_t args[0];
1043 } cframe_t;
1044
1045 extern void * find_user_regs(thread_t);
1046 extern x86_saved_state32_t *find_kern_regs(thread_t);
1047
1048 static kern_return_t
1049 do_kernel_backtrace(
1050 thread_t thread,
1051 struct x86_kernel_state *regs,
1052 uint64_t *frames,
1053 mach_msg_type_number_t *start_idx,
1054 mach_msg_type_number_t max_idx)
1055 {
1056 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1057 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1058 mach_msg_type_number_t ct = *start_idx;
1059 kern_return_t kr = KERN_FAILURE;
1060
1061 #if __LP64__
1062 uint64_t currPC = 0ULL;
1063 uint64_t currFP = 0ULL;
1064 uint64_t prevPC = 0ULL;
1065 uint64_t prevFP = 0ULL;
1066 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
1067 return KERN_FAILURE;
1068 }
1069 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
1070 return KERN_FAILURE;
1071 }
1072 #else
1073 uint32_t currPC = 0U;
1074 uint32_t currFP = 0U;
1075 uint32_t prevPC = 0U;
1076 uint32_t prevFP = 0U;
1077 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
1078 return KERN_FAILURE;
1079 }
1080 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
1081 return KERN_FAILURE;
1082 }
1083 #endif
1084
1085 if (*start_idx >= max_idx) {
1086 return KERN_RESOURCE_SHORTAGE; // no frames traced
1087 }
1088 if (!currPC) {
1089 return KERN_FAILURE;
1090 }
1091
1092 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1093
1094 // build a backtrace of this kernel state
1095 #if __LP64__
1096 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
1097 // this is the address where caller lives in the user thread
1098 uint64_t caller = currFP + sizeof(uint64_t);
1099 #else
1100 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
1101 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
1102 #endif
1103
1104 if (!currFP || !currPC) {
1105 currPC = 0;
1106 break;
1107 }
1108
1109 if (ct >= max_idx) {
1110 *start_idx = ct;
1111 return KERN_RESOURCE_SHORTAGE;
1112 }
1113
1114 /* read our caller */
1115 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
1116
1117 if (kr != KERN_SUCCESS || !currPC) {
1118 currPC = 0UL;
1119 break;
1120 }
1121
1122 /*
1123 * retrive contents of the frame pointer and advance to the next stack
1124 * frame if it's valid
1125 */
1126 prevFP = 0;
1127 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1128
1129 #if __LP64__
1130 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1131 #else
1132 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1133 #endif
1134 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1135 prevPC = currPC;
1136 }
1137 if (prevFP <= currFP) {
1138 break;
1139 } else {
1140 currFP = prevFP;
1141 }
1142 }
1143
1144 *start_idx = ct;
1145 return KERN_SUCCESS;
1146 }
1147
1148
1149
1150 static kern_return_t
1151 do_backtrace32(
1152 task_t task,
1153 thread_t thread,
1154 x86_saved_state32_t *regs,
1155 uint64_t *frames,
1156 mach_msg_type_number_t *start_idx,
1157 mach_msg_type_number_t max_idx,
1158 boolean_t supervisor)
1159 {
1160 uint32_t tmpWord = 0UL;
1161 uint64_t currPC = (uint64_t) regs->eip;
1162 uint64_t currFP = (uint64_t) regs->ebp;
1163 uint64_t prevPC = 0ULL;
1164 uint64_t prevFP = 0ULL;
1165 uint64_t kernStackMin = thread->kernel_stack;
1166 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1167 mach_msg_type_number_t ct = *start_idx;
1168 kern_return_t kr = KERN_FAILURE;
1169
1170 if (ct >= max_idx) {
1171 return KERN_RESOURCE_SHORTAGE; // no frames traced
1172 }
1173 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1174
1175 // build a backtrace of this 32 bit state.
1176 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1177 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1178
1179 if (!currFP) {
1180 currPC = 0;
1181 break;
1182 }
1183
1184 if (ct >= max_idx) {
1185 *start_idx = ct;
1186 return KERN_RESOURCE_SHORTAGE;
1187 }
1188
1189 /* read our caller */
1190 if (supervisor) {
1191 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1192 } else {
1193 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1194 }
1195
1196 if (kr != KERN_SUCCESS) {
1197 currPC = 0ULL;
1198 break;
1199 }
1200
1201 currPC = (uint64_t) tmpWord; // promote 32 bit address
1202
1203 /*
1204 * retrive contents of the frame pointer and advance to the next stack
1205 * frame if it's valid
1206 */
1207 prevFP = 0;
1208 if (supervisor) {
1209 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1210 } else {
1211 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1212 }
1213 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1214
1215 if (prevFP) {
1216 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1217 prevPC = currPC;
1218 }
1219 if (prevFP < currFP) {
1220 break;
1221 } else {
1222 currFP = prevFP;
1223 }
1224 }
1225
1226 *start_idx = ct;
1227 return KERN_SUCCESS;
1228 }
1229
1230 static kern_return_t
1231 do_backtrace64(
1232 task_t task,
1233 thread_t thread,
1234 x86_saved_state64_t *regs,
1235 uint64_t *frames,
1236 mach_msg_type_number_t *start_idx,
1237 mach_msg_type_number_t max_idx,
1238 boolean_t supervisor)
1239 {
1240 uint64_t currPC = regs->isf.rip;
1241 uint64_t currFP = regs->rbp;
1242 uint64_t prevPC = 0ULL;
1243 uint64_t prevFP = 0ULL;
1244 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1245 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1246 mach_msg_type_number_t ct = *start_idx;
1247 kern_return_t kr = KERN_FAILURE;
1248
1249 if (*start_idx >= max_idx) {
1250 return KERN_RESOURCE_SHORTAGE; // no frames traced
1251 }
1252 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1253
1254 // build a backtrace of this 32 bit state.
1255 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1256 // this is the address where caller lives in the user thread
1257 uint64_t caller = currFP + sizeof(uint64_t);
1258
1259 if (!currFP) {
1260 currPC = 0;
1261 break;
1262 }
1263
1264 if (ct >= max_idx) {
1265 *start_idx = ct;
1266 return KERN_RESOURCE_SHORTAGE;
1267 }
1268
1269 /* read our caller */
1270 if (supervisor) {
1271 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1272 } else {
1273 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1274 }
1275
1276 if (kr != KERN_SUCCESS) {
1277 currPC = 0ULL;
1278 break;
1279 }
1280
1281 /*
1282 * retrive contents of the frame pointer and advance to the next stack
1283 * frame if it's valid
1284 */
1285 prevFP = 0;
1286 if (supervisor) {
1287 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1288 } else {
1289 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1290 }
1291
1292 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1293 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1294 prevPC = currPC;
1295 }
1296 if (prevFP < currFP) {
1297 break;
1298 } else {
1299 currFP = prevFP;
1300 }
1301 }
1302
1303 *start_idx = ct;
1304 return KERN_SUCCESS;
1305 }
1306
1307 static kern_return_t
1308 chudxnu_thread_get_callstack64_internal(
1309 thread_t thread,
1310 uint64_t *callstack,
1311 mach_msg_type_number_t *count,
1312 boolean_t user_only,
1313 boolean_t kern_only)
1314 {
1315 kern_return_t kr = KERN_FAILURE;
1316 task_t task = thread->task;
1317 uint64_t currPC = 0ULL;
1318 boolean_t supervisor = FALSE;
1319 mach_msg_type_number_t bufferIndex = 0;
1320 mach_msg_type_number_t bufferMaxIndex = *count;
1321 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1322 x86_saved_state64_t *regs64 = NULL;
1323 x86_saved_state32_t *regs32 = NULL;
1324 x86_saved_state32_t *u_regs32 = NULL;
1325 x86_saved_state64_t *u_regs64 = NULL;
1326 struct x86_kernel_state *kregs = NULL;
1327
1328 if (ml_at_interrupt_context()) {
1329 if (user_only) {
1330 /* can't backtrace user state on interrupt stack. */
1331 return KERN_FAILURE;
1332 }
1333
1334 /* backtracing at interrupt context? */
1335 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1336 /*
1337 * Locate the registers for the interrupted thread, assuming it is
1338 * current_thread().
1339 */
1340 tagged_regs = current_cpu_datap()->cpu_int_state;
1341
1342 if (is_saved_state64(tagged_regs)) {
1343 /* 64 bit registers */
1344 regs64 = saved_state64(tagged_regs);
1345 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1346 } else {
1347 /* 32 bit registers */
1348 regs32 = saved_state32(tagged_regs);
1349 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1350 }
1351 }
1352 }
1353
1354 if (!ml_at_interrupt_context() && kernel_task == task) {
1355 if (!thread->kernel_stack) {
1356 return KERN_FAILURE;
1357 }
1358
1359 // Kernel thread not at interrupt context
1360 kregs = (struct x86_kernel_state *)NULL;
1361
1362 // nofault read of the thread->kernel_stack pointer
1363 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1364 return KERN_FAILURE;
1365 }
1366
1367 // Adjust to find the saved kernel state
1368 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1369
1370 supervisor = TRUE;
1371 } else if (!tagged_regs) {
1372 /*
1373 * not at interrupt context, or tracing a different thread than
1374 * current_thread() at interrupt context
1375 */
1376 tagged_regs = USER_STATE(thread);
1377 if (is_saved_state64(tagged_regs)) {
1378 /* 64 bit registers */
1379 regs64 = saved_state64(tagged_regs);
1380 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1381 } else {
1382 /* 32 bit registers */
1383 regs32 = saved_state32(tagged_regs);
1384 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1385 }
1386 }
1387
1388 *count = 0;
1389
1390 if (supervisor) {
1391 // the caller only wants a user callstack.
1392 if (user_only) {
1393 // bail - we've only got kernel state
1394 return KERN_FAILURE;
1395 }
1396 } else {
1397 // regs32(64) is not in supervisor mode.
1398 u_regs32 = regs32;
1399 u_regs64 = regs64;
1400 regs32 = NULL;
1401 regs64 = NULL;
1402 }
1403
1404 if (user_only) {
1405 /* we only want to backtrace the user mode */
1406 if (!(u_regs32 || u_regs64)) {
1407 /* no user state to look at */
1408 return KERN_FAILURE;
1409 }
1410 }
1411
1412 /*
1413 * Order of preference for top of stack:
1414 * 64 bit kernel state (not likely)
1415 * 32 bit kernel state
1416 * 64 bit user land state
1417 * 32 bit user land state
1418 */
1419
1420 if (kregs) {
1421 /*
1422 * nofault read of the registers from the kernel stack (as they can
1423 * disappear on the fly).
1424 */
1425
1426 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1427 return KERN_FAILURE;
1428 }
1429 } else if (regs64) {
1430 currPC = regs64->isf.rip;
1431 } else if (regs32) {
1432 currPC = (uint64_t) regs32->eip;
1433 } else if (u_regs64) {
1434 currPC = u_regs64->isf.rip;
1435 } else if (u_regs32) {
1436 currPC = (uint64_t) u_regs32->eip;
1437 }
1438
1439 if (!currPC) {
1440 /* no top of the stack, bail out */
1441 return KERN_FAILURE;
1442 }
1443
1444 bufferIndex = 0;
1445
1446 if (bufferMaxIndex < 1) {
1447 *count = 0;
1448 return KERN_RESOURCE_SHORTAGE;
1449 }
1450
1451 /* backtrace kernel */
1452 if (kregs) {
1453 addr64_t address = 0ULL;
1454 size_t size = 0UL;
1455
1456 // do the backtrace
1457 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1458
1459 // and do a nofault read of (r|e)sp
1460 uint64_t rsp = 0ULL;
1461 size = sizeof(uint64_t);
1462
1463 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1464 address = 0ULL;
1465 }
1466
1467 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1468 callstack[bufferIndex++] = (uint64_t)rsp;
1469 }
1470 } else if (regs64) {
1471 uint64_t rsp = 0ULL;
1472
1473 // backtrace the 64bit side.
1474 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1475 bufferMaxIndex - 1, TRUE);
1476
1477 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1478 bufferIndex < bufferMaxIndex) {
1479 callstack[bufferIndex++] = rsp;
1480 }
1481 } else if (regs32) {
1482 uint32_t esp = 0UL;
1483
1484 // backtrace the 32bit side.
1485 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1486 bufferMaxIndex - 1, TRUE);
1487
1488 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1489 bufferIndex < bufferMaxIndex) {
1490 callstack[bufferIndex++] = (uint64_t) esp;
1491 }
1492 } else if (u_regs64 && !kern_only) {
1493 /* backtrace user land */
1494 uint64_t rsp = 0ULL;
1495
1496 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1497 bufferMaxIndex - 1, FALSE);
1498
1499 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1500 bufferIndex < bufferMaxIndex) {
1501 callstack[bufferIndex++] = rsp;
1502 }
1503 } else if (u_regs32 && !kern_only) {
1504 uint32_t esp = 0UL;
1505
1506 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1507 bufferMaxIndex - 1, FALSE);
1508
1509 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1510 bufferIndex < bufferMaxIndex) {
1511 callstack[bufferIndex++] = (uint64_t) esp;
1512 }
1513 }
1514
1515 *count = bufferIndex;
1516 return kr;
1517 }
1518
1519 __private_extern__
1520 kern_return_t
1521 chudxnu_thread_get_callstack64_kperf(
1522 thread_t thread,
1523 uint64_t *callstack,
1524 mach_msg_type_number_t *count,
1525 boolean_t is_user)
1526 {
1527 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1528 }
1529 #else /* !__arm__ && !__arm64__ && !__x86_64__ */
1530 #error kperf: unsupported architecture
1531 #endif /* !__arm__ && !__arm64__ && !__x86_64__ */
mirror of XNU