]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kperf/callstack.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-7195.60.75.tar.gz
[apple/xnu.git] / osfmk / kperf / callstack.c
1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Collect kernel callstacks */
30
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <kern/cambria_layout.h>
35 #include <vm/vm_map.h>
36 #include <kperf/buffer.h>
37 #include <kperf/context.h>
38 #include <kperf/callstack.h>
39 #include <kperf/ast.h>
40 #include <sys/errno.h>
41
42 #if defined(__arm__) || defined(__arm64__)
43 #include <arm/cpu_data.h>
44 #include <arm/cpu_data_internal.h>
45 #endif
46
47 static void
48 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
49 {
50 uint64_t fixup_val = 0;
51 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
52
53 #if defined(__x86_64__)
54 user_addr_t sp_user;
55 bool user_64;
56 x86_saved_state_t *state;
57
58 state = get_user_regs(thread);
59 if (!state) {
60 goto out;
61 }
62
63 user_64 = is_saved_state64(state);
64 if (user_64) {
65 sp_user = saved_state64(state)->isf.rsp;
66 } else {
67 sp_user = saved_state32(state)->uesp;
68 }
69
70 if (thread == current_thread()) {
71 (void)copyin(sp_user, (char *)&fixup_val,
72 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
73 } else {
74 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
75 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
76 }
77
78 #elif defined(__arm64__) || defined(__arm__)
79
80 struct arm_saved_state *state = get_user_regs(thread);
81 if (!state) {
82 goto out;
83 }
84
85 /* encode thumb mode into low bit of PC */
86 if (get_saved_state_cpsr(state) & PSR_TF) {
87 cs->kpuc_frames[0] |= 1ULL;
88 }
89
90
91 fixup_val = get_saved_state_lr(state);
92
93 #else
94 #error "callstack_fixup_user: unsupported architecture"
95 #endif
96
97 out:
98 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
99 }
100
101 #if defined(__x86_64__)
102
103 __attribute__((used))
104 static kern_return_t
105 interrupted_kernel_sp_value(uintptr_t *sp_val)
106 {
107 x86_saved_state_t *state;
108 uintptr_t sp;
109 bool state_64;
110 uint64_t cs;
111 uintptr_t top, bottom;
112
113 state = current_cpu_datap()->cpu_int_state;
114 if (!state) {
115 return KERN_FAILURE;
116 }
117
118 state_64 = is_saved_state64(state);
119
120 if (state_64) {
121 cs = saved_state64(state)->isf.cs;
122 } else {
123 cs = saved_state32(state)->cs;
124 }
125 /* return early if interrupted a thread in user space */
126 if ((cs & SEL_PL) == SEL_PL_U) {
127 return KERN_FAILURE;
128 }
129
130 if (state_64) {
131 sp = saved_state64(state)->isf.rsp;
132 } else {
133 sp = saved_state32(state)->uesp;
134 }
135
136 /* make sure the stack pointer is pointing somewhere in this stack */
137 bottom = current_thread()->kernel_stack;
138 top = bottom + kernel_stack_size;
139 if (sp >= bottom && sp < top) {
140 return KERN_FAILURE;
141 }
142
143 *sp_val = *(uintptr_t *)sp;
144 return KERN_SUCCESS;
145 }
146
147 #elif defined(__arm64__)
148
149 __attribute__((used))
150 static kern_return_t
151 interrupted_kernel_lr(uintptr_t *lr)
152 {
153 struct arm_saved_state *state;
154
155 state = getCpuDatap()->cpu_int_state;
156
157 /* return early if interrupted a thread in user space */
158 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
159 return KERN_FAILURE;
160 }
161
162 *lr = get_saved_state_lr(state);
163 return KERN_SUCCESS;
164 }
165
166 #elif defined(__arm__)
167
168 __attribute__((used))
169 static kern_return_t
170 interrupted_kernel_lr(uintptr_t *lr)
171 {
172 struct arm_saved_state *state;
173
174 state = getCpuDatap()->cpu_int_state;
175
176 /* return early if interrupted a thread in user space */
177 if (PSR_IS_USER(get_saved_state_cpsr(state))) {
178 return KERN_FAILURE;
179 }
180
181 *lr = get_saved_state_lr(state);
182 return KERN_SUCCESS;
183 }
184
185 #else /* defined(__arm__) */
186 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
187 #endif /* !defined(__arm__) */
188
189
190 static void
191 callstack_fixup_interrupted(struct kp_kcallstack *cs)
192 {
193 uintptr_t fixup_val = 0;
194 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
195
196 /*
197 * Only provide arbitrary data on development or debug kernels.
198 */
199 #if DEVELOPMENT || DEBUG
200 #if defined(__x86_64__)
201 (void)interrupted_kernel_sp_value(&fixup_val);
202 #elif defined(__arm64__) || defined(__arm__)
203 (void)interrupted_kernel_lr(&fixup_val);
204 #endif /* defined(__x86_64__) */
205 #endif /* DEVELOPMENT || DEBUG */
206
207 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
208 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
209 }
210
211 void
212 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
213 {
214 thread_t thread;
215
216 assert(cs != NULL);
217 assert(context != NULL);
218
219 thread = context->cur_thread;
220 assert(thread != NULL);
221 assert(thread->continuation != NULL);
222
223 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
224 #ifdef __LP64__
225 cs->kpkc_flags |= CALLSTACK_64BIT;
226 #endif
227
228 cs->kpkc_nframes = 1;
229 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
230 }
231
232 void
233 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
234 {
235 assert(cs != NULL);
236 assert(context != NULL);
237 assert(context->cur_thread == current_thread());
238
239 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
240 #ifdef __LP64__
241 cs->kpkc_flags |= CALLSTACK_64BIT;
242 #endif
243
244 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
245
246 bool trunc = false;
247 cs->kpkc_nframes = backtrace_frame(cs->kpkc_word_frames,
248 cs->kpkc_nframes - 1, context->starting_fp, &trunc);
249 if (cs->kpkc_nframes > 0) {
250 cs->kpkc_flags |= CALLSTACK_VALID;
251 /*
252 * Fake the value pointed to by the stack pointer or the link
253 * register for symbolicators.
254 */
255 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
256 cs->kpkc_nframes += 1;
257 }
258 if (trunc) {
259 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
260 }
261
262 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
263 }
264
265 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
266 uint64_t *callStack, mach_msg_type_number_t *count,
267 boolean_t user_only);
268
269 void
270 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
271 {
272 thread_t thread;
273
274 assert(cs != NULL);
275 assert(context != NULL);
276 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
277
278 thread = context->cur_thread;
279 assert(thread != NULL);
280
281 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
282 cs->kpkc_nframes);
283
284 cs->kpkc_flags = CALLSTACK_KERNEL;
285 #ifdef __LP64__
286 cs->kpkc_flags |= CALLSTACK_64BIT;
287 #endif
288
289 if (ml_at_interrupt_context()) {
290 assert(thread == current_thread());
291 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
292 bool trunc = false;
293 cs->kpkc_nframes = backtrace_interrupted(
294 cs->kpkc_word_frames, cs->kpkc_nframes - 1, &trunc);
295 if (cs->kpkc_nframes != 0) {
296 callstack_fixup_interrupted(cs);
297 }
298 if (trunc) {
299 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
300 }
301 } else {
302 /*
303 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
304 * other threads.
305 */
306 kern_return_t kr;
307 kr = chudxnu_thread_get_callstack64_kperf(thread,
308 cs->kpkc_frames, &cs->kpkc_nframes, FALSE);
309 if (kr == KERN_SUCCESS) {
310 cs->kpkc_flags |= CALLSTACK_VALID;
311 } else if (kr == KERN_RESOURCE_SHORTAGE) {
312 cs->kpkc_flags |= CALLSTACK_VALID;
313 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
314 } else {
315 cs->kpkc_nframes = 0;
316 }
317 }
318
319 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
320 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
321 }
322
323 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
324 cs->kpkc_flags, cs->kpkc_nframes);
325 }
326
327 void
328 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
329 {
330 assert(ml_get_interrupts_enabled() == TRUE);
331
332 thread_t thread = context->cur_thread;
333 assert(thread != NULL);
334
335 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
336 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
337
338 bool user64 = false;
339 bool trunc = false;
340 int error = 0;
341 /*
342 * Leave space for the fixup information.
343 */
344 unsigned int maxnframes = cs->kpuc_nframes - 1;
345 unsigned int nframes = backtrace_thread_user(thread, cs->kpuc_frames,
346 maxnframes, &error, &user64, &trunc, true);
347 cs->kpuc_nframes = MIN(maxnframes, nframes);
348
349 /*
350 * Ignore EFAULT to get as much of the stack as possible. It will be
351 * marked as truncated, below.
352 */
353 if (error == 0 || error == EFAULT) {
354 callstack_fixup_user(cs, thread);
355 cs->kpuc_flags |= CALLSTACK_VALID;
356 } else {
357 cs->kpuc_nframes = 0;
358 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, error);
359 }
360
361 cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS | (user64 ? CALLSTACK_64BIT : 0) |
362 (trunc ? CALLSTACK_TRUNCATED : 0);
363
364 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
365 cs->kpuc_flags, cs->kpuc_nframes);
366 }
367
368 static inline uintptr_t
369 scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
370 {
371 if (frame < n_frames) {
372 if (kern) {
373 return VM_KERNEL_UNSLIDE(bt[frame]);
374 } else {
375 return bt[frame];
376 }
377 } else {
378 return 0;
379 }
380 }
381
382 static inline uintptr_t
383 scrub_frame(uint64_t *bt, int n_frames, int frame)
384 {
385 if (frame < n_frames) {
386 return (uintptr_t)(bt[frame]);
387 } else {
388 return 0;
389 }
390 }
391
392 static void
393 callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
394 unsigned int nframes, unsigned int flags)
395 {
396 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
397
398 BUF_DATA(hdrid, flags, nframes);
399
400 unsigned int nevts = nframes / 4;
401 unsigned int ovf = nframes % 4;
402 if (ovf != 0) {
403 nevts++;
404 }
405
406 bool kern = flags & CALLSTACK_KERNEL;
407
408 if (flags & CALLSTACK_KERNEL_WORDS) {
409 uintptr_t *frames = vframes;
410 for (unsigned int i = 0; i < nevts; i++) {
411 unsigned int j = i * 4;
412 BUF_DATA(dataid,
413 scrub_word(frames, nframes, j + 0, kern),
414 scrub_word(frames, nframes, j + 1, kern),
415 scrub_word(frames, nframes, j + 2, kern),
416 scrub_word(frames, nframes, j + 3, kern));
417 }
418 } else {
419 for (unsigned int i = 0; i < nevts; i++) {
420 uint64_t *frames = vframes;
421 unsigned int j = i * 4;
422 BUF_DATA(dataid,
423 scrub_frame(frames, nframes, j + 0),
424 scrub_frame(frames, nframes, j + 1),
425 scrub_frame(frames, nframes, j + 2),
426 scrub_frame(frames, nframes, j + 3));
427 }
428 }
429
430 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
431 }
432
433 void
434 kperf_kcallstack_log(struct kp_kcallstack *cs)
435 {
436 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
437 cs->kpkc_nframes, cs->kpkc_flags);
438 }
439
440 void
441 kperf_ucallstack_log(struct kp_ucallstack *cs)
442 {
443 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
444 cs->kpuc_nframes, cs->kpuc_flags);
445 }
446
447 int
448 kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
449 unsigned int actionid)
450 {
451 if (depth < 2) {
452 panic("HUH");
453 }
454 kperf_ast_set_callstack_depth(context->cur_thread, depth);
455 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
456 actionid);
457 }
458
459 static kern_return_t
460 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
461 {
462 return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
463 KERN_SUCCESS : KERN_FAILURE;
464 }
465
466 static kern_return_t
467 chudxnu_task_read(
468 task_t task,
469 void *kernaddr,
470 uint64_t usraddr,
471 vm_size_t size)
472 {
473 //ppc version ported to arm
474 kern_return_t ret = KERN_SUCCESS;
475
476 if (ml_at_interrupt_context()) {
477 return KERN_FAILURE; // can't look at tasks on interrupt stack
478 }
479
480 if (current_task() == task) {
481 if (copyin(usraddr, kernaddr, size)) {
482 ret = KERN_FAILURE;
483 }
484 } else {
485 vm_map_t map = get_task_map(task);
486 ret = vm_map_read_user(map, usraddr, kernaddr, size);
487 }
488
489 return ret;
490 }
491
492 static inline uint64_t
493 chudxnu_vm_unslide( uint64_t ptr, int kaddr )
494 {
495 if (!kaddr) {
496 return ptr;
497 }
498
499 return VM_KERNEL_UNSLIDE(ptr);
500 }
501
502 #if __arm__
503 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
504 #define CS_FLAG_EXTRASP 1 // capture extra sp register
505 static kern_return_t
506 chudxnu_thread_get_callstack64_internal(
507 thread_t thread,
508 uint64_t *callStack,
509 mach_msg_type_number_t *count,
510 boolean_t user_only,
511 int flags)
512 {
513 kern_return_t kr;
514 task_t task;
515 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
516 uint64_t prevPC = 0ULL;
517 uint32_t kernStackMin = thread->kernel_stack;
518 uint32_t kernStackMax = kernStackMin + kernel_stack_size;
519 uint64_t *buffer = callStack;
520 uint32_t frame[2];
521 int bufferIndex = 0;
522 int bufferMaxIndex = 0;
523 boolean_t supervisor = FALSE;
524 struct arm_saved_state *state = NULL;
525 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
526 uint64_t pc = 0ULL;
527
528 task = get_threadtask(thread);
529
530 bufferMaxIndex = *count;
531 //get thread state
532 if (user_only) {
533 state = find_user_regs(thread);
534 } else {
535 state = find_kern_regs(thread);
536 }
537
538 if (!state) {
539 *count = 0;
540 return KERN_FAILURE;
541 }
542
543 /* make sure it is safe to dereference before you do it */
544 supervisor = ARM_SUPERVISOR_MODE(state->cpsr);
545
546 /* can't take a kernel callstack if we've got a user frame */
547 if (!user_only && !supervisor) {
548 return KERN_FAILURE;
549 }
550
551 /*
552 * Reserve space for saving LR (and sometimes SP) at the end of the
553 * backtrace.
554 */
555 if (flags & CS_FLAG_EXTRASP) {
556 bufferMaxIndex -= 2;
557 } else {
558 bufferMaxIndex -= 1;
559 }
560
561 if (bufferMaxIndex < 2) {
562 *count = 0;
563 return KERN_RESOURCE_SHORTAGE;
564 }
565
566 currPC = (uint64_t)state->pc; /* r15 */
567 if (state->cpsr & PSR_TF) {
568 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
569 }
570 currLR = (uint64_t)state->lr; /* r14 */
571 currSP = (uint64_t)state->sp; /* r13 */
572
573 fp = (uint32_t *)state->r[7]; /* frame pointer */
574 topfp = fp;
575
576 bufferIndex = 0; // start with a stack of size zero
577 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, supervisor); // save PC in position 0.
578
579 // Now, fill buffer with stack backtraces.
580 while (bufferIndex < bufferMaxIndex) {
581 pc = 0ULL;
582 /*
583 * Below the frame pointer, the following values are saved:
584 * -> FP
585 */
586
587 /*
588 * Note that we read the pc even for the first stack frame
589 * (which, in theory, is always empty because the callee fills
590 * it in just before it lowers the stack. However, if we
591 * catch the program in between filling in the return address
592 * and lowering the stack, we want to still have a valid
593 * backtrace. FixupStack correctly disregards this value if
594 * necessary.
595 */
596
597 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
598 /* frame pointer is invalid - stop backtracing */
599 pc = 0ULL;
600 break;
601 }
602
603 if (supervisor) {
604 if (((uint32_t)fp > kernStackMax) ||
605 ((uint32_t)fp < kernStackMin)) {
606 kr = KERN_FAILURE;
607 } else {
608 kr = chudxnu_kern_read(&frame,
609 (vm_offset_t)fp,
610 (vm_size_t)sizeof(frame));
611 if (kr == KERN_SUCCESS) {
612 pc = (uint64_t)frame[1];
613 nextFramePointer = (uint32_t *) (frame[0]);
614 } else {
615 pc = 0ULL;
616 nextFramePointer = 0ULL;
617 kr = KERN_FAILURE;
618 }
619 }
620 } else {
621 kr = chudxnu_task_read(task,
622 &frame,
623 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
624 sizeof(frame));
625 if (kr == KERN_SUCCESS) {
626 pc = (uint64_t) frame[1];
627 nextFramePointer = (uint32_t *) (frame[0]);
628 } else {
629 pc = 0ULL;
630 nextFramePointer = 0ULL;
631 kr = KERN_FAILURE;
632 }
633 }
634
635 if (kr != KERN_SUCCESS) {
636 pc = 0ULL;
637 break;
638 }
639
640 if (nextFramePointer) {
641 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, supervisor);
642 prevPC = pc;
643 }
644
645 if (nextFramePointer < fp) {
646 break;
647 } else {
648 fp = nextFramePointer;
649 }
650 }
651
652 if (bufferIndex >= bufferMaxIndex) {
653 bufferIndex = bufferMaxIndex;
654 kr = KERN_RESOURCE_SHORTAGE;
655 } else {
656 kr = KERN_SUCCESS;
657 }
658
659 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
660 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, supervisor);
661 if (flags & CS_FLAG_EXTRASP) {
662 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, supervisor);
663 }
664
665 *count = bufferIndex;
666 return kr;
667 }
668
669 kern_return_t
670 chudxnu_thread_get_callstack64_kperf(
671 thread_t thread,
672 uint64_t *callStack,
673 mach_msg_type_number_t *count,
674 boolean_t user_only)
675 {
676 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
677 }
678 #elif __arm64__
679
680 #if defined(HAS_APPLE_PAC)
681 #include <ptrauth.h>
682 #endif
683
684 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
685 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
686 // after sampling has finished.
687 //
688 // For an N-entry callstack:
689 //
690 // [0] current pc
691 // [1..N-3] stack frames (including current one)
692 // [N-2] current LR (return value if we're in a leaf function)
693 // [N-1] current r0 (in case we've saved LR in r0) (optional)
694 //
695 //
696 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
697
698 #define CS_FLAG_EXTRASP 1 // capture extra sp register
699
700 static kern_return_t
701 chudxnu_thread_get_callstack64_internal(
702 thread_t thread,
703 uint64_t *callStack,
704 mach_msg_type_number_t *count,
705 boolean_t user_only,
706 int flags)
707 {
708 kern_return_t kr = KERN_SUCCESS;
709 task_t task;
710 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
711 uint64_t prevPC = 0ULL;
712 uint64_t kernStackMin = thread->kernel_stack;
713 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
714 uint64_t *buffer = callStack;
715 int bufferIndex = 0;
716 int bufferMaxIndex = 0;
717 boolean_t kernel = FALSE;
718 struct arm_saved_state *sstate = NULL;
719 uint64_t pc = 0ULL;
720
721 task = get_threadtask(thread);
722 bufferMaxIndex = *count;
723 //get thread state
724 if (user_only) {
725 sstate = find_user_regs(thread);
726 } else {
727 sstate = find_kern_regs(thread);
728 }
729
730 if (!sstate) {
731 *count = 0;
732 return KERN_FAILURE;
733 }
734
735 if (is_saved_state64(sstate)) {
736 struct arm_saved_state64 *state = NULL;
737 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
738 uint64_t frame[2];
739
740 state = saved_state64(sstate);
741
742 /* make sure it is safe to dereference before you do it */
743 kernel = PSR64_IS_KERNEL(state->cpsr);
744
745 /* can't take a kernel callstack if we've got a user frame */
746 if (!user_only && !kernel) {
747 return KERN_FAILURE;
748 }
749
750 /*
751 * Reserve space for saving LR (and sometimes SP) at the end of the
752 * backtrace.
753 */
754 if (flags & CS_FLAG_EXTRASP) {
755 bufferMaxIndex -= 2;
756 } else {
757 bufferMaxIndex -= 1;
758 }
759
760 if (bufferMaxIndex < 2) {
761 *count = 0;
762 return KERN_RESOURCE_SHORTAGE;
763 }
764
765 currPC = state->pc;
766 currLR = state->lr;
767 currSP = state->sp;
768
769 fp = (uint64_t *)state->fp; /* frame pointer */
770 topfp = fp;
771
772 bufferIndex = 0; // start with a stack of size zero
773 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
774
775 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
776
777 // Now, fill buffer with stack backtraces.
778 while (bufferIndex < bufferMaxIndex) {
779 pc = 0ULL;
780 /*
781 * Below the frame pointer, the following values are saved:
782 * -> FP
783 */
784
785 /*
786 * Note that we read the pc even for the first stack frame
787 * (which, in theory, is always empty because the callee fills
788 * it in just before it lowers the stack. However, if we
789 * catch the program in between filling in the return address
790 * and lowering the stack, we want to still have a valid
791 * backtrace. FixupStack correctly disregards this value if
792 * necessary.
793 */
794
795 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
796 /* frame pointer is invalid - stop backtracing */
797 pc = 0ULL;
798 break;
799 }
800
801 if (kernel) {
802 if (((uint64_t)fp > kernStackMax) ||
803 ((uint64_t)fp < kernStackMin)) {
804 kr = KERN_FAILURE;
805 } else {
806 kr = chudxnu_kern_read(&frame,
807 (vm_offset_t)fp,
808 (vm_size_t)sizeof(frame));
809 if (kr == KERN_SUCCESS) {
810 #if defined(HAS_APPLE_PAC)
811 /* return addresses on stack will be signed by arm64e ABI */
812 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
813 #else
814 pc = frame[1];
815 #endif
816 nextFramePointer = (uint64_t *)frame[0];
817 } else {
818 pc = 0ULL;
819 nextFramePointer = 0ULL;
820 kr = KERN_FAILURE;
821 }
822 }
823 } else {
824 kr = chudxnu_task_read(task,
825 &frame,
826 (vm_offset_t)fp,
827 (vm_size_t)sizeof(frame));
828 if (kr == KERN_SUCCESS) {
829 #if defined(HAS_APPLE_PAC)
830 /* return addresses on stack will be signed by arm64e ABI */
831 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
832 #else
833 pc = frame[1];
834 #endif
835 nextFramePointer = (uint64_t *)(frame[0]);
836 } else {
837 pc = 0ULL;
838 nextFramePointer = 0ULL;
839 kr = KERN_FAILURE;
840 }
841 }
842
843 if (kr != KERN_SUCCESS) {
844 pc = 0ULL;
845 break;
846 }
847
848 if (nextFramePointer) {
849 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
850 prevPC = pc;
851 }
852
853 if (nextFramePointer < fp) {
854 break;
855 } else {
856 fp = nextFramePointer;
857 }
858 }
859
860 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
861
862 if (bufferIndex >= bufferMaxIndex) {
863 bufferIndex = bufferMaxIndex;
864 kr = KERN_RESOURCE_SHORTAGE;
865 } else {
866 kr = KERN_SUCCESS;
867 }
868
869 // Save link register and SP at bottom of stack (used for later fixup).
870 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
871 if (flags & CS_FLAG_EXTRASP) {
872 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
873 }
874 } else {
875 struct arm_saved_state32 *state = NULL;
876 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
877
878 /* 64-bit kernel stacks, 32-bit user stacks */
879 uint64_t frame[2];
880 uint32_t frame32[2];
881
882 state = saved_state32(sstate);
883
884 /* make sure it is safe to dereference before you do it */
885 kernel = ARM_SUPERVISOR_MODE(state->cpsr);
886
887 /* can't take a kernel callstack if we've got a user frame */
888 if (!user_only && !kernel) {
889 return KERN_FAILURE;
890 }
891
892 /*
893 * Reserve space for saving LR (and sometimes SP) at the end of the
894 * backtrace.
895 */
896 if (flags & CS_FLAG_EXTRASP) {
897 bufferMaxIndex -= 2;
898 } else {
899 bufferMaxIndex -= 1;
900 }
901
902 if (bufferMaxIndex < 2) {
903 *count = 0;
904 return KERN_RESOURCE_SHORTAGE;
905 }
906
907 currPC = (uint64_t)state->pc; /* r15 */
908 if (state->cpsr & PSR_TF) {
909 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
910 }
911 currLR = (uint64_t)state->lr; /* r14 */
912 currSP = (uint64_t)state->sp; /* r13 */
913
914 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
915 topfp = fp;
916
917 bufferIndex = 0; // start with a stack of size zero
918 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
919
920 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
921
922 // Now, fill buffer with stack backtraces.
923 while (bufferIndex < bufferMaxIndex) {
924 pc = 0ULL;
925 /*
926 * Below the frame pointer, the following values are saved:
927 * -> FP
928 */
929
930 /*
931 * Note that we read the pc even for the first stack frame
932 * (which, in theory, is always empty because the callee fills
933 * it in just before it lowers the stack. However, if we
934 * catch the program in between filling in the return address
935 * and lowering the stack, we want to still have a valid
936 * backtrace. FixupStack correctly disregards this value if
937 * necessary.
938 */
939
940 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
941 /* frame pointer is invalid - stop backtracing */
942 pc = 0ULL;
943 break;
944 }
945
946 if (kernel) {
947 if (((uint32_t)fp > kernStackMax) ||
948 ((uint32_t)fp < kernStackMin)) {
949 kr = KERN_FAILURE;
950 } else {
951 kr = chudxnu_kern_read(&frame,
952 (vm_offset_t)fp,
953 (vm_size_t)sizeof(frame));
954 if (kr == KERN_SUCCESS) {
955 pc = (uint64_t)frame[1];
956 nextFramePointer = (uint32_t *) (frame[0]);
957 } else {
958 pc = 0ULL;
959 nextFramePointer = 0ULL;
960 kr = KERN_FAILURE;
961 }
962 }
963 } else {
964 kr = chudxnu_task_read(task,
965 &frame32,
966 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
967 sizeof(frame32));
968 if (kr == KERN_SUCCESS) {
969 pc = (uint64_t)frame32[1];
970 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
971 } else {
972 pc = 0ULL;
973 nextFramePointer = 0ULL;
974 kr = KERN_FAILURE;
975 }
976 }
977
978 if (kr != KERN_SUCCESS) {
979 pc = 0ULL;
980 break;
981 }
982
983 if (nextFramePointer) {
984 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
985 prevPC = pc;
986 }
987
988 if (nextFramePointer < fp) {
989 break;
990 } else {
991 fp = nextFramePointer;
992 }
993 }
994
995 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
996
997 /* clamp callstack size to max */
998 if (bufferIndex >= bufferMaxIndex) {
999 bufferIndex = bufferMaxIndex;
1000 kr = KERN_RESOURCE_SHORTAGE;
1001 } else {
1002 /* ignore all other failures */
1003 kr = KERN_SUCCESS;
1004 }
1005
1006 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
1007 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
1008 if (flags & CS_FLAG_EXTRASP) {
1009 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
1010 }
1011 }
1012
1013 *count = bufferIndex;
1014 return kr;
1015 }
1016
1017 kern_return_t
1018 chudxnu_thread_get_callstack64_kperf(
1019 thread_t thread,
1020 uint64_t *callStack,
1021 mach_msg_type_number_t *count,
1022 boolean_t user_only)
1023 {
1024 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
1025 }
1026 #elif __x86_64__
1027
1028 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
1029 // don't try to read in the hole
1030 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
1031 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
1032 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
1033
1034 typedef struct _cframe64_t {
1035 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
1036 uint64_t caller;
1037 uint64_t args[0];
1038 }cframe64_t;
1039
1040
1041 typedef struct _cframe_t {
1042 uint32_t prev; // this is really a user32-space pointer to the previous frame
1043 uint32_t caller;
1044 uint32_t args[0];
1045 } cframe_t;
1046
1047 extern void * find_user_regs(thread_t);
1048 extern x86_saved_state32_t *find_kern_regs(thread_t);
1049
1050 static kern_return_t
1051 do_kernel_backtrace(
1052 thread_t thread,
1053 struct x86_kernel_state *regs,
1054 uint64_t *frames,
1055 mach_msg_type_number_t *start_idx,
1056 mach_msg_type_number_t max_idx)
1057 {
1058 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1059 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1060 mach_msg_type_number_t ct = *start_idx;
1061 kern_return_t kr = KERN_FAILURE;
1062
1063 #if __LP64__
1064 uint64_t currPC = 0ULL;
1065 uint64_t currFP = 0ULL;
1066 uint64_t prevPC = 0ULL;
1067 uint64_t prevFP = 0ULL;
1068 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
1069 return KERN_FAILURE;
1070 }
1071 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
1072 return KERN_FAILURE;
1073 }
1074 #else
1075 uint32_t currPC = 0U;
1076 uint32_t currFP = 0U;
1077 uint32_t prevPC = 0U;
1078 uint32_t prevFP = 0U;
1079 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
1080 return KERN_FAILURE;
1081 }
1082 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
1083 return KERN_FAILURE;
1084 }
1085 #endif
1086
1087 if (*start_idx >= max_idx) {
1088 return KERN_RESOURCE_SHORTAGE; // no frames traced
1089 }
1090 if (!currPC) {
1091 return KERN_FAILURE;
1092 }
1093
1094 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1095
1096 // build a backtrace of this kernel state
1097 #if __LP64__
1098 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
1099 // this is the address where caller lives in the user thread
1100 uint64_t caller = currFP + sizeof(uint64_t);
1101 #else
1102 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
1103 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
1104 #endif
1105
1106 if (!currFP || !currPC) {
1107 currPC = 0;
1108 break;
1109 }
1110
1111 if (ct >= max_idx) {
1112 *start_idx = ct;
1113 return KERN_RESOURCE_SHORTAGE;
1114 }
1115
1116 /* read our caller */
1117 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
1118
1119 if (kr != KERN_SUCCESS || !currPC) {
1120 currPC = 0UL;
1121 break;
1122 }
1123
1124 /*
1125 * retrive contents of the frame pointer and advance to the next stack
1126 * frame if it's valid
1127 */
1128 prevFP = 0;
1129 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1130
1131 #if __LP64__
1132 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1133 #else
1134 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1135 #endif
1136 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1137 prevPC = currPC;
1138 }
1139 if (prevFP <= currFP) {
1140 break;
1141 } else {
1142 currFP = prevFP;
1143 }
1144 }
1145
1146 *start_idx = ct;
1147 return KERN_SUCCESS;
1148 }
1149
1150
1151
1152 static kern_return_t
1153 do_backtrace32(
1154 task_t task,
1155 thread_t thread,
1156 x86_saved_state32_t *regs,
1157 uint64_t *frames,
1158 mach_msg_type_number_t *start_idx,
1159 mach_msg_type_number_t max_idx,
1160 boolean_t supervisor)
1161 {
1162 uint32_t tmpWord = 0UL;
1163 uint64_t currPC = (uint64_t) regs->eip;
1164 uint64_t currFP = (uint64_t) regs->ebp;
1165 uint64_t prevPC = 0ULL;
1166 uint64_t prevFP = 0ULL;
1167 uint64_t kernStackMin = thread->kernel_stack;
1168 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1169 mach_msg_type_number_t ct = *start_idx;
1170 kern_return_t kr = KERN_FAILURE;
1171
1172 if (ct >= max_idx) {
1173 return KERN_RESOURCE_SHORTAGE; // no frames traced
1174 }
1175 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1176
1177 // build a backtrace of this 32 bit state.
1178 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1179 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1180
1181 if (!currFP) {
1182 currPC = 0;
1183 break;
1184 }
1185
1186 if (ct >= max_idx) {
1187 *start_idx = ct;
1188 return KERN_RESOURCE_SHORTAGE;
1189 }
1190
1191 /* read our caller */
1192 if (supervisor) {
1193 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1194 } else {
1195 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1196 }
1197
1198 if (kr != KERN_SUCCESS) {
1199 currPC = 0ULL;
1200 break;
1201 }
1202
1203 currPC = (uint64_t) tmpWord; // promote 32 bit address
1204
1205 /*
1206 * retrive contents of the frame pointer and advance to the next stack
1207 * frame if it's valid
1208 */
1209 prevFP = 0;
1210 if (supervisor) {
1211 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1212 } else {
1213 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1214 }
1215 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1216
1217 if (prevFP) {
1218 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1219 prevPC = currPC;
1220 }
1221 if (prevFP < currFP) {
1222 break;
1223 } else {
1224 currFP = prevFP;
1225 }
1226 }
1227
1228 *start_idx = ct;
1229 return KERN_SUCCESS;
1230 }
1231
1232 static kern_return_t
1233 do_backtrace64(
1234 task_t task,
1235 thread_t thread,
1236 x86_saved_state64_t *regs,
1237 uint64_t *frames,
1238 mach_msg_type_number_t *start_idx,
1239 mach_msg_type_number_t max_idx,
1240 boolean_t supervisor)
1241 {
1242 uint64_t currPC = regs->isf.rip;
1243 uint64_t currFP = regs->rbp;
1244 uint64_t prevPC = 0ULL;
1245 uint64_t prevFP = 0ULL;
1246 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1247 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1248 mach_msg_type_number_t ct = *start_idx;
1249 kern_return_t kr = KERN_FAILURE;
1250
1251 if (*start_idx >= max_idx) {
1252 return KERN_RESOURCE_SHORTAGE; // no frames traced
1253 }
1254 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1255
1256 // build a backtrace of this 32 bit state.
1257 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1258 // this is the address where caller lives in the user thread
1259 uint64_t caller = currFP + sizeof(uint64_t);
1260
1261 if (!currFP) {
1262 currPC = 0;
1263 break;
1264 }
1265
1266 if (ct >= max_idx) {
1267 *start_idx = ct;
1268 return KERN_RESOURCE_SHORTAGE;
1269 }
1270
1271 /* read our caller */
1272 if (supervisor) {
1273 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1274 } else {
1275 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1276 }
1277
1278 if (kr != KERN_SUCCESS) {
1279 currPC = 0ULL;
1280 break;
1281 }
1282
1283 /*
1284 * retrive contents of the frame pointer and advance to the next stack
1285 * frame if it's valid
1286 */
1287 prevFP = 0;
1288 if (supervisor) {
1289 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1290 } else {
1291 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1292 }
1293
1294 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1295 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1296 prevPC = currPC;
1297 }
1298 if (prevFP < currFP) {
1299 break;
1300 } else {
1301 currFP = prevFP;
1302 }
1303 }
1304
1305 *start_idx = ct;
1306 return KERN_SUCCESS;
1307 }
1308
1309 static kern_return_t
1310 chudxnu_thread_get_callstack64_internal(
1311 thread_t thread,
1312 uint64_t *callstack,
1313 mach_msg_type_number_t *count,
1314 boolean_t user_only,
1315 boolean_t kern_only)
1316 {
1317 kern_return_t kr = KERN_FAILURE;
1318 task_t task = thread->task;
1319 uint64_t currPC = 0ULL;
1320 boolean_t supervisor = FALSE;
1321 mach_msg_type_number_t bufferIndex = 0;
1322 mach_msg_type_number_t bufferMaxIndex = *count;
1323 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1324 x86_saved_state64_t *regs64 = NULL;
1325 x86_saved_state32_t *regs32 = NULL;
1326 x86_saved_state32_t *u_regs32 = NULL;
1327 x86_saved_state64_t *u_regs64 = NULL;
1328 struct x86_kernel_state *kregs = NULL;
1329
1330 if (ml_at_interrupt_context()) {
1331 if (user_only) {
1332 /* can't backtrace user state on interrupt stack. */
1333 return KERN_FAILURE;
1334 }
1335
1336 /* backtracing at interrupt context? */
1337 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1338 /*
1339 * Locate the registers for the interrupted thread, assuming it is
1340 * current_thread().
1341 */
1342 tagged_regs = current_cpu_datap()->cpu_int_state;
1343
1344 if (is_saved_state64(tagged_regs)) {
1345 /* 64 bit registers */
1346 regs64 = saved_state64(tagged_regs);
1347 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1348 } else {
1349 /* 32 bit registers */
1350 regs32 = saved_state32(tagged_regs);
1351 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1352 }
1353 }
1354 }
1355
1356 if (!ml_at_interrupt_context() && kernel_task == task) {
1357 if (!thread->kernel_stack) {
1358 return KERN_FAILURE;
1359 }
1360
1361 // Kernel thread not at interrupt context
1362 kregs = (struct x86_kernel_state *)NULL;
1363
1364 // nofault read of the thread->kernel_stack pointer
1365 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1366 return KERN_FAILURE;
1367 }
1368
1369 // Adjust to find the saved kernel state
1370 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1371
1372 supervisor = TRUE;
1373 } else if (!tagged_regs) {
1374 /*
1375 * not at interrupt context, or tracing a different thread than
1376 * current_thread() at interrupt context
1377 */
1378 tagged_regs = USER_STATE(thread);
1379 if (is_saved_state64(tagged_regs)) {
1380 /* 64 bit registers */
1381 regs64 = saved_state64(tagged_regs);
1382 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1383 } else {
1384 /* 32 bit registers */
1385 regs32 = saved_state32(tagged_regs);
1386 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1387 }
1388 }
1389
1390 *count = 0;
1391
1392 if (supervisor) {
1393 // the caller only wants a user callstack.
1394 if (user_only) {
1395 // bail - we've only got kernel state
1396 return KERN_FAILURE;
1397 }
1398 } else {
1399 // regs32(64) is not in supervisor mode.
1400 u_regs32 = regs32;
1401 u_regs64 = regs64;
1402 regs32 = NULL;
1403 regs64 = NULL;
1404 }
1405
1406 if (user_only) {
1407 /* we only want to backtrace the user mode */
1408 if (!(u_regs32 || u_regs64)) {
1409 /* no user state to look at */
1410 return KERN_FAILURE;
1411 }
1412 }
1413
1414 /*
1415 * Order of preference for top of stack:
1416 * 64 bit kernel state (not likely)
1417 * 32 bit kernel state
1418 * 64 bit user land state
1419 * 32 bit user land state
1420 */
1421
1422 if (kregs) {
1423 /*
1424 * nofault read of the registers from the kernel stack (as they can
1425 * disappear on the fly).
1426 */
1427
1428 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1429 return KERN_FAILURE;
1430 }
1431 } else if (regs64) {
1432 currPC = regs64->isf.rip;
1433 } else if (regs32) {
1434 currPC = (uint64_t) regs32->eip;
1435 } else if (u_regs64) {
1436 currPC = u_regs64->isf.rip;
1437 } else if (u_regs32) {
1438 currPC = (uint64_t) u_regs32->eip;
1439 }
1440
1441 if (!currPC) {
1442 /* no top of the stack, bail out */
1443 return KERN_FAILURE;
1444 }
1445
1446 bufferIndex = 0;
1447
1448 if (bufferMaxIndex < 1) {
1449 *count = 0;
1450 return KERN_RESOURCE_SHORTAGE;
1451 }
1452
1453 /* backtrace kernel */
1454 if (kregs) {
1455 addr64_t address = 0ULL;
1456 size_t size = 0UL;
1457
1458 // do the backtrace
1459 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1460
1461 // and do a nofault read of (r|e)sp
1462 uint64_t rsp = 0ULL;
1463 size = sizeof(uint64_t);
1464
1465 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1466 address = 0ULL;
1467 }
1468
1469 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1470 callstack[bufferIndex++] = (uint64_t)rsp;
1471 }
1472 } else if (regs64) {
1473 uint64_t rsp = 0ULL;
1474
1475 // backtrace the 64bit side.
1476 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1477 bufferMaxIndex - 1, TRUE);
1478
1479 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1480 bufferIndex < bufferMaxIndex) {
1481 callstack[bufferIndex++] = rsp;
1482 }
1483 } else if (regs32) {
1484 uint32_t esp = 0UL;
1485
1486 // backtrace the 32bit side.
1487 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1488 bufferMaxIndex - 1, TRUE);
1489
1490 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1491 bufferIndex < bufferMaxIndex) {
1492 callstack[bufferIndex++] = (uint64_t) esp;
1493 }
1494 } else if (u_regs64 && !kern_only) {
1495 /* backtrace user land */
1496 uint64_t rsp = 0ULL;
1497
1498 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1499 bufferMaxIndex - 1, FALSE);
1500
1501 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1502 bufferIndex < bufferMaxIndex) {
1503 callstack[bufferIndex++] = rsp;
1504 }
1505 } else if (u_regs32 && !kern_only) {
1506 uint32_t esp = 0UL;
1507
1508 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1509 bufferMaxIndex - 1, FALSE);
1510
1511 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1512 bufferIndex < bufferMaxIndex) {
1513 callstack[bufferIndex++] = (uint64_t) esp;
1514 }
1515 }
1516
1517 *count = bufferIndex;
1518 return kr;
1519 }
1520
1521 __private_extern__
1522 kern_return_t
1523 chudxnu_thread_get_callstack64_kperf(
1524 thread_t thread,
1525 uint64_t *callstack,
1526 mach_msg_type_number_t *count,
1527 boolean_t is_user)
1528 {
1529 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1530 }
1531 #else /* !__arm__ && !__arm64__ && !__x86_64__ */
1532 #error kperf: unsupported architecture
1533 #endif /* !__arm__ && !__arm64__ && !__x86_64__ */
mirror of XNU