+ return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
+ actionid);
+}
+
+static kern_return_t
+chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
+{
+ return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
+ KERN_SUCCESS : KERN_FAILURE;
+}
+
+static kern_return_t
+chudxnu_task_read(
+ task_t task,
+ void *kernaddr,
+ uint64_t usraddr,
+ vm_size_t size)
+{
+ //ppc version ported to arm
+ kern_return_t ret = KERN_SUCCESS;
+
+ if (ml_at_interrupt_context()) {
+ return KERN_FAILURE; // can't look at tasks on interrupt stack
+ }
+
+ if (current_task() == task) {
+ if (copyin(usraddr, kernaddr, size)) {
+ ret = KERN_FAILURE;
+ }
+ } else {
+ vm_map_t map = get_task_map(task);
+ ret = vm_map_read_user(map, usraddr, kernaddr, size);
+ }
+
+ return ret;
+}
+
+static inline uint64_t
+chudxnu_vm_unslide( uint64_t ptr, int kaddr )
+{
+ if (!kaddr) {
+ return ptr;
+ }
+
+ return VM_KERNEL_UNSLIDE(ptr);
+}
+
+#if __arm__
+#define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
+#define CS_FLAG_EXTRASP 1 // capture extra sp register
+static kern_return_t
+chudxnu_thread_get_callstack64_internal(
+ thread_t thread,
+ uint64_t *callStack,
+ mach_msg_type_number_t *count,
+ boolean_t user_only,
+ int flags)
+{
+ kern_return_t kr;
+ task_t task;
+ uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
+ uint64_t prevPC = 0ULL;
+ uint32_t kernStackMin = thread->kernel_stack;
+ uint32_t kernStackMax = kernStackMin + kernel_stack_size;
+ uint64_t *buffer = callStack;
+ uint32_t frame[2];
+ int bufferIndex = 0;
+ int bufferMaxIndex = 0;
+ boolean_t supervisor = FALSE;
+ struct arm_saved_state *state = NULL;
+ uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
+ uint64_t pc = 0ULL;
+
+ task = get_threadtask(thread);
+
+ bufferMaxIndex = *count;
+ //get thread state
+ if (user_only) {
+ state = find_user_regs(thread);
+ } else {
+ state = find_kern_regs(thread);
+ }
+
+ if (!state) {
+ *count = 0;
+ return KERN_FAILURE;
+ }
+
+ /* make sure it is safe to dereference before you do it */
+ supervisor = ARM_SUPERVISOR_MODE(state->cpsr);
+
+ /* can't take a kernel callstack if we've got a user frame */
+ if (!user_only && !supervisor) {
+ return KERN_FAILURE;
+ }
+
+ /*
+ * Reserve space for saving LR (and sometimes SP) at the end of the
+ * backtrace.
+ */
+ if (flags & CS_FLAG_EXTRASP) {
+ bufferMaxIndex -= 2;
+ } else {
+ bufferMaxIndex -= 1;
+ }
+
+ if (bufferMaxIndex < 2) {
+ *count = 0;
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ currPC = (uint64_t)state->pc; /* r15 */
+ if (state->cpsr & PSR_TF) {
+ currPC |= 1ULL; /* encode thumb mode into low bit of PC */
+ }
+ currLR = (uint64_t)state->lr; /* r14 */
+ currSP = (uint64_t)state->sp; /* r13 */
+
+ fp = (uint32_t *)state->r[7]; /* frame pointer */
+ topfp = fp;
+
+ bufferIndex = 0; // start with a stack of size zero
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, supervisor); // save PC in position 0.
+
+ // Now, fill buffer with stack backtraces.
+ while (bufferIndex < bufferMaxIndex) {
+ pc = 0ULL;
+ /*
+ * Below the frame pointer, the following values are saved:
+ * -> FP
+ */
+
+ /*
+ * Note that we read the pc even for the first stack frame
+ * (which, in theory, is always empty because the callee fills
+ * it in just before it lowers the stack. However, if we
+ * catch the program in between filling in the return address
+ * and lowering the stack, we want to still have a valid
+ * backtrace. FixupStack correctly disregards this value if
+ * necessary.
+ */
+
+ if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
+ /* frame pointer is invalid - stop backtracing */
+ pc = 0ULL;
+ break;
+ }
+
+ if (supervisor) {
+ if (((uint32_t)fp > kernStackMax) ||
+ ((uint32_t)fp < kernStackMin)) {
+ kr = KERN_FAILURE;
+ } else {
+ kr = chudxnu_kern_read(&frame,
+ (vm_offset_t)fp,
+ (vm_size_t)sizeof(frame));
+ if (kr == KERN_SUCCESS) {
+ pc = (uint64_t)frame[1];
+ nextFramePointer = (uint32_t *) (frame[0]);
+ } else {
+ pc = 0ULL;
+ nextFramePointer = 0ULL;
+ kr = KERN_FAILURE;
+ }
+ }
+ } else {
+ kr = chudxnu_task_read(task,
+ &frame,
+ (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
+ sizeof(frame));
+ if (kr == KERN_SUCCESS) {
+ pc = (uint64_t) frame[1];
+ nextFramePointer = (uint32_t *) (frame[0]);
+ } else {
+ pc = 0ULL;
+ nextFramePointer = 0ULL;
+ kr = KERN_FAILURE;
+ }
+ }
+
+ if (kr != KERN_SUCCESS) {
+ pc = 0ULL;
+ break;
+ }
+
+ if (nextFramePointer) {
+ buffer[bufferIndex++] = chudxnu_vm_unslide(pc, supervisor);
+ prevPC = pc;
+ }
+
+ if (nextFramePointer < fp) {
+ break;
+ } else {
+ fp = nextFramePointer;
+ }
+ }
+
+ if (bufferIndex >= bufferMaxIndex) {
+ bufferIndex = bufferMaxIndex;
+ kr = KERN_RESOURCE_SHORTAGE;
+ } else {
+ kr = KERN_SUCCESS;
+ }
+
+ // Save link register and R13 (sp) at bottom of stack (used for later fixup).
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, supervisor);
+ if (flags & CS_FLAG_EXTRASP) {
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, supervisor);
+ }
+
+ *count = bufferIndex;
+ return kr;
+}
+
+kern_return_t
+chudxnu_thread_get_callstack64_kperf(
+ thread_t thread,
+ uint64_t *callStack,
+ mach_msg_type_number_t *count,
+ boolean_t user_only)
+{
+ return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
+}
+#elif __arm64__
+
+#if defined(HAS_APPLE_PAC)
+#include <ptrauth.h>
+#endif
+
+// chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
+// fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
+// after sampling has finished.
+//
+// For an N-entry callstack:
+//
+// [0] current pc
+// [1..N-3] stack frames (including current one)
+// [N-2] current LR (return value if we're in a leaf function)
+// [N-1] current r0 (in case we've saved LR in r0) (optional)
+//
+//
+#define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
+
+#define CS_FLAG_EXTRASP 1 // capture extra sp register
+
+static kern_return_t
+chudxnu_thread_get_callstack64_internal(
+ thread_t thread,
+ uint64_t *callStack,
+ mach_msg_type_number_t *count,
+ boolean_t user_only,
+ int flags)
+{
+ kern_return_t kr = KERN_SUCCESS;
+ task_t task;
+ uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
+ uint64_t prevPC = 0ULL;
+ uint64_t kernStackMin = thread->kernel_stack;
+ uint64_t kernStackMax = kernStackMin + kernel_stack_size;
+ uint64_t *buffer = callStack;
+ int bufferIndex = 0;
+ int bufferMaxIndex = 0;
+ boolean_t kernel = FALSE;
+ struct arm_saved_state *sstate = NULL;
+ uint64_t pc = 0ULL;
+
+ task = get_threadtask(thread);
+ bufferMaxIndex = *count;
+ //get thread state
+ if (user_only) {
+ sstate = find_user_regs(thread);
+ } else {
+ sstate = find_kern_regs(thread);
+ }
+
+ if (!sstate) {
+ *count = 0;
+ return KERN_FAILURE;
+ }
+
+ if (is_saved_state64(sstate)) {
+ struct arm_saved_state64 *state = NULL;
+ uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
+ uint64_t frame[2];
+
+ state = saved_state64(sstate);
+
+ /* make sure it is safe to dereference before you do it */
+ kernel = PSR64_IS_KERNEL(state->cpsr);
+
+ /* can't take a kernel callstack if we've got a user frame */
+ if (!user_only && !kernel) {
+ return KERN_FAILURE;
+ }
+
+ /*
+ * Reserve space for saving LR (and sometimes SP) at the end of the
+ * backtrace.
+ */
+ if (flags & CS_FLAG_EXTRASP) {
+ bufferMaxIndex -= 2;
+ } else {
+ bufferMaxIndex -= 1;
+ }
+
+ if (bufferMaxIndex < 2) {
+ *count = 0;
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ currPC = state->pc;
+ currLR = state->lr;
+ currSP = state->sp;
+
+ fp = (uint64_t *)state->fp; /* frame pointer */
+ topfp = fp;
+
+ bufferIndex = 0; // start with a stack of size zero
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
+
+ BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
+
+ // Now, fill buffer with stack backtraces.
+ while (bufferIndex < bufferMaxIndex) {
+ pc = 0ULL;
+ /*
+ * Below the frame pointer, the following values are saved:
+ * -> FP
+ */
+
+ /*
+ * Note that we read the pc even for the first stack frame
+ * (which, in theory, is always empty because the callee fills
+ * it in just before it lowers the stack. However, if we
+ * catch the program in between filling in the return address
+ * and lowering the stack, we want to still have a valid
+ * backtrace. FixupStack correctly disregards this value if
+ * necessary.
+ */
+
+ if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
+ /* frame pointer is invalid - stop backtracing */
+ pc = 0ULL;
+ break;
+ }
+
+ if (kernel) {
+ if (((uint64_t)fp > kernStackMax) ||
+ ((uint64_t)fp < kernStackMin)) {
+ kr = KERN_FAILURE;
+ } else {
+ kr = chudxnu_kern_read(&frame,
+ (vm_offset_t)fp,
+ (vm_size_t)sizeof(frame));
+ if (kr == KERN_SUCCESS) {
+#if defined(HAS_APPLE_PAC)
+ /* return addresses on stack will be signed by arm64e ABI */
+ pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
+#else
+ pc = frame[1];
+#endif
+ nextFramePointer = (uint64_t *)frame[0];
+ } else {
+ pc = 0ULL;
+ nextFramePointer = 0ULL;
+ kr = KERN_FAILURE;
+ }
+ }
+ } else {
+ kr = chudxnu_task_read(task,
+ &frame,
+ (vm_offset_t)fp,
+ (vm_size_t)sizeof(frame));
+ if (kr == KERN_SUCCESS) {
+#if defined(HAS_APPLE_PAC)
+ /* return addresses on stack will be signed by arm64e ABI */
+ pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
+#else
+ pc = frame[1];
+#endif
+ nextFramePointer = (uint64_t *)(frame[0]);
+ } else {
+ pc = 0ULL;
+ nextFramePointer = 0ULL;
+ kr = KERN_FAILURE;
+ }
+ }
+
+ if (kr != KERN_SUCCESS) {
+ pc = 0ULL;
+ break;
+ }
+
+ if (nextFramePointer) {
+ buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
+ prevPC = pc;
+ }
+
+ if (nextFramePointer < fp) {
+ break;
+ } else {
+ fp = nextFramePointer;
+ }
+ }
+
+ BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
+
+ if (bufferIndex >= bufferMaxIndex) {
+ bufferIndex = bufferMaxIndex;
+ kr = KERN_RESOURCE_SHORTAGE;
+ } else {
+ kr = KERN_SUCCESS;
+ }
+
+ // Save link register and SP at bottom of stack (used for later fixup).
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
+ if (flags & CS_FLAG_EXTRASP) {
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
+ }
+ } else {
+ struct arm_saved_state32 *state = NULL;
+ uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
+
+ /* 64-bit kernel stacks, 32-bit user stacks */
+ uint64_t frame[2];
+ uint32_t frame32[2];
+
+ state = saved_state32(sstate);
+
+ /* make sure it is safe to dereference before you do it */
+ kernel = ARM_SUPERVISOR_MODE(state->cpsr);
+
+ /* can't take a kernel callstack if we've got a user frame */
+ if (!user_only && !kernel) {
+ return KERN_FAILURE;
+ }
+
+ /*
+ * Reserve space for saving LR (and sometimes SP) at the end of the
+ * backtrace.
+ */
+ if (flags & CS_FLAG_EXTRASP) {
+ bufferMaxIndex -= 2;
+ } else {
+ bufferMaxIndex -= 1;
+ }
+
+ if (bufferMaxIndex < 2) {
+ *count = 0;
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ currPC = (uint64_t)state->pc; /* r15 */
+ if (state->cpsr & PSR_TF) {
+ currPC |= 1ULL; /* encode thumb mode into low bit of PC */
+ }
+ currLR = (uint64_t)state->lr; /* r14 */
+ currSP = (uint64_t)state->sp; /* r13 */
+
+ fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
+ topfp = fp;
+
+ bufferIndex = 0; // start with a stack of size zero
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
+
+ BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
+
+ // Now, fill buffer with stack backtraces.
+ while (bufferIndex < bufferMaxIndex) {
+ pc = 0ULL;
+ /*
+ * Below the frame pointer, the following values are saved:
+ * -> FP
+ */
+
+ /*
+ * Note that we read the pc even for the first stack frame
+ * (which, in theory, is always empty because the callee fills
+ * it in just before it lowers the stack. However, if we
+ * catch the program in between filling in the return address
+ * and lowering the stack, we want to still have a valid
+ * backtrace. FixupStack correctly disregards this value if
+ * necessary.
+ */
+
+ if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
+ /* frame pointer is invalid - stop backtracing */
+ pc = 0ULL;
+ break;
+ }
+
+ if (kernel) {
+ if (((uint32_t)fp > kernStackMax) ||
+ ((uint32_t)fp < kernStackMin)) {
+ kr = KERN_FAILURE;
+ } else {
+ kr = chudxnu_kern_read(&frame,
+ (vm_offset_t)fp,
+ (vm_size_t)sizeof(frame));
+ if (kr == KERN_SUCCESS) {
+ pc = (uint64_t)frame[1];
+ nextFramePointer = (uint32_t *) (frame[0]);
+ } else {
+ pc = 0ULL;
+ nextFramePointer = 0ULL;
+ kr = KERN_FAILURE;
+ }
+ }
+ } else {
+ kr = chudxnu_task_read(task,
+ &frame32,
+ (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
+ sizeof(frame32));
+ if (kr == KERN_SUCCESS) {
+ pc = (uint64_t)frame32[1];
+ nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
+ } else {
+ pc = 0ULL;
+ nextFramePointer = 0ULL;
+ kr = KERN_FAILURE;
+ }
+ }
+
+ if (kr != KERN_SUCCESS) {
+ pc = 0ULL;
+ break;
+ }
+
+ if (nextFramePointer) {
+ buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
+ prevPC = pc;
+ }
+
+ if (nextFramePointer < fp) {
+ break;
+ } else {
+ fp = nextFramePointer;
+ }
+ }
+
+ BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
+
+ /* clamp callstack size to max */
+ if (bufferIndex >= bufferMaxIndex) {
+ bufferIndex = bufferMaxIndex;
+ kr = KERN_RESOURCE_SHORTAGE;
+ } else {
+ /* ignore all other failures */
+ kr = KERN_SUCCESS;
+ }
+
+ // Save link register and R13 (sp) at bottom of stack (used for later fixup).
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
+ if (flags & CS_FLAG_EXTRASP) {
+ buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
+ }
+ }
+
+ *count = bufferIndex;
+ return kr;
+}
+
+kern_return_t
+chudxnu_thread_get_callstack64_kperf(
+ thread_t thread,
+ uint64_t *callStack,
+ mach_msg_type_number_t *count,
+ boolean_t user_only)
+{
+ return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
+}
+#elif __x86_64__
+
+#define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
+// don't try to read in the hole
+#define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
+(supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
+((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
+
+typedef struct _cframe64_t {
+ uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
+ uint64_t caller;
+ uint64_t args[0];
+}cframe64_t;
+
+
+typedef struct _cframe_t {
+ uint32_t prev; // this is really a user32-space pointer to the previous frame
+ uint32_t caller;
+ uint32_t args[0];
+} cframe_t;
+
+extern void * find_user_regs(thread_t);
+extern x86_saved_state32_t *find_kern_regs(thread_t);
projects / apple / xnu.git / blobdiff