xnu-6153.81.5.tar.gz

[apple/xnu.git] / osfmk / kperf / callstack.c

diff --git a/osfmk/kperf/callstack.c b/osfmk/kperf/callstack.c
index 228cd9fe00f9fb2480f7040aed7d7c9d65a4769f..4a38dd7c581523f94daa101d14397c91246ddaab 100644 (file)
--- a/osfmk/kperf/callstack.c
+++ b/osfmk/kperf/callstack.c
@@ -44,10 +44,10 @@
 #endif
 
 static void
 #endif
 
 static void

-callstack_fixup_user(struct callstack *cs, thread_t thread)
+callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)

 {
        uint64_t fixup_val = 0;
 {
        uint64_t fixup_val = 0;

-       assert(cs->nframes < MAX_CALLSTACK_FRAMES);
+       assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);

 
 #if defined(__x86_64__)
        user_addr_t sp_user;
 
 #if defined(__x86_64__)
        user_addr_t sp_user;


@@ -83,7 +83,7 @@ callstack_fixup_user(struct callstack *cs, thread_t thread)
 
        /* encode thumb mode into low bit of PC */
        if (get_saved_state_cpsr(state) & PSR_TF) {
 
        /* encode thumb mode into low bit of PC */
        if (get_saved_state_cpsr(state) & PSR_TF) {

-               cs->frames[0] |= 1ULL;
+               cs->kpuc_frames[0] |= 1ULL;

        }
 
        fixup_val = get_saved_state_lr(state);
        }
 
        fixup_val = get_saved_state_lr(state);


@@ -93,7 +93,7 @@ callstack_fixup_user(struct callstack *cs, thread_t thread)
 #endif
 
 out:
 #endif
 
 out:

-       cs->frames[cs->nframes++] = fixup_val;
+       cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;

 }
 
 #if defined(__x86_64__)
 }
 
 #if defined(__x86_64__)


@@ -186,10 +186,10 @@ interrupted_kernel_lr(uintptr_t *lr)
 
 
 static void
 
 
 static void

-callstack_fixup_interrupted(struct callstack *cs)
+callstack_fixup_interrupted(struct kp_kcallstack *cs)

 {
        uintptr_t fixup_val = 0;
 {
        uintptr_t fixup_val = 0;

-       assert(cs->nframes < MAX_CALLSTACK_FRAMES);
+       assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);

 
        /*
         * Only provide arbitrary data on development or debug kernels.
 
        /*
         * Only provide arbitrary data on development or debug kernels.


@@ -202,12 +202,12 @@ callstack_fixup_interrupted(struct callstack *cs)
 #endif /* defined(__x86_64__) */
 #endif /* DEVELOPMENT || DEBUG */
 
 #endif /* defined(__x86_64__) */
 #endif /* DEVELOPMENT || DEBUG */
 

-       assert(cs->flags & CALLSTACK_KERNEL);
-       cs->frames[cs->nframes++] = fixup_val;
+       assert(cs->kpkc_flags & CALLSTACK_KERNEL);
+       cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;

 }
 
 void
 }
 
 void

-kperf_continuation_sample(struct callstack *cs, struct kperf_context *context)
+kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)

 {
        thread_t thread;
 
 {
        thread_t thread;
 


@@ -218,42 +218,46 @@ kperf_continuation_sample(struct callstack *cs, struct kperf_context *context)
        assert(thread != NULL);
        assert(thread->continuation != NULL);
 
        assert(thread != NULL);
        assert(thread->continuation != NULL);
 

-       cs->flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
+       cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;

 #ifdef __LP64__
 #ifdef __LP64__

-       cs->flags |= CALLSTACK_64BIT;
+       cs->kpkc_flags |= CALLSTACK_64BIT;

 #endif
 
 #endif
 

-       cs->nframes = 1;
-       cs->frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
+       cs->kpkc_nframes = 1;
+       cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);

 }
 
 void
 }
 
 void

-kperf_backtrace_sample(struct callstack *cs, struct kperf_context *context)
+kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)

 {
        assert(cs != NULL);
        assert(context != NULL);
        assert(context->cur_thread == current_thread());
 
 {
        assert(cs != NULL);
        assert(context != NULL);
        assert(context->cur_thread == current_thread());
 

-       cs->flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
+       cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;

 #ifdef __LP64__
 #ifdef __LP64__

-       cs->flags |= CALLSTACK_64BIT;
+       cs->kpkc_flags |= CALLSTACK_64BIT;

 #endif
 
        BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
 
 #endif
 
        BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
 

-       cs->nframes = backtrace_frame((uintptr_t *)&(cs->frames), cs->nframes - 1,
-           context->starting_fp);
-       if (cs->nframes > 0) {
-               cs->flags |= CALLSTACK_VALID;
+       bool trunc = false;
+       cs->kpkc_nframes = backtrace_frame(cs->kpkc_word_frames,
+           cs->kpkc_nframes - 1, context->starting_fp, &trunc);
+       if (cs->kpkc_nframes > 0) {
+               cs->kpkc_flags |= CALLSTACK_VALID;

                /*
                 * Fake the value pointed to by the stack pointer or the link
                 * register for symbolicators.
                 */
                /*
                 * Fake the value pointed to by the stack pointer or the link
                 * register for symbolicators.
                 */

-               cs->frames[cs->nframes + 1] = 0;
-               cs->nframes += 1;
+               cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
+               cs->kpkc_nframes += 1;
+       }
+       if (trunc) {
+               cs->kpkc_flags |= CALLSTACK_TRUNCATED;

        }
 
        }
 

-       BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->nframes);
+       BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);

 }
 
 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
 }
 
 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,


@@ -261,96 +265,96 @@ kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
     boolean_t user_only);
 
 void
     boolean_t user_only);
 
 void

-kperf_kcallstack_sample(struct callstack *cs, struct kperf_context *context)
+kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)

 {
        thread_t thread;
 
        assert(cs != NULL);
        assert(context != NULL);
 {
        thread_t thread;
 
        assert(cs != NULL);
        assert(context != NULL);

-       assert(cs->nframes <= MAX_CALLSTACK_FRAMES);
+       assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);

 
        thread = context->cur_thread;
        assert(thread != NULL);
 
        BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
 
        thread = context->cur_thread;
        assert(thread != NULL);
 
        BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),

-           cs->nframes);
-
-       cs->flags = CALLSTACK_KERNEL;
+           cs->kpkc_nframes);

 
 

+       cs->kpkc_flags = CALLSTACK_KERNEL;

 #ifdef __LP64__
 #ifdef __LP64__

-       cs->flags |= CALLSTACK_64BIT;
+       cs->kpkc_flags |= CALLSTACK_64BIT;

 #endif
 
        if (ml_at_interrupt_context()) {
                assert(thread == current_thread());
 #endif
 
        if (ml_at_interrupt_context()) {
                assert(thread == current_thread());

-               cs->flags |= CALLSTACK_KERNEL_WORDS;
-               cs->nframes = backtrace_interrupted((uintptr_t *)cs->frames,
-                   cs->nframes - 1);
-               if (cs->nframes != 0) {
+               cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
+               bool trunc = false;
+               cs->kpkc_nframes = backtrace_interrupted(
+                   cs->kpkc_word_frames, cs->kpkc_nframes - 1, &trunc);
+               if (cs->kpkc_nframes != 0) {

                        callstack_fixup_interrupted(cs);
                }
                        callstack_fixup_interrupted(cs);
                }

+               if (trunc) {
+                       cs->kpkc_flags |= CALLSTACK_TRUNCATED;
+               }

        } else {
                /*
                 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
                 * other threads.
                 */
                kern_return_t kr;
        } else {
                /*
                 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
                 * other threads.
                 */
                kern_return_t kr;

-               kr = chudxnu_thread_get_callstack64_kperf(thread, cs->frames,
-                   &cs->nframes, FALSE);
+               kr = chudxnu_thread_get_callstack64_kperf(thread,
+                   cs->kpkc_frames, &cs->kpkc_nframes, FALSE);

                if (kr == KERN_SUCCESS) {
                if (kr == KERN_SUCCESS) {

-                       cs->flags |= CALLSTACK_VALID;
+                       cs->kpkc_flags |= CALLSTACK_VALID;

                } else if (kr == KERN_RESOURCE_SHORTAGE) {
                } else if (kr == KERN_RESOURCE_SHORTAGE) {

-                       cs->flags |= CALLSTACK_VALID;
-                       cs->flags |= CALLSTACK_TRUNCATED;
+                       cs->kpkc_flags |= CALLSTACK_VALID;
+                       cs->kpkc_flags |= CALLSTACK_TRUNCATED;

                } else {
                } else {

-                       cs->nframes = 0;
+                       cs->kpkc_nframes = 0;

                }
        }
 
                }
        }
 

-       if (cs->nframes == 0) {
+       if (!(cs->kpkc_flags & CALLSTACK_VALID)) {

                BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
        }
 
                BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
        }
 

-       BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->flags, cs->nframes);
+       BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
+           cs->kpkc_flags, cs->kpkc_nframes);

 }
 
 void
 }
 
 void

-kperf_ucallstack_sample(struct callstack *cs, struct kperf_context *context)
+kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)

 {
 {

-       thread_t thread;
-       bool user_64 = false;
-       int err;
-
-       assert(cs != NULL);
-       assert(context != NULL);
-       assert(cs->nframes <= MAX_CALLSTACK_FRAMES);

        assert(ml_get_interrupts_enabled() == TRUE);
 
        assert(ml_get_interrupts_enabled() == TRUE);
 

-       thread = context->cur_thread;
+       thread_t thread = context->cur_thread;

        assert(thread != NULL);
 
        assert(thread != NULL);
 

-       BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
-           cs->nframes);
-
-       cs->flags = 0;
+       BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
+           (uintptr_t)thread_tid(thread), cs->kpuc_nframes);

 
 

-       err = backtrace_thread_user(thread, (uintptr_t *)cs->frames,
-           cs->nframes - 1, &cs->nframes, &user_64);
-       cs->flags |= CALLSTACK_KERNEL_WORDS;
-       if (user_64) {
-               cs->flags |= CALLSTACK_64BIT;
+       bool user64 = false;
+       bool trunc = false;
+       int err = backtrace_thread_user(thread, cs->kpuc_frames,
+           cs->kpuc_nframes - 1, &cs->kpuc_nframes, &user64, &trunc);
+       cs->kpuc_flags = CALLSTACK_KERNEL_WORDS;
+       if (user64) {
+               cs->kpuc_flags |= CALLSTACK_64BIT;
+       }
+       if (trunc) {
+               cs->kpuc_flags |= CALLSTACK_TRUNCATED;

        }
 
        if (!err || err == EFAULT) {
                callstack_fixup_user(cs, thread);
        }
 
        if (!err || err == EFAULT) {
                callstack_fixup_user(cs, thread);

-               cs->flags |= CALLSTACK_VALID;
+               cs->kpuc_flags |= CALLSTACK_VALID;

        } else {
        } else {

-               cs->nframes = 0;
+               cs->kpuc_nframes = 0;

                BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err);
        }
 
        BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
                BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, err);
        }
 
        BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),

-           cs->flags, cs->nframes);
+           cs->kpuc_flags, cs->kpuc_nframes);

 }
 
 static inline uintptr_t
 }
 
 static inline uintptr_t


@@ -378,38 +382,36 @@ scrub_frame(uint64_t *bt, int n_frames, int frame)
 }
 
 static void
 }
 
 static void

-callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode)
+callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
+    unsigned int nframes, unsigned int flags)

 {
 {

-       BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, cs->flags, cs->nframes);
+       BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);

 
 

-       /* framing information for the stack */
-       BUF_DATA(hcode, cs->flags, cs->nframes);
+       BUF_DATA(hdrid, flags, nframes);

 
 

-       /* how many batches of 4 */
-       unsigned int nframes = cs->nframes;
-       unsigned int n = nframes / 4;
+       unsigned int nevts = nframes / 4;

        unsigned int ovf = nframes % 4;
        if (ovf != 0) {
        unsigned int ovf = nframes % 4;
        if (ovf != 0) {

-               n++;
+               nevts++;

        }
 
        }
 

-       bool kern = cs->flags & CALLSTACK_KERNEL;
+       bool kern = flags & CALLSTACK_KERNEL;

 
 

-       if (cs->flags & CALLSTACK_KERNEL_WORDS) {
-               uintptr_t *frames = (uintptr_t *)cs->frames;
-               for (unsigned int i = 0; i < n; i++) {
+       if (flags & CALLSTACK_KERNEL_WORDS) {
+               uintptr_t *frames = vframes;
+               for (unsigned int i = 0; i < nevts; i++) {

                        unsigned int j = i * 4;
                        unsigned int j = i * 4;

-                       BUF_DATA(dcode,
+                       BUF_DATA(dataid,

                            scrub_word(frames, nframes, j + 0, kern),
                            scrub_word(frames, nframes, j + 1, kern),
                            scrub_word(frames, nframes, j + 2, kern),
                            scrub_word(frames, nframes, j + 3, kern));
                }
        } else {
                            scrub_word(frames, nframes, j + 0, kern),
                            scrub_word(frames, nframes, j + 1, kern),
                            scrub_word(frames, nframes, j + 2, kern),
                            scrub_word(frames, nframes, j + 3, kern));
                }
        } else {

-               for (unsigned int i = 0; i < n; i++) {
-                       uint64_t *frames = cs->frames;
+               for (unsigned int i = 0; i < nevts; i++) {
+                       uint64_t *frames = vframes;

                        unsigned int j = i * 4;
                        unsigned int j = i * 4;

-                       BUF_DATA(dcode,
+                       BUF_DATA(dataid,

                            scrub_frame(frames, nframes, j + 0),
                            scrub_frame(frames, nframes, j + 1),
                            scrub_frame(frames, nframes, j + 2),
                            scrub_frame(frames, nframes, j + 0),
                            scrub_frame(frames, nframes, j + 1),
                            scrub_frame(frames, nframes, j + 2),


@@ -417,28 +419,33 @@ callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode)
                }
        }
 
                }
        }
 

-       BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, cs->flags, cs->nframes);
+       BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);

 }
 
 void
 }
 
 void

-kperf_kcallstack_log( struct callstack *cs )
+kperf_kcallstack_log(struct kp_kcallstack *cs)

 {
 {

-       callstack_log(cs, PERF_CS_KHDR, PERF_CS_KDATA);
+       callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
+           cs->kpkc_nframes, cs->kpkc_flags);

 }
 
 void
 }
 
 void

-kperf_ucallstack_log( struct callstack *cs )
+kperf_ucallstack_log(struct kp_ucallstack *cs)

 {
 {

-       callstack_log(cs, PERF_CS_UHDR, PERF_CS_UDATA);
+       callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
+           cs->kpuc_nframes, cs->kpuc_flags);

 }
 
 int
 }
 
 int

-kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth)
+kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
+    unsigned int actionid)

 {
 {

-       int did_pend = kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK);
+       if (depth < 2) {
+               panic("HUH");
+       }

        kperf_ast_set_callstack_depth(context->cur_thread, depth);
        kperf_ast_set_callstack_depth(context->cur_thread, depth);

-
-       return did_pend;
+       return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
+           actionid);

 }
 
 static kern_return_t
 }
 
 static kern_return_t


@@ -662,6 +669,9 @@ chudxnu_thread_get_callstack64_kperf(
 }
 #elif __arm64__
 
 }
 #elif __arm64__
 

+#if defined(HAS_APPLE_PAC)
+#include <ptrauth.h>
+#endif

 
 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
 
 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)


@@ -789,7 +799,12 @@ chudxnu_thread_get_callstack64_internal(
                                            (vm_offset_t)fp,
                                            (vm_size_t)sizeof(frame));
                                        if (kr == KERN_SUCCESS) {
                                            (vm_offset_t)fp,
                                            (vm_size_t)sizeof(frame));
                                        if (kr == KERN_SUCCESS) {

+#if defined(HAS_APPLE_PAC)
+                                               /* return addresses on stack will be signed by arm64e ABI */
+                                               pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
+#else

                                                pc = frame[1];
                                                pc = frame[1];

+#endif

                                                nextFramePointer = (uint64_t *)frame[0];
                                        } else {
                                                pc = 0ULL;
                                                nextFramePointer = (uint64_t *)frame[0];
                                        } else {
                                                pc = 0ULL;


@@ -803,7 +818,12 @@ chudxnu_thread_get_callstack64_internal(
                                    (vm_offset_t)fp,
                                    (vm_size_t)sizeof(frame));
                                if (kr == KERN_SUCCESS) {
                                    (vm_offset_t)fp,
                                    (vm_size_t)sizeof(frame));
                                if (kr == KERN_SUCCESS) {

+#if defined(HAS_APPLE_PAC)
+                                       /* return addresses on stack will be signed by arm64e ABI */
+                                       pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
+#else

                                        pc = frame[1];
                                        pc = frame[1];

+#endif

                                        nextFramePointer = (uint64_t *)(frame[0]);
                                } else {
                                        pc = 0ULL;
                                        nextFramePointer = (uint64_t *)(frame[0]);
                                } else {
                                        pc = 0ULL;