X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d1ecb069dfe24481e4a83f44cb5217a2b06746d7..c6bf4f310a33a9262d455ea4d3f0630b1255e3fe:/bsd/dev/dtrace/systrace.c?ds=sidebyside diff --git a/bsd/dev/dtrace/systrace.c b/bsd/dev/dtrace/systrace.c index 74ab8a105..ef85a1fca 100644 --- a/bsd/dev/dtrace/systrace.c +++ b/bsd/dev/dtrace/systrace.c @@ -19,41 +19,19 @@ * CDDL HEADER END */ /* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -/* #pragma ident "@(#)systrace.c 1.6 06/09/19 SMI" */ - -#if !defined(__APPLE__) -#include -#include -#include -#include -#include -#include -#include -#include -#define SYSTRACE_ARTIFICIAL_FRAMES 1 -#else - -#ifdef KERNEL -#ifndef _KERNEL -#define _KERNEL /* Solaris vs. Darwin */ -#endif -#endif - -#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */ #include #include + /* XXX All of these should really be derived from syscall_sw.h */ -#if defined(__i386__) || defined (__x86_64__) +#if defined (__x86_64__) #define SYSCALL_CLASS_SHIFT 24 #define SYSCALL_CLASS_MASK (0xFF << SYSCALL_CLASS_SHIFT) #define SYSCALL_NUMBER_MASK (~SYSCALL_CLASS_MASK) #define I386_SYSCALL_NUMBER_MASK (0xFFFF) - -typedef x86_saved_state_t savearea_t; #endif #include @@ -63,26 +41,32 @@ typedef x86_saved_state_t savearea_t; #include #include #include +#include #include #include #include +#include #include "systrace.h" #include #include #include #include -#if defined (__ppc__) || defined (__ppc64__) -#define SYSTRACE_ARTIFICIAL_FRAMES 3 -#define MACHTRACE_ARTIFICIAL_FRAMES 4 -#elif defined(__i386__) || defined (__x86_64__) -#define SYSTRACE_ARTIFICIAL_FRAMES 2 +#include + +#if defined (__x86_64__) +#define SYSTRACE_ARTIFICIAL_FRAMES 2 +#define MACHTRACE_ARTIFICIAL_FRAMES 3 +#elif defined(__arm__) || defined(__arm64__) +#define SYSTRACE_ARTIFICIAL_FRAMES 2 #define MACHTRACE_ARTIFICIAL_FRAMES 3 #else #error Unknown Architecture #endif +#define SYSTRACE_NARGS (int)(sizeof(((uthread_t)NULL)->uu_arg) / sizeof(((uthread_t)NULL)->uu_arg[0])) + #include #define sy_callc sy_call /* Map Solaris slot name to Darwin's */ #define NSYSCALL nsysent /* and is less than 500 or so */ @@ -96,50 +80,37 @@ extern const char *syscallnames[]; #define LOADABLE_SYSCALL(a) 0 /* Not pertinent to Darwin. */ #define LOADED_SYSCALL(a) 1 /* Not pertinent to Darwin. */ +extern lck_attr_t* dtrace_lck_attr; +extern lck_grp_t* dtrace_lck_grp; +static lck_mtx_t dtrace_systrace_lock; /* probe state lock */ + systrace_sysent_t *systrace_sysent = NULL; -void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t, - uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); +void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); + +static uint64_t systrace_getargval(void *, dtrace_id_t, void *, int, int); +static void systrace_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); void systrace_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1, - uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, uint64_t arg6, uint64_t arg7) + uint64_t arg2, uint64_t arg3, uint64_t arg4) { -#pragma unused(id,arg0,arg1,arg2,arg3,arg4,arg5,arg6,arg7) +#pragma unused(id,arg0,arg1,arg2,arg3,arg4) } - int32_t dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) { - boolean_t flavor; - unsigned short code; + unsigned short code; /* The system call number */ systrace_sysent_t *sy; dtrace_id_t id; int32_t rval; -#if 0 /* XXX */ - proc_t *p; -#endif syscall_arg_t *ip = (syscall_arg_t *)uap; + uint64_t uargs[SYSTRACE_NARGS] = {0}; -#if defined (__ppc__) || defined (__ppc64__) - { - savearea_t *regs = (savearea_t *)find_user_regs(current_thread()); - - flavor = (((unsigned int)regs->save_r0) == 0)? 1: 0; - - if (flavor) - code = regs->save_r3; - else - code = regs->save_r0; - - /* - * FIXME: unix_syscall screens for "unsafe calls" and instead calls nosys(), *not* sysent[code] ! - */ - } -#elif defined(__i386__) || defined (__x86_64__) -#pragma unused(flavor) +#if defined (__x86_64__) { + pal_register_cache_state(current_thread(), VALID); x86_saved_state_t *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread()); if (is_saved_state64(tagged_regs)) { @@ -156,27 +127,83 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) code = saved_state32(tagged_regs)->eax & I386_SYSCALL_NUMBER_MASK; if (code == 0) { - vm_offset_t params = (vm_offset_t) (saved_state32(tagged_regs)->uesp + sizeof (int)); + vm_offset_t params = (vm_offset_t) (saved_state32(tagged_regs)->uesp + sizeof(int)); code = fuword(params); } } } +#elif defined(__arm__) + { + /* + * On arm, syscall numbers depend on a flavor (indirect or not) + * and can be in either r0 or r12 (always u32) + */ + + /* See bsd/dev/arm/systemcalls.c:arm_get_syscall_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + + /* Check for indirect system call */ + if (arm_regs->r[12] != 0) { + code = arm_regs->r[12]; + } else { + code = arm_regs->r[0]; + } + } +#elif defined(__arm64__) + { + /* + * On arm64, syscall numbers depend on a flavor (indirect or not) + * ... and for u32 can be in either r0 or r12 + * ... and for u64 can be in either x0 or x16 + */ + + /* see bsd/dev/arm/systemcalls.c:arm_get_syscall_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + + if (is_saved_state32(arm_regs)) { + /* Check for indirect system call */ + if (saved_state32(arm_regs)->r[12] != 0) { + code = saved_state32(arm_regs)->r[12]; + } else { + code = saved_state32(arm_regs)->r[0]; + } + } else { + /* Check for indirect system call */ + if (saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM] != 0) { + code = saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } else { + code = saved_state64(arm_regs)->x[0]; + } + } + } #else #error Unknown Architecture #endif // Bounds "check" the value of code a la unix_syscall - sy = (code >= NUM_SYSENT) ? &systrace_sysent[63] : &systrace_sysent[code]; + sy = (code >= nsysent) ? &systrace_sysent[SYS_invalid] : &systrace_sysent[code]; + + systrace_args(code, ip, uargs); if ((id = sy->stsy_entry) != DTRACE_IDNONE) { - if (ip) - (*systrace_probe)(id, *ip, *(ip+1), *(ip+2), *(ip+3), *(ip+4), *(ip+5), *(ip+6), *(ip+7)); - else - (*systrace_probe)(id, 0, 0, 0, 0, 0, 0, 0, 0); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + if (uthread) { + uthread->t_dtrace_syscall_args = uargs; + } + + static_assert(SYSTRACE_NARGS >= 5, "not enough system call arguments"); + (*systrace_probe)(id, uargs[0], uargs[1], uargs[2], uargs[3], uargs[4]); + + if (uthread) { + uthread->t_dtrace_syscall_args = NULL; + } } + + #if 0 /* XXX */ /* + * APPLE NOTE: Not implemented. * We want to explicitly allow DTrace consumers to stop a process * before it actually executes the meat of the syscall. */ @@ -193,14 +220,14 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) if ((id = sy->stsy_return) != DTRACE_IDNONE) { uint64_t munged_rv0, munged_rv1; - uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); - if (uthread) + if (uthread) { uthread->t_dtrace_errno = rval; /* Establish t_dtrace_errno now in case this enabling refers to it. */ - + } /* - * "Decode" rv for use in the call to dtrace_probe() - */ + * "Decode" rv for use in the call to dtrace_probe() + */ if (rval == ERESTART) { munged_rv0 = -1LL; /* System call will be reissued in user mode. Make DTrace report a -1 return. */ munged_rv1 = -1LL; @@ -250,20 +277,20 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) * "This is a bit of an historical artifact. At first, the syscall provider just * had its return value in arg0, and the fbt and pid providers had their return * values in arg1 (so that we could use arg0 for the offset of the return site). - * + * * We inevitably started writing scripts where we wanted to see the return * values from probes in all three providers, and we made this script easier * to write by replicating the syscall return values in arg1 to match fbt and * pid. We debated briefly about removing the return value from arg0, but * decided that it would be less confusing to have the same data in two places * than to have some non-helpful, non-intuitive value in arg0. - * + * * This change was made 4/23/2003 according to the DTrace project's putback log." - */ - (*systrace_probe)(id, munged_rv0, munged_rv0, munged_rv1, (uint64_t)rval, 0, 0, 0, 0); + */ + (*systrace_probe)(id, munged_rv0, munged_rv0, munged_rv1, (uint64_t)rval, 0); } - return (rval); + return rval; } void @@ -273,18 +300,18 @@ dtrace_systrace_syscall_return(unsigned short code, int rval, int *rv) dtrace_id_t id; // Bounds "check" the value of code a la unix_syscall_return - sy = (code >= NUM_SYSENT) ? &systrace_sysent[63] : &systrace_sysent[code]; + sy = (code >= nsysent) ? &systrace_sysent[SYS_invalid] : &systrace_sysent[code]; if ((id = sy->stsy_return) != DTRACE_IDNONE) { uint64_t munged_rv0, munged_rv1; - uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); - if (uthread) + if (uthread) { uthread->t_dtrace_errno = rval; /* Establish t_dtrace_errno now in case this enabling refers to it. */ - + } /* - * "Decode" rv for use in the call to dtrace_probe() - */ + * "Decode" rv for use in the call to dtrace_probe() + */ if (rval == ERESTART) { munged_rv0 = -1LL; /* System call will be reissued in user mode. Make DTrace report a -1 return. */ munged_rv1 = -1LL; @@ -328,66 +355,37 @@ dtrace_systrace_syscall_return(unsigned short code, int rval, int *rv) munged_rv1 = 0LL; } - (*systrace_probe)(id, munged_rv0, munged_rv0, munged_rv1, (uint64_t)rval, 0, 0, 0, 0); + (*systrace_probe)(id, munged_rv0, munged_rv0, munged_rv1, (uint64_t)rval, 0); } } -#endif /* __APPLE__ */ -#define SYSTRACE_SHIFT 16 -#define SYSTRACE_ISENTRY(x) ((int)(x) >> SYSTRACE_SHIFT) -#define SYSTRACE_SYSNUM(x) ((int)(x) & ((1 << SYSTRACE_SHIFT) - 1)) -#define SYSTRACE_ENTRY(id) ((1 << SYSTRACE_SHIFT) | (id)) -#define SYSTRACE_RETURN(id) (id) +#define SYSTRACE_SHIFT 16 +#define SYSTRACE_ISENTRY(x) ((int)(x) >> SYSTRACE_SHIFT) +#define SYSTRACE_SYSNUM(x) ((int)(x) & ((1 << SYSTRACE_SHIFT) - 1)) +#define SYSTRACE_ENTRY(id) ((1 << SYSTRACE_SHIFT) | (id)) +#define SYSTRACE_RETURN(id) (id) #if ((1 << SYSTRACE_SHIFT) <= NSYSCALL) #error 1 << SYSTRACE_SHIFT must exceed number of system calls #endif -static dev_info_t *systrace_devi; static dtrace_provider_id_t systrace_id; -#if !defined (__APPLE__) -static void -systrace_init(struct sysent *actual, systrace_sysent_t **interposed) -{ - systrace_sysent_t *sysent = *interposed; - int i; - - if (sysent == NULL) { - *interposed = sysent = kmem_zalloc(sizeof (systrace_sysent_t) * - NSYSCALL, KM_SLEEP); - } - - for (i = 0; i < NSYSCALL; i++) { - struct sysent *a = &actual[i]; - systrace_sysent_t *s = &sysent[i]; - - if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) - continue; - - if (a->sy_callc == dtrace_systrace_syscall) - continue; - -#ifdef _SYSCALL32_IMPL - if (a->sy_callc == dtrace_systrace_syscall32) - continue; -#endif +/* + * APPLE NOTE: Avoid name clash with Darwin automagic conf symbol. + * See balanced undef below. + */ +#define systrace_init _systrace_init - s->stsy_underlying = a->sy_callc; - } -} -#else -#define systrace_init _systrace_init /* Avoid name clash with Darwin automagic conf symbol */ static void systrace_init(struct sysent *actual, systrace_sysent_t **interposed) { - systrace_sysent_t *ssysent = *interposed; /* Avoid sysent shadow warning - from bsd/sys/sysent.h */ - int i; + * from bsd/sys/sysent.h */ + unsigned int i; if (ssysent == NULL) { - *interposed = ssysent = kmem_zalloc(sizeof (systrace_sysent_t) * + *interposed = ssysent = kmem_zalloc(sizeof(systrace_sysent_t) * NSYSCALL, KM_SLEEP); } @@ -395,46 +393,43 @@ systrace_init(struct sysent *actual, systrace_sysent_t **interposed) struct sysent *a = &actual[i]; systrace_sysent_t *s = &ssysent[i]; - if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) - continue; - - if (a->sy_callc == dtrace_systrace_syscall) + if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) { continue; + } -#ifdef _SYSCALL32_IMPL - if (a->sy_callc == dtrace_systrace_syscall32) + if (a->sy_callc == dtrace_systrace_syscall) { continue; -#endif + } s->stsy_underlying = a->sy_callc; s->stsy_return_type = a->sy_return_type; } + lck_mtx_init(&dtrace_systrace_lock, dtrace_lck_grp, dtrace_lck_attr); } -#endif /* __APPLE__ */ /*ARGSUSED*/ static void systrace_provide(void *arg, const dtrace_probedesc_t *desc) { #pragma unused(arg) /* __APPLE__ */ - int i; + unsigned int i; - if (desc != NULL) + if (desc != NULL) { return; + } systrace_init(sysent, &systrace_sysent); -#ifdef _SYSCALL32_IMPL - systrace_init(sysent32, &systrace_sysent32); -#endif for (i = 0; i < NSYSCALL; i++) { - if (systrace_sysent[i].stsy_underlying == NULL) + if (systrace_sysent[i].stsy_underlying == NULL) { continue; + } if (dtrace_probe_lookup(systrace_id, NULL, - syscallnames[i], "entry") != 0) + syscallnames[i], "entry") != 0) { continue; + } (void) dtrace_probe_create(systrace_id, NULL, syscallnames[i], "entry", SYSTRACE_ARTIFICIAL_FRAMES, @@ -445,15 +440,9 @@ systrace_provide(void *arg, const dtrace_probedesc_t *desc) systrace_sysent[i].stsy_entry = DTRACE_IDNONE; systrace_sysent[i].stsy_return = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[i].stsy_entry = DTRACE_IDNONE; - systrace_sysent32[i].stsy_return = DTRACE_IDNONE; -#endif } } -#if defined(__APPLE__) #undef systrace_init -#endif /*ARGSUSED*/ static void @@ -470,52 +459,39 @@ systrace_destroy(void *arg, dtrace_id_t id, void *parg) */ if (SYSTRACE_ISENTRY((uintptr_t)parg)) { ASSERT(systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE); -#ifdef _SYSCALL32_IMPL - ASSERT(systrace_sysent32[sysnum].stsy_entry == DTRACE_IDNONE); -#endif } else { ASSERT(systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE); -#ifdef _SYSCALL32_IMPL - ASSERT(systrace_sysent32[sysnum].stsy_return == DTRACE_IDNONE); -#endif } } /*ARGSUSED*/ -static void +static int systrace_enable(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg) /* __APPLE__ */ - + int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg); int enabled = (systrace_sysent[sysnum].stsy_entry != DTRACE_IDNONE || systrace_sysent[sysnum].stsy_return != DTRACE_IDNONE); if (SYSTRACE_ISENTRY((uintptr_t)parg)) { systrace_sysent[sysnum].stsy_entry = id; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_entry = id; -#endif } else { systrace_sysent[sysnum].stsy_return = id; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_return = id; -#endif } if (enabled) { ASSERT(sysent[sysnum].sy_callc == dtrace_systrace_syscall); - return; + return 0; } - (void) casptr(&sysent[sysnum].sy_callc, - (void *)systrace_sysent[sysnum].stsy_underlying, - (void *)dtrace_systrace_syscall); -#ifdef _SYSCALL32_IMPL - (void) casptr(&sysent32[sysnum].sy_callc, - (void *)systrace_sysent32[sysnum].stsy_underlying, - (void *)dtrace_systrace_syscall32); -#endif + lck_mtx_lock(&dtrace_systrace_lock); + if (sysent[sysnum].sy_callc == systrace_sysent[sysnum].stsy_underlying) { + vm_offset_t dss = (vm_offset_t)&dtrace_systrace_syscall; + ml_nofault_copy((vm_offset_t)&dss, (vm_offset_t)&sysent[sysnum].sy_callc, sizeof(vm_offset_t)); + } + lck_mtx_unlock(&dtrace_systrace_lock); + return 0; } /*ARGSUSED*/ @@ -523,83 +499,51 @@ static void systrace_disable(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg,id) /* __APPLE__ */ - + int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg); int disable = (systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE || systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE); if (disable) { - (void) casptr(&sysent[sysnum].sy_callc, - (void *)dtrace_systrace_syscall, - (void *)systrace_sysent[sysnum].stsy_underlying); - -#ifdef _SYSCALL32_IMPL - (void) casptr(&sysent32[sysnum].sy_callc, - (void *)dtrace_systrace_syscall32, - (void *)systrace_sysent32[sysnum].stsy_underlying); -#endif + lck_mtx_lock(&dtrace_systrace_lock); + if (sysent[sysnum].sy_callc == dtrace_systrace_syscall) { + ml_nofault_copy((vm_offset_t)&systrace_sysent[sysnum].stsy_underlying, (vm_offset_t)&sysent[sysnum].sy_callc, sizeof(systrace_sysent[sysnum].stsy_underlying)); + } + lck_mtx_unlock(&dtrace_systrace_lock); } if (SYSTRACE_ISENTRY((uintptr_t)parg)) { systrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_entry = DTRACE_IDNONE; -#endif } else { systrace_sysent[sysnum].stsy_return = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_return = DTRACE_IDNONE; -#endif } } static dtrace_pattr_t systrace_attr = { -{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, -{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, -{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, -{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, -{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, + { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, + { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, + { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, + { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, + { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, }; static dtrace_pops_t systrace_pops = { - systrace_provide, - NULL, - systrace_enable, - systrace_disable, - NULL, - NULL, - NULL, - NULL, - NULL, - systrace_destroy + .dtps_provide = systrace_provide, + .dtps_provide_module = NULL, + .dtps_enable = systrace_enable, + .dtps_disable = systrace_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = systrace_getargdesc, + .dtps_getargval = systrace_getargval, + .dtps_usermode = NULL, + .dtps_destroy = systrace_destroy }; static int -systrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) +systrace_attach(dev_info_t *devi) { - switch (cmd) { - case DDI_ATTACH: - break; - case DDI_RESUME: - return (DDI_SUCCESS); - default: - return (DDI_FAILURE); - } - -#if !defined(__APPLE__) - systrace_probe = (void (*)())dtrace_probe; - membar_enter(); - - if (ddi_create_minor_node(devi, "systrace", S_IFCHR, 0, - DDI_PSEUDO, NULL) == DDI_FAILURE || - dtrace_register("syscall", &systrace_attr, DTRACE_PRIV_USER, NULL, - &systrace_pops, NULL, &systrace_id) != 0) { - systrace_probe = systrace_stub; - ddi_remove_minor_node(devi, NULL); - return (DDI_FAILURE); - } -#else - systrace_probe = (void(*))&dtrace_probe; + systrace_probe = (void*)&dtrace_probe; membar_enter(); if (ddi_create_minor_node(devi, "systrace", S_IFCHR, 0, @@ -608,16 +552,16 @@ systrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) &systrace_pops, NULL, &systrace_id) != 0) { systrace_probe = systrace_stub; ddi_remove_minor_node(devi, NULL); - return (DDI_FAILURE); + return DDI_FAILURE; } -#endif /* __APPLE__ */ - ddi_report_dev(devi); - systrace_devi = devi; - - return (DDI_SUCCESS); + return DDI_SUCCESS; } + +/* + * APPLE NOTE: systrace_detach not implemented + */ #if !defined(__APPLE__) static int systrace_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) @@ -626,149 +570,55 @@ systrace_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) case DDI_DETACH: break; case DDI_SUSPEND: - return (DDI_SUCCESS); + return DDI_SUCCESS; default: - return (DDI_FAILURE); + return DDI_FAILURE; } - if (dtrace_unregister(systrace_id) != 0) - return (DDI_FAILURE); + if (dtrace_unregister(systrace_id) != 0) { + return DDI_FAILURE; + } ddi_remove_minor_node(devi, NULL); systrace_probe = systrace_stub; - return (DDI_SUCCESS); -} - -/*ARGSUSED*/ -static int -systrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) -{ - int error; - - switch (infocmd) { - case DDI_INFO_DEVT2DEVINFO: - *result = (void *)systrace_devi; - error = DDI_SUCCESS; - break; - case DDI_INFO_DEVT2INSTANCE: - *result = (void *)0; - error = DDI_SUCCESS; - break; - default: - error = DDI_FAILURE; - } - return (error); -} - -/*ARGSUSED*/ -static int -systrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) -{ - return (0); -} - -static struct cb_ops systrace_cb_ops = { - systrace_open, /* open */ - nodev, /* close */ - nulldev, /* strategy */ - nulldev, /* print */ - nodev, /* dump */ - nodev, /* read */ - nodev, /* write */ - nodev, /* ioctl */ - nodev, /* devmap */ - nodev, /* mmap */ - nodev, /* segmap */ - nochpoll, /* poll */ - ddi_prop_op, /* cb_prop_op */ - 0, /* streamtab */ - D_NEW | D_MP /* Driver compatibility flag */ -}; - -static struct dev_ops systrace_ops = { - DEVO_REV, /* devo_rev, */ - 0, /* refcnt */ - systrace_info, /* get_dev_info */ - nulldev, /* identify */ - nulldev, /* probe */ - systrace_attach, /* attach */ - systrace_detach, /* detach */ - nodev, /* reset */ - &systrace_cb_ops, /* driver operations */ - NULL, /* bus operations */ - nodev /* dev power */ -}; - -/* - * Module linkage information for the kernel. - */ -static struct modldrv modldrv = { - &mod_driverops, /* module type (this is a pseudo driver) */ - "System Call Tracing", /* name of module */ - &systrace_ops, /* driver ops */ -}; - -static struct modlinkage modlinkage = { - MODREV_1, - (void *)&modldrv, - NULL -}; - -int -_init(void) -{ - return (mod_install(&modlinkage)); + return DDI_SUCCESS; } +#endif /* __APPLE__ */ -int -_info(struct modinfo *modinfop) -{ - return (mod_info(&modlinkage, modinfop)); -} -int -_fini(void) -{ - return (mod_remove(&modlinkage)); -} -#else typedef kern_return_t (*mach_call_t)(void *); -/* XXX From #include which may be changed for 64 bit! */ -typedef void mach_munge_t(const void *, void *); +/* APPLE NOTE: From #include which may be changed for 64 bit! */ +typedef void mach_munge_t(void *); typedef struct { - int mach_trap_arg_count; - int (*mach_trap_function)(void); -#if defined(__i386__) - boolean_t mach_trap_stack; -#else - mach_munge_t *mach_trap_arg_munge32; /* system call arguments for 32-bit */ - mach_munge_t *mach_trap_arg_munge64; /* system call arguments for 64-bit */ + int mach_trap_arg_count; + kern_return_t (*mach_trap_function)(void *); +#if defined(__arm64__) || defined(__x86_64__) + mach_munge_t *mach_trap_arg_munge32; /* system call arguments for 32-bit */ #endif -#if !MACH_ASSERT - int mach_trap_unused; -#else - const char* mach_trap_name; -#endif /* !MACH_ASSERT */ + int mach_trap_u32_words; +#if MACH_ASSERT + const char* mach_trap_name; +#endif /* MACH_ASSERT */ } mach_trap_t; -extern mach_trap_t mach_trap_table[]; +extern const mach_trap_t mach_trap_table[]; /* syscall_sw.h now declares this as const */ extern int mach_trap_count; extern const char *mach_syscall_name_table[]; /* XXX From osfmk/i386/bsd_i386.c */ struct mach_call_args { - syscall_arg_t arg1; - syscall_arg_t arg2; - syscall_arg_t arg3; - syscall_arg_t arg4; - syscall_arg_t arg5; - syscall_arg_t arg6; - syscall_arg_t arg7; - syscall_arg_t arg8; - syscall_arg_t arg9; + syscall_arg_t arg1; + syscall_arg_t arg2; + syscall_arg_t arg3; + syscall_arg_t arg4; + syscall_arg_t arg5; + syscall_arg_t arg6; + syscall_arg_t arg7; + syscall_arg_t arg8; + syscall_arg_t arg9; }; #undef NSYSCALL @@ -778,21 +628,26 @@ struct mach_call_args { #error 1 << SYSTRACE_SHIFT must exceed number of Mach traps #endif -typedef systrace_sysent_t machtrace_sysent_t; +typedef struct machtrace_sysent { + dtrace_id_t stsy_entry; + dtrace_id_t stsy_return; + kern_return_t (*stsy_underlying)(void *); + int32_t stsy_return_type; +} machtrace_sysent_t; static machtrace_sysent_t *machtrace_sysent = NULL; void (*machtrace_probe)(dtrace_id_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); -static dev_info_t *machtrace_devi; +static uint64_t machtrace_getarg(void *, dtrace_id_t, void *, int, int); + static dtrace_provider_id_t machtrace_id; static kern_return_t dtrace_machtrace_syscall(struct mach_call_args *args) { - boolean_t flavor; - unsigned short code; + int code; /* The mach call number */ machtrace_sysent_t *sy; dtrace_id_t id; @@ -803,20 +658,9 @@ dtrace_machtrace_syscall(struct mach_call_args *args) syscall_arg_t *ip = (syscall_arg_t *)args; mach_call_t mach_call; -#if defined (__ppc__) || defined (__ppc64__) - { - savearea_t *regs = (savearea_t *)find_user_regs(current_thread()); - - flavor = (((unsigned int)regs->save_r0) == 0)? 1: 0; - - if (flavor) - code = -regs->save_r3; - else - code = -regs->save_r0; - } -#elif defined(__i386__) || defined (__x86_64__) -#pragma unused(flavor) +#if defined (__x86_64__) { + pal_register_cache_state(current_thread(), VALID); x86_saved_state_t *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread()); if (is_saved_state64(tagged_regs)) { @@ -825,17 +669,51 @@ dtrace_machtrace_syscall(struct mach_call_args *args) code = -saved_state32(tagged_regs)->eax; } } +#elif defined(__arm__) + { + /* r12 has the machcall number, but it is -ve */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + code = (int)arm_regs->r[12]; + ASSERT(code < 0); /* Otherwise it would be a Unix syscall */ + code = -code; + } +#elif defined(__arm64__) + { + /* From arm/thread_status.h:get_saved_state_svc_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + if (is_saved_state32(arm_regs)) { + code = (int)saved_state32(arm_regs)->r[12]; + } else { + code = (int)saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } + + /* From bsd/arm64.c:mach_syscall */ + ASSERT(code < 0); /* Otherwise it would be a Unix syscall */ + code = -code; + } #else #error Unknown Architecture #endif sy = &machtrace_sysent[code]; - if ((id = sy->stsy_entry) != DTRACE_IDNONE) - (*machtrace_probe)(id, *ip, *(ip+1), *(ip+2), *(ip+3), *(ip+4)); + if ((id = sy->stsy_entry) != DTRACE_IDNONE) { + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + if (uthread) { + uthread->t_dtrace_syscall_args = (void *)ip; + } + + (*machtrace_probe)(id, *ip, *(ip + 1), *(ip + 2), *(ip + 3), *(ip + 4)); + + if (uthread) { + uthread->t_dtrace_syscall_args = (void *)0; + } + } #if 0 /* XXX */ /* + * APPLE NOTE: Not implemented. * We want to explicitly allow DTrace consumers to stop a process * before it actually executes the meat of the syscall. */ @@ -851,34 +729,37 @@ dtrace_machtrace_syscall(struct mach_call_args *args) mach_call = (mach_call_t)(*sy->stsy_underlying); rval = mach_call(args); - if ((id = sy->stsy_return) != DTRACE_IDNONE) + if ((id = sy->stsy_return) != DTRACE_IDNONE) { (*machtrace_probe)(id, (uint64_t)rval, 0, 0, 0, 0); + } - return (rval); + return rval; } static void -machtrace_init(mach_trap_t *actual, machtrace_sysent_t **interposed) +machtrace_init(const mach_trap_t *actual, machtrace_sysent_t **interposed) { machtrace_sysent_t *msysent = *interposed; int i; if (msysent == NULL) { - *interposed = msysent = kmem_zalloc(sizeof (machtrace_sysent_t) * - NSYSCALL, KM_SLEEP); + *interposed = msysent = kmem_zalloc(sizeof(machtrace_sysent_t) * + NSYSCALL, KM_SLEEP); } for (i = 0; i < NSYSCALL; i++) { - mach_trap_t *a = &actual[i]; + const mach_trap_t *a = &actual[i]; machtrace_sysent_t *s = &msysent[i]; - if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) + if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) { continue; + } - if ((mach_call_t)(a->mach_trap_function) == (mach_call_t)(dtrace_machtrace_syscall)) + if (a->mach_trap_function == (mach_call_t)(dtrace_machtrace_syscall)) { continue; + } - s->stsy_underlying = (sy_call_t *)a->mach_trap_function; + s->stsy_underlying = a->mach_trap_function; } } @@ -887,29 +768,31 @@ static void machtrace_provide(void *arg, const dtrace_probedesc_t *desc) { #pragma unused(arg) /* __APPLE__ */ - + int i; - if (desc != NULL) + if (desc != NULL) { return; + } machtrace_init(mach_trap_table, &machtrace_sysent); for (i = 0; i < NSYSCALL; i++) { - - if (machtrace_sysent[i].stsy_underlying == NULL) + if (machtrace_sysent[i].stsy_underlying == NULL) { continue; + } if (dtrace_probe_lookup(machtrace_id, NULL, - mach_syscall_name_table[i], "entry") != 0) + mach_syscall_name_table[i], "entry") != 0) { continue; + } (void) dtrace_probe_create(machtrace_id, NULL, mach_syscall_name_table[i], - "entry", MACHTRACE_ARTIFICIAL_FRAMES, - (void *)((uintptr_t)SYSTRACE_ENTRY(i))); + "entry", MACHTRACE_ARTIFICIAL_FRAMES, + (void *)((uintptr_t)SYSTRACE_ENTRY(i))); (void) dtrace_probe_create(machtrace_id, NULL, mach_syscall_name_table[i], - "return", MACHTRACE_ARTIFICIAL_FRAMES, - (void *)((uintptr_t)SYSTRACE_RETURN(i))); + "return", MACHTRACE_ARTIFICIAL_FRAMES, + (void *)((uintptr_t)SYSTRACE_RETURN(i))); machtrace_sysent[i].stsy_entry = DTRACE_IDNONE; machtrace_sysent[i].stsy_return = DTRACE_IDNONE; @@ -922,7 +805,7 @@ machtrace_destroy(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg,id) /* __APPLE__ */ int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg); - + #pragma unused(sysnum) /* __APPLE__ */ /* @@ -937,14 +820,14 @@ machtrace_destroy(void *arg, dtrace_id_t id, void *parg) } /*ARGSUSED*/ -static void +static int machtrace_enable(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg) /* __APPLE__ */ - + int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg); int enabled = (machtrace_sysent[sysnum].stsy_entry != DTRACE_IDNONE || - machtrace_sysent[sysnum].stsy_return != DTRACE_IDNONE); + machtrace_sysent[sysnum].stsy_return != DTRACE_IDNONE); if (SYSTRACE_ISENTRY((uintptr_t)parg)) { machtrace_sysent[sysnum].stsy_entry = id; @@ -953,13 +836,20 @@ machtrace_enable(void *arg, dtrace_id_t id, void *parg) } if (enabled) { - ASSERT(sysent[sysnum].sy_callc == (void *)dtrace_machtrace_syscall); - return; + ASSERT(mach_trap_table[sysnum].mach_trap_function == (void *)dtrace_machtrace_syscall); + return 0; } - (void) casptr(&mach_trap_table[sysnum].mach_trap_function, - (void *)machtrace_sysent[sysnum].stsy_underlying, - (void *)dtrace_machtrace_syscall); + lck_mtx_lock(&dtrace_systrace_lock); + + if (mach_trap_table[sysnum].mach_trap_function == machtrace_sysent[sysnum].stsy_underlying) { + vm_offset_t dss = (vm_offset_t)&dtrace_machtrace_syscall; + ml_nofault_copy((vm_offset_t)&dss, (vm_offset_t)&mach_trap_table[sysnum].mach_trap_function, sizeof(vm_offset_t)); + } + + lck_mtx_unlock(&dtrace_systrace_lock); + + return 0; } /*ARGSUSED*/ @@ -967,16 +857,18 @@ static void machtrace_disable(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg,id) /* __APPLE__ */ - + int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg); int disable = (machtrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE || - machtrace_sysent[sysnum].stsy_return == DTRACE_IDNONE); + machtrace_sysent[sysnum].stsy_return == DTRACE_IDNONE); if (disable) { - (void) casptr(&mach_trap_table[sysnum].mach_trap_function, - (void *)dtrace_machtrace_syscall, - (void *)machtrace_sysent[sysnum].stsy_underlying); + lck_mtx_lock(&dtrace_systrace_lock); + if (mach_trap_table[sysnum].mach_trap_function == (mach_call_t)dtrace_machtrace_syscall) { + ml_nofault_copy((vm_offset_t)&machtrace_sysent[sysnum].stsy_underlying, (vm_offset_t)&mach_trap_table[sysnum].mach_trap_function, sizeof(vm_offset_t)); + } + lck_mtx_unlock(&dtrace_systrace_lock); } if (SYSTRACE_ISENTRY((uintptr_t)parg)) { @@ -987,70 +879,48 @@ machtrace_disable(void *arg, dtrace_id_t id, void *parg) } static dtrace_pattr_t machtrace_attr = { -{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, -{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, -{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, -{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, -{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, + { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, + { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, + { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, + { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, + { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, }; static dtrace_pops_t machtrace_pops = { - machtrace_provide, - NULL, - machtrace_enable, - machtrace_disable, - NULL, - NULL, - NULL, - NULL, - NULL, - machtrace_destroy + .dtps_provide = machtrace_provide, + .dtps_provide_module = NULL, + .dtps_enable = machtrace_enable, + .dtps_disable = machtrace_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = NULL, + .dtps_getargval = machtrace_getarg, + .dtps_usermode = NULL, + .dtps_destroy = machtrace_destroy }; static int -machtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) +machtrace_attach(dev_info_t *devi) { - switch (cmd) { - case DDI_ATTACH: - break; - case DDI_RESUME: - return (DDI_SUCCESS); - default: - return (DDI_FAILURE); - } - -#if !defined(__APPLE__) - machtrace_probe = (void (*)())dtrace_probe; - membar_enter(); - - if (ddi_create_minor_node(devi, "machtrace", S_IFCHR, 0, - DDI_PSEUDO, NULL) == DDI_FAILURE || - dtrace_register("mach_trap", &machtrace_attr, DTRACE_PRIV_USER, NULL, - &machtrace_pops, NULL, &machtrace_id) != 0) { - machtrace_probe = systrace_stub; -#else machtrace_probe = dtrace_probe; membar_enter(); - + if (ddi_create_minor_node(devi, "machtrace", S_IFCHR, 0, - DDI_PSEUDO, 0) == DDI_FAILURE || - dtrace_register("mach_trap", &machtrace_attr, DTRACE_PRIV_USER, NULL, - &machtrace_pops, NULL, &machtrace_id) != 0) { - machtrace_probe = (void (*))&systrace_stub; -#endif /* __APPLE__ */ + DDI_PSEUDO, 0) == DDI_FAILURE || + dtrace_register("mach_trap", &machtrace_attr, DTRACE_PRIV_USER, NULL, + &machtrace_pops, NULL, &machtrace_id) != 0) { + machtrace_probe = (void*)&systrace_stub; ddi_remove_minor_node(devi, NULL); - return (DDI_FAILURE); + return DDI_FAILURE; } - ddi_report_dev(devi); - machtrace_devi = devi; - - return (DDI_SUCCESS); + return DDI_SUCCESS; } d_open_t _systrace_open; -int _systrace_open(dev_t dev, int flags, int devtype, struct proc *p) +int +_systrace_open(dev_t dev, int flags, int devtype, struct proc *p) { #pragma unused(dev,flags,devtype,p) return 0; @@ -1064,43 +934,117 @@ int _systrace_open(dev_t dev, int flags, int devtype, struct proc *p) */ static struct cdevsw systrace_cdevsw = { - _systrace_open, /* open */ - eno_opcl, /* close */ - eno_rdwrt, /* read */ - eno_rdwrt, /* write */ - eno_ioctl, /* ioctl */ + _systrace_open, /* open */ + eno_opcl, /* close */ + eno_rdwrt, /* read */ + eno_rdwrt, /* write */ + eno_ioctl, /* ioctl */ (stop_fcn_t *)nulldev, /* stop */ (reset_fcn_t *)nulldev, /* reset */ - NULL, /* tty's */ - eno_select, /* select */ - eno_mmap, /* mmap */ - eno_strat, /* strategy */ - eno_getc, /* getc */ - eno_putc, /* putc */ - 0 /* type */ + NULL, /* tty's */ + eno_select, /* select */ + eno_mmap, /* mmap */ + eno_strat, /* strategy */ + eno_getc, /* getc */ + eno_putc, /* putc */ + 0 /* type */ }; -static int gSysTraceInited = 0; - void systrace_init( void ); -void systrace_init( void ) +void +systrace_init( void ) { - if (0 == gSysTraceInited) { - int majdevno = cdevsw_add(SYSTRACE_MAJOR, &systrace_cdevsw); + if (dtrace_sdt_probes_restricted()) { + return; + } - if (majdevno < 0) { - printf("systrace_init: failed to allocate a major number!\n"); - gSysTraceInited = 0; - return; - } + int majdevno = cdevsw_add(SYSTRACE_MAJOR, &systrace_cdevsw); - systrace_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH ); - machtrace_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH ); + if (majdevno < 0) { + printf("systrace_init: failed to allocate a major number!\n"); + return; + } - gSysTraceInited = 1; - } else - panic("systrace_init: called twice!\n"); + systrace_attach((dev_info_t*)(uintptr_t)majdevno); + machtrace_attach((dev_info_t*)(uintptr_t)majdevno); } #undef SYSTRACE_MAJOR -#endif /* __APPLE__ */ + +static uint64_t +systrace_getargval(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg,id,parg,aframes) /* __APPLE__ */ + uint64_t val = 0; + uint64_t *uargs = NULL; + + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + if (uthread) { + uargs = uthread->t_dtrace_syscall_args; + } + if (!uargs) { + return 0; + } + if (argno < 0 || argno >= SYSTRACE_NARGS) { + return 0; + } + + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = uargs[argno]; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return val; +} + +static void +systrace_getargdesc(void *arg, dtrace_id_t id, void *parg, + dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + int sysnum = SYSTRACE_SYSNUM(parg); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + uint64_t *uargs = NULL; + + if (!uthread) { + desc->dtargd_ndx = DTRACE_ARGNONE; + return; + } + + uargs = uthread->t_dtrace_syscall_args; + + if (SYSTRACE_ISENTRY((uintptr_t)parg)) { + systrace_entry_setargdesc(sysnum, desc->dtargd_ndx, + desc->dtargd_native, sizeof(desc->dtargd_native)); + } else { + systrace_return_setargdesc(sysnum, desc->dtargd_ndx, + desc->dtargd_native, sizeof(desc->dtargd_native)); + } + + if (desc->dtargd_native[0] == '\0') { + desc->dtargd_ndx = DTRACE_ARGNONE; + } +} + +static uint64_t +machtrace_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg,id,parg,aframes) /* __APPLE__ */ + uint64_t val = 0; + syscall_arg_t *stack = (syscall_arg_t *)NULL; + + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + if (uthread) { + stack = (syscall_arg_t *)uthread->t_dtrace_syscall_args; + } + + if (!stack) { + return 0; + } + + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + val = (uint64_t)*(stack + argno); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return val; +}