X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..c3c9b80d004dbbfdf763edeb97968c6997e3b45b:/bsd/sys/dtrace_impl.h diff --git a/bsd/sys/dtrace_impl.h b/bsd/sys/dtrace_impl.h index 7f42cff5e..e1ab6a060 100644 --- a/bsd/sys/dtrace_impl.h +++ b/bsd/sys/dtrace_impl.h @@ -22,13 +22,14 @@ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * + * Portions Copyright (c) 2012 by Delphix. All rights reserved. + * Portions Copyright (c) 2016 by Joyent, Inc. */ #ifndef _SYS_DTRACE_IMPL_H #define _SYS_DTRACE_IMPL_H -/* #pragma ident "@(#)dtrace_impl.h 1.23 07/02/16 SMI" */ - #ifdef __cplusplus extern "C" { #endif @@ -46,6 +47,13 @@ extern "C" { #include +/* + * DTrace Implementation Locks + */ +extern lck_attr_t dtrace_lck_attr; +extern lck_grp_t dtrace_lck_grp; +extern lck_mtx_t dtrace_procwaitfor_lock; + /* * DTrace Implementation Constants and Typedefs */ @@ -102,6 +110,8 @@ struct dtrace_probe { char *dtpr_mod; /* probe's module name */ char *dtpr_func; /* probe's function name */ char *dtpr_name; /* probe's name */ + dtrace_probe_t *dtpr_nextprov; /* next in provider hash */ + dtrace_probe_t *dtpr_prevprov; /* previous in provider hash */ dtrace_probe_t *dtpr_nextmod; /* next in module hash */ dtrace_probe_t *dtpr_prevmod; /* previous in module hash */ dtrace_probe_t *dtpr_nextfunc; /* next in function hash */ @@ -127,18 +137,21 @@ typedef struct dtrace_probekey { typedef struct dtrace_hashbucket { struct dtrace_hashbucket *dthb_next; /* next on hash chain */ - dtrace_probe_t *dthb_chain; /* chain of probes */ + void *dthb_chain; /* chain of elements */ int dthb_len; /* number of probes here */ } dtrace_hashbucket_t; +typedef const char* dtrace_strkey_f(void*, uintptr_t); + typedef struct dtrace_hash { - dtrace_hashbucket_t **dth_tab; /* hash table */ - int dth_size; /* size of hash table */ - int dth_mask; /* mask to index into table */ - int dth_nbuckets; /* total number of buckets */ - uintptr_t dth_nextoffs; /* offset of next in probe */ - uintptr_t dth_prevoffs; /* offset of prev in probe */ - uintptr_t dth_stroffs; /* offset of str in probe */ + dtrace_hashbucket_t **dth_tab; /* hash table */ + int dth_size; /* size of hash table */ + int dth_mask; /* mask to index into table */ + int dth_nbuckets; /* total number of buckets */ + uintptr_t dth_nextoffs; /* offset of next in element */ + uintptr_t dth_prevoffs; /* offset of prev in element */ + dtrace_strkey_f *dth_getstr; /* func to retrieve str in element */ + uintptr_t dth_stroffs; /* offset of str in element */ } dtrace_hash_t; /* @@ -197,15 +210,18 @@ typedef struct dtrace_hash { * predicate is non-NULL, the DIF object is executed. If the result is * non-zero, the action list is processed, with each action being executed * accordingly. When the action list has been completely executed, processing - * advances to the next ECB. processing advances to the next ECB. If the - * result is non-zero; For each ECB, it first determines the The ECB - * abstraction allows disjoint consumers to multiplex on single probes. + * advances to the next ECB. The ECB abstraction allows disjoint consumers + * to multiplex on single probes. + * + * Execution of the ECB results in consuming dte_size bytes in the buffer + * to record data. During execution, dte_needed bytes must be available in + * the buffer. This space is used for both recorded data and tuple data. */ struct dtrace_ecb { dtrace_epid_t dte_epid; /* enabled probe ID */ uint32_t dte_alignment; /* required alignment */ - size_t dte_needed; /* bytes needed */ - size_t dte_size; /* total size of payload */ + size_t dte_needed; /* space needed for execution */ + size_t dte_size; /* size of recorded payload */ dtrace_predicate_t *dte_predicate; /* predicate, if any */ dtrace_action_t *dte_action; /* actions, if any */ dtrace_ecb_t *dte_next; /* next ECB on probe */ @@ -263,27 +279,30 @@ typedef struct dtrace_aggregation { * the EPID, the consumer can determine the data layout. (The data buffer * layout is shown schematically below.) By assuring that one can determine * data layout from the EPID, the metadata stream can be separated from the - * data stream -- simplifying the data stream enormously. - * - * base of data buffer ---> +------+--------------------+------+ - * | EPID | data | EPID | - * +------+--------+------+----+------+ - * | data | EPID | data | - * +---------------+------+-----------+ - * | data, cont. | - * +------+--------------------+------+ - * | EPID | data | | - * +------+--------------------+ | - * | || | - * | || | - * | \/ | - * : : - * . . - * . . - * . . - * : : - * | | - * limit of data buffer ---> +----------------------------------+ + * data stream -- simplifying the data stream enormously. The ECB always + * proceeds the recorded data as part of the dtrace_rechdr_t structure that + * includes the EPID and a high-resolution timestamp used for output ordering + * consistency. + * + * base of data buffer ---> +--------+--------------------+--------+ + * | rechdr | data | rechdr | + * +--------+------+--------+----+--------+ + * | data | rechdr | data | + * +---------------+--------+-------------+ + * | data, cont. | + * +--------+--------------------+--------+ + * | rechdr | data | | + * +--------+--------------------+ | + * | || | + * | || | + * | \/ | + * : : + * . . + * . . + * . . + * : : + * | | + * limit of data buffer ---> +--------------------------------------+ * * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the * principal buffer (both scratch and payload) exceed the available space. If @@ -408,6 +427,8 @@ typedef struct dtrace_aggregation { typedef struct dtrace_buffer { uint64_t dtb_offset; /* current offset in buffer */ + uint64_t dtb_cur_limit; /* current limit before signaling/dropping */ + uint64_t dtb_limit; /* limit before signaling */ uint64_t dtb_size; /* size of buffer */ uint32_t dtb_flags; /* flags */ uint32_t dtb_drops; /* number of drops */ @@ -421,6 +442,9 @@ typedef struct dtrace_buffer { #ifndef _LP64 uint64_t dtb_pad1; #endif + uint64_t dtb_switched; /* time of last switch */ + uint64_t dtb_interval; /* observed switch interval */ + uint64_t dtb_pad2[4]; /* pad to avoid false sharing */ } dtrace_buffer_t; /* @@ -892,6 +916,8 @@ typedef struct dtrace_vstate { #define DTRACE_MSTATE_WALLTIMESTAMP 0x00000100 #define DTRACE_MSTATE_USTACKDEPTH 0x00000200 #define DTRACE_MSTATE_UCALLER 0x00000400 +#define DTRACE_MSTATE_MACHTIMESTAMP 0x00000800 +#define DTRACE_MSTATE_MACHCTIMESTAMP 0x00001000 typedef struct dtrace_mstate { uintptr_t dtms_scratch_base; /* base of scratch space */ @@ -902,6 +928,8 @@ typedef struct dtrace_mstate { dtrace_epid_t dtms_epid; /* current EPID */ uint64_t dtms_timestamp; /* cached timestamp */ hrtime_t dtms_walltimestamp; /* cached wall timestamp */ + uint64_t dtms_machtimestamp; /* cached mach absolute timestamp */ + uint64_t dtms_machctimestamp; /* cached mach continuous timestamp */ int dtms_stackdepth; /* cached stackdepth */ int dtms_ustackdepth; /* cached ustackdepth */ struct dtrace_probe *dtms_probe; /* current probe */ @@ -910,6 +938,7 @@ typedef struct dtrace_mstate { int dtms_ipl; /* cached interrupt pri lev */ int dtms_fltoffs; /* faulting DIFO offset */ uintptr_t dtms_strtok; /* saved strtok() pointer */ + uintptr_t dtms_strtok_limit; /* upper bound of strtok ptr */ uint32_t dtms_access; /* memory access rights */ dtrace_difo_t *dtms_difo; /* current dif object */ } dtrace_mstate_t; @@ -937,6 +966,7 @@ typedef struct dtrace_mstate { * directed acyclic graph. The activity transition diagram is as follows: * * + * * +----------+ +--------+ +--------+ * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE | * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+ @@ -978,9 +1008,9 @@ typedef enum dtrace_activity { DTRACE_ACTIVITY_KILLED /* killed */ } dtrace_activity_t; -#if defined(__APPLE__) + /* - * DTrace dof modes + * APPLE NOTE: DTrace dof modes implementation * * DTrace has four "dof modes". They are: * @@ -1047,7 +1077,6 @@ typedef enum dtrace_activity { #define DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE 2 #define DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL 3 -#endif /* __APPLE__ */ /* * DTrace Helper Implementation @@ -1109,6 +1138,7 @@ typedef struct dtrace_helpers { #define DTRACE_HELPTRACE_DONE (-2) #define DTRACE_HELPTRACE_ERR (-3) + typedef struct dtrace_helptrace { dtrace_helper_action_t *dtht_helper; /* helper action */ int dtht_where; /* where in helper action */ @@ -1159,6 +1189,13 @@ typedef struct dtrace_cred { uint16_t dcr_action; } dtrace_cred_t; +typedef struct dtrace_format { + uint64_t dtf_refcount; + char dtf_str[]; +} dtrace_format_t; + +#define DTRACE_FORMAT_SIZE(fmt) (strlen(fmt->dtf_str) + 1 + sizeof(dtrace_format_t)) + /* * DTrace Consumer State * @@ -1198,13 +1235,13 @@ struct dtrace_state { char dts_speculates; /* boolean: has speculations */ char dts_destructive; /* boolean: has dest. actions */ int dts_nformats; /* number of formats */ - char **dts_formats; /* format string array */ + dtrace_format_t **dts_formats; /* format string array */ dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */ dtrace_cred_t dts_cred; /* credentials */ size_t dts_nretained; /* number of retained enabs */ -#if defined(__APPLE__) uint64_t dts_arg_error_illval; -#endif /* __APPLE__ */ + uint32_t dts_buf_over_limit; /* number of bufs over dtb_limit */ + uint64_t **dts_rstate; /* per-CPU random state */ }; struct dtrace_provider { @@ -1215,13 +1252,15 @@ struct dtrace_provider { void *dtpv_arg; /* provider argument */ uint_t dtpv_defunct; /* boolean: defunct provider */ struct dtrace_provider *dtpv_next; /* next provider */ + uint64_t dtpv_probe_count; /* number of associated probes */ + uint64_t dtpv_ecb_count; /* number of associated enabled ECBs */ }; struct dtrace_meta { dtrace_mops_t dtm_mops; /* meta provider operations */ char *dtm_name; /* meta provider name */ void *dtm_arg; /* meta provider user arg */ - uint64_t dtm_count; /* no. of associated provs. */ + uint64_t dtm_count; /* number of associated providers */ }; /* @@ -1286,6 +1325,27 @@ typedef struct dtrace_errhash { #endif /* DTRACE_ERRDEBUG */ +typedef struct dtrace_string dtrace_string_t; + +typedef struct dtrace_string { + dtrace_string_t *dtst_next; + dtrace_string_t *dtst_prev; + uint32_t dtst_refcount; + char dtst_str[]; +} dtrace_string_t; + +/** + * DTrace Matching pre-conditions + * + * Used when matching new probes to discard matching of enablings that + * doesn't match the condition tested by dmc_func + */ +typedef struct dtrace_match_cond { + int (*dmc_func)(dtrace_probedesc_t*, void*); + void *dmc_data; +} dtrace_match_cond_t; + + /* * DTrace Toxic Ranges * @@ -1307,76 +1367,84 @@ typedef struct dtrace_toxrange { uintptr_t dtt_limit; /* limit of toxic range */ } dtrace_toxrange_t; -extern uint64_t dtrace_getarg(int, int); +extern uint64_t dtrace_getarg(int, int, dtrace_mstate_t*, dtrace_vstate_t*); extern int dtrace_getipl(void); extern uintptr_t dtrace_caller(int); extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t); extern void *dtrace_casptr(void *, void *, void *); -#if !defined(__APPLE__) -extern void dtrace_copyin(uintptr_t, uintptr_t, size_t, volatile uint16_t *); -extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); -extern void dtrace_copyout(uintptr_t, uintptr_t, size_t, volatile uint16_t *); -extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t, - volatile uint16_t *); -#else extern void dtrace_copyin(user_addr_t, uintptr_t, size_t, volatile uint16_t *); extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t, volatile uint16_t *); extern void dtrace_copyout(uintptr_t, user_addr_t, size_t, volatile uint16_t *); extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t, volatile uint16_t *); -#endif /* __APPLE__ */ extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *); -#if !defined(__APPLE__) -extern ulong_t dtrace_getreg(struct regs *, uint_t); -#else +extern uint64_t dtrace_load64(uintptr_t); +extern int dtrace_canload(uint64_t, size_t, dtrace_mstate_t*, dtrace_vstate_t*); + extern uint64_t dtrace_getreg(struct regs *, uint_t); -#endif /* __APPLE__ */ +extern uint64_t dtrace_getvmreg(uint_t); extern int dtrace_getstackdepth(int); extern void dtrace_getupcstack(uint64_t *, int); extern void dtrace_getufpstack(uint64_t *, uint64_t *, int); extern int dtrace_getustackdepth(void); extern uintptr_t dtrace_fulword(void *); -#if !defined(__APPLE__) -extern uint8_t dtrace_fuword8(void *); -extern uint16_t dtrace_fuword16(void *); -extern uint32_t dtrace_fuword32(void *); -extern uint64_t dtrace_fuword64(void *); -extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int, - int, uintptr_t); -#else extern uint8_t dtrace_fuword8(user_addr_t); extern uint16_t dtrace_fuword16(user_addr_t); extern uint32_t dtrace_fuword32(user_addr_t); extern uint64_t dtrace_fuword64(user_addr_t); +extern int dtrace_proc_waitfor(dtrace_procdesc_t*); extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int, int, uint64_t); -#endif /* __APPLE__ */ extern int dtrace_assfail(const char *, const char *, int); extern int dtrace_attached(void); extern hrtime_t dtrace_gethrestime(void); -#ifdef __sparc -extern void dtrace_flush_windows(void); -extern void dtrace_flush_user_windows(void); -extern uint_t dtrace_getotherwin(void); -extern uint_t dtrace_getfprs(void); -#else +extern void dtrace_flush_caches(void); + extern void dtrace_copy(uintptr_t, uintptr_t, size_t); extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); -#endif + +extern void* dtrace_ptrauth_strip(void*, uint64_t); +extern int dtrace_is_valid_ptrauth_key(uint64_t); + +extern uint64_t dtrace_physmem_read(uint64_t, size_t); +extern void dtrace_physmem_write(uint64_t, uint64_t, size_t); + +/* + * DTrace state handling + */ +extern minor_t dtrace_state_reserve(void); +extern dtrace_state_t* dtrace_state_allocate(minor_t minor); +extern dtrace_state_t* dtrace_state_get(minor_t minor); +extern void dtrace_state_free(minor_t minor); + +/* + * DTrace restriction checks + */ +extern void dtrace_restriction_policy_load(void); +extern boolean_t dtrace_is_restricted(void); +extern boolean_t dtrace_are_restrictions_relaxed(void); +extern boolean_t dtrace_fbt_probes_restricted(void); +extern boolean_t dtrace_sdt_probes_restricted(void); +extern boolean_t dtrace_can_attach_to_proc(proc_t); /* * DTrace Assertions * - * DTrace calls ASSERT from probe context. To assure that a failed ASSERT - * does not induce a markedly more catastrophic failure (e.g., one from which - * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that - * may safely be called from probe context. This header file must thus be - * included by any DTrace component that calls ASSERT from probe context, and - * _only_ by those components. (The only exception to this is kernel - * debugging infrastructure at user-level that doesn't depend on calling - * ASSERT.) + * DTrace calls ASSERT and VERIFY from probe context. To assure that a failed + * ASSERT or VERIFYdoes not induce a markedly more catastrophic failure (e.g., + * one from which a dump cannot be gleaned), DTrace must define its own ASSERT + * and VERIFY macros to be ones that may safely be called from probe context. + * This header file must thus be included by any DTrace component that calls + * ASSERT and/or VERIFY from probe context, and _only_ by those components. + * (The only exception to this is kernel debugging infrastructure at user-level + * that doesn't depend on calling ASSERT.) */ #undef ASSERT +#undef VERIFY + +#define VERIFY(EX) ((void)((EX) || \ + dtrace_assfail(#EX, __FILE__, __LINE__))) + #if DEBUG #define ASSERT(EX) ((void)((EX) || \ dtrace_assfail(#EX, __FILE__, __LINE__)))