#ifdef KERNEL_BUILD
#include <libkern/libkern.h>
-#include <kern/lock.h>
#include <kern/locks.h>
+#include <kern/debug.h>
#include <kern/thread_call.h>
#include <kern/thread.h>
#include <machine/machine_routines.h>
/*
* cmn_err
*/
-#define CE_CONT 0 /* continuation */
#define CE_NOTE 1 /* notice */
#define CE_WARN 2 /* warning */
-#define CE_PANIC 3 /* panic */
-#define CE_IGNORE 4 /* print nothing */
extern void cmn_err( int, const char *, ... );
proc_t* sprlock(pid_t pid);
void sprunlock(proc_t *p);
+void dtrace_sprlock(proc_t *p);
+void dtrace_sprunlock(proc_t *p);
+
/*
* uread/uwrite
*/
int fuword32(user_addr_t, uint32_t *);
int fuword64(user_addr_t, uint64_t *);
-void fuword8_noerr(user_addr_t, uint8_t *);
-void fuword16_noerr(user_addr_t, uint16_t *);
void fuword32_noerr(user_addr_t, uint32_t *);
void fuword64_noerr(user_addr_t, uint64_t *);
int suword64(user_addr_t, uint64_t value);
int suword32(user_addr_t, uint32_t value);
-int suword16(user_addr_t, uint16_t value);
-int suword8(user_addr_t, uint8_t value);
/*
* cpuvar
*/
extern lck_mtx_t cpu_lock;
+extern lck_mtx_t cyc_lock;
extern lck_mtx_t mod_lock;
+/*
+ * wrap_timer_call: wrapper of timer_call for cyclic timers.
+ */
+struct wrap_timer_call;
+
/*
* Per-CPU data.
*/
hrtime_t cpu_dtrace_chillmark; /* DTrace: chill mark time */
hrtime_t cpu_dtrace_chilled; /* DTrace: total chill time */
boolean_t cpu_dtrace_invop_underway; /* DTrace gaurds against invalid op re-entrancy */
+
+ /* Local cyclic timers on this CPU */
+ LIST_HEAD(cyc_list_head, wrap_timer_call) cpu_cyc_list;
} dtrace_cpu_t;
extern dtrace_cpu_t *cpu_list;
extern cpu_core_t *cpu_core;
+extern unsigned int dtrace_max_cpus; /* max number of enabled cpus */
+#define NCPU dtrace_max_cpus
extern int cpu_number(void); /* From #include <kern/cpu_number.h>. Called from probe context, must blacklist. */
#define CPU (&(cpu_list[cpu_number()])) /* Pointer to current CPU */
#define CPU_ON_INTR(cpup) ml_at_interrupt_context() /* always invoked on current cpu */
-#define NCPU real_ncpus
/*
* Routines used to register interest in cpu's being added to or removed
#define CPU_DTRACE_KPRIV 0x0080 /* DTrace fault: bad kernel access */
#define CPU_DTRACE_UPRIV 0x0100 /* DTrace fault: bad user access */
#define CPU_DTRACE_TUPOFLOW 0x0200 /* DTrace fault: tuple stack overflow */
-#if defined(__sparc)
-//#define CPU_DTRACE_FAKERESTORE 0x0400 /* pid provider hint to getreg */
-#endif
#define CPU_DTRACE_USTACK_FP 0x0400 /* pid provider hint to ustack() */
#define CPU_DTRACE_ENTRY 0x0800 /* pid provider hint to ustack() */
#define CPU_DTRACE_BADSTACK 0x1000 /* DTrace fault: bad stack */
char mod_modname[KMOD_MAX_NAME];
int mod_loadcnt;
char mod_loaded;
- char mod_flags; // See flags below
+ uint16_t mod_flags; // See flags below
int mod_nenabled; // # of enabled DTrace probes in module
vm_address_t mod_address; // starting address (of Mach-o header blob)
vm_size_t mod_size; // total size (of blob)
} modctl_t;
/* Definitions for mod_flags */
-#define MODCTL_IS_MACH_KERNEL 0x01 // This module represents /mach_kernel
-#define MODCTL_HAS_KERNEL_SYMBOLS 0x02 // Kernel symbols (nlist) are available
-#define MODCTL_FBT_PROBES_PROVIDED 0x04 // fbt probes have been provided
-#define MODCTL_FBT_INVALID 0x08 // Module is invalid for fbt probes
-#define MODCTL_SDT_PROBES_PROVIDED 0x10 // sdt probes have been provided
-#define MODCTL_SDT_INVALID 0x20 // Module is invalid for sdt probes
-#define MODCTL_HAS_UUID 0x40 // Module has UUID
+#define MODCTL_IS_MACH_KERNEL 0x01 // This module represents /mach_kernel
+#define MODCTL_HAS_KERNEL_SYMBOLS 0x02 // Kernel symbols (nlist) are available
+#define MODCTL_FBT_PROBES_PROVIDED 0x04 // fbt probes have been provided
+#define MODCTL_FBT_INVALID 0x08 // Module is invalid for fbt probes
+#define MODCTL_SDT_PROBES_PROVIDED 0x10 // sdt probes have been provided
+#define MODCTL_SDT_INVALID 0x20 // Module is invalid for sdt probes
+#define MODCTL_HAS_UUID 0x40 // Module has UUID
+#define MODCTL_FBT_PRIVATE_PROBES_PROVIDED 0x80 // fbt private probes have been provided
+#define MODCTL_FBT_PROVIDE_PRIVATE_PROBES 0x100 // fbt provider must provide private probes
+#define MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES 0x200 // fbt provider must provide blacklisted probes
+#define MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED 0x400 // fbt blacklisted probes have been provided
+#define MODCTL_IS_STATIC_KEXT 0x800 // module is a static kext
/* Simple/singular mod_flags accessors */
#define MOD_IS_MACH_KERNEL(mod) (mod->mod_flags & MODCTL_IS_MACH_KERNEL)
#define MOD_SDT_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_SDT_PROBES_PROVIDED)
#define MOD_SDT_INVALID(mod) (mod->mod_flags & MODCTL_SDT_INVALID)
#define MOD_HAS_UUID(mod) (mod->mod_flags & MODCTL_HAS_UUID)
+#define MOD_FBT_PRIVATE_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_FBT_PRIVATE_PROBES_PROVIDED)
+#define MOD_FBT_PROVIDE_PRIVATE_PROBES(mod) (mod->mod_flags & MODCTL_FBT_PROVIDE_PRIVATE_PROBES)
+#define MOD_FBT_BLACKLISTED_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED)
+#define MOD_FBT_PROVIDE_BLACKLISTED_PROBES(mod) (mod->mod_flags & MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES)
+#define MOD_IS_STATIC_KEXT(mod) (mod->mod_flags & MODCTL_IS_STATIC_KEXT)
/* Compound accessors */
-#define MOD_FBT_DONE(mod) (MOD_FBT_PROBES_PROVIDED(mod) || MOD_FBT_INVALID(mod))
+#define MOD_FBT_PRIVATE_PROBES_DONE(mod) (MOD_FBT_PRIVATE_PROBES_PROVIDED(mod) || !MOD_FBT_PROVIDE_PRIVATE_PROBES(mod))
+#define MOD_FBT_BLACKLISTED_PROBES_DONE(mod) (MOD_FBT_BLACKLISTED_PROBES_PROVIDED(mod) || !MOD_FBT_PROVIDE_BLACKLISTED_PROBES(mod))
+#define MOD_FBT_DONE(mod) ((MOD_FBT_PROBES_PROVIDED(mod) && MOD_FBT_PRIVATE_PROBES_DONE(mod) && MOD_FBT_BLACKLISTED_PROBES_DONE(mod)) || MOD_FBT_INVALID(mod))
#define MOD_SDT_DONE(mod) (MOD_SDT_PROBES_PROVIDED(mod) || MOD_SDT_INVALID(mod))
#define MOD_SYMBOLS_DONE(mod) (MOD_FBT_DONE(mod) && MOD_SDT_DONE(mod))
extern modctl_t *dtrace_modctl_list;
+extern int dtrace_addr_in_module(void*, struct modctl*);
+
/*
* cred_t
*/
#define CRED() kauth_cred_get() /* Can't be called from probe context! */
extern int PRIV_POLICY_CHOICE(void *, int, int);
extern int PRIV_POLICY_ONLY(void *, int, int);
-extern gid_t crgetgid(const cred_t *);
extern uid_t crgetuid(const cred_t *);
#define crgetzoneid(x) ((zoneid_t)0)
-#define crhold(a) {}
-#define crfree(a) {}
-
/*
* "cyclic"
*/
#define CY_LOW_LEVEL 0
-#define CY_LOCK_LEVEL 1
#define CY_HIGH_LEVEL 2
-#define CY_SOFT_LEVELS 2
#define CY_LEVELS 3
typedef uintptr_t cyclic_id_t;
void *cyo_arg;
} cyc_omni_handler_t;
+extern void dtrace_install_cpu_hooks(void);
+
extern cyclic_id_t cyclic_add(cyc_handler_t *, cyc_time_t *);
extern void cyclic_remove(cyclic_id_t);
extern cyclic_id_t cyclic_timer_add(cyc_handler_t *, cyc_time_t *);
extern void cyclic_timer_remove(cyclic_id_t);
-/*
- * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision)
- */
-
-thread_call_t dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos);
-
/*
* ddi
*/
#define DDI_SUCCESS 0
#define DDI_FAILURE -1
-#define DDI_DEV_T_NONE ((dev_t)-1)
-#define DDI_DEV_T_ANY ((dev_t)-2)
-#define DDI_MAJOR_T_UNKNOWN ((major_t)0)
-
#define DDI_PSEUDO "ddi_pseudo"
-typedef enum {
- DDI_ATTACH = 0,
- DDI_RESUME = 1,
- DDI_PM_RESUME = 2
-} ddi_attach_cmd_t;
-
typedef enum {
DDI_DETACH = 0,
DDI_SUSPEND = 1,
typedef struct __dev_info *dev_info_t;
-extern void ddi_report_dev(dev_info_t *);
-extern int ddi_soft_state_init(void **, size_t, size_t);
-extern void *ddi_get_soft_state(void *, int);
-extern int ddi_soft_state_free(void *, int);
-extern int ddi_soft_state_zalloc(void *, int);
-extern void ddi_soft_state_fini(void **);
-
-int ddi_getprop(dev_t dev, dev_info_t *dip, int flags, const char *name, int defvalue);
-
-extern int ddi_prop_free(void *);
-extern int ddi_prop_lookup_int_array(dev_t, dev_info_t *, uint_t, const char *, int **, uint_t *);
-
extern int ddi_driver_major(dev_info_t *);
extern int ddi_create_minor_node(dev_info_t *, const char *, int, minor_t, const char *, int);
extern major_t getemajor(dev_t);
extern minor_t getminor(dev_t);
-extern dev_t makedevice(major_t, minor_t);
-
/*
* Kernel Debug Interface
*/
-
-typedef enum kdi_dtrace_set {
- KDI_DTSET_DTRACE_ACTIVATE,
- KDI_DTSET_DTRACE_DEACTIVATE,
- KDI_DTSET_KMDB_BPT_ACTIVATE,
- KDI_DTSET_KMDB_BPT_DEACTIVATE
-} kdi_dtrace_set_t;
-
-extern int kdi_dtrace_set(kdi_dtrace_set_t);
extern void debug_enter(char *);
/*
* DTrace specific zone allocation
*/
-/*
- * To break dtrace memory usage out in a trackable
- * fashion, uncomment the #define below. This will
- * enable emulation of the general kalloc.XXX zones
- * for most dtrace allocations. (kalloc.large is not
- * emulated)
- *
- * #define DTRACE_MEMORY_ZONES 1
- *
- */
-
-#if defined(DTRACE_MEMORY_ZONES)
-void dtrace_alloc_init(void);
-void *dtrace_alloc(vm_size_t);
-void dtrace_free(void *, vm_size_t);
-#endif
-
/*
* kmem
*/
typedef struct vmem vmem_t;
typedef struct kmem_cache kmem_cache_t;
-#define kmem_alloc dt_kmem_alloc /* Avoid clash with Darwin's kmem_alloc */
#define kmem_free dt_kmem_free /* Avoid clash with Darwin's kmem_free */
-#define kmem_zalloc dt_kmem_zalloc /* Avoid clash with Darwin's kmem_zalloc */
-extern void *dt_kmem_alloc(size_t, int);
+#define kmem_free_aligned dt_kmem_free_aligned
+
+#define kmem_alloc(size, kmflag) \
+ ({ VM_ALLOC_SITE_STATIC(0, 0); \
+ dt_kmem_alloc_site(size, kmflag, &site); })
+
+extern void *dt_kmem_alloc_site(size_t, int, vm_allocation_site_t*);
extern void dt_kmem_free(void *, size_t);
-extern void *dt_kmem_zalloc(size_t, int);
-extern void *dt_kmem_alloc_aligned(size_t, size_t, int);
-extern void *dt_kmem_zalloc_aligned(size_t, size_t, int);
+#define kmem_zalloc(size, kmflag) \
+ ({ VM_ALLOC_SITE_STATIC(0, 0); \
+ dt_kmem_zalloc_site(size, kmflag, &site); })
+
+extern void *dt_kmem_zalloc_site(size_t, int, vm_allocation_site_t*);
+
+#define kmem_alloc_aligned(size, align, kmflag) \
+ ({ VM_ALLOC_SITE_STATIC(0, 0); \
+ dt_kmem_alloc_aligned_site(size, align, kmflag, &site); })
+extern void *dt_kmem_alloc_aligned_site(size_t, size_t, int, vm_allocation_site_t*);
+
+#define kmem_zalloc_aligned(size, align, kmflag) \
+ ({ VM_ALLOC_SITE_STATIC(0, 0); \
+ dt_kmem_zalloc_aligned_site(size, align, kmflag, &site); })
+extern void *dt_kmem_zalloc_aligned_site(size_t, size_t, int, vm_allocation_site_t*);
+
extern void dt_kmem_free_aligned(void*, size_t);
extern kmem_cache_t *
* proc
*/
-#define DATAMODEL_MASK 0x0FF00000
#define DATAMODEL_ILP32 0x00100000
#define DATAMODEL_LP64 0x00200000
typedef unsigned int model_t; /* For dtrace_instr_size_isa() prototype in <sys/dtrace.h> */
-/*
- * taskq
- */
-
-#define TQ_SLEEP 0x00 /* Can block for memory */
-
-typedef uint_t pri_t;
-typedef struct taskq taskq_t;
-typedef void (task_func_t)(void *);
-typedef uintptr_t taskqid_t;
-
-extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t);
-extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
-extern void taskq_destroy(taskq_t *);
-
-extern pri_t maxclsyspri;
-
/*
* vmem
*/
* Atomic
*/
-static inline void atomic_add_32( uint32_t *theValue, int32_t theAmount )
+static inline uint8_t atomic_or_8(uint8_t *addr, uint8_t mask)
+{
+ return OSBitOrAtomic8(mask, addr);
+}
+
+static inline uint32_t atomic_and_32( uint32_t *addr, int32_t mask)
+{
+ return OSBitAndAtomic(mask, addr);
+}
+
+static inline uint32_t atomic_add_32( uint32_t *theAddress, int32_t theAmount )
{
- (void)OSAddAtomic( theAmount, theValue );
+ return OSAddAtomic( theAmount, theAddress );
}
#if defined(__i386__) || defined(__x86_64__)
-static inline void atomic_add_64( uint64_t *theValue, int64_t theAmount )
+static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount )
+{
+ (void)OSAddAtomic64( theAmount, (SInt64 *)theAddress );
+}
+#elif defined(__arm__)
+static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount )
{
- (void)OSAddAtomic64( theAmount, (SInt64 *)theValue );
+ // FIXME
+ // atomic_add_64() is at present only called from fasttrap.c to increment
+ // or decrement a 64bit counter. Narrow to 32bits since arm has
+ // no convenient 64bit atomic op.
+
+ (void)OSAddAtomic( (int32_t)theAmount, &(((SInt32 *)theAddress)[0]));
+}
+#elif defined (__arm64__)
+static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount )
+{
+ (void)OSAddAtomic64( theAmount, (SInt64 *)theAddress );
}
#endif
+static inline uint32_t atomic_or_32(uint32_t *addr, uint32_t mask)
+{
+ return OSBitOrAtomic(mask, addr);
+}
+
+
/*
* Miscellaneous
*/
typedef uintptr_t pc_t;
typedef uintptr_t greg_t; /* For dtrace_impl.h prototype of dtrace_getfp() */
+#if defined(__arm__) || defined(__arm64__)
+#define regs arm_saved_state
+#endif
extern struct regs *find_user_regs( thread_t thread);
extern vm_offset_t dtrace_get_cpu_int_stack_top(void);
extern vm_offset_t max_valid_stack_address(void); /* kern/thread.h */
-extern volatile int panicwait; /* kern/debug.c */
-#define panic_quiesce (panicwait)
+#define panic_quiesce (panic_active())
#define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
-extern void delay( int ); /* kern/clock.h */
-
extern int vuprintf(const char *, va_list);
extern hrtime_t dtrace_abs_to_nano(uint64_t);
__private_extern__ const char * strstr(const char *, const char *);
+const void* bsearch(const void*, const void*, size_t, size_t, int (*compar)(const void *, const void *));
+
+int dtrace_copy_maxsize(void);
+int dtrace_buffer_copyout(const void*, user_addr_t, vm_size_t);
+
+
+#define DTRACE_NCLIENTS 32
#undef proc_t
projects / apple / xnu.git / blobdiff