bsd/sys/dtrace_impl.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Portions Copyright (c) 2012 by Delphix. All rights reserved.
  27  */
  28
  29 #ifndef _SYS_DTRACE_IMPL_H
  30 #define _SYS_DTRACE_IMPL_H
  31
  32 /* #pragma ident        "@(#)dtrace_impl.h      1.23    07/02/16 SMI" */
  33
  34 #ifdef  __cplusplus
  35 extern "C" {
  36 #endif
  37
  38 /*
  39  * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
  40  *
  41  * Note: The contents of this file are private to the implementation of the
  42  * Solaris system and DTrace subsystem and are subject to change at any time
  43  * without notice.  Applications and drivers using these interfaces will fail
  44  * to run on future releases.  These interfaces should not be used for any
  45  * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
  46  * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
  47  */
  48
  49 #include <sys/dtrace.h>
  50
  51 /*
  52  * DTrace Implementation Locks
  53  */
  54 extern lck_mtx_t dtrace_procwaitfor_lock;
  55
  56 /*
  57  * DTrace Implementation Constants and Typedefs
  58  */
  59 #define DTRACE_MAXPROPLEN               128
  60 #define DTRACE_DYNVAR_CHUNKSIZE         256
  61
  62 struct dtrace_probe;
  63 struct dtrace_ecb;
  64 struct dtrace_predicate;
  65 struct dtrace_action;
  66 struct dtrace_provider;
  67 struct dtrace_state;
  68
  69 typedef struct dtrace_probe dtrace_probe_t;
  70 typedef struct dtrace_ecb dtrace_ecb_t;
  71 typedef struct dtrace_predicate dtrace_predicate_t;
  72 typedef struct dtrace_action dtrace_action_t;
  73 typedef struct dtrace_provider dtrace_provider_t;
  74 typedef struct dtrace_meta dtrace_meta_t;
  75 typedef struct dtrace_state dtrace_state_t;
  76 typedef uint32_t dtrace_optid_t;
  77 typedef uint32_t dtrace_specid_t;
  78 typedef uint64_t dtrace_genid_t;
  79
  80 /*
  81  * DTrace Probes
  82  *
  83  * The probe is the fundamental unit of the DTrace architecture.  Probes are
  84  * created by DTrace providers, and managed by the DTrace framework.  A probe
  85  * is identified by a unique <provider, module, function, name> tuple, and has
  86  * a unique probe identifier assigned to it.  (Some probes are not associated
  87  * with a specific point in text; these are called _unanchored probes_ and have
  88  * no module or function associated with them.)  Probes are represented as a
  89  * dtrace_probe structure.  To allow quick lookups based on each element of the
  90  * probe tuple, probes are hashed by each of provider, module, function and
  91  * name.  (If a lookup is performed based on a regular expression, a
  92  * dtrace_probekey is prepared, and a linear search is performed.) Each probe
  93  * is additionally pointed to by a linear array indexed by its identifier.  The
  94  * identifier is the provider's mechanism for indicating to the DTrace
  95  * framework that a probe has fired:  the identifier is passed as the first
  96  * argument to dtrace_probe(), where it is then mapped into the corresponding
  97  * dtrace_probe structure.  From the dtrace_probe structure, dtrace_probe() can
  98  * iterate over the probe's list of enabling control blocks; see "DTrace
  99  * Enabling Control Blocks", below.)
 100  */
 101 struct dtrace_probe {
 102         dtrace_id_t dtpr_id;                    /* probe identifier */
 103         dtrace_ecb_t *dtpr_ecb;                 /* ECB list; see below */
 104         dtrace_ecb_t *dtpr_ecb_last;            /* last ECB in list */
 105         void *dtpr_arg;                         /* provider argument */
 106         dtrace_cacheid_t dtpr_predcache;        /* predicate cache ID */
 107         int dtpr_aframes;                       /* artificial frames */
 108         dtrace_provider_t *dtpr_provider;       /* pointer to provider */
 109         char *dtpr_mod;                         /* probe's module name */
 110         char *dtpr_func;                        /* probe's function name */
 111         char *dtpr_name;                        /* probe's name */
 112         dtrace_probe_t *dtpr_nextmod;           /* next in module hash */
 113         dtrace_probe_t *dtpr_prevmod;           /* previous in module hash */
 114         dtrace_probe_t *dtpr_nextfunc;          /* next in function hash */
 115         dtrace_probe_t *dtpr_prevfunc;          /* previous in function hash */
 116         dtrace_probe_t *dtpr_nextname;          /* next in name hash */
 117         dtrace_probe_t *dtpr_prevname;          /* previous in name hash */
 118         dtrace_genid_t dtpr_gen;                /* probe generation ID */
 119 };
 120
 121 typedef int dtrace_probekey_f(const char *, const char *, int);
 122
 123 typedef struct dtrace_probekey {
 124         const char *dtpk_prov;                  /* provider name to match */
 125         dtrace_probekey_f *dtpk_pmatch;         /* provider matching function */
 126         const char *dtpk_mod;                   /* module name to match */
 127         dtrace_probekey_f *dtpk_mmatch;         /* module matching function */
 128         const char *dtpk_func;                  /* func name to match */
 129         dtrace_probekey_f *dtpk_fmatch;         /* func matching function */
 130         const char *dtpk_name;                  /* name to match */
 131         dtrace_probekey_f *dtpk_nmatch;         /* name matching function */
 132         dtrace_id_t dtpk_id;                    /* identifier to match */
 133 } dtrace_probekey_t;
 134
 135 typedef struct dtrace_hashbucket {
 136         struct dtrace_hashbucket *dthb_next;    /* next on hash chain */
 137         dtrace_probe_t *dthb_chain;             /* chain of probes */
 138         int dthb_len;                           /* number of probes here */
 139 } dtrace_hashbucket_t;
 140
 141 typedef struct dtrace_hash {
 142         dtrace_hashbucket_t **dth_tab;          /* hash table */
 143         int dth_size;                           /* size of hash table */
 144         int dth_mask;                           /* mask to index into table */
 145         int dth_nbuckets;                       /* total number of buckets */
 146         uintptr_t dth_nextoffs;                 /* offset of next in probe */
 147         uintptr_t dth_prevoffs;                 /* offset of prev in probe */
 148         uintptr_t dth_stroffs;                  /* offset of str in probe */
 149 } dtrace_hash_t;
 150
 151 /*
 152  * DTrace Enabling Control Blocks
 153  *
 154  * When a provider wishes to fire a probe, it calls into dtrace_probe(),
 155  * passing the probe identifier as the first argument.  As described above,
 156  * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
 157  * structure.  This structure contains information about the probe, and a
 158  * pointer to the list of Enabling Control Blocks (ECBs).  Each ECB points to
 159  * DTrace consumer state, and contains an optional predicate, and a list of
 160  * actions.  (Shown schematically below.)  The ECB abstraction allows a single
 161  * probe to be multiplexed across disjoint consumers, or across disjoint
 162  * enablings of a single probe within one consumer.
 163  *
 164  *   Enabling Control Block
 165  *        dtrace_ecb_t
 166  * +------------------------+
 167  * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
 168  * | dtrace_state_t * ------+--------------> State associated with this ECB
 169  * | dtrace_predicate_t * --+---------+
 170  * | dtrace_action_t * -----+----+    |
 171  * | dtrace_ecb_t * ---+    |    |    |       Predicate (if any)
 172  * +-------------------+----+    |    |       dtrace_predicate_t
 173  *                     |         |    +---> +--------------------+
 174  *                     |         |          | dtrace_difo_t * ---+----> DIFO
 175  *                     |         |          +--------------------+
 176  *                     |         |
 177  *            Next ECB |         |           Action
 178  *            (if any) |         |       dtrace_action_t
 179  *                     :         +--> +-------------------+
 180  *                     :              | dtrace_actkind_t -+------> kind
 181  *                     v              | dtrace_difo_t * --+------> DIFO (if any)
 182  *                                    | dtrace_recdesc_t -+------> record descr.
 183  *                                    | dtrace_action_t * +------+
 184  *                                    +-------------------+      |
 185  *                                                               | Next action
 186  *                               +-------------------------------+  (if any)
 187  *                               |
 188  *                               |           Action
 189  *                               |       dtrace_action_t
 190  *                               +--> +-------------------+
 191  *                                    | dtrace_actkind_t -+------> kind
 192  *                                    | dtrace_difo_t * --+------> DIFO (if any)
 193  *                                    | dtrace_action_t * +------+
 194  *                                    +-------------------+      |
 195  *                                                               | Next action
 196  *                               +-------------------------------+  (if any)
 197  *                               |
 198  *                               :
 199  *                               v
 200  *
 201  *
 202  * dtrace_probe() iterates over the ECB list.  If the ECB needs less space
 203  * than is available in the principal buffer, the ECB is processed:  if the
 204  * predicate is non-NULL, the DIF object is executed.  If the result is
 205  * non-zero, the action list is processed, with each action being executed
 206  * accordingly.  When the action list has been completely executed, processing
 207  * advances to the next ECB. The ECB abstraction allows disjoint consumers
 208  * to multiplex on single probes.
 209  *
 210  * Execution of the ECB results in consuming dte_size bytes in the buffer
 211  * to record data.  During execution, dte_needed bytes must be available in
 212  * the buffer.  This space is used for both recorded data and tuple data.
 213  */
 214 struct dtrace_ecb {
 215         dtrace_epid_t dte_epid;                 /* enabled probe ID */
 216         uint32_t dte_alignment;                 /* required alignment */
 217         size_t dte_needed;                      /* space needed for execution */
 218         size_t dte_size;                        /* size of recorded payload */
 219         dtrace_predicate_t *dte_predicate;      /* predicate, if any */
 220         dtrace_action_t *dte_action;            /* actions, if any */
 221         dtrace_ecb_t *dte_next;                 /* next ECB on probe */
 222         dtrace_state_t *dte_state;              /* pointer to state */
 223         uint32_t dte_cond;                      /* security condition */
 224         dtrace_probe_t *dte_probe;              /* pointer to probe */
 225         dtrace_action_t *dte_action_last;       /* last action on ECB */
 226         uint64_t dte_uarg;                      /* library argument */
 227 };
 228
 229 struct dtrace_predicate {
 230         dtrace_difo_t *dtp_difo;                /* DIF object */
 231         dtrace_cacheid_t dtp_cacheid;           /* cache identifier */
 232         int dtp_refcnt;                         /* reference count */
 233 };
 234
 235 struct dtrace_action {
 236         dtrace_actkind_t dta_kind;              /* kind of action */
 237         uint16_t dta_intuple;                   /* boolean:  in aggregation */
 238         uint32_t dta_refcnt;                    /* reference count */
 239         dtrace_difo_t *dta_difo;                /* pointer to DIFO */
 240         dtrace_recdesc_t dta_rec;               /* record description */
 241         dtrace_action_t *dta_prev;              /* previous action */
 242         dtrace_action_t *dta_next;              /* next action */
 243 };
 244
 245 typedef struct dtrace_aggregation {
 246         dtrace_action_t dtag_action;            /* action; must be first */
 247         dtrace_aggid_t dtag_id;                 /* identifier */
 248         dtrace_ecb_t *dtag_ecb;                 /* corresponding ECB */
 249         dtrace_action_t *dtag_first;            /* first action in tuple */
 250         uint32_t dtag_base;                     /* base of aggregation */
 251         uint8_t dtag_hasarg;                    /* boolean:  has argument */
 252         uint64_t dtag_initial;                  /* initial value */
 253         void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t);
 254 } dtrace_aggregation_t;
 255
 256 /*
 257  * DTrace Buffers
 258  *
 259  * Principal buffers, aggregation buffers, and speculative buffers are all
 260  * managed with the dtrace_buffer structure.  By default, this structure
 261  * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
 262  * active and passive buffers, respectively.  For speculative buffers,
 263  * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
 264  * to a scratch buffer.  For all buffer types, the dtrace_buffer structure is
 265  * always allocated on a per-CPU basis; a single dtrace_buffer structure is
 266  * never shared among CPUs.  (That is, there is never true sharing of the
 267  * dtrace_buffer structure; to prevent false sharing of the structure, it must
 268  * always be aligned to the coherence granularity -- generally 64 bytes.)
 269  *
 270  * One of the critical design decisions of DTrace is that a given ECB always
 271  * stores the same quantity and type of data.  This is done to assure that the
 272  * only metadata required for an ECB's traced data is the EPID.  That is, from
 273  * the EPID, the consumer can determine the data layout.  (The data buffer
 274  * layout is shown schematically below.)  By assuring that one can determine
 275  * data layout from the EPID, the metadata stream can be separated from the
 276  * data stream -- simplifying the data stream enormously.  The ECB always
 277  * proceeds the recorded data as part of the dtrace_rechdr_t structure that
 278  * includes the EPID and a high-resolution timestamp used for output ordering
 279  * consistency.
 280  *
 281  *      base of data buffer --->  +--------+--------------------+--------+
 282  *                                | rechdr | data               | rechdr |
 283  *                                +--------+------+--------+----+--------+
 284  *                                | data          | rechdr | data        |
 285  *                                +---------------+--------+-------------+
 286  *                                | data, cont.                          |
 287  *                                +--------+--------------------+--------+
 288  *                                | rechdr | data               |        |
 289  *                                +--------+--------------------+        |
 290  *                                |                ||                    |
 291  *                                |                ||                    |
 292  *                                |                \/                    |
 293  *                                :                                      :
 294  *                                .                                      .
 295  *                                .                                      .
 296  *                                .                                      .
 297  *                                :                                      :
 298  *                                |                                      |
 299  *     limit of data buffer --->  +--------------------------------------+
 300  *
 301  * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
 302  * principal buffer (both scratch and payload) exceed the available space.  If
 303  * the ECB's needs exceed available space (and if the principal buffer policy
 304  * is the default "switch" policy), the ECB is dropped, the buffer's drop count
 305  * is incremented, and processing advances to the next ECB.  If the ECB's needs
 306  * can be met with the available space, the ECB is processed, but the offset in
 307  * the principal buffer is only advanced if the ECB completes processing
 308  * without error.
 309  *
 310  * When a buffer is to be switched (either because the buffer is the principal
 311  * buffer with a "switch" policy or because it is an aggregation buffer), a
 312  * cross call is issued to the CPU associated with the buffer.  In the cross
 313  * call context, interrupts are disabled, and the active and the inactive
 314  * buffers are atomically switched.  This involves switching the data pointers,
 315  * copying the various state fields (offset, drops, errors, etc.) into their
 316  * inactive equivalents, and clearing the state fields.  Because interrupts are
 317  * disabled during this procedure, the switch is guaranteed to appear atomic to
 318  * dtrace_probe().
 319  *
 320  * DTrace Ring Buffering
 321  *
 322  * To process a ring buffer correctly, one must know the oldest valid record.
 323  * Processing starts at the oldest record in the buffer and continues until
 324  * the end of the buffer is reached.  Processing then resumes starting with
 325  * the record stored at offset 0 in the buffer, and continues until the
 326  * youngest record is processed.  If trace records are of a fixed-length,
 327  * determining the oldest record is trivial:
 328  *
 329  *   - If the ring buffer has not wrapped, the oldest record is the record
 330  *     stored at offset 0.
 331  *
 332  *   - If the ring buffer has wrapped, the oldest record is the record stored
 333  *     at the current offset.
 334  *
 335  * With variable length records, however, just knowing the current offset
 336  * doesn't suffice for determining the oldest valid record:  assuming that one
 337  * allows for arbitrary data, one has no way of searching forward from the
 338  * current offset to find the oldest valid record.  (That is, one has no way
 339  * of separating data from metadata.) It would be possible to simply refuse to
 340  * process any data in the ring buffer between the current offset and the
 341  * limit, but this leaves (potentially) an enormous amount of otherwise valid
 342  * data unprocessed.
 343  *
 344  * To effect ring buffering, we track two offsets in the buffer:  the current
 345  * offset and the _wrapped_ offset.  If a request is made to reserve some
 346  * amount of data, and the buffer has wrapped, the wrapped offset is
 347  * incremented until the wrapped offset minus the current offset is greater
 348  * than or equal to the reserve request.  This is done by repeatedly looking
 349  * up the ECB corresponding to the EPID at the current wrapped offset, and
 350  * incrementing the wrapped offset by the size of the data payload
 351  * corresponding to that ECB.  If this offset is greater than or equal to the
 352  * limit of the data buffer, the wrapped offset is set to 0.  Thus, the
 353  * current offset effectively "chases" the wrapped offset around the buffer.
 354  * Schematically:
 355  *
 356  *      base of data buffer --->  +------+--------------------+------+
 357  *                                | EPID | data               | EPID |
 358  *                                +------+--------+------+----+------+
 359  *                                | data          | EPID | data      |
 360  *                                +---------------+------+-----------+
 361  *                                | data, cont.                      |
 362  *                                +------+---------------------------+
 363  *                                | EPID | data                      |
 364  *           current offset --->  +------+---------------------------+
 365  *                                | invalid data                     |
 366  *           wrapped offset --->  +------+--------------------+------+
 367  *                                | EPID | data               | EPID |
 368  *                                +------+--------+------+----+------+
 369  *                                | data          | EPID | data      |
 370  *                                +---------------+------+-----------+
 371  *                                :                                  :
 372  *                                .                                  .
 373  *                                .        ... valid data ...        .
 374  *                                .                                  .
 375  *                                :                                  :
 376  *                                +------+-------------+------+------+
 377  *                                | EPID | data        | EPID | data |
 378  *                                +------+------------++------+------+
 379  *                                | data, cont.       | leftover     |
 380  *     limit of data buffer --->  +-------------------+--------------+
 381  *
 382  * If the amount of requested buffer space exceeds the amount of space
 383  * available between the current offset and the end of the buffer:
 384  *
 385  *  (1)  all words in the data buffer between the current offset and the limit
 386  *       of the data buffer (marked "leftover", above) are set to
 387  *       DTRACE_EPIDNONE
 388  *
 389  *  (2)  the wrapped offset is set to zero
 390  *
 391  *  (3)  the iteration process described above occurs until the wrapped offset
 392  *       is greater than the amount of desired space.
 393  *
 394  * The wrapped offset is implemented by (re-)using the inactive offset.
 395  * In a "switch" buffer policy, the inactive offset stores the offset in
 396  * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
 397  * offset.
 398  *
 399  * DTrace Scratch Buffering
 400  *
 401  * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
 402  * To accommodate such requests easily, scratch memory may be allocated in
 403  * the buffer beyond the current offset plus the needed memory of the current
 404  * ECB.  If there isn't sufficient room in the buffer for the requested amount
 405  * of scratch space, the allocation fails and an error is generated.  Scratch
 406  * memory is tracked in the dtrace_mstate_t and is automatically freed when
 407  * the ECB ceases processing.  Note that ring buffers cannot allocate their
 408  * scratch from the principal buffer -- lest they needlessly overwrite older,
 409  * valid data.  Ring buffers therefore have their own dedicated scratch buffer
 410  * from which scratch is allocated.
 411  */
 412 #define DTRACEBUF_RING          0x0001          /* bufpolicy set to "ring" */
 413 #define DTRACEBUF_FILL          0x0002          /* bufpolicy set to "fill" */
 414 #define DTRACEBUF_NOSWITCH      0x0004          /* do not switch buffer */
 415 #define DTRACEBUF_WRAPPED       0x0008          /* ring buffer has wrapped */
 416 #define DTRACEBUF_DROPPED       0x0010          /* drops occurred */
 417 #define DTRACEBUF_ERROR         0x0020          /* errors occurred */
 418 #define DTRACEBUF_FULL          0x0040          /* "fill" buffer is full */
 419 #define DTRACEBUF_CONSUMED      0x0080          /* buffer has been consumed */
 420 #define DTRACEBUF_INACTIVE      0x0100          /* buffer is not yet active */
 421
 422 typedef struct dtrace_buffer {
 423         uint64_t dtb_offset;                    /* current offset in buffer */
 424         uint64_t dtb_size;                      /* size of buffer */
 425         uint32_t dtb_flags;                     /* flags */
 426         uint32_t dtb_drops;                     /* number of drops */
 427         caddr_t dtb_tomax;                      /* active buffer */
 428         caddr_t dtb_xamot;                      /* inactive buffer */
 429         uint32_t dtb_xamot_flags;               /* inactive flags */
 430         uint32_t dtb_xamot_drops;               /* drops in inactive buffer */
 431         uint64_t dtb_xamot_offset;              /* offset in inactive buffer */
 432         uint32_t dtb_errors;                    /* number of errors */
 433         uint32_t dtb_xamot_errors;              /* errors in inactive buffer */
 434 #ifndef _LP64
 435         uint64_t dtb_pad1;
 436 #endif
 437         uint64_t dtb_switched;                  /* time of last switch */
 438         uint64_t dtb_interval;                  /* observed switch interval */
 439 } dtrace_buffer_t;
 440
 441 /*
 442  * DTrace Aggregation Buffers
 443  *
 444  * Aggregation buffers use much of the same mechanism as described above
 445  * ("DTrace Buffers").  However, because an aggregation is fundamentally a
 446  * hash, there exists dynamic metadata associated with an aggregation buffer
 447  * that is not associated with other kinds of buffers.  This aggregation
 448  * metadata is _only_ relevant for the in-kernel implementation of
 449  * aggregations; it is not actually relevant to user-level consumers.  To do
 450  * this, we allocate dynamic aggregation data (hash keys and hash buckets)
 451  * starting below the _limit_ of the buffer, and we allocate data from the
 452  * _base_ of the buffer.  When the aggregation buffer is copied out, _only_ the
 453  * data is copied out; the metadata is simply discarded.  Schematically,
 454  * aggregation buffers look like:
 455  *
 456  *      base of data buffer --->  +-------+------+-----------+-------+
 457  *                                | aggid | key  | value     | aggid |
 458  *                                +-------+------+-----------+-------+
 459  *                                | key                              |
 460  *                                +-------+-------+-----+------------+
 461  *                                | value | aggid | key | value      |
 462  *                                +-------+------++-----+------+-----+
 463  *                                | aggid | key  | value       |     |
 464  *                                +-------+------+-------------+     |
 465  *                                |                ||                |
 466  *                                |                ||                |
 467  *                                |                \/                |
 468  *                                :                                  :
 469  *                                .                                  .
 470  *                                .                                  .
 471  *                                .                                  .
 472  *                                :                                  :
 473  *                                |                /\                |
 474  *                                |                ||   +------------+
 475  *                                |                ||   |            |
 476  *                                +---------------------+            |
 477  *                                | hash keys                        |
 478  *                                | (dtrace_aggkey structures)       |
 479  *                                |                                  |
 480  *                                +----------------------------------+
 481  *                                | hash buckets                     |
 482  *                                | (dtrace_aggbuffer structure)     |
 483  *                                |                                  |
 484  *     limit of data buffer --->  +----------------------------------+
 485  *
 486  *
 487  * As implied above, just as we assure that ECBs always store a constant
 488  * amount of data, we assure that a given aggregation -- identified by its
 489  * aggregation ID -- always stores data of a constant quantity and type.
 490  * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
 491  * given record.
 492  *
 493  * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
 494  * aligned.  (If this the structure changes such that this becomes false, an
 495  * assertion will fail in dtrace_aggregate().)
 496  */
 497 typedef struct dtrace_aggkey {
 498         uint32_t dtak_hashval;                  /* hash value */
 499         uint32_t dtak_action:4;                 /* action -- 4 bits */
 500         uint32_t dtak_size:28;                  /* size -- 28 bits */
 501         caddr_t dtak_data;                      /* data pointer */
 502         struct dtrace_aggkey *dtak_next;        /* next in hash chain */
 503 } dtrace_aggkey_t;
 504
 505 typedef struct dtrace_aggbuffer {
 506         uintptr_t dtagb_hashsize;               /* number of buckets */
 507         uintptr_t dtagb_free;                   /* free list of keys */
 508         dtrace_aggkey_t **dtagb_hash;           /* hash table */
 509 } dtrace_aggbuffer_t;
 510
 511 /*
 512  * DTrace Speculations
 513  *
 514  * Speculations have a per-CPU buffer and a global state.  Once a speculation
 515  * buffer has been comitted or discarded, it cannot be reused until all CPUs
 516  * have taken the same action (commit or discard) on their respective
 517  * speculative buffer.  However, because DTrace probes may execute in arbitrary
 518  * context, other CPUs cannot simply be cross-called at probe firing time to
 519  * perform the necessary commit or discard.  The speculation states thus
 520  * optimize for the case that a speculative buffer is only active on one CPU at
 521  * the time of a commit() or discard() -- for if this is the case, other CPUs
 522  * need not take action, and the speculation is immediately available for
 523  * reuse.  If the speculation is active on multiple CPUs, it must be
 524  * asynchronously cleaned -- potentially leading to a higher rate of dirty
 525  * speculative drops.  The speculation states are as follows:
 526  *
 527  *  DTRACESPEC_INACTIVE       <= Initial state; inactive speculation
 528  *  DTRACESPEC_ACTIVE         <= Allocated, but not yet speculatively traced to
 529  *  DTRACESPEC_ACTIVEONE      <= Speculatively traced to on one CPU
 530  *  DTRACESPEC_ACTIVEMANY     <= Speculatively traced to on more than one CPU
 531  *  DTRACESPEC_COMMITTING     <= Currently being commited on one CPU
 532  *  DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
 533  *  DTRACESPEC_DISCARDING     <= Currently being discarded on many CPUs
 534  *
 535  * The state transition diagram is as follows:
 536  *
 537  *     +----------------------------------------------------------+
 538  *     |                                                          |
 539  *     |                      +------------+                      |
 540  *     |  +-------------------| COMMITTING |<-----------------+   |
 541  *     |  |                   +------------+                  |   |
 542  *     |  | copied spec.            ^             commit() on |   | discard() on
 543  *     |  | into principal          |              active CPU |   | active CPU
 544  *     |  |                         | commit()                |   |
 545  *     V  V                         |                         |   |
 546  * +----------+                 +--------+                +-----------+
 547  * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
 548  * +----------+  speculation()  +--------+  speculate()   +-----------+
 549  *     ^  ^                         |                         |   |
 550  *     |  |                         | discard()               |   |
 551  *     |  | asynchronously          |            discard() on |   | speculate()
 552  *     |  | cleaned                 V            inactive CPU |   | on inactive
 553  *     |  |                   +------------+                  |   | CPU
 554  *     |  +-------------------| DISCARDING |<-----------------+   |
 555  *     |                      +------------+                      |
 556  *     | asynchronously             ^                             |
 557  *     | copied spec.               |       discard()             |
 558  *     | into principal             +------------------------+    |
 559  *     |                                                     |    V
 560  *  +----------------+             commit()              +------------+
 561  *  | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
 562  *  +----------------+                                   +------------+
 563  */
 564 typedef enum dtrace_speculation_state {
 565         DTRACESPEC_INACTIVE = 0,
 566         DTRACESPEC_ACTIVE,
 567         DTRACESPEC_ACTIVEONE,
 568         DTRACESPEC_ACTIVEMANY,
 569         DTRACESPEC_COMMITTING,
 570         DTRACESPEC_COMMITTINGMANY,
 571         DTRACESPEC_DISCARDING
 572 } dtrace_speculation_state_t;
 573
 574 typedef struct dtrace_speculation {
 575         dtrace_speculation_state_t dtsp_state;  /* current speculation state */
 576         int dtsp_cleaning;                      /* non-zero if being cleaned */
 577         dtrace_buffer_t *dtsp_buffer;           /* speculative buffer */
 578 } dtrace_speculation_t;
 579
 580 /*
 581  * DTrace Dynamic Variables
 582  *
 583  * The dynamic variable problem is obviously decomposed into two subproblems:
 584  * allocating new dynamic storage, and freeing old dynamic storage.  The
 585  * presence of the second problem makes the first much more complicated -- or
 586  * rather, the absence of the second renders the first trivial.  This is the
 587  * case with aggregations, for which there is effectively no deallocation of
 588  * dynamic storage.  (Or more accurately, all dynamic storage is deallocated
 589  * when a snapshot is taken of the aggregation.)  As DTrace dynamic variables
 590  * allow for both dynamic allocation and dynamic deallocation, the
 591  * implementation of dynamic variables is quite a bit more complicated than
 592  * that of their aggregation kin.
 593  *
 594  * We observe that allocating new dynamic storage is tricky only because the
 595  * size can vary -- the allocation problem is much easier if allocation sizes
 596  * are uniform.  We further observe that in D, the size of dynamic variables is
 597  * actually _not_ dynamic -- dynamic variable sizes may be determined by static
 598  * analysis of DIF text.  (This is true even of putatively dynamically-sized
 599  * objects like strings and stacks, the sizes of which are dictated by the
 600  * "stringsize" and "stackframes" variables, respectively.)  We exploit this by
 601  * performing this analysis on all DIF before enabling any probes.  For each
 602  * dynamic load or store, we calculate the dynamically-allocated size plus the
 603  * size of the dtrace_dynvar structure plus the storage required to key the
 604  * data.  For all DIF, we take the largest value and dub it the _chunksize_.
 605  * We then divide dynamic memory into two parts:  a hash table that is wide
 606  * enough to have every chunk in its own bucket, and a larger region of equal
 607  * chunksize units.  Whenever we wish to dynamically allocate a variable, we
 608  * always allocate a single chunk of memory.  Depending on the uniformity of
 609  * allocation, this will waste some amount of memory -- but it eliminates the
 610  * non-determinism inherent in traditional heap fragmentation.
 611  *
 612  * Dynamic objects are allocated by storing a non-zero value to them; they are
 613  * deallocated by storing a zero value to them.  Dynamic variables are
 614  * complicated enormously by being shared between CPUs.  In particular,
 615  * consider the following scenario:
 616  *
 617  *                 CPU A                                 CPU B
 618  *  +---------------------------------+   +---------------------------------+
 619  *  |                                 |   |                                 |
 620  *  | allocates dynamic object a[123] |   |                                 |
 621  *  | by storing the value 345 to it  |   |                                 |
 622  *  |                               --------->                              |
 623  *  |                                 |   | wishing to load from object     |
 624  *  |                                 |   | a[123], performs lookup in      |
 625  *  |                                 |   | dynamic variable space          |
 626  *  |                               <---------                              |
 627  *  | deallocates object a[123] by    |   |                                 |
 628  *  | storing 0 to it                 |   |                                 |
 629  *  |                                 |   |                                 |
 630  *  | allocates dynamic object b[567] |   | performs load from a[123]       |
 631  *  | by storing the value 789 to it  |   |                                 |
 632  *  :                                 :   :                                 :
 633  *  .                                 .   .                                 .
 634  *
 635  * This is obviously a race in the D program, but there are nonetheless only
 636  * two valid values for CPU B's load from a[123]:  345 or 0.  Most importantly,
 637  * CPU B may _not_ see the value 789 for a[123].
 638  *
 639  * There are essentially two ways to deal with this:
 640  *
 641  *  (1)  Explicitly spin-lock variables.  That is, if CPU B wishes to load
 642  *       from a[123], it needs to lock a[123] and hold the lock for the
 643  *       duration that it wishes to manipulate it.
 644  *
 645  *  (2)  Avoid reusing freed chunks until it is known that no CPU is referring
 646  *       to them.
 647  *
 648  * The implementation of (1) is rife with complexity, because it requires the
 649  * user of a dynamic variable to explicitly decree when they are done using it.
 650  * Were all variables by value, this perhaps wouldn't be debilitating -- but
 651  * dynamic variables of non-scalar types are tracked by reference.  That is, if
 652  * a dynamic variable is, say, a string, and that variable is to be traced to,
 653  * say, the principal buffer, the DIF emulation code returns to the main
 654  * dtrace_probe() loop a pointer to the underlying storage, not the contents of
 655  * the storage.  Further, code calling on DIF emulation would have to be aware
 656  * that the DIF emulation has returned a reference to a dynamic variable that
 657  * has been potentially locked.  The variable would have to be unlocked after
 658  * the main dtrace_probe() loop is finished with the variable, and the main
 659  * dtrace_probe() loop would have to be careful to not call any further DIF
 660  * emulation while the variable is locked to avoid deadlock.  More generally,
 661  * if one were to implement (1), DIF emulation code dealing with dynamic
 662  * variables could only deal with one dynamic variable at a time (lest deadlock
 663  * result).  To sum, (1) exports too much subtlety to the users of dynamic
 664  * variables -- increasing maintenance burden and imposing serious constraints
 665  * on future DTrace development.
 666  *
 667  * The implementation of (2) is also complex, but the complexity is more
 668  * manageable.  We need to be sure that when a variable is deallocated, it is
 669  * not placed on a traditional free list, but rather on a _dirty_ list.  Once a
 670  * variable is on a dirty list, it cannot be found by CPUs performing a
 671  * subsequent lookup of the variable -- but it may still be in use by other
 672  * CPUs.  To assure that all CPUs that may be seeing the old variable have
 673  * cleared out of probe context, a dtrace_sync() can be issued.  Once the
 674  * dtrace_sync() has completed, it can be known that all CPUs are done
 675  * manipulating the dynamic variable -- the dirty list can be atomically
 676  * appended to the free list.  Unfortunately, there's a slight hiccup in this
 677  * mechanism:  dtrace_sync() may not be issued from probe context.  The
 678  * dtrace_sync() must be therefore issued asynchronously from non-probe
 679  * context.  For this we rely on the DTrace cleaner, a cyclic that runs at the
 680  * "cleanrate" frequency.  To ease this implementation, we define several chunk
 681  * lists:
 682  *
 683  *   - Dirty.  Deallocated chunks, not yet cleaned.  Not available.
 684  *
 685  *   - Rinsing.  Formerly dirty chunks that are currently being asynchronously
 686  *     cleaned.  Not available, but will be shortly.  Dynamic variable
 687  *     allocation may not spin or block for availability, however.
 688  *
 689  *   - Clean.  Clean chunks, ready for allocation -- but not on the free list.
 690  *
 691  *   - Free.  Available for allocation.
 692  *
 693  * Moreover, to avoid absurd contention, _each_ of these lists is implemented
 694  * on a per-CPU basis.  This is only for performance, not correctness; chunks
 695  * may be allocated from another CPU's free list.  The algorithm for allocation
 696  * then is this:
 697  *
 698  *   (1)  Attempt to atomically allocate from current CPU's free list.  If list
 699  *        is non-empty and allocation is successful, allocation is complete.
 700  *
 701  *   (2)  If the clean list is non-empty, atomically move it to the free list,
 702  *        and reattempt (1).
 703  *
 704  *   (3)  If the dynamic variable space is in the CLEAN state, look for free
 705  *        and clean lists on other CPUs by setting the current CPU to the next
 706  *        CPU, and reattempting (1).  If the next CPU is the current CPU (that
 707  *        is, if all CPUs have been checked), atomically switch the state of
 708  *        the dynamic variable space based on the following:
 709  *
 710  *        - If no free chunks were found and no dirty chunks were found,
 711  *          atomically set the state to EMPTY.
 712  *
 713  *        - If dirty chunks were found, atomically set the state to DIRTY.
 714  *
 715  *        - If rinsing chunks were found, atomically set the state to RINSING.
 716  *
 717  *   (4)  Based on state of dynamic variable space state, increment appropriate
 718  *        counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
 719  *        dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
 720  *        RINSING state).  Fail the allocation.
 721  *
 722  * The cleaning cyclic operates with the following algorithm:  for all CPUs
 723  * with a non-empty dirty list, atomically move the dirty list to the rinsing
 724  * list.  Perform a dtrace_sync().  For all CPUs with a non-empty rinsing list,
 725  * atomically move the rinsing list to the clean list.  Perform another
 726  * dtrace_sync().  By this point, all CPUs have seen the new clean list; the
 727  * state of the dynamic variable space can be restored to CLEAN.
 728  *
 729  * There exist two final races that merit explanation.  The first is a simple
 730  * allocation race:
 731  *
 732  *                 CPU A                                 CPU B
 733  *  +---------------------------------+   +---------------------------------+
 734  *  |                                 |   |                                 |
 735  *  | allocates dynamic object a[123] |   | allocates dynamic object a[123] |
 736  *  | by storing the value 345 to it  |   | by storing the value 567 to it  |
 737  *  |                                 |   |                                 |
 738  *  :                                 :   :                                 :
 739  *  .                                 .   .                                 .
 740  *
 741  * Again, this is a race in the D program.  It can be resolved by having a[123]
 742  * hold the value 345 or a[123] hold the value 567 -- but it must be true that
 743  * a[123] have only _one_ of these values.  (That is, the racing CPUs may not
 744  * put the same element twice on the same hash chain.)  This is resolved
 745  * simply:  before the allocation is undertaken, the start of the new chunk's
 746  * hash chain is noted.  Later, after the allocation is complete, the hash
 747  * chain is atomically switched to point to the new element.  If this fails
 748  * (because of either concurrent allocations or an allocation concurrent with a
 749  * deletion), the newly allocated chunk is deallocated to the dirty list, and
 750  * the whole process of looking up (and potentially allocating) the dynamic
 751  * variable is reattempted.
 752  *
 753  * The final race is a simple deallocation race:
 754  *
 755  *                 CPU A                                 CPU B
 756  *  +---------------------------------+   +---------------------------------+
 757  *  |                                 |   |                                 |
 758  *  | deallocates dynamic object      |   | deallocates dynamic object      |
 759  *  | a[123] by storing the value 0   |   | a[123] by storing the value 0   |
 760  *  | to it                           |   | to it                           |
 761  *  |                                 |   |                                 |
 762  *  :                                 :   :                                 :
 763  *  .                                 .   .                                 .
 764  *
 765  * Once again, this is a race in the D program, but it is one that we must
 766  * handle without corrupting the underlying data structures.  Because
 767  * deallocations require the deletion of a chunk from the middle of a hash
 768  * chain, we cannot use a single-word atomic operation to remove it.  For this,
 769  * we add a spin lock to the hash buckets that is _only_ used for deallocations
 770  * (allocation races are handled as above).  Further, this spin lock is _only_
 771  * held for the duration of the delete; before control is returned to the DIF
 772  * emulation code, the hash bucket is unlocked.
 773  */
 774 typedef struct dtrace_key {
 775         uint64_t dttk_value;                    /* data value or data pointer */
 776         uint64_t dttk_size;                     /* 0 if by-val, >0 if by-ref */
 777 } dtrace_key_t;
 778
 779 typedef struct dtrace_tuple {
 780         uint32_t dtt_nkeys;                     /* number of keys in tuple */
 781         uint32_t dtt_pad;                       /* padding */
 782         dtrace_key_t dtt_key[1];                /* array of tuple keys */
 783 } dtrace_tuple_t;
 784
 785 typedef struct dtrace_dynvar {
 786         uint64_t dtdv_hashval;                  /* hash value -- 0 if free */
 787         struct dtrace_dynvar *dtdv_next;        /* next on list or hash chain */
 788         void *dtdv_data;                        /* pointer to data */
 789         dtrace_tuple_t dtdv_tuple;              /* tuple key */
 790 } dtrace_dynvar_t;
 791
 792 typedef enum dtrace_dynvar_op {
 793         DTRACE_DYNVAR_ALLOC,
 794         DTRACE_DYNVAR_NOALLOC,
 795         DTRACE_DYNVAR_DEALLOC
 796 } dtrace_dynvar_op_t;
 797
 798 typedef struct dtrace_dynhash {
 799         dtrace_dynvar_t *dtdh_chain;            /* hash chain for this bucket */
 800         uintptr_t dtdh_lock;                    /* deallocation lock */
 801 #ifdef _LP64
 802         uintptr_t dtdh_pad[6];                  /* pad to avoid false sharing */
 803 #else
 804         uintptr_t dtdh_pad[14];                 /* pad to avoid false sharing */
 805 #endif
 806 } dtrace_dynhash_t;
 807
 808 typedef struct dtrace_dstate_percpu {
 809         dtrace_dynvar_t *dtdsc_free;            /* free list for this CPU */
 810         dtrace_dynvar_t *dtdsc_dirty;           /* dirty list for this CPU */
 811         dtrace_dynvar_t *dtdsc_rinsing;         /* rinsing list for this CPU */
 812         dtrace_dynvar_t *dtdsc_clean;           /* clean list for this CPU */
 813         uint64_t dtdsc_drops;                   /* number of capacity drops */
 814         uint64_t dtdsc_dirty_drops;             /* number of dirty drops */
 815         uint64_t dtdsc_rinsing_drops;           /* number of rinsing drops */
 816 #ifdef _LP64
 817         uint64_t dtdsc_pad;                     /* pad to avoid false sharing */
 818 #else
 819         uint64_t dtdsc_pad[2];                  /* pad to avoid false sharing */
 820 #endif
 821 } dtrace_dstate_percpu_t;
 822
 823 typedef enum dtrace_dstate_state {
 824         DTRACE_DSTATE_CLEAN = 0,
 825         DTRACE_DSTATE_EMPTY,
 826         DTRACE_DSTATE_DIRTY,
 827         DTRACE_DSTATE_RINSING
 828 } dtrace_dstate_state_t;
 829
 830 typedef struct dtrace_dstate {
 831         void *dtds_base;                        /* base of dynamic var. space */
 832         size_t dtds_size;                       /* size of dynamic var. space */
 833         size_t dtds_hashsize;                   /* number of buckets in hash */
 834         size_t dtds_chunksize;                  /* size of each chunk */
 835         dtrace_dynhash_t *dtds_hash;            /* pointer to hash table */
 836         dtrace_dstate_state_t dtds_state;       /* current dynamic var. state */
 837         dtrace_dstate_percpu_t *dtds_percpu;    /* per-CPU dyn. var. state */
 838 } dtrace_dstate_t;
 839
 840 /*
 841  * DTrace Variable State
 842  *
 843  * The DTrace variable state tracks user-defined variables in its dtrace_vstate
 844  * structure.  Each DTrace consumer has exactly one dtrace_vstate structure,
 845  * but some dtrace_vstate structures may exist without a corresponding DTrace
 846  * consumer (see "DTrace Helpers", below).  As described in <sys/dtrace.h>,
 847  * user-defined variables can have one of three scopes:
 848  *
 849  *  DIFV_SCOPE_GLOBAL  =>  global scope
 850  *  DIFV_SCOPE_THREAD  =>  thread-local scope (i.e. "self->" variables)
 851  *  DIFV_SCOPE_LOCAL   =>  clause-local scope (i.e. "this->" variables)
 852  *
 853  * The variable state tracks variables by both their scope and their allocation
 854  * type:
 855  *
 856  *  - The dtvs_globals and dtvs_locals members each point to an array of
 857  *    dtrace_statvar structures.  These structures contain both the variable
 858  *    metadata (dtrace_difv structures) and the underlying storage for all
 859  *    statically allocated variables, including statically allocated
 860  *    DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
 861  *
 862  *  - The dtvs_tlocals member points to an array of dtrace_difv structures for
 863  *    DIFV_SCOPE_THREAD variables.  As such, this array tracks _only_ the
 864  *    variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
 865  *    is allocated out of the dynamic variable space.
 866  *
 867  *  - The dtvs_dynvars member is the dynamic variable state associated with the
 868  *    variable state.  The dynamic variable state (described in "DTrace Dynamic
 869  *    Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
 870  *    dynamically-allocated DIFV_SCOPE_GLOBAL variables.
 871  */
 872 typedef struct dtrace_statvar {
 873         uint64_t dtsv_data;                     /* data or pointer to it */
 874         size_t dtsv_size;                       /* size of pointed-to data */
 875         int dtsv_refcnt;                        /* reference count */
 876         dtrace_difv_t dtsv_var;                 /* variable metadata */
 877 } dtrace_statvar_t;
 878
 879 typedef struct dtrace_vstate {
 880         dtrace_state_t *dtvs_state;             /* back pointer to state */
 881         dtrace_statvar_t **dtvs_globals;        /* statically-allocated glbls */
 882         int dtvs_nglobals;                      /* number of globals */
 883         dtrace_difv_t *dtvs_tlocals;            /* thread-local metadata */
 884         int dtvs_ntlocals;                      /* number of thread-locals */
 885         dtrace_statvar_t **dtvs_locals;         /* clause-local data */
 886         int dtvs_nlocals;                       /* number of clause-locals */
 887         dtrace_dstate_t dtvs_dynvars;           /* dynamic variable state */
 888 } dtrace_vstate_t;
 889
 890 /*
 891  * DTrace Machine State
 892  *
 893  * In the process of processing a fired probe, DTrace needs to track and/or
 894  * cache some per-CPU state associated with that particular firing.  This is
 895  * state that is always discarded after the probe firing has completed, and
 896  * much of it is not specific to any DTrace consumer, remaining valid across
 897  * all ECBs.  This state is tracked in the dtrace_mstate structure.
 898  */
 899 #define DTRACE_MSTATE_ARGS              0x00000001
 900 #define DTRACE_MSTATE_PROBE             0x00000002
 901 #define DTRACE_MSTATE_EPID              0x00000004
 902 #define DTRACE_MSTATE_TIMESTAMP         0x00000008
 903 #define DTRACE_MSTATE_STACKDEPTH        0x00000010
 904 #define DTRACE_MSTATE_CALLER            0x00000020
 905 #define DTRACE_MSTATE_IPL               0x00000040
 906 #define DTRACE_MSTATE_FLTOFFS           0x00000080
 907 #define DTRACE_MSTATE_WALLTIMESTAMP     0x00000100
 908 #define DTRACE_MSTATE_USTACKDEPTH       0x00000200
 909 #define DTRACE_MSTATE_UCALLER           0x00000400
 910 #define DTRACE_MSTATE_MACHTIMESTAMP     0x00000800
 911
 912 typedef struct dtrace_mstate {
 913         uintptr_t dtms_scratch_base;            /* base of scratch space */
 914         uintptr_t dtms_scratch_ptr;             /* current scratch pointer */
 915         size_t dtms_scratch_size;               /* scratch size */
 916         uint32_t dtms_present;                  /* variables that are present */
 917         uint64_t dtms_arg[5];                   /* cached arguments */
 918         dtrace_epid_t dtms_epid;                /* current EPID */
 919         uint64_t dtms_timestamp;                /* cached timestamp */
 920         hrtime_t dtms_walltimestamp;            /* cached wall timestamp */
 921         uint64_t dtms_machtimestamp;            /* cached mach absolute timestamp */
 922         int dtms_stackdepth;                    /* cached stackdepth */
 923         int dtms_ustackdepth;                   /* cached ustackdepth */
 924         struct dtrace_probe *dtms_probe;        /* current probe */
 925         uintptr_t dtms_caller;                  /* cached caller */
 926         uint64_t dtms_ucaller;                  /* cached user-level caller */
 927         int dtms_ipl;                           /* cached interrupt pri lev */
 928         int dtms_fltoffs;                       /* faulting DIFO offset */
 929         uintptr_t dtms_strtok;                  /* saved strtok() pointer */
 930         uint32_t dtms_access;                   /* memory access rights */
 931         dtrace_difo_t *dtms_difo;               /* current dif object */
 932 } dtrace_mstate_t;
 933
 934 #define DTRACE_COND_OWNER       0x1
 935 #define DTRACE_COND_USERMODE    0x2
 936 #define DTRACE_COND_ZONEOWNER   0x4
 937
 938 #define DTRACE_PROBEKEY_MAXDEPTH        8       /* max glob recursion depth */
 939
 940 /*
 941  * Access flag used by dtrace_mstate.dtms_access.
 942  */
 943 #define DTRACE_ACCESS_KERNEL    0x1             /* the priv to read kmem */
 944
 945
 946 /*
 947  * DTrace Activity
 948  *
 949  * Each DTrace consumer is in one of several states, which (for purposes of
 950  * avoiding yet-another overloading of the noun "state") we call the current
 951  * _activity_.  The activity transitions on dtrace_go() (from DTRACIOCGO), on
 952  * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action.  Activities may
 953  * only transition in one direction; the activity transition diagram is a
 954  * directed acyclic graph.  The activity transition diagram is as follows:
 955  *
 956  *
 957  * +----------+                   +--------+                   +--------+
 958  * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
 959  * +----------+   dtrace_go(),    +--------+   dtrace_go(),    +--------+
 960  *                before BEGIN        |        after BEGIN       |  |  |
 961  *                                    |                          |  |  |
 962  *                      exit() action |                          |  |  |
 963  *                     from BEGIN ECB |                          |  |  |
 964  *                                    |                          |  |  |
 965  *                                    v                          |  |  |
 966  *                               +----------+     exit() action  |  |  |
 967  * +-----------------------------| DRAINING |<-------------------+  |  |
 968  * |                             +----------+                       |  |
 969  * |                                  |                             |  |
 970  * |                   dtrace_stop(), |                             |  |
 971  * |                     before END   |                             |  |
 972  * |                                  |                             |  |
 973  * |                                  v                             |  |
 974  * | +---------+                 +----------+                       |  |
 975  * | | STOPPED |<----------------| COOLDOWN |<----------------------+  |
 976  * | +---------+  dtrace_stop(), +----------+     dtrace_stop(),       |
 977  * |                after END                       before END         |
 978  * |                                                                   |
 979  * |                              +--------+                           |
 980  * +----------------------------->| KILLED |<--------------------------+
 981  *       deadman timeout or       +--------+     deadman timeout or
 982  *        killed consumer                         killed consumer
 983  *
 984  * Note that once a DTrace consumer has stopped tracing, there is no way to
 985  * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
 986  * the DTrace pseudodevice.
 987  */
 988 typedef enum dtrace_activity {
 989         DTRACE_ACTIVITY_INACTIVE = 0,           /* not yet running */
 990         DTRACE_ACTIVITY_WARMUP,                 /* while starting */
 991         DTRACE_ACTIVITY_ACTIVE,                 /* running */
 992         DTRACE_ACTIVITY_DRAINING,               /* before stopping */
 993         DTRACE_ACTIVITY_COOLDOWN,               /* while stopping */
 994         DTRACE_ACTIVITY_STOPPED,                /* after stopping */
 995         DTRACE_ACTIVITY_KILLED                  /* killed */
 996 } dtrace_activity_t;
 997
 998
 999 /*
1000  * APPLE NOTE:  DTrace dof modes implementation
1001  *
1002  * DTrace has four "dof modes". They are:
1003  *
1004  * DTRACE_DOF_MODE_NEVER        Never load any dof, period.
1005  * DTRACE_DOF_MODE_LAZY_ON      Defer loading dof until later
1006  * DTRACE_DOF_MODE_LAZY_OFF     Load all deferred dof now, and any new dof
1007  * DTRACE_DOF_MODE_NON_LAZY     Load all dof immediately.
1008  *
1009  * It is legal to transition between the two lazy modes. The NEVER and
1010  * NON_LAZY modes are permanent, and must not change once set.
1011  *
1012  * The current dof mode is kept in dtrace_dof_mode, which is protected by the
1013  * dtrace_dof_mode_lock. This is a RW lock, reads require shared access, writes
1014  * require exclusive access. Because NEVER and NON_LAZY are permanent states,
1015  * it is legal to test for those modes without holding the dof mode lock.
1016  *
1017  * Lock ordering is dof mode lock before any dtrace lock, and before the
1018  * process p_dtrace_sprlock. In general, other locks should not be held when
1019  * taking the dof mode lock. Acquiring the dof mode lock in exclusive mode
1020  * will block process fork, exec, and exit, so it should be held exclusive
1021  * for as short a time as possible.
1022  */
1023
1024 #define DTRACE_DOF_MODE_NEVER           0
1025 #define DTRACE_DOF_MODE_LAZY_ON         1
1026 #define DTRACE_DOF_MODE_LAZY_OFF        2
1027 #define DTRACE_DOF_MODE_NON_LAZY        3
1028
1029 /*
1030  * dtrace kernel symbol modes are used to control when the kernel may dispose of
1031  * symbol information used by the fbt/sdt provider. The kernel itself, as well as
1032  * every kext, has symbol table/nlist info that has historically been preserved
1033  * for dtrace's use. This allowed dtrace to be lazy about allocating fbt/sdt probes,
1034  * at the expense of keeping the symbol info in the kernel permanently.
1035  *
1036  * Starting in 10.7+, fbt probes may be created from userspace, in the same
1037  * fashion as pid probes. The kernel allows dtrace "first right of refusal"
1038  * whenever symbol data becomes available (such as a kext load). If dtrace is
1039  * active, it will immediately read/copy the needed data, and then the kernel
1040  * may free it. If dtrace is not active, it returns immediately, having done
1041  * no work or allocations, and the symbol data is freed. Should dtrace need
1042  * this data later, it is expected that the userspace client will push the
1043  * data into the kernel via ioctl calls.
1044  *
1045  * The kernel symbol modes are used to control what dtrace does with symbol data:
1046  *
1047  * DTRACE_KERNEL_SYMBOLS_NEVER                  Effectively disables fbt/sdt
1048  * DTRACE_KERNEL_SYMBOLS_FROM_KERNEL            Immediately read/copy symbol data
1049  * DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE         Wait for symbols from userspace
1050  * DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL     Immediately read/copy symbol data
1051  *
1052  * It is legal to transition between DTRACE_KERNEL_SYMBOLS_FROM_KERNEL and
1053  * DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE. The DTRACE_KERNEL_SYMBOLS_NEVER and
1054  * DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL are permanent modes, intended to
1055  * disable fbt probes entirely, or prevent any symbols being loaded from
1056  * userspace.
1057 *
1058  * The kernel symbol mode is kept in dtrace_kernel_symbol_mode, which is protected
1059  * by the dtrace_lock.
1060  */
1061
1062 #define DTRACE_KERNEL_SYMBOLS_NEVER                     0
1063 #define DTRACE_KERNEL_SYMBOLS_FROM_KERNEL               1
1064 #define DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE            2
1065 #define DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL        3
1066
1067
1068 /*
1069  * DTrace Helper Implementation
1070  *
1071  * A description of the helper architecture may be found in <sys/dtrace.h>.
1072  * Each process contains a pointer to its helpers in its p_dtrace_helpers
1073  * member.  This is a pointer to a dtrace_helpers structure, which contains an
1074  * array of pointers to dtrace_helper structures, helper variable state (shared
1075  * among a process's helpers) and a generation count.  (The generation count is
1076  * used to provide an identifier when a helper is added so that it may be
1077  * subsequently removed.)  The dtrace_helper structure is self-explanatory,
1078  * containing pointers to the objects needed to execute the helper.  Note that
1079  * helpers are _duplicated_ across fork(2), and destroyed on exec(2).  No more
1080  * than dtrace_helpers_max are allowed per-process.
1081  */
1082 #define DTRACE_HELPER_ACTION_USTACK     0
1083 #define DTRACE_NHELPER_ACTIONS          1
1084
1085 typedef struct dtrace_helper_action {
1086         int dtha_generation;                    /* helper action generation */
1087         int dtha_nactions;                      /* number of actions */
1088         dtrace_difo_t *dtha_predicate;          /* helper action predicate */
1089         dtrace_difo_t **dtha_actions;           /* array of actions */
1090         struct dtrace_helper_action *dtha_next; /* next helper action */
1091 } dtrace_helper_action_t;
1092
1093 typedef struct dtrace_helper_provider {
1094         int dthp_generation;                    /* helper provider generation */
1095         uint32_t dthp_ref;                      /* reference count */
1096         dof_helper_t dthp_prov;                 /* DOF w/ provider and probes */
1097 } dtrace_helper_provider_t;
1098
1099 typedef struct dtrace_helpers {
1100         dtrace_helper_action_t **dthps_actions; /* array of helper actions */
1101         dtrace_vstate_t dthps_vstate;           /* helper action var. state */
1102         dtrace_helper_provider_t **dthps_provs; /* array of providers */
1103         uint_t dthps_nprovs;                    /* count of providers */
1104         uint_t dthps_maxprovs;                  /* provider array size */
1105         int dthps_generation;                   /* current generation */
1106         pid_t dthps_pid;                        /* pid of associated proc */
1107         int dthps_deferred;                     /* helper in deferred list */
1108         struct dtrace_helpers *dthps_next;      /* next pointer */
1109         struct dtrace_helpers *dthps_prev;      /* prev pointer */
1110 } dtrace_helpers_t;
1111
1112 /*
1113  * DTrace Helper Action Tracing
1114  *
1115  * Debugging helper actions can be arduous.  To ease the development and
1116  * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
1117  * framework: helper tracing.  If dtrace_helptrace_enabled is non-zero (which
1118  * it is by default on DEBUG kernels), all helper activity will be traced to a
1119  * global, in-kernel ring buffer.  Each entry includes a pointer to the specific
1120  * helper, the location within the helper, and a trace of all local variables.
1121  * The ring buffer may be displayed in a human-readable format with the
1122  * ::dtrace_helptrace mdb(1) dcmd.
1123  */
1124 #define DTRACE_HELPTRACE_NEXT   (-1)
1125 #define DTRACE_HELPTRACE_DONE   (-2)
1126 #define DTRACE_HELPTRACE_ERR    (-3)
1127
1128 typedef struct dtrace_helptrace {
1129         dtrace_helper_action_t  *dtht_helper;   /* helper action */
1130         int dtht_where;                         /* where in helper action */
1131         int dtht_nlocals;                       /* number of locals */
1132         int dtht_fault;                         /* type of fault (if any) */
1133         int dtht_fltoffs;                       /* DIF offset */
1134         uint64_t dtht_illval;                   /* faulting value */
1135         uint64_t dtht_locals[1];                /* local variables */
1136 } dtrace_helptrace_t;
1137
1138 /*
1139  * DTrace Credentials
1140  *
1141  * In probe context, we have limited flexibility to examine the credentials
1142  * of the DTrace consumer that created a particular enabling.  We use
1143  * the Least Privilege interfaces to cache the consumer's cred pointer and
1144  * some facts about that credential in a dtrace_cred_t structure. These
1145  * can limit the consumer's breadth of visibility and what actions the
1146  * consumer may take.
1147  */
1148 #define DTRACE_CRV_ALLPROC              0x01
1149 #define DTRACE_CRV_KERNEL               0x02
1150 #define DTRACE_CRV_ALLZONE              0x04
1151
1152 #define DTRACE_CRV_ALL          (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \
1153         DTRACE_CRV_ALLZONE)
1154
1155 #define DTRACE_CRA_PROC                         0x0001
1156 #define DTRACE_CRA_PROC_CONTROL                 0x0002
1157 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER     0x0004
1158 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE     0x0008
1159 #define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG     0x0010
1160 #define DTRACE_CRA_KERNEL                       0x0020
1161 #define DTRACE_CRA_KERNEL_DESTRUCTIVE           0x0040
1162
1163 #define DTRACE_CRA_ALL          (DTRACE_CRA_PROC | \
1164         DTRACE_CRA_PROC_CONTROL | \
1165         DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \
1166         DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \
1167         DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \
1168         DTRACE_CRA_KERNEL | \
1169         DTRACE_CRA_KERNEL_DESTRUCTIVE)
1170
1171 typedef struct dtrace_cred {
1172         cred_t                  *dcr_cred;
1173         uint8_t                 dcr_destructive;
1174         uint8_t                 dcr_visible;
1175         uint16_t                dcr_action;
1176 } dtrace_cred_t;
1177
1178 /*
1179  * DTrace Consumer State
1180  *
1181  * Each DTrace consumer has an associated dtrace_state structure that contains
1182  * its in-kernel DTrace state -- including options, credentials, statistics and
1183  * pointers to ECBs, buffers, speculations and formats.  A dtrace_state
1184  * structure is also allocated for anonymous enablings.  When anonymous state
1185  * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
1186  * dtrace_state structure.
1187  */
1188 struct dtrace_state {
1189         dev_t dts_dev;                          /* device */
1190         int dts_necbs;                          /* total number of ECBs */
1191         dtrace_ecb_t **dts_ecbs;                /* array of ECBs */
1192         dtrace_epid_t dts_epid;                 /* next EPID to allocate */
1193         size_t dts_needed;                      /* greatest needed space */
1194         struct dtrace_state *dts_anon;          /* anon. state, if grabbed */
1195         dtrace_activity_t dts_activity;         /* current activity */
1196         dtrace_vstate_t dts_vstate;             /* variable state */
1197         dtrace_buffer_t *dts_buffer;            /* principal buffer */
1198         dtrace_buffer_t *dts_aggbuffer;         /* aggregation buffer */
1199         dtrace_speculation_t *dts_speculations; /* speculation array */
1200         int dts_nspeculations;                  /* number of speculations */
1201         int dts_naggregations;                  /* number of aggregations */
1202         dtrace_aggregation_t **dts_aggregations; /* aggregation array */
1203         vmem_t *dts_aggid_arena;                /* arena for aggregation IDs */
1204         uint64_t dts_errors;                    /* total number of errors */
1205         uint32_t dts_speculations_busy;         /* number of spec. busy */
1206         uint32_t dts_speculations_unavail;      /* number of spec unavail */
1207         uint32_t dts_stkstroverflows;           /* stack string tab overflows */
1208         uint32_t dts_dblerrors;                 /* errors in ERROR probes */
1209         uint32_t dts_reserve;                   /* space reserved for END */
1210         hrtime_t dts_laststatus;                /* time of last status */
1211         cyclic_id_t dts_cleaner;                /* cleaning cyclic */
1212         cyclic_id_t dts_deadman;                /* deadman cyclic */
1213         hrtime_t dts_alive;                     /* time last alive */
1214         char dts_speculates;                    /* boolean: has speculations */
1215         char dts_destructive;                   /* boolean: has dest. actions */
1216         int dts_nformats;                       /* number of formats */
1217         char **dts_formats;                     /* format string array */
1218         dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */
1219         dtrace_cred_t dts_cred;                 /* credentials */
1220         size_t dts_nretained;                   /* number of retained enabs */
1221         uint64_t dts_arg_error_illval;
1222 };
1223
1224 struct dtrace_provider {
1225         dtrace_pattr_t dtpv_attr;               /* provider attributes */
1226         dtrace_ppriv_t dtpv_priv;               /* provider privileges */
1227         dtrace_pops_t dtpv_pops;                /* provider operations */
1228         char *dtpv_name;                        /* provider name */
1229         void *dtpv_arg;                         /* provider argument */
1230         uint_t dtpv_defunct;                    /* boolean: defunct provider */
1231         struct dtrace_provider *dtpv_next;      /* next provider */
1232         uint64_t dtpv_probe_count;              /* number of associated probes */
1233         uint64_t dtpv_ecb_count;                /* number of associated enabled ECBs */
1234 };
1235
1236 struct dtrace_meta {
1237         dtrace_mops_t dtm_mops;                 /* meta provider operations */
1238         char *dtm_name;                         /* meta provider name */
1239         void *dtm_arg;                          /* meta provider user arg */
1240         uint64_t dtm_count;                     /* number of associated providers */
1241 };
1242
1243 /*
1244  * DTrace Enablings
1245  *
1246  * A dtrace_enabling structure is used to track a collection of ECB
1247  * descriptions -- before they have been turned into actual ECBs.  This is
1248  * created as a result of DOF processing, and is generally used to generate
1249  * ECBs immediately thereafter.  However, enablings are also generally
1250  * retained should the probes they describe be created at a later time; as
1251  * each new module or provider registers with the framework, the retained
1252  * enablings are reevaluated, with any new match resulting in new ECBs.  To
1253  * prevent probes from being matched more than once, the enabling tracks the
1254  * last probe generation matched, and only matches probes from subsequent
1255  * generations.
1256  */
1257 typedef struct dtrace_enabling {
1258         dtrace_ecbdesc_t **dten_desc;           /* all ECB descriptions */
1259         int dten_ndesc;                         /* number of ECB descriptions */
1260         int dten_maxdesc;                       /* size of ECB array */
1261         dtrace_vstate_t *dten_vstate;           /* associated variable state */
1262         dtrace_genid_t dten_probegen;           /* matched probe generation */
1263         dtrace_ecbdesc_t *dten_current;         /* current ECB description */
1264         int dten_error;                         /* current error value */
1265         int dten_primed;                        /* boolean: set if primed */
1266         struct dtrace_enabling *dten_prev;      /* previous enabling */
1267         struct dtrace_enabling *dten_next;      /* next enabling */
1268 } dtrace_enabling_t;
1269
1270 /*
1271  * DTrace Anonymous Enablings
1272  *
1273  * Anonymous enablings are DTrace enablings that are not associated with a
1274  * controlling process, but rather derive their enabling from DOF stored as
1275  * properties in the dtrace.conf file.  If there is an anonymous enabling, a
1276  * DTrace consumer state and enabling are created on attach.  The state may be
1277  * subsequently grabbed by the first consumer specifying the "grabanon"
1278  * option.  As long as an anonymous DTrace enabling exists, dtrace(7D) will
1279  * refuse to unload.
1280  */
1281 typedef struct dtrace_anon {
1282         dtrace_state_t *dta_state;              /* DTrace consumer state */
1283         dtrace_enabling_t *dta_enabling;        /* pointer to enabling */
1284         processorid_t dta_beganon;              /* which CPU BEGIN ran on */
1285 } dtrace_anon_t;
1286
1287 /*
1288  * DTrace Error Debugging
1289  */
1290 #if DEBUG
1291 #define DTRACE_ERRDEBUG
1292 #endif
1293
1294 #ifdef DTRACE_ERRDEBUG
1295
1296 typedef struct dtrace_errhash {
1297         const char      *dter_msg;      /* error message */
1298         int             dter_count;     /* number of times seen */
1299 } dtrace_errhash_t;
1300
1301 #define DTRACE_ERRHASHSZ        256     /* must be > number of err msgs */
1302
1303 #endif  /* DTRACE_ERRDEBUG */
1304
1305 /*
1306  * DTrace Toxic Ranges
1307  *
1308  * DTrace supports safe loads from probe context; if the address turns out to
1309  * be invalid, a bit will be set by the kernel indicating that DTrace
1310  * encountered a memory error, and DTrace will propagate the error to the user
1311  * accordingly.  However, there may exist some regions of memory in which an
1312  * arbitrary load can change system state, and from which it is impossible to
1313  * recover from such a load after it has been attempted.  Examples of this may
1314  * include memory in which programmable I/O registers are mapped (for which a
1315  * read may have some implications for the device) or (in the specific case of
1316  * UltraSPARC-I and -II) the virtual address hole.  The platform is required
1317  * to make DTrace aware of these toxic ranges; DTrace will then check that
1318  * target addresses are not in a toxic range before attempting to issue a
1319  * safe load.
1320  */
1321 typedef struct dtrace_toxrange {
1322         uintptr_t       dtt_base;               /* base of toxic range */
1323         uintptr_t       dtt_limit;              /* limit of toxic range */
1324 } dtrace_toxrange_t;
1325
1326 extern uint64_t dtrace_getarg(int, int);
1327 extern int dtrace_getipl(void);
1328 extern uintptr_t dtrace_caller(int);
1329 extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
1330 extern void *dtrace_casptr(void *, void *, void *);
1331 extern void dtrace_copyin(user_addr_t, uintptr_t, size_t, volatile uint16_t *);
1332 extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t, volatile uint16_t *);
1333 extern void dtrace_copyout(uintptr_t, user_addr_t, size_t, volatile uint16_t *);
1334 extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t, volatile uint16_t *);
1335 extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *);
1336 extern uint64_t dtrace_getreg(struct regs *, uint_t);
1337 extern int dtrace_getstackdepth(int);
1338 extern void dtrace_getupcstack(uint64_t *, int);
1339 extern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
1340 extern int dtrace_getustackdepth(void);
1341 extern uintptr_t dtrace_fulword(void *);
1342 extern uint8_t dtrace_fuword8(user_addr_t);
1343 extern uint16_t dtrace_fuword16(user_addr_t);
1344 extern uint32_t dtrace_fuword32(user_addr_t);
1345 extern uint64_t dtrace_fuword64(user_addr_t);
1346 extern int dtrace_proc_waitfor(dtrace_procdesc_t*);
1347 extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
1348     int, uint64_t);
1349 extern int dtrace_assfail(const char *, const char *, int);
1350 extern int dtrace_attached(void);
1351 extern hrtime_t dtrace_gethrestime(void);
1352 extern void dtrace_isa_init(void);
1353
1354 extern void dtrace_copy(uintptr_t, uintptr_t, size_t);
1355 extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1356
1357 /*
1358  * DTrace restriction checks
1359  */
1360 extern boolean_t dtrace_is_restricted(void);
1361 extern boolean_t dtrace_can_attach_to_proc(proc_t);
1362
1363 /*
1364  * DTrace Assertions
1365  *
1366  * DTrace calls ASSERT from probe context.  To assure that a failed ASSERT
1367  * does not induce a markedly more catastrophic failure (e.g., one from which
1368  * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
1369  * may safely be called from probe context.  This header file must thus be
1370  * included by any DTrace component that calls ASSERT from probe context, and
1371  * _only_ by those components.  (The only exception to this is kernel
1372  * debugging infrastructure at user-level that doesn't depend on calling
1373  * ASSERT.)
1374  */
1375 #undef ASSERT
1376 #if DEBUG
1377 #define ASSERT(EX)      ((void)((EX) || \
1378                         dtrace_assfail(#EX, __FILE__, __LINE__)))
1379 #else
1380 #define ASSERT(X)       ((void)0)
1381 #endif
1382
1383 #ifdef  __cplusplus
1384 }
1385 #endif
1386
1387 #endif /* _SYS_DTRACE_IMPL_H */
1388