4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
27 #ifndef _SYS_DTRACE_IMPL_H
28 #define _SYS_DTRACE_IMPL_H
30 /* #pragma ident "@(#)dtrace_impl.h 1.23 07/02/16 SMI" */
37 * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
39 * Note: The contents of this file are private to the implementation of the
40 * Solaris system and DTrace subsystem and are subject to change at any time
41 * without notice. Applications and drivers using these interfaces will fail
42 * to run on future releases. These interfaces should not be used for any
43 * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
44 * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
47 #include <sys/dtrace.h>
50 * DTrace Implementation Constants and Typedefs
52 #define DTRACE_MAXPROPLEN 128
53 #define DTRACE_DYNVAR_CHUNKSIZE 256
57 struct dtrace_predicate
;
59 struct dtrace_provider
;
62 typedef struct dtrace_probe dtrace_probe_t
;
63 typedef struct dtrace_ecb dtrace_ecb_t
;
64 typedef struct dtrace_predicate dtrace_predicate_t
;
65 typedef struct dtrace_action dtrace_action_t
;
66 typedef struct dtrace_provider dtrace_provider_t
;
67 typedef struct dtrace_meta dtrace_meta_t
;
68 typedef struct dtrace_state dtrace_state_t
;
69 typedef uint32_t dtrace_optid_t
;
70 typedef uint32_t dtrace_specid_t
;
71 typedef uint64_t dtrace_genid_t
;
76 * The probe is the fundamental unit of the DTrace architecture. Probes are
77 * created by DTrace providers, and managed by the DTrace framework. A probe
78 * is identified by a unique <provider, module, function, name> tuple, and has
79 * a unique probe identifier assigned to it. (Some probes are not associated
80 * with a specific point in text; these are called _unanchored probes_ and have
81 * no module or function associated with them.) Probes are represented as a
82 * dtrace_probe structure. To allow quick lookups based on each element of the
83 * probe tuple, probes are hashed by each of provider, module, function and
84 * name. (If a lookup is performed based on a regular expression, a
85 * dtrace_probekey is prepared, and a linear search is performed.) Each probe
86 * is additionally pointed to by a linear array indexed by its identifier. The
87 * identifier is the provider's mechanism for indicating to the DTrace
88 * framework that a probe has fired: the identifier is passed as the first
89 * argument to dtrace_probe(), where it is then mapped into the corresponding
90 * dtrace_probe structure. From the dtrace_probe structure, dtrace_probe() can
91 * iterate over the probe's list of enabling control blocks; see "DTrace
92 * Enabling Control Blocks", below.)
95 dtrace_id_t dtpr_id
; /* probe identifier */
96 dtrace_ecb_t
*dtpr_ecb
; /* ECB list; see below */
97 dtrace_ecb_t
*dtpr_ecb_last
; /* last ECB in list */
98 void *dtpr_arg
; /* provider argument */
99 dtrace_cacheid_t dtpr_predcache
; /* predicate cache ID */
100 int dtpr_aframes
; /* artificial frames */
101 dtrace_provider_t
*dtpr_provider
; /* pointer to provider */
102 char *dtpr_mod
; /* probe's module name */
103 char *dtpr_func
; /* probe's function name */
104 char *dtpr_name
; /* probe's name */
105 dtrace_probe_t
*dtpr_nextmod
; /* next in module hash */
106 dtrace_probe_t
*dtpr_prevmod
; /* previous in module hash */
107 dtrace_probe_t
*dtpr_nextfunc
; /* next in function hash */
108 dtrace_probe_t
*dtpr_prevfunc
; /* previous in function hash */
109 dtrace_probe_t
*dtpr_nextname
; /* next in name hash */
110 dtrace_probe_t
*dtpr_prevname
; /* previous in name hash */
111 dtrace_genid_t dtpr_gen
; /* probe generation ID */
114 typedef int dtrace_probekey_f(const char *, const char *, int);
116 typedef struct dtrace_probekey
{
117 const char *dtpk_prov
; /* provider name to match */
118 dtrace_probekey_f
*dtpk_pmatch
; /* provider matching function */
119 const char *dtpk_mod
; /* module name to match */
120 dtrace_probekey_f
*dtpk_mmatch
; /* module matching function */
121 const char *dtpk_func
; /* func name to match */
122 dtrace_probekey_f
*dtpk_fmatch
; /* func matching function */
123 const char *dtpk_name
; /* name to match */
124 dtrace_probekey_f
*dtpk_nmatch
; /* name matching function */
125 dtrace_id_t dtpk_id
; /* identifier to match */
128 typedef struct dtrace_hashbucket
{
129 struct dtrace_hashbucket
*dthb_next
; /* next on hash chain */
130 dtrace_probe_t
*dthb_chain
; /* chain of probes */
131 int dthb_len
; /* number of probes here */
132 } dtrace_hashbucket_t
;
134 typedef struct dtrace_hash
{
135 dtrace_hashbucket_t
**dth_tab
; /* hash table */
136 int dth_size
; /* size of hash table */
137 int dth_mask
; /* mask to index into table */
138 int dth_nbuckets
; /* total number of buckets */
139 uintptr_t dth_nextoffs
; /* offset of next in probe */
140 uintptr_t dth_prevoffs
; /* offset of prev in probe */
141 uintptr_t dth_stroffs
; /* offset of str in probe */
145 * DTrace Enabling Control Blocks
147 * When a provider wishes to fire a probe, it calls into dtrace_probe(),
148 * passing the probe identifier as the first argument. As described above,
149 * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
150 * structure. This structure contains information about the probe, and a
151 * pointer to the list of Enabling Control Blocks (ECBs). Each ECB points to
152 * DTrace consumer state, and contains an optional predicate, and a list of
153 * actions. (Shown schematically below.) The ECB abstraction allows a single
154 * probe to be multiplexed across disjoint consumers, or across disjoint
155 * enablings of a single probe within one consumer.
157 * Enabling Control Block
159 * +------------------------+
160 * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
161 * | dtrace_state_t * ------+--------------> State associated with this ECB
162 * | dtrace_predicate_t * --+---------+
163 * | dtrace_action_t * -----+----+ |
164 * | dtrace_ecb_t * ---+ | | | Predicate (if any)
165 * +-------------------+----+ | | dtrace_predicate_t
166 * | | +---> +--------------------+
167 * | | | dtrace_difo_t * ---+----> DIFO
168 * | | +--------------------+
170 * Next ECB | | Action
171 * (if any) | | dtrace_action_t
172 * : +--> +-------------------+
173 * : | dtrace_actkind_t -+------> kind
174 * v | dtrace_difo_t * --+------> DIFO (if any)
175 * | dtrace_recdesc_t -+------> record descr.
176 * | dtrace_action_t * +------+
177 * +-------------------+ |
179 * +-------------------------------+ (if any)
183 * +--> +-------------------+
184 * | dtrace_actkind_t -+------> kind
185 * | dtrace_difo_t * --+------> DIFO (if any)
186 * | dtrace_action_t * +------+
187 * +-------------------+ |
189 * +-------------------------------+ (if any)
195 * dtrace_probe() iterates over the ECB list. If the ECB needs less space
196 * than is available in the principal buffer, the ECB is processed: if the
197 * predicate is non-NULL, the DIF object is executed. If the result is
198 * non-zero, the action list is processed, with each action being executed
199 * accordingly. When the action list has been completely executed, processing
200 * advances to the next ECB. processing advances to the next ECB. If the
201 * result is non-zero; For each ECB, it first determines the The ECB
202 * abstraction allows disjoint consumers to multiplex on single probes.
205 dtrace_epid_t dte_epid
; /* enabled probe ID */
206 uint32_t dte_alignment
; /* required alignment */
207 size_t dte_needed
; /* bytes needed */
208 size_t dte_size
; /* total size of payload */
209 dtrace_predicate_t
*dte_predicate
; /* predicate, if any */
210 dtrace_action_t
*dte_action
; /* actions, if any */
211 dtrace_ecb_t
*dte_next
; /* next ECB on probe */
212 dtrace_state_t
*dte_state
; /* pointer to state */
213 uint32_t dte_cond
; /* security condition */
214 dtrace_probe_t
*dte_probe
; /* pointer to probe */
215 dtrace_action_t
*dte_action_last
; /* last action on ECB */
216 uint64_t dte_uarg
; /* library argument */
219 struct dtrace_predicate
{
220 dtrace_difo_t
*dtp_difo
; /* DIF object */
221 dtrace_cacheid_t dtp_cacheid
; /* cache identifier */
222 int dtp_refcnt
; /* reference count */
225 struct dtrace_action
{
226 dtrace_actkind_t dta_kind
; /* kind of action */
227 uint16_t dta_intuple
; /* boolean: in aggregation */
228 uint32_t dta_refcnt
; /* reference count */
229 dtrace_difo_t
*dta_difo
; /* pointer to DIFO */
230 dtrace_recdesc_t dta_rec
; /* record description */
231 dtrace_action_t
*dta_prev
; /* previous action */
232 dtrace_action_t
*dta_next
; /* next action */
235 typedef struct dtrace_aggregation
{
236 dtrace_action_t dtag_action
; /* action; must be first */
237 dtrace_aggid_t dtag_id
; /* identifier */
238 dtrace_ecb_t
*dtag_ecb
; /* corresponding ECB */
239 dtrace_action_t
*dtag_first
; /* first action in tuple */
240 uint32_t dtag_base
; /* base of aggregation */
241 uint8_t dtag_hasarg
; /* boolean: has argument */
242 uint64_t dtag_initial
; /* initial value */
243 void (*dtag_aggregate
)(uint64_t *, uint64_t, uint64_t);
244 } dtrace_aggregation_t
;
249 * Principal buffers, aggregation buffers, and speculative buffers are all
250 * managed with the dtrace_buffer structure. By default, this structure
251 * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
252 * active and passive buffers, respectively. For speculative buffers,
253 * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
254 * to a scratch buffer. For all buffer types, the dtrace_buffer structure is
255 * always allocated on a per-CPU basis; a single dtrace_buffer structure is
256 * never shared among CPUs. (That is, there is never true sharing of the
257 * dtrace_buffer structure; to prevent false sharing of the structure, it must
258 * always be aligned to the coherence granularity -- generally 64 bytes.)
260 * One of the critical design decisions of DTrace is that a given ECB always
261 * stores the same quantity and type of data. This is done to assure that the
262 * only metadata required for an ECB's traced data is the EPID. That is, from
263 * the EPID, the consumer can determine the data layout. (The data buffer
264 * layout is shown schematically below.) By assuring that one can determine
265 * data layout from the EPID, the metadata stream can be separated from the
266 * data stream -- simplifying the data stream enormously.
268 * base of data buffer ---> +------+--------------------+------+
269 * | EPID | data | EPID |
270 * +------+--------+------+----+------+
271 * | data | EPID | data |
272 * +---------------+------+-----------+
274 * +------+--------------------+------+
276 * +------+--------------------+ |
286 * limit of data buffer ---> +----------------------------------+
288 * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
289 * principal buffer (both scratch and payload) exceed the available space. If
290 * the ECB's needs exceed available space (and if the principal buffer policy
291 * is the default "switch" policy), the ECB is dropped, the buffer's drop count
292 * is incremented, and processing advances to the next ECB. If the ECB's needs
293 * can be met with the available space, the ECB is processed, but the offset in
294 * the principal buffer is only advanced if the ECB completes processing
297 * When a buffer is to be switched (either because the buffer is the principal
298 * buffer with a "switch" policy or because it is an aggregation buffer), a
299 * cross call is issued to the CPU associated with the buffer. In the cross
300 * call context, interrupts are disabled, and the active and the inactive
301 * buffers are atomically switched. This involves switching the data pointers,
302 * copying the various state fields (offset, drops, errors, etc.) into their
303 * inactive equivalents, and clearing the state fields. Because interrupts are
304 * disabled during this procedure, the switch is guaranteed to appear atomic to
307 * DTrace Ring Buffering
309 * To process a ring buffer correctly, one must know the oldest valid record.
310 * Processing starts at the oldest record in the buffer and continues until
311 * the end of the buffer is reached. Processing then resumes starting with
312 * the record stored at offset 0 in the buffer, and continues until the
313 * youngest record is processed. If trace records are of a fixed-length,
314 * determining the oldest record is trivial:
316 * - If the ring buffer has not wrapped, the oldest record is the record
317 * stored at offset 0.
319 * - If the ring buffer has wrapped, the oldest record is the record stored
320 * at the current offset.
322 * With variable length records, however, just knowing the current offset
323 * doesn't suffice for determining the oldest valid record: assuming that one
324 * allows for arbitrary data, one has no way of searching forward from the
325 * current offset to find the oldest valid record. (That is, one has no way
326 * of separating data from metadata.) It would be possible to simply refuse to
327 * process any data in the ring buffer between the current offset and the
328 * limit, but this leaves (potentially) an enormous amount of otherwise valid
331 * To effect ring buffering, we track two offsets in the buffer: the current
332 * offset and the _wrapped_ offset. If a request is made to reserve some
333 * amount of data, and the buffer has wrapped, the wrapped offset is
334 * incremented until the wrapped offset minus the current offset is greater
335 * than or equal to the reserve request. This is done by repeatedly looking
336 * up the ECB corresponding to the EPID at the current wrapped offset, and
337 * incrementing the wrapped offset by the size of the data payload
338 * corresponding to that ECB. If this offset is greater than or equal to the
339 * limit of the data buffer, the wrapped offset is set to 0. Thus, the
340 * current offset effectively "chases" the wrapped offset around the buffer.
343 * base of data buffer ---> +------+--------------------+------+
344 * | EPID | data | EPID |
345 * +------+--------+------+----+------+
346 * | data | EPID | data |
347 * +---------------+------+-----------+
349 * +------+---------------------------+
351 * current offset ---> +------+---------------------------+
353 * wrapped offset ---> +------+--------------------+------+
354 * | EPID | data | EPID |
355 * +------+--------+------+----+------+
356 * | data | EPID | data |
357 * +---------------+------+-----------+
360 * . ... valid data ... .
363 * +------+-------------+------+------+
364 * | EPID | data | EPID | data |
365 * +------+------------++------+------+
366 * | data, cont. | leftover |
367 * limit of data buffer ---> +-------------------+--------------+
369 * If the amount of requested buffer space exceeds the amount of space
370 * available between the current offset and the end of the buffer:
372 * (1) all words in the data buffer between the current offset and the limit
373 * of the data buffer (marked "leftover", above) are set to
376 * (2) the wrapped offset is set to zero
378 * (3) the iteration process described above occurs until the wrapped offset
379 * is greater than the amount of desired space.
381 * The wrapped offset is implemented by (re-)using the inactive offset.
382 * In a "switch" buffer policy, the inactive offset stores the offset in
383 * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
386 * DTrace Scratch Buffering
388 * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
389 * To accommodate such requests easily, scratch memory may be allocated in
390 * the buffer beyond the current offset plus the needed memory of the current
391 * ECB. If there isn't sufficient room in the buffer for the requested amount
392 * of scratch space, the allocation fails and an error is generated. Scratch
393 * memory is tracked in the dtrace_mstate_t and is automatically freed when
394 * the ECB ceases processing. Note that ring buffers cannot allocate their
395 * scratch from the principal buffer -- lest they needlessly overwrite older,
396 * valid data. Ring buffers therefore have their own dedicated scratch buffer
397 * from which scratch is allocated.
399 #define DTRACEBUF_RING 0x0001 /* bufpolicy set to "ring" */
400 #define DTRACEBUF_FILL 0x0002 /* bufpolicy set to "fill" */
401 #define DTRACEBUF_NOSWITCH 0x0004 /* do not switch buffer */
402 #define DTRACEBUF_WRAPPED 0x0008 /* ring buffer has wrapped */
403 #define DTRACEBUF_DROPPED 0x0010 /* drops occurred */
404 #define DTRACEBUF_ERROR 0x0020 /* errors occurred */
405 #define DTRACEBUF_FULL 0x0040 /* "fill" buffer is full */
406 #define DTRACEBUF_CONSUMED 0x0080 /* buffer has been consumed */
407 #define DTRACEBUF_INACTIVE 0x0100 /* buffer is not yet active */
409 typedef struct dtrace_buffer
{
410 uint64_t dtb_offset
; /* current offset in buffer */
411 uint64_t dtb_size
; /* size of buffer */
412 uint32_t dtb_flags
; /* flags */
413 uint32_t dtb_drops
; /* number of drops */
414 caddr_t dtb_tomax
; /* active buffer */
415 caddr_t dtb_xamot
; /* inactive buffer */
416 uint32_t dtb_xamot_flags
; /* inactive flags */
417 uint32_t dtb_xamot_drops
; /* drops in inactive buffer */
418 uint64_t dtb_xamot_offset
; /* offset in inactive buffer */
419 uint32_t dtb_errors
; /* number of errors */
420 uint32_t dtb_xamot_errors
; /* errors in inactive buffer */
427 * DTrace Aggregation Buffers
429 * Aggregation buffers use much of the same mechanism as described above
430 * ("DTrace Buffers"). However, because an aggregation is fundamentally a
431 * hash, there exists dynamic metadata associated with an aggregation buffer
432 * that is not associated with other kinds of buffers. This aggregation
433 * metadata is _only_ relevant for the in-kernel implementation of
434 * aggregations; it is not actually relevant to user-level consumers. To do
435 * this, we allocate dynamic aggregation data (hash keys and hash buckets)
436 * starting below the _limit_ of the buffer, and we allocate data from the
437 * _base_ of the buffer. When the aggregation buffer is copied out, _only_ the
438 * data is copied out; the metadata is simply discarded. Schematically,
439 * aggregation buffers look like:
441 * base of data buffer ---> +-------+------+-----------+-------+
442 * | aggid | key | value | aggid |
443 * +-------+------+-----------+-------+
445 * +-------+-------+-----+------------+
446 * | value | aggid | key | value |
447 * +-------+------++-----+------+-----+
448 * | aggid | key | value | |
449 * +-------+------+-------------+ |
459 * | || +------------+
461 * +---------------------+ |
463 * | (dtrace_aggkey structures) |
465 * +----------------------------------+
467 * | (dtrace_aggbuffer structure) |
469 * limit of data buffer ---> +----------------------------------+
472 * As implied above, just as we assure that ECBs always store a constant
473 * amount of data, we assure that a given aggregation -- identified by its
474 * aggregation ID -- always stores data of a constant quantity and type.
475 * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
478 * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
479 * aligned. (If this the structure changes such that this becomes false, an
480 * assertion will fail in dtrace_aggregate().)
482 typedef struct dtrace_aggkey
{
483 uint32_t dtak_hashval
; /* hash value */
484 uint32_t dtak_action
:4; /* action -- 4 bits */
485 uint32_t dtak_size
:28; /* size -- 28 bits */
486 caddr_t dtak_data
; /* data pointer */
487 struct dtrace_aggkey
*dtak_next
; /* next in hash chain */
490 typedef struct dtrace_aggbuffer
{
491 uintptr_t dtagb_hashsize
; /* number of buckets */
492 uintptr_t dtagb_free
; /* free list of keys */
493 dtrace_aggkey_t
**dtagb_hash
; /* hash table */
494 } dtrace_aggbuffer_t
;
497 * DTrace Speculations
499 * Speculations have a per-CPU buffer and a global state. Once a speculation
500 * buffer has been comitted or discarded, it cannot be reused until all CPUs
501 * have taken the same action (commit or discard) on their respective
502 * speculative buffer. However, because DTrace probes may execute in arbitrary
503 * context, other CPUs cannot simply be cross-called at probe firing time to
504 * perform the necessary commit or discard. The speculation states thus
505 * optimize for the case that a speculative buffer is only active on one CPU at
506 * the time of a commit() or discard() -- for if this is the case, other CPUs
507 * need not take action, and the speculation is immediately available for
508 * reuse. If the speculation is active on multiple CPUs, it must be
509 * asynchronously cleaned -- potentially leading to a higher rate of dirty
510 * speculative drops. The speculation states are as follows:
512 * DTRACESPEC_INACTIVE <= Initial state; inactive speculation
513 * DTRACESPEC_ACTIVE <= Allocated, but not yet speculatively traced to
514 * DTRACESPEC_ACTIVEONE <= Speculatively traced to on one CPU
515 * DTRACESPEC_ACTIVEMANY <= Speculatively traced to on more than one CPU
516 * DTRACESPEC_COMMITTING <= Currently being commited on one CPU
517 * DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
518 * DTRACESPEC_DISCARDING <= Currently being discarded on many CPUs
520 * The state transition diagram is as follows:
522 * +----------------------------------------------------------+
525 * | +-------------------| COMMITTING |<-----------------+ |
526 * | | +------------+ | |
527 * | | copied spec. ^ commit() on | | discard() on
528 * | | into principal | active CPU | | active CPU
531 * +----------+ +--------+ +-----------+
532 * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
533 * +----------+ speculation() +--------+ speculate() +-----------+
535 * | | | discard() | |
536 * | | asynchronously | discard() on | | speculate()
537 * | | cleaned V inactive CPU | | on inactive
538 * | | +------------+ | | CPU
539 * | +-------------------| DISCARDING |<-----------------+ |
541 * | asynchronously ^ |
542 * | copied spec. | discard() |
543 * | into principal +------------------------+ |
545 * +----------------+ commit() +------------+
546 * | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
547 * +----------------+ +------------+
549 typedef enum dtrace_speculation_state
{
550 DTRACESPEC_INACTIVE
= 0,
552 DTRACESPEC_ACTIVEONE
,
553 DTRACESPEC_ACTIVEMANY
,
554 DTRACESPEC_COMMITTING
,
555 DTRACESPEC_COMMITTINGMANY
,
556 DTRACESPEC_DISCARDING
557 } dtrace_speculation_state_t
;
559 typedef struct dtrace_speculation
{
560 dtrace_speculation_state_t dtsp_state
; /* current speculation state */
561 int dtsp_cleaning
; /* non-zero if being cleaned */
562 dtrace_buffer_t
*dtsp_buffer
; /* speculative buffer */
563 } dtrace_speculation_t
;
566 * DTrace Dynamic Variables
568 * The dynamic variable problem is obviously decomposed into two subproblems:
569 * allocating new dynamic storage, and freeing old dynamic storage. The
570 * presence of the second problem makes the first much more complicated -- or
571 * rather, the absence of the second renders the first trivial. This is the
572 * case with aggregations, for which there is effectively no deallocation of
573 * dynamic storage. (Or more accurately, all dynamic storage is deallocated
574 * when a snapshot is taken of the aggregation.) As DTrace dynamic variables
575 * allow for both dynamic allocation and dynamic deallocation, the
576 * implementation of dynamic variables is quite a bit more complicated than
577 * that of their aggregation kin.
579 * We observe that allocating new dynamic storage is tricky only because the
580 * size can vary -- the allocation problem is much easier if allocation sizes
581 * are uniform. We further observe that in D, the size of dynamic variables is
582 * actually _not_ dynamic -- dynamic variable sizes may be determined by static
583 * analysis of DIF text. (This is true even of putatively dynamically-sized
584 * objects like strings and stacks, the sizes of which are dictated by the
585 * "stringsize" and "stackframes" variables, respectively.) We exploit this by
586 * performing this analysis on all DIF before enabling any probes. For each
587 * dynamic load or store, we calculate the dynamically-allocated size plus the
588 * size of the dtrace_dynvar structure plus the storage required to key the
589 * data. For all DIF, we take the largest value and dub it the _chunksize_.
590 * We then divide dynamic memory into two parts: a hash table that is wide
591 * enough to have every chunk in its own bucket, and a larger region of equal
592 * chunksize units. Whenever we wish to dynamically allocate a variable, we
593 * always allocate a single chunk of memory. Depending on the uniformity of
594 * allocation, this will waste some amount of memory -- but it eliminates the
595 * non-determinism inherent in traditional heap fragmentation.
597 * Dynamic objects are allocated by storing a non-zero value to them; they are
598 * deallocated by storing a zero value to them. Dynamic variables are
599 * complicated enormously by being shared between CPUs. In particular,
600 * consider the following scenario:
603 * +---------------------------------+ +---------------------------------+
605 * | allocates dynamic object a[123] | | |
606 * | by storing the value 345 to it | | |
608 * | | | wishing to load from object |
609 * | | | a[123], performs lookup in |
610 * | | | dynamic variable space |
612 * | deallocates object a[123] by | | |
613 * | storing 0 to it | | |
615 * | allocates dynamic object b[567] | | performs load from a[123] |
616 * | by storing the value 789 to it | | |
620 * This is obviously a race in the D program, but there are nonetheless only
621 * two valid values for CPU B's load from a[123]: 345 or 0. Most importantly,
622 * CPU B may _not_ see the value 789 for a[123].
624 * There are essentially two ways to deal with this:
626 * (1) Explicitly spin-lock variables. That is, if CPU B wishes to load
627 * from a[123], it needs to lock a[123] and hold the lock for the
628 * duration that it wishes to manipulate it.
630 * (2) Avoid reusing freed chunks until it is known that no CPU is referring
633 * The implementation of (1) is rife with complexity, because it requires the
634 * user of a dynamic variable to explicitly decree when they are done using it.
635 * Were all variables by value, this perhaps wouldn't be debilitating -- but
636 * dynamic variables of non-scalar types are tracked by reference. That is, if
637 * a dynamic variable is, say, a string, and that variable is to be traced to,
638 * say, the principal buffer, the DIF emulation code returns to the main
639 * dtrace_probe() loop a pointer to the underlying storage, not the contents of
640 * the storage. Further, code calling on DIF emulation would have to be aware
641 * that the DIF emulation has returned a reference to a dynamic variable that
642 * has been potentially locked. The variable would have to be unlocked after
643 * the main dtrace_probe() loop is finished with the variable, and the main
644 * dtrace_probe() loop would have to be careful to not call any further DIF
645 * emulation while the variable is locked to avoid deadlock. More generally,
646 * if one were to implement (1), DIF emulation code dealing with dynamic
647 * variables could only deal with one dynamic variable at a time (lest deadlock
648 * result). To sum, (1) exports too much subtlety to the users of dynamic
649 * variables -- increasing maintenance burden and imposing serious constraints
650 * on future DTrace development.
652 * The implementation of (2) is also complex, but the complexity is more
653 * manageable. We need to be sure that when a variable is deallocated, it is
654 * not placed on a traditional free list, but rather on a _dirty_ list. Once a
655 * variable is on a dirty list, it cannot be found by CPUs performing a
656 * subsequent lookup of the variable -- but it may still be in use by other
657 * CPUs. To assure that all CPUs that may be seeing the old variable have
658 * cleared out of probe context, a dtrace_sync() can be issued. Once the
659 * dtrace_sync() has completed, it can be known that all CPUs are done
660 * manipulating the dynamic variable -- the dirty list can be atomically
661 * appended to the free list. Unfortunately, there's a slight hiccup in this
662 * mechanism: dtrace_sync() may not be issued from probe context. The
663 * dtrace_sync() must be therefore issued asynchronously from non-probe
664 * context. For this we rely on the DTrace cleaner, a cyclic that runs at the
665 * "cleanrate" frequency. To ease this implementation, we define several chunk
668 * - Dirty. Deallocated chunks, not yet cleaned. Not available.
670 * - Rinsing. Formerly dirty chunks that are currently being asynchronously
671 * cleaned. Not available, but will be shortly. Dynamic variable
672 * allocation may not spin or block for availability, however.
674 * - Clean. Clean chunks, ready for allocation -- but not on the free list.
676 * - Free. Available for allocation.
678 * Moreover, to avoid absurd contention, _each_ of these lists is implemented
679 * on a per-CPU basis. This is only for performance, not correctness; chunks
680 * may be allocated from another CPU's free list. The algorithm for allocation
683 * (1) Attempt to atomically allocate from current CPU's free list. If list
684 * is non-empty and allocation is successful, allocation is complete.
686 * (2) If the clean list is non-empty, atomically move it to the free list,
689 * (3) If the dynamic variable space is in the CLEAN state, look for free
690 * and clean lists on other CPUs by setting the current CPU to the next
691 * CPU, and reattempting (1). If the next CPU is the current CPU (that
692 * is, if all CPUs have been checked), atomically switch the state of
693 * the dynamic variable space based on the following:
695 * - If no free chunks were found and no dirty chunks were found,
696 * atomically set the state to EMPTY.
698 * - If dirty chunks were found, atomically set the state to DIRTY.
700 * - If rinsing chunks were found, atomically set the state to RINSING.
702 * (4) Based on state of dynamic variable space state, increment appropriate
703 * counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
704 * dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
705 * RINSING state). Fail the allocation.
707 * The cleaning cyclic operates with the following algorithm: for all CPUs
708 * with a non-empty dirty list, atomically move the dirty list to the rinsing
709 * list. Perform a dtrace_sync(). For all CPUs with a non-empty rinsing list,
710 * atomically move the rinsing list to the clean list. Perform another
711 * dtrace_sync(). By this point, all CPUs have seen the new clean list; the
712 * state of the dynamic variable space can be restored to CLEAN.
714 * There exist two final races that merit explanation. The first is a simple
718 * +---------------------------------+ +---------------------------------+
720 * | allocates dynamic object a[123] | | allocates dynamic object a[123] |
721 * | by storing the value 345 to it | | by storing the value 567 to it |
726 * Again, this is a race in the D program. It can be resolved by having a[123]
727 * hold the value 345 or a[123] hold the value 567 -- but it must be true that
728 * a[123] have only _one_ of these values. (That is, the racing CPUs may not
729 * put the same element twice on the same hash chain.) This is resolved
730 * simply: before the allocation is undertaken, the start of the new chunk's
731 * hash chain is noted. Later, after the allocation is complete, the hash
732 * chain is atomically switched to point to the new element. If this fails
733 * (because of either concurrent allocations or an allocation concurrent with a
734 * deletion), the newly allocated chunk is deallocated to the dirty list, and
735 * the whole process of looking up (and potentially allocating) the dynamic
736 * variable is reattempted.
738 * The final race is a simple deallocation race:
741 * +---------------------------------+ +---------------------------------+
743 * | deallocates dynamic object | | deallocates dynamic object |
744 * | a[123] by storing the value 0 | | a[123] by storing the value 0 |
745 * | to it | | to it |
750 * Once again, this is a race in the D program, but it is one that we must
751 * handle without corrupting the underlying data structures. Because
752 * deallocations require the deletion of a chunk from the middle of a hash
753 * chain, we cannot use a single-word atomic operation to remove it. For this,
754 * we add a spin lock to the hash buckets that is _only_ used for deallocations
755 * (allocation races are handled as above). Further, this spin lock is _only_
756 * held for the duration of the delete; before control is returned to the DIF
757 * emulation code, the hash bucket is unlocked.
759 typedef struct dtrace_key
{
760 uint64_t dttk_value
; /* data value or data pointer */
761 uint64_t dttk_size
; /* 0 if by-val, >0 if by-ref */
764 typedef struct dtrace_tuple
{
765 uint32_t dtt_nkeys
; /* number of keys in tuple */
766 uint32_t dtt_pad
; /* padding */
767 dtrace_key_t dtt_key
[1]; /* array of tuple keys */
770 typedef struct dtrace_dynvar
{
771 uint64_t dtdv_hashval
; /* hash value -- 0 if free */
772 struct dtrace_dynvar
*dtdv_next
; /* next on list or hash chain */
773 void *dtdv_data
; /* pointer to data */
774 dtrace_tuple_t dtdv_tuple
; /* tuple key */
777 typedef enum dtrace_dynvar_op
{
779 DTRACE_DYNVAR_NOALLOC
,
780 DTRACE_DYNVAR_DEALLOC
781 } dtrace_dynvar_op_t
;
783 typedef struct dtrace_dynhash
{
784 dtrace_dynvar_t
*dtdh_chain
; /* hash chain for this bucket */
785 uintptr_t dtdh_lock
; /* deallocation lock */
787 uintptr_t dtdh_pad
[6]; /* pad to avoid false sharing */
789 uintptr_t dtdh_pad
[14]; /* pad to avoid false sharing */
793 typedef struct dtrace_dstate_percpu
{
794 dtrace_dynvar_t
*dtdsc_free
; /* free list for this CPU */
795 dtrace_dynvar_t
*dtdsc_dirty
; /* dirty list for this CPU */
796 dtrace_dynvar_t
*dtdsc_rinsing
; /* rinsing list for this CPU */
797 dtrace_dynvar_t
*dtdsc_clean
; /* clean list for this CPU */
798 uint64_t dtdsc_drops
; /* number of capacity drops */
799 uint64_t dtdsc_dirty_drops
; /* number of dirty drops */
800 uint64_t dtdsc_rinsing_drops
; /* number of rinsing drops */
802 uint64_t dtdsc_pad
; /* pad to avoid false sharing */
804 uint64_t dtdsc_pad
[2]; /* pad to avoid false sharing */
806 } dtrace_dstate_percpu_t
;
808 typedef enum dtrace_dstate_state
{
809 DTRACE_DSTATE_CLEAN
= 0,
812 DTRACE_DSTATE_RINSING
813 } dtrace_dstate_state_t
;
815 typedef struct dtrace_dstate
{
816 void *dtds_base
; /* base of dynamic var. space */
817 size_t dtds_size
; /* size of dynamic var. space */
818 size_t dtds_hashsize
; /* number of buckets in hash */
819 size_t dtds_chunksize
; /* size of each chunk */
820 dtrace_dynhash_t
*dtds_hash
; /* pointer to hash table */
821 dtrace_dstate_state_t dtds_state
; /* current dynamic var. state */
822 dtrace_dstate_percpu_t
*dtds_percpu
; /* per-CPU dyn. var. state */
826 * DTrace Variable State
828 * The DTrace variable state tracks user-defined variables in its dtrace_vstate
829 * structure. Each DTrace consumer has exactly one dtrace_vstate structure,
830 * but some dtrace_vstate structures may exist without a corresponding DTrace
831 * consumer (see "DTrace Helpers", below). As described in <sys/dtrace.h>,
832 * user-defined variables can have one of three scopes:
834 * DIFV_SCOPE_GLOBAL => global scope
835 * DIFV_SCOPE_THREAD => thread-local scope (i.e. "self->" variables)
836 * DIFV_SCOPE_LOCAL => clause-local scope (i.e. "this->" variables)
838 * The variable state tracks variables by both their scope and their allocation
841 * - The dtvs_globals and dtvs_locals members each point to an array of
842 * dtrace_statvar structures. These structures contain both the variable
843 * metadata (dtrace_difv structures) and the underlying storage for all
844 * statically allocated variables, including statically allocated
845 * DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
847 * - The dtvs_tlocals member points to an array of dtrace_difv structures for
848 * DIFV_SCOPE_THREAD variables. As such, this array tracks _only_ the
849 * variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
850 * is allocated out of the dynamic variable space.
852 * - The dtvs_dynvars member is the dynamic variable state associated with the
853 * variable state. The dynamic variable state (described in "DTrace Dynamic
854 * Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
855 * dynamically-allocated DIFV_SCOPE_GLOBAL variables.
857 typedef struct dtrace_statvar
{
858 uint64_t dtsv_data
; /* data or pointer to it */
859 size_t dtsv_size
; /* size of pointed-to data */
860 int dtsv_refcnt
; /* reference count */
861 dtrace_difv_t dtsv_var
; /* variable metadata */
864 typedef struct dtrace_vstate
{
865 dtrace_state_t
*dtvs_state
; /* back pointer to state */
866 dtrace_statvar_t
**dtvs_globals
; /* statically-allocated glbls */
867 int dtvs_nglobals
; /* number of globals */
868 dtrace_difv_t
*dtvs_tlocals
; /* thread-local metadata */
869 int dtvs_ntlocals
; /* number of thread-locals */
870 dtrace_statvar_t
**dtvs_locals
; /* clause-local data */
871 int dtvs_nlocals
; /* number of clause-locals */
872 dtrace_dstate_t dtvs_dynvars
; /* dynamic variable state */
876 * DTrace Machine State
878 * In the process of processing a fired probe, DTrace needs to track and/or
879 * cache some per-CPU state associated with that particular firing. This is
880 * state that is always discarded after the probe firing has completed, and
881 * much of it is not specific to any DTrace consumer, remaining valid across
882 * all ECBs. This state is tracked in the dtrace_mstate structure.
884 #define DTRACE_MSTATE_ARGS 0x00000001
885 #define DTRACE_MSTATE_PROBE 0x00000002
886 #define DTRACE_MSTATE_EPID 0x00000004
887 #define DTRACE_MSTATE_TIMESTAMP 0x00000008
888 #define DTRACE_MSTATE_STACKDEPTH 0x00000010
889 #define DTRACE_MSTATE_CALLER 0x00000020
890 #define DTRACE_MSTATE_IPL 0x00000040
891 #define DTRACE_MSTATE_FLTOFFS 0x00000080
892 #define DTRACE_MSTATE_WALLTIMESTAMP 0x00000100
893 #define DTRACE_MSTATE_USTACKDEPTH 0x00000200
894 #define DTRACE_MSTATE_UCALLER 0x00000400
896 typedef struct dtrace_mstate
{
897 uintptr_t dtms_scratch_base
; /* base of scratch space */
898 uintptr_t dtms_scratch_ptr
; /* current scratch pointer */
899 size_t dtms_scratch_size
; /* scratch size */
900 uint32_t dtms_present
; /* variables that are present */
901 uint64_t dtms_arg
[5]; /* cached arguments */
902 dtrace_epid_t dtms_epid
; /* current EPID */
903 uint64_t dtms_timestamp
; /* cached timestamp */
904 hrtime_t dtms_walltimestamp
; /* cached wall timestamp */
905 int dtms_stackdepth
; /* cached stackdepth */
906 int dtms_ustackdepth
; /* cached ustackdepth */
907 struct dtrace_probe
*dtms_probe
; /* current probe */
908 uintptr_t dtms_caller
; /* cached caller */
909 uint64_t dtms_ucaller
; /* cached user-level caller */
910 int dtms_ipl
; /* cached interrupt pri lev */
911 int dtms_fltoffs
; /* faulting DIFO offset */
912 uintptr_t dtms_strtok
; /* saved strtok() pointer */
913 uint32_t dtms_access
; /* memory access rights */
914 dtrace_difo_t
*dtms_difo
; /* current dif object */
917 #define DTRACE_COND_OWNER 0x1
918 #define DTRACE_COND_USERMODE 0x2
919 #define DTRACE_COND_ZONEOWNER 0x4
921 #define DTRACE_PROBEKEY_MAXDEPTH 8 /* max glob recursion depth */
924 * Access flag used by dtrace_mstate.dtms_access.
926 #define DTRACE_ACCESS_KERNEL 0x1 /* the priv to read kmem */
932 * Each DTrace consumer is in one of several states, which (for purposes of
933 * avoiding yet-another overloading of the noun "state") we call the current
934 * _activity_. The activity transitions on dtrace_go() (from DTRACIOCGO), on
935 * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action. Activities may
936 * only transition in one direction; the activity transition diagram is a
937 * directed acyclic graph. The activity transition diagram is as follows:
940 * +----------+ +--------+ +--------+
941 * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
942 * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+
943 * before BEGIN | after BEGIN | | |
945 * exit() action | | | |
946 * from BEGIN ECB | | | |
949 * +----------+ exit() action | | |
950 * +-----------------------------| DRAINING |<-------------------+ | |
953 * | dtrace_stop(), | | |
957 * | +---------+ +----------+ | |
958 * | | STOPPED |<----------------| COOLDOWN |<----------------------+ |
959 * | +---------+ dtrace_stop(), +----------+ dtrace_stop(), |
960 * | after END before END |
963 * +----------------------------->| KILLED |<--------------------------+
964 * deadman timeout or +--------+ deadman timeout or
965 * killed consumer killed consumer
967 * Note that once a DTrace consumer has stopped tracing, there is no way to
968 * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
969 * the DTrace pseudodevice.
971 typedef enum dtrace_activity
{
972 DTRACE_ACTIVITY_INACTIVE
= 0, /* not yet running */
973 DTRACE_ACTIVITY_WARMUP
, /* while starting */
974 DTRACE_ACTIVITY_ACTIVE
, /* running */
975 DTRACE_ACTIVITY_DRAINING
, /* before stopping */
976 DTRACE_ACTIVITY_COOLDOWN
, /* while stopping */
977 DTRACE_ACTIVITY_STOPPED
, /* after stopping */
978 DTRACE_ACTIVITY_KILLED
/* killed */
981 #if defined(__APPLE__)
985 * DTrace has four "dof modes". They are:
987 * DTRACE_DOF_MODE_NEVER Never load any dof, period.
988 * DTRACE_DOF_MODE_LAZY_ON Defer loading dof until later
989 * DTRACE_DOF_MODE_LAZY_OFF Load all deferred dof now, and any new dof
990 * DTRACE_DOF_MODE_NON_LAZY Load all dof immediately.
992 * It is legal to transition between the two lazy modes. The NEVER and
993 * NON_LAZY modes are permanent, and must not change once set.
995 * The current dof mode is kept in dtrace_dof_mode, which is protected by the
996 * dtrace_dof_mode_lock. This is a RW lock, reads require shared access, writes
997 * require exclusive access. Because NEVER and NON_LAZY are permanent states,
998 * it is legal to test for those modes without holding the dof mode lock.
1000 * Lock ordering is dof mode lock before any dtrace lock, and before the
1001 * process p_dtrace_sprlock. In general, other locks should not be held when
1002 * taking the dof mode lock. Acquiring the dof mode lock in exclusive mode
1003 * will block process fork, exec, and exit, so it should be held exclusive
1004 * for as short a time as possible.
1007 #define DTRACE_DOF_MODE_NEVER 0
1008 #define DTRACE_DOF_MODE_LAZY_ON 1
1009 #define DTRACE_DOF_MODE_LAZY_OFF 2
1010 #define DTRACE_DOF_MODE_NON_LAZY 3
1011 #endif /* __APPLE__ */
1014 * DTrace Helper Implementation
1016 * A description of the helper architecture may be found in <sys/dtrace.h>.
1017 * Each process contains a pointer to its helpers in its p_dtrace_helpers
1018 * member. This is a pointer to a dtrace_helpers structure, which contains an
1019 * array of pointers to dtrace_helper structures, helper variable state (shared
1020 * among a process's helpers) and a generation count. (The generation count is
1021 * used to provide an identifier when a helper is added so that it may be
1022 * subsequently removed.) The dtrace_helper structure is self-explanatory,
1023 * containing pointers to the objects needed to execute the helper. Note that
1024 * helpers are _duplicated_ across fork(2), and destroyed on exec(2). No more
1025 * than dtrace_helpers_max are allowed per-process.
1027 #define DTRACE_HELPER_ACTION_USTACK 0
1028 #define DTRACE_NHELPER_ACTIONS 1
1030 typedef struct dtrace_helper_action
{
1031 int dtha_generation
; /* helper action generation */
1032 int dtha_nactions
; /* number of actions */
1033 dtrace_difo_t
*dtha_predicate
; /* helper action predicate */
1034 dtrace_difo_t
**dtha_actions
; /* array of actions */
1035 struct dtrace_helper_action
*dtha_next
; /* next helper action */
1036 } dtrace_helper_action_t
;
1038 typedef struct dtrace_helper_provider
{
1039 int dthp_generation
; /* helper provider generation */
1040 uint32_t dthp_ref
; /* reference count */
1041 dof_helper_t dthp_prov
; /* DOF w/ provider and probes */
1042 } dtrace_helper_provider_t
;
1044 typedef struct dtrace_helpers
{
1045 dtrace_helper_action_t
**dthps_actions
; /* array of helper actions */
1046 dtrace_vstate_t dthps_vstate
; /* helper action var. state */
1047 dtrace_helper_provider_t
**dthps_provs
; /* array of providers */
1048 uint_t dthps_nprovs
; /* count of providers */
1049 uint_t dthps_maxprovs
; /* provider array size */
1050 int dthps_generation
; /* current generation */
1051 pid_t dthps_pid
; /* pid of associated proc */
1052 int dthps_deferred
; /* helper in deferred list */
1053 struct dtrace_helpers
*dthps_next
; /* next pointer */
1054 struct dtrace_helpers
*dthps_prev
; /* prev pointer */
1058 * DTrace Helper Action Tracing
1060 * Debugging helper actions can be arduous. To ease the development and
1061 * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
1062 * framework: helper tracing. If dtrace_helptrace_enabled is non-zero (which
1063 * it is by default on DEBUG kernels), all helper activity will be traced to a
1064 * global, in-kernel ring buffer. Each entry includes a pointer to the specific
1065 * helper, the location within the helper, and a trace of all local variables.
1066 * The ring buffer may be displayed in a human-readable format with the
1067 * ::dtrace_helptrace mdb(1) dcmd.
1069 #define DTRACE_HELPTRACE_NEXT (-1)
1070 #define DTRACE_HELPTRACE_DONE (-2)
1071 #define DTRACE_HELPTRACE_ERR (-3)
1073 typedef struct dtrace_helptrace
{
1074 dtrace_helper_action_t
*dtht_helper
; /* helper action */
1075 int dtht_where
; /* where in helper action */
1076 int dtht_nlocals
; /* number of locals */
1077 int dtht_fault
; /* type of fault (if any) */
1078 int dtht_fltoffs
; /* DIF offset */
1079 uint64_t dtht_illval
; /* faulting value */
1080 uint64_t dtht_locals
[1]; /* local variables */
1081 } dtrace_helptrace_t
;
1084 * DTrace Credentials
1086 * In probe context, we have limited flexibility to examine the credentials
1087 * of the DTrace consumer that created a particular enabling. We use
1088 * the Least Privilege interfaces to cache the consumer's cred pointer and
1089 * some facts about that credential in a dtrace_cred_t structure. These
1090 * can limit the consumer's breadth of visibility and what actions the
1091 * consumer may take.
1093 #define DTRACE_CRV_ALLPROC 0x01
1094 #define DTRACE_CRV_KERNEL 0x02
1095 #define DTRACE_CRV_ALLZONE 0x04
1097 #define DTRACE_CRV_ALL (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \
1100 #define DTRACE_CRA_PROC 0x0001
1101 #define DTRACE_CRA_PROC_CONTROL 0x0002
1102 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER 0x0004
1103 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE 0x0008
1104 #define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG 0x0010
1105 #define DTRACE_CRA_KERNEL 0x0020
1106 #define DTRACE_CRA_KERNEL_DESTRUCTIVE 0x0040
1108 #define DTRACE_CRA_ALL (DTRACE_CRA_PROC | \
1109 DTRACE_CRA_PROC_CONTROL | \
1110 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \
1111 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \
1112 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \
1113 DTRACE_CRA_KERNEL | \
1114 DTRACE_CRA_KERNEL_DESTRUCTIVE)
1116 typedef struct dtrace_cred
{
1118 uint8_t dcr_destructive
;
1119 uint8_t dcr_visible
;
1120 uint16_t dcr_action
;
1124 * DTrace Consumer State
1126 * Each DTrace consumer has an associated dtrace_state structure that contains
1127 * its in-kernel DTrace state -- including options, credentials, statistics and
1128 * pointers to ECBs, buffers, speculations and formats. A dtrace_state
1129 * structure is also allocated for anonymous enablings. When anonymous state
1130 * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
1131 * dtrace_state structure.
1133 struct dtrace_state
{
1134 dev_t dts_dev
; /* device */
1135 int dts_necbs
; /* total number of ECBs */
1136 dtrace_ecb_t
**dts_ecbs
; /* array of ECBs */
1137 dtrace_epid_t dts_epid
; /* next EPID to allocate */
1138 size_t dts_needed
; /* greatest needed space */
1139 struct dtrace_state
*dts_anon
; /* anon. state, if grabbed */
1140 dtrace_activity_t dts_activity
; /* current activity */
1141 dtrace_vstate_t dts_vstate
; /* variable state */
1142 dtrace_buffer_t
*dts_buffer
; /* principal buffer */
1143 dtrace_buffer_t
*dts_aggbuffer
; /* aggregation buffer */
1144 dtrace_speculation_t
*dts_speculations
; /* speculation array */
1145 int dts_nspeculations
; /* number of speculations */
1146 int dts_naggregations
; /* number of aggregations */
1147 dtrace_aggregation_t
**dts_aggregations
; /* aggregation array */
1148 vmem_t
*dts_aggid_arena
; /* arena for aggregation IDs */
1149 uint64_t dts_errors
; /* total number of errors */
1150 uint32_t dts_speculations_busy
; /* number of spec. busy */
1151 uint32_t dts_speculations_unavail
; /* number of spec unavail */
1152 uint32_t dts_stkstroverflows
; /* stack string tab overflows */
1153 uint32_t dts_dblerrors
; /* errors in ERROR probes */
1154 uint32_t dts_reserve
; /* space reserved for END */
1155 hrtime_t dts_laststatus
; /* time of last status */
1156 cyclic_id_t dts_cleaner
; /* cleaning cyclic */
1157 cyclic_id_t dts_deadman
; /* deadman cyclic */
1158 hrtime_t dts_alive
; /* time last alive */
1159 char dts_speculates
; /* boolean: has speculations */
1160 char dts_destructive
; /* boolean: has dest. actions */
1161 int dts_nformats
; /* number of formats */
1162 char **dts_formats
; /* format string array */
1163 dtrace_optval_t dts_options
[DTRACEOPT_MAX
]; /* options */
1164 dtrace_cred_t dts_cred
; /* credentials */
1165 size_t dts_nretained
; /* number of retained enabs */
1166 #if defined(__APPLE__)
1167 uint64_t dts_arg_error_illval
;
1168 #endif /* __APPLE__ */
1171 struct dtrace_provider
{
1172 dtrace_pattr_t dtpv_attr
; /* provider attributes */
1173 dtrace_ppriv_t dtpv_priv
; /* provider privileges */
1174 dtrace_pops_t dtpv_pops
; /* provider operations */
1175 char *dtpv_name
; /* provider name */
1176 void *dtpv_arg
; /* provider argument */
1177 uint_t dtpv_defunct
; /* boolean: defunct provider */
1178 struct dtrace_provider
*dtpv_next
; /* next provider */
1181 struct dtrace_meta
{
1182 dtrace_mops_t dtm_mops
; /* meta provider operations */
1183 char *dtm_name
; /* meta provider name */
1184 void *dtm_arg
; /* meta provider user arg */
1185 uint64_t dtm_count
; /* no. of associated provs. */
1191 * A dtrace_enabling structure is used to track a collection of ECB
1192 * descriptions -- before they have been turned into actual ECBs. This is
1193 * created as a result of DOF processing, and is generally used to generate
1194 * ECBs immediately thereafter. However, enablings are also generally
1195 * retained should the probes they describe be created at a later time; as
1196 * each new module or provider registers with the framework, the retained
1197 * enablings are reevaluated, with any new match resulting in new ECBs. To
1198 * prevent probes from being matched more than once, the enabling tracks the
1199 * last probe generation matched, and only matches probes from subsequent
1202 typedef struct dtrace_enabling
{
1203 dtrace_ecbdesc_t
**dten_desc
; /* all ECB descriptions */
1204 int dten_ndesc
; /* number of ECB descriptions */
1205 int dten_maxdesc
; /* size of ECB array */
1206 dtrace_vstate_t
*dten_vstate
; /* associated variable state */
1207 dtrace_genid_t dten_probegen
; /* matched probe generation */
1208 dtrace_ecbdesc_t
*dten_current
; /* current ECB description */
1209 int dten_error
; /* current error value */
1210 int dten_primed
; /* boolean: set if primed */
1211 struct dtrace_enabling
*dten_prev
; /* previous enabling */
1212 struct dtrace_enabling
*dten_next
; /* next enabling */
1213 } dtrace_enabling_t
;
1216 * DTrace Anonymous Enablings
1218 * Anonymous enablings are DTrace enablings that are not associated with a
1219 * controlling process, but rather derive their enabling from DOF stored as
1220 * properties in the dtrace.conf file. If there is an anonymous enabling, a
1221 * DTrace consumer state and enabling are created on attach. The state may be
1222 * subsequently grabbed by the first consumer specifying the "grabanon"
1223 * option. As long as an anonymous DTrace enabling exists, dtrace(7D) will
1226 typedef struct dtrace_anon
{
1227 dtrace_state_t
*dta_state
; /* DTrace consumer state */
1228 dtrace_enabling_t
*dta_enabling
; /* pointer to enabling */
1229 processorid_t dta_beganon
; /* which CPU BEGIN ran on */
1233 * DTrace Error Debugging
1236 #define DTRACE_ERRDEBUG
1239 #ifdef DTRACE_ERRDEBUG
1241 typedef struct dtrace_errhash
{
1242 const char *dter_msg
; /* error message */
1243 int dter_count
; /* number of times seen */
1246 #define DTRACE_ERRHASHSZ 256 /* must be > number of err msgs */
1248 #endif /* DTRACE_ERRDEBUG */
1251 * DTrace Toxic Ranges
1253 * DTrace supports safe loads from probe context; if the address turns out to
1254 * be invalid, a bit will be set by the kernel indicating that DTrace
1255 * encountered a memory error, and DTrace will propagate the error to the user
1256 * accordingly. However, there may exist some regions of memory in which an
1257 * arbitrary load can change system state, and from which it is impossible to
1258 * recover from such a load after it has been attempted. Examples of this may
1259 * include memory in which programmable I/O registers are mapped (for which a
1260 * read may have some implications for the device) or (in the specific case of
1261 * UltraSPARC-I and -II) the virtual address hole. The platform is required
1262 * to make DTrace aware of these toxic ranges; DTrace will then check that
1263 * target addresses are not in a toxic range before attempting to issue a
1266 typedef struct dtrace_toxrange
{
1267 uintptr_t dtt_base
; /* base of toxic range */
1268 uintptr_t dtt_limit
; /* limit of toxic range */
1269 } dtrace_toxrange_t
;
1271 extern uint64_t dtrace_getarg(int, int);
1272 extern int dtrace_getipl(void);
1273 extern uintptr_t dtrace_caller(int);
1274 extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
1275 extern void *dtrace_casptr(void *, void *, void *);
1276 #if !defined(__APPLE__)
1277 extern void dtrace_copyin(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1278 extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1279 extern void dtrace_copyout(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1280 extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t,
1281 volatile uint16_t *);
1283 extern void dtrace_copyin(user_addr_t
, uintptr_t, size_t, volatile uint16_t *);
1284 extern void dtrace_copyinstr(user_addr_t
, uintptr_t, size_t, volatile uint16_t *);
1285 extern void dtrace_copyout(uintptr_t, user_addr_t
, size_t, volatile uint16_t *);
1286 extern void dtrace_copyoutstr(uintptr_t, user_addr_t
, size_t, volatile uint16_t *);
1287 #endif /* __APPLE__ */
1288 extern void dtrace_getpcstack(pc_t
*, int, int, uint32_t *);
1289 #if !defined(__APPLE__)
1290 extern ulong_t
dtrace_getreg(struct regs
*, uint_t
);
1292 extern uint64_t dtrace_getreg(struct regs
*, uint_t
);
1293 #endif /* __APPLE__ */
1294 extern int dtrace_getstackdepth(int);
1295 extern void dtrace_getupcstack(uint64_t *, int);
1296 extern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
1297 extern int dtrace_getustackdepth(void);
1298 extern uintptr_t dtrace_fulword(void *);
1299 #if !defined(__APPLE__)
1300 extern uint8_t dtrace_fuword8(void *);
1301 extern uint16_t dtrace_fuword16(void *);
1302 extern uint32_t dtrace_fuword32(void *);
1303 extern uint64_t dtrace_fuword64(void *);
1304 extern void dtrace_probe_error(dtrace_state_t
*, dtrace_epid_t
, int, int,
1307 extern uint8_t dtrace_fuword8(user_addr_t
);
1308 extern uint16_t dtrace_fuword16(user_addr_t
);
1309 extern uint32_t dtrace_fuword32(user_addr_t
);
1310 extern uint64_t dtrace_fuword64(user_addr_t
);
1311 extern void dtrace_probe_error(dtrace_state_t
*, dtrace_epid_t
, int, int,
1313 #endif /* __APPLE__ */
1314 extern int dtrace_assfail(const char *, const char *, int);
1315 extern int dtrace_attached(void);
1316 extern hrtime_t
dtrace_gethrestime(void);
1319 extern void dtrace_flush_windows(void);
1320 extern void dtrace_flush_user_windows(void);
1321 extern uint_t
dtrace_getotherwin(void);
1322 extern uint_t
dtrace_getfprs(void);
1324 extern void dtrace_copy(uintptr_t, uintptr_t, size_t);
1325 extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1331 * DTrace calls ASSERT from probe context. To assure that a failed ASSERT
1332 * does not induce a markedly more catastrophic failure (e.g., one from which
1333 * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
1334 * may safely be called from probe context. This header file must thus be
1335 * included by any DTrace component that calls ASSERT from probe context, and
1336 * _only_ by those components. (The only exception to this is kernel
1337 * debugging infrastructure at user-level that doesn't depend on calling
1342 #define ASSERT(EX) ((void)((EX) || \
1343 dtrace_assfail(#EX, __FILE__, __LINE__)))
1345 #define ASSERT(X) ((void)0)
1352 #endif /* _SYS_DTRACE_IMPL_H */
projects / apple / xnu.git / blob