[apple/xnu.git] / iokit / IOKit / IOStatisticsPrivate.h

/*
 * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef __IOKIT_STATISTICS_PRIVATE_H
#define __IOKIT_STATISTICS_PRIVATE_H

#if IOKITSTATS

#include <sys/queue.h>
#include <sys/tree.h>

#include <libkern/c++/OSKext.h>
#include <libkern/OSDebug.h>

#include <IOKit/IOMemoryDescriptor.h>
#include <IOKit/IOStatistics.h>

#ifndef KERNEL
#error IOStatisticsPrivate.h is for kernel use only
#endif

/* Defines */
#define IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS 20

#ifndef __probable
#define __probable(x) x
#endif

/* Forward declarations */
class IOWorkLoop;
class IOUserClient;
class IOEventSource;

struct IOEventSourceCounter;
struct IOUserClientCounter;
struct IOWorkLoopCounter;
struct IOUserClientProcessEntry;

struct KextNode;

/* Allocation tracking */

enum {
	kIOStatisticsMalloc = 0,
	kIOStatisticsFree,
	kIOStatisticsMallocAligned,
	kIOStatisticsFreeAligned,
	kIOStatisticsMallocContiguous,
	kIOStatisticsFreeContiguous,
	kIOStatisticsMallocPageable,
	kIOStatisticsFreePageable,
	kIOStatisticsAllocCount
};

TAILQ_HEAD(ProcessEntryList, IOUserClientProcessEntry);

/* Tree and list structs */

typedef struct ClassNode {
	RB_ENTRY(ClassNode) tLink;
	SLIST_ENTRY(ClassNode) lLink;
	struct KextNode *parentKext;
	uint32_t classID;
	uint32_t superClassID;
	const OSMetaClass *metaClass;
	SLIST_HEAD(, IOEventSourceCounter) counterList;
	SLIST_HEAD(, IOUserClientCounter) userClientList;
} ClassNode;

typedef struct KextNode {
	RB_ENTRY(KextNode) link;
	RB_ENTRY(KextNode) addressLink;
	OSKext *kext;
	OSKextLoadTag loadTag;
	vm_offset_t address;
	vm_offset_t address_end;
	uint32_t memoryCounters[kIOStatisticsAllocCount];
	uint32_t classes;
	SLIST_HEAD(, ClassNode) classList;
	SLIST_HEAD(, IOWorkLoopCounter) workLoopList;
	ProcessEntryList userClientCallList;
} KextNode;

/* User client tracing */

typedef struct IOUserClientProcessEntry {
	TAILQ_ENTRY(IOUserClientProcessEntry) link;
	char processName[kIOStatisticsProcessNameLength];
	int32_t pid;
	uint32_t calls;
} IOUserClientProcessEntry;

/* Counters */

typedef struct IOInterruptEventSourceCounter {
	uint32_t produced;
	uint32_t checksForWork;
} IOInterruptEventSourceCounter;

typedef struct IOTimerEventSourceCounter {
	uint32_t timeouts;
	uint32_t checksForWork;
} IOTimerEventSourceCounter;

typedef struct IOCommandGateCounter {
	uint32_t actionCalls;
} IOCommandGateCounter;

typedef struct IOCommandQueueCounter {
	uint32_t actionCalls;
} IOCommandQueueCounter;

typedef struct IOEventSourceCounter {
	SLIST_ENTRY(IOEventSourceCounter) link;
	ClassNode *parentClass;
	IOStatisticsCounterType type;
	uint64_t startTimeStamp;
	uint64_t timeOnGate;
	uint32_t closeGateCalls;
	uint32_t openGateCalls;
	union {
		IOInterruptEventSourceCounter interrupt;
		IOInterruptEventSourceCounter filter;
		IOTimerEventSourceCounter timer;
		IOCommandGateCounter commandGate;
		IOCommandQueueCounter commandQueue;
	} u;
} IOEventSourceCounter;

typedef struct IOWorkLoopDependency {
	RB_ENTRY(IOWorkLoopDependency) link;
	OSKextLoadTag loadTag;
} IOWorkLoopDependency;

typedef struct IOWorkLoopCounter {    
	SLIST_ENTRY(IOWorkLoopCounter) link;
	KextNode *parentKext;
	int attachedEventSources;
	IOWorkLoop *workLoop;
	uint64_t startTimeStamp;
    uint64_t timeOnGate;
	uint32_t closeGateCalls;
	uint32_t openGateCalls;
	typedef RB_HEAD(DependencyTree, IOWorkLoopDependency) DependencyTreeHead;
	DependencyTreeHead dependencyHead;
	static int loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2);
	RB_PROTOTYPE_SC(static, DependencyTree, IOWorkLoopDependency, dependencyLink, KextTagCompare);
} IOWorkLoopCounter;

typedef struct IOUserClientCounter {
	SLIST_ENTRY(IOUserClientCounter) link;
	ClassNode *parentClass;
	uint32_t clientCalls;
} IOUserClientCounter;

class IOStatistics {
	static bool enabled;

	static IORWLock *lock;

	static uint32_t sequenceID;

	static uint32_t lastKextIndex;
	static uint32_t lastClassIndex;

	static uint32_t loadedKexts;
	static uint32_t registeredClasses;
	static uint32_t registeredCounters;
	static uint32_t registeredWorkloops;

	static uint32_t attachedEventSources;

	static KextNode *kextHint;
	
	static IOWorkLoopDependency *nextWorkLoopDependency;

	typedef RB_HEAD(KextTree, KextNode) KextTreeHead;
	static KextTreeHead kextHead;
	static int kextNodeCompare(KextNode *e1, KextNode *e2);
	RB_PROTOTYPE_SC(static, KextTree, KextNode, link, kextNodeCompare);

	typedef RB_HEAD(KextAddressTree, KextNode) KextAddressTreeHead;
	static KextAddressTreeHead kextAddressHead;
	static int kextAddressNodeCompare(KextNode *e1, KextNode *e2);
	RB_PROTOTYPE_SC(static, KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);

	typedef RB_HEAD(ClassTree, ClassNode) ClassTreeHead;
	static ClassTreeHead classHead;
	static int classNodeCompare(ClassNode *e1, ClassNode *e2);
	RB_PROTOTYPE_SC(static, ClassTree, ClassNode, tLink, classNodeCompare);

	static int oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req);

	static uint32_t copyGlobalStatistics(IOStatisticsGlobal *stats);
	static uint32_t copyKextStatistics(IOStatisticsKext *stats);
	static uint32_t copyMemoryStatistics(IOStatisticsMemory *stats);
	static uint32_t copyClassStatistics(IOStatisticsClass *stats);
	static uint32_t copyCounterStatistics(IOStatisticsCounter *stats);
	static uint32_t copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs);
	static uint32_t copyClassNames(IOStatisticsClassName *classNames);

	static uint32_t copyWorkLoopStatistics(IOStatisticsWorkLoop *workLoopStats);

	static uint32_t copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag);

	static void updateAllocationCounter(vm_offset_t address, uint32_t index, vm_size_t size);

	static void storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter);

	static KextNode *getKextNodeFromBacktrace(boolean_t write);
	static void releaseKextNode(KextNode *node);

public:
	
	static void initialize();

	static void onKextLoad(OSKext *kext, kmod_info_t *kmod_info);
	static void onKextUnload(OSKext *kext);
	static void onClassAdded(OSKext *parentKext, OSMetaClass *metaClass);
	static void onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass);

	static IOEventSourceCounter *registerEventSource(OSObject *inOwner);
	static void unregisterEventSource(IOEventSourceCounter *counter);
	
	static IOWorkLoopCounter *registerWorkLoop(IOWorkLoop *workLoop);
	static void unregisterWorkLoop(IOWorkLoopCounter *counter);

	static IOUserClientCounter *registerUserClient(IOUserClient *userClient);
	static void unregisterUserClient(IOUserClientCounter *counter);

	static int getStatistics(sysctl_req *req);
	static int getWorkLoopStatistics(sysctl_req *req);
	static int getUserClientStatistics(sysctl_req *req);

	/* Inlines for counter manipulation.
	 * 				
	 * NOTE: counter access is not expressly guarded here so as not to incur performance penalties
	 * in the instrumented parent objects. Writes are arranged so as to be protected by pre-existing
	 * locks in the parent where appropriate, but reads have no such guarantee. Counters should
	 * therefore be regarded as providing an indication of current state, rather than precisely
	 * accurate statistics. 
	 */
	
	static inline void setCounterType(IOEventSourceCounter *counter, IOStatisticsCounterType type) {
		if (counter) {
			counter->type = type;
		}
	}

	static inline void countOpenGate(IOEventSourceCounter *counter) {
		if (counter) {
			counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
			counter->openGateCalls++;
		}
	}

	static inline void countCloseGate(IOEventSourceCounter *counter) {
		if (counter) {
			counter->startTimeStamp = mach_absolute_time();
			counter->closeGateCalls++;
		}
	}

	/* Interrupt */
	static inline void countInterruptCheckForWork(IOEventSourceCounter *counter) {
		if (counter) {
			counter->u.interrupt.checksForWork++;
		}
	}

	static inline void countInterrupt(IOEventSourceCounter *counter) {
		if (counter) {
			counter->u.interrupt.produced++;
		}
	}

	/* CommandQueue */
	static inline void countCommandQueueActionCall(IOEventSourceCounter *counter) {
		if (counter) {
			counter->u.commandQueue.actionCalls++;
		}
	}

	/* CommandGate */
	static inline void countCommandGateActionCall(IOEventSourceCounter *counter) {
		if (counter) {
			counter->u.commandGate.actionCalls++;
		}
	}

	/* Timer */
	static inline void countTimerTimeout(IOEventSourceCounter *counter) {
		if (counter) {
			counter->u.timer.timeouts++;
		}
	}

	/* WorkLoop */
	static void attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc);
	static void detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc);

	static inline void countWorkLoopOpenGate(IOWorkLoopCounter *counter) {
		if (counter) {
			counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
			counter->openGateCalls++;
		}
	}

	static inline void countWorkLoopCloseGate(IOWorkLoopCounter *counter) {
		if (counter) {
			counter->startTimeStamp = mach_absolute_time();
			counter->closeGateCalls++;
		}
	}

	/* IOLib allocations */	
	static void countAlloc(uint32_t index, vm_size_t size);

	/* UserClient */
	static void countUserClientCall(IOUserClient *client);
};

#else

/* Statistics disabled */

class IOStatistics {
public:
	static void initialize() {}
};

#endif /* IOKITSTATS */

#endif /* __IOKIT_STATISTICS_PRIVATE_H */
Commit	Line	Data
6d2010ae A	1	/*
	2	* Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#ifndef __IOKIT_STATISTICS_PRIVATE_H
	30	#define __IOKIT_STATISTICS_PRIVATE_H
	31
	32	#if IOKITSTATS
	33
	34	#include <sys/queue.h>
	35	#include <sys/tree.h>
	36
	37	#include <libkern/c++/OSKext.h>
	38	#include <libkern/OSDebug.h>
	39
	40	#include <IOKit/IOMemoryDescriptor.h>
	41	#include <IOKit/IOStatistics.h>
	42
	43	#ifndef KERNEL
	44	#error IOStatisticsPrivate.h is for kernel use only
	45	#endif
	46
	47	/* Defines */
	48	#define IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS 20
	49
	50	#ifndef __probable
	51	#define __probable(x) x
	52	#endif
	53
	54	/* Forward declarations */
	55	class IOWorkLoop;
	56	class IOUserClient;
	57	class IOEventSource;
	58
	59	struct IOEventSourceCounter;
	60	struct IOUserClientCounter;
	61	struct IOWorkLoopCounter;
	62	struct IOUserClientProcessEntry;
	63
	64	struct KextNode;
65
66	/* Allocation tracking */
67
68	enum {
69	kIOStatisticsMalloc = 0,
70	kIOStatisticsFree,
71	kIOStatisticsMallocAligned,
72	kIOStatisticsFreeAligned,
73	kIOStatisticsMallocContiguous,
74	kIOStatisticsFreeContiguous,
75	kIOStatisticsMallocPageable,
76	kIOStatisticsFreePageable,
77	kIOStatisticsAllocCount
78	};
79
80	TAILQ_HEAD(ProcessEntryList, IOUserClientProcessEntry);
81
82	/* Tree and list structs */
83
84	typedef struct ClassNode {
85	RB_ENTRY(ClassNode) tLink;
86	SLIST_ENTRY(ClassNode) lLink;
87	struct KextNode *parentKext;
88	uint32_t classID;
89	uint32_t superClassID;
90	const OSMetaClass *metaClass;
91	SLIST_HEAD(, IOEventSourceCounter) counterList;
92	SLIST_HEAD(, IOUserClientCounter) userClientList;
93	} ClassNode;
94
95	typedef struct KextNode {
96	RB_ENTRY(KextNode) link;
97	RB_ENTRY(KextNode) addressLink;
98	OSKext *kext;
99	OSKextLoadTag loadTag;
100	vm_offset_t address;
101	vm_offset_t address_end;
102	uint32_t memoryCounters[kIOStatisticsAllocCount];
103	uint32_t classes;
104	SLIST_HEAD(, ClassNode) classList;
105	SLIST_HEAD(, IOWorkLoopCounter) workLoopList;
106	ProcessEntryList userClientCallList;
107	} KextNode;
108
109	/* User client tracing */
110
111	typedef struct IOUserClientProcessEntry {
112	TAILQ_ENTRY(IOUserClientProcessEntry) link;
113	char processName[kIOStatisticsProcessNameLength];
114	int32_t pid;
115	uint32_t calls;
116	} IOUserClientProcessEntry;
117
118	/* Counters */
119
120	typedef struct IOInterruptEventSourceCounter {
121	uint32_t produced;
122	uint32_t checksForWork;
123	} IOInterruptEventSourceCounter;
124
125	typedef struct IOTimerEventSourceCounter {
126	uint32_t timeouts;
127	uint32_t checksForWork;
128	} IOTimerEventSourceCounter;
129
130	typedef struct IOCommandGateCounter {
131	uint32_t actionCalls;
132	} IOCommandGateCounter;
133
134	typedef struct IOCommandQueueCounter {
135	uint32_t actionCalls;
136	} IOCommandQueueCounter;
137
138	typedef struct IOEventSourceCounter {
139	SLIST_ENTRY(IOEventSourceCounter) link;
140	ClassNode *parentClass;
141	IOStatisticsCounterType type;
142	uint64_t startTimeStamp;
143	uint64_t timeOnGate;
144	uint32_t closeGateCalls;
145	uint32_t openGateCalls;
146	union {
147	IOInterruptEventSourceCounter interrupt;
148	IOInterruptEventSourceCounter filter;
149	IOTimerEventSourceCounter timer;
150	IOCommandGateCounter commandGate;
151	IOCommandQueueCounter commandQueue;
152	} u;
153	} IOEventSourceCounter;
154
155	typedef struct IOWorkLoopDependency {
156	RB_ENTRY(IOWorkLoopDependency) link;
157	OSKextLoadTag loadTag;
158	} IOWorkLoopDependency;
159
160	typedef struct IOWorkLoopCounter {
161	SLIST_ENTRY(IOWorkLoopCounter) link;
162	KextNode *parentKext;
163	int attachedEventSources;
164	IOWorkLoop *workLoop;
165	uint64_t startTimeStamp;
166	uint64_t timeOnGate;
167	uint32_t closeGateCalls;
168	uint32_t openGateCalls;
169	typedef RB_HEAD(DependencyTree, IOWorkLoopDependency) DependencyTreeHead;
170	DependencyTreeHead dependencyHead;
171	static int loadTagCompare(IOWorkLoopDependency e1, IOWorkLoopDependency e2);
172	RB_PROTOTYPE_SC(static, DependencyTree, IOWorkLoopDependency, dependencyLink, KextTagCompare);
173	} IOWorkLoopCounter;
174
175	typedef struct IOUserClientCounter {
176	SLIST_ENTRY(IOUserClientCounter) link;
177	ClassNode *parentClass;
178	uint32_t clientCalls;
179	} IOUserClientCounter;
180
181	class IOStatistics {
182	static bool enabled;
183
184	static IORWLock *lock;
185
186	static uint32_t sequenceID;
187
188	static uint32_t lastKextIndex;
189	static uint32_t lastClassIndex;
190
191	static uint32_t loadedKexts;
192	static uint32_t registeredClasses;
193	static uint32_t registeredCounters;
194	static uint32_t registeredWorkloops;
195
196	static uint32_t attachedEventSources;
197
198	static KextNode *kextHint;
199
200	static IOWorkLoopDependency *nextWorkLoopDependency;
201
202	typedef RB_HEAD(KextTree, KextNode) KextTreeHead;
203	static KextTreeHead kextHead;
204	static int kextNodeCompare(KextNode e1, KextNode e2);
205	RB_PROTOTYPE_SC(static, KextTree, KextNode, link, kextNodeCompare);
206
207	typedef RB_HEAD(KextAddressTree, KextNode) KextAddressTreeHead;
208	static KextAddressTreeHead kextAddressHead;
209	static int kextAddressNodeCompare(KextNode e1, KextNode e2);
210	RB_PROTOTYPE_SC(static, KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
211
212	typedef RB_HEAD(ClassTree, ClassNode) ClassTreeHead;
213	static ClassTreeHead classHead;
214	static int classNodeCompare(ClassNode e1, ClassNode e2);
215	RB_PROTOTYPE_SC(static, ClassTree, ClassNode, tLink, classNodeCompare);
216
217	static int oid_sysctl(__unused struct sysctl_oid oidp, __unused void arg1, int arg2, struct sysctl_req *req);
218
219	static uint32_t copyGlobalStatistics(IOStatisticsGlobal *stats);
220	static uint32_t copyKextStatistics(IOStatisticsKext *stats);
221	static uint32_t copyMemoryStatistics(IOStatisticsMemory *stats);
222	static uint32_t copyClassStatistics(IOStatisticsClass *stats);
223	static uint32_t copyCounterStatistics(IOStatisticsCounter *stats);
224	static uint32_t copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs);
225	static uint32_t copyClassNames(IOStatisticsClassName *classNames);
226
227	static uint32_t copyWorkLoopStatistics(IOStatisticsWorkLoop *workLoopStats);
228
229	static uint32_t copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag);
230
231	static void updateAllocationCounter(vm_offset_t address, uint32_t index, vm_size_t size);
232
233	static void storeUserClientCallInfo(IOUserClient userClient, IOUserClientCounter counter);
234
235	static KextNode *getKextNodeFromBacktrace(boolean_t write);
236	static void releaseKextNode(KextNode *node);
237
238	public:
239
240	static void initialize();
241
242	static void onKextLoad(OSKext kext, kmod_info_t kmod_info);
243	static void onKextUnload(OSKext *kext);
244	static void onClassAdded(OSKext parentKext, OSMetaClass metaClass);
245	static void onClassRemoved(OSKext parentKext, OSMetaClass metaClass);
246
247	static IOEventSourceCounter registerEventSource(OSObject inOwner);
248	static void unregisterEventSource(IOEventSourceCounter *counter);
249
250	static IOWorkLoopCounter registerWorkLoop(IOWorkLoop workLoop);
251	static void unregisterWorkLoop(IOWorkLoopCounter *counter);
252
253	static IOUserClientCounter registerUserClient(IOUserClient userClient);
254	static void unregisterUserClient(IOUserClientCounter *counter);
255
256	static int getStatistics(sysctl_req *req);
257	static int getWorkLoopStatistics(sysctl_req *req);
258	static int getUserClientStatistics(sysctl_req *req);
259
260	/* Inlines for counter manipulation.
261	*
262	* NOTE: counter access is not expressly guarded here so as not to incur performance penalties
263	* in the instrumented parent objects. Writes are arranged so as to be protected by pre-existing
264	* locks in the parent where appropriate, but reads have no such guarantee. Counters should
265	* therefore be regarded as providing an indication of current state, rather than precisely
266	* accurate statistics.
267	*/
268
269	static inline void setCounterType(IOEventSourceCounter *counter, IOStatisticsCounterType type) {
270	if (counter) {
271	counter->type = type;
272	}
273	}
274
275	static inline void countOpenGate(IOEventSourceCounter *counter) {
276	if (counter) {
277	counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
278	counter->openGateCalls++;
279	}
280	}
281
282	static inline void countCloseGate(IOEventSourceCounter *counter) {
283	if (counter) {
284	counter->startTimeStamp = mach_absolute_time();
285	counter->closeGateCalls++;
286	}
287	}
288
289	/* Interrupt */
290	static inline void countInterruptCheckForWork(IOEventSourceCounter *counter) {
291	if (counter) {
292	counter->u.interrupt.checksForWork++;
293	}
294	}
295
296	static inline void countInterrupt(IOEventSourceCounter *counter) {
297	if (counter) {
298	counter->u.interrupt.produced++;
299	}
300	}
301
302	/* CommandQueue */
303	static inline void countCommandQueueActionCall(IOEventSourceCounter *counter) {
304	if (counter) {
305	counter->u.commandQueue.actionCalls++;
306	}
307	}
308
309	/* CommandGate */
310	static inline void countCommandGateActionCall(IOEventSourceCounter *counter) {
311	if (counter) {
312	counter->u.commandGate.actionCalls++;
313	}
314	}
315
316	/* Timer */
317	static inline void countTimerTimeout(IOEventSourceCounter *counter) {
318	if (counter) {
319	counter->u.timer.timeouts++;
320	}
321	}
322
323	/* WorkLoop */
324	static void attachWorkLoopEventSource(IOWorkLoopCounter wlc, IOEventSourceCounter esc);
325	static void detachWorkLoopEventSource(IOWorkLoopCounter wlc, IOEventSourceCounter esc);
326
327	static inline void countWorkLoopOpenGate(IOWorkLoopCounter *counter) {
328	if (counter) {
329	counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
330	counter->openGateCalls++;
331	}
332	}
333
334	static inline void countWorkLoopCloseGate(IOWorkLoopCounter *counter) {
335	if (counter) {
336	counter->startTimeStamp = mach_absolute_time();
337	counter->closeGateCalls++;
338	}
339	}
340
341	/* IOLib allocations */
342	static void countAlloc(uint32_t index, vm_size_t size);
343
344	/* UserClient */
345	static void countUserClientCall(IOUserClient *client);
346	};
347
348	#else
349
350	/* Statistics disabled */
351
352	class IOStatistics {
353	public:
354	static void initialize() {}
355	};
356
357	#endif /* IOKITSTATS */
358
359	#endif /* __IOKIT_STATISTICS_PRIVATE_H */