[apple/xnu.git] / iokit / IOKit / IOStatisticsPrivate.h

/*
 * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef __IOKIT_STATISTICS_PRIVATE_H
#define __IOKIT_STATISTICS_PRIVATE_H

#if IOKITSTATS

#include <sys/queue.h>
#include <sys/tree.h>

#include <libkern/c++/OSKext.h>
#include <libkern/OSDebug.h>

#include <IOKit/IOMemoryDescriptor.h>
#include <IOKit/IOStatistics.h>

#ifndef KERNEL
#error IOStatisticsPrivate.h is for kernel use only
#endif

/* Defines */
#define IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS 20

#ifndef __probable
#define __probable(x) x
#endif

/* Forward declarations */
class IOWorkLoop;
class IOUserClient;
class IOEventSource;

struct IOEventSourceCounter;
struct IOUserClientCounter;
struct IOWorkLoopCounter;
struct IOUserClientProcessEntry;

struct KextNode;

/* Allocation tracking */

enum {
	kIOStatisticsMalloc = 0,
	kIOStatisticsFree,
	kIOStatisticsMallocAligned,
	kIOStatisticsFreeAligned,
	kIOStatisticsMallocContiguous,
	kIOStatisticsFreeContiguous,
	kIOStatisticsMallocPageable,
	kIOStatisticsFreePageable,
	kIOStatisticsAllocCount
};

TAILQ_HEAD(ProcessEntryList, IOUserClientProcessEntry);

/* Tree and list structs */

typedef struct ClassNode {
	RB_ENTRY(ClassNode) tLink;
	SLIST_ENTRY(ClassNode) lLink;
	struct KextNode *parentKext;
	uint32_t classID;
	uint32_t superClassID;
	const OSMetaClass *metaClass;
	SLIST_HEAD(, IOEventSourceCounter) counterList;
	SLIST_HEAD(, IOUserClientCounter) userClientList;
} ClassNode;

typedef struct KextNode {
	RB_ENTRY(KextNode) link;
	RB_ENTRY(KextNode) addressLink;
	OSKext *kext;
	OSKextLoadTag loadTag;
	vm_offset_t address;
	vm_offset_t address_end;
	uint32_t memoryCounters[kIOStatisticsAllocCount];
	uint32_t classes;
	SLIST_HEAD(, ClassNode) classList;
	SLIST_HEAD(, IOWorkLoopCounter) workLoopList;
	ProcessEntryList userClientCallList;
} KextNode;

/* User client tracing */

typedef struct IOUserClientProcessEntry {
	TAILQ_ENTRY(IOUserClientProcessEntry) link;
	char processName[kIOStatisticsProcessNameLength];
	int32_t pid;
	uint32_t calls;
} IOUserClientProcessEntry;

/* Counters */

typedef struct IOInterruptEventSourceCounter {
	uint32_t produced;
	uint32_t checksForWork;
} IOInterruptEventSourceCounter;

typedef struct IOTimerEventSourceCounter {
	uint32_t timeouts;
	uint32_t checksForWork;
} IOTimerEventSourceCounter;

typedef struct IOCommandGateCounter {
	uint32_t actionCalls;
} IOCommandGateCounter;

typedef struct IOCommandQueueCounter {
	uint32_t actionCalls;
} IOCommandQueueCounter;

typedef struct IOEventSourceCounter {
	SLIST_ENTRY(IOEventSourceCounter) link;
	ClassNode *parentClass;
	IOStatisticsCounterType type;
	uint64_t startTimeStamp;
	uint64_t timeOnGate;
	uint32_t closeGateCalls;
	uint32_t openGateCalls;
	union {
		IOInterruptEventSourceCounter interrupt;
		IOInterruptEventSourceCounter filter;
		IOTimerEventSourceCounter timer;
		IOCommandGateCounter commandGate;
		IOCommandQueueCounter commandQueue;
	} u;
} IOEventSourceCounter;

typedef struct IOWorkLoopDependency {
	RB_ENTRY(IOWorkLoopDependency) link;
	OSKextLoadTag loadTag;
} IOWorkLoopDependency;

typedef struct IOWorkLoopCounter {
	SLIST_ENTRY(IOWorkLoopCounter) link;
	KextNode *parentKext;
	int attachedEventSources;
	IOWorkLoop *workLoop;
	uint64_t startTimeStamp;
	uint64_t timeOnGate;
	uint32_t closeGateCalls;
	uint32_t openGateCalls;
	typedef RB_HEAD(DependencyTree, IOWorkLoopDependency) DependencyTreeHead;
	DependencyTreeHead dependencyHead;
	static int loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2);
	RB_PROTOTYPE_SC(static, DependencyTree, IOWorkLoopDependency, dependencyLink, KextTagCompare);
} IOWorkLoopCounter;

typedef struct IOUserClientCounter {
	SLIST_ENTRY(IOUserClientCounter) link;
	ClassNode *parentClass;
	uint32_t clientCalls;
} IOUserClientCounter;

class IOStatistics {
	static bool enabled;

	static IORWLock *lock;

	static uint32_t sequenceID;

	static uint32_t lastKextIndex;
	static uint32_t lastClassIndex;

	static uint32_t loadedKexts;
	static uint32_t registeredClasses;
	static uint32_t registeredCounters;
	static uint32_t registeredWorkloops;

	static uint32_t attachedEventSources;

	static KextNode *kextHint;

	static IOWorkLoopDependency *nextWorkLoopDependency;

	typedef RB_HEAD(KextTree, KextNode) KextTreeHead;
	static KextTreeHead kextHead;
	static int kextNodeCompare(KextNode *e1, KextNode *e2);
	RB_PROTOTYPE_SC(static, KextTree, KextNode, link, kextNodeCompare);

	typedef RB_HEAD(KextAddressTree, KextNode) KextAddressTreeHead;
	static KextAddressTreeHead kextAddressHead;
	static int kextAddressNodeCompare(KextNode *e1, KextNode *e2);
	RB_PROTOTYPE_SC(static, KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);

	typedef RB_HEAD(ClassTree, ClassNode) ClassTreeHead;
	static ClassTreeHead classHead;
	static int classNodeCompare(ClassNode *e1, ClassNode *e2);
	RB_PROTOTYPE_SC(static, ClassTree, ClassNode, tLink, classNodeCompare);

	static int oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req);

	static uint32_t copyGlobalStatistics(IOStatisticsGlobal *stats);
	static uint32_t copyKextStatistics(IOStatisticsKext *stats);
	static uint32_t copyMemoryStatistics(IOStatisticsMemory *stats);
	static uint32_t copyClassStatistics(IOStatisticsClass *stats);
	static uint32_t copyCounterStatistics(IOStatisticsCounter *stats);
	static uint32_t copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs);
	static uint32_t copyClassNames(IOStatisticsClassName *classNames);

	static uint32_t copyWorkLoopStatistics(IOStatisticsWorkLoop *workLoopStats);

	static uint32_t copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag);

	static void updateAllocationCounter(vm_offset_t address, uint32_t index, vm_size_t size);

	static void storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter);

	static KextNode *getKextNodeFromBacktrace(boolean_t write);
	static void releaseKextNode(KextNode *node);

public:

	static void initialize();

	inline static bool
	isEnabled()
	{
		return enabled;
	}

	static void onKextLoad(OSKext *kext, kmod_info_t *kmod_info);
	static void onKextUnload(OSKext *kext);
	static void onClassAdded(OSKext *parentKext, OSMetaClass *metaClass);
	static void onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass);

	static IOEventSourceCounter *registerEventSource(OSObject *inOwner);
	static void unregisterEventSource(IOEventSourceCounter *counter);

	static IOWorkLoopCounter *registerWorkLoop(IOWorkLoop *workLoop);
	static void unregisterWorkLoop(IOWorkLoopCounter *counter);

	static IOUserClientCounter *registerUserClient(IOUserClient *userClient);
	static void unregisterUserClient(IOUserClientCounter *counter);

	static int getStatistics(sysctl_req *req);
	static int getWorkLoopStatistics(sysctl_req *req);
	static int getUserClientStatistics(sysctl_req *req);

/* Inlines for counter manipulation.
 *
 * NOTE: counter access is not expressly guarded here so as not to incur performance penalties
 * in the instrumented parent objects. Writes are arranged so as to be protected by pre-existing
 * locks in the parent where appropriate, but reads have no such guarantee. Counters should
 * therefore be regarded as providing an indication of current state, rather than precisely
 * accurate statistics.
 */

	static inline void
	setCounterType(IOEventSourceCounter *counter, IOStatisticsCounterType type)
	{
		if (counter) {
			counter->type = type;
		}
	}

	static inline void
	countOpenGate(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
			counter->openGateCalls++;
		}
	}

	static inline void
	countCloseGate(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->startTimeStamp = mach_absolute_time();
			counter->closeGateCalls++;
		}
	}

/* Interrupt */
	static inline void
	countInterruptCheckForWork(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->u.interrupt.checksForWork++;
		}
	}

	static inline void
	countInterrupt(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->u.interrupt.produced++;
		}
	}

/* CommandQueue */
	static inline void
	countCommandQueueActionCall(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->u.commandQueue.actionCalls++;
		}
	}

/* CommandGate */
	static inline void
	countCommandGateActionCall(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->u.commandGate.actionCalls++;
		}
	}

/* Timer */
	static inline void
	countTimerTimeout(IOEventSourceCounter *counter)
	{
		if (counter) {
			counter->u.timer.timeouts++;
		}
	}

/* WorkLoop */
	static void attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc);
	static void detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc);

	static inline void
	countWorkLoopOpenGate(IOWorkLoopCounter *counter)
	{
		if (counter) {
			counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
			counter->openGateCalls++;
		}
	}

	static inline void
	countWorkLoopCloseGate(IOWorkLoopCounter *counter)
	{
		if (counter) {
			counter->startTimeStamp = mach_absolute_time();
			counter->closeGateCalls++;
		}
	}

/* IOLib allocations */
	static void countAlloc(uint32_t index, vm_size_t size);

/* UserClient */
	static void countUserClientCall(IOUserClient *client);
};

#else

/* Statistics disabled */

class IOStatistics {
public:
	static void
	initialize()
	{
	}
};

#endif /* IOKITSTATS */

#endif /* __IOKIT_STATISTICS_PRIVATE_H */
Commit	Line	Data
6d2010ae A	1	/*
	2	* Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
6d2010ae A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
6d2010ae A	15	* Please obtain a copy of the License at
6d2010ae A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
6d2010ae A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
6d2010ae A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#ifndef __IOKIT_STATISTICS_PRIVATE_H
	30	#define __IOKIT_STATISTICS_PRIVATE_H
	31
	32	#if IOKITSTATS
	33
	34	#include <sys/queue.h>
	35	#include <sys/tree.h>
	36
	37	#include <libkern/c++/OSKext.h>
	38	#include <libkern/OSDebug.h>
	39
	40	#include <IOKit/IOMemoryDescriptor.h>
	41	#include <IOKit/IOStatistics.h>
	42
	43	#ifndef KERNEL
	44	#error IOStatisticsPrivate.h is for kernel use only
	45	#endif
	46
	47	/* Defines */
	48	#define IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS 20
	49
	50	#ifndef __probable
	51	#define __probable(x) x
	52	#endif
	53
	54	/* Forward declarations */
	55	class IOWorkLoop;
	56	class IOUserClient;
	57	class IOEventSource;
	58
	59	struct IOEventSourceCounter;
	60	struct IOUserClientCounter;
	61	struct IOWorkLoopCounter;
	62	struct IOUserClientProcessEntry;
	63
	64	struct KextNode;
	65
	66	/* Allocation tracking */
	67
	68	enum {
	69	kIOStatisticsMalloc = 0,
	70	kIOStatisticsFree,
	71	kIOStatisticsMallocAligned,
	72	kIOStatisticsFreeAligned,
	73	kIOStatisticsMallocContiguous,
	74	kIOStatisticsFreeContiguous,
	75	kIOStatisticsMallocPageable,
	76	kIOStatisticsFreePageable,
	77	kIOStatisticsAllocCount
	78	};
	79
	80	TAILQ_HEAD(ProcessEntryList, IOUserClientProcessEntry);
	81
	82	/* Tree and list structs */
	83
	84	typedef struct ClassNode {
	85	RB_ENTRY(ClassNode) tLink;
	86	SLIST_ENTRY(ClassNode) lLink;
	87	struct KextNode *parentKext;
	88	uint32_t classID;
	89	uint32_t superClassID;
90	const OSMetaClass *metaClass;
91	SLIST_HEAD(, IOEventSourceCounter) counterList;
92	SLIST_HEAD(, IOUserClientCounter) userClientList;
93	} ClassNode;
94
95	typedef struct KextNode {
96	RB_ENTRY(KextNode) link;
97	RB_ENTRY(KextNode) addressLink;
98	OSKext *kext;
99	OSKextLoadTag loadTag;
100	vm_offset_t address;
101	vm_offset_t address_end;
102	uint32_t memoryCounters[kIOStatisticsAllocCount];
103	uint32_t classes;
104	SLIST_HEAD(, ClassNode) classList;
105	SLIST_HEAD(, IOWorkLoopCounter) workLoopList;
106	ProcessEntryList userClientCallList;
107	} KextNode;
108
109	/* User client tracing */
110
111	typedef struct IOUserClientProcessEntry {
112	TAILQ_ENTRY(IOUserClientProcessEntry) link;
113	char processName[kIOStatisticsProcessNameLength];
114	int32_t pid;
115	uint32_t calls;
116	} IOUserClientProcessEntry;
117
118	/* Counters */
119
120	typedef struct IOInterruptEventSourceCounter {
121	uint32_t produced;
122	uint32_t checksForWork;
123	} IOInterruptEventSourceCounter;
124
125	typedef struct IOTimerEventSourceCounter {
126	uint32_t timeouts;
127	uint32_t checksForWork;
128	} IOTimerEventSourceCounter;
129
130	typedef struct IOCommandGateCounter {
131	uint32_t actionCalls;
132	} IOCommandGateCounter;
133
134	typedef struct IOCommandQueueCounter {
135	uint32_t actionCalls;
136	} IOCommandQueueCounter;
137
138	typedef struct IOEventSourceCounter {
139	SLIST_ENTRY(IOEventSourceCounter) link;
140	ClassNode *parentClass;
141	IOStatisticsCounterType type;
142	uint64_t startTimeStamp;
143	uint64_t timeOnGate;
144	uint32_t closeGateCalls;
145	uint32_t openGateCalls;
146	union {
147	IOInterruptEventSourceCounter interrupt;
148	IOInterruptEventSourceCounter filter;
149	IOTimerEventSourceCounter timer;
150	IOCommandGateCounter commandGate;
151	IOCommandQueueCounter commandQueue;
152	} u;
153	} IOEventSourceCounter;
154
155	typedef struct IOWorkLoopDependency {
156	RB_ENTRY(IOWorkLoopDependency) link;
157	OSKextLoadTag loadTag;
158	} IOWorkLoopDependency;
159
0a7de745	160	typedef struct IOWorkLoopCounter {
6d2010ae A	161	SLIST_ENTRY(IOWorkLoopCounter) link;
	162	KextNode *parentKext;
	163	int attachedEventSources;
	164	IOWorkLoop *workLoop;
	165	uint64_t startTimeStamp;
0a7de745	166	uint64_t timeOnGate;
6d2010ae A	167	uint32_t closeGateCalls;
	168	uint32_t openGateCalls;
	169	typedef RB_HEAD(DependencyTree, IOWorkLoopDependency) DependencyTreeHead;
	170	DependencyTreeHead dependencyHead;
	171	static int loadTagCompare(IOWorkLoopDependency e1, IOWorkLoopDependency e2);
	172	RB_PROTOTYPE_SC(static, DependencyTree, IOWorkLoopDependency, dependencyLink, KextTagCompare);
	173	} IOWorkLoopCounter;
	174
	175	typedef struct IOUserClientCounter {
	176	SLIST_ENTRY(IOUserClientCounter) link;
	177	ClassNode *parentClass;
	178	uint32_t clientCalls;
	179	} IOUserClientCounter;
	180
	181	class IOStatistics {
	182	static bool enabled;
	183
	184	static IORWLock *lock;
	185
	186	static uint32_t sequenceID;
	187
	188	static uint32_t lastKextIndex;
	189	static uint32_t lastClassIndex;
	190
	191	static uint32_t loadedKexts;
	192	static uint32_t registeredClasses;
	193	static uint32_t registeredCounters;
	194	static uint32_t registeredWorkloops;
	195
	196	static uint32_t attachedEventSources;
	197
	198	static KextNode *kextHint;
0a7de745	199
6d2010ae A	200	static IOWorkLoopDependency *nextWorkLoopDependency;
	201
	202	typedef RB_HEAD(KextTree, KextNode) KextTreeHead;
	203	static KextTreeHead kextHead;
	204	static int kextNodeCompare(KextNode e1, KextNode e2);
	205	RB_PROTOTYPE_SC(static, KextTree, KextNode, link, kextNodeCompare);
	206
	207	typedef RB_HEAD(KextAddressTree, KextNode) KextAddressTreeHead;
	208	static KextAddressTreeHead kextAddressHead;
	209	static int kextAddressNodeCompare(KextNode e1, KextNode e2);
	210	RB_PROTOTYPE_SC(static, KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
	211
	212	typedef RB_HEAD(ClassTree, ClassNode) ClassTreeHead;
	213	static ClassTreeHead classHead;
	214	static int classNodeCompare(ClassNode e1, ClassNode e2);
	215	RB_PROTOTYPE_SC(static, ClassTree, ClassNode, tLink, classNodeCompare);
	216
	217	static int oid_sysctl(__unused struct sysctl_oid oidp, __unused void arg1, int arg2, struct sysctl_req *req);
	218
	219	static uint32_t copyGlobalStatistics(IOStatisticsGlobal *stats);
	220	static uint32_t copyKextStatistics(IOStatisticsKext *stats);
	221	static uint32_t copyMemoryStatistics(IOStatisticsMemory *stats);
	222	static uint32_t copyClassStatistics(IOStatisticsClass *stats);
	223	static uint32_t copyCounterStatistics(IOStatisticsCounter *stats);
	224	static uint32_t copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs);
	225	static uint32_t copyClassNames(IOStatisticsClassName *classNames);
	226
	227	static uint32_t copyWorkLoopStatistics(IOStatisticsWorkLoop *workLoopStats);
	228
	229	static uint32_t copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag);
	230
	231	static void updateAllocationCounter(vm_offset_t address, uint32_t index, vm_size_t size);
	232
	233	static void storeUserClientCallInfo(IOUserClient userClient, IOUserClientCounter counter);
	234
	235	static KextNode *getKextNodeFromBacktrace(boolean_t write);
	236	static void releaseKextNode(KextNode *node);
	237
	238	public:
0a7de745	239
6d2010ae A	240	static void initialize();
6d2010ae A	241
c3c9b80d A	242	inline static bool
	243	isEnabled()
	244	{
	245	return enabled;
	246	}
	247
6d2010ae A	248	static void onKextLoad(OSKext kext, kmod_info_t kmod_info);
	249	static void onKextUnload(OSKext *kext);
	250	static void onClassAdded(OSKext parentKext, OSMetaClass metaClass);
	251	static void onClassRemoved(OSKext parentKext, OSMetaClass metaClass);
	252
	253	static IOEventSourceCounter registerEventSource(OSObject inOwner);
	254	static void unregisterEventSource(IOEventSourceCounter *counter);
0a7de745	255
6d2010ae A	256	static IOWorkLoopCounter registerWorkLoop(IOWorkLoop workLoop);
	257	static void unregisterWorkLoop(IOWorkLoopCounter *counter);
	258
	259	static IOUserClientCounter registerUserClient(IOUserClient userClient);
	260	static void unregisterUserClient(IOUserClientCounter *counter);
	261
	262	static int getStatistics(sysctl_req *req);
	263	static int getWorkLoopStatistics(sysctl_req *req);
	264	static int getUserClientStatistics(sysctl_req *req);
	265
0a7de745 A	266	/* Inlines for counter manipulation.
	267	*
	268	* NOTE: counter access is not expressly guarded here so as not to incur performance penalties
	269	* in the instrumented parent objects. Writes are arranged so as to be protected by pre-existing
	270	* locks in the parent where appropriate, but reads have no such guarantee. Counters should
	271	* therefore be regarded as providing an indication of current state, rather than precisely
	272	* accurate statistics.
	273	*/
	274
	275	static inline void
	276	setCounterType(IOEventSourceCounter *counter, IOStatisticsCounterType type)
	277	{
6d2010ae A	278	if (counter) {
	279	counter->type = type;
	280	}
	281	}
	282
0a7de745 A	283	static inline void
	284	countOpenGate(IOEventSourceCounter *counter)
	285	{
6d2010ae A	286	if (counter) {
	287	counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
	288	counter->openGateCalls++;
	289	}
	290	}
	291
0a7de745 A	292	static inline void
	293	countCloseGate(IOEventSourceCounter *counter)
	294	{
6d2010ae A	295	if (counter) {
	296	counter->startTimeStamp = mach_absolute_time();
	297	counter->closeGateCalls++;
	298	}
	299	}
	300
0a7de745 A	301	/* Interrupt */
	302	static inline void
	303	countInterruptCheckForWork(IOEventSourceCounter *counter)
	304	{
6d2010ae A	305	if (counter) {
	306	counter->u.interrupt.checksForWork++;
	307	}
	308	}
	309
0a7de745 A	310	static inline void
	311	countInterrupt(IOEventSourceCounter *counter)
	312	{
6d2010ae A	313	if (counter) {
	314	counter->u.interrupt.produced++;
	315	}
	316	}
	317
0a7de745 A	318	/* CommandQueue */
	319	static inline void
	320	countCommandQueueActionCall(IOEventSourceCounter *counter)
	321	{
6d2010ae A	322	if (counter) {
	323	counter->u.commandQueue.actionCalls++;
	324	}
	325	}
	326
0a7de745 A	327	/* CommandGate */
	328	static inline void
	329	countCommandGateActionCall(IOEventSourceCounter *counter)
	330	{
6d2010ae A	331	if (counter) {
	332	counter->u.commandGate.actionCalls++;
	333	}
	334	}
	335
0a7de745 A	336	/* Timer */
	337	static inline void
	338	countTimerTimeout(IOEventSourceCounter *counter)
	339	{
6d2010ae A	340	if (counter) {
	341	counter->u.timer.timeouts++;
	342	}
	343	}
	344
0a7de745	345	/* WorkLoop */
6d2010ae A	346	static void attachWorkLoopEventSource(IOWorkLoopCounter wlc, IOEventSourceCounter esc);
	347	static void detachWorkLoopEventSource(IOWorkLoopCounter wlc, IOEventSourceCounter esc);
	348
0a7de745 A	349	static inline void
	350	countWorkLoopOpenGate(IOWorkLoopCounter *counter)
	351	{
6d2010ae A	352	if (counter) {
	353	counter->timeOnGate += mach_absolute_time() - counter->startTimeStamp;
	354	counter->openGateCalls++;
	355	}
	356	}
	357
0a7de745 A	358	static inline void
	359	countWorkLoopCloseGate(IOWorkLoopCounter *counter)
	360	{
6d2010ae A	361	if (counter) {
	362	counter->startTimeStamp = mach_absolute_time();
	363	counter->closeGateCalls++;
	364	}
	365	}
	366
0a7de745	367	/* IOLib allocations */
6d2010ae A	368	static void countAlloc(uint32_t index, vm_size_t size);
6d2010ae A	369
0a7de745	370	/* UserClient */
6d2010ae A	371	static void countUserClientCall(IOUserClient *client);
	372	};
	373
	374	#else
	375
	376	/* Statistics disabled */
	377
	378	class IOStatistics {
	379	public:
0a7de745 A	380	static void
	381	initialize()
	382	{
	383	}
6d2010ae A	384	};
	385
	386	#endif /* IOKITSTATS */
	387
	388	#endif /* __IOKIT_STATISTICS_PRIVATE_H */