[apple/xnu.git] / iokit / IOKit / perfcontrol / IOPerfControl.h

/*
 * Copyright (c) 2017 Apple Inc. All rights reserved.
 */

#pragma once

#ifdef KERNEL_PRIVATE
#ifdef __cplusplus

#include <IOKit/IOService.h>
#include <stdatomic.h>
#include <kern/bits.h>

struct thread_group;

enum{
	kIOPerfControlClientWorkUntracked = 0,
};

/*!
 * @class IOPerfControlClient : public OSObject
 * @abstract Class which implements an interface allowing device drivers to participate in performance control.
 * @discussion TODO
 */
class IOPerfControlClient final : public OSObject
{
	OSDeclareDefaultStructors(IOPerfControlClient);

protected:
	virtual bool init(IOService *driver, uint64_t maxWorkCapacity);

public:
/*!
 * @function copyClient
 * @abstract Return a retained reference to a client object, to be released by the driver. It may be
 * shared with other drivers in the system.
 * @param driver The device driver that will be using this interface.
 * @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver.
 * @returns An instance of IOPerfControlClient.
 */
	static IOPerfControlClient *copyClient(IOService *driver, uint64_t maxWorkCapacity);

/*!
 * @function registerDevice
 * @abstract Inform the system that work will be dispatched to a device in the future.
 * @discussion The system will do some one-time setup work associated with the device, and may block the
 * current thread during the setup. Devices should not be passed to work workSubmit, workSubmitAndBegin,
 * workBegin, or workEnd until they have been successfully registered. The unregistration process happens
 * automatically when the device object is deallocated.
 * @param device The device object. Some platforms require device to be a specific subclass of IOService.
 * @returns kIOReturnSuccess or an IOReturn error code
 */
	virtual IOReturn registerDevice(IOService *driver, IOService *device);

/*!
 * @function unregisterDevice
 * @abstract Inform the system that work will be no longer be dispatched to a device in the future.
 * @discussion This call is optional as the unregistration process happens automatically when the device
 * object is deallocated. This call may block the current thread and/or acquire locks. It should not be
 * called until after all submitted work has been ended using workEnd.
 * @param device The device object. Some platforms require device to be a specific subclass of IOService.
 */
	virtual void unregisterDevice(IOService *driver, IOService *device);

/*!
 * @struct WorkSubmitArgs
 * @discussion Drivers may submit additional device-specific arguments related to the submission of a work item
 * by passing a struct with WorkSubmitArgs as its first member. Note: Drivers are responsible for publishing
 * a header file describing these arguments.
 */
	struct WorkSubmitArgs {
		uint32_t version;
		uint32_t size;
		uint64_t submit_time;
		uint64_t reserved[4];
		void *driver_data;
	};

/*!
 * @function workSubmit
 * @abstract Tell the performance controller that work was submitted.
 * @param device The device that will execute the work. Some platforms require device to be a
 * specific subclass of IOService.
 * @param args Optional device-specific arguments related to the submission of this work item.
 * @returns A token representing this work item, which must be passed to workEnd when the work is finished
 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks
 * and a degradation of system performance.
 */
	virtual uint64_t workSubmit(IOService *device, WorkSubmitArgs *args = nullptr);

/*!
 * @struct WorkBeginArgs
 * @discussion Drivers may submit additional device-specific arguments related to the start of a work item
 * by passing a struct with WorkBeginArgs as its first member. Note: Drivers are responsible for publishing
 * a header file describing these arguments.
 */
	struct WorkBeginArgs {
		uint32_t version;
		uint32_t size;
		uint64_t begin_time;
		uint64_t reserved[4];
		void *driver_data;
	};

/*!
 * @function workSubmitAndBegin
 * @abstract Tell the performance controller that work was submitted and immediately began executing.
 * @param device The device that is executing the work. Some platforms require device to be a
 * specific subclass of IOService.
 * @param submitArgs Optional device-specific arguments related to the submission of this work item.
 * @param beginArgs Optional device-specific arguments related to the start of this work item.
 * @returns A token representing this work item, which must be passed to workEnd when the work is finished
 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks
 * and a degradation of system performance.
 */
	virtual uint64_t workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs = nullptr,
	    WorkBeginArgs *beginArgs = nullptr);

/*!
 * @function workBegin
 * @abstract Tell the performance controller that previously submitted work began executing.
 * @param device The device that is executing the work. Some platforms require device to be a
 * specific subclass of IOService.
 * @param args Optional device-specific arguments related to the start of this work item.
 */
	virtual void workBegin(IOService *device, uint64_t token, WorkBeginArgs *args = nullptr);

/*!
 * @struct WorkEndArgs
 * @discussion Drivers may submit additional device-specific arguments related to the end of a work item
 * by passing a struct with WorkEndArgs as its first member. Note: Drivers are responsible for publishing
 * a header file describing these arguments.
 */
	struct WorkEndArgs {
		uint32_t version;
		uint32_t size;
		uint64_t end_time;
		uint64_t reserved[4];
		void *driver_data;
	};

/*!
 * @function workEnd
 * @abstract Tell the performance controller that previously started work finished executing.
 * @param device The device that executed the work. Some platforms require device to be a
 * specific subclass of IOService.
 * @param args Optional device-specific arguments related to the end of this work item.
 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for
 * calling workBegin when the work resumes and workEnd with done set to True when it has completed.
 */
	virtual void workEnd(IOService *device, uint64_t token, WorkEndArgs *args = nullptr, bool done = true);

/*!
 * @struct PerfControllerInterface
 * @discussion Function pointers necessary to register a performance controller. Not for general driver use.
 */
	struct PerfControllerInterface {
		struct WorkState {
			uint64_t thread_group_id;
			void *thread_group_data;
			void *work_data;
			uint32_t work_data_size;
		};

		using RegisterDeviceFunction = IOReturn (*)(IOService *);
		using WorkCanSubmitFunction = bool (*)(IOService *, WorkState *, WorkSubmitArgs *);
		using WorkSubmitFunction = void (*)(IOService *, uint64_t, WorkState *, WorkSubmitArgs *);
		using WorkBeginFunction = void (*)(IOService *, uint64_t, WorkState *, WorkBeginArgs *);
		using WorkEndFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, bool);

		uint64_t version;
		RegisterDeviceFunction registerDevice;
		RegisterDeviceFunction unregisterDevice;
		WorkCanSubmitFunction workCanSubmit;
		WorkSubmitFunction workSubmit;
		WorkBeginFunction workBegin;
		WorkEndFunction workEnd;
	};

	struct IOPerfControlClientShared {
		atomic_uint_fast8_t maxDriverIndex;
		PerfControllerInterface interface;
		IOLock *interfaceLock;
		OSSet *deviceRegistrationList;
	};

/*!
 * @function registerPerformanceController
 * @abstract Register a performance controller to receive callbacks. Not for general driver use.
 * @param interface Struct containing callback functions implemented by the performance controller.
 * @returns kIOReturnSuccess or kIOReturnError if the interface was already registered.
 */
	virtual IOReturn registerPerformanceController(PerfControllerInterface interface);

private:
	struct WorkTableEntry {
		struct thread_group *thread_group;
		bool started;
		uint8_t perfcontrol_data[32];
	};

	static constexpr size_t kMaxWorkTableNumEntries = 1024;
	static constexpr size_t kWorkTableIndexBits = 24;
	static constexpr size_t kWorkTableMaxSize = (1 << kWorkTableIndexBits) - 1; // - 1 since
	// kIOPerfControlClientWorkUntracked takes number 0
	static constexpr size_t kWorkTableIndexMask = mask(kWorkTableIndexBits);

	uint64_t allocateToken(thread_group *thread_group);
	void deallocateToken(uint64_t token);
	bool getEntryForToken(uint64_t token, WorkTableEntry &entry);
	void markEntryStarted(uint64_t token, bool started);
	inline uint64_t tokenToGlobalUniqueToken(uint64_t token);

	uint8_t driverIndex;
	IOPerfControlClientShared *shared;
	WorkTableEntry *workTable;
	size_t workTableLength;
	size_t workTableNextIndex;
	IOSimpleLock *workTableLock;
};

#endif /* __cplusplus */
#endif /* KERNEL_PRIVATE */
Commit	Line	Data
d9a64523 A	1	/*
	2	* Copyright (c) 2017 Apple Inc. All rights reserved.
	3	*/
	4
	5	#pragma once
	6
	7	#ifdef KERNEL_PRIVATE
	8	#ifdef __cplusplus
	9
	10	#include <IOKit/IOService.h>
cb323159 A	11	#include <stdatomic.h>
cb323159 A	12	#include <kern/bits.h>
d9a64523 A	13
	14	struct thread_group;
	15
0a7de745 A	16	enum{
0a7de745 A	17	kIOPerfControlClientWorkUntracked = 0,
d9a64523 A	18	};
	19
	20	/*!
	21	* @class IOPerfControlClient : public OSObject
	22	* @abstract Class which implements an interface allowing device drivers to participate in performance control.
	23	* @discussion TODO
	24	*/
	25	class IOPerfControlClient final : public OSObject
	26	{
0a7de745	27	OSDeclareDefaultStructors(IOPerfControlClient);
d9a64523 A	28
d9a64523 A	29	protected:
0a7de745	30	virtual bool init(IOService *driver, uint64_t maxWorkCapacity);
d9a64523 A	31
d9a64523 A	32	public:
0a7de745 A	33	/*!
	34	* @function copyClient
	35	* @abstract Return a retained reference to a client object, to be released by the driver. It may be
	36	* shared with other drivers in the system.
	37	* @param driver The device driver that will be using this interface.
	38	* @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver.
	39	* @returns An instance of IOPerfControlClient.
	40	*/
	41	static IOPerfControlClient copyClient(IOService driver, uint64_t maxWorkCapacity);
	42
	43	/*!
	44	* @function registerDevice
	45	* @abstract Inform the system that work will be dispatched to a device in the future.
	46	* @discussion The system will do some one-time setup work associated with the device, and may block the
	47	* current thread during the setup. Devices should not be passed to work workSubmit, workSubmitAndBegin,
	48	* workBegin, or workEnd until they have been successfully registered. The unregistration process happens
	49	* automatically when the device object is deallocated.
	50	* @param device The device object. Some platforms require device to be a specific subclass of IOService.
	51	* @returns kIOReturnSuccess or an IOReturn error code
	52	*/
	53	virtual IOReturn registerDevice(IOService driver, IOService device);
	54
	55	/*!
	56	* @function unregisterDevice
	57	* @abstract Inform the system that work will be no longer be dispatched to a device in the future.
	58	* @discussion This call is optional as the unregistration process happens automatically when the device
	59	* object is deallocated. This call may block the current thread and/or acquire locks. It should not be
	60	* called until after all submitted work has been ended using workEnd.
	61	* @param device The device object. Some platforms require device to be a specific subclass of IOService.
	62	*/
	63	virtual void unregisterDevice(IOService driver, IOService device);
	64
	65	/*!
	66	* @struct WorkSubmitArgs
	67	* @discussion Drivers may submit additional device-specific arguments related to the submission of a work item
	68	* by passing a struct with WorkSubmitArgs as its first member. Note: Drivers are responsible for publishing
	69	* a header file describing these arguments.
	70	*/
	71	struct WorkSubmitArgs {
	72	uint32_t version;
	73	uint32_t size;
	74	uint64_t submit_time;
	75	uint64_t reserved[4];
	76	void *driver_data;
	77	};
	78
	79	/*!
	80	* @function workSubmit
	81	* @abstract Tell the performance controller that work was submitted.
	82	* @param device The device that will execute the work. Some platforms require device to be a
	83	* specific subclass of IOService.
	84	* @param args Optional device-specific arguments related to the submission of this work item.
	85	* @returns A token representing this work item, which must be passed to workEnd when the work is finished
	86	* unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks
	87	* and a degradation of system performance.
	88	*/
	89	virtual uint64_t workSubmit(IOService device, WorkSubmitArgs args = nullptr);
	90
	91	/*!
	92	* @struct WorkBeginArgs
	93	* @discussion Drivers may submit additional device-specific arguments related to the start of a work item
	94	* by passing a struct with WorkBeginArgs as its first member. Note: Drivers are responsible for publishing
	95	* a header file describing these arguments.
	96	*/
97	struct WorkBeginArgs {
98	uint32_t version;
99	uint32_t size;
100	uint64_t begin_time;
101	uint64_t reserved[4];
102	void *driver_data;
103	};
104
105	/*!
106	* @function workSubmitAndBegin
107	* @abstract Tell the performance controller that work was submitted and immediately began executing.
108	* @param device The device that is executing the work. Some platforms require device to be a
109	* specific subclass of IOService.
110	* @param submitArgs Optional device-specific arguments related to the submission of this work item.
111	* @param beginArgs Optional device-specific arguments related to the start of this work item.
112	* @returns A token representing this work item, which must be passed to workEnd when the work is finished
113	* unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks
114	* and a degradation of system performance.
115	*/
116	virtual uint64_t workSubmitAndBegin(IOService device, WorkSubmitArgs submitArgs = nullptr,
117	WorkBeginArgs *beginArgs = nullptr);
118
119	/*!
120	* @function workBegin
121	* @abstract Tell the performance controller that previously submitted work began executing.
122	* @param device The device that is executing the work. Some platforms require device to be a
123	* specific subclass of IOService.
124	* @param args Optional device-specific arguments related to the start of this work item.
125	*/
126	virtual void workBegin(IOService device, uint64_t token, WorkBeginArgs args = nullptr);
127
128	/*!
129	* @struct WorkEndArgs
130	* @discussion Drivers may submit additional device-specific arguments related to the end of a work item
131	* by passing a struct with WorkEndArgs as its first member. Note: Drivers are responsible for publishing
132	* a header file describing these arguments.
133	*/
134	struct WorkEndArgs {
135	uint32_t version;
136	uint32_t size;
137	uint64_t end_time;
138	uint64_t reserved[4];
139	void *driver_data;
140	};
141
142	/*!
143	* @function workEnd
144	* @abstract Tell the performance controller that previously started work finished executing.
145	* @param device The device that executed the work. Some platforms require device to be a
146	* specific subclass of IOService.
147	* @param args Optional device-specific arguments related to the end of this work item.
148	* @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for
149	* calling workBegin when the work resumes and workEnd with done set to True when it has completed.
150	*/
151	virtual void workEnd(IOService device, uint64_t token, WorkEndArgs args = nullptr, bool done = true);
152
153	/*!
154	* @struct PerfControllerInterface
155	* @discussion Function pointers necessary to register a performance controller. Not for general driver use.
156	*/
157	struct PerfControllerInterface {
158	struct WorkState {
159	uint64_t thread_group_id;
160	void *thread_group_data;
161	void *work_data;
162	uint32_t work_data_size;
163	};
164
165	using RegisterDeviceFunction = IOReturn ()(IOService );
166	using WorkCanSubmitFunction = bool ()(IOService , WorkState , WorkSubmitArgs );
167	using WorkSubmitFunction = void ()(IOService , uint64_t, WorkState , WorkSubmitArgs );
168	using WorkBeginFunction = void ()(IOService , uint64_t, WorkState , WorkBeginArgs );
169	using WorkEndFunction = void ()(IOService , uint64_t, WorkState , WorkEndArgs , bool);
170
171	uint64_t version;
172	RegisterDeviceFunction registerDevice;
173	RegisterDeviceFunction unregisterDevice;
174	WorkCanSubmitFunction workCanSubmit;
175	WorkSubmitFunction workSubmit;
176	WorkBeginFunction workBegin;
177	WorkEndFunction workEnd;
178	};
179
cb323159 A	180	struct IOPerfControlClientShared {
	181	atomic_uint_fast8_t maxDriverIndex;
	182	PerfControllerInterface interface;
	183	IOLock *interfaceLock;
	184	OSSet *deviceRegistrationList;
	185	};
	186
0a7de745 A	187	/*!
	188	* @function registerPerformanceController
	189	* @abstract Register a performance controller to receive callbacks. Not for general driver use.
	190	* @param interface Struct containing callback functions implemented by the performance controller.
	191	* @returns kIOReturnSuccess or kIOReturnError if the interface was already registered.
	192	*/
	193	virtual IOReturn registerPerformanceController(PerfControllerInterface interface);
d9a64523 A	194
d9a64523 A	195	private:
0a7de745 A	196	struct WorkTableEntry {
	197	struct thread_group *thread_group;
	198	bool started;
	199	uint8_t perfcontrol_data[32];
	200	};
	201
cb323159 A	202	static constexpr size_t kMaxWorkTableNumEntries = 1024;
	203	static constexpr size_t kWorkTableIndexBits = 24;
	204	static constexpr size_t kWorkTableMaxSize = (1 << kWorkTableIndexBits) - 1; // - 1 since
	205	// kIOPerfControlClientWorkUntracked takes number 0
	206	static constexpr size_t kWorkTableIndexMask = mask(kWorkTableIndexBits);
0a7de745 A	207
	208	uint64_t allocateToken(thread_group *thread_group);
	209	void deallocateToken(uint64_t token);
	210	bool getEntryForToken(uint64_t token, WorkTableEntry &entry);
	211	void markEntryStarted(uint64_t token, bool started);
cb323159	212	inline uint64_t tokenToGlobalUniqueToken(uint64_t token);
0a7de745	213
cb323159 A	214	uint8_t driverIndex;
	215	IOPerfControlClientShared *shared;
	216	WorkTableEntry *workTable;
	217	size_t workTableLength;
0a7de745 A	218	size_t workTableNextIndex;
0a7de745 A	219	IOSimpleLock *workTableLock;
d9a64523 A	220	};
	221
	222	#endif /* __cplusplus */
	223	#endif /* KERNEL_PRIVATE */