#define super OSObject
OSDefineMetaClassAndStructors(IOPerfControlClient, OSObject);
+static IOPerfControlClient::IOPerfControlClientShared *_Atomic gIOPerfControlClientShared;
+
bool
IOPerfControlClient::init(IOService *driver, uint64_t maxWorkCapacity)
{
+ // TODO: Remove this limit and implement dynamic table growth if workloads are found that exceed this
+ if (maxWorkCapacity > kMaxWorkTableNumEntries) {
+ maxWorkCapacity = kMaxWorkTableNumEntries;
+ }
+
if (!super::init()) {
return false;
}
- interface = PerfControllerInterface{
- .version = 0,
- .registerDevice =
- [](IOService *device) {
- return kIOReturnSuccess;
- },
- .unregisterDevice =
+ shared = atomic_load_explicit(&gIOPerfControlClientShared, memory_order_acquire);
+ if (shared == nullptr) {
+ IOPerfControlClient::IOPerfControlClientShared *expected = shared;
+ shared = reinterpret_cast<IOPerfControlClient::IOPerfControlClientShared*>(kalloc(sizeof(IOPerfControlClientShared)));
+ if (!shared) {
+ return false;
+ }
+
+ atomic_init(&shared->maxDriverIndex, 0);
+
+ shared->interface = PerfControllerInterface{
+ .version = 0,
+ .registerDevice =
[](IOService *device) {
return kIOReturnSuccess;
},
- .workCanSubmit =
- [](IOService *device, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
- return false;
- },
- .workSubmit =
- [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
- },
- .workBegin =
- [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkBeginArgs *args) {
- },
- .workEnd =
- [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
- },
- };
+ .unregisterDevice =
+ [](IOService *device) {
+ return kIOReturnSuccess;
+ },
+ .workCanSubmit =
+ [](IOService *device, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
+ return false;
+ },
+ .workSubmit =
+ [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
+ },
+ .workBegin =
+ [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkBeginArgs *args) {
+ },
+ .workEnd =
+ [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
+ },
+ };
+
+ shared->interfaceLock = IOLockAlloc();
+ if (!shared->interfaceLock) {
+ goto shared_init_error;
+ }
- interfaceLock = IOLockAlloc();
- if (!interfaceLock) {
- goto error;
- }
+ shared->deviceRegistrationList = OSSet::withCapacity(4);
+ if (!shared->deviceRegistrationList) {
+ goto shared_init_error;
+ }
- deviceRegistrationList = OSSet::withCapacity(4);
- if (!deviceRegistrationList) {
- goto error;
+ if (!atomic_compare_exchange_strong_explicit(&gIOPerfControlClientShared, &expected, shared, memory_order_acq_rel,
+ memory_order_acquire)) {
+ IOLockFree(shared->interfaceLock);
+ shared->deviceRegistrationList->release();
+ kfree(shared, sizeof(*shared));
+ shared = expected;
+ }
}
- bzero(workTable, sizeof(workTable));
- memset(&workTable[kIOPerfControlClientWorkUntracked], ~0, sizeof(WorkTableEntry));
- workTableNextIndex = kIOPerfControlClientWorkUntracked + 1;
+ driverIndex = atomic_fetch_add_explicit(&shared->maxDriverIndex, 1, memory_order_relaxed) + 1;
+ assertf(driverIndex != 0, "Overflow in driverIndex. Too many IOPerfControlClients created.\n");
- workTableLock = IOSimpleLockAlloc();
- if (!workTableLock) {
- goto error;
- }
+ // + 1 since index 0 is unused for kIOPerfControlClientWorkUntracked
+ workTableLength = maxWorkCapacity + 1;
+ assertf(workTableLength <= kWorkTableMaxSize, "%zu exceeds max allowed capacity of %zu", workTableLength, kWorkTableMaxSize);
+ if (maxWorkCapacity > 0) {
+ workTable = reinterpret_cast<WorkTableEntry*>(kalloc(workTableLength * sizeof(WorkTableEntry)));
+ if (!workTable) {
+ goto error;
+ }
+ bzero(workTable, workTableLength * sizeof(WorkTableEntry));
+ workTableNextIndex = 1;
- // TODO: check sum(maxWorkCapacities) < table size
+ workTableLock = IOSimpleLockAlloc();
+ if (!workTableLock) {
+ goto error;
+ }
+ }
return true;
error:
- if (interfaceLock) {
- IOLockFree(interfaceLock);
- }
- if (deviceRegistrationList) {
- deviceRegistrationList->release();
+ if (workTable) {
+ kfree(workTable, maxWorkCapacity * sizeof(WorkTableEntry));
}
if (workTableLock) {
IOSimpleLockFree(workTableLock);
}
return false;
+shared_init_error:
+ if (shared) {
+ if (shared->interfaceLock) {
+ IOLockFree(shared->interfaceLock);
+ }
+ if (shared->deviceRegistrationList) {
+ shared->deviceRegistrationList->release();
+ }
+ kfree(shared, sizeof(*shared));
+ shared = nullptr;
+ }
+ return false;
}
-IOPerfControlClient *_Atomic gSharedClient = nullptr;
-
IOPerfControlClient *
IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
{
- IOPerfControlClient *client = atomic_load_explicit(&gSharedClient, memory_order_acquire);
- if (client == nullptr) {
- IOPerfControlClient *expected = client;
- client = new IOPerfControlClient;
- if (!client || !client->init(driver, maxWorkCapacity)) {
- panic("could not create IOPerfControlClient");
- }
- if (!atomic_compare_exchange_strong_explicit(&gSharedClient, &expected, client, memory_order_acq_rel,
- memory_order_acquire)) {
- client->release();
- client = expected;
- }
+ IOPerfControlClient *client = new IOPerfControlClient;
+ if (!client || !client->init(driver, maxWorkCapacity)) {
+ panic("could not create IOPerfControlClient");
}
- // TODO: add maxWorkCapacity to existing client
- client->retain();
return client;
}
+/* Convert the per driver token into a globally unique token for the performance
+ * controller's consumption. This is achieved by setting the driver's unique
+ * index onto the high order bits. The performance controller is shared between
+ * all drivers and must track all instances separately, while each driver has
+ * its own token table, so this step is needed to avoid token collisions between
+ * drivers.
+ */
+inline uint64_t
+IOPerfControlClient::tokenToGlobalUniqueToken(uint64_t token)
+{
+ return token | (static_cast<uint64_t>(driverIndex) << kWorkTableIndexBits);
+}
+
+/* With this implementation, tokens returned to the driver differ from tokens
+ * passed to the performance controller. This implementation has the nice
+ * property that tokens returns to the driver will aways be between 1 and
+ * the value of maxWorkCapacity passed by the driver to copyClient. The tokens
+ * the performance controller sees will match on the lower order bits and have
+ * the driver index set on the high order bits.
+ */
uint64_t
IOPerfControlClient::allocateToken(thread_group *thread_group)
{
return false;
}
- if (token >= kWorkTableNumEntries) {
+ if (token >= workTableLength) {
panic("Invalid work token (%llu): index out of bounds.", token);
}
return;
}
- if (token >= kWorkTableNumEntries) {
+ if (token >= workTableLength) {
panic("Invalid work token (%llu): index out of bounds.", token);
}
{
IOReturn ret = kIOReturnSuccess;
- IOLockLock(interfaceLock);
+ IOLockLock(shared->interfaceLock);
- if (interface.version > 0) {
- ret = interface.registerDevice(device);
+ if (shared->interface.version > 0) {
+ ret = shared->interface.registerDevice(device);
} else {
- deviceRegistrationList->setObject(device);
+ shared->deviceRegistrationList->setObject(device);
}
- IOLockUnlock(interfaceLock);
+ IOLockUnlock(shared->interfaceLock);
return ret;
}
void
IOPerfControlClient::unregisterDevice(__unused IOService *driver, IOService *device)
{
- IOLockLock(interfaceLock);
+ IOLockLock(shared->interfaceLock);
- if (interface.version > 0) {
- interface.unregisterDevice(device);
+ if (shared->interface.version > 0) {
+ shared->interface.unregisterDevice(device);
} else {
- deviceRegistrationList->removeObject(device);
+ shared->deviceRegistrationList->removeObject(device);
}
- IOLockUnlock(interfaceLock);
+ IOLockUnlock(shared->interfaceLock);
}
uint64_t
{
IOReturn result = kIOReturnError;
- IOLockLock(interfaceLock);
+ IOLockLock(shared->interfaceLock);
- if (interface.version == 0 && pci.version > 0) {
+ if (shared->interface.version == 0 && pci.version > 0) {
assert(pci.registerDevice && pci.unregisterDevice && pci.workCanSubmit && pci.workSubmit && pci.workBegin && pci.workEnd);
result = kIOReturnSuccess;
OSObject *obj;
- while ((obj = deviceRegistrationList->getAnyObject())) {
+ while ((obj = shared->deviceRegistrationList->getAnyObject())) {
IOService *device = OSDynamicCast(IOService, obj);
if (device) {
pci.registerDevice(device);
}
- deviceRegistrationList->removeObject(obj);
+ shared->deviceRegistrationList->removeObject(obj);
}
- interface = pci;
+ shared->interface = pci;
}
- IOLockUnlock(interfaceLock);
+ IOLockUnlock(shared->interfaceLock);
return result;
}
projects / apple / xnu.git / blobdiff