xnu-6153.41.3.tar.gz

[apple/xnu.git] / iokit / Kernel / IOPerfControl.cpp

diff --git a/iokit/Kernel/IOPerfControl.cpp b/iokit/Kernel/IOPerfControl.cpp
index f90699c34774610f7d08093d1272caab54909b36..b3d1a5aac617b9be8c766f8a641a76e07865edc9 100644 (file)
--- a/iokit/Kernel/IOPerfControl.cpp
+++ b/iokit/Kernel/IOPerfControl.cpp
@@ -12,97 +12,148 @@
 #define super OSObject
 OSDefineMetaClassAndStructors(IOPerfControlClient, OSObject);
 
 #define super OSObject
 OSDefineMetaClassAndStructors(IOPerfControlClient, OSObject);
 

+static IOPerfControlClient::IOPerfControlClientShared *_Atomic gIOPerfControlClientShared;
+

 bool
 IOPerfControlClient::init(IOService *driver, uint64_t maxWorkCapacity)
 {
 bool
 IOPerfControlClient::init(IOService *driver, uint64_t maxWorkCapacity)
 {

+       // TODO: Remove this limit and implement dynamic table growth if workloads are found that exceed this
+       if (maxWorkCapacity > kMaxWorkTableNumEntries) {
+               maxWorkCapacity = kMaxWorkTableNumEntries;
+       }
+

        if (!super::init()) {
                return false;
        }
 
        if (!super::init()) {
                return false;
        }
 

-       interface = PerfControllerInterface{
-               .version = 0,
-               .registerDevice =
-                   [](IOService *device) {
-                           return kIOReturnSuccess;
-                   },
-               .unregisterDevice =
+       shared = atomic_load_explicit(&gIOPerfControlClientShared, memory_order_acquire);
+       if (shared == nullptr) {
+               IOPerfControlClient::IOPerfControlClientShared *expected = shared;
+               shared = reinterpret_cast<IOPerfControlClient::IOPerfControlClientShared*>(kalloc(sizeof(IOPerfControlClientShared)));
+               if (!shared) {
+                       return false;
+               }
+
+               atomic_init(&shared->maxDriverIndex, 0);
+
+               shared->interface = PerfControllerInterface{
+                       .version = 0,
+                       .registerDevice =

                    [](IOService *device) {
                            return kIOReturnSuccess;
                    },
                    [](IOService *device) {
                            return kIOReturnSuccess;
                    },

-               .workCanSubmit =
-                   [](IOService *device, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
-                           return false;
-                   },
-               .workSubmit =
-                   [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
-                   },
-               .workBegin =
-                   [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkBeginArgs *args) {
-                   },
-               .workEnd =
-                   [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
-                   },
-       };
+                       .unregisterDevice =
+                           [](IOService *device) {
+                                   return kIOReturnSuccess;
+                           },
+                       .workCanSubmit =
+                           [](IOService *device, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
+                                   return false;
+                           },
+                       .workSubmit =
+                           [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
+                           },
+                       .workBegin =
+                           [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkBeginArgs *args) {
+                           },
+                       .workEnd =
+                           [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
+                           },
+               };
+
+               shared->interfaceLock = IOLockAlloc();
+               if (!shared->interfaceLock) {
+                       goto shared_init_error;
+               }

 
 

-       interfaceLock = IOLockAlloc();
-       if (!interfaceLock) {
-               goto error;
-       }
+               shared->deviceRegistrationList = OSSet::withCapacity(4);
+               if (!shared->deviceRegistrationList) {
+                       goto shared_init_error;
+               }

 
 

-       deviceRegistrationList = OSSet::withCapacity(4);
-       if (!deviceRegistrationList) {
-               goto error;
+               if (!atomic_compare_exchange_strong_explicit(&gIOPerfControlClientShared, &expected, shared, memory_order_acq_rel,
+                   memory_order_acquire)) {
+                       IOLockFree(shared->interfaceLock);
+                       shared->deviceRegistrationList->release();
+                       kfree(shared, sizeof(*shared));
+                       shared = expected;
+               }

        }
 
        }
 

-       bzero(workTable, sizeof(workTable));
-       memset(&workTable[kIOPerfControlClientWorkUntracked], ~0, sizeof(WorkTableEntry));
-       workTableNextIndex = kIOPerfControlClientWorkUntracked + 1;
+       driverIndex = atomic_fetch_add_explicit(&shared->maxDriverIndex, 1, memory_order_relaxed) + 1;
+       assertf(driverIndex != 0, "Overflow in driverIndex. Too many IOPerfControlClients created.\n");

 
 

-       workTableLock = IOSimpleLockAlloc();
-       if (!workTableLock) {
-               goto error;
-       }
+       // + 1 since index 0 is unused for kIOPerfControlClientWorkUntracked
+       workTableLength = maxWorkCapacity + 1;
+       assertf(workTableLength <= kWorkTableMaxSize, "%zu exceeds max allowed capacity of %zu", workTableLength, kWorkTableMaxSize);
+       if (maxWorkCapacity > 0) {
+               workTable = reinterpret_cast<WorkTableEntry*>(kalloc(workTableLength * sizeof(WorkTableEntry)));
+               if (!workTable) {
+                       goto error;
+               }
+               bzero(workTable, workTableLength * sizeof(WorkTableEntry));
+               workTableNextIndex = 1;

 
 

-       // TODO: check sum(maxWorkCapacities) < table size
+               workTableLock = IOSimpleLockAlloc();
+               if (!workTableLock) {
+                       goto error;
+               }
+       }

 
        return true;
 
 error:
 
        return true;
 
 error:

-       if (interfaceLock) {
-               IOLockFree(interfaceLock);
-       }
-       if (deviceRegistrationList) {
-               deviceRegistrationList->release();
+       if (workTable) {
+               kfree(workTable, maxWorkCapacity * sizeof(WorkTableEntry));

        }
        if (workTableLock) {
                IOSimpleLockFree(workTableLock);
        }
        return false;
        }
        if (workTableLock) {
                IOSimpleLockFree(workTableLock);
        }
        return false;

+shared_init_error:
+       if (shared) {
+               if (shared->interfaceLock) {
+                       IOLockFree(shared->interfaceLock);
+               }
+               if (shared->deviceRegistrationList) {
+                       shared->deviceRegistrationList->release();
+               }
+               kfree(shared, sizeof(*shared));
+               shared = nullptr;
+       }
+       return false;

 }
 
 }
 

-IOPerfControlClient *_Atomic gSharedClient = nullptr;
-

 IOPerfControlClient *
 IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
 {
 IOPerfControlClient *
 IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
 {

-       IOPerfControlClient *client = atomic_load_explicit(&gSharedClient, memory_order_acquire);
-       if (client == nullptr) {
-               IOPerfControlClient *expected = client;
-               client = new IOPerfControlClient;
-               if (!client || !client->init(driver, maxWorkCapacity)) {
-                       panic("could not create IOPerfControlClient");
-               }
-               if (!atomic_compare_exchange_strong_explicit(&gSharedClient, &expected, client, memory_order_acq_rel,
-                   memory_order_acquire)) {
-                       client->release();
-                       client = expected;
-               }
+       IOPerfControlClient *client = new IOPerfControlClient;
+       if (!client || !client->init(driver, maxWorkCapacity)) {
+               panic("could not create IOPerfControlClient");

        }
        }

-       // TODO: add maxWorkCapacity to existing client
-       client->retain();

        return client;
 }
 
        return client;
 }
 

+/* Convert the per driver token into a globally unique token for the performance
+ * controller's consumption. This is achieved by setting the driver's unique
+ * index onto the high order bits. The performance controller is shared between
+ * all drivers and must track all instances separately, while each driver has
+ * its own token table, so this step is needed to avoid token collisions between
+ * drivers.
+ */
+inline uint64_t
+IOPerfControlClient::tokenToGlobalUniqueToken(uint64_t token)
+{
+       return token | (static_cast<uint64_t>(driverIndex) << kWorkTableIndexBits);
+}
+
+/* With this implementation, tokens returned to the driver differ from tokens
+ * passed to the performance controller. This implementation has the nice
+ * property that tokens returns to the driver will aways be between 1 and
+ * the value of maxWorkCapacity passed by the driver to copyClient. The tokens
+ * the performance controller sees will match on the lower order bits and have
+ * the driver index set on the high order bits.
+ */

 uint64_t
 IOPerfControlClient::allocateToken(thread_group *thread_group)
 {
 uint64_t
 IOPerfControlClient::allocateToken(thread_group *thread_group)
 {


@@ -124,7 +175,7 @@ IOPerfControlClient::getEntryForToken(uint64_t token, IOPerfControlClient::WorkT
                return false;
        }
 
                return false;
        }
 

-       if (token >= kWorkTableNumEntries) {
+       if (token >= workTableLength) {

                panic("Invalid work token (%llu): index out of bounds.", token);
        }
 
                panic("Invalid work token (%llu): index out of bounds.", token);
        }
 


@@ -141,7 +192,7 @@ IOPerfControlClient::markEntryStarted(uint64_t token, bool started)
                return;
        }
 
                return;
        }
 

-       if (token >= kWorkTableNumEntries) {
+       if (token >= workTableLength) {

                panic("Invalid work token (%llu): index out of bounds.", token);
        }
 
                panic("Invalid work token (%llu): index out of bounds.", token);
        }
 


@@ -153,15 +204,15 @@ IOPerfControlClient::registerDevice(__unused IOService *driver, IOService *devic
 {
        IOReturn ret = kIOReturnSuccess;
 
 {
        IOReturn ret = kIOReturnSuccess;
 

-       IOLockLock(interfaceLock);
+       IOLockLock(shared->interfaceLock);

 
 

-       if (interface.version > 0) {
-               ret = interface.registerDevice(device);
+       if (shared->interface.version > 0) {
+               ret = shared->interface.registerDevice(device);

        } else {
        } else {

-               deviceRegistrationList->setObject(device);
+               shared->deviceRegistrationList->setObject(device);

        }
 
        }
 

-       IOLockUnlock(interfaceLock);
+       IOLockUnlock(shared->interfaceLock);

 
        return ret;
 }
 
        return ret;
 }


@@ -169,15 +220,15 @@ IOPerfControlClient::registerDevice(__unused IOService *driver, IOService *devic
 void
 IOPerfControlClient::unregisterDevice(__unused IOService *driver, IOService *device)
 {
 void
 IOPerfControlClient::unregisterDevice(__unused IOService *driver, IOService *device)
 {

-       IOLockLock(interfaceLock);
+       IOLockLock(shared->interfaceLock);

 
 

-       if (interface.version > 0) {
-               interface.unregisterDevice(device);
+       if (shared->interface.version > 0) {
+               shared->interface.unregisterDevice(device);

        } else {
        } else {

-               deviceRegistrationList->removeObject(device);
+               shared->deviceRegistrationList->removeObject(device);

        }
 
        }
 

-       IOLockUnlock(interfaceLock);
+       IOLockUnlock(shared->interfaceLock);

 }
 
 uint64_t
 }
 
 uint64_t


@@ -207,25 +258,25 @@ IOPerfControlClient::registerPerformanceController(PerfControllerInterface pci)
 {
        IOReturn result = kIOReturnError;
 
 {
        IOReturn result = kIOReturnError;
 

-       IOLockLock(interfaceLock);
+       IOLockLock(shared->interfaceLock);

 
 

-       if (interface.version == 0 && pci.version > 0) {
+       if (shared->interface.version == 0 && pci.version > 0) {

                assert(pci.registerDevice && pci.unregisterDevice && pci.workCanSubmit && pci.workSubmit && pci.workBegin && pci.workEnd);
                result = kIOReturnSuccess;
 
                OSObject *obj;
                assert(pci.registerDevice && pci.unregisterDevice && pci.workCanSubmit && pci.workSubmit && pci.workBegin && pci.workEnd);
                result = kIOReturnSuccess;
 
                OSObject *obj;

-               while ((obj = deviceRegistrationList->getAnyObject())) {
+               while ((obj = shared->deviceRegistrationList->getAnyObject())) {

                        IOService *device = OSDynamicCast(IOService, obj);
                        if (device) {
                                pci.registerDevice(device);
                        }
                        IOService *device = OSDynamicCast(IOService, obj);
                        if (device) {
                                pci.registerDevice(device);
                        }

-                       deviceRegistrationList->removeObject(obj);
+                       shared->deviceRegistrationList->removeObject(obj);

                }
 
                }
 

-               interface = pci;
+               shared->interface = pci;

        }
 
        }
 

-       IOLockUnlock(interfaceLock);
+       IOLockUnlock(shared->interfaceLock);

 
        return result;
 }
 
        return result;
 }