iokit/Kernel/IOPerfControl.cpp

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  */
   4
   5 #include <IOKit/perfcontrol/IOPerfControl.h>
   6
   7 #include <stdatomic.h>
   8
   9 #include <kern/thread_group.h>
  10
  11 #undef super
  12 #define super OSObject
  13 OSDefineMetaClassAndStructors(IOPerfControlClient, OSObject);
  14
  15 static IOPerfControlClient::IOPerfControlClientShared *_Atomic gIOPerfControlClientShared;
  16
  17 bool
  18 IOPerfControlClient::init(IOService *driver, uint64_t maxWorkCapacity)
  19 {
  20         // TODO: Remove this limit and implement dynamic table growth if workloads are found that exceed this
  21         if (maxWorkCapacity > kMaxWorkTableNumEntries) {
  22                 maxWorkCapacity = kMaxWorkTableNumEntries;
  23         }
  24
  25         if (!super::init()) {
  26                 return false;
  27         }
  28
  29         shared = atomic_load_explicit(&gIOPerfControlClientShared, memory_order_acquire);
  30         if (shared == nullptr) {
  31                 IOPerfControlClient::IOPerfControlClientShared *expected = shared;
  32                 shared = reinterpret_cast<IOPerfControlClient::IOPerfControlClientShared*>(kalloc(sizeof(IOPerfControlClientShared)));
  33                 if (!shared) {
  34                         return false;
  35                 }
  36
  37                 atomic_init(&shared->maxDriverIndex, 0);
  38
  39                 shared->interface = PerfControllerInterface{
  40                         .version = 0,
  41                         .registerDevice =
  42                     [](IOService *device) {
  43                             return kIOReturnSuccess;
  44                     },
  45                         .unregisterDevice =
  46                             [](IOService *device) {
  47                                     return kIOReturnSuccess;
  48                             },
  49                         .workCanSubmit =
  50                             [](IOService *device, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
  51                                     return false;
  52                             },
  53                         .workSubmit =
  54                             [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkSubmitArgs *args) {
  55                             },
  56                         .workBegin =
  57                             [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkBeginArgs *args) {
  58                             },
  59                         .workEnd =
  60                             [](IOService *device, uint64_t token, PerfControllerInterface::WorkState *state, WorkEndArgs *args, bool done) {
  61                             },
  62                 };
  63
  64                 shared->interfaceLock = IOLockAlloc();
  65                 if (!shared->interfaceLock) {
  66                         goto shared_init_error;
  67                 }
  68
  69                 shared->deviceRegistrationList = OSSet::withCapacity(4);
  70                 if (!shared->deviceRegistrationList) {
  71                         goto shared_init_error;
  72                 }
  73
  74                 if (!atomic_compare_exchange_strong_explicit(&gIOPerfControlClientShared, &expected, shared, memory_order_acq_rel,
  75                     memory_order_acquire)) {
  76                         IOLockFree(shared->interfaceLock);
  77                         shared->deviceRegistrationList->release();
  78                         kfree(shared, sizeof(*shared));
  79                         shared = expected;
  80                 }
  81         }
  82
  83         driverIndex = atomic_fetch_add_explicit(&shared->maxDriverIndex, 1, memory_order_relaxed) + 1;
  84         assertf(driverIndex != 0, "Overflow in driverIndex. Too many IOPerfControlClients created.\n");
  85
  86         // + 1 since index 0 is unused for kIOPerfControlClientWorkUntracked
  87         workTableLength = maxWorkCapacity + 1;
  88         assertf(workTableLength <= kWorkTableMaxSize, "%zu exceeds max allowed capacity of %zu", workTableLength, kWorkTableMaxSize);
  89         if (maxWorkCapacity > 0) {
  90                 workTable = reinterpret_cast<WorkTableEntry*>(kalloc(workTableLength * sizeof(WorkTableEntry)));
  91                 if (!workTable) {
  92                         goto error;
  93                 }
  94                 bzero(workTable, workTableLength * sizeof(WorkTableEntry));
  95                 workTableNextIndex = 1;
  96
  97                 workTableLock = IOSimpleLockAlloc();
  98                 if (!workTableLock) {
  99                         goto error;
 100                 }
 101         }
 102
 103         return true;
 104
 105 error:
 106         if (workTable) {
 107                 kfree(workTable, maxWorkCapacity * sizeof(WorkTableEntry));
 108         }
 109         if (workTableLock) {
 110                 IOSimpleLockFree(workTableLock);
 111         }
 112         return false;
 113 shared_init_error:
 114         if (shared) {
 115                 if (shared->interfaceLock) {
 116                         IOLockFree(shared->interfaceLock);
 117                 }
 118                 if (shared->deviceRegistrationList) {
 119                         shared->deviceRegistrationList->release();
 120                 }
 121                 kfree(shared, sizeof(*shared));
 122                 shared = nullptr;
 123         }
 124         return false;
 125 }
 126
 127 IOPerfControlClient *
 128 IOPerfControlClient::copyClient(IOService *driver, uint64_t maxWorkCapacity)
 129 {
 130         IOPerfControlClient *client = new IOPerfControlClient;
 131         if (!client || !client->init(driver, maxWorkCapacity)) {
 132                 panic("could not create IOPerfControlClient");
 133         }
 134         return client;
 135 }
 136
 137 /* Convert the per driver token into a globally unique token for the performance
 138  * controller's consumption. This is achieved by setting the driver's unique
 139  * index onto the high order bits. The performance controller is shared between
 140  * all drivers and must track all instances separately, while each driver has
 141  * its own token table, so this step is needed to avoid token collisions between
 142  * drivers.
 143  */
 144 inline uint64_t
 145 IOPerfControlClient::tokenToGlobalUniqueToken(uint64_t token)
 146 {
 147         return token | (static_cast<uint64_t>(driverIndex) << kWorkTableIndexBits);
 148 }
 149
 150 /* With this implementation, tokens returned to the driver differ from tokens
 151  * passed to the performance controller. This implementation has the nice
 152  * property that tokens returns to the driver will aways be between 1 and
 153  * the value of maxWorkCapacity passed by the driver to copyClient. The tokens
 154  * the performance controller sees will match on the lower order bits and have
 155  * the driver index set on the high order bits.
 156  */
 157 uint64_t
 158 IOPerfControlClient::allocateToken(thread_group *thread_group)
 159 {
 160         uint64_t token = kIOPerfControlClientWorkUntracked;
 161
 162
 163         return token;
 164 }
 165
 166 void
 167 IOPerfControlClient::deallocateToken(uint64_t token)
 168 {
 169 }
 170
 171 bool
 172 IOPerfControlClient::getEntryForToken(uint64_t token, IOPerfControlClient::WorkTableEntry &entry)
 173 {
 174         if (token == kIOPerfControlClientWorkUntracked) {
 175                 return false;
 176         }
 177
 178         if (token >= workTableLength) {
 179                 panic("Invalid work token (%llu): index out of bounds.", token);
 180         }
 181
 182         entry = workTable[token];
 183         auto *thread_group = entry.thread_group;
 184         assertf(thread_group, "Invalid work token: %llu", token);
 185         return thread_group != nullptr;
 186 }
 187
 188 void
 189 IOPerfControlClient::markEntryStarted(uint64_t token, bool started)
 190 {
 191         if (token == kIOPerfControlClientWorkUntracked) {
 192                 return;
 193         }
 194
 195         if (token >= workTableLength) {
 196                 panic("Invalid work token (%llu): index out of bounds.", token);
 197         }
 198
 199         workTable[token].started = started;
 200 }
 201
 202 IOReturn
 203 IOPerfControlClient::registerDevice(__unused IOService *driver, IOService *device)
 204 {
 205         IOReturn ret = kIOReturnSuccess;
 206
 207         IOLockLock(shared->interfaceLock);
 208
 209         if (shared->interface.version > 0) {
 210                 ret = shared->interface.registerDevice(device);
 211         } else {
 212                 shared->deviceRegistrationList->setObject(device);
 213         }
 214
 215         IOLockUnlock(shared->interfaceLock);
 216
 217         return ret;
 218 }
 219
 220 void
 221 IOPerfControlClient::unregisterDevice(__unused IOService *driver, IOService *device)
 222 {
 223         IOLockLock(shared->interfaceLock);
 224
 225         if (shared->interface.version > 0) {
 226                 shared->interface.unregisterDevice(device);
 227         } else {
 228                 shared->deviceRegistrationList->removeObject(device);
 229         }
 230
 231         IOLockUnlock(shared->interfaceLock);
 232 }
 233
 234 uint64_t
 235 IOPerfControlClient::workSubmit(IOService *device, WorkSubmitArgs *args)
 236 {
 237         return kIOPerfControlClientWorkUntracked;
 238 }
 239
 240 uint64_t
 241 IOPerfControlClient::workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs, WorkBeginArgs *beginArgs)
 242 {
 243         return kIOPerfControlClientWorkUntracked;
 244 }
 245
 246 void
 247 IOPerfControlClient::workBegin(IOService *device, uint64_t token, WorkBeginArgs *args)
 248 {
 249 }
 250
 251 void
 252 IOPerfControlClient::workEnd(IOService *device, uint64_t token, WorkEndArgs *args, bool done)
 253 {
 254 }
 255
 256 IOReturn
 257 IOPerfControlClient::registerPerformanceController(PerfControllerInterface pci)
 258 {
 259         IOReturn result = kIOReturnError;
 260
 261         IOLockLock(shared->interfaceLock);
 262
 263         if (shared->interface.version == 0 && pci.version > 0) {
 264                 assert(pci.registerDevice && pci.unregisterDevice && pci.workCanSubmit && pci.workSubmit && pci.workBegin && pci.workEnd);
 265                 result = kIOReturnSuccess;
 266
 267                 OSObject *obj;
 268                 while ((obj = shared->deviceRegistrationList->getAnyObject())) {
 269                         IOService *device = OSDynamicCast(IOService, obj);
 270                         if (device) {
 271                                 pci.registerDevice(device);
 272                         }
 273                         shared->deviceRegistrationList->removeObject(obj);
 274                 }
 275
 276                 shared->interface = pci;
 277         }
 278
 279         IOLockUnlock(shared->interfaceLock);
 280
 281         return result;
 282 }