iokit/Kernel/IOWorkLoop.cpp

   1 /*
   2  * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <pexpert/pexpert.h>
  30 #include <IOKit/IOWorkLoop.h>
  31 #include <IOKit/IOEventSource.h>
  32 #include <IOKit/IOInterruptEventSource.h>
  33 #include <IOKit/IOCommandGate.h>
  34 #include <IOKit/IOTimeStamp.h>
  35 #include <IOKit/IOKitDebug.h>
  36 #include <libkern/OSDebug.h>
  37
  38 #define super OSObject
  39
  40 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
  41
  42 // Block of unused functions intended for future use
  43 #if __LP64__
  44 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
  45 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
  46 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
  47 #else
  48 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
  49 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
  50 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
  51 #endif
  52 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
  53 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
  54 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
  55 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
  56 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
  57
  58 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
  59 #ifdef __ppc__
  60 static inline void SETP(void *addr, unsigned int flag)
  61     { unsigned int *num = (unsigned int *) addr; *num |= flag; }
  62 static inline void CLRP(void *addr, unsigned int flag)
  63     { unsigned int *num = (unsigned int *) addr; *num &= ~flag; }
  64 static inline bool ISSETP(void *addr, unsigned int flag)
  65     { unsigned int *num = (unsigned int *) addr; return (*num & flag) != 0; }
  66 #else
  67 static inline void SETP(void *addr, unsigned int flag)
  68     { unsigned char *num = (unsigned char *) addr; *num |= flag; }
  69 static inline void CLRP(void *addr, unsigned int flag)
  70     { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
  71 static inline bool ISSETP(void *addr, unsigned int flag)
  72     { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
  73 #endif
  74
  75 #define fFlags loopRestart
  76
  77
  78 bool IOWorkLoop::init()
  79 {
  80     // The super init and gateLock allocation MUST be done first
  81     if ( !super::init() )
  82         return false;
  83
  84     if ( gateLock == NULL ) {
  85         if ( !( gateLock = IORecursiveLockAlloc()) )
  86             return false;
  87     }
  88
  89     if ( workToDoLock == NULL ) {
  90         if ( !(workToDoLock = IOSimpleLockAlloc()) )
  91             return false;
  92         IOSimpleLockInit(workToDoLock);
  93         workToDo = false;
  94     }
  95
  96     if ( controlG == NULL ) {
  97         controlG = IOCommandGate::commandGate(
  98             this,
  99             OSMemberFunctionCast(
 100                 IOCommandGate::Action,
 101                 this,
 102                 &IOWorkLoop::_maintRequest));
 103
 104         if ( !controlG )
 105             return false;
 106         // Point the controlGate at the workLoop.  Usually addEventSource
 107         // does this automatically.  The problem is in this case addEventSource
 108         // uses the control gate and it has to be bootstrapped.
 109         controlG->setWorkLoop(this);
 110         if (addEventSource(controlG) != kIOReturnSuccess)
 111             return false;
 112     }
 113
 114     if ( workThread == NULL ) {
 115         thread_continue_t cptr = OSMemberFunctionCast(
 116             thread_continue_t,
 117             this,
 118             &IOWorkLoop::threadMain);
 119         if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
 120             return false;
 121     }
 122
 123     return true;
 124 }
 125
 126 IOWorkLoop *
 127 IOWorkLoop::workLoop()
 128 {
 129     return IOWorkLoop::workLoopWithOptions(0);
 130 }
 131
 132 IOWorkLoop *
 133 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
 134 {
 135     IOWorkLoop *me = new IOWorkLoop;
 136
 137     if (me && options) {
 138         me->reserved = IONew(ExpansionData, 1);
 139         if (!me->reserved) {
 140             me->release();
 141             return 0;
 142         }
 143         me->reserved->options = options;
 144     }
 145
 146     if (me && !me->init()) {
 147         me->release();
 148         return 0;
 149     }
 150
 151     return me;
 152 }
 153
 154 // Free is called twice:
 155 // First when the atomic retainCount transitions from 1 -> 0
 156 // Secondly when the work loop itself is commiting hari kari
 157 // Hence the each leg of the free must be single threaded.
 158 void IOWorkLoop::free()
 159 {
 160     if (workThread) {
 161         IOInterruptState is;
 162
 163         // If we are here then we must be trying to shut down this work loop
 164         // in this case disable all of the event source, mark the loop
 165         // as terminating and wakeup the work thread itself and return
 166         // Note: we hold the gate across the entire operation mainly for the
 167         // benefit of our event sources so we can disable them cleanly.
 168         closeGate();
 169
 170         disableAllEventSources();
 171
 172         is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 173         SETP(&fFlags, kLoopTerminate);
 174         thread_wakeup_one((void *) &workToDo);
 175         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 176
 177         openGate();
 178     }
 179     else /* !workThread */ {
 180         IOEventSource *event, *next;
 181
 182         for (event = eventChain; event; event = next) {
 183             next = event->getNext();
 184             event->setWorkLoop(0);
 185             event->setNext(0);
 186             event->release();
 187         }
 188         eventChain = 0;
 189
 190         // Either we have a partial initialization to clean up
 191         // or the workThread itself is performing hari-kari.
 192         // Either way clean up all of our resources and return.
 193
 194         if (controlG) {
 195             controlG->release();
 196             controlG = 0;
 197         }
 198
 199         if (workToDoLock) {
 200             IOSimpleLockFree(workToDoLock);
 201             workToDoLock = 0;
 202         }
 203
 204         if (gateLock) {
 205             IORecursiveLockFree(gateLock);
 206             gateLock = 0;
 207         }
 208         if (reserved) {
 209             IODelete(reserved, ExpansionData, 1);
 210             reserved = 0;
 211         }
 212
 213         super::free();
 214     }
 215 }
 216
 217 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
 218 {
 219     return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
 220 }
 221
 222 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
 223 {
 224     return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
 225 }
 226
 227 void IOWorkLoop::enableAllEventSources() const
 228 {
 229     IOEventSource *event;
 230
 231     for (event = eventChain; event; event = event->getNext())
 232         event->enable();
 233 }
 234
 235 void IOWorkLoop::disableAllEventSources() const
 236 {
 237     IOEventSource *event;
 238
 239     for (event = eventChain; event; event = event->getNext())
 240         if (event != controlG)  // Don't disable the control gate
 241             event->disable();
 242 }
 243
 244 void IOWorkLoop::enableAllInterrupts() const
 245 {
 246     IOEventSource *event;
 247
 248     for (event = eventChain; event; event = event->getNext())
 249         if (OSDynamicCast(IOInterruptEventSource, event))
 250             event->enable();
 251 }
 252
 253 void IOWorkLoop::disableAllInterrupts() const
 254 {
 255     IOEventSource *event;
 256
 257     for (event = eventChain; event; event = event->getNext())
 258         if (OSDynamicCast(IOInterruptEventSource, event))
 259             event->disable();
 260 }
 261
 262 #if KDEBUG
 263 #define IOTimeClientS()                                                 \
 264 do {                                                                    \
 265     IOTimeStampStart(IODBG_WORKLOOP(IOWL_CLIENT),                       \
 266                      (unsigned int) this, (unsigned int) event);        \
 267 } while(0)
 268
 269 #define IOTimeClientE()                                                 \
 270 do {                                                                    \
 271     IOTimeStampEnd(IODBG_WORKLOOP(IOWL_CLIENT),                         \
 272                    (unsigned int) this, (unsigned int) event);          \
 273 } while(0)
 274
 275 #define IOTimeWorkS()                                                   \
 276 do {                                                                    \
 277     IOTimeStampStart(IODBG_WORKLOOP(IOWL_WORK), (unsigned int) this);   \
 278 } while(0)
 279
 280 #define IOTimeWorkE()                                                   \
 281 do {                                                                    \
 282     IOTimeStampEnd(IODBG_WORKLOOP(IOWL_WORK),(unsigned int) this);      \
 283 } while(0)
 284
 285 #else /* !KDEBUG */
 286
 287 #define IOTimeClientS()
 288 #define IOTimeClientE()
 289 #define IOTimeWorkS()
 290 #define IOTimeWorkE()
 291
 292 #endif /* KDEBUG */
 293
 294 /* virtual */ bool IOWorkLoop::runEventSources()
 295 {
 296     bool res = false;
 297     bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
 298     bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
 299
 300     closeGate();
 301     if (ISSETP(&fFlags, kLoopTerminate))
 302         goto abort;
 303
 304     if (traceWL)
 305         IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 306
 307     bool more;
 308     do {
 309         CLRP(&fFlags, kLoopRestart);
 310         more = false;
 311         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 312         workToDo = false;
 313         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 314         for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
 315
 316                 if (traceES)
 317                         IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 318
 319             more |= evnt->checkForWork();
 320
 321                 if (traceES)
 322                         IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 323
 324             if (ISSETP(&fFlags, kLoopTerminate))
 325                 goto abort;
 326             else if (fFlags & kLoopRestart) {
 327                 more = true;
 328                 break;
 329             }
 330         }
 331     } while (more);
 332
 333     res = true;
 334
 335     if (traceWL)
 336         IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 337
 338 abort:
 339     openGate();
 340     return res;
 341 }
 342
 343 /* virtual */ void IOWorkLoop::threadMain()
 344 {
 345 restartThread:
 346     do {
 347         if ( !runEventSources() )
 348             goto exitThread;
 349
 350         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 351         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 352             assert_wait((void *) &workToDo, false);
 353             IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 354             thread_continue_t cptr = NULL;
 355             if (!reserved || !(kPreciousStack & reserved->options))
 356                 cptr = OSMemberFunctionCast(
 357                         thread_continue_t, this, &IOWorkLoop::threadMain);
 358             thread_block_parameter(cptr, this);
 359             goto restartThread;
 360             /* NOTREACHED */
 361         }
 362
 363         // At this point we either have work to do or we need
 364         // to commit suicide.  But no matter
 365         // Clear the simple lock and retore the interrupt state
 366         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 367     } while(workToDo);
 368
 369 exitThread:
 370         thread_t thread = workThread;
 371     workThread = 0;     // Say we don't have a loop and free ourselves
 372     free();
 373
 374         thread_deallocate(thread);
 375     (void) thread_terminate(thread);
 376 }
 377
 378 IOThread IOWorkLoop::getThread() const
 379 {
 380     return workThread;
 381 }
 382
 383 bool IOWorkLoop::onThread() const
 384 {
 385     return (IOThreadSelf() == workThread);
 386 }
 387
 388 bool IOWorkLoop::inGate() const
 389 {
 390     return IORecursiveLockHaveLock(gateLock);
 391 }
 392
 393 // Internal APIs used by event sources to control the thread
 394 void IOWorkLoop::signalWorkAvailable()
 395 {
 396     if (workToDoLock) {
 397         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 398         workToDo = true;
 399         thread_wakeup_one((void *) &workToDo);
 400         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 401     }
 402 }
 403
 404 void IOWorkLoop::openGate()
 405 {
 406     IORecursiveLockUnlock(gateLock);
 407 }
 408
 409 void IOWorkLoop::closeGate()
 410 {
 411     IORecursiveLockLock(gateLock);
 412 }
 413
 414 bool IOWorkLoop::tryCloseGate()
 415 {
 416     return IORecursiveLockTryLock(gateLock) != 0;
 417 }
 418
 419 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
 420 {
 421     return IORecursiveLockSleep(gateLock, event, interuptibleType);
 422 }
 423
 424 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
 425 {
 426     return IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
 427 }
 428
 429 void IOWorkLoop::wakeupGate(void *event, bool oneThread)
 430 {
 431     IORecursiveLockWakeup(gateLock, event, oneThread);
 432 }
 433
 434 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
 435                                   void *arg0, void *arg1,
 436                                   void *arg2, void *arg3)
 437 {
 438     IOReturn res;
 439
 440     // closeGate is recursive so don't worry if we already hold the lock.
 441     closeGate();
 442     res = (*inAction)(target, arg0, arg1, arg2, arg3);
 443     openGate();
 444
 445     return res;
 446 }
 447
 448 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
 449 {
 450     maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
 451     IOEventSource *inEvent = (IOEventSource *) inD;
 452     IOReturn res = kIOReturnSuccess;
 453
 454     switch (command)
 455     {
 456     case mAddEvent:
 457         if (!inEvent->getWorkLoop()) {
 458             SETP(&fFlags, kLoopRestart);
 459
 460             inEvent->retain();
 461             inEvent->setWorkLoop(this);
 462             inEvent->setNext(0);
 463
 464             if (!eventChain)
 465                 eventChain = inEvent;
 466             else {
 467                 IOEventSource *event, *next;
 468
 469                 for (event = eventChain; (next = event->getNext()); event = next)
 470                     ;
 471                 event->setNext(inEvent);
 472             }
 473         }
 474         break;
 475
 476     case mRemoveEvent:
 477         if (inEvent->getWorkLoop()) {
 478             if (eventChain == inEvent)
 479                 eventChain = inEvent->getNext();
 480             else {
 481                 IOEventSource *event, *next;
 482
 483                 event = eventChain;
 484                 while ((next = event->getNext()) && next != inEvent)
 485                     event = next;
 486
 487                 if (!next) {
 488                     res = kIOReturnBadArgument;
 489                     break;
 490                 }
 491                 event->setNext(inEvent->getNext());
 492             }
 493
 494             inEvent->setWorkLoop(0);
 495             inEvent->setNext(0);
 496             inEvent->release();
 497             SETP(&fFlags, kLoopRestart);
 498         }
 499         break;
 500
 501     default:
 502         return kIOReturnUnsupported;
 503     }
 504
 505     return res;
 506 }