iokit/Kernel/IOWorkLoop.cpp

   1 /*
   2  * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <pexpert/pexpert.h>
  30 #include <IOKit/IOWorkLoop.h>
  31 #include <IOKit/IOEventSource.h>
  32 #include <IOKit/IOInterruptEventSource.h>
  33 #include <IOKit/IOCommandGate.h>
  34 #include <IOKit/IOTimeStamp.h>
  35 #include <IOKit/IOKitDebug.h>
  36 #include <libkern/OSDebug.h>
  37 #include <kern/thread.h>
  38
  39 #define super OSObject
  40
  41 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
  42
  43 // Block of unused functions intended for future use
  44 #if __LP64__
  45 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
  46 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
  47 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
  48 #else
  49 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
  50 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
  51 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
  52 #endif
  53 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
  54 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
  55 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
  56 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
  57 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
  58
  59 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
  60 static inline void SETP(void *addr, unsigned int flag)
  61     { unsigned char *num = (unsigned char *) addr; *num |= flag; }
  62 static inline void CLRP(void *addr, unsigned int flag)
  63     { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
  64 static inline bool ISSETP(void *addr, unsigned int flag)
  65     { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
  66
  67 #define fFlags loopRestart
  68
  69 #define passiveEventChain       reserved->passiveEventChain
  70
  71 #if IOKITSTATS
  72
  73 #define IOStatisticsRegisterCounter() \
  74 do { \
  75         reserved->counter = IOStatistics::registerWorkLoop(this); \
  76 } while(0)
  77
  78 #define IOStatisticsUnregisterCounter() \
  79 do { \
  80         if (reserved) \
  81                 IOStatistics::unregisterWorkLoop(reserved->counter); \
  82 } while(0)
  83
  84 #define IOStatisticsOpenGate() \
  85 do { \
  86         IOStatistics::countWorkLoopOpenGate(reserved->counter); \
  87 } while(0)
  88
  89 #define IOStatisticsCloseGate() \
  90 do { \
  91         IOStatistics::countWorkLoopCloseGate(reserved->counter); \
  92 } while(0)
  93
  94 #define IOStatisticsAttachEventSource() \
  95 do { \
  96         IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
  97 } while(0)
  98
  99 #define IOStatisticsDetachEventSource() \
 100 do { \
 101         IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
 102 } while(0)
 103
 104 #else
 105
 106 #define IOStatisticsRegisterCounter()
 107 #define IOStatisticsUnregisterCounter()
 108 #define IOStatisticsOpenGate()
 109 #define IOStatisticsCloseGate()
 110 #define IOStatisticsAttachEventSource()
 111 #define IOStatisticsDetachEventSource()
 112
 113 #endif /* IOKITSTATS */
 114
 115 bool IOWorkLoop::init()
 116 {
 117     // The super init and gateLock allocation MUST be done first.
 118     if ( !super::init() )
 119         return false;
 120
 121         // Allocate our ExpansionData if it hasn't been allocated already.
 122         if ( !reserved )
 123         {
 124                 reserved = IONew(ExpansionData,1);
 125                 if ( !reserved )
 126                         return false;
 127
 128                 bzero(reserved,sizeof(ExpansionData));
 129         }
 130
 131 #if DEBUG
 132         OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
 133 #endif
 134
 135     if ( gateLock == NULL ) {
 136         if ( !( gateLock = IORecursiveLockAlloc()) )
 137             return false;
 138     }
 139
 140     if ( workToDoLock == NULL ) {
 141         if ( !(workToDoLock = IOSimpleLockAlloc()) )
 142             return false;
 143         IOSimpleLockInit(workToDoLock);
 144         workToDo = false;
 145     }
 146
 147     IOStatisticsRegisterCounter();
 148
 149     if ( controlG == NULL ) {
 150         controlG = IOCommandGate::commandGate(
 151             this,
 152             OSMemberFunctionCast(
 153                 IOCommandGate::Action,
 154                 this,
 155                 &IOWorkLoop::_maintRequest));
 156
 157         if ( !controlG )
 158             return false;
 159         // Point the controlGate at the workLoop.  Usually addEventSource
 160         // does this automatically.  The problem is in this case addEventSource
 161         // uses the control gate and it has to be bootstrapped.
 162         controlG->setWorkLoop(this);
 163         if (addEventSource(controlG) != kIOReturnSuccess)
 164             return false;
 165     }
 166
 167     if ( workThread == NULL ) {
 168         thread_continue_t cptr = OSMemberFunctionCast(
 169             thread_continue_t,
 170             this,
 171             &IOWorkLoop::threadMain);
 172         if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
 173             return false;
 174     }
 175
 176     (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
 177     return true;
 178 }
 179
 180 IOWorkLoop *
 181 IOWorkLoop::workLoop()
 182 {
 183     return IOWorkLoop::workLoopWithOptions(0);
 184 }
 185
 186 IOWorkLoop *
 187 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
 188 {
 189         IOWorkLoop *me = new IOWorkLoop;
 190
 191         if (me && options) {
 192                 me->reserved = IONew(ExpansionData,1);
 193                 if (!me->reserved) {
 194                         me->release();
 195                         return 0;
 196                 }
 197                 bzero(me->reserved,sizeof(ExpansionData));
 198                 me->reserved->options = options;
 199         }
 200
 201         if (me && !me->init()) {
 202                 me->release();
 203                 return 0;
 204         }
 205
 206         return me;
 207 }
 208
 209 // Free is called twice:
 210 // First when the atomic retainCount transitions from 1 -> 0
 211 // Secondly when the work loop itself is commiting hari kari
 212 // Hence the each leg of the free must be single threaded.
 213 void IOWorkLoop::free()
 214 {
 215     if (workThread) {
 216         IOInterruptState is;
 217
 218         // If we are here then we must be trying to shut down this work loop
 219         // in this case disable all of the event source, mark the loop
 220         // as terminating and wakeup the work thread itself and return
 221         // Note: we hold the gate across the entire operation mainly for the
 222         // benefit of our event sources so we can disable them cleanly.
 223         closeGate();
 224
 225         disableAllEventSources();
 226
 227         is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 228         SETP(&fFlags, kLoopTerminate);
 229         thread_wakeup_one((void *) &workToDo);
 230         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 231
 232         openGate();
 233     }
 234     else /* !workThread */ {
 235         IOEventSource *event, *next;
 236
 237         for (event = eventChain; event; event = next) {
 238             next = event->getNext();
 239             event->setWorkLoop(0);
 240             event->setNext(0);
 241             event->release();
 242         }
 243         eventChain = 0;
 244
 245         for (event = passiveEventChain; event; event = next) {
 246             next = event->getNext();
 247             event->setWorkLoop(0);
 248             event->setNext(0);
 249             event->release();
 250         }
 251         passiveEventChain = 0;
 252
 253         // Either we have a partial initialization to clean up
 254         // or the workThread itself is performing hari-kari.
 255         // Either way clean up all of our resources and return.
 256
 257         if (controlG) {
 258             controlG->workLoop = 0;
 259             controlG->release();
 260             controlG = 0;
 261         }
 262
 263         if (workToDoLock) {
 264             IOSimpleLockFree(workToDoLock);
 265             workToDoLock = 0;
 266         }
 267
 268         if (gateLock) {
 269             IORecursiveLockFree(gateLock);
 270             gateLock = 0;
 271         }
 272
 273         IOStatisticsUnregisterCounter();
 274
 275         if (reserved) {
 276             IODelete(reserved, ExpansionData, 1);
 277             reserved = 0;
 278         }
 279
 280         super::free();
 281     }
 282 }
 283
 284 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
 285 {
 286     return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
 287 }
 288
 289 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
 290 {
 291     return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
 292 }
 293
 294 void IOWorkLoop::enableAllEventSources() const
 295 {
 296     IOEventSource *event;
 297
 298     for (event = eventChain; event; event = event->getNext())
 299         event->enable();
 300
 301     for (event = passiveEventChain; event; event = event->getNext())
 302         event->enable();
 303 }
 304
 305 void IOWorkLoop::disableAllEventSources() const
 306 {
 307     IOEventSource *event;
 308
 309     for (event = eventChain; event; event = event->getNext())
 310                 event->disable();
 311
 312         /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
 313     for (event = passiveEventChain; event; event = event->getNext())
 314         if (event != controlG)  // Don't disable the control gate
 315             event->disable();
 316 }
 317
 318 void IOWorkLoop::enableAllInterrupts() const
 319 {
 320     IOEventSource *event;
 321
 322     for (event = eventChain; event; event = event->getNext())
 323         if (OSDynamicCast(IOInterruptEventSource, event))
 324             event->enable();
 325 }
 326
 327 void IOWorkLoop::disableAllInterrupts() const
 328 {
 329     IOEventSource *event;
 330
 331     for (event = eventChain; event; event = event->getNext())
 332         if (OSDynamicCast(IOInterruptEventSource, event))
 333             event->disable();
 334 }
 335
 336
 337 /* virtual */ bool IOWorkLoop::runEventSources()
 338 {
 339     bool res = false;
 340     bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
 341     bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
 342
 343     closeGate();
 344     if (ISSETP(&fFlags, kLoopTerminate))
 345                 goto abort;
 346
 347     if (traceWL)
 348         IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 349
 350     bool more;
 351     do {
 352                 CLRP(&fFlags, kLoopRestart);
 353                 more = false;
 354                 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 355                 workToDo = false;
 356                 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 357                 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
 358                 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
 359
 360                         if (traceES)
 361                                 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 362
 363                         more |= evnt->checkForWork();
 364
 365                         if (traceES)
 366                                 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 367
 368                         if (ISSETP(&fFlags, kLoopTerminate))
 369                                 goto abort;
 370                         else if (fFlags & kLoopRestart) {
 371                                 more = true;
 372                                 break;
 373                         }
 374                 }
 375     } while (more);
 376
 377     res = true;
 378
 379     if (traceWL)
 380         IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 381
 382 abort:
 383     openGate();
 384     return res;
 385 }
 386
 387 /* virtual */ void IOWorkLoop::threadMain()
 388 {
 389 restartThread:
 390     do {
 391         if ( !runEventSources() )
 392             goto exitThread;
 393
 394         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 395         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 396             assert_wait((void *) &workToDo, false);
 397             IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 398             thread_continue_t cptr = NULL;
 399             if (!reserved || !(kPreciousStack & reserved->options))
 400                 cptr = OSMemberFunctionCast(
 401                         thread_continue_t, this, &IOWorkLoop::threadMain);
 402             thread_block_parameter(cptr, this);
 403             goto restartThread;
 404             /* NOTREACHED */
 405         }
 406
 407         // At this point we either have work to do or we need
 408         // to commit suicide.  But no matter
 409         // Clear the simple lock and retore the interrupt state
 410         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 411     } while(workToDo);
 412
 413 exitThread:
 414         thread_t thread = workThread;
 415     workThread = 0;     // Say we don't have a loop and free ourselves
 416     free();
 417
 418         thread_deallocate(thread);
 419     (void) thread_terminate(thread);
 420 }
 421
 422 IOThread IOWorkLoop::getThread() const
 423 {
 424     return workThread;
 425 }
 426
 427 bool IOWorkLoop::onThread() const
 428 {
 429     return (IOThreadSelf() == workThread);
 430 }
 431
 432 bool IOWorkLoop::inGate() const
 433 {
 434     return IORecursiveLockHaveLock(gateLock);
 435 }
 436
 437 // Internal APIs used by event sources to control the thread
 438 void IOWorkLoop::signalWorkAvailable()
 439 {
 440     if (workToDoLock) {
 441         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 442         workToDo = true;
 443         thread_wakeup_one((void *) &workToDo);
 444         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 445     }
 446 }
 447
 448 void IOWorkLoop::openGate()
 449 {
 450     IOStatisticsOpenGate();
 451     IORecursiveLockUnlock(gateLock);
 452 }
 453
 454 void IOWorkLoop::closeGate()
 455 {
 456     IORecursiveLockLock(gateLock);
 457     IOStatisticsCloseGate();
 458 }
 459
 460 bool IOWorkLoop::tryCloseGate()
 461 {
 462     bool res = (IORecursiveLockTryLock(gateLock) != 0);
 463     if (res) {
 464         IOStatisticsCloseGate();
 465     }
 466     return res;
 467 }
 468
 469 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
 470 {
 471     int res;
 472     IOStatisticsOpenGate();
 473     res = IORecursiveLockSleep(gateLock, event, interuptibleType);
 474     IOStatisticsCloseGate();
 475     return res;
 476 }
 477
 478 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
 479 {
 480     int res;
 481     IOStatisticsOpenGate();
 482     res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
 483     IOStatisticsCloseGate();
 484     return res;
 485 }
 486
 487 void IOWorkLoop::wakeupGate(void *event, bool oneThread)
 488 {
 489     IORecursiveLockWakeup(gateLock, event, oneThread);
 490 }
 491
 492 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
 493                                   void *arg0, void *arg1,
 494                                   void *arg2, void *arg3)
 495 {
 496     IOReturn res;
 497
 498     // closeGate is recursive so don't worry if we already hold the lock.
 499     closeGate();
 500     res = (*inAction)(target, arg0, arg1, arg2, arg3);
 501     openGate();
 502
 503     return res;
 504 }
 505
 506 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
 507 {
 508     maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
 509     IOEventSource *inEvent = (IOEventSource *) inD;
 510     IOReturn res = kIOReturnSuccess;
 511
 512     switch (command)
 513     {
 514     case mAddEvent:
 515         if (!inEvent->getWorkLoop()) {
 516             SETP(&fFlags, kLoopRestart);
 517
 518             inEvent->retain();
 519             inEvent->setWorkLoop(this);
 520             inEvent->setNext(0);
 521
 522                 /* Check if this is a passive or active event source being added */
 523                 if (eventSourcePerformsWork(inEvent)) {
 524
 525                     if (!eventChain)
 526                     eventChain = inEvent;
 527                     else {
 528                     IOEventSource *event, *next;
 529
 530                         for (event = eventChain; (next = event->getNext()); event = next)
 531                         ;
 532                         event->setNext(inEvent);
 533
 534                 }
 535
 536             }
 537             else {
 538
 539                     if (!passiveEventChain)
 540                     passiveEventChain = inEvent;
 541                     else {
 542                     IOEventSource *event, *next;
 543
 544                         for (event = passiveEventChain; (next = event->getNext()); event = next)
 545                         ;
 546                         event->setNext(inEvent);
 547
 548                 }
 549
 550             }
 551             IOStatisticsAttachEventSource();
 552         }
 553         break;
 554
 555     case mRemoveEvent:
 556         if (inEvent->getWorkLoop()) {
 557                 IOStatisticsDetachEventSource();
 558
 559                 if (eventSourcePerformsWork(inEvent)) {
 560                                 if (eventChain == inEvent)
 561                                         eventChain = inEvent->getNext();
 562                                 else {
 563                                         IOEventSource *event, *next = 0;
 564
 565                                         event = eventChain;
 566                                         if (event) while ((next = event->getNext()) && (next != inEvent))
 567                                                 event = next;
 568
 569                                         if (!next) {
 570                                                 res = kIOReturnBadArgument;
 571                                                 break;
 572                                         }
 573                                         event->setNext(inEvent->getNext());
 574                                 }
 575                 }
 576                 else {
 577                                 if (passiveEventChain == inEvent)
 578                                         passiveEventChain = inEvent->getNext();
 579                                 else {
 580                                         IOEventSource *event, *next = 0;
 581
 582                                         event = passiveEventChain;
 583                                         if (event) while ((next = event->getNext()) && (next != inEvent))
 584                                                 event = next;
 585
 586                                         if (!next) {
 587                                                 res = kIOReturnBadArgument;
 588                                                 break;
 589                                         }
 590                                         event->setNext(inEvent->getNext());
 591                                 }
 592                 }
 593
 594             inEvent->setWorkLoop(0);
 595             inEvent->setNext(0);
 596             inEvent->release();
 597             SETP(&fFlags, kLoopRestart);
 598         }
 599         break;
 600
 601     default:
 602         return kIOReturnUnsupported;
 603     }
 604
 605     return res;
 606 }
 607
 608 bool
 609 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
 610 {
 611         bool    result = true;
 612
 613         /*
 614          * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
 615          * The assumption is that if you override checkForWork(), you need to be
 616          * active and not passive.
 617          *
 618          * We picked a known quantity controlG that does not override
 619          * IOEventSource::checkForWork(), namely the IOCommandGate associated with
 620          * the workloop to which this event source is getting attached.
 621          *
 622          * We do a pointer comparison on the offset in the vtable for inNewEvent against
 623          * the offset in the vtable for inReferenceEvent. This works because
 624          * IOCommandGate's slot for checkForWork() has the address of
 625          * IOEventSource::checkForWork() in it.
 626          *
 627          * Think of OSMemberFunctionCast yielding the value at the vtable offset for
 628          * checkForWork() here. We're just testing to see if it's the same or not.
 629          *
 630          */
 631         if (controlG) {
 632                 void *  ptr1;
 633                 void *  ptr2;
 634
 635                 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
 636                 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
 637
 638                 if (ptr1 == ptr2)
 639                         result = false;
 640         }
 641
 642     return result;
 643 }