iokit/Kernel/IOWorkLoop.cpp

   1 /*
   2  * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <pexpert/pexpert.h>
  30 #include <IOKit/IOWorkLoop.h>
  31 #include <IOKit/IOEventSource.h>
  32 #include <IOKit/IOInterruptEventSource.h>
  33 #include <IOKit/IOCommandGate.h>
  34 #include <IOKit/IOTimeStamp.h>
  35 #include <IOKit/IOKitDebug.h>
  36 #include <libkern/OSDebug.h>
  37 #include <kern/thread.h>
  38
  39 #define super OSObject
  40
  41 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
  42
  43 // Block of unused functions intended for future use
  44 #if __LP64__
  45 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
  46 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
  47 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
  48 #else
  49 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
  50 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
  51 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
  52 #endif
  53 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
  54 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
  55 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
  56 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
  57 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
  58
  59 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
  60 static inline void SETP(void *addr, unsigned int flag)
  61     { unsigned char *num = (unsigned char *) addr; *num |= flag; }
  62 static inline void CLRP(void *addr, unsigned int flag)
  63     { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
  64 static inline bool ISSETP(void *addr, unsigned int flag)
  65     { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
  66
  67 #define fFlags loopRestart
  68
  69 #define passiveEventChain       reserved->passiveEventChain
  70
  71 #if IOKITSTATS
  72
  73 #define IOStatisticsRegisterCounter() \
  74 do { \
  75         reserved->counter = IOStatistics::registerWorkLoop(this); \
  76 } while(0)
  77
  78 #define IOStatisticsUnregisterCounter() \
  79 do { \
  80         if (reserved) \
  81                 IOStatistics::unregisterWorkLoop(reserved->counter); \
  82 } while(0)
  83
  84 #define IOStatisticsOpenGate() \
  85 do { \
  86         IOStatistics::countWorkLoopOpenGate(reserved->counter); \
  87 } while(0)
  88
  89 #define IOStatisticsCloseGate() \
  90 do { \
  91         IOStatistics::countWorkLoopCloseGate(reserved->counter); \
  92 } while(0)
  93
  94 #define IOStatisticsAttachEventSource() \
  95 do { \
  96         IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
  97 } while(0)
  98
  99 #define IOStatisticsDetachEventSource() \
 100 do { \
 101         IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
 102 } while(0)
 103
 104 #else
 105
 106 #define IOStatisticsRegisterCounter()
 107 #define IOStatisticsUnregisterCounter()
 108 #define IOStatisticsOpenGate()
 109 #define IOStatisticsCloseGate()
 110 #define IOStatisticsAttachEventSource()
 111 #define IOStatisticsDetachEventSource()
 112
 113 #endif /* IOKITSTATS */
 114
 115 bool IOWorkLoop::init()
 116 {
 117     // The super init and gateLock allocation MUST be done first.
 118     if ( !super::init() )
 119         return false;
 120
 121         // Allocate our ExpansionData if it hasn't been allocated already.
 122         if ( !reserved )
 123         {
 124                 reserved = IONew(ExpansionData,1);
 125                 if ( !reserved )
 126                         return false;
 127
 128                 bzero(reserved,sizeof(ExpansionData));
 129         }
 130
 131 #if DEBUG
 132         OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
 133 #endif
 134
 135     if ( gateLock == NULL ) {
 136         if ( !( gateLock = IORecursiveLockAlloc()) )
 137             return false;
 138     }
 139
 140     if ( workToDoLock == NULL ) {
 141         if ( !(workToDoLock = IOSimpleLockAlloc()) )
 142             return false;
 143         IOSimpleLockInit(workToDoLock);
 144         workToDo = false;
 145     }
 146
 147     if (!reserved) {
 148         reserved = IONew(ExpansionData, 1);
 149         reserved->options = 0;
 150     }
 151
 152     IOStatisticsRegisterCounter();
 153
 154     if ( controlG == NULL ) {
 155         controlG = IOCommandGate::commandGate(
 156             this,
 157             OSMemberFunctionCast(
 158                 IOCommandGate::Action,
 159                 this,
 160                 &IOWorkLoop::_maintRequest));
 161
 162         if ( !controlG )
 163             return false;
 164         // Point the controlGate at the workLoop.  Usually addEventSource
 165         // does this automatically.  The problem is in this case addEventSource
 166         // uses the control gate and it has to be bootstrapped.
 167         controlG->setWorkLoop(this);
 168         if (addEventSource(controlG) != kIOReturnSuccess)
 169             return false;
 170     }
 171
 172     if ( workThread == NULL ) {
 173         thread_continue_t cptr = OSMemberFunctionCast(
 174             thread_continue_t,
 175             this,
 176             &IOWorkLoop::threadMain);
 177         if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
 178             return false;
 179     }
 180
 181     (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
 182     return true;
 183 }
 184
 185 IOWorkLoop *
 186 IOWorkLoop::workLoop()
 187 {
 188     return IOWorkLoop::workLoopWithOptions(0);
 189 }
 190
 191 IOWorkLoop *
 192 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
 193 {
 194         IOWorkLoop *me = new IOWorkLoop;
 195
 196         if (me && options) {
 197                 me->reserved = IONew(ExpansionData,1);
 198                 if (!me->reserved) {
 199                         me->release();
 200                         return 0;
 201                 }
 202                 bzero(me->reserved,sizeof(ExpansionData));
 203                 me->reserved->options = options;
 204         }
 205
 206         if (me && !me->init()) {
 207                 me->release();
 208                 return 0;
 209         }
 210
 211         return me;
 212 }
 213
 214 // Free is called twice:
 215 // First when the atomic retainCount transitions from 1 -> 0
 216 // Secondly when the work loop itself is commiting hari kari
 217 // Hence the each leg of the free must be single threaded.
 218 void IOWorkLoop::free()
 219 {
 220     if (workThread) {
 221         IOInterruptState is;
 222
 223         // If we are here then we must be trying to shut down this work loop
 224         // in this case disable all of the event source, mark the loop
 225         // as terminating and wakeup the work thread itself and return
 226         // Note: we hold the gate across the entire operation mainly for the
 227         // benefit of our event sources so we can disable them cleanly.
 228         closeGate();
 229
 230         disableAllEventSources();
 231
 232         is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 233         SETP(&fFlags, kLoopTerminate);
 234         thread_wakeup_one((void *) &workToDo);
 235         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 236
 237         openGate();
 238     }
 239     else /* !workThread */ {
 240         IOEventSource *event, *next;
 241
 242         for (event = eventChain; event; event = next) {
 243             next = event->getNext();
 244             event->setWorkLoop(0);
 245             event->setNext(0);
 246             event->release();
 247         }
 248         eventChain = 0;
 249
 250         for (event = passiveEventChain; event; event = next) {
 251             next = event->getNext();
 252             event->setWorkLoop(0);
 253             event->setNext(0);
 254             event->release();
 255         }
 256         passiveEventChain = 0;
 257
 258         // Either we have a partial initialization to clean up
 259         // or the workThread itself is performing hari-kari.
 260         // Either way clean up all of our resources and return.
 261
 262         if (controlG) {
 263             controlG->release();
 264             controlG = 0;
 265         }
 266
 267         if (workToDoLock) {
 268             IOSimpleLockFree(workToDoLock);
 269             workToDoLock = 0;
 270         }
 271
 272         if (gateLock) {
 273             IORecursiveLockFree(gateLock);
 274             gateLock = 0;
 275         }
 276
 277         IOStatisticsUnregisterCounter();
 278
 279         if (reserved) {
 280             IODelete(reserved, ExpansionData, 1);
 281             reserved = 0;
 282         }
 283
 284         super::free();
 285     }
 286 }
 287
 288 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
 289 {
 290     return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
 291 }
 292
 293 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
 294 {
 295     return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
 296 }
 297
 298 void IOWorkLoop::enableAllEventSources() const
 299 {
 300     IOEventSource *event;
 301
 302     for (event = eventChain; event; event = event->getNext())
 303         event->enable();
 304
 305     for (event = passiveEventChain; event; event = event->getNext())
 306         event->enable();
 307 }
 308
 309 void IOWorkLoop::disableAllEventSources() const
 310 {
 311     IOEventSource *event;
 312
 313     for (event = eventChain; event; event = event->getNext())
 314                 event->disable();
 315
 316         /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
 317     for (event = passiveEventChain; event; event = event->getNext())
 318         if (event != controlG)  // Don't disable the control gate
 319             event->disable();
 320 }
 321
 322 void IOWorkLoop::enableAllInterrupts() const
 323 {
 324     IOEventSource *event;
 325
 326     for (event = eventChain; event; event = event->getNext())
 327         if (OSDynamicCast(IOInterruptEventSource, event))
 328             event->enable();
 329 }
 330
 331 void IOWorkLoop::disableAllInterrupts() const
 332 {
 333     IOEventSource *event;
 334
 335     for (event = eventChain; event; event = event->getNext())
 336         if (OSDynamicCast(IOInterruptEventSource, event))
 337             event->disable();
 338 }
 339
 340
 341 /* virtual */ bool IOWorkLoop::runEventSources()
 342 {
 343     bool res = false;
 344     bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
 345     bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
 346
 347     closeGate();
 348     if (ISSETP(&fFlags, kLoopTerminate))
 349                 goto abort;
 350
 351     if (traceWL)
 352         IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 353
 354     bool more;
 355     do {
 356                 CLRP(&fFlags, kLoopRestart);
 357                 more = false;
 358                 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 359                 workToDo = false;
 360                 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 361                 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
 362                 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
 363
 364                         if (traceES)
 365                                 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 366
 367                         more |= evnt->checkForWork();
 368
 369                         if (traceES)
 370                                 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 371
 372                         if (ISSETP(&fFlags, kLoopTerminate))
 373                                 goto abort;
 374                         else if (fFlags & kLoopRestart) {
 375                                 more = true;
 376                                 break;
 377                         }
 378                 }
 379     } while (more);
 380
 381     res = true;
 382
 383     if (traceWL)
 384         IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 385
 386 abort:
 387     openGate();
 388     return res;
 389 }
 390
 391 /* virtual */ void IOWorkLoop::threadMain()
 392 {
 393 restartThread:
 394     do {
 395         if ( !runEventSources() )
 396             goto exitThread;
 397
 398         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 399         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 400             assert_wait((void *) &workToDo, false);
 401             IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 402             thread_continue_t cptr = NULL;
 403             if (!reserved || !(kPreciousStack & reserved->options))
 404                 cptr = OSMemberFunctionCast(
 405                         thread_continue_t, this, &IOWorkLoop::threadMain);
 406             thread_block_parameter(cptr, this);
 407             goto restartThread;
 408             /* NOTREACHED */
 409         }
 410
 411         // At this point we either have work to do or we need
 412         // to commit suicide.  But no matter
 413         // Clear the simple lock and retore the interrupt state
 414         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 415     } while(workToDo);
 416
 417 exitThread:
 418         thread_t thread = workThread;
 419     workThread = 0;     // Say we don't have a loop and free ourselves
 420     free();
 421
 422         thread_deallocate(thread);
 423     (void) thread_terminate(thread);
 424 }
 425
 426 IOThread IOWorkLoop::getThread() const
 427 {
 428     return workThread;
 429 }
 430
 431 bool IOWorkLoop::onThread() const
 432 {
 433     return (IOThreadSelf() == workThread);
 434 }
 435
 436 bool IOWorkLoop::inGate() const
 437 {
 438     return IORecursiveLockHaveLock(gateLock);
 439 }
 440
 441 // Internal APIs used by event sources to control the thread
 442 void IOWorkLoop::signalWorkAvailable()
 443 {
 444     if (workToDoLock) {
 445         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 446         workToDo = true;
 447         thread_wakeup_one((void *) &workToDo);
 448         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 449     }
 450 }
 451
 452 void IOWorkLoop::openGate()
 453 {
 454     IOStatisticsOpenGate();
 455     IORecursiveLockUnlock(gateLock);
 456 }
 457
 458 void IOWorkLoop::closeGate()
 459 {
 460     IORecursiveLockLock(gateLock);
 461     IOStatisticsCloseGate();
 462 }
 463
 464 bool IOWorkLoop::tryCloseGate()
 465 {
 466     bool res = (IORecursiveLockTryLock(gateLock) != 0);
 467     if (res) {
 468         IOStatisticsCloseGate();
 469     }
 470     return res;
 471 }
 472
 473 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
 474 {
 475     int res;
 476     IOStatisticsOpenGate();
 477     res = IORecursiveLockSleep(gateLock, event, interuptibleType);
 478     IOStatisticsCloseGate();
 479     return res;
 480 }
 481
 482 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
 483 {
 484     int res;
 485     IOStatisticsOpenGate();
 486     res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
 487     IOStatisticsCloseGate();
 488     return res;
 489 }
 490
 491 void IOWorkLoop::wakeupGate(void *event, bool oneThread)
 492 {
 493     IORecursiveLockWakeup(gateLock, event, oneThread);
 494 }
 495
 496 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
 497                                   void *arg0, void *arg1,
 498                                   void *arg2, void *arg3)
 499 {
 500     IOReturn res;
 501
 502     // closeGate is recursive so don't worry if we already hold the lock.
 503     closeGate();
 504     res = (*inAction)(target, arg0, arg1, arg2, arg3);
 505     openGate();
 506
 507     return res;
 508 }
 509
 510 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
 511 {
 512     maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
 513     IOEventSource *inEvent = (IOEventSource *) inD;
 514     IOReturn res = kIOReturnSuccess;
 515
 516     switch (command)
 517     {
 518     case mAddEvent:
 519         if (!inEvent->getWorkLoop()) {
 520             SETP(&fFlags, kLoopRestart);
 521
 522             inEvent->retain();
 523             inEvent->setWorkLoop(this);
 524             inEvent->setNext(0);
 525
 526                 /* Check if this is a passive or active event source being added */
 527                 if (eventSourcePerformsWork(inEvent)) {
 528
 529                     if (!eventChain)
 530                     eventChain = inEvent;
 531                     else {
 532                     IOEventSource *event, *next;
 533
 534                         for (event = eventChain; (next = event->getNext()); event = next)
 535                         ;
 536                         event->setNext(inEvent);
 537
 538                 }
 539
 540             }
 541             else {
 542
 543                     if (!passiveEventChain)
 544                     passiveEventChain = inEvent;
 545                     else {
 546                     IOEventSource *event, *next;
 547
 548                         for (event = passiveEventChain; (next = event->getNext()); event = next)
 549                         ;
 550                         event->setNext(inEvent);
 551
 552                 }
 553
 554             }
 555             IOStatisticsAttachEventSource();
 556         }
 557         break;
 558
 559     case mRemoveEvent:
 560         if (inEvent->getWorkLoop()) {
 561                 IOStatisticsDetachEventSource();
 562
 563                 if (eventSourcePerformsWork(inEvent)) {
 564                                 if (eventChain == inEvent)
 565                                         eventChain = inEvent->getNext();
 566                                 else {
 567                                         IOEventSource *event, *next;
 568
 569                                         event = eventChain;
 570                                         while ((next = event->getNext()) && next != inEvent)
 571                                                 event = next;
 572
 573                                         if (!next) {
 574                                                 res = kIOReturnBadArgument;
 575                                                 break;
 576                                         }
 577                                         event->setNext(inEvent->getNext());
 578                                 }
 579                 }
 580                 else {
 581                                 if (passiveEventChain == inEvent)
 582                                         passiveEventChain = inEvent->getNext();
 583                                 else {
 584                                         IOEventSource *event, *next;
 585
 586                                         event = passiveEventChain;
 587                                         while ((next = event->getNext()) && next != inEvent)
 588                                                 event = next;
 589
 590                                         if (!next) {
 591                                                 res = kIOReturnBadArgument;
 592                                                 break;
 593                                         }
 594                                         event->setNext(inEvent->getNext());
 595                                 }
 596                 }
 597
 598             inEvent->setWorkLoop(0);
 599             inEvent->setNext(0);
 600             inEvent->release();
 601             SETP(&fFlags, kLoopRestart);
 602         }
 603         break;
 604
 605     default:
 606         return kIOReturnUnsupported;
 607     }
 608
 609     return res;
 610 }
 611
 612 bool
 613 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
 614 {
 615         bool    result = true;
 616
 617         /*
 618          * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
 619          * The assumption is that if you override checkForWork(), you need to be
 620          * active and not passive.
 621          *
 622          * We picked a known quantity controlG that does not override
 623          * IOEventSource::checkForWork(), namely the IOCommandGate associated with
 624          * the workloop to which this event source is getting attached.
 625          *
 626          * We do a pointer comparison on the offset in the vtable for inNewEvent against
 627          * the offset in the vtable for inReferenceEvent. This works because
 628          * IOCommandGate's slot for checkForWork() has the address of
 629          * IOEventSource::checkForWork() in it.
 630          *
 631          * Think of OSMemberFunctionCast yielding the value at the vtable offset for
 632          * checkForWork() here. We're just testing to see if it's the same or not.
 633          *
 634          */
 635         if (controlG) {
 636                 void *  ptr1;
 637                 void *  ptr2;
 638
 639                 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
 640                 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
 641
 642                 if (ptr1 == ptr2)
 643                         result = false;
 644         }
 645
 646     return result;
 647 }