iokit/Kernel/IOWorkLoop.cpp

   1 /*
   2  * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <pexpert/pexpert.h>
  30 #include <IOKit/IOWorkLoop.h>
  31 #include <IOKit/IOEventSource.h>
  32 #include <IOKit/IOInterruptEventSource.h>
  33 #include <IOKit/IOCommandGate.h>
  34 #include <IOKit/IOTimeStamp.h>
  35 #include <IOKit/IOKitDebug.h>
  36 #include <libkern/OSDebug.h>
  37
  38 #define super OSObject
  39
  40 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
  41
  42 // Block of unused functions intended for future use
  43 #if __LP64__
  44 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
  45 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
  46 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
  47 #else
  48 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
  49 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
  50 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
  51 #endif
  52 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
  53 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
  54 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
  55 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
  56 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
  57
  58 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
  59 static inline void SETP(void *addr, unsigned int flag)
  60     { unsigned char *num = (unsigned char *) addr; *num |= flag; }
  61 static inline void CLRP(void *addr, unsigned int flag)
  62     { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
  63 static inline bool ISSETP(void *addr, unsigned int flag)
  64     { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
  65
  66 #define fFlags loopRestart
  67
  68 #define passiveEventChain       reserved->passiveEventChain
  69
  70 #if IOKITSTATS
  71
  72 #define IOStatisticsRegisterCounter() \
  73 do { \
  74         reserved->counter = IOStatistics::registerWorkLoop(this); \
  75 } while(0)
  76
  77 #define IOStatisticsUnregisterCounter() \
  78 do { \
  79         if (reserved) \
  80                 IOStatistics::unregisterWorkLoop(reserved->counter); \
  81 } while(0)
  82
  83 #define IOStatisticsOpenGate() \
  84 do { \
  85         IOStatistics::countWorkLoopOpenGate(reserved->counter); \
  86 } while(0)
  87
  88 #define IOStatisticsCloseGate() \
  89 do { \
  90         IOStatistics::countWorkLoopCloseGate(reserved->counter); \
  91 } while(0)
  92
  93 #define IOStatisticsAttachEventSource() \
  94 do { \
  95         IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
  96 } while(0)
  97
  98 #define IOStatisticsDetachEventSource() \
  99 do { \
 100         IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
 101 } while(0)
 102
 103 #else
 104
 105 #define IOStatisticsRegisterCounter()
 106 #define IOStatisticsUnregisterCounter()
 107 #define IOStatisticsOpenGate()
 108 #define IOStatisticsCloseGate()
 109 #define IOStatisticsAttachEventSource()
 110 #define IOStatisticsDetachEventSource()
 111
 112 #endif /* IOKITSTATS */
 113
 114 bool IOWorkLoop::init()
 115 {
 116     // The super init and gateLock allocation MUST be done first.
 117     if ( !super::init() )
 118         return false;
 119
 120         // Allocate our ExpansionData if it hasn't been allocated already.
 121         if ( !reserved )
 122         {
 123                 reserved = IONew(ExpansionData,1);
 124                 if ( !reserved )
 125                         return false;
 126
 127                 bzero(reserved,sizeof(ExpansionData));
 128         }
 129
 130 #if DEBUG
 131         OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
 132 #endif
 133
 134     if ( gateLock == NULL ) {
 135         if ( !( gateLock = IORecursiveLockAlloc()) )
 136             return false;
 137     }
 138
 139     if ( workToDoLock == NULL ) {
 140         if ( !(workToDoLock = IOSimpleLockAlloc()) )
 141             return false;
 142         IOSimpleLockInit(workToDoLock);
 143         workToDo = false;
 144     }
 145
 146     if (!reserved) {
 147         reserved = IONew(ExpansionData, 1);
 148         reserved->options = 0;
 149     }
 150
 151     IOStatisticsRegisterCounter();
 152
 153     if ( controlG == NULL ) {
 154         controlG = IOCommandGate::commandGate(
 155             this,
 156             OSMemberFunctionCast(
 157                 IOCommandGate::Action,
 158                 this,
 159                 &IOWorkLoop::_maintRequest));
 160
 161         if ( !controlG )
 162             return false;
 163         // Point the controlGate at the workLoop.  Usually addEventSource
 164         // does this automatically.  The problem is in this case addEventSource
 165         // uses the control gate and it has to be bootstrapped.
 166         controlG->setWorkLoop(this);
 167         if (addEventSource(controlG) != kIOReturnSuccess)
 168             return false;
 169     }
 170
 171     if ( workThread == NULL ) {
 172         thread_continue_t cptr = OSMemberFunctionCast(
 173             thread_continue_t,
 174             this,
 175             &IOWorkLoop::threadMain);
 176         if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
 177             return false;
 178     }
 179
 180     return true;
 181 }
 182
 183 IOWorkLoop *
 184 IOWorkLoop::workLoop()
 185 {
 186     return IOWorkLoop::workLoopWithOptions(0);
 187 }
 188
 189 IOWorkLoop *
 190 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
 191 {
 192         IOWorkLoop *me = new IOWorkLoop;
 193
 194         if (me && options) {
 195                 me->reserved = IONew(ExpansionData,1);
 196                 if (!me->reserved) {
 197                         me->release();
 198                         return 0;
 199                 }
 200                 bzero(me->reserved,sizeof(ExpansionData));
 201                 me->reserved->options = options;
 202         }
 203
 204         if (me && !me->init()) {
 205                 me->release();
 206                 return 0;
 207         }
 208
 209         return me;
 210 }
 211
 212 // Free is called twice:
 213 // First when the atomic retainCount transitions from 1 -> 0
 214 // Secondly when the work loop itself is commiting hari kari
 215 // Hence the each leg of the free must be single threaded.
 216 void IOWorkLoop::free()
 217 {
 218     if (workThread) {
 219         IOInterruptState is;
 220
 221         // If we are here then we must be trying to shut down this work loop
 222         // in this case disable all of the event source, mark the loop
 223         // as terminating and wakeup the work thread itself and return
 224         // Note: we hold the gate across the entire operation mainly for the
 225         // benefit of our event sources so we can disable them cleanly.
 226         closeGate();
 227
 228         disableAllEventSources();
 229
 230         is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 231         SETP(&fFlags, kLoopTerminate);
 232         thread_wakeup_one((void *) &workToDo);
 233         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 234
 235         openGate();
 236     }
 237     else /* !workThread */ {
 238         IOEventSource *event, *next;
 239
 240         for (event = eventChain; event; event = next) {
 241             next = event->getNext();
 242             event->setWorkLoop(0);
 243             event->setNext(0);
 244             event->release();
 245         }
 246         eventChain = 0;
 247
 248         for (event = passiveEventChain; event; event = next) {
 249             next = event->getNext();
 250             event->setWorkLoop(0);
 251             event->setNext(0);
 252             event->release();
 253         }
 254         passiveEventChain = 0;
 255
 256         // Either we have a partial initialization to clean up
 257         // or the workThread itself is performing hari-kari.
 258         // Either way clean up all of our resources and return.
 259
 260         if (controlG) {
 261             controlG->release();
 262             controlG = 0;
 263         }
 264
 265         if (workToDoLock) {
 266             IOSimpleLockFree(workToDoLock);
 267             workToDoLock = 0;
 268         }
 269
 270         if (gateLock) {
 271             IORecursiveLockFree(gateLock);
 272             gateLock = 0;
 273         }
 274
 275         IOStatisticsUnregisterCounter();
 276
 277         if (reserved) {
 278             IODelete(reserved, ExpansionData, 1);
 279             reserved = 0;
 280         }
 281
 282         super::free();
 283     }
 284 }
 285
 286 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
 287 {
 288     return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
 289 }
 290
 291 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
 292 {
 293     return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
 294 }
 295
 296 void IOWorkLoop::enableAllEventSources() const
 297 {
 298     IOEventSource *event;
 299
 300     for (event = eventChain; event; event = event->getNext())
 301         event->enable();
 302
 303     for (event = passiveEventChain; event; event = event->getNext())
 304         event->enable();
 305 }
 306
 307 void IOWorkLoop::disableAllEventSources() const
 308 {
 309     IOEventSource *event;
 310
 311     for (event = eventChain; event; event = event->getNext())
 312                 event->disable();
 313
 314         /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
 315     for (event = passiveEventChain; event; event = event->getNext())
 316         if (event != controlG)  // Don't disable the control gate
 317             event->disable();
 318 }
 319
 320 void IOWorkLoop::enableAllInterrupts() const
 321 {
 322     IOEventSource *event;
 323
 324     for (event = eventChain; event; event = event->getNext())
 325         if (OSDynamicCast(IOInterruptEventSource, event))
 326             event->enable();
 327 }
 328
 329 void IOWorkLoop::disableAllInterrupts() const
 330 {
 331     IOEventSource *event;
 332
 333     for (event = eventChain; event; event = event->getNext())
 334         if (OSDynamicCast(IOInterruptEventSource, event))
 335             event->disable();
 336 }
 337
 338
 339 /* virtual */ bool IOWorkLoop::runEventSources()
 340 {
 341     bool res = false;
 342     bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
 343     bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
 344
 345     closeGate();
 346     if (ISSETP(&fFlags, kLoopTerminate))
 347                 goto abort;
 348
 349     if (traceWL)
 350         IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 351
 352     bool more;
 353     do {
 354                 CLRP(&fFlags, kLoopRestart);
 355                 more = false;
 356                 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 357                 workToDo = false;
 358                 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 359                 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
 360                 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
 361
 362                         if (traceES)
 363                                 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 364
 365                         more |= evnt->checkForWork();
 366
 367                         if (traceES)
 368                                 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 369
 370                         if (ISSETP(&fFlags, kLoopTerminate))
 371                                 goto abort;
 372                         else if (fFlags & kLoopRestart) {
 373                                 more = true;
 374                                 break;
 375                         }
 376                 }
 377     } while (more);
 378
 379     res = true;
 380
 381     if (traceWL)
 382         IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 383
 384 abort:
 385     openGate();
 386     return res;
 387 }
 388
 389 /* virtual */ void IOWorkLoop::threadMain()
 390 {
 391 restartThread:
 392     do {
 393         if ( !runEventSources() )
 394             goto exitThread;
 395
 396         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 397         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 398             assert_wait((void *) &workToDo, false);
 399             IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 400             thread_continue_t cptr = NULL;
 401             if (!reserved || !(kPreciousStack & reserved->options))
 402                 cptr = OSMemberFunctionCast(
 403                         thread_continue_t, this, &IOWorkLoop::threadMain);
 404             thread_block_parameter(cptr, this);
 405             goto restartThread;
 406             /* NOTREACHED */
 407         }
 408
 409         // At this point we either have work to do or we need
 410         // to commit suicide.  But no matter
 411         // Clear the simple lock and retore the interrupt state
 412         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 413     } while(workToDo);
 414
 415 exitThread:
 416         thread_t thread = workThread;
 417     workThread = 0;     // Say we don't have a loop and free ourselves
 418     free();
 419
 420         thread_deallocate(thread);
 421     (void) thread_terminate(thread);
 422 }
 423
 424 IOThread IOWorkLoop::getThread() const
 425 {
 426     return workThread;
 427 }
 428
 429 bool IOWorkLoop::onThread() const
 430 {
 431     return (IOThreadSelf() == workThread);
 432 }
 433
 434 bool IOWorkLoop::inGate() const
 435 {
 436     return IORecursiveLockHaveLock(gateLock);
 437 }
 438
 439 // Internal APIs used by event sources to control the thread
 440 void IOWorkLoop::signalWorkAvailable()
 441 {
 442     if (workToDoLock) {
 443         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 444         workToDo = true;
 445         thread_wakeup_one((void *) &workToDo);
 446         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 447     }
 448 }
 449
 450 void IOWorkLoop::openGate()
 451 {
 452     IOStatisticsOpenGate();
 453     IORecursiveLockUnlock(gateLock);
 454 }
 455
 456 void IOWorkLoop::closeGate()
 457 {
 458     IORecursiveLockLock(gateLock);
 459     IOStatisticsCloseGate();
 460 }
 461
 462 bool IOWorkLoop::tryCloseGate()
 463 {
 464     bool res = (IORecursiveLockTryLock(gateLock) != 0);
 465     if (res) {
 466         IOStatisticsCloseGate();
 467     }
 468     return res;
 469 }
 470
 471 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
 472 {
 473     int res;
 474     IOStatisticsOpenGate();
 475     res = IORecursiveLockSleep(gateLock, event, interuptibleType);
 476     IOStatisticsCloseGate();
 477     return res;
 478 }
 479
 480 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
 481 {
 482     int res;
 483     IOStatisticsOpenGate();
 484     res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
 485     IOStatisticsCloseGate();
 486     return res;
 487 }
 488
 489 void IOWorkLoop::wakeupGate(void *event, bool oneThread)
 490 {
 491     IORecursiveLockWakeup(gateLock, event, oneThread);
 492 }
 493
 494 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
 495                                   void *arg0, void *arg1,
 496                                   void *arg2, void *arg3)
 497 {
 498     IOReturn res;
 499
 500     // closeGate is recursive so don't worry if we already hold the lock.
 501     closeGate();
 502     res = (*inAction)(target, arg0, arg1, arg2, arg3);
 503     openGate();
 504
 505     return res;
 506 }
 507
 508 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
 509 {
 510     maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
 511     IOEventSource *inEvent = (IOEventSource *) inD;
 512     IOReturn res = kIOReturnSuccess;
 513
 514     switch (command)
 515     {
 516     case mAddEvent:
 517         if (!inEvent->getWorkLoop()) {
 518             SETP(&fFlags, kLoopRestart);
 519
 520             inEvent->retain();
 521             inEvent->setWorkLoop(this);
 522             inEvent->setNext(0);
 523
 524                 /* Check if this is a passive or active event source being added */
 525                 if (eventSourcePerformsWork(inEvent)) {
 526
 527                     if (!eventChain)
 528                     eventChain = inEvent;
 529                     else {
 530                     IOEventSource *event, *next;
 531
 532                         for (event = eventChain; (next = event->getNext()); event = next)
 533                         ;
 534                         event->setNext(inEvent);
 535
 536                 }
 537
 538             }
 539             else {
 540
 541                     if (!passiveEventChain)
 542                     passiveEventChain = inEvent;
 543                     else {
 544                     IOEventSource *event, *next;
 545
 546                         for (event = passiveEventChain; (next = event->getNext()); event = next)
 547                         ;
 548                         event->setNext(inEvent);
 549
 550                 }
 551
 552             }
 553             IOStatisticsAttachEventSource();
 554         }
 555         break;
 556
 557     case mRemoveEvent:
 558         if (inEvent->getWorkLoop()) {
 559                 IOStatisticsDetachEventSource();
 560
 561                 if (eventSourcePerformsWork(inEvent)) {
 562                                 if (eventChain == inEvent)
 563                                         eventChain = inEvent->getNext();
 564                                 else {
 565                                         IOEventSource *event, *next;
 566
 567                                         event = eventChain;
 568                                         while ((next = event->getNext()) && next != inEvent)
 569                                                 event = next;
 570
 571                                         if (!next) {
 572                                                 res = kIOReturnBadArgument;
 573                                                 break;
 574                                         }
 575                                         event->setNext(inEvent->getNext());
 576                                 }
 577                 }
 578                 else {
 579                                 if (passiveEventChain == inEvent)
 580                                         passiveEventChain = inEvent->getNext();
 581                                 else {
 582                                         IOEventSource *event, *next;
 583
 584                                         event = passiveEventChain;
 585                                         while ((next = event->getNext()) && next != inEvent)
 586                                                 event = next;
 587
 588                                         if (!next) {
 589                                                 res = kIOReturnBadArgument;
 590                                                 break;
 591                                         }
 592                                         event->setNext(inEvent->getNext());
 593                                 }
 594                 }
 595
 596             inEvent->setWorkLoop(0);
 597             inEvent->setNext(0);
 598             inEvent->release();
 599             SETP(&fFlags, kLoopRestart);
 600         }
 601         break;
 602
 603     default:
 604         return kIOReturnUnsupported;
 605     }
 606
 607     return res;
 608 }
 609
 610 bool
 611 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
 612 {
 613         bool    result = true;
 614
 615         /*
 616          * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
 617          * The assumption is that if you override checkForWork(), you need to be
 618          * active and not passive.
 619          *
 620          * We picked a known quantity controlG that does not override
 621          * IOEventSource::checkForWork(), namely the IOCommandGate associated with
 622          * the workloop to which this event source is getting attached.
 623          *
 624          * We do a pointer comparison on the offset in the vtable for inNewEvent against
 625          * the offset in the vtable for inReferenceEvent. This works because
 626          * IOCommandGate's slot for checkForWork() has the address of
 627          * IOEventSource::checkForWork() in it.
 628          *
 629          * Think of OSMemberFunctionCast yielding the value at the vtable offset for
 630          * checkForWork() here. We're just testing to see if it's the same or not.
 631          *
 632          */
 633         if (controlG) {
 634                 void *  ptr1;
 635                 void *  ptr2;
 636
 637                 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
 638                 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
 639
 640                 if (ptr1 == ptr2)
 641                         result = false;
 642         }
 643
 644     return result;
 645 }