iokit/Kernel/IOWorkLoop.cpp

   1 /*
   2  * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <pexpert/pexpert.h>
  30 #include <IOKit/IOWorkLoop.h>
  31 #include <IOKit/IOEventSource.h>
  32 #include <IOKit/IOInterruptEventSource.h>
  33 #include <IOKit/IOCommandGate.h>
  34 #include <IOKit/IOCommandPool.h>
  35 #include <IOKit/IOTimeStamp.h>
  36 #include <IOKit/IOKitDebug.h>
  37 #include <libkern/OSDebug.h>
  38 #include <kern/thread.h>
  39
  40 #define super OSObject
  41
  42 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
  43
  44 // Block of unused functions intended for future use
  45 #if __LP64__
  46 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
  47 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
  48 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
  49 #else
  50 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
  51 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
  52 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
  53 #endif
  54 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
  55 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
  56 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
  57 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
  58 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
  59
  60 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
  61 static inline void SETP(void *addr, unsigned int flag)
  62     { unsigned char *num = (unsigned char *) addr; *num |= flag; }
  63 static inline void CLRP(void *addr, unsigned int flag)
  64     { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
  65 static inline bool ISSETP(void *addr, unsigned int flag)
  66     { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
  67
  68 #define fFlags loopRestart
  69
  70 #define passiveEventChain       reserved->passiveEventChain
  71
  72 #if IOKITSTATS
  73
  74 #define IOStatisticsRegisterCounter() \
  75 do { \
  76         reserved->counter = IOStatistics::registerWorkLoop(this); \
  77 } while(0)
  78
  79 #define IOStatisticsUnregisterCounter() \
  80 do { \
  81         if (reserved) \
  82                 IOStatistics::unregisterWorkLoop(reserved->counter); \
  83 } while(0)
  84
  85 #define IOStatisticsOpenGate() \
  86 do { \
  87         IOStatistics::countWorkLoopOpenGate(reserved->counter); \
  88         if (reserved->lockInterval) lockTime();                 \
  89 } while(0)
  90 #define IOStatisticsCloseGate() \
  91 do { \
  92         IOStatistics::countWorkLoopCloseGate(reserved->counter);                    \
  93         if (reserved->lockInterval) reserved->lockTime = mach_absolute_time();      \
  94 } while(0)
  95
  96 #define IOStatisticsAttachEventSource() \
  97 do { \
  98         IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
  99 } while(0)
 100
 101 #define IOStatisticsDetachEventSource() \
 102 do { \
 103         IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
 104 } while(0)
 105
 106 #else
 107
 108 #define IOStatisticsRegisterCounter()
 109 #define IOStatisticsUnregisterCounter()
 110 #define IOStatisticsOpenGate()
 111 #define IOStatisticsCloseGate()
 112 #define IOStatisticsAttachEventSource()
 113 #define IOStatisticsDetachEventSource()
 114
 115 #endif /* IOKITSTATS */
 116
 117 bool IOWorkLoop::init()
 118 {
 119     // The super init and gateLock allocation MUST be done first.
 120     if ( !super::init() )
 121         return false;
 122
 123         // Allocate our ExpansionData if it hasn't been allocated already.
 124         if ( !reserved )
 125         {
 126                 reserved = IONew(ExpansionData,1);
 127                 if ( !reserved )
 128                         return false;
 129
 130                 bzero(reserved,sizeof(ExpansionData));
 131         }
 132
 133 #if DEBUG
 134         OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
 135 #endif
 136
 137     if ( gateLock == NULL ) {
 138         if ( !( gateLock = IORecursiveLockAlloc()) )
 139             return false;
 140     }
 141
 142     if ( workToDoLock == NULL ) {
 143         if ( !(workToDoLock = IOSimpleLockAlloc()) )
 144             return false;
 145         IOSimpleLockInit(workToDoLock);
 146         workToDo = false;
 147     }
 148
 149     IOStatisticsRegisterCounter();
 150
 151     if ( controlG == NULL ) {
 152         controlG = IOCommandGate::commandGate(
 153             this,
 154             OSMemberFunctionCast(
 155                 IOCommandGate::Action,
 156                 this,
 157                 &IOWorkLoop::_maintRequest));
 158
 159         if ( !controlG )
 160             return false;
 161         // Point the controlGate at the workLoop.  Usually addEventSource
 162         // does this automatically.  The problem is in this case addEventSource
 163         // uses the control gate and it has to be bootstrapped.
 164         controlG->setWorkLoop(this);
 165         if (addEventSource(controlG) != kIOReturnSuccess)
 166             return false;
 167     }
 168
 169     if ( workThread == NULL ) {
 170         thread_continue_t cptr = OSMemberFunctionCast(
 171             thread_continue_t,
 172             this,
 173             &IOWorkLoop::threadMain);
 174         if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
 175             return false;
 176     }
 177
 178     (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
 179     return true;
 180 }
 181
 182 IOWorkLoop *
 183 IOWorkLoop::workLoop()
 184 {
 185     return IOWorkLoop::workLoopWithOptions(0);
 186 }
 187
 188 IOWorkLoop *
 189 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
 190 {
 191         IOWorkLoop *me = new IOWorkLoop;
 192
 193         if (me && options) {
 194                 me->reserved = IONew(ExpansionData,1);
 195                 if (!me->reserved) {
 196                         me->release();
 197                         return 0;
 198                 }
 199                 bzero(me->reserved,sizeof(ExpansionData));
 200                 me->reserved->options = options;
 201         }
 202
 203         if (me && !me->init()) {
 204                 me->release();
 205                 return 0;
 206         }
 207
 208         return me;
 209 }
 210
 211 // Free is called twice:
 212 // First when the atomic retainCount transitions from 1 -> 0
 213 // Secondly when the work loop itself is commiting hari kari
 214 // Hence the each leg of the free must be single threaded.
 215 void IOWorkLoop::free()
 216 {
 217     if (workThread) {
 218         IOInterruptState is;
 219
 220         // If we are here then we must be trying to shut down this work loop
 221         // in this case disable all of the event source, mark the loop
 222         // as terminating and wakeup the work thread itself and return
 223         // Note: we hold the gate across the entire operation mainly for the
 224         // benefit of our event sources so we can disable them cleanly.
 225         closeGate();
 226
 227         disableAllEventSources();
 228
 229         is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 230         SETP(&fFlags, kLoopTerminate);
 231         thread_wakeup_one((void *) &workToDo);
 232         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 233
 234         openGate();
 235     }
 236     else /* !workThread */ {
 237         IOEventSource *event, *next;
 238
 239         for (event = eventChain; event; event = next) {
 240             next = event->getNext();
 241             event->setWorkLoop(0);
 242             event->setNext(0);
 243             event->release();
 244         }
 245         eventChain = 0;
 246
 247         for (event = passiveEventChain; event; event = next) {
 248             next = event->getNext();
 249             event->setWorkLoop(0);
 250             event->setNext(0);
 251             event->release();
 252         }
 253         passiveEventChain = 0;
 254
 255         // Either we have a partial initialization to clean up
 256         // or the workThread itself is performing hari-kari.
 257         // Either way clean up all of our resources and return.
 258
 259         if (controlG) {
 260             controlG->workLoop = 0;
 261             controlG->release();
 262             controlG = 0;
 263         }
 264
 265         if (workToDoLock) {
 266             IOSimpleLockFree(workToDoLock);
 267             workToDoLock = 0;
 268         }
 269
 270         if (gateLock) {
 271             IORecursiveLockFree(gateLock);
 272             gateLock = 0;
 273         }
 274
 275         IOStatisticsUnregisterCounter();
 276
 277         if (reserved) {
 278             IODelete(reserved, ExpansionData, 1);
 279             reserved = 0;
 280         }
 281
 282         super::free();
 283     }
 284 }
 285
 286 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
 287 {
 288     if ((workThread)
 289       && !thread_has_thread_name(workThread)
 290       && (newEvent->owner)
 291       && !OSDynamicCast(IOCommandPool, newEvent->owner)) {
 292         thread_set_thread_name(workThread, newEvent->owner->getMetaClass()->getClassName());
 293     }
 294
 295     return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
 296 }
 297
 298 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
 299 {
 300     return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
 301 }
 302
 303 void IOWorkLoop::enableAllEventSources() const
 304 {
 305     IOEventSource *event;
 306
 307     for (event = eventChain; event; event = event->getNext())
 308         event->enable();
 309
 310     for (event = passiveEventChain; event; event = event->getNext())
 311         event->enable();
 312 }
 313
 314 void IOWorkLoop::disableAllEventSources() const
 315 {
 316     IOEventSource *event;
 317
 318     for (event = eventChain; event; event = event->getNext())
 319                 event->disable();
 320
 321         /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
 322     for (event = passiveEventChain; event; event = event->getNext())
 323         if (event != controlG)  // Don't disable the control gate
 324             event->disable();
 325 }
 326
 327 void IOWorkLoop::enableAllInterrupts() const
 328 {
 329     IOEventSource *event;
 330
 331     for (event = eventChain; event; event = event->getNext())
 332         if (OSDynamicCast(IOInterruptEventSource, event))
 333             event->enable();
 334 }
 335
 336 void IOWorkLoop::disableAllInterrupts() const
 337 {
 338     IOEventSource *event;
 339
 340     for (event = eventChain; event; event = event->getNext())
 341         if (OSDynamicCast(IOInterruptEventSource, event))
 342             event->disable();
 343 }
 344
 345
 346 /* virtual */ bool IOWorkLoop::runEventSources()
 347 {
 348     bool res = false;
 349     bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
 350     bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
 351
 352     closeGate();
 353     if (ISSETP(&fFlags, kLoopTerminate))
 354                 goto abort;
 355
 356     if (traceWL)
 357         IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 358
 359     bool more;
 360     do {
 361                 CLRP(&fFlags, kLoopRestart);
 362                 more = false;
 363                 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 364                 workToDo = false;
 365                 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 366                 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
 367                 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
 368
 369                         if (traceES)
 370                                 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 371
 372                         more |= evnt->checkForWork();
 373
 374                         if (traceES)
 375                                 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
 376
 377                         if (ISSETP(&fFlags, kLoopTerminate))
 378                                 goto abort;
 379                         else if (fFlags & kLoopRestart) {
 380                                 more = true;
 381                                 break;
 382                         }
 383                 }
 384     } while (more);
 385
 386     res = true;
 387
 388     if (traceWL)
 389         IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
 390
 391 abort:
 392     openGate();
 393     return res;
 394 }
 395
 396 /* virtual */ void IOWorkLoop::threadMain()
 397 {
 398 restartThread:
 399     do {
 400         if ( !runEventSources() )
 401             goto exitThread;
 402
 403         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 404         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 405             assert_wait((void *) &workToDo, false);
 406             IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 407             thread_continue_t cptr = NULL;
 408             if (!reserved || !(kPreciousStack & reserved->options))
 409                 cptr = OSMemberFunctionCast(
 410                         thread_continue_t, this, &IOWorkLoop::threadMain);
 411             thread_block_parameter(cptr, this);
 412             goto restartThread;
 413             /* NOTREACHED */
 414         }
 415
 416         // At this point we either have work to do or we need
 417         // to commit suicide.  But no matter
 418         // Clear the simple lock and retore the interrupt state
 419         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 420     } while(workToDo);
 421
 422 exitThread:
 423         thread_t thread = workThread;
 424     workThread = 0;     // Say we don't have a loop and free ourselves
 425     free();
 426
 427         thread_deallocate(thread);
 428     (void) thread_terminate(thread);
 429 }
 430
 431 IOThread IOWorkLoop::getThread() const
 432 {
 433     return workThread;
 434 }
 435
 436 bool IOWorkLoop::onThread() const
 437 {
 438     return (IOThreadSelf() == workThread);
 439 }
 440
 441 bool IOWorkLoop::inGate() const
 442 {
 443     return IORecursiveLockHaveLock(gateLock);
 444 }
 445
 446 // Internal APIs used by event sources to control the thread
 447 void IOWorkLoop::signalWorkAvailable()
 448 {
 449     if (workToDoLock) {
 450         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
 451         workToDo = true;
 452         thread_wakeup_one((void *) &workToDo);
 453         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 454     }
 455 }
 456
 457 void IOWorkLoop::openGate()
 458 {
 459     IOStatisticsOpenGate();
 460     IORecursiveLockUnlock(gateLock);
 461 }
 462
 463 void IOWorkLoop::closeGate()
 464 {
 465     IORecursiveLockLock(gateLock);
 466     IOStatisticsCloseGate();
 467 }
 468
 469 bool IOWorkLoop::tryCloseGate()
 470 {
 471     bool res = (IORecursiveLockTryLock(gateLock) != 0);
 472     if (res) {
 473         IOStatisticsCloseGate();
 474     }
 475     return res;
 476 }
 477
 478 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
 479 {
 480     int res;
 481     IOStatisticsOpenGate();
 482     res = IORecursiveLockSleep(gateLock, event, interuptibleType);
 483     IOStatisticsCloseGate();
 484     return res;
 485 }
 486
 487 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
 488 {
 489     int res;
 490     IOStatisticsOpenGate();
 491     res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
 492     IOStatisticsCloseGate();
 493     return res;
 494 }
 495
 496 void IOWorkLoop::wakeupGate(void *event, bool oneThread)
 497 {
 498     IORecursiveLockWakeup(gateLock, event, oneThread);
 499 }
 500
 501 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
 502                                   void *arg0, void *arg1,
 503                                   void *arg2, void *arg3)
 504 {
 505     IOReturn res;
 506
 507     // closeGate is recursive so don't worry if we already hold the lock.
 508     closeGate();
 509     res = (*inAction)(target, arg0, arg1, arg2, arg3);
 510     openGate();
 511
 512     return res;
 513 }
 514
 515 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
 516 {
 517     maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
 518     IOEventSource *inEvent = (IOEventSource *) inD;
 519     IOReturn res = kIOReturnSuccess;
 520
 521     switch (command)
 522     {
 523     case mAddEvent:
 524         if (!inEvent->getWorkLoop()) {
 525             SETP(&fFlags, kLoopRestart);
 526
 527             inEvent->retain();
 528             inEvent->setWorkLoop(this);
 529             inEvent->setNext(0);
 530
 531                 /* Check if this is a passive or active event source being added */
 532                 if (eventSourcePerformsWork(inEvent)) {
 533
 534                     if (!eventChain)
 535                     eventChain = inEvent;
 536                     else {
 537                     IOEventSource *event, *next;
 538
 539                         for (event = eventChain; (next = event->getNext()); event = next)
 540                         ;
 541                         event->setNext(inEvent);
 542
 543                 }
 544
 545             }
 546             else {
 547
 548                     if (!passiveEventChain)
 549                     passiveEventChain = inEvent;
 550                     else {
 551                     IOEventSource *event, *next;
 552
 553                         for (event = passiveEventChain; (next = event->getNext()); event = next)
 554                         ;
 555                         event->setNext(inEvent);
 556
 557                 }
 558
 559             }
 560             IOStatisticsAttachEventSource();
 561         }
 562         break;
 563
 564     case mRemoveEvent:
 565         if (inEvent->getWorkLoop()) {
 566                 IOStatisticsDetachEventSource();
 567
 568                 if (eventSourcePerformsWork(inEvent)) {
 569                                 if (eventChain == inEvent)
 570                                         eventChain = inEvent->getNext();
 571                                 else {
 572                                         IOEventSource *event, *next = 0;
 573
 574                                         event = eventChain;
 575                                         if (event) while ((next = event->getNext()) && (next != inEvent))
 576                                                 event = next;
 577
 578                                         if (!next) {
 579                                                 res = kIOReturnBadArgument;
 580                                                 break;
 581                                         }
 582                                         event->setNext(inEvent->getNext());
 583                                 }
 584                 }
 585                 else {
 586                                 if (passiveEventChain == inEvent)
 587                                         passiveEventChain = inEvent->getNext();
 588                                 else {
 589                                         IOEventSource *event, *next = 0;
 590
 591                                         event = passiveEventChain;
 592                                         if (event) while ((next = event->getNext()) && (next != inEvent))
 593                                                 event = next;
 594
 595                                         if (!next) {
 596                                                 res = kIOReturnBadArgument;
 597                                                 break;
 598                                         }
 599                                         event->setNext(inEvent->getNext());
 600                                 }
 601                 }
 602
 603             inEvent->setWorkLoop(0);
 604             inEvent->setNext(0);
 605             inEvent->release();
 606             SETP(&fFlags, kLoopRestart);
 607         }
 608         break;
 609
 610     default:
 611         return kIOReturnUnsupported;
 612     }
 613
 614     return res;
 615 }
 616
 617 bool
 618 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
 619 {
 620         bool    result = true;
 621
 622         /*
 623          * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
 624          * The assumption is that if you override checkForWork(), you need to be
 625          * active and not passive.
 626          *
 627          * We picked a known quantity controlG that does not override
 628          * IOEventSource::checkForWork(), namely the IOCommandGate associated with
 629          * the workloop to which this event source is getting attached.
 630          *
 631          * We do a pointer comparison on the offset in the vtable for inNewEvent against
 632          * the offset in the vtable for inReferenceEvent. This works because
 633          * IOCommandGate's slot for checkForWork() has the address of
 634          * IOEventSource::checkForWork() in it.
 635          *
 636          * Think of OSMemberFunctionCast yielding the value at the vtable offset for
 637          * checkForWork() here. We're just testing to see if it's the same or not.
 638          *
 639          */
 640         if (controlG) {
 641                 void *  ptr1;
 642                 void *  ptr2;
 643
 644                 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
 645                 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
 646
 647                 if (ptr1 == ptr2)
 648                         result = false;
 649         }
 650
 651     return result;
 652 }
 653
 654 void
 655 IOWorkLoop::lockTime(void)
 656 {
 657     uint64_t time;
 658     time = mach_absolute_time() - reserved->lockTime;
 659     if (time > reserved->lockInterval)
 660     {
 661         absolutetime_to_nanoseconds(time, &time);
 662         if (kTimeLockPanics & reserved->options) panic("IOWorkLoop %p lock time %qd us", this, time / 1000ULL);
 663         else                     OSReportWithBacktrace("IOWorkLoop %p lock time %qd us", this, time / 1000ULL);
 664     }
 665 }
 666
 667 void
 668 IOWorkLoop::setMaximumLockTime(uint64_t interval, uint32_t options)
 669 {
 670     IORecursiveLockLock(gateLock);
 671     reserved->lockInterval = interval;
 672     reserved->options = (reserved->options & ~kTimeLockPanics) | (options & kTimeLockPanics);
 673     IORecursiveLockUnlock(gateLock);
 674 }