X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4452a7af2eac33dbad800bcc91f2399d62c18f53..7e41aa883dd258f888d0470250eead40a53ef1f5:/iokit/Kernel/IOWorkLoop.cpp diff --git a/iokit/Kernel/IOWorkLoop.cpp b/iokit/Kernel/IOWorkLoop.cpp index c5f757160..6207b1ea1 100644 --- a/iokit/Kernel/IOWorkLoop.cpp +++ b/iokit/Kernel/IOWorkLoop.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1998-2010 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -25,28 +25,31 @@ * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -/* -Copyright (c) 1998 Apple Computer, Inc. All rights reserved. -HISTORY - 1998-7-13 Godfrey van der Linden(gvdl) - Created. -*/ +#include #include #include #include #include #include +#include +#include +#include #define super OSObject OSDefineMetaClassAndStructors(IOWorkLoop, OSObject); // Block of unused functions intended for future use +#if __LP64__ +OSMetaClassDefineReservedUnused(IOWorkLoop, 0); +OSMetaClassDefineReservedUnused(IOWorkLoop, 1); +OSMetaClassDefineReservedUnused(IOWorkLoop, 2); +#else OSMetaClassDefineReservedUsed(IOWorkLoop, 0); OSMetaClassDefineReservedUsed(IOWorkLoop, 1); - -OSMetaClassDefineReservedUnused(IOWorkLoop, 2); +OSMetaClassDefineReservedUsed(IOWorkLoop, 2); +#endif OSMetaClassDefineReservedUnused(IOWorkLoop, 3); OSMetaClassDefineReservedUnused(IOWorkLoop, 4); OSMetaClassDefineReservedUnused(IOWorkLoop, 5); @@ -55,62 +58,152 @@ OSMetaClassDefineReservedUnused(IOWorkLoop, 7); enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 }; static inline void SETP(void *addr, unsigned int flag) - { unsigned int *num = (unsigned int *) addr; *num |= flag; } + { unsigned char *num = (unsigned char *) addr; *num |= flag; } static inline void CLRP(void *addr, unsigned int flag) - { unsigned int *num = (unsigned int *) addr; *num &= ~flag; } + { unsigned char *num = (unsigned char *) addr; *num &= ~flag; } static inline bool ISSETP(void *addr, unsigned int flag) - { unsigned int *num = (unsigned int *) addr; return (*num & flag) != 0; } + { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; } #define fFlags loopRestart -bool IOWorkLoop::init() -{ - // The super init and gateLock allocation MUST be done first - if ( !super::init() ) - return false; +#define passiveEventChain reserved->passiveEventChain - if ( !(gateLock = IORecursiveLockAlloc()) ) - return false; +#if IOKITSTATS - if ( !(workToDoLock = IOSimpleLockAlloc()) ) - return false; +#define IOStatisticsRegisterCounter() \ +do { \ + reserved->counter = IOStatistics::registerWorkLoop(this); \ +} while(0) - controlG = IOCommandGate:: - commandGate(this, OSMemberFunctionCast(IOCommandGate::Action, - this, &IOWorkLoop::_maintRequest)); - if ( !controlG ) - return false; +#define IOStatisticsUnregisterCounter() \ +do { \ + if (reserved) \ + IOStatistics::unregisterWorkLoop(reserved->counter); \ +} while(0) + +#define IOStatisticsOpenGate() \ +do { \ + IOStatistics::countWorkLoopOpenGate(reserved->counter); \ +} while(0) - IOSimpleLockInit(workToDoLock); - workToDo = false; +#define IOStatisticsCloseGate() \ +do { \ + IOStatistics::countWorkLoopCloseGate(reserved->counter); \ +} while(0) - // Point the controlGate at the workLoop. Usually addEventSource - // does this automatically. The problem is in this case addEventSource - // uses the control gate and it has to be bootstrapped. - controlG->setWorkLoop(this); - if (addEventSource(controlG) != kIOReturnSuccess) - return false; +#define IOStatisticsAttachEventSource() \ +do { \ + IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ +} while(0) + +#define IOStatisticsDetachEventSource() \ +do { \ + IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ +} while(0) + +#else - IOThreadFunc cptr = - OSMemberFunctionCast(IOThreadFunc, this, &IOWorkLoop::threadMain); - workThread = IOCreateThread(cptr, this); - if (!workThread) +#define IOStatisticsRegisterCounter() +#define IOStatisticsUnregisterCounter() +#define IOStatisticsOpenGate() +#define IOStatisticsCloseGate() +#define IOStatisticsAttachEventSource() +#define IOStatisticsDetachEventSource() + +#endif /* IOKITSTATS */ + +bool IOWorkLoop::init() +{ + // The super init and gateLock allocation MUST be done first. + if ( !super::init() ) return false; + + // Allocate our ExpansionData if it hasn't been allocated already. + if ( !reserved ) + { + reserved = IONew(ExpansionData,1); + if ( !reserved ) + return false; + + bzero(reserved,sizeof(ExpansionData)); + } + +#if DEBUG + OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) ); +#endif + + if ( gateLock == NULL ) { + if ( !( gateLock = IORecursiveLockAlloc()) ) + return false; + } + + if ( workToDoLock == NULL ) { + if ( !(workToDoLock = IOSimpleLockAlloc()) ) + return false; + IOSimpleLockInit(workToDoLock); + workToDo = false; + } + + IOStatisticsRegisterCounter(); + + if ( controlG == NULL ) { + controlG = IOCommandGate::commandGate( + this, + OSMemberFunctionCast( + IOCommandGate::Action, + this, + &IOWorkLoop::_maintRequest)); + + if ( !controlG ) + return false; + // Point the controlGate at the workLoop. Usually addEventSource + // does this automatically. The problem is in this case addEventSource + // uses the control gate and it has to be bootstrapped. + controlG->setWorkLoop(this); + if (addEventSource(controlG) != kIOReturnSuccess) + return false; + } + if ( workThread == NULL ) { + thread_continue_t cptr = OSMemberFunctionCast( + thread_continue_t, + this, + &IOWorkLoop::threadMain); + if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) + return false; + } + + (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP); return true; } IOWorkLoop * IOWorkLoop::workLoop() { - IOWorkLoop *me = new IOWorkLoop; - - if (me && !me->init()) { - me->release(); - return 0; - } + return IOWorkLoop::workLoopWithOptions(0); +} - return me; +IOWorkLoop * +IOWorkLoop::workLoopWithOptions(IOOptionBits options) +{ + IOWorkLoop *me = new IOWorkLoop; + + if (me && options) { + me->reserved = IONew(ExpansionData,1); + if (!me->reserved) { + me->release(); + return 0; + } + bzero(me->reserved,sizeof(ExpansionData)); + me->reserved->options = options; + } + + if (me && !me->init()) { + me->release(); + return 0; + } + + return me; } // Free is called twice: @@ -123,7 +216,7 @@ void IOWorkLoop::free() IOInterruptState is; // If we are here then we must be trying to shut down this work loop - // in this case disable all of the event source, mark the loop for + // in this case disable all of the event source, mark the loop // as terminating and wakeup the work thread itself and return // Note: we hold the gate across the entire operation mainly for the // benefit of our event sources so we can disable them cleanly. @@ -149,11 +242,20 @@ void IOWorkLoop::free() } eventChain = 0; - // Either we have a partial initialisation to clean up - // or we the workThread itself is performing hari-kari. - // either way clean up all of our resources and return. + for (event = passiveEventChain; event; event = next) { + next = event->getNext(); + event->setWorkLoop(0); + event->setNext(0); + event->release(); + } + passiveEventChain = 0; + + // Either we have a partial initialization to clean up + // or the workThread itself is performing hari-kari. + // Either way clean up all of our resources and return. if (controlG) { + controlG->workLoop = 0; controlG->release(); controlG = 0; } @@ -167,6 +269,13 @@ void IOWorkLoop::free() IORecursiveLockFree(gateLock); gateLock = 0; } + + IOStatisticsUnregisterCounter(); + + if (reserved) { + IODelete(reserved, ExpansionData, 1); + reserved = 0; + } super::free(); } @@ -188,6 +297,9 @@ void IOWorkLoop::enableAllEventSources() const for (event = eventChain; event; event = event->getNext()) event->enable(); + + for (event = passiveEventChain; event; event = event->getNext()) + event->enable(); } void IOWorkLoop::disableAllEventSources() const @@ -195,6 +307,10 @@ void IOWorkLoop::disableAllEventSources() const IOEventSource *event; for (event = eventChain; event; event = event->getNext()) + event->disable(); + + /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */ + for (event = passiveEventChain; event; event = event->getNext()) if (event != controlG) // Don't disable the control gate event->disable(); } @@ -202,7 +318,7 @@ void IOWorkLoop::disableAllEventSources() const void IOWorkLoop::enableAllInterrupts() const { IOEventSource *event; - + for (event = eventChain; event; event = event->getNext()) if (OSDynamicCast(IOInterruptEventSource, event)) event->enable(); @@ -211,74 +327,58 @@ void IOWorkLoop::enableAllInterrupts() const void IOWorkLoop::disableAllInterrupts() const { IOEventSource *event; - + for (event = eventChain; event; event = event->getNext()) if (OSDynamicCast(IOInterruptEventSource, event)) event->disable(); } -#if KDEBUG -#define IOTimeClientS() \ -do { \ - IOTimeStampStart(IODBG_WORKLOOP(IOWL_CLIENT), \ - (unsigned int) this, (unsigned int) event); \ -} while(0) - -#define IOTimeClientE() \ -do { \ - IOTimeStampEnd(IODBG_WORKLOOP(IOWL_CLIENT), \ - (unsigned int) this, (unsigned int) event); \ -} while(0) - -#define IOTimeWorkS() \ -do { \ - IOTimeStampStart(IODBG_WORKLOOP(IOWL_WORK), (unsigned int) this); \ -} while(0) - -#define IOTimeWorkE() \ -do { \ - IOTimeStampEnd(IODBG_WORKLOOP(IOWL_WORK),(unsigned int) this); \ -} while(0) - -#else /* !KDEBUG */ - -#define IOTimeClientS() -#define IOTimeClientE() -#define IOTimeWorkS() -#define IOTimeWorkE() - -#endif /* KDEBUG */ /* virtual */ bool IOWorkLoop::runEventSources() { bool res = false; + bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false; + bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false; + closeGate(); if (ISSETP(&fFlags, kLoopTerminate)) - goto abort; - - IOTimeWorkS(); + goto abort; + + if (traceWL) + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + bool more; do { - CLRP(&fFlags, kLoopRestart); - workToDo = more = false; - for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { - - IOTimeClientS(); - more |= evnt->checkForWork(); - IOTimeClientE(); - - if (ISSETP(&fFlags, kLoopTerminate)) - goto abort; - else if (fFlags & kLoopRestart) { - more = true; - break; - } - } + CLRP(&fFlags, kLoopRestart); + more = false; + IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); + workToDo = false; + IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); + /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */ + for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { + + if (traceES) + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + + more |= evnt->checkForWork(); + + if (traceES) + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + + if (ISSETP(&fFlags, kLoopTerminate)) + goto abort; + else if (fFlags & kLoopRestart) { + more = true; + break; + } + } } while (more); - + res = true; - IOTimeWorkE(); - + + if (traceWL) + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + abort: openGate(); return res; @@ -286,6 +386,7 @@ abort: /* virtual */ void IOWorkLoop::threadMain() { +restartThread: do { if ( !runEventSources() ) goto exitThread; @@ -294,10 +395,12 @@ abort: if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) { assert_wait((void *) &workToDo, false); IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); - - thread_continue_t cptr = OSMemberFunctionCast( - thread_continue_t, this, &IOWorkLoop::threadMain); + thread_continue_t cptr = NULL; + if (!reserved || !(kPreciousStack & reserved->options)) + cptr = OSMemberFunctionCast( + thread_continue_t, this, &IOWorkLoop::threadMain); thread_block_parameter(cptr, this); + goto restartThread; /* NOTREACHED */ } @@ -308,9 +411,12 @@ abort: } while(workToDo); exitThread: + thread_t thread = workThread; workThread = 0; // Say we don't have a loop and free ourselves free(); - IOExitThread(); + + thread_deallocate(thread); + (void) thread_terminate(thread); } IOThread IOWorkLoop::getThread() const @@ -341,22 +447,41 @@ void IOWorkLoop::signalWorkAvailable() void IOWorkLoop::openGate() { + IOStatisticsOpenGate(); IORecursiveLockUnlock(gateLock); } void IOWorkLoop::closeGate() { IORecursiveLockLock(gateLock); + IOStatisticsCloseGate(); } bool IOWorkLoop::tryCloseGate() { - return IORecursiveLockTryLock(gateLock) != 0; + bool res = (IORecursiveLockTryLock(gateLock) != 0); + if (res) { + IOStatisticsCloseGate(); + } + return res; } int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType) { - return IORecursiveLockSleep(gateLock, event, interuptibleType); + int res; + IOStatisticsOpenGate(); + res = IORecursiveLockSleep(gateLock, event, interuptibleType); + IOStatisticsCloseGate(); + return res; +} + +int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType) +{ + int res; + IOStatisticsOpenGate(); + res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType); + IOStatisticsCloseGate(); + return res; } void IOWorkLoop::wakeupGate(void *event, bool oneThread) @@ -380,7 +505,7 @@ IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) { - maintCommandEnum command = (maintCommandEnum) (vm_address_t) inC; + maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC; IOEventSource *inEvent = (IOEventSource *) inD; IOReturn res = kIOReturnSuccess; @@ -393,37 +518,79 @@ IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) inEvent->retain(); inEvent->setWorkLoop(this); inEvent->setNext(0); + + /* Check if this is a passive or active event source being added */ + if (eventSourcePerformsWork(inEvent)) { + + if (!eventChain) + eventChain = inEvent; + else { + IOEventSource *event, *next; - if (!eventChain) - eventChain = inEvent; + for (event = eventChain; (next = event->getNext()); event = next) + ; + event->setNext(inEvent); + + } + + } else { - IOEventSource *event, *next; + + if (!passiveEventChain) + passiveEventChain = inEvent; + else { + IOEventSource *event, *next; - for (event = eventChain; (next = event->getNext()); event = next) - ; - event->setNext(inEvent); + for (event = passiveEventChain; (next = event->getNext()); event = next) + ; + event->setNext(inEvent); + + } + } + IOStatisticsAttachEventSource(); } break; case mRemoveEvent: if (inEvent->getWorkLoop()) { - if (eventChain == inEvent) - eventChain = inEvent->getNext(); - else { - IOEventSource *event, *next; - - event = eventChain; - while ((next = event->getNext()) && next != inEvent) - event = next; - - if (!next) { - res = kIOReturnBadArgument; - break; - } - event->setNext(inEvent->getNext()); - } - + IOStatisticsDetachEventSource(); + + if (eventSourcePerformsWork(inEvent)) { + if (eventChain == inEvent) + eventChain = inEvent->getNext(); + else { + IOEventSource *event, *next = 0; + + event = eventChain; + if (event) while ((next = event->getNext()) && (next != inEvent)) + event = next; + + if (!next) { + res = kIOReturnBadArgument; + break; + } + event->setNext(inEvent->getNext()); + } + } + else { + if (passiveEventChain == inEvent) + passiveEventChain = inEvent->getNext(); + else { + IOEventSource *event, *next = 0; + + event = passiveEventChain; + if (event) while ((next = event->getNext()) && (next != inEvent)) + event = next; + + if (!next) { + res = kIOReturnBadArgument; + break; + } + event->setNext(inEvent->getNext()); + } + } + inEvent->setWorkLoop(0); inEvent->setNext(0); inEvent->release(); @@ -437,3 +604,40 @@ IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) return res; } + +bool +IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) +{ + bool result = true; + + /* + * The idea here is to see if the subclass of IOEventSource has overridden checkForWork(). + * The assumption is that if you override checkForWork(), you need to be + * active and not passive. + * + * We picked a known quantity controlG that does not override + * IOEventSource::checkForWork(), namely the IOCommandGate associated with + * the workloop to which this event source is getting attached. + * + * We do a pointer comparison on the offset in the vtable for inNewEvent against + * the offset in the vtable for inReferenceEvent. This works because + * IOCommandGate's slot for checkForWork() has the address of + * IOEventSource::checkForWork() in it. + * + * Think of OSMemberFunctionCast yielding the value at the vtable offset for + * checkForWork() here. We're just testing to see if it's the same or not. + * + */ + if (controlG) { + void * ptr1; + void * ptr2; + + ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork); + ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork); + + if (ptr1 == ptr2) + result = false; + } + + return result; +}