X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/43866e378188c25dd1e2208016ab3cbeb086ae6c..7e41aa883dd258f888d0470250eead40a53ef1f5:/iokit/Kernel/IOWorkLoop.cpp diff --git a/iokit/Kernel/IOWorkLoop.cpp b/iokit/Kernel/IOWorkLoop.cpp index a6bfd4673..6207b1ea1 100644 --- a/iokit/Kernel/IOWorkLoop.cpp +++ b/iokit/Kernel/IOWorkLoop.cpp @@ -1,16 +1,19 @@ /* - * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1998-2010 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ - * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -20,30 +23,33 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -/* -Copyright (c) 1998 Apple Computer, Inc. All rights reserved. -HISTORY - 1998-7-13 Godfrey van der Linden(gvdl) - Created. -*/ +#include #include #include #include #include #include +#include +#include +#include #define super OSObject OSDefineMetaClassAndStructors(IOWorkLoop, OSObject); // Block of unused functions intended for future use -OSMetaClassDefineReservedUsed(IOWorkLoop, 0); - +#if __LP64__ +OSMetaClassDefineReservedUnused(IOWorkLoop, 0); OSMetaClassDefineReservedUnused(IOWorkLoop, 1); OSMetaClassDefineReservedUnused(IOWorkLoop, 2); +#else +OSMetaClassDefineReservedUsed(IOWorkLoop, 0); +OSMetaClassDefineReservedUsed(IOWorkLoop, 1); +OSMetaClassDefineReservedUsed(IOWorkLoop, 2); +#endif OSMetaClassDefineReservedUnused(IOWorkLoop, 3); OSMetaClassDefineReservedUnused(IOWorkLoop, 4); OSMetaClassDefineReservedUnused(IOWorkLoop, 5); @@ -52,71 +58,152 @@ OSMetaClassDefineReservedUnused(IOWorkLoop, 7); enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 }; static inline void SETP(void *addr, unsigned int flag) - { unsigned int *num = (unsigned int *) addr; *num |= flag; } + { unsigned char *num = (unsigned char *) addr; *num |= flag; } static inline void CLRP(void *addr, unsigned int flag) - { unsigned int *num = (unsigned int *) addr; *num &= ~flag; } + { unsigned char *num = (unsigned char *) addr; *num &= ~flag; } static inline bool ISSETP(void *addr, unsigned int flag) - { unsigned int *num = (unsigned int *) addr; return (*num & flag) != 0; } + { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; } #define fFlags loopRestart -extern "C" extern void stack_privilege( thread_t thread); +#define passiveEventChain reserved->passiveEventChain -void IOWorkLoop::launchThreadMain(void *self) -{ - register thread_t mythread = current_thread(); +#if IOKITSTATS - // Make sure that this thread always has a kernel stack - stack_privilege(mythread); - thread_set_cont_arg((int) self); - threadMainContinuation(); -} +#define IOStatisticsRegisterCounter() \ +do { \ + reserved->counter = IOStatistics::registerWorkLoop(this); \ +} while(0) -bool IOWorkLoop::init() -{ - // The super init and gateLock allocation MUST be done first - if ( !super::init() ) - return false; +#define IOStatisticsUnregisterCounter() \ +do { \ + if (reserved) \ + IOStatistics::unregisterWorkLoop(reserved->counter); \ +} while(0) - if ( !(gateLock = IORecursiveLockAlloc()) ) - return false; +#define IOStatisticsOpenGate() \ +do { \ + IOStatistics::countWorkLoopOpenGate(reserved->counter); \ +} while(0) - if ( !(workToDoLock = IOSimpleLockAlloc()) ) - return false; +#define IOStatisticsCloseGate() \ +do { \ + IOStatistics::countWorkLoopCloseGate(reserved->counter); \ +} while(0) - controlG = IOCommandGate:: - commandGate(this, (IOCommandGate::Action) &IOWorkLoop::_maintRequest); - if ( !controlG ) - return false; +#define IOStatisticsAttachEventSource() \ +do { \ + IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ +} while(0) + +#define IOStatisticsDetachEventSource() \ +do { \ + IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ +} while(0) - IOSimpleLockInit(workToDoLock); - workToDo = false; +#else - // Point the controlGate at the workLoop. Usually addEventSource - // does this automatically. The problem is in this case addEventSource - // uses the control gate and it has to be bootstrapped. - controlG->setWorkLoop(this); - if (addEventSource(controlG) != kIOReturnSuccess) - return false; +#define IOStatisticsRegisterCounter() +#define IOStatisticsUnregisterCounter() +#define IOStatisticsOpenGate() +#define IOStatisticsCloseGate() +#define IOStatisticsAttachEventSource() +#define IOStatisticsDetachEventSource() - workThread = IOCreateThread(launchThreadMain, (void *) this); - if (!workThread) +#endif /* IOKITSTATS */ + +bool IOWorkLoop::init() +{ + // The super init and gateLock allocation MUST be done first. + if ( !super::init() ) return false; + + // Allocate our ExpansionData if it hasn't been allocated already. + if ( !reserved ) + { + reserved = IONew(ExpansionData,1); + if ( !reserved ) + return false; + + bzero(reserved,sizeof(ExpansionData)); + } + +#if DEBUG + OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) ); +#endif + + if ( gateLock == NULL ) { + if ( !( gateLock = IORecursiveLockAlloc()) ) + return false; + } + + if ( workToDoLock == NULL ) { + if ( !(workToDoLock = IOSimpleLockAlloc()) ) + return false; + IOSimpleLockInit(workToDoLock); + workToDo = false; + } + + IOStatisticsRegisterCounter(); + + if ( controlG == NULL ) { + controlG = IOCommandGate::commandGate( + this, + OSMemberFunctionCast( + IOCommandGate::Action, + this, + &IOWorkLoop::_maintRequest)); + + if ( !controlG ) + return false; + // Point the controlGate at the workLoop. Usually addEventSource + // does this automatically. The problem is in this case addEventSource + // uses the control gate and it has to be bootstrapped. + controlG->setWorkLoop(this); + if (addEventSource(controlG) != kIOReturnSuccess) + return false; + } + + if ( workThread == NULL ) { + thread_continue_t cptr = OSMemberFunctionCast( + thread_continue_t, + this, + &IOWorkLoop::threadMain); + if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) + return false; + } + (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP); return true; } IOWorkLoop * IOWorkLoop::workLoop() { - IOWorkLoop *me = new IOWorkLoop; - - if (me && !me->init()) { - me->free(); - return 0; - } + return IOWorkLoop::workLoopWithOptions(0); +} - return me; +IOWorkLoop * +IOWorkLoop::workLoopWithOptions(IOOptionBits options) +{ + IOWorkLoop *me = new IOWorkLoop; + + if (me && options) { + me->reserved = IONew(ExpansionData,1); + if (!me->reserved) { + me->release(); + return 0; + } + bzero(me->reserved,sizeof(ExpansionData)); + me->reserved->options = options; + } + + if (me && !me->init()) { + me->release(); + return 0; + } + + return me; } // Free is called twice: @@ -129,7 +216,7 @@ void IOWorkLoop::free() IOInterruptState is; // If we are here then we must be trying to shut down this work loop - // in this case disable all of the event source, mark the loop for + // in this case disable all of the event source, mark the loop // as terminating and wakeup the work thread itself and return // Note: we hold the gate across the entire operation mainly for the // benefit of our event sources so we can disable them cleanly. @@ -155,11 +242,20 @@ void IOWorkLoop::free() } eventChain = 0; - // Either we have a partial initialisation to clean up - // or we the workThread itself is performing hari-kari. - // either way clean up all of our resources and return. + for (event = passiveEventChain; event; event = next) { + next = event->getNext(); + event->setWorkLoop(0); + event->setNext(0); + event->release(); + } + passiveEventChain = 0; + + // Either we have a partial initialization to clean up + // or the workThread itself is performing hari-kari. + // Either way clean up all of our resources and return. if (controlG) { + controlG->workLoop = 0; controlG->release(); controlG = 0; } @@ -173,6 +269,13 @@ void IOWorkLoop::free() IORecursiveLockFree(gateLock); gateLock = 0; } + + IOStatisticsUnregisterCounter(); + + if (reserved) { + IODelete(reserved, ExpansionData, 1); + reserved = 0; + } super::free(); } @@ -194,6 +297,9 @@ void IOWorkLoop::enableAllEventSources() const for (event = eventChain; event; event = event->getNext()) event->enable(); + + for (event = passiveEventChain; event; event = event->getNext()) + event->enable(); } void IOWorkLoop::disableAllEventSources() const @@ -201,6 +307,10 @@ void IOWorkLoop::disableAllEventSources() const IOEventSource *event; for (event = eventChain; event; event = event->getNext()) + event->disable(); + + /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */ + for (event = passiveEventChain; event; event = event->getNext()) if (event != controlG) // Don't disable the control gate event->disable(); } @@ -208,7 +318,7 @@ void IOWorkLoop::disableAllEventSources() const void IOWorkLoop::enableAllInterrupts() const { IOEventSource *event; - + for (event = eventChain; event; event = event->getNext()) if (OSDynamicCast(IOInterruptEventSource, event)) event->enable(); @@ -217,94 +327,80 @@ void IOWorkLoop::enableAllInterrupts() const void IOWorkLoop::disableAllInterrupts() const { IOEventSource *event; - + for (event = eventChain; event; event = event->getNext()) if (OSDynamicCast(IOInterruptEventSource, event)) event->disable(); } -#if KDEBUG -#define IOTimeClientS() \ -do { \ - IOTimeStampStart(IODBG_WORKLOOP(IOWL_CLIENT), \ - (unsigned int) this, (unsigned int) event); \ -} while(0) - -#define IOTimeClientE() \ -do { \ - IOTimeStampEnd(IODBG_WORKLOOP(IOWL_CLIENT), \ - (unsigned int) this, (unsigned int) event); \ -} while(0) - -#define IOTimeWorkS() \ -do { \ - IOTimeStampStart(IODBG_WORKLOOP(IOWL_WORK), (unsigned int) this); \ -} while(0) - -#define IOTimeWorkE() \ -do { \ - IOTimeStampEnd(IODBG_WORKLOOP(IOWL_WORK),(unsigned int) this); \ -} while(0) - -#else /* !KDEBUG */ - -#define IOTimeClientS() -#define IOTimeClientE() -#define IOTimeWorkS() -#define IOTimeWorkE() - -#endif /* KDEBUG */ -void IOWorkLoop::threadMainContinuation() +/* virtual */ bool IOWorkLoop::runEventSources() { - IOWorkLoop* self; - self = (IOWorkLoop *) thread_get_cont_arg(); - - self->threadMain(); + bool res = false; + bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false; + bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false; + + closeGate(); + if (ISSETP(&fFlags, kLoopTerminate)) + goto abort; + + if (traceWL) + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + + bool more; + do { + CLRP(&fFlags, kLoopRestart); + more = false; + IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); + workToDo = false; + IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); + /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */ + for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { + + if (traceES) + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + + more |= evnt->checkForWork(); + + if (traceES) + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + + if (ISSETP(&fFlags, kLoopTerminate)) + goto abort; + else if (fFlags & kLoopRestart) { + more = true; + break; + } + } + } while (more); + + res = true; + + if (traceWL) + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + +abort: + openGate(); + return res; } -void IOWorkLoop::threadMain() +/* virtual */ void IOWorkLoop::threadMain() { - CLRP(&fFlags, kLoopRestart); - - for (;;) { - bool more; - IOInterruptState is; - - IOTimeWorkS(); - - closeGate(); - if (ISSETP(&fFlags, kLoopTerminate)) +restartThread: + do { + if ( !runEventSources() ) goto exitThread; - do { - workToDo = more = false; - for (IOEventSource *event = eventChain; event; event = event->getNext()) { - - IOTimeClientS(); - more |= event->checkForWork(); - IOTimeClientE(); - - if (ISSETP(&fFlags, kLoopTerminate)) - goto exitThread; - else if (fFlags & kLoopRestart) { - CLRP(&fFlags, kLoopRestart); - continue; - } - } - } while (more); - - IOTimeWorkE(); - - openGate(); - - is = IOSimpleLockLockDisableInterrupt(workToDoLock); + IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) { assert_wait((void *) &workToDo, false); IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); - - thread_set_cont_arg((int) this); - thread_block(&threadMainContinuation); + thread_continue_t cptr = NULL; + if (!reserved || !(kPreciousStack & reserved->options)) + cptr = OSMemberFunctionCast( + thread_continue_t, this, &IOWorkLoop::threadMain); + thread_block_parameter(cptr, this); + goto restartThread; /* NOTREACHED */ } @@ -312,16 +408,15 @@ void IOWorkLoop::threadMain() // to commit suicide. But no matter // Clear the simple lock and retore the interrupt state IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); - if (workToDo) - continue; - else - break; - } + } while(workToDo); exitThread: + thread_t thread = workThread; workThread = 0; // Say we don't have a loop and free ourselves free(); - IOExitThread(); + + thread_deallocate(thread); + (void) thread_terminate(thread); } IOThread IOWorkLoop::getThread() const @@ -352,22 +447,41 @@ void IOWorkLoop::signalWorkAvailable() void IOWorkLoop::openGate() { + IOStatisticsOpenGate(); IORecursiveLockUnlock(gateLock); } void IOWorkLoop::closeGate() { IORecursiveLockLock(gateLock); + IOStatisticsCloseGate(); } bool IOWorkLoop::tryCloseGate() { - return IORecursiveLockTryLock(gateLock) != 0; + bool res = (IORecursiveLockTryLock(gateLock) != 0); + if (res) { + IOStatisticsCloseGate(); + } + return res; } int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType) { - return IORecursiveLockSleep(gateLock, event, interuptibleType); + int res; + IOStatisticsOpenGate(); + res = IORecursiveLockSleep(gateLock, event, interuptibleType); + IOStatisticsCloseGate(); + return res; +} + +int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType) +{ + int res; + IOStatisticsOpenGate(); + res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType); + IOStatisticsCloseGate(); + return res; } void IOWorkLoop::wakeupGate(void *event, bool oneThread) @@ -376,8 +490,8 @@ void IOWorkLoop::wakeupGate(void *event, bool oneThread) } IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, - void *arg0 = 0, void *arg1 = 0, - void *arg2 = 0, void *arg3 = 0) + void *arg0, void *arg1, + void *arg2, void *arg3) { IOReturn res; @@ -391,7 +505,7 @@ IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) { - maintCommandEnum command = (maintCommandEnum) (vm_address_t) inC; + maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC; IOEventSource *inEvent = (IOEventSource *) inD; IOReturn res = kIOReturnSuccess; @@ -404,37 +518,79 @@ IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) inEvent->retain(); inEvent->setWorkLoop(this); inEvent->setNext(0); + + /* Check if this is a passive or active event source being added */ + if (eventSourcePerformsWork(inEvent)) { + + if (!eventChain) + eventChain = inEvent; + else { + IOEventSource *event, *next; - if (!eventChain) - eventChain = inEvent; + for (event = eventChain; (next = event->getNext()); event = next) + ; + event->setNext(inEvent); + + } + + } else { - IOEventSource *event, *next; + + if (!passiveEventChain) + passiveEventChain = inEvent; + else { + IOEventSource *event, *next; - for (event = eventChain; (next = event->getNext()); event = next) - ; - event->setNext(inEvent); + for (event = passiveEventChain; (next = event->getNext()); event = next) + ; + event->setNext(inEvent); + + } + } + IOStatisticsAttachEventSource(); } break; case mRemoveEvent: if (inEvent->getWorkLoop()) { - if (eventChain == inEvent) - eventChain = inEvent->getNext(); - else { - IOEventSource *event, *next; - - event = eventChain; - while ((next = event->getNext()) && next != inEvent) - event = next; - - if (!next) { - res = kIOReturnBadArgument; - break; - } - event->setNext(inEvent->getNext()); - } - + IOStatisticsDetachEventSource(); + + if (eventSourcePerformsWork(inEvent)) { + if (eventChain == inEvent) + eventChain = inEvent->getNext(); + else { + IOEventSource *event, *next = 0; + + event = eventChain; + if (event) while ((next = event->getNext()) && (next != inEvent)) + event = next; + + if (!next) { + res = kIOReturnBadArgument; + break; + } + event->setNext(inEvent->getNext()); + } + } + else { + if (passiveEventChain == inEvent) + passiveEventChain = inEvent->getNext(); + else { + IOEventSource *event, *next = 0; + + event = passiveEventChain; + if (event) while ((next = event->getNext()) && (next != inEvent)) + event = next; + + if (!next) { + res = kIOReturnBadArgument; + break; + } + event->setNext(inEvent->getNext()); + } + } + inEvent->setWorkLoop(0); inEvent->setNext(0); inEvent->release(); @@ -448,3 +604,40 @@ IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) return res; } + +bool +IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) +{ + bool result = true; + + /* + * The idea here is to see if the subclass of IOEventSource has overridden checkForWork(). + * The assumption is that if you override checkForWork(), you need to be + * active and not passive. + * + * We picked a known quantity controlG that does not override + * IOEventSource::checkForWork(), namely the IOCommandGate associated with + * the workloop to which this event source is getting attached. + * + * We do a pointer comparison on the offset in the vtable for inNewEvent against + * the offset in the vtable for inReferenceEvent. This works because + * IOCommandGate's slot for checkForWork() has the address of + * IOEventSource::checkForWork() in it. + * + * Think of OSMemberFunctionCast yielding the value at the vtable offset for + * checkForWork() here. We're just testing to see if it's the same or not. + * + */ + if (controlG) { + void * ptr1; + void * ptr2; + + ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork); + ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork); + + if (ptr1 == ptr2) + result = false; + } + + return result; +}