]>
git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOWorkLoop.cpp
a9aff9f93e54355cac32fcb39cc318cad5af7d12
2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <pexpert/pexpert.h>
30 #include <IOKit/IOWorkLoop.h>
31 #include <IOKit/IOEventSource.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOCommandGate.h>
34 #include <IOKit/IOCommandPool.h>
35 #include <IOKit/IOTimeStamp.h>
36 #include <IOKit/IOKitDebug.h>
37 #include <libkern/OSDebug.h>
38 #include <kern/thread.h>
40 #define super OSObject
42 OSDefineMetaClassAndStructors(IOWorkLoop
, OSObject
);
44 // Block of unused functions intended for future use
46 OSMetaClassDefineReservedUnused(IOWorkLoop
, 0);
47 OSMetaClassDefineReservedUnused(IOWorkLoop
, 1);
48 OSMetaClassDefineReservedUnused(IOWorkLoop
, 2);
50 OSMetaClassDefineReservedUsed(IOWorkLoop
, 0);
51 OSMetaClassDefineReservedUsed(IOWorkLoop
, 1);
52 OSMetaClassDefineReservedUsed(IOWorkLoop
, 2);
54 OSMetaClassDefineReservedUnused(IOWorkLoop
, 3);
55 OSMetaClassDefineReservedUnused(IOWorkLoop
, 4);
56 OSMetaClassDefineReservedUnused(IOWorkLoop
, 5);
57 OSMetaClassDefineReservedUnused(IOWorkLoop
, 6);
58 OSMetaClassDefineReservedUnused(IOWorkLoop
, 7);
60 enum IOWorkLoopState
{ kLoopRestart
= 0x1, kLoopTerminate
= 0x2 };
62 SETP(void *addr
, unsigned int flag
)
64 unsigned char *num
= (unsigned char *) addr
; *num
|= flag
;
67 CLRP(void *addr
, unsigned int flag
)
69 unsigned char *num
= (unsigned char *) addr
; *num
&= ~flag
;
72 ISSETP(void *addr
, unsigned int flag
)
74 unsigned char *num
= (unsigned char *) addr
; return (*num
& flag
) != 0;
77 #define fFlags loopRestart
79 #define passiveEventChain reserved->passiveEventChain
83 #define IOStatisticsRegisterCounter() \
85 reserved->counter = IOStatistics::registerWorkLoop(this); \
88 #define IOStatisticsUnregisterCounter() \
91 IOStatistics::unregisterWorkLoop(reserved->counter); \
94 #define IOStatisticsOpenGate() \
96 IOStatistics::countWorkLoopOpenGate(reserved->counter); \
97 if (reserved->lockInterval) lockTime(); \
99 #define IOStatisticsCloseGate() \
101 IOStatistics::countWorkLoopCloseGate(reserved->counter); \
102 if (reserved->lockInterval) reserved->lockTime = mach_absolute_time(); \
105 #define IOStatisticsAttachEventSource() \
107 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
110 #define IOStatisticsDetachEventSource() \
112 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
117 #define IOStatisticsRegisterCounter()
118 #define IOStatisticsUnregisterCounter()
119 #define IOStatisticsOpenGate()
120 #define IOStatisticsCloseGate()
121 #define IOStatisticsAttachEventSource()
122 #define IOStatisticsDetachEventSource()
124 #endif /* IOKITSTATS */
129 // The super init and gateLock allocation MUST be done first.
130 if (!super::init()) {
134 // Allocate our ExpansionData if it hasn't been allocated already.
136 reserved
= IONew(ExpansionData
, 1);
141 bzero(reserved
, sizeof(ExpansionData
));
144 if (gateLock
== NULL
) {
145 if (!(gateLock
= IORecursiveLockAlloc())) {
150 if (workToDoLock
== NULL
) {
151 if (!(workToDoLock
= IOSimpleLockAlloc())) {
154 IOSimpleLockInit(workToDoLock
);
158 IOStatisticsRegisterCounter();
160 if (controlG
== NULL
) {
161 controlG
= IOCommandGate::commandGate(
163 OSMemberFunctionCast(
164 IOCommandGate::Action
,
166 &IOWorkLoop::_maintRequest
));
171 // Point the controlGate at the workLoop. Usually addEventSource
172 // does this automatically. The problem is in this case addEventSource
173 // uses the control gate and it has to be bootstrapped.
174 controlG
->setWorkLoop(this);
175 if (addEventSource(controlG
) != kIOReturnSuccess
) {
180 if (workThread
== NULL
) {
181 thread_continue_t cptr
= OSMemberFunctionCast(
184 &IOWorkLoop::threadMain
);
185 if (KERN_SUCCESS
!= kernel_thread_start(cptr
, this, &workThread
)) {
190 (void) thread_set_tag(workThread
, THREAD_TAG_IOWORKLOOP
);
195 IOWorkLoop::workLoop()
197 return IOWorkLoop::workLoopWithOptions(0);
201 IOWorkLoop::workLoopWithOptions(IOOptionBits options
)
203 IOWorkLoop
*me
= new IOWorkLoop
;
206 me
->reserved
= IONew(ExpansionData
, 1);
211 bzero(me
->reserved
, sizeof(ExpansionData
));
212 me
->reserved
->options
= options
;
215 if (me
&& !me
->init()) {
223 // Free is called twice:
224 // First when the atomic retainCount transitions from 1 -> 0
225 // Secondly when the work loop itself is commiting hari kari
226 // Hence the each leg of the free must be single threaded.
233 // If we are here then we must be trying to shut down this work loop
234 // in this case disable all of the event source, mark the loop
235 // as terminating and wakeup the work thread itself and return
236 // Note: we hold the gate across the entire operation mainly for the
237 // benefit of our event sources so we can disable them cleanly.
240 disableAllEventSources();
242 is
= IOSimpleLockLockDisableInterrupt(workToDoLock
);
243 SETP(&fFlags
, kLoopTerminate
);
244 thread_wakeup_thread((void *) &workToDo
, workThread
);
245 IOSimpleLockUnlockEnableInterrupt(workToDoLock
, is
);
248 } else { /* !workThread */
249 IOEventSource
*event
, *next
;
251 for (event
= eventChain
; event
; event
= next
) {
252 next
= event
->getNext();
253 event
->setWorkLoop(0);
259 for (event
= passiveEventChain
; event
; event
= next
) {
260 next
= event
->getNext();
261 event
->setWorkLoop(0);
265 passiveEventChain
= 0;
267 // Either we have a partial initialization to clean up
268 // or the workThread itself is performing hari-kari.
269 // Either way clean up all of our resources and return.
272 controlG
->workLoop
= 0;
278 IOSimpleLockFree(workToDoLock
);
283 IORecursiveLockFree(gateLock
);
287 IOStatisticsUnregisterCounter();
290 IODelete(reserved
, ExpansionData
, 1);
299 IOWorkLoop::addEventSource(IOEventSource
*newEvent
)
302 && !thread_has_thread_name(workThread
)
304 && !OSDynamicCast(IOCommandPool
, newEvent
->owner
)) {
305 thread_set_thread_name(workThread
, newEvent
->owner
->getMetaClass()->getClassName());
308 return controlG
->runCommand((void *) mAddEvent
, (void *) newEvent
);
312 IOWorkLoop::removeEventSource(IOEventSource
*toRemove
)
314 return controlG
->runCommand((void *) mRemoveEvent
, (void *) toRemove
);
318 IOWorkLoop::enableAllEventSources() const
320 IOEventSource
*event
;
322 for (event
= eventChain
; event
; event
= event
->getNext()) {
326 for (event
= passiveEventChain
; event
; event
= event
->getNext()) {
332 IOWorkLoop::disableAllEventSources() const
334 IOEventSource
*event
;
336 for (event
= eventChain
; event
; event
= event
->getNext()) {
340 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
341 for (event
= passiveEventChain
; event
; event
= event
->getNext()) {
342 if (event
!= controlG
) { // Don't disable the control gate
349 IOWorkLoop::enableAllInterrupts() const
351 IOEventSource
*event
;
353 for (event
= eventChain
; event
; event
= event
->getNext()) {
354 if (OSDynamicCast(IOInterruptEventSource
, event
)) {
361 IOWorkLoop::disableAllInterrupts() const
363 IOEventSource
*event
;
365 for (event
= eventChain
; event
; event
= event
->getNext()) {
366 if (OSDynamicCast(IOInterruptEventSource
, event
)) {
374 IOWorkLoop::runEventSources()
377 bool traceWL
= (gIOKitTrace
& kIOTraceWorkLoops
) ? true : false;
378 bool traceES
= (gIOKitTrace
& kIOTraceEventSources
) ? true : false;
381 if (ISSETP(&fFlags
, kLoopTerminate
)) {
386 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK
), VM_KERNEL_ADDRHIDE(this));
391 CLRP(&fFlags
, kLoopRestart
);
393 IOInterruptState is
= IOSimpleLockLockDisableInterrupt(workToDoLock
);
395 IOSimpleLockUnlockEnableInterrupt(workToDoLock
, is
);
396 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
397 for (IOEventSource
*evnt
= eventChain
; evnt
; evnt
= evnt
->getNext()) {
399 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT
), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt
));
402 more
|= evnt
->checkForWork();
405 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT
), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt
));
408 if (ISSETP(&fFlags
, kLoopTerminate
)) {
410 } else if (fFlags
& kLoopRestart
) {
420 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK
), VM_KERNEL_ADDRHIDE(this));
429 IOWorkLoop::threadMain()
433 if (!runEventSources()) {
437 IOInterruptState is
= IOSimpleLockLockDisableInterrupt(workToDoLock
);
438 if (!ISSETP(&fFlags
, kLoopTerminate
) && !workToDo
) {
439 assert_wait((void *) &workToDo
, false);
440 IOSimpleLockUnlockEnableInterrupt(workToDoLock
, is
);
441 thread_continue_t cptr
= NULL
;
442 if (!reserved
|| !(kPreciousStack
& reserved
->options
)) {
443 cptr
= OSMemberFunctionCast(
444 thread_continue_t
, this, &IOWorkLoop::threadMain
);
446 thread_block_parameter(cptr
, this);
451 // At this point we either have work to do or we need
452 // to commit suicide. But no matter
453 // Clear the simple lock and retore the interrupt state
454 IOSimpleLockUnlockEnableInterrupt(workToDoLock
, is
);
459 thread_t thread
= workThread
;
460 workThread
= 0; // Say we don't have a loop and free ourselves
465 thread_deallocate(thread
);
466 (void) thread_terminate(thread
);
470 IOWorkLoop::getThread() const
476 IOWorkLoop::onThread() const
478 return IOThreadSelf() == workThread
;
482 IOWorkLoop::inGate() const
484 return IORecursiveLockHaveLock(gateLock
);
487 // Internal APIs used by event sources to control the thread
489 IOWorkLoop::signalWorkAvailable()
492 IOInterruptState is
= IOSimpleLockLockDisableInterrupt(workToDoLock
);
494 thread_wakeup_thread((void *) &workToDo
, workThread
);
495 IOSimpleLockUnlockEnableInterrupt(workToDoLock
, is
);
500 IOWorkLoop::openGate()
502 IOStatisticsOpenGate();
503 IORecursiveLockUnlock(gateLock
);
507 IOWorkLoop::closeGate()
509 IORecursiveLockLock(gateLock
);
510 IOStatisticsCloseGate();
514 IOWorkLoop::tryCloseGate()
516 bool res
= (IORecursiveLockTryLock(gateLock
) != 0);
518 IOStatisticsCloseGate();
524 IOWorkLoop::sleepGate(void *event
, UInt32 interuptibleType
)
527 IOStatisticsOpenGate();
528 res
= IORecursiveLockSleep(gateLock
, event
, interuptibleType
);
529 IOStatisticsCloseGate();
534 IOWorkLoop::sleepGate(void *event
, AbsoluteTime deadline
, UInt32 interuptibleType
)
537 IOStatisticsOpenGate();
538 res
= IORecursiveLockSleepDeadline(gateLock
, event
, deadline
, interuptibleType
);
539 IOStatisticsCloseGate();
544 IOWorkLoop::wakeupGate(void *event
, bool oneThread
)
546 IORecursiveLockWakeup(gateLock
, event
, oneThread
);
550 IOWorkLoopActionToBlock(OSObject
*owner
,
551 void *arg0
, void *arg1
,
552 void *arg2
, void *arg3
)
554 return ((IOWorkLoop::ActionBlock
) arg0
)();
558 IOWorkLoop::runActionBlock(ActionBlock action
)
560 return runAction(&IOWorkLoopActionToBlock
, this, action
);
564 IOWorkLoop::runAction(Action inAction
, OSObject
*target
,
565 void *arg0
, void *arg1
,
566 void *arg2
, void *arg3
)
570 // closeGate is recursive so don't worry if we already hold the lock.
572 res
= (*inAction
)(target
, arg0
, arg1
, arg2
, arg3
);
579 IOWorkLoop::_maintRequest(void *inC
, void *inD
, void *, void *)
581 maintCommandEnum command
= (maintCommandEnum
) (uintptr_t) inC
;
582 IOEventSource
*inEvent
= (IOEventSource
*) inD
;
583 IOReturn res
= kIOReturnSuccess
;
587 if (!inEvent
->getWorkLoop()) {
588 SETP(&fFlags
, kLoopRestart
);
591 inEvent
->setWorkLoop(this);
594 /* Check if this is a passive or active event source being added */
595 if (eventSourcePerformsWork(inEvent
)) {
597 eventChain
= inEvent
;
599 IOEventSource
*event
, *next
;
601 for (event
= eventChain
; (next
= event
->getNext()); event
= next
) {
604 event
->setNext(inEvent
);
607 if (!passiveEventChain
) {
608 passiveEventChain
= inEvent
;
610 IOEventSource
*event
, *next
;
612 for (event
= passiveEventChain
; (next
= event
->getNext()); event
= next
) {
615 event
->setNext(inEvent
);
618 IOStatisticsAttachEventSource();
623 if (inEvent
->getWorkLoop()) {
624 IOStatisticsDetachEventSource();
626 if (eventSourcePerformsWork(inEvent
)) {
627 if (eventChain
== inEvent
) {
628 eventChain
= inEvent
->getNext();
630 IOEventSource
*event
, *next
= 0;
634 while ((next
= event
->getNext()) && (next
!= inEvent
)) {
640 res
= kIOReturnBadArgument
;
643 event
->setNext(inEvent
->getNext());
646 if (passiveEventChain
== inEvent
) {
647 passiveEventChain
= inEvent
->getNext();
649 IOEventSource
*event
, *next
= 0;
651 event
= passiveEventChain
;
653 while ((next
= event
->getNext()) && (next
!= inEvent
)) {
659 res
= kIOReturnBadArgument
;
662 event
->setNext(inEvent
->getNext());
666 inEvent
->setWorkLoop(0);
669 SETP(&fFlags
, kLoopRestart
);
674 return kIOReturnUnsupported
;
681 IOWorkLoop::eventSourcePerformsWork(IOEventSource
*inEventSource
)
686 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
687 * The assumption is that if you override checkForWork(), you need to be
688 * active and not passive.
690 * We picked a known quantity controlG that does not override
691 * IOEventSource::checkForWork(), namely the IOCommandGate associated with
692 * the workloop to which this event source is getting attached.
694 * We do a pointer comparison on the offset in the vtable for inNewEvent against
695 * the offset in the vtable for inReferenceEvent. This works because
696 * IOCommandGate's slot for checkForWork() has the address of
697 * IOEventSource::checkForWork() in it.
699 * Think of OSMemberFunctionCast yielding the value at the vtable offset for
700 * checkForWork() here. We're just testing to see if it's the same or not.
704 if (IOEventSource::kPassive
& inEventSource
->flags
) {
706 } else if (IOEventSource::kActive
& inEventSource
->flags
) {
708 } else if (controlG
) {
712 ptr1
= OSMemberFunctionCast(void*, inEventSource
, &IOEventSource::checkForWork
);
713 ptr2
= OSMemberFunctionCast(void*, controlG
, &IOEventSource::checkForWork
);
724 IOWorkLoop::lockTime(void)
727 time
= mach_absolute_time() - reserved
->lockTime
;
728 if (time
> reserved
->lockInterval
) {
729 absolutetime_to_nanoseconds(time
, &time
);
730 if (kTimeLockPanics
& reserved
->options
) {
731 panic("IOWorkLoop %p lock time %qd us", this, time
/ 1000ULL);
733 OSReportWithBacktrace("IOWorkLoop %p lock time %qd us", this, time
/ 1000ULL);
739 IOWorkLoop::setMaximumLockTime(uint64_t interval
, uint32_t options
)
741 IORecursiveLockLock(gateLock
);
742 reserved
->lockInterval
= interval
;
743 reserved
->options
= (reserved
->options
& ~kTimeLockPanics
) | (options
& kTimeLockPanics
);
744 IORecursiveLockUnlock(gateLock
);