]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOWorkLoop.cpp
a9aff9f93e54355cac32fcb39cc318cad5af7d12
[apple/xnu.git] / iokit / Kernel / IOWorkLoop.cpp
1 /*
2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <pexpert/pexpert.h>
30 #include <IOKit/IOWorkLoop.h>
31 #include <IOKit/IOEventSource.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOCommandGate.h>
34 #include <IOKit/IOCommandPool.h>
35 #include <IOKit/IOTimeStamp.h>
36 #include <IOKit/IOKitDebug.h>
37 #include <libkern/OSDebug.h>
38 #include <kern/thread.h>
39
40 #define super OSObject
41
42 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
43
44 // Block of unused functions intended for future use
45 #if __LP64__
46 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
47 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
48 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
49 #else
50 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
51 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
52 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
53 #endif
54 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
55 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
56 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
57 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
58 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
59
60 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
61 static inline void
62 SETP(void *addr, unsigned int flag)
63 {
64 unsigned char *num = (unsigned char *) addr; *num |= flag;
65 }
66 static inline void
67 CLRP(void *addr, unsigned int flag)
68 {
69 unsigned char *num = (unsigned char *) addr; *num &= ~flag;
70 }
71 static inline bool
72 ISSETP(void *addr, unsigned int flag)
73 {
74 unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0;
75 }
76
77 #define fFlags loopRestart
78
79 #define passiveEventChain reserved->passiveEventChain
80
81 #if IOKITSTATS
82
83 #define IOStatisticsRegisterCounter() \
84 do { \
85 reserved->counter = IOStatistics::registerWorkLoop(this); \
86 } while(0)
87
88 #define IOStatisticsUnregisterCounter() \
89 do { \
90 if (reserved) \
91 IOStatistics::unregisterWorkLoop(reserved->counter); \
92 } while(0)
93
94 #define IOStatisticsOpenGate() \
95 do { \
96 IOStatistics::countWorkLoopOpenGate(reserved->counter); \
97 if (reserved->lockInterval) lockTime(); \
98 } while(0)
99 #define IOStatisticsCloseGate() \
100 do { \
101 IOStatistics::countWorkLoopCloseGate(reserved->counter); \
102 if (reserved->lockInterval) reserved->lockTime = mach_absolute_time(); \
103 } while(0)
104
105 #define IOStatisticsAttachEventSource() \
106 do { \
107 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
108 } while(0)
109
110 #define IOStatisticsDetachEventSource() \
111 do { \
112 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
113 } while(0)
114
115 #else
116
117 #define IOStatisticsRegisterCounter()
118 #define IOStatisticsUnregisterCounter()
119 #define IOStatisticsOpenGate()
120 #define IOStatisticsCloseGate()
121 #define IOStatisticsAttachEventSource()
122 #define IOStatisticsDetachEventSource()
123
124 #endif /* IOKITSTATS */
125
126 bool
127 IOWorkLoop::init()
128 {
129 // The super init and gateLock allocation MUST be done first.
130 if (!super::init()) {
131 return false;
132 }
133
134 // Allocate our ExpansionData if it hasn't been allocated already.
135 if (!reserved) {
136 reserved = IONew(ExpansionData, 1);
137 if (!reserved) {
138 return false;
139 }
140
141 bzero(reserved, sizeof(ExpansionData));
142 }
143
144 if (gateLock == NULL) {
145 if (!(gateLock = IORecursiveLockAlloc())) {
146 return false;
147 }
148 }
149
150 if (workToDoLock == NULL) {
151 if (!(workToDoLock = IOSimpleLockAlloc())) {
152 return false;
153 }
154 IOSimpleLockInit(workToDoLock);
155 workToDo = false;
156 }
157
158 IOStatisticsRegisterCounter();
159
160 if (controlG == NULL) {
161 controlG = IOCommandGate::commandGate(
162 this,
163 OSMemberFunctionCast(
164 IOCommandGate::Action,
165 this,
166 &IOWorkLoop::_maintRequest));
167
168 if (!controlG) {
169 return false;
170 }
171 // Point the controlGate at the workLoop. Usually addEventSource
172 // does this automatically. The problem is in this case addEventSource
173 // uses the control gate and it has to be bootstrapped.
174 controlG->setWorkLoop(this);
175 if (addEventSource(controlG) != kIOReturnSuccess) {
176 return false;
177 }
178 }
179
180 if (workThread == NULL) {
181 thread_continue_t cptr = OSMemberFunctionCast(
182 thread_continue_t,
183 this,
184 &IOWorkLoop::threadMain);
185 if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) {
186 return false;
187 }
188 }
189
190 (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
191 return true;
192 }
193
194 IOWorkLoop *
195 IOWorkLoop::workLoop()
196 {
197 return IOWorkLoop::workLoopWithOptions(0);
198 }
199
200 IOWorkLoop *
201 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
202 {
203 IOWorkLoop *me = new IOWorkLoop;
204
205 if (me && options) {
206 me->reserved = IONew(ExpansionData, 1);
207 if (!me->reserved) {
208 me->release();
209 return 0;
210 }
211 bzero(me->reserved, sizeof(ExpansionData));
212 me->reserved->options = options;
213 }
214
215 if (me && !me->init()) {
216 me->release();
217 return 0;
218 }
219
220 return me;
221 }
222
223 // Free is called twice:
224 // First when the atomic retainCount transitions from 1 -> 0
225 // Secondly when the work loop itself is commiting hari kari
226 // Hence the each leg of the free must be single threaded.
227 void
228 IOWorkLoop::free()
229 {
230 if (workThread) {
231 IOInterruptState is;
232
233 // If we are here then we must be trying to shut down this work loop
234 // in this case disable all of the event source, mark the loop
235 // as terminating and wakeup the work thread itself and return
236 // Note: we hold the gate across the entire operation mainly for the
237 // benefit of our event sources so we can disable them cleanly.
238 closeGate();
239
240 disableAllEventSources();
241
242 is = IOSimpleLockLockDisableInterrupt(workToDoLock);
243 SETP(&fFlags, kLoopTerminate);
244 thread_wakeup_thread((void *) &workToDo, workThread);
245 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
246
247 openGate();
248 } else { /* !workThread */
249 IOEventSource *event, *next;
250
251 for (event = eventChain; event; event = next) {
252 next = event->getNext();
253 event->setWorkLoop(0);
254 event->setNext(0);
255 event->release();
256 }
257 eventChain = 0;
258
259 for (event = passiveEventChain; event; event = next) {
260 next = event->getNext();
261 event->setWorkLoop(0);
262 event->setNext(0);
263 event->release();
264 }
265 passiveEventChain = 0;
266
267 // Either we have a partial initialization to clean up
268 // or the workThread itself is performing hari-kari.
269 // Either way clean up all of our resources and return.
270
271 if (controlG) {
272 controlG->workLoop = 0;
273 controlG->release();
274 controlG = 0;
275 }
276
277 if (workToDoLock) {
278 IOSimpleLockFree(workToDoLock);
279 workToDoLock = 0;
280 }
281
282 if (gateLock) {
283 IORecursiveLockFree(gateLock);
284 gateLock = 0;
285 }
286
287 IOStatisticsUnregisterCounter();
288
289 if (reserved) {
290 IODelete(reserved, ExpansionData, 1);
291 reserved = 0;
292 }
293
294 super::free();
295 }
296 }
297
298 IOReturn
299 IOWorkLoop::addEventSource(IOEventSource *newEvent)
300 {
301 if ((workThread)
302 && !thread_has_thread_name(workThread)
303 && (newEvent->owner)
304 && !OSDynamicCast(IOCommandPool, newEvent->owner)) {
305 thread_set_thread_name(workThread, newEvent->owner->getMetaClass()->getClassName());
306 }
307
308 return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
309 }
310
311 IOReturn
312 IOWorkLoop::removeEventSource(IOEventSource *toRemove)
313 {
314 return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
315 }
316
317 void
318 IOWorkLoop::enableAllEventSources() const
319 {
320 IOEventSource *event;
321
322 for (event = eventChain; event; event = event->getNext()) {
323 event->enable();
324 }
325
326 for (event = passiveEventChain; event; event = event->getNext()) {
327 event->enable();
328 }
329 }
330
331 void
332 IOWorkLoop::disableAllEventSources() const
333 {
334 IOEventSource *event;
335
336 for (event = eventChain; event; event = event->getNext()) {
337 event->disable();
338 }
339
340 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
341 for (event = passiveEventChain; event; event = event->getNext()) {
342 if (event != controlG) { // Don't disable the control gate
343 event->disable();
344 }
345 }
346 }
347
348 void
349 IOWorkLoop::enableAllInterrupts() const
350 {
351 IOEventSource *event;
352
353 for (event = eventChain; event; event = event->getNext()) {
354 if (OSDynamicCast(IOInterruptEventSource, event)) {
355 event->enable();
356 }
357 }
358 }
359
360 void
361 IOWorkLoop::disableAllInterrupts() const
362 {
363 IOEventSource *event;
364
365 for (event = eventChain; event; event = event->getNext()) {
366 if (OSDynamicCast(IOInterruptEventSource, event)) {
367 event->disable();
368 }
369 }
370 }
371
372
373 /* virtual */ bool
374 IOWorkLoop::runEventSources()
375 {
376 bool res = false;
377 bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
378 bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
379
380 closeGate();
381 if (ISSETP(&fFlags, kLoopTerminate)) {
382 goto abort;
383 }
384
385 if (traceWL) {
386 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this));
387 }
388
389 bool more;
390 do {
391 CLRP(&fFlags, kLoopRestart);
392 more = false;
393 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
394 workToDo = false;
395 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
396 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
397 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
398 if (traceES) {
399 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt));
400 }
401
402 more |= evnt->checkForWork();
403
404 if (traceES) {
405 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt));
406 }
407
408 if (ISSETP(&fFlags, kLoopTerminate)) {
409 goto abort;
410 } else if (fFlags & kLoopRestart) {
411 more = true;
412 break;
413 }
414 }
415 } while (more);
416
417 res = true;
418
419 if (traceWL) {
420 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this));
421 }
422
423 abort:
424 openGate();
425 return res;
426 }
427
428 /* virtual */ void
429 IOWorkLoop::threadMain()
430 {
431 restartThread:
432 do {
433 if (!runEventSources()) {
434 goto exitThread;
435 }
436
437 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
438 if (!ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
439 assert_wait((void *) &workToDo, false);
440 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
441 thread_continue_t cptr = NULL;
442 if (!reserved || !(kPreciousStack & reserved->options)) {
443 cptr = OSMemberFunctionCast(
444 thread_continue_t, this, &IOWorkLoop::threadMain);
445 }
446 thread_block_parameter(cptr, this);
447 goto restartThread;
448 /* NOTREACHED */
449 }
450
451 // At this point we either have work to do or we need
452 // to commit suicide. But no matter
453 // Clear the simple lock and retore the interrupt state
454 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
455 } while (workToDo);
456
457 exitThread:
458 closeGate();
459 thread_t thread = workThread;
460 workThread = 0; // Say we don't have a loop and free ourselves
461 openGate();
462
463 free();
464
465 thread_deallocate(thread);
466 (void) thread_terminate(thread);
467 }
468
469 IOThread
470 IOWorkLoop::getThread() const
471 {
472 return workThread;
473 }
474
475 bool
476 IOWorkLoop::onThread() const
477 {
478 return IOThreadSelf() == workThread;
479 }
480
481 bool
482 IOWorkLoop::inGate() const
483 {
484 return IORecursiveLockHaveLock(gateLock);
485 }
486
487 // Internal APIs used by event sources to control the thread
488 void
489 IOWorkLoop::signalWorkAvailable()
490 {
491 if (workToDoLock) {
492 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
493 workToDo = true;
494 thread_wakeup_thread((void *) &workToDo, workThread);
495 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
496 }
497 }
498
499 void
500 IOWorkLoop::openGate()
501 {
502 IOStatisticsOpenGate();
503 IORecursiveLockUnlock(gateLock);
504 }
505
506 void
507 IOWorkLoop::closeGate()
508 {
509 IORecursiveLockLock(gateLock);
510 IOStatisticsCloseGate();
511 }
512
513 bool
514 IOWorkLoop::tryCloseGate()
515 {
516 bool res = (IORecursiveLockTryLock(gateLock) != 0);
517 if (res) {
518 IOStatisticsCloseGate();
519 }
520 return res;
521 }
522
523 int
524 IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
525 {
526 int res;
527 IOStatisticsOpenGate();
528 res = IORecursiveLockSleep(gateLock, event, interuptibleType);
529 IOStatisticsCloseGate();
530 return res;
531 }
532
533 int
534 IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
535 {
536 int res;
537 IOStatisticsOpenGate();
538 res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
539 IOStatisticsCloseGate();
540 return res;
541 }
542
543 void
544 IOWorkLoop::wakeupGate(void *event, bool oneThread)
545 {
546 IORecursiveLockWakeup(gateLock, event, oneThread);
547 }
548
549 static IOReturn
550 IOWorkLoopActionToBlock(OSObject *owner,
551 void *arg0, void *arg1,
552 void *arg2, void *arg3)
553 {
554 return ((IOWorkLoop::ActionBlock) arg0)();
555 }
556
557 IOReturn
558 IOWorkLoop::runActionBlock(ActionBlock action)
559 {
560 return runAction(&IOWorkLoopActionToBlock, this, action);
561 }
562
563 IOReturn
564 IOWorkLoop::runAction(Action inAction, OSObject *target,
565 void *arg0, void *arg1,
566 void *arg2, void *arg3)
567 {
568 IOReturn res;
569
570 // closeGate is recursive so don't worry if we already hold the lock.
571 closeGate();
572 res = (*inAction)(target, arg0, arg1, arg2, arg3);
573 openGate();
574
575 return res;
576 }
577
578 IOReturn
579 IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
580 {
581 maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
582 IOEventSource *inEvent = (IOEventSource *) inD;
583 IOReturn res = kIOReturnSuccess;
584
585 switch (command) {
586 case mAddEvent:
587 if (!inEvent->getWorkLoop()) {
588 SETP(&fFlags, kLoopRestart);
589
590 inEvent->retain();
591 inEvent->setWorkLoop(this);
592 inEvent->setNext(0);
593
594 /* Check if this is a passive or active event source being added */
595 if (eventSourcePerformsWork(inEvent)) {
596 if (!eventChain) {
597 eventChain = inEvent;
598 } else {
599 IOEventSource *event, *next;
600
601 for (event = eventChain; (next = event->getNext()); event = next) {
602 ;
603 }
604 event->setNext(inEvent);
605 }
606 } else {
607 if (!passiveEventChain) {
608 passiveEventChain = inEvent;
609 } else {
610 IOEventSource *event, *next;
611
612 for (event = passiveEventChain; (next = event->getNext()); event = next) {
613 ;
614 }
615 event->setNext(inEvent);
616 }
617 }
618 IOStatisticsAttachEventSource();
619 }
620 break;
621
622 case mRemoveEvent:
623 if (inEvent->getWorkLoop()) {
624 IOStatisticsDetachEventSource();
625
626 if (eventSourcePerformsWork(inEvent)) {
627 if (eventChain == inEvent) {
628 eventChain = inEvent->getNext();
629 } else {
630 IOEventSource *event, *next = 0;
631
632 event = eventChain;
633 if (event) {
634 while ((next = event->getNext()) && (next != inEvent)) {
635 event = next;
636 }
637 }
638
639 if (!next) {
640 res = kIOReturnBadArgument;
641 break;
642 }
643 event->setNext(inEvent->getNext());
644 }
645 } else {
646 if (passiveEventChain == inEvent) {
647 passiveEventChain = inEvent->getNext();
648 } else {
649 IOEventSource *event, *next = 0;
650
651 event = passiveEventChain;
652 if (event) {
653 while ((next = event->getNext()) && (next != inEvent)) {
654 event = next;
655 }
656 }
657
658 if (!next) {
659 res = kIOReturnBadArgument;
660 break;
661 }
662 event->setNext(inEvent->getNext());
663 }
664 }
665
666 inEvent->setWorkLoop(0);
667 inEvent->setNext(0);
668 inEvent->release();
669 SETP(&fFlags, kLoopRestart);
670 }
671 break;
672
673 default:
674 return kIOReturnUnsupported;
675 }
676
677 return res;
678 }
679
680 bool
681 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
682 {
683 bool result = true;
684
685 /*
686 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
687 * The assumption is that if you override checkForWork(), you need to be
688 * active and not passive.
689 *
690 * We picked a known quantity controlG that does not override
691 * IOEventSource::checkForWork(), namely the IOCommandGate associated with
692 * the workloop to which this event source is getting attached.
693 *
694 * We do a pointer comparison on the offset in the vtable for inNewEvent against
695 * the offset in the vtable for inReferenceEvent. This works because
696 * IOCommandGate's slot for checkForWork() has the address of
697 * IOEventSource::checkForWork() in it.
698 *
699 * Think of OSMemberFunctionCast yielding the value at the vtable offset for
700 * checkForWork() here. We're just testing to see if it's the same or not.
701 *
702 */
703
704 if (IOEventSource::kPassive & inEventSource->flags) {
705 result = false;
706 } else if (IOEventSource::kActive & inEventSource->flags) {
707 result = true;
708 } else if (controlG) {
709 void * ptr1;
710 void * ptr2;
711
712 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
713 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
714
715 if (ptr1 == ptr2) {
716 result = false;
717 }
718 }
719
720 return result;
721 }
722
723 void
724 IOWorkLoop::lockTime(void)
725 {
726 uint64_t time;
727 time = mach_absolute_time() - reserved->lockTime;
728 if (time > reserved->lockInterval) {
729 absolutetime_to_nanoseconds(time, &time);
730 if (kTimeLockPanics & reserved->options) {
731 panic("IOWorkLoop %p lock time %qd us", this, time / 1000ULL);
732 } else {
733 OSReportWithBacktrace("IOWorkLoop %p lock time %qd us", this, time / 1000ULL);
734 }
735 }
736 }
737
738 void
739 IOWorkLoop::setMaximumLockTime(uint64_t interval, uint32_t options)
740 {
741 IORecursiveLockLock(gateLock);
742 reserved->lockInterval = interval;
743 reserved->options = (reserved->options & ~kTimeLockPanics) | (options & kTimeLockPanics);
744 IORecursiveLockUnlock(gateLock);
745 }