X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/c910b4d9d2451126ae3917b931cd4390c11e1d52..HEAD:/iokit/Kernel/IOTimerEventSource.cpp?ds=sidebyside diff --git a/iokit/Kernel/IOTimerEventSource.cpp b/iokit/Kernel/IOTimerEventSource.cpp index d32a17887..644d840c3 100644 --- a/iokit/Kernel/IOTimerEventSource.cpp +++ b/iokit/Kernel/IOTimerEventSource.cpp @@ -1,8 +1,8 @@ /* - * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,21 +22,11 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ -/* - * Copyright (c) 1999 Apple Computer, Inc. All rights reserved. - * - * IOTimerEventSource.cpp - * - * HISTORY - * 2-Feb-1999 Joe Liu (jliu) created. - * 1999-10-14 Godfrey van der Linden(gvdl) - * Revamped to use thread_call APIs * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include #include __BEGIN_DECLS @@ -51,272 +41,542 @@ __END_DECLS #include #include +#include +#if CONFIG_DTRACE +#include +#endif + +#include +#include + #define super IOEventSource OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource) -OSMetaClassDefineReservedUnused(IOTimerEventSource, 0); -OSMetaClassDefineReservedUnused(IOTimerEventSource, 1); -OSMetaClassDefineReservedUnused(IOTimerEventSource, 2); +OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0); +OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1); +OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2); OSMetaClassDefineReservedUnused(IOTimerEventSource, 3); OSMetaClassDefineReservedUnused(IOTimerEventSource, 4); OSMetaClassDefineReservedUnused(IOTimerEventSource, 5); OSMetaClassDefineReservedUnused(IOTimerEventSource, 6); OSMetaClassDefineReservedUnused(IOTimerEventSource, 7); -// +#if IOKITSTATS + +#define IOStatisticsInitializeCounter() \ +do { \ + IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \ +} while (0) + +#define IOStatisticsOpenGate() \ +do { \ + IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \ +} while (0) + +#define IOStatisticsCloseGate() \ +do { \ + IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \ +} while (0) + +#define IOStatisticsTimeout() \ +do { \ + IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \ +} while (0) + +#else + +#define IOStatisticsInitializeCounter() +#define IOStatisticsOpenGate() +#define IOStatisticsCloseGate() +#define IOStatisticsTimeout() + +#endif /* IOKITSTATS */ + +// // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, -// not a subclassed implementation. +// not a subclassed implementation. // -bool IOTimerEventSource::checkForWork() { return false; } - // Timeout handler function. This function is called by the kernel when // the timeout interval expires. // -void IOTimerEventSource::timeout(void *self) + +__inline__ void +IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts, + OSObject * _owner, IOWorkLoop * _workLoop) { - IOTimerEventSource *me = (IOTimerEventSource *) self; - - if (me->enabled && me->action) - { - IOWorkLoop * - wl = me->workLoop; - if (wl) - { - Action doit; - wl->closeGate(); - doit = (Action) me->action; - if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) - { - IOTimeStampConstant(IODBG_TIMES(IOTIMES_ACTION), - (unsigned int) doit, (unsigned int) me->owner); - (*doit)(me->owner, me); - } - wl->openGate(); - } - } + bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; + void * address; + + if (kActionBlock & flags) { + address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock)); + } else { + address = ptrauth_nop_cast(void *, _action); + } + + if (trace) { + IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), + VM_KERNEL_ADDRHIDE(address), + VM_KERNEL_ADDRHIDE(_owner)); + } + + if (kActionBlock & flags) { + ((IOTimerEventSource::ActionBlock) actionBlock)(ts); + } else { + ((IOTimerEventSource::Action)_action)(_owner, ts); + } + +#if CONFIG_DTRACE + DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop); +#endif + + if (trace) { + IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), + VM_KERNEL_UNSLIDE(address), + VM_KERNEL_ADDRHIDE(_owner)); + } } -void IOTimerEventSource::timeoutAndRelease(void * self, void * count) +void +IOTimerEventSource::timeout(void *self) { - IOTimerEventSource *me = (IOTimerEventSource *) self; - - if (me->enabled && me->action) - { - IOWorkLoop * - wl = me->reserved->workLoop; - if (wl) - { - Action doit; - wl->closeGate(); - doit = (Action) me->action; - if (doit && (me->reserved->calloutGeneration == (SInt32) count)) - { - IOTimeStampConstant(IODBG_TIMES(IOTIMES_ACTION), - (unsigned int) doit, (unsigned int) me->owner); - (*doit)(me->owner, me); - } - wl->openGate(); - } - } - - me->reserved->workLoop->release(); - me->release(); + IOTimerEventSource *me = (IOTimerEventSource *) self; + + IOStatisticsTimeout(); + + if (me->enabled && me->action) { + IOWorkLoop * + wl = me->workLoop; + if (wl) { + IOEventSource::Action doit; + wl->closeGate(); + IOStatisticsCloseGate(); + doit = me->action; + if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) { + me->invokeAction(doit, me, me->owner, me->workLoop); + } + IOStatisticsOpenGate(); + wl->openGate(); + } + } } -void IOTimerEventSource::setTimeoutFunc() +void +IOTimerEventSource::timeoutAndRelease(void * self, void * c) { - // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, - // not a subclassed implementation - reserved = IONew(ExpansionData, 1); - calloutEntry = (void *) thread_call_allocate((thread_call_func_t) &IOTimerEventSource::timeoutAndRelease, - (thread_call_param_t) this); + IOTimerEventSource *me = (IOTimerEventSource *) self; + /* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count" + * must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */ + SInt32 count = (SInt32) (long) c; + + IOStatisticsTimeout(); + + if (me->enabled && me->action) { + IOWorkLoop * + wl = me->reserved->workLoop; + if (wl) { + IOEventSource::Action doit; + wl->closeGate(); + IOStatisticsCloseGate(); + doit = me->action; + if (doit && (me->reserved->calloutGeneration == count)) { + thread_call_start_iotes_invocation((thread_call_t)me->calloutEntry); + me->invokeAction(doit, me, me->owner, me->workLoop); + } + IOStatisticsOpenGate(); + wl->openGate(); + } + } + + me->reserved->workLoop->release(); + me->release(); } -bool IOTimerEventSource::init(OSObject *inOwner, Action inAction) +// -- work loop delivery + +bool +IOTimerEventSource::checkForWork() { - if (!super::init(inOwner, (IOEventSource::Action) inAction) ) - return false; + IOEventSource::Action doit; - setTimeoutFunc(); - if (!calloutEntry) - return false; + if (reserved + && (reserved->calloutGenerationSignaled == reserved->calloutGeneration) + && enabled && (doit = action)) { + reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; + invokeAction(doit, this, owner, workLoop); + } - return true; + return false; } -IOTimerEventSource * -IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) +void +IOTimerEventSource::timeoutSignaled(void * self, void * c) +{ + IOTimerEventSource *me = (IOTimerEventSource *) self; + + me->reserved->calloutGenerationSignaled = (SInt32)(long) c; + if (me->enabled) { + me->signalWorkAvailable(); + } +} + +// -- + +void +IOTimerEventSource::setTimeoutFunc() { - IOTimerEventSource *me = new IOTimerEventSource; + thread_call_priority_t pri; + uint32_t options; + + if (reserved) { + panic("setTimeoutFunc already %p, %p", this, reserved); + } + + // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, + // not a subclassed implementation + reserved = IONewZero(ExpansionData, 1); + + reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; + // make use of an existing ivar for parameter passing + options = (uint32_t) abstime; + abstime = 0; + + thread_call_options_t tcoptions = 0; + thread_call_func_t func = NULL; + + switch (kIOTimerEventSourceOptionsPriorityMask & options) { + case kIOTimerEventSourceOptionsPriorityHigh: + pri = THREAD_CALL_PRIORITY_HIGH; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityKernel: + pri = THREAD_CALL_PRIORITY_KERNEL; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityKernelHigh: + pri = THREAD_CALL_PRIORITY_KERNEL_HIGH; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityUser: + pri = THREAD_CALL_PRIORITY_USER; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityLow: + pri = THREAD_CALL_PRIORITY_LOW; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityWorkLoop: + pri = THREAD_CALL_PRIORITY_KERNEL; + tcoptions |= THREAD_CALL_OPTIONS_SIGNAL; + if (kIOTimerEventSourceOptionsAllowReenter & options) { + break; + } + func = &IOTimerEventSource::timeoutSignaled; + break; + + default: + break; + } + + assertf(func, "IOTimerEventSource options 0x%x", options); + if (!func) { + return; // init will fail + } + if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) { + flags |= kActive; + } else { + flags |= kPassive; + } + + if (!(kIOTimerEventSourceOptionsAllowReenter & options)) { + tcoptions |= THREAD_CALL_OPTIONS_ONCE; + } + + calloutEntry = (void *) thread_call_allocate_with_options(func, + (thread_call_param_t) this, pri, tcoptions); + assert(calloutEntry); +} + +bool +IOTimerEventSource::init(OSObject *inOwner, Action inAction) +{ + if (!super::init(inOwner, (IOEventSource::Action) inAction)) { + return false; + } + + setTimeoutFunc(); + if (!calloutEntry) { + return false; + } - if (me && !me->init(inOwner, inAction)) { - me->release(); - return 0; - } + IOStatisticsInitializeCounter(); - return me; + return true; } -void IOTimerEventSource::free() +bool +IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction) +{ + // make use of an existing ivar for parameter passing + abstime = options; + return init(inOwner, inAction); +} + +IOTimerEventSource * +IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction) { - if (calloutEntry) { - cancelTimeout(); - thread_call_free((thread_call_t) calloutEntry); - } + IOTimerEventSource *me = new IOTimerEventSource; - if (reserved) - IODelete(reserved, ExpansionData, 1); + if (me && !me->init(inOptions, inOwner, inAction)) { + me->release(); + return NULL; + } - super::free(); + return me; } -void IOTimerEventSource::cancelTimeout() +IOTimerEventSource * +IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action) { - if (reserved) - reserved->calloutGeneration++; - bool active = thread_call_cancel((thread_call_t) calloutEntry); - AbsoluteTime_to_scalar(&abstime) = 0; - if (active && reserved) - { - release(); - workLoop->release(); - } + IOTimerEventSource * tes; + tes = IOTimerEventSource::timerEventSource(options, inOwner, (Action) NULL); + if (tes) { + tes->setActionBlock((IOEventSource::ActionBlock) _action); + } + + return tes; } -void IOTimerEventSource::enable() +#define _thread_call_cancel(tc) ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc))) + +IOTimerEventSource * +IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) { - super::enable(); - if (kIOReturnSuccess != wakeAtTime(abstime)) - super::disable(); // Problem re-scheduling timeout ignore enable + return IOTimerEventSource::timerEventSource( + kIOTimerEventSourceOptionsPriorityKernelHigh, + inOwner, inAction); } -void IOTimerEventSource::disable() +void +IOTimerEventSource::free() { - if (reserved) - reserved->calloutGeneration++; - bool active = thread_call_cancel((thread_call_t) calloutEntry); - super::disable(); - if (active && reserved) - { - release(); - workLoop->release(); - } + if (calloutEntry) { + __assert_only bool freed; + + cancelTimeout(); + + freed = thread_call_free((thread_call_t) calloutEntry); + assert(freed); + } + + if (reserved) { + IODelete(reserved, ExpansionData, 1); + } + + super::free(); } -IOReturn IOTimerEventSource::setTimeoutTicks(UInt32 ticks) +void +IOTimerEventSource::cancelTimeout() { - return setTimeout(ticks, kTickScale); + if (reserved) { + reserved->calloutGeneration++; + } + bool active = _thread_call_cancel((thread_call_t) calloutEntry); + AbsoluteTime_to_scalar(&abstime) = 0; + if (active && reserved && (kPassive & flags)) { + release(); + workLoop->release(); + } } -IOReturn IOTimerEventSource::setTimeoutMS(UInt32 ms) +void +IOTimerEventSource::enable() { - return setTimeout(ms, kMillisecondScale); + super::enable(); + if (kIOReturnSuccess != wakeAtTime(abstime)) { + super::disable(); // Problem re-scheduling timeout ignore enable + } } -IOReturn IOTimerEventSource::setTimeoutUS(UInt32 us) +void +IOTimerEventSource::disable() { - return setTimeout(us, kMicrosecondScale); + if (reserved) { + reserved->calloutGeneration++; + } + bool active = _thread_call_cancel((thread_call_t) calloutEntry); + super::disable(); + if (active && reserved && (kPassive & flags)) { + release(); + workLoop->release(); + } } -IOReturn IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor) +IOReturn +IOTimerEventSource::setTimeoutTicks(UInt32 ticks) { - AbsoluteTime end; + return setTimeout(ticks, kTickScale); +} - clock_interval_to_deadline(interval, scale_factor, &end); - return wakeAtTime(end); +IOReturn +IOTimerEventSource::setTimeoutMS(UInt32 ms) +{ + return setTimeout(ms, kMillisecondScale); } -IOReturn IOTimerEventSource::setTimeout(mach_timespec_t interval) +IOReturn +IOTimerEventSource::setTimeoutUS(UInt32 us) { - AbsoluteTime end, nsecs; + return setTimeout(us, kMicrosecondScale); +} - clock_interval_to_absolutetime_interval - (interval.tv_nsec, kNanosecondScale, &nsecs); - clock_interval_to_deadline - (interval.tv_sec, NSEC_PER_SEC, &end); - ADD_ABSOLUTETIME(&end, &nsecs); +IOReturn +IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor) +{ + AbsoluteTime end; - return wakeAtTime(end); + clock_interval_to_deadline(interval, scale_factor, &end); + return wakeAtTime(end); } -IOReturn IOTimerEventSource::setTimeout(AbsoluteTime interval) +#if !defined(__LP64__) +IOReturn +IOTimerEventSource::setTimeout(mach_timespec_t interval) { - AbsoluteTime end; + AbsoluteTime end, nsecs; - clock_get_uptime(&end); - ADD_ABSOLUTETIME(&end, &interval); + clock_interval_to_absolutetime_interval + (interval.tv_nsec, kNanosecondScale, &nsecs); + clock_interval_to_deadline + (interval.tv_sec, NSEC_PER_SEC, &end); + ADD_ABSOLUTETIME(&end, &nsecs); - return wakeAtTime(end); + return wakeAtTime(end); } +#endif -IOReturn IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks) +IOReturn +IOTimerEventSource::setTimeout(AbsoluteTime interval) { - return wakeAtTime(ticks, kTickScale); + AbsoluteTime end; + clock_absolutetime_interval_to_deadline(interval, &end); + return wakeAtTime(end); } -IOReturn IOTimerEventSource::wakeAtTimeMS(UInt32 ms) +IOReturn +IOTimerEventSource::setTimeout(uint32_t options, + AbsoluteTime abstime, AbsoluteTime leeway) { - return wakeAtTime(ms, kMillisecondScale); + AbsoluteTime end; + if (options & kIOTimeOptionsContinuous) { + clock_continuoustime_interval_to_deadline(abstime, &end); + } else { + clock_absolutetime_interval_to_deadline(abstime, &end); + } + + return wakeAtTime(options, end, leeway); } -IOReturn IOTimerEventSource::wakeAtTimeUS(UInt32 us) +IOReturn +IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks) { - return wakeAtTime(us, kMicrosecondScale); + return wakeAtTime(ticks, kTickScale); } -IOReturn IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor) +IOReturn +IOTimerEventSource::wakeAtTimeMS(UInt32 ms) { - AbsoluteTime end; - clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end); + return wakeAtTime(ms, kMillisecondScale); +} - return wakeAtTime(end); +IOReturn +IOTimerEventSource::wakeAtTimeUS(UInt32 us) +{ + return wakeAtTime(us, kMicrosecondScale); } -IOReturn IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime) +IOReturn +IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor) { - AbsoluteTime end, nsecs; + AbsoluteTime end; + clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end); - clock_interval_to_absolutetime_interval - (inAbstime.tv_nsec, kNanosecondScale, &nsecs); - clock_interval_to_absolutetime_interval - (inAbstime.tv_sec, kSecondScale, &end); - ADD_ABSOLUTETIME(&end, &nsecs); + return wakeAtTime(end); +} + +#if !defined(__LP64__) +IOReturn +IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime) +{ + AbsoluteTime end, nsecs; + + clock_interval_to_absolutetime_interval + (inAbstime.tv_nsec, kNanosecondScale, &nsecs); + clock_interval_to_absolutetime_interval + (inAbstime.tv_sec, kSecondScale, &end); + ADD_ABSOLUTETIME(&end, &nsecs); - return wakeAtTime(end); + return wakeAtTime(end); +} +#endif + +void +IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) +{ + super::setWorkLoop(inWorkLoop); + if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) { + wakeAtTime(abstime); + } } -void IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) +IOReturn +IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) { - super::setWorkLoop(inWorkLoop); - if ( enabled && AbsoluteTime_to_scalar(&abstime) && workLoop ) - wakeAtTime(abstime); + return wakeAtTime(0, inAbstime, 0); } -IOReturn IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) +IOReturn +IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway) { - if (!action) - return kIOReturnNoResources; - - abstime = inAbstime; - if ( enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop ) - { - if (reserved) - { - retain(); - workLoop->retain(); - reserved->workLoop = workLoop; - reserved->calloutGeneration++; - if (thread_call_enter1_delayed((thread_call_t) calloutEntry, - (void *) reserved->calloutGeneration, inAbstime)) - { - release(); - workLoop->release(); - } - } - else - thread_call_enter_delayed((thread_call_t) calloutEntry, inAbstime); - } - - return kIOReturnSuccess; + if (!action) { + return kIOReturnNoResources; + } + + abstime = inAbstime; + if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) { + uint32_t tcoptions = 0; + + if (kIOTimeOptionsWithLeeway & options) { + tcoptions |= THREAD_CALL_DELAY_LEEWAY; + } + if (kIOTimeOptionsContinuous & options) { + tcoptions |= THREAD_CALL_CONTINUOUS; + } + + if (reserved) { + if (kPassive & flags) { + retain(); + workLoop->retain(); + } + reserved->workLoop = workLoop; + reserved->calloutGeneration++; + if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry, + (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions) + && (kPassive & flags)) { + release(); + workLoop->release(); + } + } else { + thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry, + NULL, inAbstime, leeway, tcoptions); + } + } + + return kIOReturnSuccess; }