iokit/Kernel/IOTimerEventSource.cpp

   1 /*
   2  * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30
  31 __BEGIN_DECLS
  32 #include <kern/thread_call.h>
  33 __END_DECLS
  34
  35 #include <IOKit/assert.h>
  36 #include <IOKit/system.h>
  37
  38 #include <IOKit/IOLib.h>
  39 #include <IOKit/IOTimerEventSource.h>
  40 #include <IOKit/IOWorkLoop.h>
  41
  42 #include <IOKit/IOTimeStamp.h>
  43 #include <IOKit/IOKitDebug.h>
  44 #if CONFIG_DTRACE
  45 #include <mach/sdt.h>
  46 #endif
  47
  48 #include <libkern/Block.h>
  49
  50
  51 #define super IOEventSource
  52 OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource)
  53 OSMetaClassDefineReservedUsed(IOTimerEventSource, 0);
  54 OSMetaClassDefineReservedUsed(IOTimerEventSource, 1);
  55 OSMetaClassDefineReservedUsed(IOTimerEventSource, 2);
  56 OSMetaClassDefineReservedUnused(IOTimerEventSource, 3);
  57 OSMetaClassDefineReservedUnused(IOTimerEventSource, 4);
  58 OSMetaClassDefineReservedUnused(IOTimerEventSource, 5);
  59 OSMetaClassDefineReservedUnused(IOTimerEventSource, 6);
  60 OSMetaClassDefineReservedUnused(IOTimerEventSource, 7);
  61
  62 #if IOKITSTATS
  63
  64 #define IOStatisticsInitializeCounter() \
  65 do { \
  66         IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \
  67 } while (0)
  68
  69 #define IOStatisticsOpenGate() \
  70 do { \
  71         IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \
  72 } while (0)
  73
  74 #define IOStatisticsCloseGate() \
  75 do { \
  76         IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \
  77 } while (0)
  78
  79 #define IOStatisticsTimeout() \
  80 do { \
  81         IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \
  82 } while (0)
  83
  84 #else
  85
  86 #define IOStatisticsInitializeCounter()
  87 #define IOStatisticsOpenGate()
  88 #define IOStatisticsCloseGate()
  89 #define IOStatisticsTimeout()
  90
  91 #endif /* IOKITSTATS */
  92
  93 //
  94 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
  95 // not a subclassed implementation.
  96 //
  97
  98 // Timeout handler function. This function is called by the kernel when
  99 // the timeout interval expires.
 100 //
 101
 102 __inline__ void
 103 IOTimerEventSource::invokeAction(IOTimerEventSource::Action _action, IOTimerEventSource * ts,
 104     OSObject * _owner, IOWorkLoop * _workLoop)
 105 {
 106         bool    trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
 107
 108         if (trace) {
 109                 IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
 110                     VM_KERNEL_ADDRHIDE(_action), VM_KERNEL_ADDRHIDE(_owner));
 111         }
 112
 113         if (kActionBlock & flags) {
 114                 ((IOTimerEventSource::ActionBlock) actionBlock)(ts);
 115         } else {
 116                 (*_action)(_owner, ts);
 117         }
 118
 119 #if CONFIG_DTRACE
 120         DTRACE_TMR3(iotescallout__expire, Action, _action, OSObject, _owner, void, _workLoop);
 121 #endif
 122
 123         if (trace) {
 124                 IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
 125                     VM_KERNEL_UNSLIDE(_action), VM_KERNEL_ADDRHIDE(_owner));
 126         }
 127 }
 128
 129 void
 130 IOTimerEventSource::timeout(void *self)
 131 {
 132         IOTimerEventSource *me = (IOTimerEventSource *) self;
 133
 134         IOStatisticsTimeout();
 135
 136         if (me->enabled && me->action) {
 137                 IOWorkLoop *
 138                     wl = me->workLoop;
 139                 if (wl) {
 140                         Action doit;
 141                         wl->closeGate();
 142                         IOStatisticsCloseGate();
 143                         doit = (Action) me->action;
 144                         if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) {
 145                                 me->invokeAction(doit, me, me->owner, me->workLoop);
 146                         }
 147                         IOStatisticsOpenGate();
 148                         wl->openGate();
 149                 }
 150         }
 151 }
 152
 153 void
 154 IOTimerEventSource::timeoutAndRelease(void * self, void * c)
 155 {
 156         IOTimerEventSource *me = (IOTimerEventSource *) self;
 157         /* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count"
 158          *  must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */
 159         SInt32 count = (SInt32) (long) c;
 160
 161         IOStatisticsTimeout();
 162
 163         if (me->enabled && me->action) {
 164                 IOWorkLoop *
 165                     wl = me->reserved->workLoop;
 166                 if (wl) {
 167                         Action doit;
 168                         wl->closeGate();
 169                         IOStatisticsCloseGate();
 170                         doit = (Action) me->action;
 171                         if (doit && (me->reserved->calloutGeneration == count)) {
 172                                 me->invokeAction(doit, me, me->owner, me->workLoop);
 173                         }
 174                         IOStatisticsOpenGate();
 175                         wl->openGate();
 176                 }
 177         }
 178
 179         me->reserved->workLoop->release();
 180         me->release();
 181 }
 182
 183 // -- work loop delivery
 184
 185 bool
 186 IOTimerEventSource::checkForWork()
 187 {
 188         Action doit;
 189
 190         if (reserved
 191             && (reserved->calloutGenerationSignaled == reserved->calloutGeneration)
 192             && enabled && (doit = (Action) action)) {
 193                 reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
 194                 invokeAction(doit, this, owner, workLoop);
 195         }
 196
 197         return false;
 198 }
 199
 200 void
 201 IOTimerEventSource::timeoutSignaled(void * self, void * c)
 202 {
 203         IOTimerEventSource *me = (IOTimerEventSource *) self;
 204
 205         me->reserved->calloutGenerationSignaled = (SInt32)(long) c;
 206         if (me->enabled) {
 207                 me->signalWorkAvailable();
 208         }
 209 }
 210
 211 // --
 212
 213 void
 214 IOTimerEventSource::setTimeoutFunc()
 215 {
 216         thread_call_priority_t pri;
 217         uint32_t options;
 218
 219         if (reserved) {
 220                 panic("setTimeoutFunc already %p, %p", this, reserved);
 221         }
 222
 223         // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
 224         // not a subclassed implementation
 225         reserved = IONew(ExpansionData, 1);
 226         reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
 227         options = abstime;
 228         abstime = 0;
 229
 230         thread_call_options_t tcoptions = 0;
 231         thread_call_func_t    func      = NULL;
 232
 233         switch (kIOTimerEventSourceOptionsPriorityMask & options) {
 234         case kIOTimerEventSourceOptionsPriorityHigh:
 235                 pri = THREAD_CALL_PRIORITY_HIGH;
 236                 func = &IOTimerEventSource::timeoutAndRelease;
 237                 break;
 238
 239         case kIOTimerEventSourceOptionsPriorityKernel:
 240                 pri = THREAD_CALL_PRIORITY_KERNEL;
 241                 func = &IOTimerEventSource::timeoutAndRelease;
 242                 break;
 243
 244         case kIOTimerEventSourceOptionsPriorityKernelHigh:
 245                 pri = THREAD_CALL_PRIORITY_KERNEL_HIGH;
 246                 func = &IOTimerEventSource::timeoutAndRelease;
 247                 break;
 248
 249         case kIOTimerEventSourceOptionsPriorityUser:
 250                 pri = THREAD_CALL_PRIORITY_USER;
 251                 func = &IOTimerEventSource::timeoutAndRelease;
 252                 break;
 253
 254         case kIOTimerEventSourceOptionsPriorityLow:
 255                 pri = THREAD_CALL_PRIORITY_LOW;
 256                 func = &IOTimerEventSource::timeoutAndRelease;
 257                 break;
 258
 259         case kIOTimerEventSourceOptionsPriorityWorkLoop:
 260                 pri = THREAD_CALL_PRIORITY_KERNEL;
 261                 tcoptions |= THREAD_CALL_OPTIONS_SIGNAL;
 262                 if (kIOTimerEventSourceOptionsAllowReenter & options) {
 263                         break;
 264                 }
 265                 func = &IOTimerEventSource::timeoutSignaled;
 266                 break;
 267
 268         default:
 269                 break;
 270         }
 271
 272         assertf(func, "IOTimerEventSource options 0x%x", options);
 273         if (!func) {
 274                 return;                                              // init will fail
 275         }
 276         if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) {
 277                 flags |= kActive;
 278         } else {
 279                 flags |= kPassive;
 280         }
 281
 282         if (!(kIOTimerEventSourceOptionsAllowReenter & options)) {
 283                 tcoptions |= THREAD_CALL_OPTIONS_ONCE;
 284         }
 285
 286         calloutEntry = (void *) thread_call_allocate_with_options(func,
 287             (thread_call_param_t) this, pri, tcoptions);
 288         assert(calloutEntry);
 289 }
 290
 291 bool
 292 IOTimerEventSource::init(OSObject *inOwner, Action inAction)
 293 {
 294         if (!super::init(inOwner, (IOEventSource::Action) inAction)) {
 295                 return false;
 296         }
 297
 298         setTimeoutFunc();
 299         if (!calloutEntry) {
 300                 return false;
 301         }
 302
 303         IOStatisticsInitializeCounter();
 304
 305         return true;
 306 }
 307
 308 bool
 309 IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction)
 310 {
 311         abstime = options;
 312         return init(inOwner, inAction);
 313 }
 314
 315 IOTimerEventSource *
 316 IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction)
 317 {
 318         IOTimerEventSource *me = new IOTimerEventSource;
 319
 320         if (me && !me->init(inOptions, inOwner, inAction)) {
 321                 me->release();
 322                 return NULL;
 323         }
 324
 325         return me;
 326 }
 327
 328 IOTimerEventSource *
 329 IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action)
 330 {
 331         IOTimerEventSource * tes;
 332         tes = IOTimerEventSource::timerEventSource(options, inOwner, (Action) NULL);
 333         if (tes) {
 334                 tes->setActionBlock((IOEventSource::ActionBlock) _action);
 335         }
 336
 337         return tes;
 338 }
 339
 340 #define _thread_call_cancel(tc)   ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc)))
 341
 342 IOTimerEventSource *
 343 IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction)
 344 {
 345         return IOTimerEventSource::timerEventSource(
 346                 kIOTimerEventSourceOptionsPriorityKernelHigh,
 347                 inOwner, inAction);
 348 }
 349
 350 void
 351 IOTimerEventSource::free()
 352 {
 353         if (calloutEntry) {
 354                 __assert_only bool freed;
 355
 356                 cancelTimeout();
 357
 358                 freed = thread_call_free((thread_call_t) calloutEntry);
 359                 assert(freed);
 360         }
 361
 362         if (reserved) {
 363                 IODelete(reserved, ExpansionData, 1);
 364         }
 365
 366         super::free();
 367 }
 368
 369 void
 370 IOTimerEventSource::cancelTimeout()
 371 {
 372         if (reserved) {
 373                 reserved->calloutGeneration++;
 374         }
 375         bool active = _thread_call_cancel((thread_call_t) calloutEntry);
 376         AbsoluteTime_to_scalar(&abstime) = 0;
 377         if (active && reserved && (kPassive & flags)) {
 378                 release();
 379                 workLoop->release();
 380         }
 381 }
 382
 383 void
 384 IOTimerEventSource::enable()
 385 {
 386         super::enable();
 387         if (kIOReturnSuccess != wakeAtTime(abstime)) {
 388                 super::disable(); // Problem re-scheduling timeout ignore enable
 389         }
 390 }
 391
 392 void
 393 IOTimerEventSource::disable()
 394 {
 395         if (reserved) {
 396                 reserved->calloutGeneration++;
 397         }
 398         bool active = _thread_call_cancel((thread_call_t) calloutEntry);
 399         super::disable();
 400         if (active && reserved && (kPassive & flags)) {
 401                 release();
 402                 workLoop->release();
 403         }
 404 }
 405
 406 IOReturn
 407 IOTimerEventSource::setTimeoutTicks(UInt32 ticks)
 408 {
 409         return setTimeout(ticks, kTickScale);
 410 }
 411
 412 IOReturn
 413 IOTimerEventSource::setTimeoutMS(UInt32 ms)
 414 {
 415         return setTimeout(ms, kMillisecondScale);
 416 }
 417
 418 IOReturn
 419 IOTimerEventSource::setTimeoutUS(UInt32 us)
 420 {
 421         return setTimeout(us, kMicrosecondScale);
 422 }
 423
 424 IOReturn
 425 IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
 426 {
 427         AbsoluteTime end;
 428
 429         clock_interval_to_deadline(interval, scale_factor, &end);
 430         return wakeAtTime(end);
 431 }
 432
 433 #if !defined(__LP64__)
 434 IOReturn
 435 IOTimerEventSource::setTimeout(mach_timespec_t interval)
 436 {
 437         AbsoluteTime end, nsecs;
 438
 439         clock_interval_to_absolutetime_interval
 440         (interval.tv_nsec, kNanosecondScale, &nsecs);
 441         clock_interval_to_deadline
 442         (interval.tv_sec, NSEC_PER_SEC, &end);
 443         ADD_ABSOLUTETIME(&end, &nsecs);
 444
 445         return wakeAtTime(end);
 446 }
 447 #endif
 448
 449 IOReturn
 450 IOTimerEventSource::setTimeout(AbsoluteTime interval)
 451 {
 452         AbsoluteTime end;
 453         clock_absolutetime_interval_to_deadline(interval, &end);
 454         return wakeAtTime(end);
 455 }
 456
 457 IOReturn
 458 IOTimerEventSource::setTimeout(uint32_t options,
 459     AbsoluteTime abstime, AbsoluteTime leeway)
 460 {
 461         AbsoluteTime end;
 462         if (options & kIOTimeOptionsContinuous) {
 463                 clock_continuoustime_interval_to_deadline(abstime, &end);
 464         } else {
 465                 clock_absolutetime_interval_to_deadline(abstime, &end);
 466         }
 467
 468         return wakeAtTime(options, end, leeway);
 469 }
 470
 471 IOReturn
 472 IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks)
 473 {
 474         return wakeAtTime(ticks, kTickScale);
 475 }
 476
 477 IOReturn
 478 IOTimerEventSource::wakeAtTimeMS(UInt32 ms)
 479 {
 480         return wakeAtTime(ms, kMillisecondScale);
 481 }
 482
 483 IOReturn
 484 IOTimerEventSource::wakeAtTimeUS(UInt32 us)
 485 {
 486         return wakeAtTime(us, kMicrosecondScale);
 487 }
 488
 489 IOReturn
 490 IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor)
 491 {
 492         AbsoluteTime end;
 493         clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end);
 494
 495         return wakeAtTime(end);
 496 }
 497
 498 #if !defined(__LP64__)
 499 IOReturn
 500 IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime)
 501 {
 502         AbsoluteTime end, nsecs;
 503
 504         clock_interval_to_absolutetime_interval
 505         (inAbstime.tv_nsec, kNanosecondScale, &nsecs);
 506         clock_interval_to_absolutetime_interval
 507         (inAbstime.tv_sec, kSecondScale, &end);
 508         ADD_ABSOLUTETIME(&end, &nsecs);
 509
 510         return wakeAtTime(end);
 511 }
 512 #endif
 513
 514 void
 515 IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
 516 {
 517         super::setWorkLoop(inWorkLoop);
 518         if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) {
 519                 wakeAtTime(abstime);
 520         }
 521 }
 522
 523 IOReturn
 524 IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime)
 525 {
 526         return wakeAtTime(0, inAbstime, 0);
 527 }
 528
 529 IOReturn
 530 IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway)
 531 {
 532         if (!action) {
 533                 return kIOReturnNoResources;
 534         }
 535
 536         abstime = inAbstime;
 537         if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) {
 538                 uint32_t tcoptions = 0;
 539
 540                 if (kIOTimeOptionsWithLeeway & options) {
 541                         tcoptions |= THREAD_CALL_DELAY_LEEWAY;
 542                 }
 543                 if (kIOTimeOptionsContinuous & options) {
 544                         tcoptions |= THREAD_CALL_CONTINUOUS;
 545                 }
 546
 547                 if (reserved) {
 548                         if (kPassive & flags) {
 549                                 retain();
 550                                 workLoop->retain();
 551                         }
 552                         reserved->workLoop = workLoop;
 553                         reserved->calloutGeneration++;
 554                         if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
 555                             (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions)
 556                             && (kPassive & flags)) {
 557                                 release();
 558                                 workLoop->release();
 559                         }
 560                 } else {
 561                         thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
 562                             NULL, inAbstime, leeway, tcoptions);
 563                 }
 564         }
 565
 566         return kIOReturnSuccess;
 567 }