iokit/Kernel/IOTimerEventSource.cpp

   1 /*
   2  * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <ptrauth.h>
  30 #include <sys/cdefs.h>
  31
  32 __BEGIN_DECLS
  33 #include <kern/thread_call.h>
  34 __END_DECLS
  35
  36 #include <IOKit/assert.h>
  37 #include <IOKit/system.h>
  38
  39 #include <IOKit/IOLib.h>
  40 #include <IOKit/IOTimerEventSource.h>
  41 #include <IOKit/IOWorkLoop.h>
  42
  43 #include <IOKit/IOTimeStamp.h>
  44 #include <IOKit/IOKitDebug.h>
  45 #if CONFIG_DTRACE
  46 #include <mach/sdt.h>
  47 #endif
  48
  49 #include <libkern/Block.h>
  50 #include <libkern/Block_private.h>
  51
  52
  53 #define super IOEventSource
  54 OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource)
  55 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0);
  56 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1);
  57 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2);
  58 OSMetaClassDefineReservedUnused(IOTimerEventSource, 3);
  59 OSMetaClassDefineReservedUnused(IOTimerEventSource, 4);
  60 OSMetaClassDefineReservedUnused(IOTimerEventSource, 5);
  61 OSMetaClassDefineReservedUnused(IOTimerEventSource, 6);
  62 OSMetaClassDefineReservedUnused(IOTimerEventSource, 7);
  63
  64 #if IOKITSTATS
  65
  66 #define IOStatisticsInitializeCounter() \
  67 do { \
  68         IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \
  69 } while (0)
  70
  71 #define IOStatisticsOpenGate() \
  72 do { \
  73         IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \
  74 } while (0)
  75
  76 #define IOStatisticsCloseGate() \
  77 do { \
  78         IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \
  79 } while (0)
  80
  81 #define IOStatisticsTimeout() \
  82 do { \
  83         IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \
  84 } while (0)
  85
  86 #else
  87
  88 #define IOStatisticsInitializeCounter()
  89 #define IOStatisticsOpenGate()
  90 #define IOStatisticsCloseGate()
  91 #define IOStatisticsTimeout()
  92
  93 #endif /* IOKITSTATS */
  94
  95 //
  96 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
  97 // not a subclassed implementation.
  98 //
  99
 100 // Timeout handler function. This function is called by the kernel when
 101 // the timeout interval expires.
 102 //
 103
 104 __inline__ void
 105 IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts,
 106     OSObject * _owner, IOWorkLoop * _workLoop)
 107 {
 108         bool    trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
 109         void  * address;
 110
 111         if (kActionBlock & flags) {
 112                 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock));
 113         } else {
 114                 address = ptrauth_nop_cast(void *, _action);
 115         }
 116
 117         if (trace) {
 118                 IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
 119                     VM_KERNEL_ADDRHIDE(address),
 120                     VM_KERNEL_ADDRHIDE(_owner));
 121         }
 122
 123         if (kActionBlock & flags) {
 124                 ((IOTimerEventSource::ActionBlock) actionBlock)(ts);
 125         } else {
 126                 ((IOTimerEventSource::Action)_action)(_owner, ts);
 127         }
 128
 129 #if CONFIG_DTRACE
 130         DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop);
 131 #endif
 132
 133         if (trace) {
 134                 IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
 135                     VM_KERNEL_UNSLIDE(address),
 136                     VM_KERNEL_ADDRHIDE(_owner));
 137         }
 138 }
 139
 140 void
 141 IOTimerEventSource::timeout(void *self)
 142 {
 143         IOTimerEventSource *me = (IOTimerEventSource *) self;
 144
 145         IOStatisticsTimeout();
 146
 147         if (me->enabled && me->action) {
 148                 IOWorkLoop *
 149                     wl = me->workLoop;
 150                 if (wl) {
 151                         IOEventSource::Action doit;
 152                         wl->closeGate();
 153                         IOStatisticsCloseGate();
 154                         doit = me->action;
 155                         if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) {
 156                                 me->invokeAction(doit, me, me->owner, me->workLoop);
 157                         }
 158                         IOStatisticsOpenGate();
 159                         wl->openGate();
 160                 }
 161         }
 162 }
 163
 164 void
 165 IOTimerEventSource::timeoutAndRelease(void * self, void * c)
 166 {
 167         IOTimerEventSource *me = (IOTimerEventSource *) self;
 168         /* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count"
 169          *  must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */
 170         SInt32 count = (SInt32) (long) c;
 171
 172         IOStatisticsTimeout();
 173
 174         if (me->enabled && me->action) {
 175                 IOWorkLoop *
 176                     wl = me->reserved->workLoop;
 177                 if (wl) {
 178                         IOEventSource::Action doit;
 179                         wl->closeGate();
 180                         IOStatisticsCloseGate();
 181                         doit = me->action;
 182                         if (doit && (me->reserved->calloutGeneration == count)) {
 183                                 thread_call_start_iotes_invocation((thread_call_t)me->calloutEntry);
 184                                 me->invokeAction(doit, me, me->owner, me->workLoop);
 185                         }
 186                         IOStatisticsOpenGate();
 187                         wl->openGate();
 188                 }
 189         }
 190
 191         me->reserved->workLoop->release();
 192         me->release();
 193 }
 194
 195 // -- work loop delivery
 196
 197 bool
 198 IOTimerEventSource::checkForWork()
 199 {
 200         IOEventSource::Action doit;
 201
 202         if (reserved
 203             && (reserved->calloutGenerationSignaled == reserved->calloutGeneration)
 204             && enabled && (doit = action)) {
 205                 reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
 206                 invokeAction(doit, this, owner, workLoop);
 207         }
 208
 209         return false;
 210 }
 211
 212 void
 213 IOTimerEventSource::timeoutSignaled(void * self, void * c)
 214 {
 215         IOTimerEventSource *me = (IOTimerEventSource *) self;
 216
 217         me->reserved->calloutGenerationSignaled = (SInt32)(long) c;
 218         if (me->enabled) {
 219                 me->signalWorkAvailable();
 220         }
 221 }
 222
 223 // --
 224
 225 void
 226 IOTimerEventSource::setTimeoutFunc()
 227 {
 228         thread_call_priority_t pri;
 229         uint32_t options;
 230
 231         if (reserved) {
 232                 panic("setTimeoutFunc already %p, %p", this, reserved);
 233         }
 234
 235         // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
 236         // not a subclassed implementation
 237         reserved = IONewZero(ExpansionData, 1);
 238
 239         reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
 240         // make use of an existing ivar for parameter passing
 241         options = (uint32_t) abstime;
 242         abstime = 0;
 243
 244         thread_call_options_t tcoptions = 0;
 245         thread_call_func_t    func      = NULL;
 246
 247         switch (kIOTimerEventSourceOptionsPriorityMask & options) {
 248         case kIOTimerEventSourceOptionsPriorityHigh:
 249                 pri = THREAD_CALL_PRIORITY_HIGH;
 250                 func = &IOTimerEventSource::timeoutAndRelease;
 251                 break;
 252
 253         case kIOTimerEventSourceOptionsPriorityKernel:
 254                 pri = THREAD_CALL_PRIORITY_KERNEL;
 255                 func = &IOTimerEventSource::timeoutAndRelease;
 256                 break;
 257
 258         case kIOTimerEventSourceOptionsPriorityKernelHigh:
 259                 pri = THREAD_CALL_PRIORITY_KERNEL_HIGH;
 260                 func = &IOTimerEventSource::timeoutAndRelease;
 261                 break;
 262
 263         case kIOTimerEventSourceOptionsPriorityUser:
 264                 pri = THREAD_CALL_PRIORITY_USER;
 265                 func = &IOTimerEventSource::timeoutAndRelease;
 266                 break;
 267
 268         case kIOTimerEventSourceOptionsPriorityLow:
 269                 pri = THREAD_CALL_PRIORITY_LOW;
 270                 func = &IOTimerEventSource::timeoutAndRelease;
 271                 break;
 272
 273         case kIOTimerEventSourceOptionsPriorityWorkLoop:
 274                 pri = THREAD_CALL_PRIORITY_KERNEL;
 275                 tcoptions |= THREAD_CALL_OPTIONS_SIGNAL;
 276                 if (kIOTimerEventSourceOptionsAllowReenter & options) {
 277                         break;
 278                 }
 279                 func = &IOTimerEventSource::timeoutSignaled;
 280                 break;
 281
 282         default:
 283                 break;
 284         }
 285
 286         assertf(func, "IOTimerEventSource options 0x%x", options);
 287         if (!func) {
 288                 return;                                              // init will fail
 289         }
 290         if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) {
 291                 flags |= kActive;
 292         } else {
 293                 flags |= kPassive;
 294         }
 295
 296         if (!(kIOTimerEventSourceOptionsAllowReenter & options)) {
 297                 tcoptions |= THREAD_CALL_OPTIONS_ONCE;
 298         }
 299
 300         calloutEntry = (void *) thread_call_allocate_with_options(func,
 301             (thread_call_param_t) this, pri, tcoptions);
 302         assert(calloutEntry);
 303 }
 304
 305 bool
 306 IOTimerEventSource::init(OSObject *inOwner, Action inAction)
 307 {
 308         if (!super::init(inOwner, (IOEventSource::Action) inAction)) {
 309                 return false;
 310         }
 311
 312         setTimeoutFunc();
 313         if (!calloutEntry) {
 314                 return false;
 315         }
 316
 317         IOStatisticsInitializeCounter();
 318
 319         return true;
 320 }
 321
 322 bool
 323 IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction)
 324 {
 325         // make use of an existing ivar for parameter passing
 326         abstime = options;
 327         return init(inOwner, inAction);
 328 }
 329
 330 IOTimerEventSource *
 331 IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction)
 332 {
 333         IOTimerEventSource *me = new IOTimerEventSource;
 334
 335         if (me && !me->init(inOptions, inOwner, inAction)) {
 336                 me->release();
 337                 return NULL;
 338         }
 339
 340         return me;
 341 }
 342
 343 IOTimerEventSource *
 344 IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action)
 345 {
 346         IOTimerEventSource * tes;
 347         tes = IOTimerEventSource::timerEventSource(options, inOwner, (Action) NULL);
 348         if (tes) {
 349                 tes->setActionBlock((IOEventSource::ActionBlock) _action);
 350         }
 351
 352         return tes;
 353 }
 354
 355 #define _thread_call_cancel(tc)   ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc)))
 356
 357 IOTimerEventSource *
 358 IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction)
 359 {
 360         return IOTimerEventSource::timerEventSource(
 361                 kIOTimerEventSourceOptionsPriorityKernelHigh,
 362                 inOwner, inAction);
 363 }
 364
 365 void
 366 IOTimerEventSource::free()
 367 {
 368         if (calloutEntry) {
 369                 __assert_only bool freed;
 370
 371                 cancelTimeout();
 372
 373                 freed = thread_call_free((thread_call_t) calloutEntry);
 374                 assert(freed);
 375         }
 376
 377         if (reserved) {
 378                 IODelete(reserved, ExpansionData, 1);
 379         }
 380
 381         super::free();
 382 }
 383
 384 void
 385 IOTimerEventSource::cancelTimeout()
 386 {
 387         if (reserved) {
 388                 reserved->calloutGeneration++;
 389         }
 390         bool active = _thread_call_cancel((thread_call_t) calloutEntry);
 391         AbsoluteTime_to_scalar(&abstime) = 0;
 392         if (active && reserved && (kPassive & flags)) {
 393                 release();
 394                 workLoop->release();
 395         }
 396 }
 397
 398 void
 399 IOTimerEventSource::enable()
 400 {
 401         super::enable();
 402         if (kIOReturnSuccess != wakeAtTime(abstime)) {
 403                 super::disable(); // Problem re-scheduling timeout ignore enable
 404         }
 405 }
 406
 407 void
 408 IOTimerEventSource::disable()
 409 {
 410         if (reserved) {
 411                 reserved->calloutGeneration++;
 412         }
 413         bool active = _thread_call_cancel((thread_call_t) calloutEntry);
 414         super::disable();
 415         if (active && reserved && (kPassive & flags)) {
 416                 release();
 417                 workLoop->release();
 418         }
 419 }
 420
 421 IOReturn
 422 IOTimerEventSource::setTimeoutTicks(UInt32 ticks)
 423 {
 424         return setTimeout(ticks, kTickScale);
 425 }
 426
 427 IOReturn
 428 IOTimerEventSource::setTimeoutMS(UInt32 ms)
 429 {
 430         return setTimeout(ms, kMillisecondScale);
 431 }
 432
 433 IOReturn
 434 IOTimerEventSource::setTimeoutUS(UInt32 us)
 435 {
 436         return setTimeout(us, kMicrosecondScale);
 437 }
 438
 439 IOReturn
 440 IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
 441 {
 442         AbsoluteTime end;
 443
 444         clock_interval_to_deadline(interval, scale_factor, &end);
 445         return wakeAtTime(end);
 446 }
 447
 448 #if !defined(__LP64__)
 449 IOReturn
 450 IOTimerEventSource::setTimeout(mach_timespec_t interval)
 451 {
 452         AbsoluteTime end, nsecs;
 453
 454         clock_interval_to_absolutetime_interval
 455         (interval.tv_nsec, kNanosecondScale, &nsecs);
 456         clock_interval_to_deadline
 457         (interval.tv_sec, NSEC_PER_SEC, &end);
 458         ADD_ABSOLUTETIME(&end, &nsecs);
 459
 460         return wakeAtTime(end);
 461 }
 462 #endif
 463
 464 IOReturn
 465 IOTimerEventSource::setTimeout(AbsoluteTime interval)
 466 {
 467         AbsoluteTime end;
 468         clock_absolutetime_interval_to_deadline(interval, &end);
 469         return wakeAtTime(end);
 470 }
 471
 472 IOReturn
 473 IOTimerEventSource::setTimeout(uint32_t options,
 474     AbsoluteTime abstime, AbsoluteTime leeway)
 475 {
 476         AbsoluteTime end;
 477         if (options & kIOTimeOptionsContinuous) {
 478                 clock_continuoustime_interval_to_deadline(abstime, &end);
 479         } else {
 480                 clock_absolutetime_interval_to_deadline(abstime, &end);
 481         }
 482
 483         return wakeAtTime(options, end, leeway);
 484 }
 485
 486 IOReturn
 487 IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks)
 488 {
 489         return wakeAtTime(ticks, kTickScale);
 490 }
 491
 492 IOReturn
 493 IOTimerEventSource::wakeAtTimeMS(UInt32 ms)
 494 {
 495         return wakeAtTime(ms, kMillisecondScale);
 496 }
 497
 498 IOReturn
 499 IOTimerEventSource::wakeAtTimeUS(UInt32 us)
 500 {
 501         return wakeAtTime(us, kMicrosecondScale);
 502 }
 503
 504 IOReturn
 505 IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor)
 506 {
 507         AbsoluteTime end;
 508         clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end);
 509
 510         return wakeAtTime(end);
 511 }
 512
 513 #if !defined(__LP64__)
 514 IOReturn
 515 IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime)
 516 {
 517         AbsoluteTime end, nsecs;
 518
 519         clock_interval_to_absolutetime_interval
 520         (inAbstime.tv_nsec, kNanosecondScale, &nsecs);
 521         clock_interval_to_absolutetime_interval
 522         (inAbstime.tv_sec, kSecondScale, &end);
 523         ADD_ABSOLUTETIME(&end, &nsecs);
 524
 525         return wakeAtTime(end);
 526 }
 527 #endif
 528
 529 void
 530 IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
 531 {
 532         super::setWorkLoop(inWorkLoop);
 533         if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) {
 534                 wakeAtTime(abstime);
 535         }
 536 }
 537
 538 IOReturn
 539 IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime)
 540 {
 541         return wakeAtTime(0, inAbstime, 0);
 542 }
 543
 544 IOReturn
 545 IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway)
 546 {
 547         if (!action) {
 548                 return kIOReturnNoResources;
 549         }
 550
 551         abstime = inAbstime;
 552         if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) {
 553                 uint32_t tcoptions = 0;
 554
 555                 if (kIOTimeOptionsWithLeeway & options) {
 556                         tcoptions |= THREAD_CALL_DELAY_LEEWAY;
 557                 }
 558                 if (kIOTimeOptionsContinuous & options) {
 559                         tcoptions |= THREAD_CALL_CONTINUOUS;
 560                 }
 561
 562                 if (reserved) {
 563                         if (kPassive & flags) {
 564                                 retain();
 565                                 workLoop->retain();
 566                         }
 567                         reserved->workLoop = workLoop;
 568                         reserved->calloutGeneration++;
 569                         if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
 570                             (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions)
 571                             && (kPassive & flags)) {
 572                                 release();
 573                                 workLoop->release();
 574                         }
 575                 } else {
 576                         thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
 577                             NULL, inAbstime, leeway, tcoptions);
 578                 }
 579         }
 580
 581         return kIOReturnSuccess;
 582 }