iokit/Kernel/IOInterruptEventSource.cpp

   1 /*
   2  * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/IOInterruptEventSource.h>
  30 #include <IOKit/IOKitDebug.h>
  31 #include <IOKit/IOLib.h>
  32 #include <IOKit/IOService.h>
  33 #include <IOKit/IOInterrupts.h>
  34 #include <IOKit/IOTimeStamp.h>
  35 #include <IOKit/IOWorkLoop.h>
  36 #include <IOKit/IOInterruptAccountingPrivate.h>
  37
  38 #if IOKITSTATS
  39
  40 #define IOStatisticsInitializeCounter() \
  41 do { \
  42         IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
  43 } while (0)
  44
  45 #define IOStatisticsCheckForWork() \
  46 do { \
  47         IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
  48 } while (0)
  49
  50 #define IOStatisticsInterrupt() \
  51 do { \
  52         IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
  53 } while (0)
  54
  55 #else
  56
  57 #define IOStatisticsInitializeCounter()
  58 #define IOStatisticsCheckForWork()
  59 #define IOStatisticsInterrupt()
  60
  61 #endif // IOKITSTATS
  62
  63 #define super IOEventSource
  64
  65 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
  66 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
  67 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
  68 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
  69 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
  70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
  71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
  72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
  73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
  74
  75 bool
  76 IOInterruptEventSource::init(OSObject *inOwner,
  77     Action inAction,
  78     IOService *inProvider,
  79     int inIntIndex)
  80 {
  81         bool res = true;
  82
  83         if (!super::init(inOwner, (IOEventSourceAction) inAction)) {
  84                 return false;
  85         }
  86
  87         reserved = IONew(ExpansionData, 1);
  88
  89         if (!reserved) {
  90                 return false;
  91         }
  92
  93         bzero(reserved, sizeof(ExpansionData));
  94
  95         provider = inProvider;
  96         producerCount = consumerCount = 0;
  97         autoDisable = explicitDisable = false;
  98         intIndex = ~inIntIndex;
  99
 100         // Assumes inOwner holds a reference(retain) on the provider
 101         if (inProvider) {
 102                 if (IA_ANY_STATISTICS_ENABLED) {
 103                         /*
 104                          * We only treat this as an "interrupt" if it has a provider; if it does,
 105                          * set up the objects necessary to track interrupt statistics.  Interrupt
 106                          * event sources without providers are most likely being used as simple
 107                          * event source in order to poke at workloops and kick off work.
 108                          *
 109                          * We also avoid try to avoid interrupt accounting overhead if none of
 110                          * the statistics are enabled.
 111                          */
 112                         reserved->statistics = IONew(IOInterruptAccountingData, 1);
 113
 114                         if (!reserved->statistics) {
 115                                 /*
 116                                  * We rely on the free() routine to clean up after us if init fails
 117                                  * midway.
 118                                  */
 119                                 return false;
 120                         }
 121
 122                         bzero(reserved->statistics, sizeof(IOInterruptAccountingData));
 123
 124                         reserved->statistics->owner = this;
 125                 }
 126
 127                 res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
 128
 129                 if (res) {
 130                         intIndex = inIntIndex;
 131                 }
 132         }
 133
 134         IOStatisticsInitializeCounter();
 135
 136         return res;
 137 }
 138
 139 IOReturn
 140 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
 141     int inIntIndex)
 142 {
 143         IOReturn ret;
 144         int intType;
 145         IOInterruptAction intHandler;
 146
 147         ret = inProvider->getInterruptType(inIntIndex, &intType);
 148         if (kIOReturnSuccess != ret) {
 149                 return ret;
 150         }
 151
 152         autoDisable = (intType == kIOInterruptTypeLevel);
 153         if (autoDisable) {
 154                 intHandler = OSMemberFunctionCast(IOInterruptAction,
 155                     this, &IOInterruptEventSource::disableInterruptOccurred);
 156         } else {
 157                 intHandler = OSMemberFunctionCast(IOInterruptAction,
 158                     this, &IOInterruptEventSource::normalInterruptOccurred);
 159         }
 160
 161         ret = provider->registerInterrupt(inIntIndex, this, intHandler);
 162
 163         /*
 164          * Add statistics to the provider.  The setWorkLoop convention should ensure
 165          * that we always go down the unregister path before we register (outside of
 166          * init()), so we don't have to worry that we will invoke addInterruptStatistics
 167          * erroneously.
 168          */
 169         if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
 170                 /*
 171                  * Stash the normal index value, for the sake of debugging.
 172                  */
 173                 reserved->statistics->interruptIndex = inIntIndex;
 174
 175                 /*
 176                  * We need to hook the interrupt information up to the provider so that it
 177                  * can find the statistics for this interrupt when desired.  The provider is
 178                  * responsible for maintaining the reporter for a particular interrupt, and
 179                  * needs a handle on the statistics so that it can request that the reporter
 180                  * be updated as needed.  Errors are considered "soft" for the moment (it
 181                  * will either panic, or fail in a way such that we can still service the
 182                  * interrupt).
 183                  */
 184                 provider->addInterruptStatistics(reserved->statistics, inIntIndex);
 185
 186                 /*
 187                  * Add the statistics object to the global list of statistics objects; this
 188                  * is an aid to debugging (we can trivially find statistics for all eligible
 189                  * interrupts, and dump them; potentially helpful if the system is wedged
 190                  * due to interrupt activity).
 191                  */
 192                 interruptAccountingDataAddToList(reserved->statistics);
 193         }
 194
 195         return ret;
 196 }
 197
 198 void
 199 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
 200     int inIntIndex)
 201 {
 202         if (reserved->statistics) {
 203                 interruptAccountingDataRemoveFromList(reserved->statistics);
 204                 provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
 205         }
 206
 207         provider->unregisterInterrupt(inIntIndex);
 208 }
 209
 210
 211 IOInterruptEventSource *
 212 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
 213     Action inAction,
 214     IOService *inProvider,
 215     int inIntIndex)
 216 {
 217         IOInterruptEventSource *me = new IOInterruptEventSource;
 218
 219         if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
 220                 me->release();
 221                 return NULL;
 222         }
 223
 224         return me;
 225 }
 226
 227 IOInterruptEventSource *
 228 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
 229     IOService *inProvider,
 230     int inIntIndex,
 231     ActionBlock inAction)
 232 {
 233         IOInterruptEventSource * ies;
 234         ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex);
 235         if (ies) {
 236                 ies->setActionBlock((IOEventSource::ActionBlock) inAction);
 237         }
 238
 239         return ies;
 240 }
 241
 242 void
 243 IOInterruptEventSource::free()
 244 {
 245         if (provider && intIndex >= 0) {
 246                 unregisterInterruptHandler(provider, intIndex);
 247         }
 248
 249         if (reserved) {
 250                 if (reserved->statistics) {
 251                         IODelete(reserved->statistics, IOInterruptAccountingData, 1);
 252                 }
 253
 254                 IODelete(reserved, ExpansionData, 1);
 255         }
 256
 257         super::free();
 258 }
 259
 260 void
 261 IOInterruptEventSource::enable()
 262 {
 263         if (provider && intIndex >= 0) {
 264                 provider->enableInterrupt(intIndex);
 265                 explicitDisable = false;
 266                 enabled = true;
 267         }
 268 }
 269
 270 void
 271 IOInterruptEventSource::disable()
 272 {
 273         if (provider && intIndex >= 0) {
 274                 provider->disableInterrupt(intIndex);
 275                 explicitDisable = true;
 276                 enabled = false;
 277         }
 278 }
 279
 280 void
 281 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
 282 {
 283         if (inWorkLoop) {
 284                 super::setWorkLoop(inWorkLoop);
 285         }
 286
 287         if (provider) {
 288                 if (!inWorkLoop) {
 289                         if (intIndex >= 0) {
 290                                 /*
 291                                  * It isn't necessarily safe to wait until free() to unregister the interrupt;
 292                                  * our provider may disappear.
 293                                  */
 294                                 unregisterInterruptHandler(provider, intIndex);
 295                                 intIndex = ~intIndex;
 296                         }
 297                 } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
 298                         intIndex = ~intIndex;
 299                 }
 300         }
 301
 302         if (!inWorkLoop) {
 303                 super::setWorkLoop(inWorkLoop);
 304         }
 305 }
 306
 307 const IOService *
 308 IOInterruptEventSource::getProvider() const
 309 {
 310         return provider;
 311 }
 312
 313 int
 314 IOInterruptEventSource::getIntIndex() const
 315 {
 316         return intIndex;
 317 }
 318
 319 bool
 320 IOInterruptEventSource::getAutoDisable() const
 321 {
 322         return autoDisable;
 323 }
 324
 325 bool
 326 IOInterruptEventSource::checkForWork()
 327 {
 328         uint64_t startSystemTime = 0;
 329         uint64_t endSystemTime = 0;
 330         uint64_t startCPUTime = 0;
 331         uint64_t endCPUTime = 0;
 332         unsigned int cacheProdCount = producerCount;
 333         int numInts = cacheProdCount - consumerCount;
 334         IOInterruptEventAction intAction = (IOInterruptEventAction) action;
 335         ActionBlock intActionBlock = (ActionBlock) actionBlock;
 336         bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
 337
 338         IOStatisticsCheckForWork();
 339
 340         if (numInts > 0) {
 341                 if (trace) {
 342                         IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
 343                             VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
 344                             VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
 345                 }
 346
 347                 if (reserved->statistics) {
 348                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
 349                                 startSystemTime = mach_absolute_time();
 350                         }
 351
 352                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
 353                                 startCPUTime = thread_get_runtime_self();
 354                         }
 355                 }
 356
 357                 // Call the handler
 358                 if (kActionBlock & flags) {
 359                         (intActionBlock)(this, numInts);
 360                 } else {
 361                         (*intAction)(owner, this, numInts);
 362                 }
 363
 364                 if (reserved->statistics) {
 365                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
 366                                 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
 367                         }
 368
 369                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
 370                                 endCPUTime = thread_get_runtime_self();
 371                                 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
 372                         }
 373
 374                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
 375                                 endSystemTime = mach_absolute_time();
 376                                 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
 377                         }
 378                 }
 379
 380                 if (trace) {
 381                         IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
 382                             VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
 383                             VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
 384                 }
 385
 386                 consumerCount = cacheProdCount;
 387                 if (autoDisable && !explicitDisable) {
 388                         enable();
 389                 }
 390         } else if (numInts < 0) {
 391                 if (trace) {
 392                         IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
 393                             VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
 394                             VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
 395                 }
 396
 397                 if (reserved->statistics) {
 398                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
 399                                 startSystemTime = mach_absolute_time();
 400                         }
 401
 402                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
 403                                 startCPUTime = thread_get_runtime_self();
 404                         }
 405                 }
 406
 407                 // Call the handler
 408                 if (kActionBlock & flags) {
 409                         (intActionBlock)(this, numInts);
 410                 } else {
 411                         (*intAction)(owner, this, numInts);
 412                 }
 413
 414                 if (reserved->statistics) {
 415                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
 416                                 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
 417                         }
 418
 419                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
 420                                 endCPUTime = thread_get_runtime_self();
 421                                 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
 422                         }
 423
 424                         if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
 425                                 endSystemTime = mach_absolute_time();
 426                                 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
 427                         }
 428                 }
 429
 430                 if (trace) {
 431                         IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
 432                             VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
 433                             VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
 434                 }
 435
 436                 consumerCount = cacheProdCount;
 437                 if (autoDisable && !explicitDisable) {
 438                         enable();
 439                 }
 440         }
 441
 442         return false;
 443 }
 444
 445 void
 446 IOInterruptEventSource::normalInterruptOccurred
 447 (void */*refcon*/, IOService */*prov*/, int /*source*/)
 448 {
 449         bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
 450
 451         IOStatisticsInterrupt();
 452         producerCount++;
 453
 454         if (trace) {
 455                 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
 456         }
 457
 458         if (reserved->statistics) {
 459                 if (reserved->statistics->enablePrimaryTimestamp) {
 460                         reserved->statistics->primaryTimestamp = mach_absolute_time();
 461                 }
 462                 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
 463                         IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
 464                 }
 465         }
 466
 467         signalWorkAvailable();
 468
 469         if (trace) {
 470                 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
 471         }
 472 }
 473
 474 void
 475 IOInterruptEventSource::disableInterruptOccurred
 476 (void */*refcon*/, IOService *prov, int source)
 477 {
 478         bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
 479
 480         prov->disableInterrupt(source); /* disable the interrupt */
 481
 482         IOStatisticsInterrupt();
 483         producerCount++;
 484
 485         if (trace) {
 486                 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
 487         }
 488
 489         if (reserved->statistics) {
 490                 if (reserved->statistics->enablePrimaryTimestamp) {
 491                         reserved->statistics->primaryTimestamp = mach_absolute_time();
 492                 }
 493                 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
 494                         IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
 495                 }
 496         }
 497
 498         signalWorkAvailable();
 499
 500         if (trace) {
 501                 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
 502         }
 503 }
 504
 505 void
 506 IOInterruptEventSource::interruptOccurred
 507 (void *_refcon, IOService *prov, int source)
 508 {
 509         if (autoDisable && prov) {
 510                 disableInterruptOccurred(_refcon, prov, source);
 511         } else {
 512                 normalInterruptOccurred(_refcon, prov, source);
 513         }
 514 }
 515
 516 IOReturn
 517 IOInterruptEventSource::warmCPU
 518 (uint64_t abstime)
 519 {
 520         return ml_interrupt_prewarm(abstime);
 521 }
 522
 523 void
 524 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable)
 525 {
 526         if (reserved->statistics) {
 527                 reserved->statistics->enablePrimaryTimestamp = enable;
 528         }
 529 }
 530
 531 uint64_t
 532 IOInterruptEventSource::getPimaryInterruptTimestamp()
 533 {
 534         if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) {
 535                 return reserved->statistics->primaryTimestamp;
 536         }
 537         return -1ULL;
 538 }