]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOInterruptEventSource.cpp
projects / apple / xnu.git / blob
? search:


a720e9a301b9c547d0b8b0330836125e72c154b7
[apple/xnu.git] / iokit / Kernel / IOInterruptEventSource.cpp
1 /*
2 * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #define IOKIT_ENABLE_SHARED_PTR
30
31 #include <ptrauth.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOKitDebug.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOService.h>
36 #include <IOKit/IOInterrupts.h>
37 #include <IOKit/IOTimeStamp.h>
38 #include <IOKit/IOWorkLoop.h>
39 #include <IOKit/IOInterruptAccountingPrivate.h>
40 #include <libkern/Block_private.h>
41
42 #if IOKITSTATS
43
44 #define IOStatisticsInitializeCounter() \
45 do { \
46 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
47 } while (0)
48
49 #define IOStatisticsCheckForWork() \
50 do { \
51 IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
52 } while (0)
53
54 #define IOStatisticsInterrupt() \
55 do { \
56 IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
57 } while (0)
58
59 #else
60
61 #define IOStatisticsInitializeCounter()
62 #define IOStatisticsCheckForWork()
63 #define IOStatisticsInterrupt()
64
65 #endif // IOKITSTATS
66
67 #define super IOEventSource
68
69 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
74 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
75 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
76 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
77 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
78
79 bool
80 IOInterruptEventSource::init(OSObject *inOwner,
81 Action inAction,
82 IOService *inProvider,
83 int inIntIndex)
84 {
85 bool res = true;
86
87 if (!super::init(inOwner, (IOEventSourceAction) inAction)) {
88 return false;
89 }
90
91 reserved = IONew(ExpansionData, 1);
92
93 if (!reserved) {
94 return false;
95 }
96
97 bzero(reserved, sizeof(ExpansionData));
98
99 provider = inProvider;
100 producerCount = consumerCount = 0;
101 autoDisable = explicitDisable = false;
102 intIndex = ~inIntIndex;
103
104 // Assumes inOwner holds a reference(retain) on the provider
105 if (inProvider) {
106 if (IA_ANY_STATISTICS_ENABLED) {
107 /*
108 * We only treat this as an "interrupt" if it has a provider; if it does,
109 * set up the objects necessary to track interrupt statistics. Interrupt
110 * event sources without providers are most likely being used as simple
111 * event source in order to poke at workloops and kick off work.
112 *
113 * We also avoid try to avoid interrupt accounting overhead if none of
114 * the statistics are enabled.
115 */
116 reserved->statistics = IONew(IOInterruptAccountingData, 1);
117
118 if (!reserved->statistics) {
119 /*
120 * We rely on the free() routine to clean up after us if init fails
121 * midway.
122 */
123 return false;
124 }
125
126 bzero(reserved->statistics, sizeof(IOInterruptAccountingData));
127
128 reserved->statistics->owner = this;
129 }
130
131 res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
132
133 if (res) {
134 intIndex = inIntIndex;
135 }
136 }
137
138 IOStatisticsInitializeCounter();
139
140 return res;
141 }
142
143 IOReturn
144 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
145 int inIntIndex)
146 {
147 IOReturn ret;
148 int intType;
149 IOInterruptAction intHandler;
150
151 ret = inProvider->getInterruptType(inIntIndex, &intType);
152 if (kIOReturnSuccess != ret) {
153 return ret;
154 }
155
156 autoDisable = (intType == kIOInterruptTypeLevel);
157 if (autoDisable) {
158 intHandler = OSMemberFunctionCast(IOInterruptAction,
159 this, &IOInterruptEventSource::disableInterruptOccurred);
160 } else {
161 intHandler = OSMemberFunctionCast(IOInterruptAction,
162 this, &IOInterruptEventSource::normalInterruptOccurred);
163 }
164
165 ret = provider->registerInterrupt(inIntIndex, this, intHandler);
166
167 /*
168 * Add statistics to the provider. The setWorkLoop convention should ensure
169 * that we always go down the unregister path before we register (outside of
170 * init()), so we don't have to worry that we will invoke addInterruptStatistics
171 * erroneously.
172 */
173 if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
174 /*
175 * Stash the normal index value, for the sake of debugging.
176 */
177 reserved->statistics->interruptIndex = inIntIndex;
178
179 /*
180 * We need to hook the interrupt information up to the provider so that it
181 * can find the statistics for this interrupt when desired. The provider is
182 * responsible for maintaining the reporter for a particular interrupt, and
183 * needs a handle on the statistics so that it can request that the reporter
184 * be updated as needed. Errors are considered "soft" for the moment (it
185 * will either panic, or fail in a way such that we can still service the
186 * interrupt).
187 */
188 provider->addInterruptStatistics(reserved->statistics, inIntIndex);
189
190 /*
191 * Add the statistics object to the global list of statistics objects; this
192 * is an aid to debugging (we can trivially find statistics for all eligible
193 * interrupts, and dump them; potentially helpful if the system is wedged
194 * due to interrupt activity).
195 */
196 interruptAccountingDataAddToList(reserved->statistics);
197 }
198
199 return ret;
200 }
201
202 void
203 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
204 int inIntIndex)
205 {
206 if (reserved->statistics) {
207 interruptAccountingDataRemoveFromList(reserved->statistics);
208 provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
209 }
210
211 provider->unregisterInterrupt(inIntIndex);
212 }
213
214
215 OSSharedPtr<IOInterruptEventSource>
216 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
217 Action inAction,
218 IOService *inProvider,
219 int inIntIndex)
220 {
221 OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>();
222
223 if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
224 return nullptr;
225 }
226
227 return me;
228 }
229
230 OSSharedPtr<IOInterruptEventSource>
231 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
232 IOService *inProvider,
233 int inIntIndex,
234 ActionBlock inAction)
235 {
236 OSSharedPtr<IOInterruptEventSource> ies;
237 ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex);
238 if (ies) {
239 ies->setActionBlock((IOEventSource::ActionBlock) inAction);
240 }
241
242 return ies;
243 }
244
245 void
246 IOInterruptEventSource::free()
247 {
248 if (provider && intIndex >= 0) {
249 unregisterInterruptHandler(provider, intIndex);
250 }
251
252 if (reserved) {
253 if (reserved->statistics) {
254 IODelete(reserved->statistics, IOInterruptAccountingData, 1);
255 }
256
257 IODelete(reserved, ExpansionData, 1);
258 }
259
260 super::free();
261 }
262
263 void
264 IOInterruptEventSource::enable()
265 {
266 if (provider && intIndex >= 0) {
267 provider->enableInterrupt(intIndex);
268 explicitDisable = false;
269 enabled = true;
270 }
271 }
272
273 void
274 IOInterruptEventSource::disable()
275 {
276 if (provider && intIndex >= 0) {
277 provider->disableInterrupt(intIndex);
278 explicitDisable = true;
279 enabled = false;
280 }
281 }
282
283 void
284 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
285 {
286 if (inWorkLoop) {
287 super::setWorkLoop(inWorkLoop);
288 }
289
290 if (provider) {
291 if (!inWorkLoop) {
292 if (intIndex >= 0) {
293 /*
294 * It isn't necessarily safe to wait until free() to unregister the interrupt;
295 * our provider may disappear.
296 */
297 unregisterInterruptHandler(provider, intIndex);
298 intIndex = ~intIndex;
299 }
300 } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
301 intIndex = ~intIndex;
302 }
303 }
304
305 if (!inWorkLoop) {
306 super::setWorkLoop(inWorkLoop);
307 }
308 }
309
310 const IOService *
311 IOInterruptEventSource::getProvider() const
312 {
313 return provider;
314 }
315
316 int
317 IOInterruptEventSource::getIntIndex() const
318 {
319 return intIndex;
320 }
321
322 bool
323 IOInterruptEventSource::getAutoDisable() const
324 {
325 return autoDisable;
326 }
327
328 bool
329 IOInterruptEventSource::checkForWork()
330 {
331 uint64_t startSystemTime = 0;
332 uint64_t endSystemTime = 0;
333 uint64_t startCPUTime = 0;
334 uint64_t endCPUTime = 0;
335 unsigned int cacheProdCount = producerCount;
336 int numInts = cacheProdCount - consumerCount;
337 IOEventSource::Action intAction = action;
338 ActionBlock intActionBlock = (ActionBlock) actionBlock;
339 void *address;
340 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
341
342 if (kActionBlock & flags) {
343 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock));
344 } else {
345 address = ptrauth_nop_cast(void *, intAction);
346 }
347
348 IOStatisticsCheckForWork();
349
350 if (numInts > 0) {
351 if (trace) {
352 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
353 VM_KERNEL_ADDRHIDE(address),
354 VM_KERNEL_ADDRHIDE(owner),
355 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
356 }
357
358 if (reserved->statistics) {
359 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
360 startSystemTime = mach_absolute_time();
361 }
362
363 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
364 startCPUTime = thread_get_runtime_self();
365 }
366 }
367
368 // Call the handler
369 if (kActionBlock & flags) {
370 (intActionBlock)(this, numInts);
371 } else {
372 ((IOInterruptEventAction)intAction)(owner, this, numInts);
373 }
374
375 if (reserved->statistics) {
376 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
377 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
378 }
379
380 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
381 endCPUTime = thread_get_runtime_self();
382 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
383 }
384
385 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
386 endSystemTime = mach_absolute_time();
387 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
388 }
389 }
390
391 if (trace) {
392 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
393 VM_KERNEL_ADDRHIDE(address),
394 VM_KERNEL_ADDRHIDE(owner),
395 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
396 }
397
398 consumerCount = cacheProdCount;
399 if (autoDisable && !explicitDisable) {
400 enable();
401 }
402 } else if (numInts < 0) {
403 if (trace) {
404 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
405 VM_KERNEL_ADDRHIDE(address),
406 VM_KERNEL_ADDRHIDE(owner),
407 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
408 }
409
410 if (reserved->statistics) {
411 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
412 startSystemTime = mach_absolute_time();
413 }
414
415 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
416 startCPUTime = thread_get_runtime_self();
417 }
418 }
419
420 // Call the handler
421 if (kActionBlock & flags) {
422 (intActionBlock)(this, numInts);
423 } else {
424 ((IOInterruptEventAction)intAction)(owner, this, numInts);
425 }
426
427 if (reserved->statistics) {
428 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
429 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
430 }
431
432 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
433 endCPUTime = thread_get_runtime_self();
434 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
435 }
436
437 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
438 endSystemTime = mach_absolute_time();
439 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
440 }
441 }
442
443 if (trace) {
444 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
445 VM_KERNEL_ADDRHIDE(address),
446 VM_KERNEL_ADDRHIDE(owner),
447 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
448 }
449
450 consumerCount = cacheProdCount;
451 if (autoDisable && !explicitDisable) {
452 enable();
453 }
454 }
455
456 return false;
457 }
458
459 void
460 IOInterruptEventSource::normalInterruptOccurred
461 (void */*refcon*/, IOService */*prov*/, int /*source*/)
462 {
463 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
464
465 IOStatisticsInterrupt();
466 producerCount++;
467
468 if (trace) {
469 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
470 }
471
472 if (reserved->statistics) {
473 if (reserved->statistics->enablePrimaryTimestamp) {
474 reserved->statistics->primaryTimestamp = mach_absolute_time();
475 }
476 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
477 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
478 }
479 }
480
481 signalWorkAvailable();
482
483 if (trace) {
484 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
485 }
486 }
487
488 void
489 IOInterruptEventSource::disableInterruptOccurred
490 (void */*refcon*/, IOService *prov, int source)
491 {
492 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
493
494 prov->disableInterrupt(source); /* disable the interrupt */
495
496 IOStatisticsInterrupt();
497 producerCount++;
498
499 if (trace) {
500 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
501 }
502
503 if (reserved->statistics) {
504 if (reserved->statistics->enablePrimaryTimestamp) {
505 reserved->statistics->primaryTimestamp = mach_absolute_time();
506 }
507 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
508 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
509 }
510 }
511
512 signalWorkAvailable();
513
514 if (trace) {
515 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
516 }
517 }
518
519 void
520 IOInterruptEventSource::interruptOccurred
521 (void *_refcon, IOService *prov, int source)
522 {
523 if (autoDisable && prov) {
524 disableInterruptOccurred(_refcon, prov, source);
525 } else {
526 normalInterruptOccurred(_refcon, prov, source);
527 }
528 }
529
530 IOReturn
531 IOInterruptEventSource::warmCPU
532 (uint64_t abstime)
533 {
534 return ml_interrupt_prewarm(abstime);
535 }
536
537 void
538 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable)
539 {
540 if (reserved->statistics) {
541 reserved->statistics->enablePrimaryTimestamp = enable;
542 }
543 }
544
545 uint64_t
546 IOInterruptEventSource::getPrimaryInterruptTimestamp()
547 {
548 if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) {
549 return reserved->statistics->primaryTimestamp;
550 }
551 return -1ULL;
552 }
mirror of XNU