iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <mach/sync_policy.h>
  54 #include <IOKit/assert.h>
  55 #include <sys/errno.h>
  56
  57 #include <machine/pal_routines.h>
  58
  59 #define LOG kprintf
  60 //#define LOG IOLog
  61 #define MATCH_DEBUG     0
  62 #define OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  63
  64 #include "IOServicePrivate.h"
  65 #include "IOKitKernelInternal.h"
  66
  67 // take lockForArbitration before LOCKNOTIFY
  68
  69 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  70
  71 #define super IORegistryEntry
  72
  73 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  74
  75 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  76
  77 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  78
  79 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  80
  81 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  82
  83 OSDefineMetaClassAndStructors(IOResources, IOService)
  84
  85 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  86
  87 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  88
  89 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  90
  91 static IOPlatformExpert *       gIOPlatform;
  92 static class IOPMrootDomain *   gIOPMRootDomain;
  93 const IORegistryPlane *         gIOServicePlane;
  94 const IORegistryPlane *         gIOPowerPlane;
  95 const OSSymbol *                gIODeviceMemoryKey;
  96 const OSSymbol *                gIOInterruptControllersKey;
  97 const OSSymbol *                gIOInterruptSpecifiersKey;
  98
  99 const OSSymbol *                gIOResourcesKey;
 100 const OSSymbol *                gIOResourceMatchKey;
 101 const OSSymbol *                gIOProviderClassKey;
 102 const OSSymbol *                gIONameMatchKey;
 103 const OSSymbol *                gIONameMatchedKey;
 104 const OSSymbol *                gIOPropertyMatchKey;
 105 const OSSymbol *                gIOLocationMatchKey;
 106 const OSSymbol *                gIOParentMatchKey;
 107 const OSSymbol *                gIOPathMatchKey;
 108 const OSSymbol *                gIOMatchCategoryKey;
 109 const OSSymbol *                gIODefaultMatchCategoryKey;
 110 const OSSymbol *                gIOMatchedServiceCountKey;
 111
 112 const OSSymbol *                gIOMapperIDKey;
 113 const OSSymbol *                gIOUserClientClassKey;
 114 const OSSymbol *                gIOKitDebugKey;
 115
 116 const OSSymbol *                gIOCommandPoolSizeKey;
 117
 118 const OSSymbol *                gIOConsoleLockedKey;
 119 const OSSymbol *                gIOConsoleUsersKey;
 120 const OSSymbol *                gIOConsoleSessionUIDKey;
 121 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 122 const OSSymbol *                gIOConsoleUsersSeedKey;
 123 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 124 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 125 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 126 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 127
 128 clock_sec_t                     gIOConsoleLockTime;
 129 static bool                     gIOConsoleLoggedIn;
 130 #if HIBERNATION
 131 static uint32_t                 gIOScreenLockState;
 132 #endif
 133 static IORegistryEntry *        gIOChosenEntry;
 134
 135 static int                      gIOResourceGenerationCount;
 136
 137 const OSSymbol *                gIOServiceKey;
 138 const OSSymbol *                gIOPublishNotification;
 139 const OSSymbol *                gIOFirstPublishNotification;
 140 const OSSymbol *                gIOMatchedNotification;
 141 const OSSymbol *                gIOFirstMatchNotification;
 142 const OSSymbol *                gIOTerminatedNotification;
 143
 144 const OSSymbol *                gIOGeneralInterest;
 145 const OSSymbol *                gIOBusyInterest;
 146 const OSSymbol *                gIOAppPowerStateInterest;
 147 const OSSymbol *                gIOPriorityPowerStateInterest;
 148 const OSSymbol *                gIOConsoleSecurityInterest;
 149
 150 static OSDictionary *           gNotifications;
 151 static IORecursiveLock *        gNotificationLock;
 152
 153 static IOService *              gIOResources;
 154 static IOService *              gIOServiceRoot;
 155
 156 static OSOrderedSet *           gJobs;
 157 static semaphore_port_t         gJobsSemaphore;
 158 static IOLock *                 gJobsLock;
 159 static int                      gOutstandingJobs;
 160 static int                      gNumConfigThreads;
 161 static int                      gNumWaitingThreads;
 162 static IOLock *                 gIOServiceBusyLock;
 163
 164 static thread_t                 gIOTerminateThread;
 165 static UInt32                   gIOTerminateWork;
 166 static OSArray *                gIOTerminatePhase2List;
 167 static OSArray *                gIOStopList;
 168 static OSArray *                gIOStopProviderList;
 169 static OSArray *                gIOFinalizeList;
 170
 171 static SInt32                   gIOConsoleUsersSeed;
 172 static OSData *                 gIOConsoleUsersSeedValue;
 173
 174 extern const OSSymbol *         gIODTPHandleKey;
 175
 176 const OSSymbol *                gIOPlatformSleepActionKey;
 177 const OSSymbol *                gIOPlatformWakeActionKey;
 178 const OSSymbol *                gIOPlatformQuiesceActionKey;
 179 const OSSymbol *                gIOPlatformActiveActionKey;
 180 const OSSymbol *                gIOPlatformHaltRestartActionKey;
 181
 182 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 183
 184 static IOLock *                 gIOConsoleUsersLock;
 185 static thread_call_t            gIOConsoleLockCallout;
 186
 187 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 188
 189 #define LOCKREADNOTIFY()        \
 190     IORecursiveLockLock( gNotificationLock )
 191 #define LOCKWRITENOTIFY()       \
 192     IORecursiveLockLock( gNotificationLock )
 193 #define LOCKWRITE2READNOTIFY()
 194 #define UNLOCKNOTIFY()          \
 195     IORecursiveLockUnlock( gNotificationLock )
 196 #define SLEEPNOTIFY(event) \
 197     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 198 #define SLEEPNOTIFYTO(event, deadline) \
 199     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 200 #define WAKEUPNOTIFY(event) \
 201         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 202
 203 #define randomDelay()   \
 204         int del = read_processor_clock();                               \
 205         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 206         IOSleep( del );
 207
 208 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 209
 210 #define queue_element(entry, element, type, field) do { \
 211         vm_address_t __ele = (vm_address_t) (entry);    \
 212         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 213         (element) = (type) __ele;                       \
 214     } while(0)
 215
 216 #define iterqueue(que, elt)                             \
 217         for (queue_entry_t elt = queue_first(que);      \
 218              !queue_end(que, elt);                      \
 219              elt = queue_next(elt))
 220
 221 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 222
 223 struct IOInterruptAccountingReporter {
 224     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 225     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 226 };
 227
 228 struct ArbitrationLockQueueElement {
 229     queue_chain_t link;
 230     IOThread      thread;
 231     IOService *   service;
 232     unsigned      count;
 233     bool          required;
 234     bool          aborted;
 235 };
 236
 237 static queue_head_t gArbitrationLockQueueActive;
 238 static queue_head_t gArbitrationLockQueueWaiting;
 239 static queue_head_t gArbitrationLockQueueFree;
 240 static IOLock *     gArbitrationLockQueueLock;
 241
 242 bool IOService::isInactive( void ) const
 243     { return( 0 != (kIOServiceInactiveState & getState())); }
 244
 245 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 246
 247 #if defined(__i386__) || defined(__x86_64__)
 248
 249 // Only used by the intel implementation of
 250 //     IOService::requireMaxBusStall(UInt32 ns)
 251 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 252 struct CpuDelayEntry
 253 {
 254     IOService * fService;
 255     UInt32      fMaxDelay;
 256     UInt32      fDelayType;
 257 };
 258
 259 enum {
 260     kCpuDelayBusStall, kCpuDelayInterrupt,
 261     kCpuNumDelayTypes
 262 };
 263
 264 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 265 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 266 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 267 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 268 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 269 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 270
 271 static void
 272 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 273 static IOReturn
 274 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 275
 276 #endif /* defined(__i386__) || defined(__x86_64__) */
 277
 278 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 279
 280 void IOService::initialize( void )
 281 {
 282     kern_return_t       err;
 283
 284     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 285     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 286
 287     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 288     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 289     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 290     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 291     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 292     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 293     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 294
 295     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 296     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 297                                         kIODefaultMatchCategoryKey );
 298     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 299                                         kIOMatchedServiceCountKey );
 300
 301     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 302
 303     gIOResourcesKey     = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 304     gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 305
 306     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 307     gIOInterruptControllersKey
 308         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 309     gIOInterruptSpecifiersKey
 310         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 311
 312     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 313
 314     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 315
 316     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 317
 318     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 319     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 320     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 321     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 322     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 323
 324     gNotifications              = OSDictionary::withCapacity( 1 );
 325     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 326                                                  kIOPublishNotification );
 327     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 328                                                  kIOFirstPublishNotification );
 329     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 330                                                  kIOMatchedNotification );
 331     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 332                                                  kIOFirstMatchNotification );
 333     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 334                                                  kIOTerminatedNotification );
 335     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 336
 337     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 338     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 339     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 340     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 341
 342     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 343     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 344     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 345     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 346     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 347
 348     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 349
 350     gIOPlatformSleepActionKey       = OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey);
 351     gIOPlatformWakeActionKey        = OSSymbol::withCStringNoCopy(kIOPlatformWakeActionKey);
 352     gIOPlatformQuiesceActionKey     = OSSymbol::withCStringNoCopy(kIOPlatformQuiesceActionKey);
 353     gIOPlatformActiveActionKey      = OSSymbol::withCStringNoCopy(kIOPlatformActiveActionKey);
 354     gIOPlatformHaltRestartActionKey = OSSymbol::withCStringNoCopy(kIOPlatformHaltRestartActionKey);
 355
 356     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 357 #if defined(__i386__) || defined(__x86_64__)
 358     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 359     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 360     uint32_t  idx;
 361     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 362     {
 363         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 364         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 365         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 366     }
 367 #endif
 368     gNotificationLock           = IORecursiveLockAlloc();
 369
 370     assert( gIOServicePlane && gIODeviceMemoryKey
 371         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 372         && gIOResourcesKey && gNotifications && gNotificationLock
 373         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 374         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 375         && gIOPublishNotification && gIOMatchedNotification
 376         && gIOTerminatedNotification && gIOServiceKey
 377         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 378     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 379         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 380
 381     gJobsLock   = IOLockAlloc();
 382     gJobs       = OSOrderedSet::withCapacity( 10 );
 383
 384     gIOServiceBusyLock = IOLockAlloc();
 385
 386     gIOConsoleUsersLock = IOLockAlloc();
 387
 388     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 389
 390     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 391
 392     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 393
 394     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 395                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 396
 397     gIOResources = IOResources::resources();
 398     assert( gIOResources );
 399
 400     gArbitrationLockQueueLock = IOLockAlloc();
 401     queue_init(&gArbitrationLockQueueActive);
 402     queue_init(&gArbitrationLockQueueWaiting);
 403     queue_init(&gArbitrationLockQueueFree);
 404
 405     assert( gArbitrationLockQueueLock );
 406
 407     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 408     gIOStopList            = OSArray::withCapacity( 16 );
 409     gIOStopProviderList    = OSArray::withCapacity( 16 );
 410     gIOFinalizeList        = OSArray::withCapacity( 16 );
 411     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 412 }
 413
 414 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 415
 416 #if IOMATCHDEBUG
 417 static UInt64 getDebugFlags( OSDictionary * props )
 418 {
 419     OSNumber *  debugProp;
 420     UInt64      debugFlags;
 421
 422     debugProp = OSDynamicCast( OSNumber,
 423                 props->getObject( gIOKitDebugKey ));
 424     if( debugProp)
 425         debugFlags = debugProp->unsigned64BitValue();
 426     else
 427         debugFlags = gIOKitDebug;
 428
 429     return( debugFlags );
 430 }
 431 #endif
 432
 433 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 434
 435 // Probe a matched service and return an instance to be started.
 436 // The default score is from the property table, & may be altered
 437 // during probe to change the start order.
 438
 439 IOService * IOService::probe(   IOService * provider,
 440                                 SInt32    * score )
 441 {
 442     return( this );
 443 }
 444
 445 bool IOService::start( IOService * provider )
 446 {
 447     return( true );
 448 }
 449
 450 void IOService::stop( IOService * provider )
 451 {
 452 }
 453
 454 bool IOService::init( OSDictionary * dictionary )
 455 {
 456     bool ret = false;
 457
 458     ret = super::init(dictionary);
 459
 460     if (!ret)
 461         goto done;
 462
 463     reserved = IONew(ExpansionData, 1);
 464
 465     if (!reserved) {
 466         ret = false;
 467         goto done;
 468     }
 469
 470     bzero(reserved, sizeof(*reserved));
 471
 472     /*
 473      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 474      * lock based on the presence of specifiers ran into issues as some
 475      * platforms set up the specifiers after IOService initialization.
 476      *
 477      * We may be able to get away with a global lock, as this should only be
 478      * contended by IOReporting clients and driver start/stop (unless a
 479      * driver wants to remove/add handlers in the course of normal operation,
 480      * which should be unlikely).
 481      */
 482     reserved->interruptStatisticsLock = IOLockAlloc();
 483
 484     if (!reserved->interruptStatisticsLock) {
 485         ret = false;
 486         goto done;
 487     }
 488
 489 done:
 490     return ret;
 491 }
 492
 493 bool IOService::init( IORegistryEntry * from,
 494                       const IORegistryPlane * inPlane )
 495 {
 496     bool ret = false;
 497
 498     ret = super::init(from, inPlane);
 499
 500     if (!ret)
 501         goto done;
 502
 503     reserved = IONew(ExpansionData, 1);
 504
 505     if (!reserved) {
 506         ret = false;
 507         goto done;
 508     }
 509
 510     bzero(reserved, sizeof(*reserved));
 511
 512     /*
 513      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 514      * lock based on the presence of specifiers ran into issues as some
 515      * platforms set up the specifiers after IOService initialization.
 516      *
 517      * We may be able to get away with a global lock, as this should only be
 518      * contended by IOReporting clients and driver start/stop (unless a
 519      * driver wants to remove/add handlers in the course of normal operation,
 520      * which should be unlikely).
 521      */
 522     reserved->interruptStatisticsLock = IOLockAlloc();
 523
 524     if (!reserved->interruptStatisticsLock) {
 525         ret = false;
 526         goto done;
 527     }
 528
 529 done:
 530     return ret;
 531 }
 532
 533 void IOService::free( void )
 534 {
 535     int i = 0;
 536     requireMaxBusStall(0);
 537     requireMaxInterruptDelay(0);
 538     if( getPropertyTable())
 539         unregisterAllInterest();
 540     PMfree();
 541
 542     if (reserved) {
 543         if (reserved->interruptStatisticsArray) {
 544             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 545                 if (reserved->interruptStatisticsArray[i].reporter)
 546                     reserved->interruptStatisticsArray[i].reporter->release();
 547             }
 548
 549             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 550         }
 551
 552         if (reserved->interruptStatisticsLock)
 553             IOLockFree(reserved->interruptStatisticsLock);
 554         IODelete(reserved, ExpansionData, 1);
 555     }
 556
 557     super::free();
 558 }
 559
 560 /*
 561  * Attach in service plane
 562  */
 563 bool IOService::attach( IOService * provider )
 564 {
 565     bool        ok;
 566
 567     if( provider) {
 568
 569         if( gIOKitDebug & kIOLogAttach)
 570             LOG( "%s::attach(%s)\n", getName(),
 571                     provider->getName());
 572
 573         provider->lockForArbitration();
 574         if( provider->__state[0] & kIOServiceInactiveState)
 575             ok = false;
 576         else
 577             ok = attachToParent( provider, gIOServicePlane);
 578         provider->unlockForArbitration();
 579
 580     } else {
 581         gIOServiceRoot = this;
 582         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 583     }
 584
 585     if (ok && !__provider) (void) getProvider();
 586
 587     return( ok );
 588 }
 589
 590 IOService * IOService::getServiceRoot( void )
 591 {
 592     return( gIOServiceRoot );
 593 }
 594
 595 void IOService::detach( IOService * provider )
 596 {
 597     IOService * newProvider = 0;
 598     SInt32      busy;
 599     bool        adjParent;
 600
 601     if( gIOKitDebug & kIOLogAttach)
 602         LOG("%s::detach(%s)\n", getName(), provider->getName());
 603
 604     lockForArbitration();
 605
 606     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 607                && (provider == getProvider()));
 608
 609     detachFromParent( provider, gIOServicePlane );
 610
 611     if( busy) {
 612         newProvider = getProvider();
 613         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 614             _adjustBusy( -busy );
 615     }
 616
 617     if (kIOServiceInactiveState & __state[0]) {
 618         getMetaClass()->removeInstance(this);
 619     }
 620
 621     unlockForArbitration();
 622
 623     if( newProvider) {
 624         newProvider->lockForArbitration();
 625         newProvider->_adjustBusy(1);
 626         newProvider->unlockForArbitration();
 627     }
 628
 629     // check for last client detach from a terminated service
 630     if( provider->lockForArbitration( true ))
 631     {
 632         if (kIOServiceStartState & __state[1])
 633         {
 634             provider->scheduleTerminatePhase2();
 635         }
 636         if( adjParent) provider->_adjustBusy( -1 );
 637         if( (provider->__state[1] & kIOServiceTermPhase3State)
 638          && (0 == provider->getClient())) {
 639             provider->scheduleFinalize();
 640         }
 641         provider->unlockForArbitration();
 642     }
 643 }
 644
 645 /*
 646  * Register instance - publish it for matching
 647  */
 648
 649 void IOService::registerService( IOOptionBits options )
 650 {
 651     char *              pathBuf;
 652     const char *        path;
 653     char *              skip;
 654     int                 len;
 655     enum { kMaxPathLen  = 256 };
 656     enum { kMaxChars    = 63 };
 657
 658     IORegistryEntry * parent = this;
 659     IORegistryEntry * root = getRegistryRoot();
 660     while( parent && (parent != root))
 661         parent = parent->getParentEntry( gIOServicePlane);
 662
 663     if( parent != root) {
 664         IOLog("%s: not registry member at registerService()\n", getName());
 665         return;
 666     }
 667
 668     // Allow the Platform Expert to adjust this node.
 669     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 670         return;
 671
 672     if( (this != gIOResources)
 673      && (kIOLogRegister & gIOKitDebug)) {
 674
 675         pathBuf = (char *) IOMalloc( kMaxPathLen );
 676
 677         IOLog( "Registering: " );
 678
 679         len = kMaxPathLen;
 680         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 681
 682             path = pathBuf;
 683             if( len > kMaxChars) {
 684                 IOLog("..");
 685                 len -= kMaxChars;
 686                 path += len;
 687                 if( (skip = strchr( path, '/')))
 688                     path = skip;
 689             }
 690         } else
 691             path = getName();
 692
 693         IOLog( "%s\n", path );
 694
 695         if( pathBuf)
 696             IOFree( pathBuf, kMaxPathLen );
 697     }
 698
 699     startMatching( options );
 700 }
 701
 702 void IOService::startMatching( IOOptionBits options )
 703 {
 704     IOService * provider;
 705     UInt32      prevBusy = 0;
 706     bool        needConfig;
 707     bool        needWake = false;
 708     bool        ok;
 709     bool        sync;
 710     bool        waitAgain;
 711
 712     lockForArbitration();
 713
 714     sync = (options & kIOServiceSynchronous)
 715         || ((provider = getProvider())
 716                 && (provider->__state[1] & kIOServiceSynchronousState));
 717
 718         if ( options & kIOServiceAsynchronous )
 719                 sync = false;
 720
 721     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState)))
 722                && (0 == (__state[0] & kIOServiceInactiveState));
 723
 724     __state[1] |= kIOServiceNeedConfigState;
 725
 726 //    __state[0] &= ~kIOServiceInactiveState;
 727
 728 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 729 //                      OSKernelStackRemaining(), getName());
 730
 731     if( needConfig) {
 732         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 733     }
 734
 735     if( sync)
 736         __state[1] |= kIOServiceSynchronousState;
 737     else
 738         __state[1] &= ~kIOServiceSynchronousState;
 739
 740     if( needConfig) prevBusy = _adjustBusy( 1 );
 741
 742     unlockForArbitration();
 743
 744     if( needConfig) {
 745
 746         if( needWake) {
 747             IOLockLock( gIOServiceBusyLock );
 748             thread_wakeup( (event_t) this/*&__state[1]*/ );
 749             IOLockUnlock( gIOServiceBusyLock );
 750
 751         } else if( !sync || (kIOServiceAsynchronous & options)) {
 752
 753             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 754
 755         } else do {
 756
 757             if( (__state[1] & kIOServiceNeedConfigState))
 758                 doServiceMatch( options );
 759
 760             lockForArbitration();
 761             IOLockLock( gIOServiceBusyLock );
 762
 763             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 764                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 765
 766             if( waitAgain)
 767                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 768             else
 769                 __state[1] &= ~kIOServiceSyncPubState;
 770
 771             unlockForArbitration();
 772
 773             if( waitAgain)
 774                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 775
 776             IOLockUnlock( gIOServiceBusyLock );
 777             if( waitAgain)
 778                 thread_block(THREAD_CONTINUE_NULL);
 779
 780         } while( waitAgain );
 781     }
 782 }
 783
 784 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 785 {
 786     OSDictionary *      table;
 787     OSSet *             set;
 788     OSSet *             allSet = 0;
 789     IOService *         service;
 790 #if IOMATCHDEBUG
 791     SInt32              count = 0;
 792 #endif
 793
 794     newTables->retain();
 795
 796     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 797
 798         LOCKWRITENOTIFY();
 799         set = (OSSet *) copyExistingServices( table,
 800                                                 kIOServiceRegisteredState,
 801                                                 kIOServiceExistingSet);
 802         UNLOCKNOTIFY();
 803         if( set) {
 804
 805 #if IOMATCHDEBUG
 806             count += set->getCount();
 807 #endif
 808             if (allSet) {
 809                 allSet->merge((const OSSet *) set);
 810                 set->release();
 811             }
 812             else
 813                 allSet = set;
 814         }
 815
 816 #if IOMATCHDEBUG
 817         if( getDebugFlags( table ) & kIOLogMatch)
 818             LOG("Matching service count = %ld\n", (long)count);
 819 #endif
 820         newTables->removeObject(table);
 821     }
 822
 823     if (allSet) {
 824         while( (service = (IOService *) allSet->getAnyObject())) {
 825             service->startMatching(kIOServiceAsynchronous);
 826             allSet->removeObject(service);
 827         }
 828         allSet->release();
 829     }
 830
 831     newTables->release();
 832
 833     return( kIOReturnSuccess );
 834 }
 835
 836  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 837                                                 IOOptionBits options )
 838 {
 839     _IOServiceJob *     job;
 840
 841     job = new _IOServiceJob;
 842     if( job && !job->init()) {
 843         job->release();
 844         job = 0;
 845     }
 846
 847     if( job) {
 848         job->type       = type;
 849         job->nub        = nub;
 850         job->options    = options;
 851         nub->retain();                  // thread will release()
 852         pingConfig( job );
 853     }
 854
 855     return( job );
 856 }
 857
 858 /*
 859  * Called on a registered service to see if it matches
 860  * a property table.
 861  */
 862
 863 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 864 {
 865     return( matchPropertyTable(table) );
 866 }
 867
 868 bool IOService::matchPropertyTable( OSDictionary * table )
 869 {
 870     return( true );
 871 }
 872
 873 /*
 874  * Called on a matched service to allocate resources
 875  * before first driver is attached.
 876  */
 877
 878 IOReturn IOService::getResources( void )
 879 {
 880     return( kIOReturnSuccess);
 881 }
 882
 883 /*
 884  * Client/provider accessors
 885  */
 886
 887 IOService * IOService::getProvider( void ) const
 888 {
 889     IOService * self = (IOService *) this;
 890     IOService * parent;
 891     SInt32      generation;
 892
 893     generation = getGenerationCount();
 894     if( __providerGeneration == generation)
 895         return( __provider );
 896
 897     parent = (IOService *) getParentEntry( gIOServicePlane);
 898     if( parent == IORegistryEntry::getRegistryRoot())
 899         /* root is not an IOService */
 900         parent = 0;
 901
 902     self->__provider = parent;
 903     OSMemoryBarrier();
 904     // save the count from before call to getParentEntry()
 905     self->__providerGeneration = generation;
 906
 907     return( parent );
 908 }
 909
 910 IOWorkLoop * IOService::getWorkLoop() const
 911 {
 912     IOService *provider = getProvider();
 913
 914     if (provider)
 915         return provider->getWorkLoop();
 916     else
 917         return 0;
 918 }
 919
 920 OSIterator * IOService::getProviderIterator( void ) const
 921 {
 922     return( getParentIterator( gIOServicePlane));
 923 }
 924
 925 IOService * IOService::getClient( void ) const
 926 {
 927     return( (IOService *) getChildEntry( gIOServicePlane));
 928 }
 929
 930 OSIterator * IOService::getClientIterator( void ) const
 931 {
 932     return( getChildIterator( gIOServicePlane));
 933 }
 934
 935 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
 936                                                 const IOService * client,
 937                                                 const IOService * provider )
 938 {
 939     _IOOpenServiceIterator * inst;
 940
 941     if( !_iter)
 942         return( 0 );
 943
 944     inst = new _IOOpenServiceIterator;
 945
 946     if( inst && !inst->init()) {
 947         inst->release();
 948         inst = 0;
 949     }
 950     if( inst) {
 951         inst->iter = _iter;
 952         inst->client = client;
 953         inst->provider = provider;
 954     }
 955
 956     return( inst );
 957 }
 958
 959 void _IOOpenServiceIterator::free()
 960 {
 961     iter->release();
 962     if( last)
 963         last->unlockForArbitration();
 964     OSIterator::free();
 965 }
 966
 967 OSObject * _IOOpenServiceIterator::getNextObject()
 968 {
 969     IOService * next;
 970
 971     if( last)
 972         last->unlockForArbitration();
 973
 974     while( (next = (IOService *) iter->getNextObject())) {
 975
 976         next->lockForArbitration();
 977         if( (client && (next->isOpen( client )))
 978          || (provider && (provider->isOpen( next ))) )
 979             break;
 980         next->unlockForArbitration();
 981     }
 982
 983     last = next;
 984
 985     return( next );
 986 }
 987
 988 bool _IOOpenServiceIterator::isValid()
 989 {
 990     return( iter->isValid() );
 991 }
 992
 993 void _IOOpenServiceIterator::reset()
 994 {
 995     if( last) {
 996         last->unlockForArbitration();
 997         last = 0;
 998     }
 999     iter->reset();
1000 }
1001
1002 OSIterator * IOService::getOpenProviderIterator( void ) const
1003 {
1004     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1005 }
1006
1007 OSIterator * IOService::getOpenClientIterator( void ) const
1008 {
1009     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1010 }
1011
1012
1013 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1014                                           bool waitForFunction,
1015                                           void *param1, void *param2,
1016                                           void *param3, void *param4 )
1017 {
1018   IOReturn  result = kIOReturnUnsupported;
1019   IOService *provider;
1020
1021   if (gIOPlatformFunctionHandlerSet == functionName)
1022   {
1023 #if defined(__i386__) || defined(__x86_64__)
1024     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1025     IOService *      target              = (IOService *) param2;
1026     bool             enable              = (param3 != 0);
1027
1028     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1029         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1030     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1031         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1032 #endif /* defined(__i386__) || defined(__x86_64__) */
1033   }
1034
1035   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1036     result = provider->callPlatformFunction(functionName, waitForFunction,
1037                                             param1, param2, param3, param4);
1038   }
1039
1040   return result;
1041 }
1042
1043 IOReturn IOService::callPlatformFunction( const char * functionName,
1044                                           bool waitForFunction,
1045                                           void *param1, void *param2,
1046                                           void *param3, void *param4 )
1047 {
1048   IOReturn result = kIOReturnNoMemory;
1049   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1050
1051   if (functionSymbol != 0) {
1052     result = callPlatformFunction(functionSymbol, waitForFunction,
1053                                   param1, param2, param3, param4);
1054     functionSymbol->release();
1055   }
1056
1057   return result;
1058 }
1059
1060
1061 /*
1062  * Accessors for global services
1063  */
1064
1065 IOPlatformExpert * IOService::getPlatform( void )
1066 {
1067     return( gIOPlatform);
1068 }
1069
1070 class IOPMrootDomain * IOService::getPMRootDomain( void )
1071 {
1072     return( gIOPMRootDomain);
1073 }
1074
1075 IOService * IOService::getResourceService( void )
1076 {
1077     return( gIOResources );
1078 }
1079
1080 void IOService::setPlatform( IOPlatformExpert * platform)
1081 {
1082     gIOPlatform = platform;
1083     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1084
1085 #if defined(__i386__) || defined(__x86_64__)
1086
1087     static const char * keys[kCpuNumDelayTypes] = {
1088         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1089     const OSObject * objs[2];
1090     OSArray * array;
1091     uint32_t  idx;
1092
1093     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1094     {
1095         objs[0] = sCPULatencySet[idx];
1096         objs[1] = sCPULatencyHolder[idx];
1097         array   = OSArray::withObjects(objs, 2);
1098         if (!array) break;
1099         platform->setProperty(keys[idx], array);
1100         array->release();
1101     }
1102 #endif /* defined(__i386__) || defined(__x86_64__) */
1103 }
1104
1105 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1106 {
1107     gIOPMRootDomain = rootDomain;
1108     publishResource("IOKit");
1109 }
1110
1111 /*
1112  * Stacking change
1113  */
1114
1115 bool IOService::lockForArbitration( bool isSuccessRequired )
1116 {
1117     bool                          found;
1118     bool                          success;
1119     ArbitrationLockQueueElement * element;
1120     ArbitrationLockQueueElement * active;
1121     ArbitrationLockQueueElement * waiting;
1122
1123     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1124
1125     // lock global access
1126     IOTakeLock( gArbitrationLockQueueLock );
1127
1128     // obtain an unused queue element
1129     if( !queue_empty( &gArbitrationLockQueueFree )) {
1130         queue_remove_first( &gArbitrationLockQueueFree,
1131                             element,
1132                             ArbitrationLockQueueElement *,
1133                             link );
1134     } else {
1135         element = IONew( ArbitrationLockQueueElement, 1 );
1136         assert( element );
1137     }
1138
1139     // prepare the queue element
1140     element->thread   = IOThreadSelf();
1141     element->service  = this;
1142     element->count    = 1;
1143     element->required = isSuccessRequired;
1144     element->aborted  = false;
1145
1146     // determine whether this object is already locked (ie. on active queue)
1147     found = false;
1148     queue_iterate( &gArbitrationLockQueueActive,
1149                     active,
1150                     ArbitrationLockQueueElement *,
1151                     link )
1152     {
1153         if( active->service == element->service ) {
1154             found = true;
1155             break;
1156         }
1157     }
1158
1159     if( found ) { // this object is already locked
1160
1161         // determine whether it is the same or a different thread trying to lock
1162         if( active->thread != element->thread ) { // it is a different thread
1163
1164             ArbitrationLockQueueElement * victim = 0;
1165
1166             // before placing this new thread on the waiting queue, we look for
1167             // a deadlock cycle...
1168
1169             while( 1 ) {
1170                 // determine whether the active thread holding the object we
1171                 // want is waiting for another object to be unlocked
1172                 found = false;
1173                 queue_iterate( &gArbitrationLockQueueWaiting,
1174                                waiting,
1175                                ArbitrationLockQueueElement *,
1176                                link )
1177                 {
1178                     if( waiting->thread == active->thread ) {
1179                         assert( false == waiting->aborted );
1180                         found = true;
1181                         break;
1182                     }
1183                 }
1184
1185                 if( found ) { // yes, active thread waiting for another object
1186
1187                     // this may be a candidate for rejection if the required
1188                     // flag is not set, should we detect a deadlock later on
1189                     if( false == waiting->required )
1190                         victim = waiting;
1191
1192                     // find the thread that is holding this other object, that
1193                     // is blocking the active thread from proceeding (fun :-)
1194                     found = false;
1195                     queue_iterate( &gArbitrationLockQueueActive,
1196                                    active,      // (reuse active queue element)
1197                                    ArbitrationLockQueueElement *,
1198                                    link )
1199                     {
1200                         if( active->service == waiting->service ) {
1201                             found = true;
1202                             break;
1203                         }
1204                     }
1205
1206                     // someone must be holding it or it wouldn't be waiting
1207                     assert( found );
1208
1209                     if( active->thread == element->thread ) {
1210
1211                         // doh, it's waiting for the thread that originated
1212                         // this whole lock (ie. current thread) -> deadlock
1213                         if( false == element->required ) { // willing to fail?
1214
1215                             // the originating thread doesn't have the required
1216                             // flag, so it can fail
1217                             success = false; // (fail originating lock request)
1218                             break; // (out of while)
1219
1220                         } else { // originating thread is not willing to fail
1221
1222                             // see if we came across a waiting thread that did
1223                             // not have the 'required' flag set: we'll fail it
1224                             if( victim ) {
1225
1226                                 // we do have a willing victim, fail it's lock
1227                                 victim->aborted = true;
1228
1229                                 // take the victim off the waiting queue
1230                                 queue_remove( &gArbitrationLockQueueWaiting,
1231                                               victim,
1232                                               ArbitrationLockQueueElement *,
1233                                               link );
1234
1235                                 // wake the victim
1236                                 IOLockWakeup( gArbitrationLockQueueLock,
1237                                               victim,
1238                                               /* one thread */ true );
1239
1240                                 // allow this thread to proceed (ie. wait)
1241                                 success = true; // (put request on wait queue)
1242                                 break; // (out of while)
1243                             } else {
1244
1245                                 // all the waiting threads we came across in
1246                                 // finding this loop had the 'required' flag
1247                                 // set, so we've got a deadlock we can't avoid
1248                                 panic("I/O Kit: Unrecoverable deadlock.");
1249                             }
1250                         }
1251                     } else {
1252                         // repeat while loop, redefining active thread to be the
1253                         // thread holding "this other object" (see above), and
1254                         // looking for threads waiting on it; note the active
1255                         // variable points to "this other object" already... so
1256                         // there nothing to do in this else clause.
1257                     }
1258                 } else { // no, active thread is not waiting for another object
1259
1260                     success = true; // (put request on wait queue)
1261                     break; // (out of while)
1262                 }
1263             } // while forever
1264
1265             if( success ) { // put the request on the waiting queue?
1266                 kern_return_t wait_result;
1267
1268                 // place this thread on the waiting queue and put it to sleep;
1269                 // we place it at the tail of the queue...
1270                 queue_enter( &gArbitrationLockQueueWaiting,
1271                              element,
1272                              ArbitrationLockQueueElement *,
1273                              link );
1274
1275                 // declare that this thread will wait for a given event
1276 restart_sleep:  wait_result = assert_wait( element,
1277                                            element->required ? THREAD_UNINT
1278                                            : THREAD_INTERRUPTIBLE );
1279
1280                 // unlock global access
1281                 IOUnlock( gArbitrationLockQueueLock );
1282
1283                 // put thread to sleep, waiting for our event to fire...
1284                 if (wait_result == THREAD_WAITING)
1285                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1286
1287
1288                 // ...and we've been woken up; we might be in one of two states:
1289                 // (a) we've been aborted and our queue element is not on
1290                 //     any of the three queues, but is floating around
1291                 // (b) we're allowed to proceed with the lock and we have
1292                 //     already been moved from the waiting queue to the
1293                 //     active queue.
1294                 // ...plus a 3rd state, should the thread have been interrupted:
1295                 // (c) we're still on the waiting queue
1296
1297                 // determine whether we were interrupted out of our sleep
1298                 if( THREAD_INTERRUPTED == wait_result ) {
1299
1300                     // re-lock global access
1301                     IOTakeLock( gArbitrationLockQueueLock );
1302
1303                     // determine whether we're still on the waiting queue
1304                     found = false;
1305                     queue_iterate( &gArbitrationLockQueueWaiting,
1306                                    waiting,     // (reuse waiting queue element)
1307                                    ArbitrationLockQueueElement *,
1308                                    link )
1309                     {
1310                         if( waiting == element ) {
1311                             found = true;
1312                             break;
1313                         }
1314                     }
1315
1316                     if( found ) { // yes, we're still on the waiting queue
1317
1318                         // determine whether we're willing to fail
1319                         if( false == element->required ) {
1320
1321                             // mark us as aborted
1322                             element->aborted = true;
1323
1324                             // take us off the waiting queue
1325                             queue_remove( &gArbitrationLockQueueWaiting,
1326                                           element,
1327                                           ArbitrationLockQueueElement *,
1328                                           link );
1329                         } else { // we are not willing to fail
1330
1331                             // ignore interruption, go back to sleep
1332                             goto restart_sleep;
1333                         }
1334                     }
1335
1336                     // unlock global access
1337                     IOUnlock( gArbitrationLockQueueLock );
1338
1339                     // proceed as though this were a normal wake up
1340                     wait_result = THREAD_AWAKENED;
1341                 }
1342
1343                 assert( THREAD_AWAKENED == wait_result );
1344
1345                 // determine whether we've been aborted while we were asleep
1346                 if( element->aborted ) {
1347                     assert( false == element->required );
1348
1349                     // re-lock global access
1350                     IOTakeLock( gArbitrationLockQueueLock );
1351
1352                     action = kPutOnFreeQueue;
1353                     success = false;
1354                 } else { // we weren't aborted, so we must be ready to go :-)
1355
1356                     // we've already been moved from waiting to active queue
1357                     return true;
1358                 }
1359
1360             } else { // the lock request is to be failed
1361
1362                 // return unused queue element to queue
1363                 action = kPutOnFreeQueue;
1364             }
1365         } else { // it is the same thread, recursive access is allowed
1366
1367             // add one level of recursion
1368             active->count++;
1369
1370             // return unused queue element to queue
1371             action = kPutOnFreeQueue;
1372             success = true;
1373         }
1374     } else { // this object is not already locked, so let this thread through
1375         action = kPutOnActiveQueue;
1376         success = true;
1377     }
1378
1379     // put the new element on a queue
1380     if( kPutOnActiveQueue == action ) {
1381         queue_enter( &gArbitrationLockQueueActive,
1382                      element,
1383                      ArbitrationLockQueueElement *,
1384                      link );
1385     } else if( kPutOnFreeQueue == action ) {
1386         queue_enter( &gArbitrationLockQueueFree,
1387                      element,
1388                      ArbitrationLockQueueElement *,
1389                      link );
1390     } else {
1391         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1392     }
1393
1394     // unlock global access
1395     IOUnlock( gArbitrationLockQueueLock );
1396
1397     return( success );
1398 }
1399
1400 void IOService::unlockForArbitration( void )
1401 {
1402     bool                          found;
1403     ArbitrationLockQueueElement * element;
1404
1405     // lock global access
1406     IOTakeLock( gArbitrationLockQueueLock );
1407
1408     // find the lock element for this object (ie. on active queue)
1409     found = false;
1410     queue_iterate( &gArbitrationLockQueueActive,
1411                     element,
1412                     ArbitrationLockQueueElement *,
1413                     link )
1414     {
1415         if( element->service == this ) {
1416             found = true;
1417             break;
1418         }
1419     }
1420
1421     assert( found );
1422
1423     // determine whether the lock has been taken recursively
1424     if( element->count > 1 ) {
1425         // undo one level of recursion
1426         element->count--;
1427
1428     } else {
1429
1430         // remove it from the active queue
1431         queue_remove( &gArbitrationLockQueueActive,
1432                       element,
1433                       ArbitrationLockQueueElement *,
1434                       link );
1435
1436         // put it on the free queue
1437         queue_enter( &gArbitrationLockQueueFree,
1438                      element,
1439                      ArbitrationLockQueueElement *,
1440                      link );
1441
1442         // determine whether a thread is waiting for object (head to tail scan)
1443         found = false;
1444         queue_iterate( &gArbitrationLockQueueWaiting,
1445                        element,
1446                        ArbitrationLockQueueElement *,
1447                        link )
1448         {
1449             if( element->service == this ) {
1450                 found = true;
1451                 break;
1452             }
1453         }
1454
1455         if ( found ) { // we found an interested thread on waiting queue
1456
1457             // remove it from the waiting queue
1458             queue_remove( &gArbitrationLockQueueWaiting,
1459                           element,
1460                           ArbitrationLockQueueElement *,
1461                           link );
1462
1463             // put it on the active queue
1464             queue_enter( &gArbitrationLockQueueActive,
1465                          element,
1466                          ArbitrationLockQueueElement *,
1467                          link );
1468
1469             // wake the waiting thread
1470             IOLockWakeup( gArbitrationLockQueueLock,
1471                           element,
1472                           /* one thread */ true );
1473         }
1474     }
1475
1476     // unlock global access
1477     IOUnlock( gArbitrationLockQueueLock );
1478 }
1479
1480 void IOService::applyToProviders( IOServiceApplierFunction applier,
1481                                   void * context )
1482 {
1483     applyToParents( (IORegistryEntryApplierFunction) applier,
1484                     context, gIOServicePlane );
1485 }
1486
1487 void IOService::applyToClients( IOServiceApplierFunction applier,
1488                                 void * context )
1489 {
1490     applyToChildren( (IORegistryEntryApplierFunction) applier,
1491                      context, gIOServicePlane );
1492 }
1493
1494
1495 /*
1496  * Client messages
1497  */
1498
1499
1500 // send a message to a client or interested party of this service
1501 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1502                                    void * argument, vm_size_t argSize )
1503 {
1504     IOReturn                            ret;
1505     IOService *                         service;
1506     _IOServiceInterestNotifier *        notify;
1507
1508     if( (service = OSDynamicCast( IOService, client)))
1509         ret = service->message( type, this, argument );
1510
1511     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1512
1513         _IOServiceNotifierInvocation invocation;
1514         bool                     willNotify;
1515
1516         invocation.thread = current_thread();
1517
1518         LOCKWRITENOTIFY();
1519         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1520
1521         if( willNotify) {
1522             queue_enter( &notify->handlerInvocations, &invocation,
1523                             _IOServiceNotifierInvocation *, link );
1524         }
1525         UNLOCKNOTIFY();
1526
1527         if( willNotify) {
1528
1529             ret = (*notify->handler)( notify->target, notify->ref,
1530                                           type, this, argument, argSize );
1531
1532             LOCKWRITENOTIFY();
1533             queue_remove( &notify->handlerInvocations, &invocation,
1534                             _IOServiceNotifierInvocation *, link );
1535             if( kIOServiceNotifyWaiter & notify->state) {
1536                 notify->state &= ~kIOServiceNotifyWaiter;
1537                 WAKEUPNOTIFY( notify );
1538             }
1539             UNLOCKNOTIFY();
1540
1541         } else
1542             ret = kIOReturnSuccess;
1543
1544     } else
1545         ret = kIOReturnBadArgument;
1546
1547     return( ret );
1548 }
1549
1550 static void
1551 applyToInterestNotifiers(const IORegistryEntry *target,
1552                          const OSSymbol * typeOfInterest,
1553                          OSObjectApplierFunction applier,
1554                          void * context )
1555 {
1556     OSArray *           copyArray = 0;
1557
1558     LOCKREADNOTIFY();
1559
1560     IOCommand *notifyList =
1561         OSDynamicCast( IOCommand, target->getProperty( typeOfInterest ));
1562
1563     if( notifyList) {
1564         copyArray = OSArray::withCapacity(1);
1565
1566         // iterate over queue, entry is set to each element in the list
1567         iterqueue(&notifyList->fCommandChain, entry) {
1568             _IOServiceInterestNotifier * notify;
1569
1570             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1571             copyArray->setObject(notify);
1572         }
1573     }
1574     UNLOCKNOTIFY();
1575
1576     if( copyArray) {
1577         unsigned int    index;
1578         OSObject *      next;
1579
1580         for( index = 0; (next = copyArray->getObject( index )); index++)
1581             (*applier)(next, context);
1582         copyArray->release();
1583     }
1584 }
1585
1586 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1587                                    OSObjectApplierFunction applier,
1588                                    void * context )
1589 {
1590     if (gIOGeneralInterest == typeOfInterest)
1591         applyToClients( (IOServiceApplierFunction) applier, context );
1592     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1593 }
1594
1595 struct MessageClientsContext {
1596     IOService * service;
1597     UInt32      type;
1598     void *      argument;
1599     vm_size_t   argSize;
1600     IOReturn    ret;
1601 };
1602
1603 static void messageClientsApplier( OSObject * object, void * ctx )
1604 {
1605     IOReturn                ret;
1606     MessageClientsContext * context = (MessageClientsContext *) ctx;
1607
1608     ret = context->service->messageClient( context->type,
1609                                            object, context->argument, context->argSize );
1610     if( kIOReturnSuccess != ret)
1611         context->ret = ret;
1612 }
1613
1614 // send a message to all clients
1615 IOReturn IOService::messageClients( UInt32 type,
1616                                     void * argument, vm_size_t argSize )
1617 {
1618     MessageClientsContext       context;
1619
1620     context.service     = this;
1621     context.type        = type;
1622     context.argument    = argument;
1623     context.argSize     = argSize;
1624     context.ret         = kIOReturnSuccess;
1625
1626     applyToInterested( gIOGeneralInterest,
1627                        &messageClientsApplier, &context );
1628
1629     return( context.ret );
1630 }
1631
1632 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1633                                               IOOptionBits response )
1634 {
1635     return( kIOReturnUnsupported );
1636 }
1637
1638 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1639                   IOServiceInterestHandler handler, void * target, void * ref )
1640 {
1641     _IOServiceInterestNotifier * notify = 0;
1642     IOReturn rc = kIOReturnError;
1643
1644     notify = new _IOServiceInterestNotifier;
1645     if (!notify) return NULL;
1646
1647     if(notify->init()) {
1648         rc = registerInterestForNotifer(notify, typeOfInterest,
1649                               handler, target, ref);
1650     }
1651
1652     if (rc != kIOReturnSuccess) {
1653         notify->release();
1654         notify = 0;
1655     }
1656
1657     return( notify );
1658 }
1659
1660 IOReturn IOService::registerInterestForNotifer( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1661                   IOServiceInterestHandler handler, void * target, void * ref )
1662 {
1663     IOReturn rc = kIOReturnSuccess;
1664     _IOServiceInterestNotifier  *notify = 0;
1665
1666     if( (typeOfInterest != gIOGeneralInterest)
1667      && (typeOfInterest != gIOBusyInterest)
1668      && (typeOfInterest != gIOAppPowerStateInterest)
1669      && (typeOfInterest != gIOConsoleSecurityInterest)
1670      && (typeOfInterest != gIOPriorityPowerStateInterest))
1671         return( kIOReturnBadArgument );
1672
1673     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1674         return( kIOReturnBadArgument );
1675
1676     lockForArbitration();
1677     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1678
1679         notify->handler = handler;
1680         notify->target = target;
1681         notify->ref = ref;
1682         notify->state = kIOServiceNotifyEnable;
1683
1684         ////// queue
1685
1686         LOCKWRITENOTIFY();
1687
1688         // Get the head of the notifier linked list
1689         IOCommand *notifyList = (IOCommand *) getProperty( typeOfInterest );
1690         if (!notifyList || !OSDynamicCast(IOCommand, notifyList)) {
1691             notifyList = OSTypeAlloc(IOCommand);
1692             if (notifyList) {
1693                 notifyList->init();
1694                 setProperty( typeOfInterest, notifyList);
1695                 notifyList->release();
1696             }
1697         }
1698
1699         if (notifyList) {
1700             enqueue(&notifyList->fCommandChain, &notify->chain);
1701             notify->retain();   // ref'ed while in list
1702         }
1703
1704         UNLOCKNOTIFY();
1705     }
1706     else {
1707         rc = kIOReturnNotReady;
1708     }
1709     unlockForArbitration();
1710
1711     return rc;
1712 }
1713
1714 static void cleanInterestList( OSObject * head )
1715 {
1716     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1717     if (!notifyHead)
1718         return;
1719
1720     LOCKWRITENOTIFY();
1721     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1722         queue_next(entry) = queue_prev(entry) = 0;
1723
1724         _IOServiceInterestNotifier * notify;
1725
1726         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1727         notify->release();
1728     }
1729     UNLOCKNOTIFY();
1730 }
1731
1732 void IOService::unregisterAllInterest( void )
1733 {
1734     cleanInterestList( getProperty( gIOGeneralInterest ));
1735     cleanInterestList( getProperty( gIOBusyInterest ));
1736     cleanInterestList( getProperty( gIOAppPowerStateInterest ));
1737     cleanInterestList( getProperty( gIOPriorityPowerStateInterest ));
1738     cleanInterestList( getProperty( gIOConsoleSecurityInterest ));
1739 }
1740
1741 /*
1742  * _IOServiceInterestNotifier
1743  */
1744
1745 // wait for all threads, other than the current one,
1746 //  to exit the handler
1747
1748 void _IOServiceInterestNotifier::wait()
1749 {
1750     _IOServiceNotifierInvocation * next;
1751     bool doWait;
1752
1753     do {
1754         doWait = false;
1755         queue_iterate( &handlerInvocations, next,
1756                         _IOServiceNotifierInvocation *, link) {
1757             if( next->thread != current_thread() ) {
1758                 doWait = true;
1759                 break;
1760             }
1761         }
1762         if( doWait) {
1763             state |= kIOServiceNotifyWaiter;
1764                SLEEPNOTIFY(this);
1765         }
1766
1767     } while( doWait );
1768 }
1769
1770 void _IOServiceInterestNotifier::free()
1771 {
1772     assert( queue_empty( &handlerInvocations ));
1773     OSObject::free();
1774 }
1775
1776 void _IOServiceInterestNotifier::remove()
1777 {
1778     LOCKWRITENOTIFY();
1779
1780     if( queue_next( &chain )) {
1781         remqueue(&chain);
1782         queue_next( &chain) = queue_prev( &chain) = 0;
1783         release();
1784     }
1785
1786     state &= ~kIOServiceNotifyEnable;
1787
1788     wait();
1789
1790     UNLOCKNOTIFY();
1791
1792     release();
1793 }
1794
1795 bool _IOServiceInterestNotifier::disable()
1796 {
1797     bool        ret;
1798
1799     LOCKWRITENOTIFY();
1800
1801     ret = (0 != (kIOServiceNotifyEnable & state));
1802     state &= ~kIOServiceNotifyEnable;
1803     if( ret)
1804         wait();
1805
1806     UNLOCKNOTIFY();
1807
1808     return( ret );
1809 }
1810
1811 void _IOServiceInterestNotifier::enable( bool was )
1812 {
1813     LOCKWRITENOTIFY();
1814     if( was)
1815         state |= kIOServiceNotifyEnable;
1816     else
1817         state &= ~kIOServiceNotifyEnable;
1818     UNLOCKNOTIFY();
1819 }
1820
1821 bool _IOServiceInterestNotifier::init()
1822 {
1823     queue_init( &handlerInvocations );
1824     return (OSObject::init());
1825 }
1826 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1827
1828 /*
1829  * Termination
1830  */
1831
1832 #define tailQ(o)                setObject(o)
1833 #define headQ(o)                setObject(0, o)
1834 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
1835
1836 static void _workLoopAction( IOWorkLoop::Action action,
1837                              IOService * service,
1838                              void * p0 = 0, void * p1 = 0,
1839                              void * p2 = 0, void * p3 = 0 )
1840 {
1841     IOWorkLoop * wl;
1842
1843     if( (wl = service->getWorkLoop())) {
1844         wl->retain();
1845         wl->runAction( action, service, p0, p1, p2, p3 );
1846         wl->release();
1847     } else
1848         (*action)( service, p0, p1, p2, p3 );
1849 }
1850
1851 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
1852 {
1853     bool ok;
1854
1855     // if its our only provider
1856     ok = isParent( provider, gIOServicePlane, true);
1857
1858     // -- compat
1859     if( ok) {
1860         provider->terminateClient( this, options | kIOServiceRecursing );
1861         ok = (0 != (__state[1] & kIOServiceRecursing));
1862     }
1863     // --
1864
1865     return( ok );
1866 }
1867
1868 bool IOService::terminatePhase1( IOOptionBits options )
1869 {
1870     IOService *  victim;
1871     IOService *  client;
1872     OSIterator * iter;
1873     OSArray *    makeInactive;
1874         int          waitResult = THREAD_AWAKENED;
1875         bool         wait;
1876     bool                 ok;
1877     bool                 didInactive;
1878     bool                 startPhase2 = false;
1879
1880     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
1881
1882     uint64_t regID = getRegistryEntryID();
1883     IOServiceTrace(
1884         IOSERVICE_TERMINATE_PHASE1,
1885         (uintptr_t) regID,
1886         (uintptr_t) (regID >> 32),
1887         (uintptr_t) this,
1888         (uintptr_t) options);
1889
1890     // -- compat
1891     if( options & kIOServiceRecursing) {
1892         lockForArbitration();
1893         __state[0] |= kIOServiceInactiveState;
1894         __state[1] |= kIOServiceRecursing;
1895         unlockForArbitration();
1896
1897         return( true );
1898     }
1899     // --
1900
1901     makeInactive = OSArray::withCapacity( 16 );
1902     if( !makeInactive)
1903         return( false );
1904
1905     victim = this;
1906     victim->retain();
1907
1908     while( victim ) {
1909
1910         didInactive = victim->lockForArbitration( true );
1911         if( didInactive) {
1912             didInactive = (0 == (victim->__state[0] & kIOServiceInactiveState))
1913                          || (victim->__state[1] & kIOServiceRecursing);
1914             if( didInactive) {
1915                 victim->__state[0] |= kIOServiceInactiveState;
1916                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
1917                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
1918                 victim->__state[1] &= ~kIOServiceRecursing;
1919
1920                 if (victim == this)
1921                 {
1922                     victim->__state[1] |= kIOServiceTermPhase1State;
1923                     if (kIOServiceTerminateNeedWillTerminate & options)
1924                     {
1925                         victim->__state[1] |= kIOServiceNeedWillTerminate;
1926                     }
1927                 }
1928
1929                 victim->_adjustBusy( 1 );
1930
1931             } else if (victim != this) do {
1932
1933                 IOLockLock(gIOServiceBusyLock);
1934                 wait = (victim->__state[1] & kIOServiceTermPhase1State);
1935                 if( wait) {
1936                     TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
1937                         getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
1938                         victim->__state[1] |= kIOServiceTerm1WaiterState;
1939                         victim->unlockForArbitration();
1940                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
1941                 }
1942                 IOLockUnlock(gIOServiceBusyLock);
1943                 if( wait) {
1944                     waitResult = thread_block(THREAD_CONTINUE_NULL);
1945                     TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
1946                         getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
1947                         victim->lockForArbitration();
1948                 }
1949             } while( wait && (waitResult != THREAD_TIMED_OUT));
1950
1951             victim->unlockForArbitration();
1952         }
1953         if( victim == this)
1954             startPhase2 = didInactive;
1955         if( didInactive) {
1956
1957             victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff );
1958             IOUserClient::destroyUserReferences( victim );
1959
1960             iter = victim->getClientIterator();
1961             if( iter) {
1962                 while( (client = (IOService *) iter->getNextObject())) {
1963                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
1964                             client->getName(), client->getRegistryEntryID(),
1965                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
1966                     ok = client->requestTerminate( victim, options );
1967                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
1968                             client->getName(), client->getRegistryEntryID(),
1969                             victim->getName(), victim->getRegistryEntryID(), ok);
1970
1971                     uint64_t regID1 = client->getRegistryEntryID();
1972                     uint64_t regID2 = victim->getRegistryEntryID();
1973                     IOServiceTrace(
1974                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
1975                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
1976                         (uintptr_t) regID1,
1977                         (uintptr_t) (regID1 >> 32),
1978                         (uintptr_t) regID2,
1979                         (uintptr_t) (regID2 >> 32));
1980
1981                     if( ok)
1982                         makeInactive->setObject( client );
1983                 }
1984                 iter->release();
1985             }
1986         }
1987         victim->release();
1988         victim = (IOService *) makeInactive->getObject(0);
1989         if( victim) {
1990             victim->retain();
1991             makeInactive->removeObject(0);
1992         }
1993     }
1994
1995     makeInactive->release();
1996
1997     if( startPhase2)
1998     {
1999         lockForArbitration();
2000         __state[1] &= ~kIOServiceTermPhase1State;
2001         if (kIOServiceTerm1WaiterState & __state[1])
2002         {
2003             __state[1] &= ~kIOServiceTerm1WaiterState;
2004             TLOG("%s[0x%qx]::wakePhase1\n", getName(), getRegistryEntryID());
2005             IOLockLock( gIOServiceBusyLock );
2006             thread_wakeup( (event_t) &__state[1]);
2007             IOLockUnlock( gIOServiceBusyLock );
2008         }
2009         unlockForArbitration();
2010         scheduleTerminatePhase2( options );
2011     }
2012
2013     return( true );
2014 }
2015
2016 void IOService::setTerminateDefer(IOService * provider, bool defer)
2017 {
2018     lockForArbitration();
2019     if (defer)  __state[1] |= kIOServiceStartState;
2020     else        __state[1] &= ~kIOServiceStartState;
2021     unlockForArbitration();
2022
2023     if (provider && !defer)
2024     {
2025         provider->lockForArbitration();
2026         provider->scheduleTerminatePhase2();
2027         provider->unlockForArbitration();
2028     }
2029 }
2030
2031 // call with lockForArbitration
2032 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2033 {
2034     AbsoluteTime        deadline;
2035     int                 waitResult = THREAD_AWAKENED;
2036     bool                wait, haveDeadline = false;
2037
2038     if (!(__state[0] & kIOServiceInactiveState)
2039       || (__state[1] & kIOServiceTermPhase1State))              return;
2040
2041     options |= kIOServiceRequired;
2042
2043     retain();
2044
2045     IOLockLock( gJobsLock );
2046
2047     if( (options & kIOServiceSynchronous)
2048         && (current_thread() != gIOTerminateThread)) {
2049
2050         do {
2051             wait = (gIOTerminateThread != 0);
2052             if( wait) {
2053                 // wait to become the terminate thread
2054                 IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2055             }
2056         } while( wait );
2057
2058         gIOTerminateThread = current_thread();
2059         gIOTerminatePhase2List->setObject( this );
2060         gIOTerminateWork++;
2061
2062         do {
2063             while( gIOTerminateWork )
2064                 terminateWorker( options );
2065             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2066             if( wait) {
2067                 // wait for the victim to go non-busy
2068                 if( !haveDeadline) {
2069                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2070                     haveDeadline = true;
2071                 }
2072                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2073                                                   deadline, THREAD_UNINT );
2074                 if( waitResult == THREAD_TIMED_OUT) {
2075                     IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2076                 }
2077             }
2078         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2079
2080         gIOTerminateThread = 0;
2081         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2082
2083     } else {
2084         // ! kIOServiceSynchronous
2085
2086         gIOTerminatePhase2List->setObject( this );
2087         if( 0 == gIOTerminateWork++) {
2088             if( !gIOTerminateThread)
2089                 kernel_thread_start(&terminateThread, (void *)(uintptr_t) options, &gIOTerminateThread);
2090             else
2091                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2092         }
2093     }
2094
2095     IOLockUnlock( gJobsLock );
2096
2097     release();
2098 }
2099
2100 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2101 {
2102     IOLockLock( gJobsLock );
2103
2104     while (gIOTerminateWork)
2105         terminateWorker( (uintptr_t) arg );
2106
2107     thread_deallocate(gIOTerminateThread);
2108     gIOTerminateThread = 0;
2109     IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2110
2111     IOLockUnlock( gJobsLock );
2112 }
2113
2114 void IOService::scheduleStop( IOService * provider )
2115 {
2116     uint64_t regID1 = getRegistryEntryID();
2117     uint64_t regID2 = provider->getRegistryEntryID();
2118
2119     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2120     IOServiceTrace(
2121         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2122         (uintptr_t) regID1,
2123         (uintptr_t) (regID1 >> 32),
2124         (uintptr_t) regID2,
2125         (uintptr_t) (regID2 >> 32));
2126
2127     IOLockLock( gJobsLock );
2128     gIOStopList->tailQ( this );
2129     gIOStopProviderList->tailQ( provider );
2130
2131     if( 0 == gIOTerminateWork++) {
2132         if( !gIOTerminateThread)
2133             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2134         else
2135             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2136     }
2137
2138     IOLockUnlock( gJobsLock );
2139 }
2140
2141 void IOService::scheduleFinalize( void )
2142 {
2143     uint64_t regID1 = getRegistryEntryID();
2144
2145     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2146     IOServiceTrace(
2147         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2148         (uintptr_t) regID1,
2149         (uintptr_t) (regID1 >> 32),
2150         0, 0);
2151
2152     IOLockLock( gJobsLock );
2153     gIOFinalizeList->tailQ( this );
2154
2155     if( 0 == gIOTerminateWork++) {
2156         if( !gIOTerminateThread)
2157             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2158         else
2159             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2160     }
2161
2162     IOLockUnlock( gJobsLock );
2163 }
2164
2165 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2166 {
2167     return( true );
2168 }
2169
2170 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2171 {
2172     if( false == *defer) {
2173
2174         if( lockForArbitration( true )) {
2175             if( false == provider->handleIsOpen( this ))
2176                 scheduleStop( provider );
2177             // -- compat
2178             else {
2179                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2180                 if( false == provider->handleIsOpen( this ))
2181                     scheduleStop( provider );
2182             }
2183             // --
2184             unlockForArbitration();
2185         }
2186     }
2187
2188     return( true );
2189 }
2190
2191 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2192                                      OSArray * doPhase2List,
2193                                      void *unused2 __unused,
2194                                      void *unused3 __unused  )
2195 {
2196     OSIterator * iter;
2197     IOService *  client;
2198     bool         ok;
2199     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2200
2201     iter = victim->getClientIterator();
2202     if( iter) {
2203         while( (client = (IOService *) iter->getNextObject())) {
2204
2205             regID1 = client->getRegistryEntryID();
2206             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2207                     client->getName(), regID1,
2208                     victim->getName(), regID2, (long long)options);
2209             IOServiceTrace(
2210                 IOSERVICE_TERMINATE_WILL,
2211                 (uintptr_t) regID1,
2212                 (uintptr_t) (regID1 >> 32),
2213                 (uintptr_t) regID2,
2214                 (uintptr_t) (regID2 >> 32));
2215
2216             ok = client->willTerminate( victim, options );
2217             doPhase2List->tailQ( client );
2218         }
2219         iter->release();
2220     }
2221 }
2222
2223 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2224                             void *unused1 __unused, void *unused2 __unused,
2225                             void *unused3 __unused )
2226 {
2227     OSIterator * iter;
2228     IOService *  client;
2229     bool defer = false;
2230     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2231
2232     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2233
2234     iter = victim->getClientIterator();
2235     if( iter) {
2236         while( (client = (IOService *) iter->getNextObject())) {
2237
2238             regID1 = client->getRegistryEntryID();
2239             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2240                     client->getName(), regID1,
2241                     victim->getName(), regID2, (long long)options);
2242             client->didTerminate( victim, options, &defer );
2243
2244             IOServiceTrace(
2245                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2246                        : IOSERVICE_TERMINATE_DID),
2247                 (uintptr_t) regID1,
2248                 (uintptr_t) (regID1 >> 32),
2249                 (uintptr_t) regID2,
2250                 (uintptr_t) (regID2 >> 32));
2251
2252             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2253                     client->getName(), regID1,
2254                     victim->getName(), regID2, defer);
2255         }
2256         iter->release();
2257     }
2258 }
2259
2260
2261 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2262                             void *unused1 __unused, void *unused2 __unused,
2263                             void *unused3 __unused )
2264 {
2265     OSIterator * iter;
2266     IOService *  provider;
2267     bool         ok;
2268     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2269
2270     iter = victim->getProviderIterator();
2271     if( iter) {
2272         while( (provider = (IOService *) iter->getNextObject())) {
2273
2274             regID1 = provider->getRegistryEntryID();
2275             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2276                     victim->getName(), regID2,
2277                     provider->getName(), regID1, (long long)options);
2278             IOServiceTrace(
2279                 IOSERVICE_TERMINATE_WILL,
2280                 (uintptr_t) regID2,
2281                 (uintptr_t) (regID2 >> 32),
2282                 (uintptr_t) regID1,
2283                 (uintptr_t) (regID1 >> 32));
2284
2285             ok = victim->willTerminate( provider, options );
2286         }
2287         iter->release();
2288     }
2289 }
2290
2291 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2292                             void *unused1 __unused, void *unused2 __unused,
2293                             void *unused3 __unused )
2294 {
2295     OSIterator * iter;
2296     IOService *  provider;
2297     bool defer = false;
2298     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2299
2300     iter = victim->getProviderIterator();
2301     if( iter) {
2302         while( (provider = (IOService *) iter->getNextObject())) {
2303
2304             regID1 = provider->getRegistryEntryID();
2305             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2306                     victim->getName(), regID2,
2307                     provider->getName(), regID1, (long long)options);
2308             victim->didTerminate( provider, options, &defer );
2309
2310             IOServiceTrace(
2311                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2312                        : IOSERVICE_TERMINATE_DID),
2313                 (uintptr_t) regID2,
2314                 (uintptr_t) (regID2 >> 32),
2315                 (uintptr_t) regID1,
2316                 (uintptr_t) (regID1 >> 32));
2317
2318             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2319                     victim->getName(), regID2,
2320                     provider->getName(), regID1, defer);
2321         }
2322         iter->release();
2323     }
2324 }
2325
2326
2327 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2328                             void *unused1 __unused, void *unused2 __unused,
2329                             void *unused3 __unused )
2330 {
2331     uint64_t regID1 = victim->getRegistryEntryID();
2332     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2333     IOServiceTrace(
2334         IOSERVICE_TERMINATE_FINALIZE,
2335         (uintptr_t) regID1,
2336         (uintptr_t) (regID1 >> 32),
2337         0, 0);
2338
2339     victim->finalize( options );
2340 }
2341
2342 void IOService::actionStop( IOService * provider, IOService * client,
2343                             void *unused1 __unused, void *unused2 __unused,
2344                             void *unused3 __unused )
2345 {
2346     uint64_t regID1 = provider->getRegistryEntryID();
2347     uint64_t regID2 = client->getRegistryEntryID();
2348
2349     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2350     IOServiceTrace(
2351         IOSERVICE_TERMINATE_STOP,
2352         (uintptr_t) regID1,
2353         (uintptr_t) (regID1 >> 32),
2354         (uintptr_t) regID2,
2355         (uintptr_t) (regID2 >> 32));
2356
2357     client->stop( provider );
2358     if( provider->isOpen( client ))
2359         provider->close( client );
2360
2361     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2362     client->detach( provider );
2363 }
2364
2365 void IOService::terminateWorker( IOOptionBits options )
2366 {
2367     OSArray *           doPhase2List;
2368     OSArray *           didPhase2List;
2369     OSSet *             freeList;
2370     OSIterator *        iter;
2371     UInt32              workDone;
2372     IOService *         victim;
2373     IOService *         client;
2374     IOService *         provider;
2375     unsigned int        idx;
2376     bool                moreToDo;
2377     bool                doPhase2;
2378     bool                doPhase3;
2379
2380     options |= kIOServiceRequired;
2381
2382     doPhase2List  = OSArray::withCapacity( 16 );
2383     didPhase2List = OSArray::withCapacity( 16 );
2384     freeList      = OSSet::withCapacity( 16 );
2385     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2386         return;
2387
2388     do {
2389         workDone = gIOTerminateWork;
2390
2391         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2392
2393             victim->retain();
2394             gIOTerminatePhase2List->removeObject(0);
2395             IOLockUnlock( gJobsLock );
2396
2397             while( victim ) {
2398
2399                 doPhase2 = victim->lockForArbitration( true );
2400                 if( doPhase2) {
2401                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2402                     if( doPhase2) {
2403                         doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State))
2404                                 && (0 == (victim->__state[1] & kIOServiceConfigState));
2405
2406                         if (doPhase2 && (iter = victim->getClientIterator())) {
2407                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2408                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2409
2410                                 if (!doPhase2) TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2411                                                victim->getName(), victim->getRegistryEntryID(),
2412                                                client->getName(), client->getRegistryEntryID());
2413                             }
2414                             iter->release();
2415                         }
2416                         if( doPhase2)
2417                             victim->__state[1] |= kIOServiceTermPhase2State;
2418                     }
2419                     victim->unlockForArbitration();
2420                 }
2421                 if( doPhase2) {
2422
2423                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2424                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2425                                             victim, (void *)(uintptr_t) options, NULL );
2426                     }
2427
2428                     if( 0 == victim->getClient()) {
2429                         // no clients - will go to finalize
2430                         IOLockLock( gJobsLock );
2431                         gIOFinalizeList->tailQ( victim );
2432                         IOLockUnlock( gJobsLock );
2433                     } else {
2434                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2435                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2436                     }
2437                     didPhase2List->headQ( victim );
2438                 }
2439                 victim->release();
2440                 victim = (IOService *) doPhase2List->getObject(0);
2441                 if( victim) {
2442                     victim->retain();
2443                     doPhase2List->removeObject(0);
2444                 }
2445             }
2446
2447             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2448
2449                 if( victim->lockForArbitration( true )) {
2450                     victim->__state[1] |= kIOServiceTermPhase3State;
2451                     victim->unlockForArbitration();
2452                 }
2453                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2454                                     victim, (void *)(uintptr_t) options );
2455                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2456                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2457                                         victim, (void *)(uintptr_t) options, NULL );
2458                 }
2459                 didPhase2List->removeObject(0);
2460             }
2461             IOLockLock( gJobsLock );
2462         }
2463
2464         // phase 3
2465         do {
2466             doPhase3 = false;
2467             // finalize leaves
2468             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2469
2470                 IOLockUnlock( gJobsLock );
2471                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2472                                     victim, (void *)(uintptr_t) options );
2473                 IOLockLock( gJobsLock );
2474                 // hold off free
2475                 freeList->setObject( victim );
2476                 // safe if finalize list is append only
2477                 gIOFinalizeList->removeObject(0);
2478             }
2479
2480             for( idx = 0;
2481                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2482
2483                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2484                 assert( provider );
2485
2486                 uint64_t regID1 = provider->getRegistryEntryID();
2487                 uint64_t regID2 = client->getRegistryEntryID();
2488
2489                 if( !provider->isChild( client, gIOServicePlane )) {
2490                     // may be multiply queued - nop it
2491                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2492                     IOServiceTrace(
2493                         IOSERVICE_TERMINATE_STOP_NOP,
2494                         (uintptr_t) regID1,
2495                         (uintptr_t) (regID1 >> 32),
2496                         (uintptr_t) regID2,
2497                         (uintptr_t) (regID2 >> 32));
2498
2499                 } else {
2500                     // a terminated client is not ready for stop if it has clients, skip it
2501                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2502                         TLOG("%s[0x%qx]::defer stop(%s[0x%qx])\n",
2503                                 client->getName(), regID2,
2504                                 client->getClient()->getName(), client->getClient()->getRegistryEntryID());
2505                         IOServiceTrace(
2506                             IOSERVICE_TERMINATE_STOP_DEFER,
2507                             (uintptr_t) regID1,
2508                             (uintptr_t) (regID1 >> 32),
2509                             (uintptr_t) regID2,
2510                             (uintptr_t) (regID2 >> 32));
2511
2512                         idx++;
2513                         continue;
2514                     }
2515
2516                     IOLockUnlock( gJobsLock );
2517                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2518                                      provider, (void *) client );
2519                     IOLockLock( gJobsLock );
2520                     // check the finalize list now
2521                     doPhase3 = true;
2522                 }
2523                 // hold off free
2524                 freeList->setObject( client );
2525                 freeList->setObject( provider );
2526
2527                 // safe if stop list is append only
2528                 gIOStopList->removeObject( idx );
2529                 gIOStopProviderList->removeObject( idx );
2530                 idx = 0;
2531             }
2532
2533         } while( doPhase3 );
2534
2535         gIOTerminateWork -= workDone;
2536         moreToDo = (gIOTerminateWork != 0);
2537
2538         if( !moreToDo) {
2539             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2540             IOServiceTrace(
2541                 IOSERVICE_TERMINATE_DONE,
2542                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2543         }
2544
2545     } while( moreToDo );
2546
2547     IOLockUnlock( gJobsLock );
2548
2549     freeList->release();
2550     doPhase2List->release();
2551     didPhase2List->release();
2552
2553     IOLockLock( gJobsLock );
2554 }
2555
2556 bool IOService::finalize( IOOptionBits options )
2557 {
2558     OSIterator *  iter;
2559     IOService *   provider;
2560     uint64_t      regID1, regID2 = getRegistryEntryID();
2561
2562     iter = getProviderIterator();
2563     assert( iter );
2564
2565     if( iter) {
2566         while( (provider = (IOService *) iter->getNextObject())) {
2567
2568             // -- compat
2569             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2570                 /* we come down here on programmatic terminate */
2571
2572                 regID1 = provider->getRegistryEntryID();
2573                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2574                 IOServiceTrace(
2575                     IOSERVICE_TERMINATE_STOP,
2576                     (uintptr_t) regID1,
2577                     (uintptr_t) (regID1 >> 32),
2578                     (uintptr_t) regID2,
2579                     (uintptr_t) (regID2 >> 32));
2580
2581                 stop( provider );
2582                 if( provider->isOpen( this ))
2583                     provider->close( this );
2584                 detach( provider );
2585             } else {
2586             //--
2587                 if( provider->lockForArbitration( true )) {
2588                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2589                         scheduleStop( provider );
2590                     provider->unlockForArbitration();
2591                 }
2592             }
2593         }
2594         iter->release();
2595     }
2596
2597     return( true );
2598 }
2599
2600 #undef tailQ
2601 #undef headQ
2602
2603 /*
2604  * Terminate
2605  */
2606
2607 void IOService::doServiceTerminate( IOOptionBits options )
2608 {
2609 }
2610
2611 // a method in case someone needs to override it
2612 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2613 {
2614     bool ok;
2615
2616     if( client->isParent( this, gIOServicePlane, true))
2617         // we are the clients only provider
2618         ok = client->terminate( options );
2619     else
2620         ok = true;
2621
2622     return( ok );
2623 }
2624
2625 bool IOService::terminate( IOOptionBits options )
2626 {
2627     options |= kIOServiceTerminate;
2628
2629     return( terminatePhase1( options ));
2630 }
2631
2632 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2633
2634 /*
2635  * Open & close
2636  */
2637
2638 struct ServiceOpenMessageContext
2639 {
2640     IOService *  service;
2641     UInt32       type;
2642     IOService *  excludeClient;
2643     IOOptionBits options;
2644 };
2645
2646 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2647 {
2648     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2649
2650     if( object != context->excludeClient)
2651         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2652 }
2653
2654 bool IOService::open(   IOService *     forClient,
2655                         IOOptionBits    options,
2656                         void *          arg )
2657 {
2658     bool                        ok;
2659     ServiceOpenMessageContext   context;
2660
2661     context.service             = this;
2662     context.type                = kIOMessageServiceIsAttemptingOpen;
2663     context.excludeClient       = forClient;
2664     context.options             = options;
2665
2666     applyToInterested( gIOGeneralInterest,
2667                         &serviceOpenMessageApplier, &context );
2668
2669     if( false == lockForArbitration(false) )
2670         return false;
2671
2672     ok = (0 == (__state[0] & kIOServiceInactiveState));
2673     if( ok)
2674         ok = handleOpen( forClient, options, arg );
2675
2676     unlockForArbitration();
2677
2678     return( ok );
2679 }
2680
2681 void IOService::close(  IOService *     forClient,
2682                         IOOptionBits    options )
2683 {
2684     bool                wasClosed;
2685     bool                last = false;
2686
2687     lockForArbitration();
2688
2689     wasClosed = handleIsOpen( forClient );
2690     if( wasClosed) {
2691         handleClose( forClient, options );
2692         last = (__state[1] & kIOServiceTermPhase3State);
2693     }
2694
2695     unlockForArbitration();
2696
2697     if( last)
2698         forClient->scheduleStop( this );
2699
2700     else if( wasClosed) {
2701
2702         ServiceOpenMessageContext context;
2703
2704         context.service         = this;
2705         context.type            = kIOMessageServiceWasClosed;
2706         context.excludeClient   = forClient;
2707         context.options         = options;
2708
2709         applyToInterested( gIOGeneralInterest,
2710                             &serviceOpenMessageApplier, &context );
2711     }
2712 }
2713
2714 bool IOService::isOpen( const IOService * forClient ) const
2715 {
2716     IOService * self = (IOService *) this;
2717     bool ok;
2718
2719     self->lockForArbitration();
2720
2721     ok = handleIsOpen( forClient );
2722
2723     self->unlockForArbitration();
2724
2725     return( ok );
2726 }
2727
2728 bool IOService::handleOpen(     IOService *     forClient,
2729                                 IOOptionBits    options,
2730                                 void *          arg )
2731 {
2732     bool        ok;
2733
2734     ok = (0 == __owner);
2735     if( ok )
2736         __owner = forClient;
2737
2738     else if( options & kIOServiceSeize ) {
2739         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2740                                 __owner, (void *)(uintptr_t) options ));
2741         if( ok && (0 == __owner ))
2742             __owner = forClient;
2743         else
2744             ok = false;
2745     }
2746     return( ok );
2747 }
2748
2749 void IOService::handleClose(    IOService *     forClient,
2750                                 IOOptionBits    options )
2751 {
2752     if( __owner == forClient)
2753         __owner = 0;
2754 }
2755
2756 bool IOService::handleIsOpen(   const IOService * forClient ) const
2757 {
2758     if( forClient)
2759         return( __owner == forClient );
2760     else
2761         return( __owner != forClient );
2762 }
2763
2764 /*
2765  * Probing & starting
2766  */
2767 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2768 {
2769     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
2770     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
2771     SInt32             val1;
2772     SInt32             val2;
2773
2774     val1 = 0;
2775     val2 = 0;
2776
2777     if ( obj1 )
2778         val1 = obj1->priority;
2779
2780     if ( obj2 )
2781         val2 = obj2->priority;
2782
2783     return ( val1 - val2 );
2784 }
2785
2786 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
2787 {
2788     OSDictionary *      dict;
2789     IOService *         service;
2790     _IOServiceNotifier * notify;
2791     OSSymbol *          key = (OSSymbol *) ref;
2792     OSNumber *          offset;
2793
2794     if( (dict = OSDynamicCast( OSDictionary, entry)))
2795         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
2796     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
2797         return( notify->priority );
2798
2799     else if( (service = OSDynamicCast( IOService, entry)))
2800         offset = OSDynamicCast(OSNumber, service->getProperty( key ));
2801     else {
2802         assert( false );
2803         offset = 0;
2804     }
2805
2806     if( offset)
2807         return( (SInt32) offset->unsigned32BitValue());
2808     else
2809         return( kIODefaultProbeScore );
2810 }
2811
2812 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2813 {
2814     const OSObject *    obj1 = (const OSObject *) inObj1;
2815     const OSObject *    obj2 = (const OSObject *) inObj2;
2816     SInt32               val1;
2817     SInt32               val2;
2818
2819     val1 = 0;
2820     val2 = 0;
2821
2822     if ( obj1 )
2823         val1 = IOServiceObjectOrder( obj1, ref );
2824
2825     if ( obj2 )
2826         val2 = IOServiceObjectOrder( obj2, ref );
2827
2828     return ( val1 - val2 );
2829 }
2830
2831 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
2832 {
2833     IOService *         service = 0;
2834     OSIterator *        iter;
2835     const OSSymbol *    nextCat;
2836
2837     iter = getClientIterator();
2838     if( iter) {
2839         while( (service = (IOService *) iter->getNextObject())) {
2840             if( kIOServiceInactiveState & service->__state[0])
2841                 continue;
2842             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
2843                         service->getProperty( gIOMatchCategoryKey ));
2844             if( category == nextCat)
2845             {
2846                 service->retain();
2847                 break;
2848             }
2849         }
2850         iter->release();
2851     }
2852     return( service );
2853 }
2854
2855 IOService * IOService::getClientWithCategory( const OSSymbol * category )
2856 {
2857     IOService *
2858     service = copyClientWithCategory(category);
2859     if (service)
2860         service->release();
2861     return (service);
2862 }
2863
2864 bool IOService::invokeNotifer( _IOServiceNotifier * notify )
2865 {
2866     _IOServiceNotifierInvocation invocation;
2867     bool                         willNotify;
2868     bool                         ret = true;
2869
2870     invocation.thread = current_thread();
2871
2872     LOCKWRITENOTIFY();
2873     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2874
2875     if( willNotify) {
2876         queue_enter( &notify->handlerInvocations, &invocation,
2877                         _IOServiceNotifierInvocation *, link );
2878     }
2879     UNLOCKNOTIFY();
2880
2881     if( willNotify) {
2882
2883         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
2884
2885         LOCKWRITENOTIFY();
2886         queue_remove( &notify->handlerInvocations, &invocation,
2887                         _IOServiceNotifierInvocation *, link );
2888         if( kIOServiceNotifyWaiter & notify->state) {
2889             notify->state &= ~kIOServiceNotifyWaiter;
2890             WAKEUPNOTIFY( notify );
2891         }
2892         UNLOCKNOTIFY();
2893     }
2894
2895     return( ret );
2896 }
2897
2898 /*
2899  * Alloc and probe matching classes,
2900  * called on the provider instance
2901  */
2902
2903 void IOService::probeCandidates( OSOrderedSet * matches )
2904 {
2905     OSDictionary        *       match = 0;
2906     OSSymbol            *       symbol;
2907     IOService           *       inst;
2908     IOService           *       newInst;
2909     OSDictionary        *       props;
2910     SInt32                      score;
2911     OSNumber            *       newPri;
2912     OSOrderedSet        *       familyMatches = 0;
2913     OSOrderedSet        *       startList;
2914     OSDictionary        *       startDict = 0;
2915     const OSSymbol      *       category;
2916     OSIterator          *       iter;
2917     _IOServiceNotifier  *               notify;
2918     OSObject            *       nextMatch = 0;
2919     bool                        started;
2920     bool                        needReloc = false;
2921 #if IOMATCHDEBUG
2922     SInt64                      debugFlags;
2923 #endif
2924     IOService * client = NULL;
2925
2926
2927     assert( matches );
2928     while( !needReloc && (nextMatch = matches->getFirstObject())) {
2929
2930         nextMatch->retain();
2931         matches->removeObject(nextMatch);
2932
2933         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
2934
2935             lockForArbitration();
2936             if( 0 == (__state[0] & kIOServiceInactiveState))
2937                 invokeNotifer( notify );
2938             unlockForArbitration();
2939             nextMatch->release();
2940             nextMatch = 0;
2941             continue;
2942
2943         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
2944             nextMatch->release();
2945             nextMatch = 0;
2946             continue;
2947         }
2948
2949         props = 0;
2950 #if IOMATCHDEBUG
2951         debugFlags = getDebugFlags( match );
2952 #endif
2953
2954         do {
2955             category = OSDynamicCast( OSSymbol,
2956                         match->getObject( gIOMatchCategoryKey ));
2957             if( 0 == category)
2958                 category = gIODefaultMatchCategoryKey;
2959
2960             if( (client = copyClientWithCategory(category)) ) {
2961 #if IOMATCHDEBUG
2962                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
2963                     LOG("%s: match category %s exists\n", getName(),
2964                                 category->getCStringNoCopy());
2965 #endif
2966                 nextMatch->release();
2967                 nextMatch = 0;
2968
2969                 client->release();
2970                 client = NULL;
2971
2972                 continue;
2973             }
2974
2975             // create a copy now in case its modified during matching
2976             props = OSDictionary::withDictionary( match, match->getCount());
2977             if( 0 == props)
2978                 continue;
2979             props->setCapacityIncrement(1);
2980
2981             // check the nub matches
2982             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
2983                 continue;
2984
2985             // Check to see if driver reloc has been loaded.
2986             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
2987             if( needReloc) {
2988 #if IOMATCHDEBUG
2989                 if( debugFlags & kIOLogCatalogue)
2990                     LOG("%s: stalling for module\n", getName());
2991 #endif
2992                 // If reloc hasn't been loaded, exit;
2993                 // reprobing will occur after reloc has been loaded.
2994                 continue;
2995             }
2996
2997             // reorder on family matchPropertyTable score.
2998             if( 0 == familyMatches)
2999                 familyMatches = OSOrderedSet::withCapacity( 1,
3000                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3001             if( familyMatches)
3002                 familyMatches->setObject( props );
3003
3004         } while( false );
3005
3006         if (nextMatch) {
3007             nextMatch->release();
3008             nextMatch = 0;
3009         }
3010         if( props)
3011             props->release();
3012     }
3013     matches->release();
3014     matches = 0;
3015
3016     if( familyMatches) {
3017
3018         while( !needReloc
3019              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3020
3021             props->retain();
3022             familyMatches->removeObject( props );
3023
3024             inst = 0;
3025             newInst = 0;
3026 #if IOMATCHDEBUG
3027             debugFlags = getDebugFlags( props );
3028 #endif
3029             do {
3030                 symbol = OSDynamicCast( OSSymbol,
3031                                 props->getObject( gIOClassKey));
3032                 if( !symbol)
3033                     continue;
3034
3035                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), OBFUSCATE(symbol), OBFUSCATE(props));
3036
3037                 // alloc the driver instance
3038                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3039
3040                 if( !inst) {
3041                     IOLog("Couldn't alloc class \"%s\"\n",
3042                         symbol->getCStringNoCopy());
3043                     continue;
3044                 }
3045
3046                 // init driver instance
3047                 if( !(inst->init( props ))) {
3048 #if IOMATCHDEBUG
3049                     if( debugFlags & kIOLogStart)
3050                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3051 #endif
3052                     continue;
3053                 }
3054                 if( __state[1] & kIOServiceSynchronousState)
3055                     inst->__state[1] |= kIOServiceSynchronousState;
3056
3057                 // give the driver the default match category if not specified
3058                 category = OSDynamicCast( OSSymbol,
3059                             props->getObject( gIOMatchCategoryKey ));
3060                 if( 0 == category)
3061                     category = gIODefaultMatchCategoryKey;
3062                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3063                 // attach driver instance
3064                 if( !(inst->attach( this )))
3065                         continue;
3066
3067                 // pass in score from property table
3068                 score = familyMatches->orderObject( props );
3069
3070                 // & probe the new driver instance
3071 #if IOMATCHDEBUG
3072                 if( debugFlags & kIOLogProbe)
3073                     LOG("%s::probe(%s)\n",
3074                         inst->getMetaClass()->getClassName(), getName());
3075 #endif
3076
3077                 newInst = inst->probe( this, &score );
3078                 inst->detach( this );
3079                 if( 0 == newInst) {
3080 #if IOMATCHDEBUG
3081                     if( debugFlags & kIOLogProbe)
3082                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3083 #endif
3084                     continue;
3085                 }
3086
3087                 // save the score
3088                 newPri = OSNumber::withNumber( score, 32 );
3089                 if( newPri) {
3090                     newInst->setProperty( gIOProbeScoreKey, newPri );
3091                     newPri->release();
3092                 }
3093
3094                 // add to start list for the match category
3095                 if( 0 == startDict)
3096                     startDict = OSDictionary::withCapacity( 1 );
3097                 assert( startDict );
3098                 startList = (OSOrderedSet *)
3099                                 startDict->getObject( category );
3100                 if( 0 == startList) {
3101                     startList = OSOrderedSet::withCapacity( 1,
3102                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3103                     if( startDict && startList) {
3104                         startDict->setObject( category, startList );
3105                         startList->release();
3106                     }
3107                 }
3108                 assert( startList );
3109                 if( startList)
3110                     startList->setObject( newInst );
3111
3112             } while( false );
3113
3114             props->release();
3115             if( inst)
3116                 inst->release();
3117         }
3118         familyMatches->release();
3119         familyMatches = 0;
3120     }
3121
3122     // start the best (until success) of each category
3123
3124     iter = OSCollectionIterator::withCollection( startDict );
3125     if( iter) {
3126         while( (category = (const OSSymbol *) iter->getNextObject())) {
3127
3128             startList = (OSOrderedSet *) startDict->getObject( category );
3129             assert( startList );
3130             if( !startList)
3131                 continue;
3132
3133             started = false;
3134             while( true // (!started)
3135                    && (inst = (IOService *)startList->getFirstObject())) {
3136
3137                 inst->retain();
3138                 startList->removeObject(inst);
3139
3140 #if IOMATCHDEBUG
3141                 debugFlags = getDebugFlags( inst->getPropertyTable() );
3142
3143                 if( debugFlags & kIOLogStart) {
3144                     if( started)
3145                         LOG( "match category exists, skipping " );
3146                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3147                          getName(), inst->getRetainCount());
3148                 }
3149 #endif
3150                 if( false == started)
3151                     started = startCandidate( inst );
3152 #if IOMATCHDEBUG
3153                 if( (debugFlags & kIOLogStart) && (false == started))
3154                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3155                          inst->getRetainCount());
3156 #endif
3157                 inst->release();
3158             }
3159         }
3160         iter->release();
3161     }
3162
3163
3164     // adjust the busy count by +1 if matching is stalled for a module,
3165     // or -1 if a previously stalled matching is complete.
3166     lockForArbitration();
3167     SInt32 adjBusy = 0;
3168     uint64_t regID = getRegistryEntryID();
3169
3170     if( needReloc) {
3171         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3172         if( adjBusy) {
3173
3174             IOServiceTrace(
3175                 IOSERVICE_MODULESTALL,
3176                 (uintptr_t) regID,
3177                 (uintptr_t) (regID >> 32),
3178                 (uintptr_t) this,
3179                 0);
3180
3181             __state[1] |= kIOServiceModuleStallState;
3182         }
3183
3184     } else if( __state[1] & kIOServiceModuleStallState) {
3185
3186             IOServiceTrace(
3187             IOSERVICE_MODULEUNSTALL,
3188             (uintptr_t) regID,
3189             (uintptr_t) (regID >> 32),
3190             (uintptr_t) this,
3191             0);
3192
3193         __state[1] &= ~kIOServiceModuleStallState;
3194         adjBusy = -1;
3195     }
3196     if( adjBusy)
3197         _adjustBusy( adjBusy );
3198     unlockForArbitration();
3199
3200     if( startDict)
3201         startDict->release();
3202 }
3203
3204 /*
3205  * Start a previously attached & probed instance,
3206  * called on exporting object instance
3207  */
3208
3209 bool IOService::startCandidate( IOService * service )
3210 {
3211     bool                ok;
3212
3213     ok = service->attach( this );
3214
3215     if( ok)
3216     {
3217         if (this != gIOResources)
3218         {
3219             // stall for any nub resources
3220             checkResources();
3221             // stall for any driver resources
3222             service->checkResources();
3223         }
3224
3225         AbsoluteTime startTime;
3226         AbsoluteTime endTime;
3227         UInt64       nano;
3228
3229         if (kIOLogStart & gIOKitDebug)
3230             clock_get_uptime(&startTime);
3231
3232         ok = service->start(this);
3233
3234         if (kIOLogStart & gIOKitDebug)
3235         {
3236             clock_get_uptime(&endTime);
3237
3238             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3239             {
3240                 SUB_ABSOLUTETIME(&endTime, &startTime);
3241                 absolutetime_to_nanoseconds(endTime, &nano);
3242                 if (nano > 500000000ULL)
3243                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3244             }
3245         }
3246         if( !ok)
3247             service->detach( this );
3248     }
3249     return( ok );
3250 }
3251
3252 void IOService::publishResource( const char * key, OSObject * value )
3253 {
3254     const OSSymbol *    sym;
3255
3256     if( (sym = OSSymbol::withCString( key))) {
3257         publishResource( sym, value);
3258         sym->release();
3259     }
3260 }
3261
3262 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3263 {
3264     if( 0 == value)
3265         value = (OSObject *) gIOServiceKey;
3266
3267     gIOResources->setProperty( key, value);
3268
3269     if( IORecursiveLockHaveLock( gNotificationLock))
3270         return;
3271
3272     gIOResourceGenerationCount++;
3273     gIOResources->registerService();
3274 }
3275
3276 bool IOService::addNeededResource( const char * key )
3277 {
3278     OSObject *  resourcesProp;
3279     OSSet *     set;
3280     OSString *  newKey;
3281     bool ret;
3282
3283     resourcesProp = getProperty( gIOResourceMatchKey );
3284
3285     newKey = OSString::withCString( key );
3286     if( (0 == resourcesProp) || (0 == newKey))
3287         return( false);
3288
3289     set = OSDynamicCast( OSSet, resourcesProp );
3290     if( !set) {
3291         set = OSSet::withCapacity( 1 );
3292         if( set)
3293             set->setObject( resourcesProp );
3294     }
3295     else
3296         set->retain();
3297
3298     set->setObject( newKey );
3299     newKey->release();
3300     ret = setProperty( gIOResourceMatchKey, set );
3301     set->release();
3302
3303     return( ret );
3304 }
3305
3306 bool IOService::checkResource( OSObject * matching )
3307 {
3308     OSString *          str;
3309     OSDictionary *      table;
3310
3311     if( (str = OSDynamicCast( OSString, matching ))) {
3312         if( gIOResources->getProperty( str ))
3313             return( true );
3314     }
3315
3316     if( str)
3317         table = resourceMatching( str );
3318     else if( (table = OSDynamicCast( OSDictionary, matching )))
3319         table->retain();
3320     else {
3321         IOLog("%s: Can't match using: %s\n", getName(),
3322                 matching->getMetaClass()->getClassName());
3323         /* false would stall forever */
3324         return( true );
3325     }
3326
3327     if( gIOKitDebug & kIOLogConfig)
3328         LOG("config(%p): stalling %s\n", OBFUSCATE(IOThreadSelf()), getName());
3329
3330     waitForService( table );
3331
3332     if( gIOKitDebug & kIOLogConfig)
3333         LOG("config(%p): waking\n", OBFUSCATE(IOThreadSelf()) );
3334
3335     return( true );
3336 }
3337
3338 bool IOService::checkResources( void )
3339 {
3340     OSObject *          resourcesProp;
3341     OSSet *             set;
3342     OSIterator *        iter;
3343     bool                ok;
3344
3345     resourcesProp = getProperty( gIOResourceMatchKey );
3346     if( 0 == resourcesProp)
3347         return( true );
3348
3349     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3350
3351         iter = OSCollectionIterator::withCollection( set );
3352         ok = (0 != iter);
3353         while( ok && (resourcesProp = iter->getNextObject()) )
3354             ok = checkResource( resourcesProp );
3355         if( iter)
3356             iter->release();
3357
3358     } else
3359         ok = checkResource( resourcesProp );
3360
3361     return( ok );
3362 }
3363
3364
3365 void _IOConfigThread::configThread( void )
3366 {
3367     _IOConfigThread *   inst;
3368
3369     do {
3370         if( !(inst = new _IOConfigThread))
3371             continue;
3372         if( !inst->init())
3373             continue;
3374         thread_t unused;
3375         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3376             continue;
3377
3378         return;
3379
3380     } while( false);
3381
3382     if( inst)
3383         inst->release();
3384
3385     return;
3386 }
3387
3388 void _IOConfigThread::free( void )
3389 {
3390     thread_deallocate(current_thread());
3391     OSObject::free();
3392 }
3393
3394 void IOService::doServiceMatch( IOOptionBits options )
3395 {
3396     _IOServiceNotifier * notify;
3397     OSIterator *        iter;
3398     OSOrderedSet *      matches;
3399     SInt32              catalogGeneration;
3400     bool                keepGuessing = true;
3401     bool                reRegistered = true;
3402     bool                didRegister;
3403
3404 //    job->nub->deliverNotification( gIOPublishNotification,
3405 //                              kIOServiceRegisteredState, 0xffffffff );
3406
3407     while( keepGuessing ) {
3408
3409         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3410         // the matches list should always be created by findDrivers()
3411         if( matches) {
3412
3413             lockForArbitration();
3414             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3415                 getMetaClass()->addInstance(this);
3416                 deliverNotification( gIOFirstPublishNotification,
3417                                      kIOServiceFirstPublishState, 0xffffffff );
3418             }
3419             LOCKREADNOTIFY();
3420             __state[1] &= ~kIOServiceNeedConfigState;
3421             __state[1] |= kIOServiceConfigState;
3422             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3423             __state[0] |= kIOServiceRegisteredState;
3424
3425             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3426             if (reRegistered && keepGuessing) {
3427                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3428                         gNotifications->getObject( gIOPublishNotification ) );
3429                 if( iter) {
3430                     while((notify = (_IOServiceNotifier *)
3431                            iter->getNextObject())) {
3432
3433                         if( matchPassive(notify->matching, 0)
3434                          && (kIOServiceNotifyEnable & notify->state))
3435                             matches->setObject( notify );
3436                     }
3437                     iter->release();
3438                 }
3439             }
3440
3441             UNLOCKNOTIFY();
3442             unlockForArbitration();
3443
3444             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3445                 probeCandidates( matches );
3446             else
3447                 matches->release();
3448         }
3449
3450         lockForArbitration();
3451         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3452         keepGuessing =
3453                    (reRegistered || (catalogGeneration !=
3454                                         gIOCatalogue->getGenerationCount()))
3455                 && (0 == (__state[0] & kIOServiceInactiveState));
3456
3457         if( keepGuessing)
3458             unlockForArbitration();
3459     }
3460
3461     if( (0 == (__state[0] & kIOServiceInactiveState))
3462      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3463         deliverNotification( gIOMatchedNotification,
3464                 kIOServiceMatchedState, 0xffffffff );
3465         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3466             deliverNotification( gIOFirstMatchNotification,
3467                 kIOServiceFirstMatchState, 0xffffffff );
3468     }
3469
3470     __state[1] &= ~kIOServiceConfigState;
3471     scheduleTerminatePhase2();
3472
3473     _adjustBusy( -1 );
3474     unlockForArbitration();
3475 }
3476
3477 UInt32 IOService::_adjustBusy( SInt32 delta )
3478 {
3479     IOService * next;
3480     UInt32      count;
3481     UInt32      result;
3482     bool        wasQuiet, nowQuiet, needWake;
3483
3484     next = this;
3485     result = __state[1] & kIOServiceBusyStateMask;
3486
3487     if( delta) do {
3488         if( next != this)
3489             next->lockForArbitration();
3490         count = next->__state[1] & kIOServiceBusyStateMask;
3491         wasQuiet = (0 == count);
3492         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3493             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3494         else
3495             count += delta;
3496         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3497         nowQuiet = (0 == count);
3498         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3499
3500         if( needWake) {
3501             next->__state[1] &= ~kIOServiceBusyWaiterState;
3502             IOLockLock( gIOServiceBusyLock );
3503             thread_wakeup( (event_t) next);
3504             IOLockUnlock( gIOServiceBusyLock );
3505         }
3506         if( next != this)
3507             next->unlockForArbitration();
3508
3509         if( (wasQuiet || nowQuiet) ) {
3510             uint64_t regID = next->getRegistryEntryID();
3511
3512                 IOServiceTrace(
3513                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3514                 (uintptr_t) regID,
3515                 (uintptr_t) (regID >> 32),
3516                 (uintptr_t) next,
3517                 0);
3518
3519             if (wasQuiet)
3520             {
3521                 next->__timeBusy = mach_absolute_time();
3522             }
3523             else
3524             {
3525                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3526                 next->__timeBusy = 0;
3527             }
3528
3529             MessageClientsContext context;
3530
3531             context.service  = next;
3532             context.type     = kIOMessageServiceBusyStateChange;
3533             context.argument = (void *) wasQuiet;       /*nowBusy*/
3534             context.argSize  = 0;
3535
3536             applyToInterestNotifiers( next, gIOBusyInterest,
3537                                      &messageClientsApplier, &context );
3538
3539 #if !NO_KEXTD
3540             if( nowQuiet && (next == gIOServiceRoot)) {
3541                 OSKext::considerUnloads();
3542                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3543             }
3544 #endif
3545         }
3546
3547         delta = nowQuiet ? -1 : +1;
3548
3549     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3550
3551     return( result );
3552 }
3553
3554 void IOService::adjustBusy( SInt32 delta )
3555 {
3556     lockForArbitration();
3557     _adjustBusy( delta );
3558     unlockForArbitration();
3559 }
3560
3561 uint64_t IOService::getAccumulatedBusyTime( void )
3562 {
3563     uint64_t accumBusy = __accumBusy;
3564     uint64_t timeBusy = __timeBusy;
3565     uint64_t nano;
3566
3567     do
3568     {
3569         accumBusy = __accumBusy;
3570         timeBusy  = __timeBusy;
3571         if (timeBusy)
3572             accumBusy += mach_absolute_time() - timeBusy;
3573     }
3574     while (timeBusy != __timeBusy);
3575
3576     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3577
3578     return (nano);
3579 }
3580
3581 UInt32 IOService::getBusyState( void )
3582 {
3583     return( __state[1] & kIOServiceBusyStateMask );
3584 }
3585
3586 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3587                                   mach_timespec_t * timeout )
3588 {
3589     panic("waitForState");
3590     return (kIOReturnUnsupported);
3591 }
3592
3593 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3594                                    uint64_t timeout )
3595 {
3596     bool            wait;
3597     int             waitResult = THREAD_AWAKENED;
3598     bool            computeDeadline = true;
3599     AbsoluteTime    abstime;
3600
3601     do {
3602         lockForArbitration();
3603         IOLockLock( gIOServiceBusyLock );
3604         wait = (value != (__state[1] & mask));
3605         if( wait) {
3606             __state[1] |= kIOServiceBusyWaiterState;
3607             unlockForArbitration();
3608             if( timeout != UINT64_MAX ) {
3609                 if( computeDeadline ) {
3610                     AbsoluteTime  nsinterval;
3611                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3612                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3613                     computeDeadline = false;
3614                 }
3615                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3616             }
3617             else
3618                 assert_wait((event_t)this, THREAD_UNINT );
3619         } else
3620             unlockForArbitration();
3621         IOLockUnlock( gIOServiceBusyLock );
3622         if( wait)
3623             waitResult = thread_block(THREAD_CONTINUE_NULL);
3624
3625     } while( wait && (waitResult != THREAD_TIMED_OUT));
3626
3627     if( waitResult == THREAD_TIMED_OUT)
3628         return( kIOReturnTimeout );
3629     else
3630         return( kIOReturnSuccess );
3631 }
3632
3633 IOReturn IOService::waitQuiet( uint64_t timeout )
3634 {
3635         IOReturn ret;
3636     ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3637         if ((kIOReturnTimeout == ret) && (timeout >= 30000000000) && (kIOWaitQuietPanics & gIOKitDebug))
3638         {
3639                 panic("IOService 0x%llx (%s) busy timeout", getRegistryEntryID(), getName());
3640         }
3641         return (ret);
3642 }
3643
3644 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
3645 {
3646     uint64_t    timeoutNS;
3647
3648     if (timeout)
3649     {
3650         timeoutNS = timeout->tv_sec;
3651         timeoutNS *= kSecondScale;
3652         timeoutNS += timeout->tv_nsec;
3653     }
3654     else
3655         timeoutNS = UINT64_MAX;
3656
3657     return (waitQuiet(timeoutNS));
3658 }
3659
3660 bool IOService::serializeProperties( OSSerialize * s ) const
3661 {
3662 #if 0
3663     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
3664                 sizeof( __state), "__state");
3665 #endif
3666     return( super::serializeProperties(s) );
3667 }
3668
3669
3670 void _IOConfigThread::main(void * arg, wait_result_t result)
3671 {
3672     _IOConfigThread * self = (_IOConfigThread *) arg;
3673     _IOServiceJob * job;
3674     IOService   *   nub;
3675     bool            alive = true;
3676     kern_return_t   kr;
3677     thread_precedence_policy_data_t precedence = { -1 };
3678
3679     kr = thread_policy_set(current_thread(),
3680                             THREAD_PRECEDENCE_POLICY,
3681                             (thread_policy_t) &precedence,
3682                             THREAD_PRECEDENCE_POLICY_COUNT);
3683     if (KERN_SUCCESS != kr)
3684         IOLog("thread_policy_set(%d)\n", kr);
3685
3686     do {
3687
3688 //      randomDelay();
3689
3690         semaphore_wait( gJobsSemaphore );
3691
3692         IOTakeLock( gJobsLock );
3693         job = (_IOServiceJob *) gJobs->getFirstObject();
3694         job->retain();
3695         gJobs->removeObject(job);
3696         if( job) {
3697             gOutstandingJobs--;
3698 //          gNumConfigThreads--;        // we're out of service
3699             gNumWaitingThreads--;       // we're out of service
3700         }
3701         IOUnlock( gJobsLock );
3702
3703         if( job) {
3704
3705           nub = job->nub;
3706
3707           if( gIOKitDebug & kIOLogConfig)
3708             LOG("config(%p): starting on %s, %d\n",
3709                         OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
3710
3711           switch( job->type) {
3712
3713             case kMatchNubJob:
3714                 nub->doServiceMatch( job->options );
3715                 break;
3716
3717             default:
3718                 LOG("config(%p): strange type (%d)\n",
3719                         OBFUSCATE(IOThreadSelf()), job->type );
3720                 break;
3721             }
3722
3723             nub->release();
3724             job->release();
3725
3726             IOTakeLock( gJobsLock );
3727             alive = (gOutstandingJobs > gNumWaitingThreads);
3728             if( alive)
3729                 gNumWaitingThreads++;   // back in service
3730 //              gNumConfigThreads++;
3731             else {
3732                 if( 0 == --gNumConfigThreads) {
3733 //                    IOLog("MATCH IDLE\n");
3734                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
3735                 }
3736             }
3737             IOUnlock( gJobsLock );
3738         }
3739
3740     } while( alive );
3741
3742     if( gIOKitDebug & kIOLogConfig)
3743         LOG("config(%p): terminating\n", OBFUSCATE(IOThreadSelf()) );
3744
3745     self->release();
3746 }
3747
3748 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
3749 {
3750     bool            wait;
3751     int             waitResult = THREAD_AWAKENED;
3752     bool            computeDeadline = true;
3753     AbsoluteTime    deadline;
3754
3755     IOLockLock( gJobsLock );
3756     do {
3757         wait = (0 != gNumConfigThreads);
3758         if( wait) {
3759             if( msToWait) {
3760                 if( computeDeadline ) {
3761                     clock_interval_to_deadline(
3762                           msToWait, kMillisecondScale, &deadline );
3763                     computeDeadline = false;
3764                 }
3765                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
3766                                                                 deadline, THREAD_UNINT );
3767                    } else {
3768                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
3769                                                                 THREAD_UNINT );
3770                 }
3771         }
3772     } while( wait && (waitResult != THREAD_TIMED_OUT));
3773         IOLockUnlock( gJobsLock );
3774
3775     if( waitResult == THREAD_TIMED_OUT)
3776         return( kIOReturnTimeout );
3777     else
3778         return( kIOReturnSuccess );
3779 }
3780
3781 void _IOServiceJob::pingConfig( _IOServiceJob * job )
3782 {
3783     int         count;
3784     bool        create;
3785
3786     assert( job );
3787
3788     IOTakeLock( gJobsLock );
3789
3790     gOutstandingJobs++;
3791     gJobs->setLastObject( job );
3792
3793     count = gNumWaitingThreads;
3794 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
3795
3796     create = (  (gOutstandingJobs > count)
3797                 && (gNumConfigThreads < kMaxConfigThreads) );
3798     if( create) {
3799         gNumConfigThreads++;
3800         gNumWaitingThreads++;
3801     }
3802
3803     IOUnlock( gJobsLock );
3804
3805     job->release();
3806
3807     if( create) {
3808         if( gIOKitDebug & kIOLogConfig)
3809             LOG("config(%d): creating\n", gNumConfigThreads - 1);
3810         _IOConfigThread::configThread();
3811     }
3812
3813     semaphore_signal( gJobsSemaphore );
3814 }
3815
3816 struct IOServiceMatchContext
3817 {
3818     OSDictionary * table;
3819     OSObject *     result;
3820     uint32_t       options;
3821     uint32_t       state;
3822     uint32_t       count;
3823     uint32_t       done;
3824 };
3825
3826 bool IOService::instanceMatch(const OSObject * entry, void * context)
3827 {
3828     IOServiceMatchContext * ctx = (typeof(ctx)) context;
3829     IOService *    service = (typeof(service)) entry;
3830     OSDictionary * table   = ctx->table;
3831     uint32_t       options = ctx->options;
3832     uint32_t       state   = ctx->state;
3833     uint32_t       done;
3834     bool           match;
3835
3836     done = 0;
3837     do
3838     {
3839         match = ((state == (state & service->__state[0]))
3840                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
3841         if (!match) break;
3842         ctx->count += table->getCount();
3843         match = service->matchInternal(table, options, &done);
3844         ctx->done += done;
3845     }
3846     while (false);
3847     if (!match)
3848         return (false);
3849
3850     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
3851     {
3852         service->retain();
3853         ctx->result = service;
3854         return (true);
3855     }
3856     else if (!ctx->result)
3857     {
3858         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
3859     }
3860     else
3861     {
3862         ((OSSet *)ctx->result)->setObject(service);
3863     }
3864     return (false);
3865 }
3866
3867 // internal - call with gNotificationLock
3868 OSObject * IOService::copyExistingServices( OSDictionary * matching,
3869                  IOOptionBits inState, IOOptionBits options )
3870 {
3871     OSObject *   current = 0;
3872     OSIterator * iter;
3873     IOService *  service;
3874     OSObject *   obj;
3875     OSString *   str;
3876
3877     if( !matching)
3878         return( 0 );
3879
3880 #if MATCH_DEBUG
3881     OSSerialize * s = OSSerialize::withCapacity(128);
3882     matching->serialize(s);
3883 #endif
3884
3885     if((obj = matching->getObject(gIOProviderClassKey))
3886       && gIOResourcesKey
3887       && gIOResourcesKey->isEqualTo(obj)
3888       && (service = gIOResources))
3889     {
3890         if( (inState == (service->__state[0] & inState))
3891           && (0 == (service->__state[0] & kIOServiceInactiveState))
3892           &&  service->matchPassive(matching, options))
3893         {
3894             if( options & kIONotifyOnce)
3895             {
3896                 service->retain();
3897                 current = service;
3898             }
3899             else
3900                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
3901         }
3902     }
3903     else
3904     {
3905         IOServiceMatchContext ctx;
3906         ctx.table   = matching;
3907         ctx.state   = inState;
3908         ctx.count   = 0;
3909         ctx.done    = 0;
3910         ctx.options = options;
3911         ctx.result  = 0;
3912
3913         if ((str = OSDynamicCast(OSString, obj)))
3914         {
3915             const OSSymbol * sym = OSSymbol::withString(str);
3916             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
3917             sym->release();
3918         }
3919         else
3920         {
3921             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
3922         }
3923
3924
3925         current = ctx.result;
3926
3927         options |= kIOServiceInternalDone | kIOServiceClassDone;
3928         if (current && (ctx.done != ctx.count))
3929         {
3930             OSSet *
3931             source = OSDynamicCast(OSSet, current);
3932             current = 0;
3933             while ((service = (IOService *) source->getAnyObject()))
3934             {
3935                 if (service->matchPassive(matching, options))
3936                 {
3937                     if( options & kIONotifyOnce)
3938                     {
3939                         service->retain();
3940                         current = service;
3941                         break;
3942                     }
3943                     if( current)
3944                     {
3945                         ((OSSet *)current)->setObject( service );
3946                     }
3947                     else
3948                     {
3949                         current = OSSet::withObjects(
3950                                         (const OSObject **) &service, 1, 1 );
3951                     }
3952                 }
3953                 source->removeObject(service);
3954             }
3955             source->release();
3956         }
3957     }
3958
3959 #if MATCH_DEBUG
3960     {
3961         OSObject * _current = 0;
3962
3963         iter = IORegistryIterator::iterateOver( gIOServicePlane,
3964                                             kIORegistryIterateRecursively );
3965         if( iter) {
3966             do {
3967                 iter->reset();
3968                 while( (service = (IOService *) iter->getNextObject())) {
3969                     if( (inState == (service->__state[0] & inState))
3970                     && (0 == (service->__state[0] & kIOServiceInactiveState))
3971                     &&  service->matchPassive(matching, 0)) {
3972
3973                         if( options & kIONotifyOnce) {
3974                             service->retain();
3975                             _current = service;
3976                             break;
3977                         }
3978                         if( _current)
3979                             ((OSSet *)_current)->setObject( service );
3980                         else
3981                             _current = OSSet::withObjects(
3982                                             (const OSObject **) &service, 1, 1 );
3983                     }
3984                 }
3985             } while( !service && !iter->isValid());
3986             iter->release();
3987         }
3988
3989
3990         if ( ((current != 0) != (_current != 0))
3991         || (current && _current && !current->isEqualTo(_current)))
3992         {
3993             OSSerialize * s1 = OSSerialize::withCapacity(128);
3994             OSSerialize * s2 = OSSerialize::withCapacity(128);
3995             current->serialize(s1);
3996             _current->serialize(s2);
3997             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", OBFUSCATE(current),
3998                 OBFUSCATE(_current), s->text(), s1->text(), s2->text());
3999             s1->release();
4000             s2->release();
4001         }
4002
4003         if (_current) _current->release();
4004     }
4005
4006     s->release();
4007 #endif
4008
4009     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4010         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4011         current->release();
4012         current = iter;
4013     }
4014
4015     return( current );
4016 }
4017
4018 // public version
4019 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4020 {
4021     OSIterator *        iter;
4022
4023     // is a lock even needed?
4024     LOCKWRITENOTIFY();
4025
4026     iter = (OSIterator *) copyExistingServices( matching,
4027                                                 kIOServiceMatchedState );
4028
4029     UNLOCKNOTIFY();
4030
4031     return( iter );
4032 }
4033
4034 IOService * IOService::copyMatchingService( OSDictionary * matching )
4035 {
4036     IOService * service;
4037
4038     // is a lock even needed?
4039     LOCKWRITENOTIFY();
4040
4041     service = (IOService *) copyExistingServices( matching,
4042                                                 kIOServiceMatchedState, kIONotifyOnce );
4043
4044     UNLOCKNOTIFY();
4045
4046     return( service );
4047 }
4048
4049 struct _IOServiceMatchingNotificationHandlerRef
4050 {
4051     IOServiceNotificationHandler handler;
4052     void * ref;
4053 };
4054
4055 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4056                                                    IOService * newService,
4057                                                    IONotifier * notifier )
4058 {
4059     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4060 }
4061
4062 // internal - call with gNotificationLock
4063 IONotifier * IOService::setNotification(
4064             const OSSymbol * type, OSDictionary * matching,
4065             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4066             SInt32 priority )
4067 {
4068     _IOServiceNotifier * notify = 0;
4069     OSOrderedSet *      set;
4070
4071     if( !matching)
4072         return( 0 );
4073
4074     notify = new _IOServiceNotifier;
4075     if( notify && !notify->init()) {
4076         notify->release();
4077         notify = 0;
4078     }
4079
4080     if( notify) {
4081         notify->handler = handler;
4082         notify->target = target;
4083         notify->matching = matching;
4084         matching->retain();
4085         if (handler == &_IOServiceMatchingNotificationHandler)
4086         {
4087             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4088             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4089         }
4090         else
4091             notify->ref = ref;
4092         notify->priority = priority;
4093         notify->state = kIOServiceNotifyEnable;
4094         queue_init( &notify->handlerInvocations );
4095
4096         ////// queue
4097
4098         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4099             set = OSOrderedSet::withCapacity( 1,
4100                         IONotifyOrdering, 0 );
4101             if( set) {
4102                 gNotifications->setObject( type, set );
4103                 set->release();
4104             }
4105         }
4106         notify->whence = set;
4107         if( set)
4108             set->setObject( notify );
4109     }
4110
4111     return( notify );
4112 }
4113
4114 // internal - call with gNotificationLock
4115 IONotifier * IOService::doInstallNotification(
4116                         const OSSymbol * type, OSDictionary * matching,
4117                         IOServiceMatchingNotificationHandler handler,
4118                         void * target, void * ref,
4119                         SInt32 priority, OSIterator ** existing )
4120 {
4121     OSIterator *        exist;
4122     IONotifier *        notify;
4123     IOOptionBits        inState;
4124
4125     if( !matching)
4126         return( 0 );
4127
4128     if( type == gIOPublishNotification)
4129         inState = kIOServiceRegisteredState;
4130
4131     else if( type == gIOFirstPublishNotification)
4132         inState = kIOServiceFirstPublishState;
4133
4134     else if( (type == gIOMatchedNotification)
4135           || (type == gIOFirstMatchNotification))
4136         inState = kIOServiceMatchedState;
4137     else if( type == gIOTerminatedNotification)
4138         inState = 0;
4139     else
4140         return( 0 );
4141
4142     notify = setNotification( type, matching, handler, target, ref, priority );
4143
4144     if( inState)
4145         // get the current set
4146         exist = (OSIterator *) copyExistingServices( matching, inState );
4147     else
4148         exist = 0;
4149
4150     *existing = exist;
4151
4152     return( notify );
4153 }
4154
4155 #if !defined(__LP64__)
4156 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4157                                         IOServiceNotificationHandler handler,
4158                                         void * target, void * refCon,
4159                                         SInt32 priority, OSIterator ** existing )
4160 {
4161     IONotifier * result;
4162     _IOServiceMatchingNotificationHandlerRef ref;
4163     ref.handler = handler;
4164     ref.ref     = refCon;
4165
4166     result = (_IOServiceNotifier *) installNotification( type, matching,
4167                          &_IOServiceMatchingNotificationHandler,
4168                         target, &ref, priority, existing );
4169     if (result)
4170         matching->release();
4171
4172     return (result);
4173 }
4174 #endif /* !defined(__LP64__) */
4175
4176
4177 IONotifier * IOService::installNotification(
4178                         const OSSymbol * type, OSDictionary * matching,
4179                         IOServiceMatchingNotificationHandler handler,
4180                         void * target, void * ref,
4181                         SInt32 priority, OSIterator ** existing )
4182 {
4183     IONotifier * notify;
4184
4185     LOCKWRITENOTIFY();
4186
4187     notify = doInstallNotification( type, matching, handler, target, ref,
4188                 priority, existing );
4189
4190     UNLOCKNOTIFY();
4191
4192     return( notify );
4193 }
4194
4195 IONotifier * IOService::addNotification(
4196                         const OSSymbol * type, OSDictionary * matching,
4197                         IOServiceNotificationHandler handler,
4198                         void * target, void * refCon,
4199                         SInt32 priority )
4200 {
4201     IONotifier * result;
4202     _IOServiceMatchingNotificationHandlerRef ref;
4203
4204     ref.handler = handler;
4205     ref.ref     = refCon;
4206
4207     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4208                             target, &ref, priority);
4209
4210     if (result)
4211         matching->release();
4212
4213     return (result);
4214 }
4215
4216 IONotifier * IOService::addMatchingNotification(
4217                         const OSSymbol * type, OSDictionary * matching,
4218                         IOServiceMatchingNotificationHandler handler,
4219                         void * target, void * ref,
4220                         SInt32 priority )
4221 {
4222     OSIterator *                existing = NULL;
4223     _IOServiceNotifier *        notify;
4224     IOService *                 next;
4225
4226     notify = (_IOServiceNotifier *) installNotification( type, matching,
4227                 handler, target, ref, priority, &existing );
4228
4229     // send notifications for existing set
4230     if( existing) {
4231
4232         notify->retain();               // in case handler remove()s
4233         while( (next = (IOService *) existing->getNextObject())) {
4234
4235             next->lockForArbitration();
4236             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4237                 next->invokeNotifer( notify );
4238             next->unlockForArbitration();
4239         }
4240         notify->release();
4241         existing->release();
4242     }
4243
4244     return( notify );
4245 }
4246
4247 bool IOService::syncNotificationHandler(
4248                         void * /* target */, void * ref,
4249                         IOService * newService,
4250                         IONotifier * notifier )
4251 {
4252
4253     LOCKWRITENOTIFY();
4254     if (!*((IOService **) ref))
4255     {
4256         newService->retain();
4257         (*(IOService **) ref) = newService;
4258         WAKEUPNOTIFY(ref);
4259     }
4260     UNLOCKNOTIFY();
4261
4262     return( false );
4263 }
4264
4265 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4266                                                 uint64_t timeout)
4267 {
4268     IONotifier *        notify = 0;
4269     // priority doesn't help us much since we need a thread wakeup
4270     SInt32              priority = 0;
4271     IOService *         result;
4272
4273     if (!matching)
4274         return( 0 );
4275
4276     result = NULL;
4277
4278     LOCKWRITENOTIFY();
4279     do
4280     {
4281         result = (IOService *) copyExistingServices( matching,
4282                             kIOServiceMatchedState, kIONotifyOnce );
4283         if (result)
4284             break;
4285         notify = IOService::setNotification( gIOMatchedNotification, matching,
4286                     &IOService::syncNotificationHandler, (void *) 0,
4287                     &result, priority );
4288          if (!notify)
4289             break;
4290         if (UINT64_MAX != timeout)
4291         {
4292             AbsoluteTime deadline;
4293             nanoseconds_to_absolutetime(timeout, &deadline);
4294             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4295             SLEEPNOTIFYTO(&result, deadline);
4296         }
4297         else
4298         {
4299             SLEEPNOTIFY(&result);
4300         }
4301     }
4302     while( false );
4303
4304     UNLOCKNOTIFY();
4305
4306     if (notify)
4307         notify->remove();       // dequeues
4308
4309     return( result );
4310 }
4311
4312 IOService * IOService::waitForService( OSDictionary * matching,
4313                                         mach_timespec_t * timeout )
4314 {
4315     IOService * result;
4316     uint64_t    timeoutNS;
4317
4318     if (timeout)
4319     {
4320         timeoutNS = timeout->tv_sec;
4321         timeoutNS *= kSecondScale;
4322         timeoutNS += timeout->tv_nsec;
4323     }
4324     else
4325         timeoutNS = UINT64_MAX;
4326
4327     result = waitForMatchingService(matching, timeoutNS);
4328
4329     matching->release();
4330     if (result)
4331         result->release();
4332
4333     return (result);
4334 }
4335
4336 void IOService::deliverNotification( const OSSymbol * type,
4337                             IOOptionBits orNewState, IOOptionBits andNewState )
4338 {
4339     _IOServiceNotifier * notify;
4340     OSIterator *         iter;
4341     OSArray *            willSend = 0;
4342
4343     lockForArbitration();
4344
4345     if( (0 == (__state[0] & kIOServiceInactiveState))
4346      || (type == gIOTerminatedNotification)) {
4347
4348         LOCKREADNOTIFY();
4349
4350         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4351                     gNotifications->getObject( type ) );
4352
4353         if( iter) {
4354             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4355
4356                 if( matchPassive(notify->matching, 0)
4357                   && (kIOServiceNotifyEnable & notify->state)) {
4358                     if( 0 == willSend)
4359                         willSend = OSArray::withCapacity(8);
4360                     if( willSend)
4361                         willSend->setObject( notify );
4362                 }
4363             }
4364             iter->release();
4365         }
4366
4367         __state[0] = (__state[0] | orNewState) & andNewState;
4368
4369         UNLOCKNOTIFY();
4370     }
4371
4372     if( willSend) {
4373         for( unsigned int idx = 0;
4374              (notify = (_IOServiceNotifier *) willSend->getObject(idx));
4375              idx++) {
4376             invokeNotifer( notify );
4377         }
4378         willSend->release();
4379     }
4380     unlockForArbitration();
4381 }
4382
4383 IOOptionBits IOService::getState( void ) const
4384 {
4385     return( __state[0] );
4386 }
4387
4388 /*
4389  * Helpers to make matching objects for simple cases
4390  */
4391
4392 OSDictionary * IOService::serviceMatching( const OSString * name,
4393                         OSDictionary * table )
4394 {
4395
4396     const OSString *    str;
4397
4398     str = OSSymbol::withString(name);
4399     if( !str)
4400         return( 0 );
4401
4402     if( !table)
4403         table = OSDictionary::withCapacity( 2 );
4404     if( table)
4405         table->setObject(gIOProviderClassKey, (OSObject *)str );
4406     str->release();
4407
4408     return( table );
4409 }
4410
4411 OSDictionary * IOService::serviceMatching( const char * name,
4412                         OSDictionary * table )
4413 {
4414     const OSString *    str;
4415
4416     str = OSSymbol::withCString( name );
4417     if( !str)
4418         return( 0 );
4419
4420     table = serviceMatching( str, table );
4421     str->release();
4422     return( table );
4423 }
4424
4425 OSDictionary * IOService::nameMatching( const OSString * name,
4426                         OSDictionary * table )
4427 {
4428     if( !table)
4429         table = OSDictionary::withCapacity( 2 );
4430     if( table)
4431         table->setObject( gIONameMatchKey, (OSObject *)name );
4432
4433     return( table );
4434 }
4435
4436 OSDictionary * IOService::nameMatching( const char * name,
4437                         OSDictionary * table )
4438 {
4439     const OSString *    str;
4440
4441     str = OSSymbol::withCString( name );
4442     if( !str)
4443         return( 0 );
4444
4445     table = nameMatching( str, table );
4446     str->release();
4447     return( table );
4448 }
4449
4450 OSDictionary * IOService::resourceMatching( const OSString * str,
4451                         OSDictionary * table )
4452 {
4453     table = serviceMatching( gIOResourcesKey, table );
4454     if( table)
4455         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4456
4457     return( table );
4458 }
4459
4460 OSDictionary * IOService::resourceMatching( const char * name,
4461                         OSDictionary * table )
4462 {
4463     const OSSymbol *    str;
4464
4465     str = OSSymbol::withCString( name );
4466     if( !str)
4467         return( 0 );
4468
4469     table = resourceMatching( str, table );
4470     str->release();
4471
4472     return( table );
4473 }
4474
4475 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4476                         OSDictionary * table )
4477 {
4478     OSDictionary * properties;
4479
4480     properties = OSDictionary::withCapacity( 2 );
4481     if( !properties)
4482         return( 0 );
4483     properties->setObject( key, value );
4484
4485     if( !table)
4486         table = OSDictionary::withCapacity( 2 );
4487     if( table)
4488         table->setObject( gIOPropertyMatchKey, properties );
4489
4490     properties->release();
4491
4492     return( table );
4493 }
4494
4495 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4496                         OSDictionary * table )
4497 {
4498     OSNumber *     num;
4499
4500     num = OSNumber::withNumber( entryID, 64 );
4501     if( !num)
4502         return( 0 );
4503
4504     if( !table)
4505         table = OSDictionary::withCapacity( 2 );
4506     if( table)
4507         table->setObject( gIORegistryEntryIDKey, num );
4508
4509     if (num)
4510         num->release();
4511
4512     return( table );
4513 }
4514
4515
4516 /*
4517  * _IOServiceNotifier
4518  */
4519
4520 // wait for all threads, other than the current one,
4521 //  to exit the handler
4522
4523 void _IOServiceNotifier::wait()
4524 {
4525     _IOServiceNotifierInvocation * next;
4526     bool doWait;
4527
4528     do {
4529         doWait = false;
4530         queue_iterate( &handlerInvocations, next,
4531                         _IOServiceNotifierInvocation *, link) {
4532             if( next->thread != current_thread() ) {
4533                 doWait = true;
4534                 break;
4535             }
4536         }
4537         if( doWait) {
4538             state |= kIOServiceNotifyWaiter;
4539             SLEEPNOTIFY(this);
4540         }
4541
4542     } while( doWait );
4543 }
4544
4545 void _IOServiceNotifier::free()
4546 {
4547     assert( queue_empty( &handlerInvocations ));
4548     OSObject::free();
4549 }
4550
4551 void _IOServiceNotifier::remove()
4552 {
4553     LOCKWRITENOTIFY();
4554
4555     if( whence) {
4556         whence->removeObject( (OSObject *) this );
4557         whence = 0;
4558     }
4559     if( matching) {
4560         matching->release();
4561         matching = 0;
4562     }
4563
4564     state &= ~kIOServiceNotifyEnable;
4565
4566     wait();
4567
4568     UNLOCKNOTIFY();
4569
4570     release();
4571 }
4572
4573 bool _IOServiceNotifier::disable()
4574 {
4575     bool        ret;
4576
4577     LOCKWRITENOTIFY();
4578
4579     ret = (0 != (kIOServiceNotifyEnable & state));
4580     state &= ~kIOServiceNotifyEnable;
4581     if( ret)
4582         wait();
4583
4584     UNLOCKNOTIFY();
4585
4586     return( ret );
4587 }
4588
4589 void _IOServiceNotifier::enable( bool was )
4590 {
4591     LOCKWRITENOTIFY();
4592     if( was)
4593         state |= kIOServiceNotifyEnable;
4594     else
4595         state &= ~kIOServiceNotifyEnable;
4596     UNLOCKNOTIFY();
4597 }
4598
4599 /*
4600  * IOResources
4601  */
4602
4603 IOService * IOResources::resources( void )
4604 {
4605     IOResources *       inst;
4606
4607     inst = new IOResources;
4608     if( inst && !inst->init()) {
4609         inst->release();
4610         inst = 0;
4611     }
4612
4613     return( inst );
4614 }
4615
4616 bool IOResources::init( OSDictionary * dictionary )
4617 {
4618     // Do super init first
4619     if ( !super::init() )
4620         return false;
4621
4622     // Allow PAL layer to publish a value
4623     const char *property_name;
4624     int property_value;
4625
4626     pal_get_resource_property( &property_name, &property_value );
4627
4628     if( property_name ) {
4629         OSNumber *num;
4630         const OSSymbol *        sym;
4631
4632         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
4633             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
4634                 this->setProperty( sym, num );
4635                 sym->release();
4636             }
4637             num->release();
4638         }
4639     }
4640
4641     return true;
4642 }
4643
4644 IOWorkLoop * IOResources::getWorkLoop() const
4645 {
4646     // If we are the resource root
4647     // then use the platform's workloop
4648     if (this == (IOResources *) gIOResources)
4649         return getPlatform()->getWorkLoop();
4650     else
4651         return IOService::getWorkLoop();
4652 }
4653
4654 bool IOResources::matchPropertyTable( OSDictionary * table )
4655 {
4656     OSObject *          prop;
4657     OSString *          str;
4658     OSSet *             set;
4659     OSIterator *        iter;
4660     bool                ok = true;
4661
4662     prop = table->getObject( gIOResourceMatchKey );
4663     str = OSDynamicCast( OSString, prop );
4664     if( str)
4665         ok = (0 != getProperty( str ));
4666
4667     else if( (set = OSDynamicCast( OSSet, prop))) {
4668
4669         iter = OSCollectionIterator::withCollection( set );
4670         ok = (iter != 0);
4671         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
4672             ok = (0 != getProperty( str ));
4673
4674         if( iter)
4675             iter->release();
4676     }
4677
4678     return( ok );
4679 }
4680
4681 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
4682 {
4683     IOService::updateConsoleUsers(NULL, 0);
4684 }
4685
4686 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
4687 {
4688     IORegistryEntry * regEntry;
4689     OSObject *        locked = kOSBooleanFalse;
4690     uint32_t          idx;
4691     bool              publish;
4692     OSDictionary *    user;
4693     static IOMessage  sSystemPower;
4694
4695     regEntry = IORegistryEntry::getRegistryRoot();
4696
4697     if (!gIOChosenEntry)
4698         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
4699
4700     IOLockLock(gIOConsoleUsersLock);
4701
4702     if (systemMessage)
4703     {
4704         sSystemPower = systemMessage;
4705 #if HIBERNATION
4706         if ((kIOMessageSystemHasPoweredOn == systemMessage) && IOHibernateWasScreenLocked())
4707         {
4708             locked = kOSBooleanTrue;
4709         }
4710 #endif /* HIBERNATION */
4711     }
4712
4713     if (consoleUsers)
4714     {
4715         OSNumber * num = 0;
4716         gIOConsoleLoggedIn = false;
4717         for (idx = 0;
4718               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
4719               idx++)
4720         {
4721             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
4722                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
4723             if (!num)
4724             {
4725                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
4726             }
4727         }
4728         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
4729     }
4730
4731     if (!gIOConsoleLoggedIn
4732      || (kIOMessageSystemWillSleep == sSystemPower)
4733      || (kIOMessageSystemPagingOff == sSystemPower))
4734     {
4735         locked = kOSBooleanTrue;
4736     }
4737     else if (gIOConsoleLockTime)
4738     {
4739         clock_sec_t  now;
4740         clock_usec_t microsecs;
4741
4742         clock_get_calendar_microtime(&now, &microsecs);
4743         if (gIOConsoleLockTime > now)
4744         {
4745             AbsoluteTime deadline;
4746             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
4747             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
4748         }
4749         else
4750         {
4751             locked = kOSBooleanTrue;
4752         }
4753     }
4754
4755     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
4756     if (publish)
4757     {
4758         regEntry->setProperty(gIOConsoleLockedKey, locked);
4759         if (consoleUsers)
4760         {
4761             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
4762         }
4763         OSIncrementAtomic( &gIOConsoleUsersSeed );
4764     }
4765
4766 #if HIBERNATION
4767     if (gIOChosenEntry)
4768     {
4769         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
4770         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
4771         else                          gIOScreenLockState = kIOScreenLockNoLock;
4772         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
4773     }
4774 #endif /* HIBERNATION */
4775
4776     IOLockUnlock(gIOConsoleUsersLock);
4777
4778     if (publish)
4779     {
4780         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
4781
4782         MessageClientsContext context;
4783
4784         context.service  = getServiceRoot();
4785         context.type     = kIOMessageConsoleSecurityChange;
4786         context.argument = (void *) regEntry;
4787         context.argSize  = 0;
4788
4789         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
4790                                  &messageClientsApplier, &context );
4791     }
4792 }
4793
4794 IOReturn IOResources::setProperties( OSObject * properties )
4795 {
4796     IOReturn                    err;
4797     const OSSymbol *            key;
4798     OSDictionary *              dict;
4799     OSCollectionIterator *      iter;
4800
4801     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
4802     if ( kIOReturnSuccess != err)
4803         return( err );
4804
4805     dict = OSDynamicCast(OSDictionary, properties);
4806     if( 0 == dict)
4807         return( kIOReturnBadArgument);
4808
4809     iter = OSCollectionIterator::withCollection( dict);
4810     if( 0 == iter)
4811         return( kIOReturnBadArgument);
4812
4813     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
4814     {
4815         if (gIOConsoleUsersKey == key) do
4816         {
4817             OSArray * consoleUsers;
4818             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
4819             if (!consoleUsers)
4820                 continue;
4821             IOService::updateConsoleUsers(consoleUsers, 0);
4822         }
4823         while (false);
4824
4825         publishResource( key, dict->getObject(key) );
4826     }
4827
4828     iter->release();
4829
4830     return( kIOReturnSuccess );
4831 }
4832
4833 /*
4834  * Helpers for matching dictionaries.
4835  * Keys existing in matching are checked in properties.
4836  * Keys may be a string or OSCollection of IOStrings
4837  */
4838
4839 bool IOService::compareProperty( OSDictionary * matching,
4840                                  const char *   key )
4841 {
4842     OSObject *  value;
4843     bool        ok;
4844
4845     value = matching->getObject( key );
4846     if( value)
4847         ok = value->isEqualTo( getProperty( key ));
4848     else
4849         ok = true;
4850
4851     return( ok );
4852 }
4853
4854
4855 bool IOService::compareProperty( OSDictionary *   matching,
4856                                  const OSString * key )
4857 {
4858     OSObject *  value;
4859     bool        ok;
4860
4861     value = matching->getObject( key );
4862     if( value)
4863         ok = value->isEqualTo( getProperty( key ));
4864     else
4865         ok = true;
4866
4867     return( ok );
4868 }
4869
4870 bool IOService::compareProperties( OSDictionary * matching,
4871                                    OSCollection * keys )
4872 {
4873     OSCollectionIterator *      iter;
4874     const OSString *            key;
4875     bool                        ok = true;
4876
4877     if( !matching || !keys)
4878         return( false );
4879
4880     iter = OSCollectionIterator::withCollection( keys );
4881
4882     if( iter) {
4883         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
4884             ok = compareProperty( matching, key );
4885
4886         iter->release();
4887     }
4888     keys->release();    // !! consume a ref !!
4889
4890     return( ok );
4891 }
4892
4893 /* Helper to add a location matching dict to the table */
4894
4895 OSDictionary * IOService::addLocation( OSDictionary * table )
4896 {
4897     OSDictionary *      dict;
4898
4899     if( !table)
4900         return( 0 );
4901
4902     dict = OSDictionary::withCapacity( 1 );
4903     if( dict) {
4904         table->setObject( gIOLocationMatchKey, dict );
4905         dict->release();
4906     }
4907
4908     return( dict );
4909 }
4910
4911 /*
4912  * Go looking for a provider to match a location dict.
4913  */
4914
4915 IOService * IOService::matchLocation( IOService * /* client */ )
4916 {
4917     IOService * parent;
4918
4919     parent = getProvider();
4920
4921     if( parent)
4922         parent = parent->matchLocation( this );
4923
4924     return( parent );
4925 }
4926
4927 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
4928 {
4929     OSString *          matched;
4930     OSObject *          obj;
4931     OSString *          str;
4932     IORegistryEntry *   entry;
4933     OSNumber *          num;
4934     bool                match = true;
4935     bool                changesOK = (0 != (kIOServiceChangesOK & options));
4936     uint32_t            count;
4937     uint32_t            done;
4938
4939     do
4940     {
4941         count = table->getCount();
4942         done = 0;
4943         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
4944         if (str) {
4945             done++;
4946             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
4947 #if MATCH_DEBUG
4948             match = (0 != metaCast( str ));
4949             if ((kIOServiceClassDone & options) && !match) panic("classDone");
4950 #endif
4951             if ((!match) || (done == count)) break;
4952         }
4953
4954         obj = table->getObject( gIONameMatchKey );
4955         if( obj) {
4956             done++;
4957             match = compareNames( obj, changesOK ? &matched : 0 );
4958             if (!match) break;
4959             if( changesOK && matched) {
4960                 // leave a hint as to which name matched
4961                 table->setObject( gIONameMatchedKey, matched );
4962                 matched->release();
4963             }
4964             if (done == count) break;
4965         }
4966
4967         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
4968         if (str)
4969         {
4970             const OSSymbol * sym;
4971             done++;
4972             match = false;
4973             sym = copyLocation();
4974             if (sym) {
4975                 match = sym->isEqualTo( str );
4976                 sym->release();
4977             }
4978             if ((!match) || (done == count)) break;
4979         }
4980
4981         obj = table->getObject( gIOPropertyMatchKey );
4982         if( obj)
4983         {
4984             OSDictionary * dict;
4985             OSDictionary * nextDict;
4986             OSIterator *   iter;
4987             done++;
4988             match = false;
4989             dict = dictionaryWithProperties();
4990             if( dict) {
4991                 nextDict = OSDynamicCast( OSDictionary, obj);
4992                 if( nextDict)
4993                     iter = 0;
4994                 else
4995                     iter = OSCollectionIterator::withCollection(
4996                                 OSDynamicCast(OSCollection, obj));
4997
4998                 while( nextDict
4999                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5000                                             iter->getNextObject()))))) {
5001                     match = dict->isEqualTo( nextDict, nextDict);
5002                     if( match)
5003                         break;
5004                     nextDict = 0;
5005                 }
5006                 dict->release();
5007                 if( iter)
5008                     iter->release();
5009             }
5010             if ((!match) || (done == count)) break;
5011         }
5012
5013         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5014         if( str) {
5015             done++;
5016             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5017             match = (this == entry);
5018             if( entry)
5019                 entry->release();
5020             if ((!match) || (done == count)) break;
5021         }
5022
5023         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5024         if (num) {
5025             done++;
5026             match = (getRegistryEntryID() == num->unsigned64BitValue());
5027             if ((!match) || (done == count)) break;
5028         }
5029
5030         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5031         if( num)
5032         {
5033             OSIterator *        iter;
5034             IOService *         service = 0;
5035             UInt32              serviceCount = 0;
5036
5037             done++;
5038             iter = getClientIterator();
5039             if( iter) {
5040                 while( (service = (IOService *) iter->getNextObject())) {
5041                     if( kIOServiceInactiveState & service->__state[0])
5042                         continue;
5043                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5044                         continue;
5045                     ++serviceCount;
5046                 }
5047                 iter->release();
5048             }
5049             match = (serviceCount == num->unsigned32BitValue());
5050             if ((!match) || (done == count)) break;
5051         }
5052
5053 #define propMatch(key)                                  \
5054         obj = table->getObject(key);                    \
5055         if (obj)                                        \
5056         {                                               \
5057             OSObject * prop;                            \
5058             done++;                                     \
5059             prop = copyProperty(key);                   \
5060             match = obj->isEqualTo(prop);               \
5061             if (prop) prop->release();                  \
5062             if ((!match) || (done == count)) break;     \
5063         }
5064         propMatch(kIOBSDNameKey)
5065         propMatch(kIOBSDMajorKey)
5066         propMatch(kIOBSDMinorKey)
5067         propMatch(kIOBSDUnitKey)
5068 #undef propMatch
5069     }
5070     while (false);
5071
5072     if (did) *did = done;
5073     return (match);
5074 }
5075
5076 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5077 {
5078     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5079 }
5080
5081 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5082 {
5083     IOService *         where;
5084     OSDictionary *      nextTable;
5085     SInt32              score;
5086     OSNumber *          newPri;
5087     bool                match = true;
5088     bool                matchParent = false;
5089     uint32_t            count;
5090     uint32_t            done;
5091
5092     assert( table );
5093
5094 #if MATCH_DEBUG
5095     OSDictionary * root = table;
5096 #endif
5097
5098     where = this;
5099     do
5100     {
5101         do
5102         {
5103             count = table->getCount();
5104             if (!(kIOServiceInternalDone & options))
5105             {
5106                 match = where->matchInternal(table, options, &done);
5107                 // don't call family if we've done all the entries in the table
5108                 if ((!match) || (done == count)) break;
5109             }
5110
5111             // pass in score from property table
5112             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5113
5114             // do family specific matching
5115             match = where->matchPropertyTable( table, &score );
5116
5117             if( !match) {
5118 #if IOMATCHDEBUG
5119                 if( kIOLogMatch & getDebugFlags( table ))
5120                     LOG("%s: family specific matching fails\n", where->getName());
5121 #endif
5122                 break;
5123             }
5124
5125             if (kIOServiceChangesOK & options) {
5126                 // save the score
5127                 newPri = OSNumber::withNumber( score, 32 );
5128                 if( newPri) {
5129                     table->setObject( gIOProbeScoreKey, newPri );
5130                     newPri->release();
5131                 }
5132             }
5133
5134             options = 0;
5135             matchParent = false;
5136
5137             nextTable = OSDynamicCast(OSDictionary,
5138                   table->getObject( gIOParentMatchKey ));
5139             if( nextTable) {
5140                 // look for a matching entry anywhere up to root
5141                 match = false;
5142                 matchParent = true;
5143                 table = nextTable;
5144                 break;
5145             }
5146
5147             table = OSDynamicCast(OSDictionary,
5148                     table->getObject( gIOLocationMatchKey ));
5149             if (table) {
5150                 // look for a matching entry at matchLocation()
5151                 match = false;
5152                 where = where->getProvider();
5153                 if (where && (where = where->matchLocation(where))) continue;
5154             }
5155             break;
5156         }
5157         while (true);
5158     }
5159     while( matchParent && (!match) && (where = where->getProvider()) );
5160
5161 #if MATCH_DEBUG
5162     if (where != this)
5163     {
5164         OSSerialize * s = OSSerialize::withCapacity(128);
5165         root->serialize(s);
5166         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5167         s->release();
5168     }
5169 #endif
5170
5171     return( match );
5172 }
5173
5174
5175 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5176                                     UInt32 type,  OSDictionary * properties,
5177                                     IOUserClient ** handler )
5178 {
5179     const OSSymbol *userClientClass = 0;
5180     IOUserClient *client;
5181     OSObject *temp;
5182
5183     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5184         return kIOReturnSuccess;
5185
5186     // First try my own properties for a user client class name
5187     temp = getProperty(gIOUserClientClassKey);
5188     if (temp) {
5189         if (OSDynamicCast(OSSymbol, temp))
5190             userClientClass = (const OSSymbol *) temp;
5191         else if (OSDynamicCast(OSString, temp)) {
5192             userClientClass = OSSymbol::withString((OSString *) temp);
5193             if (userClientClass)
5194                 setProperty(kIOUserClientClassKey,
5195                             (OSObject *) userClientClass);
5196         }
5197     }
5198
5199     // Didn't find one so lets just bomb out now without further ado.
5200     if (!userClientClass)
5201         return kIOReturnUnsupported;
5202
5203     // This reference is consumed by the IOServiceOpen call
5204     temp = OSMetaClass::allocClassWithName(userClientClass);
5205     if (!temp)
5206         return kIOReturnNoMemory;
5207
5208     if (OSDynamicCast(IOUserClient, temp))
5209         client = (IOUserClient *) temp;
5210     else {
5211         temp->release();
5212         return kIOReturnUnsupported;
5213     }
5214
5215     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5216         client->release();
5217         return kIOReturnBadArgument;
5218     }
5219
5220     if ( !client->attach(this) ) {
5221         client->release();
5222         return kIOReturnUnsupported;
5223     }
5224
5225     if ( !client->start(this) ) {
5226         client->detach(this);
5227         client->release();
5228         return kIOReturnUnsupported;
5229     }
5230
5231     *handler = client;
5232     return kIOReturnSuccess;
5233 }
5234
5235 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5236                                     UInt32 type, IOUserClient ** handler )
5237 {
5238     return( kIOReturnUnsupported );
5239 }
5240
5241 IOReturn IOService::requestProbe( IOOptionBits options )
5242 {
5243     return( kIOReturnUnsupported);
5244 }
5245
5246 /*
5247  * Convert an IOReturn to text. Subclasses which add additional
5248  * IOReturn's should override this method and call
5249  * super::stringFromReturn if the desired value is not found.
5250  */
5251
5252 const char * IOService::stringFromReturn( IOReturn rtn )
5253 {
5254     static const IONamedValue IOReturn_values[] = {
5255         {kIOReturnSuccess,          "success"                           },
5256         {kIOReturnError,            "general error"                     },
5257         {kIOReturnNoMemory,         "memory allocation error"           },
5258         {kIOReturnNoResources,      "resource shortage"                 },
5259         {kIOReturnIPCError,         "Mach IPC failure"                  },
5260         {kIOReturnNoDevice,         "no such device"                    },
5261         {kIOReturnNotPrivileged,    "privilege violation"               },
5262         {kIOReturnBadArgument,      "invalid argument"                  },
5263         {kIOReturnLockedRead,       "device is read locked"             },
5264         {kIOReturnLockedWrite,      "device is write locked"            },
5265         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5266         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5267         {kIOReturnUnsupported,      "unsupported function"              },
5268         {kIOReturnVMError,          "virtual memory error"              },
5269         {kIOReturnInternalError,    "internal driver error"             },
5270         {kIOReturnIOError,          "I/O error"                         },
5271         {kIOReturnCannotLock,       "cannot acquire lock"               },
5272         {kIOReturnNotOpen,          "device is not open"                },
5273         {kIOReturnNotReadable,      "device is not readable"            },
5274         {kIOReturnNotWritable,      "device is not writeable"           },
5275         {kIOReturnNotAligned,       "alignment error"                   },
5276         {kIOReturnBadMedia,         "media error"                       },
5277         {kIOReturnStillOpen,        "device is still open"              },
5278         {kIOReturnRLDError,         "rld failure"                       },
5279         {kIOReturnDMAError,         "DMA failure"                       },
5280         {kIOReturnBusy,             "device is busy"                    },
5281         {kIOReturnTimeout,          "I/O timeout"                       },
5282         {kIOReturnOffline,          "device is offline"                 },
5283         {kIOReturnNotReady,         "device is not ready"               },
5284         {kIOReturnNotAttached,      "device/channel is not attached"    },
5285         {kIOReturnNoChannels,       "no DMA channels available"         },
5286         {kIOReturnNoSpace,          "no space for data"                 },
5287         {kIOReturnPortExists,       "device port already exists"        },
5288         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5289         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5290         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5291         {kIOReturnMessageTooLarge,  "message is too large"              },
5292         {kIOReturnNotPermitted,     "operation is not permitted"        },
5293         {kIOReturnNoPower,          "device is without power"           },
5294         {kIOReturnNoMedia,          "media is not present"              },
5295         {kIOReturnUnformattedMedia, "media is not formatted"            },
5296         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5297         {kIOReturnUnderrun,         "data underrun"                     },
5298         {kIOReturnOverrun,          "data overrun"                      },
5299         {kIOReturnDeviceError,      "device error"                      },
5300         {kIOReturnNoCompletion,     "no completion routine"             },
5301         {kIOReturnAborted,          "operation was aborted"             },
5302         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5303         {kIOReturnNotResponding,    "device is not responding"          },
5304         {kIOReturnInvalid,          "unanticipated driver error"        },
5305         {0,                         NULL                                }
5306     };
5307
5308     return IOFindNameForValue(rtn, IOReturn_values);
5309 }
5310
5311 /*
5312  * Convert an IOReturn to an errno.
5313  */
5314 int IOService::errnoFromReturn( IOReturn rtn )
5315 {
5316     if (unix_err(err_get_code(rtn)) == rtn)
5317         return err_get_code(rtn);
5318
5319     switch(rtn) {
5320         // (obvious match)
5321         case kIOReturnSuccess:
5322             return(0);
5323         case kIOReturnNoMemory:
5324             return(ENOMEM);
5325         case kIOReturnNoDevice:
5326             return(ENXIO);
5327         case kIOReturnVMError:
5328             return(EFAULT);
5329         case kIOReturnNotPermitted:
5330             return(EPERM);
5331         case kIOReturnNotPrivileged:
5332             return(EACCES);
5333         case kIOReturnIOError:
5334             return(EIO);
5335         case kIOReturnNotWritable:
5336             return(EROFS);
5337         case kIOReturnBadArgument:
5338             return(EINVAL);
5339         case kIOReturnUnsupported:
5340             return(ENOTSUP);
5341         case kIOReturnBusy:
5342             return(EBUSY);
5343         case kIOReturnNoPower:
5344             return(EPWROFF);
5345         case kIOReturnDeviceError:
5346             return(EDEVERR);
5347         case kIOReturnTimeout:
5348             return(ETIMEDOUT);
5349         case kIOReturnMessageTooLarge:
5350             return(EMSGSIZE);
5351         case kIOReturnNoSpace:
5352             return(ENOSPC);
5353         case kIOReturnCannotLock:
5354             return(ENOLCK);
5355
5356         // (best match)
5357         case kIOReturnBadMessageID:
5358         case kIOReturnNoCompletion:
5359         case kIOReturnNotAligned:
5360             return(EINVAL);
5361         case kIOReturnNotReady:
5362             return(EBUSY);
5363         case kIOReturnRLDError:
5364             return(EBADMACHO);
5365         case kIOReturnPortExists:
5366         case kIOReturnStillOpen:
5367             return(EEXIST);
5368         case kIOReturnExclusiveAccess:
5369         case kIOReturnLockedRead:
5370         case kIOReturnLockedWrite:
5371         case kIOReturnNotOpen:
5372         case kIOReturnNotReadable:
5373             return(EACCES);
5374         case kIOReturnCannotWire:
5375         case kIOReturnNoResources:
5376             return(ENOMEM);
5377         case kIOReturnAborted:
5378         case kIOReturnOffline:
5379         case kIOReturnNotResponding:
5380             return(EBUSY);
5381         case kIOReturnBadMedia:
5382         case kIOReturnNoMedia:
5383         case kIOReturnNotAttached:
5384         case kIOReturnUnformattedMedia:
5385             return(ENXIO); // (media error)
5386         case kIOReturnDMAError:
5387         case kIOReturnOverrun:
5388         case kIOReturnUnderrun:
5389             return(EIO); // (transfer error)
5390         case kIOReturnNoBandwidth:
5391         case kIOReturnNoChannels:
5392         case kIOReturnNoFrames:
5393         case kIOReturnNoInterrupt:
5394             return(EIO); // (hardware error)
5395         case kIOReturnError:
5396         case kIOReturnInternalError:
5397         case kIOReturnInvalid:
5398             return(EIO); // (generic error)
5399         case kIOReturnIPCError:
5400             return(EIO); // (ipc error)
5401         default:
5402             return(EIO); // (all other errors)
5403     }
5404 }
5405
5406 IOReturn IOService::message( UInt32 type, IOService * provider,
5407                                 void * argument )
5408 {
5409     /*
5410      * Generic entry point for calls from the provider.  A return value of
5411      * kIOReturnSuccess indicates that the message was received, and where
5412      * applicable, that it was successful.
5413      */
5414
5415     return kIOReturnUnsupported;
5416 }
5417
5418 /*
5419  * Device memory
5420  */
5421
5422 IOItemCount IOService::getDeviceMemoryCount( void )
5423 {
5424     OSArray *           array;
5425     IOItemCount         count;
5426
5427     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5428     if( array)
5429         count = array->getCount();
5430     else
5431         count = 0;
5432
5433     return( count);
5434 }
5435
5436 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
5437 {
5438     OSArray *           array;
5439     IODeviceMemory *    range;
5440
5441     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5442     if( array)
5443         range = (IODeviceMemory *) array->getObject( index );
5444     else
5445         range = 0;
5446
5447     return( range);
5448 }
5449
5450 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
5451                                                 IOOptionBits options )
5452 {
5453     IODeviceMemory *    range;
5454     IOMemoryMap *       map;
5455
5456     range = getDeviceMemoryWithIndex( index );
5457     if( range)
5458         map = range->map( options );
5459     else
5460         map = 0;
5461
5462     return( map );
5463 }
5464
5465 OSArray * IOService::getDeviceMemory( void )
5466 {
5467     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
5468 }
5469
5470
5471 void IOService::setDeviceMemory( OSArray * array )
5472 {
5473     setProperty( gIODeviceMemoryKey, array);
5474 }
5475
5476 /*
5477  * For machines where the transfers on an I/O bus can stall because
5478  * the CPU is in an idle mode, These APIs allow a driver to specify
5479  * the maximum bus stall that they can handle.  0 indicates no limit.
5480  */
5481 void IOService::
5482 setCPUSnoopDelay(UInt32 __unused ns)
5483 {
5484 #if defined(__i386__) || defined(__x86_64__)
5485     ml_set_maxsnoop(ns);
5486 #endif /* defined(__i386__) || defined(__x86_64__) */
5487 }
5488
5489 UInt32 IOService::
5490 getCPUSnoopDelay()
5491 {
5492 #if defined(__i386__) || defined(__x86_64__)
5493     return ml_get_maxsnoop();
5494 #else
5495     return 0;
5496 #endif /* defined(__i386__) || defined(__x86_64__) */
5497 }
5498
5499 #if defined(__i386__) || defined(__x86_64__)
5500 static void
5501 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
5502 {
5503     static const UInt kNoReplace = -1U; // Must be an illegal index
5504     UInt replace = kNoReplace;
5505     bool setCpuDelay = false;
5506
5507     IORecursiveLockLock(sCpuDelayLock);
5508
5509     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5510     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5511     IOService * holder = NULL;
5512
5513     if (ns) {
5514         const CpuDelayEntry ne = {service, ns, delayType};
5515         holder = service;
5516         // Set maximum delay.
5517         for (UInt i = 0; i < count; i++) {
5518             IOService *thisService = entries[i].fService;
5519             bool sameType = (delayType == entries[i].fDelayType);
5520             if ((service == thisService) && sameType)
5521                 replace = i;
5522             else if (!thisService) {
5523                 if (kNoReplace == replace)
5524                     replace = i;
5525             }
5526             else if (sameType) {
5527                 const UInt32 thisMax = entries[i].fMaxDelay;
5528                 if (thisMax < ns)
5529                 {
5530                     ns = thisMax;
5531                     holder = thisService;
5532                 }
5533             }
5534         }
5535
5536         setCpuDelay = true;
5537         if (kNoReplace == replace)
5538             sCpuDelayData->appendBytes(&ne, sizeof(ne));
5539         else
5540             entries[replace] = ne;
5541     }
5542     else {
5543         ns = -1U;       // Set to max unsigned, i.e. no restriction
5544
5545         for (UInt i = 0; i < count; i++) {
5546             // Clear a maximum delay.
5547             IOService *thisService = entries[i].fService;
5548             if (thisService && (delayType == entries[i].fDelayType)) {
5549                 UInt32 thisMax = entries[i].fMaxDelay;
5550                 if (service == thisService)
5551                     replace = i;
5552                 else if (thisMax < ns) {
5553                     ns = thisMax;
5554                     holder = thisService;
5555                 }
5556             }
5557         }
5558
5559         // Check if entry found
5560         if (kNoReplace != replace) {
5561             entries[replace].fService = 0;      // Null the entry
5562             setCpuDelay = true;
5563         }
5564     }
5565
5566     if (setCpuDelay)
5567     {
5568         // Must be safe to call from locked context
5569         if (delayType == kCpuDelayBusStall)
5570         {
5571             ml_set_maxbusdelay(ns);
5572         }
5573         else if (delayType == kCpuDelayInterrupt)
5574         {
5575             ml_set_maxintdelay(ns);
5576         }
5577         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
5578         sCPULatencySet   [delayType]->setValue(ns);
5579
5580         OSArray * handlers = sCpuLatencyHandlers[delayType];
5581         IOService * target;
5582         if (handlers) for (unsigned int idx = 0;
5583                             (target = (IOService *) handlers->getObject(idx));
5584                             idx++)
5585         {
5586             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5587                                             (void *) (uintptr_t) ns, holder,
5588                                             NULL, NULL);
5589         }
5590     }
5591
5592     IORecursiveLockUnlock(sCpuDelayLock);
5593 }
5594
5595 static IOReturn
5596 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
5597 {
5598     IOReturn result = kIOReturnNotFound;
5599     OSArray * array;
5600     unsigned int idx;
5601
5602     IORecursiveLockLock(sCpuDelayLock);
5603
5604     do
5605     {
5606         if (enable && !sCpuLatencyHandlers[delayType])
5607             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
5608         array = sCpuLatencyHandlers[delayType];
5609         if (!array)
5610             break;
5611         idx = array->getNextIndexOfObject(target, 0);
5612         if (!enable)
5613         {
5614             if (-1U != idx)
5615             {
5616                 array->removeObject(idx);
5617                 result = kIOReturnSuccess;
5618             }
5619         }
5620         else
5621         {
5622             if (-1U != idx) {
5623                 result = kIOReturnExclusiveAccess;
5624                 break;
5625             }
5626             array->setObject(target);
5627
5628             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5629             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5630             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
5631             IOService * holder = NULL;
5632
5633             for (UInt i = 0; i < count; i++) {
5634                 if (entries[i].fService
5635                   && (delayType == entries[i].fDelayType)
5636                   && (entries[i].fMaxDelay < ns)) {
5637                     ns = entries[i].fMaxDelay;
5638                     holder = entries[i].fService;
5639                 }
5640             }
5641             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5642                                             (void *) (uintptr_t) ns, holder,
5643                                             NULL, NULL);
5644             result = kIOReturnSuccess;
5645         }
5646     }
5647     while (false);
5648
5649     IORecursiveLockUnlock(sCpuDelayLock);
5650
5651     return (result);
5652 }
5653
5654 #endif /* defined(__i386__) || defined(__x86_64__) */
5655
5656 void IOService::
5657 requireMaxBusStall(UInt32 __unused ns)
5658 {
5659 #if defined(__i386__) || defined(__x86_64__)
5660     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
5661 #endif
5662 }
5663
5664 void IOService::
5665 requireMaxInterruptDelay(uint32_t __unused ns)
5666 {
5667 #if defined(__i386__) || defined(__x86_64__)
5668     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
5669 #endif
5670 }
5671
5672 /*
5673  * Device interrupts
5674  */
5675
5676 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
5677 {
5678   IOInterruptController *interruptController;
5679   OSArray               *array;
5680   OSData                *data;
5681   OSSymbol              *interruptControllerName;
5682   long                  numSources;
5683   IOInterruptSource     *interruptSources;
5684
5685   // Get the parents list from the nub.
5686   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
5687   if (array == 0) return kIOReturnNoResources;
5688
5689   // Allocate space for the IOInterruptSources if needed... then return early.
5690   if (nub->_interruptSources == 0) {
5691     numSources = array->getCount();
5692     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
5693     if (interruptSources == 0) return kIOReturnNoMemory;
5694
5695     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
5696
5697     nub->_numInterruptSources = numSources;
5698     nub->_interruptSources = interruptSources;
5699     return kIOReturnSuccess;
5700   }
5701
5702   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
5703   if (interruptControllerName == 0) return kIOReturnNoResources;
5704
5705   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
5706   if (interruptController == 0) return kIOReturnNoResources;
5707
5708   // Get the interrupt numbers from the nub.
5709   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
5710   if (array == 0) return kIOReturnNoResources;
5711   data = OSDynamicCast(OSData, array->getObject(source));
5712   if (data == 0) return kIOReturnNoResources;
5713
5714   // Set the interruptController and interruptSource in the nub's table.
5715   interruptSources = nub->_interruptSources;
5716   interruptSources[source].interruptController = interruptController;
5717   interruptSources[source].vectorData = data;
5718
5719   return kIOReturnSuccess;
5720 }
5721
5722 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
5723 {
5724   IOReturn ret;
5725
5726   /* Make sure the _interruptSources are set */
5727   if (_interruptSources == 0) {
5728     ret = resolveInterrupt(this, source);
5729     if (ret != kIOReturnSuccess) return ret;
5730   }
5731
5732   /* Make sure the local source number is valid */
5733   if ((source < 0) || (source >= _numInterruptSources))
5734     return kIOReturnNoInterrupt;
5735
5736   /* Look up the contoller for the local source */
5737   *interruptController = _interruptSources[source].interruptController;
5738
5739   if (*interruptController == NULL) {
5740     if (!resolve) return kIOReturnNoInterrupt;
5741
5742     /* Try to reslove the interrupt */
5743     ret = resolveInterrupt(this, source);
5744     if (ret != kIOReturnSuccess) return ret;
5745
5746     *interruptController = _interruptSources[source].interruptController;
5747   }
5748
5749   return kIOReturnSuccess;
5750 }
5751
5752 IOReturn IOService::registerInterrupt(int source, OSObject *target,
5753                                       IOInterruptAction handler,
5754                                       void *refCon)
5755 {
5756   IOInterruptController *interruptController;
5757   IOReturn              ret;
5758
5759   ret = lookupInterrupt(source, true, &interruptController);
5760   if (ret != kIOReturnSuccess) return ret;
5761
5762   /* Register the source */
5763   return interruptController->registerInterrupt(this, source, target,
5764                                                 (IOInterruptHandler)handler,
5765                                                 refCon);
5766 }
5767
5768 IOReturn IOService::unregisterInterrupt(int source)
5769 {
5770   IOInterruptController *interruptController;
5771   IOReturn              ret;
5772
5773   ret = lookupInterrupt(source, false, &interruptController);
5774   if (ret != kIOReturnSuccess) return ret;
5775
5776   /* Unregister the source */
5777   return interruptController->unregisterInterrupt(this, source);
5778 }
5779
5780 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
5781 {
5782   IOReportLegend * legend = NULL;
5783   IOInterruptAccountingData * oldValue = NULL;
5784   IOInterruptAccountingReporter * newArray = NULL;
5785   int newArraySize = 0;
5786   int i = 0;
5787
5788   if (source < 0) {
5789     return kIOReturnBadArgument;
5790   }
5791
5792   /*
5793    * We support statistics on a maximum of 256 interrupts per nub; if a nub
5794    * has more than 256 interrupt specifiers associated with it, and tries
5795    * to register a high interrupt index with interrupt accounting, panic.
5796    * Having more than 256 interrupts associated with a single nub is
5797    * probably a sign that something fishy is going on.
5798    */
5799   if (source > IA_INDEX_MAX) {
5800     panic("addInterruptStatistics called for an excessively large index (%d)", source);
5801   }
5802
5803   /*
5804    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
5805    * leaving it as is because the likelihood of contention where we are
5806    * actually growing the array is minimal (we would realistically need
5807    * to be starting a driver for the first time, with an IOReporting
5808    * client already in place).  Nonetheless, cleanup that can be done
5809    * to adhere to best practices; it'll make the code more complicated,
5810    * unfortunately.
5811    */
5812   IOLockLock(reserved->interruptStatisticsLock);
5813
5814   /*
5815    * Lazily allocate the statistics array.
5816    */
5817   if (!reserved->interruptStatisticsArray) {
5818     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
5819     assert(reserved->interruptStatisticsArray);
5820     reserved->interruptStatisticsArrayCount = 1;
5821     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
5822   }
5823
5824   if (source >= reserved->interruptStatisticsArrayCount) {
5825     /*
5826      * We're still within the range of supported indices, but we are out
5827      * of space in the current array.  Do a nasty realloc (because
5828      * IORealloc isn't a thing) here.  We'll double the size with each
5829      * reallocation.
5830      *
5831      * Yes, the "next power of 2" could be more efficient; but this will
5832      * be invoked incredibly rarely.  Who cares.
5833      */
5834     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
5835
5836     while (newArraySize <= source)
5837       newArraySize = (newArraySize << 1);
5838     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
5839
5840     assert(newArray);
5841
5842     /*
5843      * TODO: This even zeroes the memory it is about to overwrite.
5844      * Shameful; fix it.  Not particularly high impact, however.
5845      */
5846     bzero(newArray, newArraySize * sizeof(*newArray));
5847     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
5848     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
5849     reserved->interruptStatisticsArray = newArray;
5850     reserved->interruptStatisticsArrayCount = newArraySize;
5851   }
5852
5853   if (!reserved->interruptStatisticsArray[source].reporter) {
5854     /*
5855      * We don't have a reporter associated with this index yet, so we
5856      * need to create one.
5857      */
5858     /*
5859      * TODO: Some statistics do in fact have common units (time); should this be
5860      * split into separate reporters to communicate this?
5861      */
5862      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryInterrupt, kIOReportUnitNone);
5863
5864     /*
5865      * Each statistic is given an identifier based on the interrupt index (which
5866      * should be unique relative to any single nub) and the statistic involved.
5867      * We should now have a sane (small and positive) index, so start
5868      * constructing the channels for statistics.
5869      */
5870     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
5871       /*
5872        * TODO: Currently, this does not add channels for disabled statistics.
5873        * Will this be confusing for clients?  If so, we should just add the
5874        * channels; we can avoid updating the channels even if they exist.
5875        */
5876       if (IA_GET_STATISTIC_ENABLED(i))
5877         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
5878     }
5879
5880     /*
5881      * We now need to add the legend for this reporter to the registry.
5882      */
5883     legend = IOReportLegend::with(OSDynamicCast(OSArray, getProperty(kIOReportLegendKey)));
5884
5885     if ((source >= IA_MAX_SUBGROUP_NAME) || (source < 0)) {
5886       /*
5887        * Either we're using a nonsensical index (should never happen), or the
5888        * index is larger than anticipated (may happen, almost certainly won't).
5889        * This may move to live generation of the names in the future, but for
5890        * now, point both cases to a generic subgroup name (this will confuse
5891        * clients, unfortunately).
5892        */
5893       legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, kInterruptAccountingGenericSubgroupName);
5894     } else {
5895       legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, kInterruptAccountingSubgroupNames[source]);
5896     }
5897
5898     setProperty(kIOReportLegendKey, legend->getLegend());
5899     legend->release();
5900
5901     /*
5902      * TODO: Is this a good idea?  Probably not; my assumption is it opts
5903      * all entities who register interrupts into public disclosure of all
5904      * IOReporting channels.  Unfortunately, this appears to be as fine
5905      * grain as it gets.
5906      */
5907     setProperty(kIOReportLegendPublicKey, true);
5908   }
5909
5910   /*
5911    * Don't stomp existing entries.  If we are about to, panic; this
5912    * probably means we failed to tear down our old interrupt source
5913    * correctly.
5914    */
5915   oldValue = reserved->interruptStatisticsArray[source].statistics;
5916
5917   if (oldValue) {
5918     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
5919   }
5920
5921   reserved->interruptStatisticsArray[source].statistics = statistics;
5922
5923   /*
5924    * Inherit the reporter values for each statistic.  The target may
5925    * be torn down as part of the runtime of the service (especially
5926    * for sleep/wake), so we inherit in order to avoid having values
5927    * reset for no apparent reason.  Our statistics are ultimately
5928    * tied to the index and the sevice, not to an individual target,
5929    * so we should maintain them accordingly.
5930    */
5931   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
5932
5933   IOLockUnlock(reserved->interruptStatisticsLock);
5934
5935   return kIOReturnSuccess;
5936 }
5937
5938 IOReturn IOService::removeInterruptStatistics(int source)
5939 {
5940   IOInterruptAccountingData * value = NULL;
5941
5942   if (source < 0) {
5943     return kIOReturnBadArgument;
5944   }
5945
5946   IOLockLock(reserved->interruptStatisticsLock);
5947
5948   /*
5949    * We dynamically grow the statistics array, so an excessively
5950    * large index value has NEVER been registered.  This either
5951    * means our cap on the array size is too small (unlikely), or
5952    * that we have been passed a corrupt index (this must be passed
5953    * the plain index into the interrupt specifier list).
5954    */
5955   if (source >= reserved->interruptStatisticsArrayCount) {
5956     panic("removeInterruptStatistics called for index %d, which was never registered", source);
5957   }
5958
5959   assert(reserved->interruptStatisticsArray);
5960
5961   /*
5962    * If there is no existing entry, we are most likely trying to
5963    * free an interrupt owner twice, or we have corrupted the
5964    * index value.
5965    */
5966   value = reserved->interruptStatisticsArray[source].statistics;
5967
5968   if (!value) {
5969     panic("removeInterruptStatistics called for empty index %d", source);
5970   }
5971
5972   /*
5973    * We update the statistics, so that any delta with the reporter
5974    * state is not lost.
5975    */
5976   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
5977   reserved->interruptStatisticsArray[source].statistics = NULL;
5978   IOLockUnlock(reserved->interruptStatisticsLock);
5979
5980   return kIOReturnSuccess;
5981 }
5982
5983 IOReturn IOService::getInterruptType(int source, int *interruptType)
5984 {
5985   IOInterruptController *interruptController;
5986   IOReturn              ret;
5987
5988   ret = lookupInterrupt(source, true, &interruptController);
5989   if (ret != kIOReturnSuccess) return ret;
5990
5991   /* Return the type */
5992   return interruptController->getInterruptType(this, source, interruptType);
5993 }
5994
5995 IOReturn IOService::enableInterrupt(int source)
5996 {
5997   IOInterruptController *interruptController;
5998   IOReturn              ret;
5999
6000   ret = lookupInterrupt(source, false, &interruptController);
6001   if (ret != kIOReturnSuccess) return ret;
6002
6003   /* Enable the source */
6004   return interruptController->enableInterrupt(this, source);
6005 }
6006
6007 IOReturn IOService::disableInterrupt(int source)
6008 {
6009   IOInterruptController *interruptController;
6010   IOReturn              ret;
6011
6012   ret = lookupInterrupt(source, false, &interruptController);
6013   if (ret != kIOReturnSuccess) return ret;
6014
6015   /* Disable the source */
6016   return interruptController->disableInterrupt(this, source);
6017 }
6018
6019 IOReturn IOService::causeInterrupt(int source)
6020 {
6021   IOInterruptController *interruptController;
6022   IOReturn              ret;
6023
6024   ret = lookupInterrupt(source, false, &interruptController);
6025   if (ret != kIOReturnSuccess) return ret;
6026
6027   /* Cause an interrupt for the source */
6028   return interruptController->causeInterrupt(this, source);
6029 }
6030
6031 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6032                                     IOReportConfigureAction action,
6033                                     void                   *result,
6034                                     void                   *destination)
6035 {
6036     unsigned cnt;
6037
6038     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6039         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6040             if (pwrMgt) configurePowerStatesReport(action, result);
6041             else  return kIOReturnUnsupported;
6042         }
6043         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6044             if (pwrMgt) configureSimplePowerReport(action, result);
6045             else  return kIOReturnUnsupported;
6046         }
6047     }
6048
6049     IOLockLock(reserved->interruptStatisticsLock);
6050
6051     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6052     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6053         if (reserved->interruptStatisticsArray[cnt].reporter) {
6054             /*
6055              * If the reporter is currently associated with the statistics
6056              * for an event source, we may need to update the reporter.
6057              */
6058             if (reserved->interruptStatisticsArray[cnt].statistics)
6059                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6060
6061             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6062         }
6063     }
6064
6065     IOLockUnlock(reserved->interruptStatisticsLock);
6066
6067     return kIOReturnSuccess;
6068 }
6069
6070 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6071                                  IOReportUpdateAction      action,
6072                                  void                     *result,
6073                                  void                     *destination)
6074 {
6075     unsigned cnt;
6076
6077     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6078         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6079             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6080             else return kIOReturnUnsupported;
6081         }
6082         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6083             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6084             else return kIOReturnUnsupported;
6085         }
6086     }
6087
6088     IOLockLock(reserved->interruptStatisticsLock);
6089
6090     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6091     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6092         if (reserved->interruptStatisticsArray[cnt].reporter) {
6093             /*
6094              * If the reporter is currently associated with the statistics
6095              * for an event source, we need to update the reporter.
6096              */
6097             if (reserved->interruptStatisticsArray[cnt].statistics)
6098                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6099
6100             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6101         }
6102     }
6103
6104     IOLockUnlock(reserved->interruptStatisticsLock);
6105
6106     return kIOReturnSuccess;
6107 }
6108
6109 uint64_t IOService::getAuthorizationID( void )
6110 {
6111     return reserved->authorizationID;
6112 }
6113
6114 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6115 {
6116     OSObject * entitlement;
6117     IOReturn status;
6118
6119     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6120
6121     if ( entitlement )
6122     {
6123         if ( entitlement == kOSBooleanTrue )
6124         {
6125             reserved->authorizationID = authorizationID;
6126
6127             status = kIOReturnSuccess;
6128         }
6129         else
6130         {
6131             status = kIOReturnNotPrivileged;
6132         }
6133
6134         entitlement->release( );
6135     }
6136     else
6137     {
6138         status = kIOReturnNotPrivileged;
6139     }
6140
6141     return status;
6142 }
6143
6144 #if __LP64__
6145 OSMetaClassDefineReservedUsed(IOService, 0);
6146 OSMetaClassDefineReservedUsed(IOService, 1);
6147 OSMetaClassDefineReservedUnused(IOService, 2);
6148 OSMetaClassDefineReservedUnused(IOService, 3);
6149 OSMetaClassDefineReservedUnused(IOService, 4);
6150 OSMetaClassDefineReservedUnused(IOService, 5);
6151 OSMetaClassDefineReservedUnused(IOService, 6);
6152 OSMetaClassDefineReservedUnused(IOService, 7);
6153 #else
6154 OSMetaClassDefineReservedUsed(IOService, 0);
6155 OSMetaClassDefineReservedUsed(IOService, 1);
6156 OSMetaClassDefineReservedUsed(IOService, 2);
6157 OSMetaClassDefineReservedUsed(IOService, 3);
6158 OSMetaClassDefineReservedUsed(IOService, 4);
6159 OSMetaClassDefineReservedUsed(IOService, 5);
6160 OSMetaClassDefineReservedUsed(IOService, 6);
6161 OSMetaClassDefineReservedUsed(IOService, 7);
6162 #endif
6163 OSMetaClassDefineReservedUnused(IOService, 8);
6164 OSMetaClassDefineReservedUnused(IOService, 9);
6165 OSMetaClassDefineReservedUnused(IOService, 10);
6166 OSMetaClassDefineReservedUnused(IOService, 11);
6167 OSMetaClassDefineReservedUnused(IOService, 12);
6168 OSMetaClassDefineReservedUnused(IOService, 13);
6169 OSMetaClassDefineReservedUnused(IOService, 14);
6170 OSMetaClassDefineReservedUnused(IOService, 15);
6171 OSMetaClassDefineReservedUnused(IOService, 16);
6172 OSMetaClassDefineReservedUnused(IOService, 17);
6173 OSMetaClassDefineReservedUnused(IOService, 18);
6174 OSMetaClassDefineReservedUnused(IOService, 19);
6175 OSMetaClassDefineReservedUnused(IOService, 20);
6176 OSMetaClassDefineReservedUnused(IOService, 21);
6177 OSMetaClassDefineReservedUnused(IOService, 22);
6178 OSMetaClassDefineReservedUnused(IOService, 23);
6179 OSMetaClassDefineReservedUnused(IOService, 24);
6180 OSMetaClassDefineReservedUnused(IOService, 25);
6181 OSMetaClassDefineReservedUnused(IOService, 26);
6182 OSMetaClassDefineReservedUnused(IOService, 27);
6183 OSMetaClassDefineReservedUnused(IOService, 28);
6184 OSMetaClassDefineReservedUnused(IOService, 29);
6185 OSMetaClassDefineReservedUnused(IOService, 30);
6186 OSMetaClassDefineReservedUnused(IOService, 31);
6187 OSMetaClassDefineReservedUnused(IOService, 32);
6188 OSMetaClassDefineReservedUnused(IOService, 33);
6189 OSMetaClassDefineReservedUnused(IOService, 34);
6190 OSMetaClassDefineReservedUnused(IOService, 35);
6191 OSMetaClassDefineReservedUnused(IOService, 36);
6192 OSMetaClassDefineReservedUnused(IOService, 37);
6193 OSMetaClassDefineReservedUnused(IOService, 38);
6194 OSMetaClassDefineReservedUnused(IOService, 39);
6195 OSMetaClassDefineReservedUnused(IOService, 40);
6196 OSMetaClassDefineReservedUnused(IOService, 41);
6197 OSMetaClassDefineReservedUnused(IOService, 42);
6198 OSMetaClassDefineReservedUnused(IOService, 43);
6199 OSMetaClassDefineReservedUnused(IOService, 44);
6200 OSMetaClassDefineReservedUnused(IOService, 45);
6201 OSMetaClassDefineReservedUnused(IOService, 46);
6202 OSMetaClassDefineReservedUnused(IOService, 47);