iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceMemory.h>
  39 #include <IOKit/IOInterrupts.h>
  40 #include <IOKit/IOInterruptController.h>
  41 #include <IOKit/IOPlatformExpert.h>
  42 #include <IOKit/IOMessage.h>
  43 #include <IOKit/IOLib.h>
  44 #include <IOKit/IOKitKeysPrivate.h>
  45 #include <IOKit/IOBSD.h>
  46 #include <IOKit/IOUserClient.h>
  47 #include <IOKit/IOWorkLoop.h>
  48 #include <IOKit/IOTimeStamp.h>
  49 #include <mach/sync_policy.h>
  50 #include <IOKit/assert.h>
  51 #include <sys/errno.h>
  52
  53 #include <machine/pal_routines.h>
  54
  55 #define LOG kprintf
  56 //#define LOG IOLog
  57
  58 #include "IOServicePrivate.h"
  59 #include "IOKitKernelInternal.h"
  60
  61 // take lockForArbitration before LOCKNOTIFY
  62
  63 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  64
  65 #define super IORegistryEntry
  66
  67 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  68
  69 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  70
  71 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  72
  73 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  74
  75 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  76
  77 OSDefineMetaClassAndStructors(IOResources, IOService)
  78
  79 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  80
  81 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  82
  83 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  84
  85 static IOPlatformExpert *       gIOPlatform;
  86 static class IOPMrootDomain *   gIOPMRootDomain;
  87 const IORegistryPlane *         gIOServicePlane;
  88 const IORegistryPlane *         gIOPowerPlane;
  89 const OSSymbol *                gIODeviceMemoryKey;
  90 const OSSymbol *                gIOInterruptControllersKey;
  91 const OSSymbol *                gIOInterruptSpecifiersKey;
  92
  93 const OSSymbol *                gIOResourcesKey;
  94 const OSSymbol *                gIOResourceMatchKey;
  95 const OSSymbol *                gIOProviderClassKey;
  96 const OSSymbol *                gIONameMatchKey;
  97 const OSSymbol *                gIONameMatchedKey;
  98 const OSSymbol *                gIOPropertyMatchKey;
  99 const OSSymbol *                gIOLocationMatchKey;
 100 const OSSymbol *                gIOParentMatchKey;
 101 const OSSymbol *                gIOPathMatchKey;
 102 const OSSymbol *                gIOMatchCategoryKey;
 103 const OSSymbol *                gIODefaultMatchCategoryKey;
 104 const OSSymbol *                gIOMatchedServiceCountKey;
 105
 106 const OSSymbol *                gIOMapperIDKey;
 107 const OSSymbol *                gIOUserClientClassKey;
 108 const OSSymbol *                gIOKitDebugKey;
 109
 110 const OSSymbol *                gIOCommandPoolSizeKey;
 111
 112 const OSSymbol *                gIOConsoleLockedKey;
 113 const OSSymbol *                gIOConsoleUsersKey;
 114 const OSSymbol *                gIOConsoleSessionUIDKey;
 115 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 116 const OSSymbol *                gIOConsoleUsersSeedKey;
 117 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 118 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 119 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 120
 121 static clock_sec_t              gIOConsoleLockTime;
 122
 123 static int                      gIOResourceGenerationCount;
 124
 125 const OSSymbol *                gIOServiceKey;
 126 const OSSymbol *                gIOPublishNotification;
 127 const OSSymbol *                gIOFirstPublishNotification;
 128 const OSSymbol *                gIOMatchedNotification;
 129 const OSSymbol *                gIOFirstMatchNotification;
 130 const OSSymbol *                gIOTerminatedNotification;
 131
 132 const OSSymbol *                gIOGeneralInterest;
 133 const OSSymbol *                gIOBusyInterest;
 134 const OSSymbol *                gIOAppPowerStateInterest;
 135 const OSSymbol *                gIOPriorityPowerStateInterest;
 136 const OSSymbol *                gIOConsoleSecurityInterest;
 137
 138 static OSDictionary *           gNotifications;
 139 static IORecursiveLock *        gNotificationLock;
 140
 141 static IOService *              gIOResources;
 142 static IOService *              gIOServiceRoot;
 143
 144 static OSOrderedSet *           gJobs;
 145 static semaphore_port_t         gJobsSemaphore;
 146 static IOLock *                 gJobsLock;
 147 static int                      gOutstandingJobs;
 148 static int                      gNumConfigThreads;
 149 static int                      gNumWaitingThreads;
 150 static IOLock *                 gIOServiceBusyLock;
 151
 152 static thread_t                 gIOTerminateThread;
 153 static UInt32                   gIOTerminateWork;
 154 static OSArray *                gIOTerminatePhase2List;
 155 static OSArray *                gIOStopList;
 156 static OSArray *                gIOStopProviderList;
 157 static OSArray *                gIOFinalizeList;
 158
 159 static SInt32                   gIOConsoleUsersSeed;
 160 static OSData *                 gIOConsoleUsersSeedValue;
 161
 162 extern const OSSymbol *         gIODTPHandleKey;
 163
 164 const OSSymbol *                gIOPlatformSleepActionKey;
 165 const OSSymbol *                gIOPlatformWakeActionKey;
 166 const OSSymbol *                gIOPlatformQuiesceActionKey;
 167 const OSSymbol *                gIOPlatformActiveActionKey;
 168
 169 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 170
 171 static IOLock *                 gIOConsoleUsersLock;
 172 static thread_call_t            gIOConsoleLockCallout;
 173
 174 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 175
 176 #define LOCKREADNOTIFY()        \
 177     IORecursiveLockLock( gNotificationLock )
 178 #define LOCKWRITENOTIFY()       \
 179     IORecursiveLockLock( gNotificationLock )
 180 #define LOCKWRITE2READNOTIFY()
 181 #define UNLOCKNOTIFY()          \
 182     IORecursiveLockUnlock( gNotificationLock )
 183 #define SLEEPNOTIFY(event) \
 184     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 185 #define SLEEPNOTIFYTO(event, deadline) \
 186     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 187 #define WAKEUPNOTIFY(event) \
 188         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 189
 190 #define randomDelay()   \
 191         int del = read_processor_clock();                               \
 192         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 193         IOSleep( del );
 194
 195 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 196
 197 #define queue_element(entry, element, type, field) do { \
 198         vm_address_t __ele = (vm_address_t) (entry);    \
 199         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 200         (element) = (type) __ele;                       \
 201     } while(0)
 202
 203 #define iterqueue(que, elt)                             \
 204         for (queue_entry_t elt = queue_first(que);      \
 205              !queue_end(que, elt);                      \
 206              elt = queue_next(elt))
 207
 208 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 209
 210 struct ArbitrationLockQueueElement {
 211     queue_chain_t link;
 212     IOThread      thread;
 213     IOService *   service;
 214     unsigned      count;
 215     bool          required;
 216     bool          aborted;
 217 };
 218
 219 static queue_head_t gArbitrationLockQueueActive;
 220 static queue_head_t gArbitrationLockQueueWaiting;
 221 static queue_head_t gArbitrationLockQueueFree;
 222 static IOLock *     gArbitrationLockQueueLock;
 223
 224 bool IOService::isInactive( void ) const
 225     { return( 0 != (kIOServiceInactiveState & getState())); }
 226
 227
 228 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 229
 230 #if defined(__i386__) || defined(__x86_64__)
 231
 232 // Only used by the intel implementation of
 233 //     IOService::requireMaxBusStall(UInt32 ns)
 234 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 235 struct CpuDelayEntry
 236 {
 237     IOService * fService;
 238     UInt32      fMaxDelay;
 239     UInt32      fDelayType;
 240 };
 241
 242 enum {
 243     kCpuDelayBusStall, kCpuDelayInterrupt,
 244     kCpuNumDelayTypes
 245 };
 246
 247 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 248 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 249 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 250 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 251
 252 static void
 253 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 254 static IOReturn
 255 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 256
 257 #endif /* defined(__i386__) || defined(__x86_64__) */
 258
 259 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 260
 261 void IOService::initialize( void )
 262 {
 263     kern_return_t       err;
 264
 265     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 266     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 267
 268     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 269     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 270     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 271     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 272     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 273     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 274     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 275
 276     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 277     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 278                                         kIODefaultMatchCategoryKey );
 279     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 280                                         kIOMatchedServiceCountKey );
 281
 282     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 283
 284     gIOResourcesKey     = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 285     gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 286
 287     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 288     gIOInterruptControllersKey
 289         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 290     gIOInterruptSpecifiersKey
 291         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 292
 293     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 294
 295     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 296
 297     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 298
 299     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 300     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 301     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 302     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 303     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 304
 305     gNotifications              = OSDictionary::withCapacity( 1 );
 306     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 307                                                  kIOPublishNotification );
 308     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 309                                                  kIOFirstPublishNotification );
 310     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 311                                                  kIOMatchedNotification );
 312     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 313                                                  kIOFirstMatchNotification );
 314     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 315                                                  kIOTerminatedNotification );
 316     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 317
 318     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 319     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 320     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 321     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 322
 323     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 324     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 325     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 326     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 327
 328     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 329
 330     gIOPlatformSleepActionKey   = OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey);
 331     gIOPlatformWakeActionKey    = OSSymbol::withCStringNoCopy(kIOPlatformWakeActionKey);
 332     gIOPlatformQuiesceActionKey = OSSymbol::withCStringNoCopy(kIOPlatformQuiesceActionKey);
 333     gIOPlatformActiveActionKey  = OSSymbol::withCStringNoCopy(kIOPlatformActiveActionKey);
 334
 335     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 336 #if defined(__i386__) || defined(__x86_64__)
 337     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 338     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 339 #endif
 340     gNotificationLock           = IORecursiveLockAlloc();
 341
 342     assert( gIOServicePlane && gIODeviceMemoryKey
 343         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 344         && gIOResourcesKey && gNotifications && gNotificationLock
 345         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 346         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 347         && gIOPublishNotification && gIOMatchedNotification
 348         && gIOTerminatedNotification && gIOServiceKey
 349         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 350     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 351         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 352
 353     gJobsLock   = IOLockAlloc();
 354     gJobs       = OSOrderedSet::withCapacity( 10 );
 355
 356     gIOServiceBusyLock = IOLockAlloc();
 357
 358     gIOConsoleUsersLock = IOLockAlloc();
 359
 360     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 361
 362     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 363
 364     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 365                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 366
 367     gIOResources = IOResources::resources();
 368     assert( gIOResources );
 369
 370     gArbitrationLockQueueLock = IOLockAlloc();
 371     queue_init(&gArbitrationLockQueueActive);
 372     queue_init(&gArbitrationLockQueueWaiting);
 373     queue_init(&gArbitrationLockQueueFree);
 374
 375     assert( gArbitrationLockQueueLock );
 376
 377     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 378     gIOStopList            = OSArray::withCapacity( 16 );
 379     gIOStopProviderList    = OSArray::withCapacity( 16 );
 380     gIOFinalizeList        = OSArray::withCapacity( 16 );
 381     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 382 }
 383
 384 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 385
 386 #if IOMATCHDEBUG
 387 static UInt64 getDebugFlags( OSDictionary * props )
 388 {
 389     OSNumber *  debugProp;
 390     UInt64      debugFlags;
 391
 392     debugProp = OSDynamicCast( OSNumber,
 393                 props->getObject( gIOKitDebugKey ));
 394     if( debugProp)
 395         debugFlags = debugProp->unsigned64BitValue();
 396     else
 397         debugFlags = gIOKitDebug;
 398
 399     return( debugFlags );
 400 }
 401 #endif
 402
 403 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 404
 405 // Probe a matched service and return an instance to be started.
 406 // The default score is from the property table, & may be altered
 407 // during probe to change the start order.
 408
 409 IOService * IOService::probe(   IOService * provider,
 410                                 SInt32    * score )
 411 {
 412     return( this );
 413 }
 414
 415 bool IOService::start( IOService * provider )
 416 {
 417     return( true );
 418 }
 419
 420 void IOService::stop( IOService * provider )
 421 {
 422 }
 423
 424 void IOService::free( void )
 425 {
 426     requireMaxBusStall(0);
 427     requireMaxInterruptDelay(0);
 428     if( getPropertyTable())
 429         unregisterAllInterest();
 430     PMfree();
 431     super::free();
 432 }
 433
 434 /*
 435  * Attach in service plane
 436  */
 437 bool IOService::attach( IOService * provider )
 438 {
 439     bool        ok;
 440
 441     if( provider) {
 442
 443         if( gIOKitDebug & kIOLogAttach)
 444             LOG( "%s::attach(%s)\n", getName(),
 445                     provider->getName());
 446
 447         provider->lockForArbitration();
 448         if( provider->__state[0] & kIOServiceInactiveState)
 449             ok = false;
 450         else
 451             ok = attachToParent( provider, gIOServicePlane);
 452         provider->unlockForArbitration();
 453
 454     } else {
 455         gIOServiceRoot = this;
 456         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 457     }
 458
 459     return( ok );
 460 }
 461
 462 IOService * IOService::getServiceRoot( void )
 463 {
 464     return( gIOServiceRoot );
 465 }
 466
 467 void IOService::detach( IOService * provider )
 468 {
 469     IOService * newProvider = 0;
 470     SInt32      busy;
 471     bool        adjParent;
 472
 473     if( gIOKitDebug & kIOLogAttach)
 474         LOG("%s::detach(%s)\n", getName(), provider->getName());
 475
 476     lockForArbitration();
 477
 478     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 479                && (provider == getProvider()));
 480
 481     detachFromParent( provider, gIOServicePlane );
 482
 483     if( busy) {
 484         newProvider = getProvider();
 485         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 486             _adjustBusy( -busy );
 487     }
 488
 489     unlockForArbitration();
 490
 491     if( newProvider) {
 492         newProvider->lockForArbitration();
 493         newProvider->_adjustBusy(1);
 494         newProvider->unlockForArbitration();
 495     }
 496
 497     // check for last client detach from a terminated service
 498     if( provider->lockForArbitration( true )) {
 499         if( adjParent)
 500             provider->_adjustBusy( -1 );
 501         if( (provider->__state[1] & kIOServiceTermPhase3State)
 502          && (0 == provider->getClient())) {
 503             provider->scheduleFinalize();
 504         }
 505         provider->unlockForArbitration();
 506     }
 507 }
 508
 509 /*
 510  * Register instance - publish it for matching
 511  */
 512
 513 void IOService::registerService( IOOptionBits options )
 514 {
 515     char *              pathBuf;
 516     const char *        path;
 517     char *              skip;
 518     int                 len;
 519     enum { kMaxPathLen  = 256 };
 520     enum { kMaxChars    = 63 };
 521
 522     IORegistryEntry * parent = this;
 523     IORegistryEntry * root = getRegistryRoot();
 524     while( parent && (parent != root))
 525         parent = parent->getParentEntry( gIOServicePlane);
 526
 527     if( parent != root) {
 528         IOLog("%s: not registry member at registerService()\n", getName());
 529         return;
 530     }
 531
 532     // Allow the Platform Expert to adjust this node.
 533     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 534         return;
 535
 536     if( (this != gIOResources)
 537      && (kIOLogRegister & gIOKitDebug)) {
 538
 539         pathBuf = (char *) IOMalloc( kMaxPathLen );
 540
 541         IOLog( "Registering: " );
 542
 543         len = kMaxPathLen;
 544         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 545
 546             path = pathBuf;
 547             if( len > kMaxChars) {
 548                 IOLog("..");
 549                 len -= kMaxChars;
 550                 path += len;
 551                 if( (skip = strchr( path, '/')))
 552                     path = skip;
 553             }
 554         } else
 555             path = getName();
 556
 557         IOLog( "%s\n", path );
 558
 559         if( pathBuf)
 560             IOFree( pathBuf, kMaxPathLen );
 561     }
 562
 563     startMatching( options );
 564 }
 565
 566 void IOService::startMatching( IOOptionBits options )
 567 {
 568     IOService * provider;
 569     UInt32      prevBusy = 0;
 570     bool        needConfig;
 571     bool        needWake = false;
 572     bool        ok;
 573     bool        sync;
 574     bool        waitAgain;
 575
 576     lockForArbitration();
 577
 578     sync = (options & kIOServiceSynchronous)
 579         || ((provider = getProvider())
 580                 && (provider->__state[1] & kIOServiceSynchronousState));
 581
 582         if ( options & kIOServiceAsynchronous )
 583                 sync = false;
 584
 585     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState)))
 586                && (0 == (__state[0] & kIOServiceInactiveState));
 587
 588     __state[1] |= kIOServiceNeedConfigState;
 589
 590 //    __state[0] &= ~kIOServiceInactiveState;
 591
 592 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 593 //                      OSKernelStackRemaining(), getName());
 594
 595     if( needConfig) {
 596         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 597     }
 598
 599     if( sync)
 600         __state[1] |= kIOServiceSynchronousState;
 601     else
 602         __state[1] &= ~kIOServiceSynchronousState;
 603
 604     unlockForArbitration();
 605
 606     if( needConfig) {
 607
 608         prevBusy = _adjustBusy( 1 );
 609
 610         if( needWake) {
 611             IOLockLock( gIOServiceBusyLock );
 612             thread_wakeup( (event_t) this/*&__state[1]*/ );
 613             IOLockUnlock( gIOServiceBusyLock );
 614
 615         } else if( !sync || (kIOServiceAsynchronous & options)) {
 616
 617             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 618
 619         } else do {
 620
 621             if( (__state[1] & kIOServiceNeedConfigState))
 622                 doServiceMatch( options );
 623
 624             lockForArbitration();
 625             IOLockLock( gIOServiceBusyLock );
 626
 627             waitAgain = (prevBusy < (__state[1] & kIOServiceBusyStateMask));
 628             if( waitAgain)
 629                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 630             else
 631                 __state[1] &= ~kIOServiceSyncPubState;
 632
 633             unlockForArbitration();
 634
 635             if( waitAgain)
 636                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 637
 638             IOLockUnlock( gIOServiceBusyLock );
 639             if( waitAgain)
 640                 thread_block(THREAD_CONTINUE_NULL);
 641
 642         } while( waitAgain );
 643     }
 644 }
 645
 646 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 647 {
 648     OSDictionary *      table;
 649     OSSet *             set;
 650     OSSet *             allSet = 0;
 651     IOService *         service;
 652 #if IOMATCHDEBUG
 653     SInt32              count = 0;
 654 #endif
 655
 656     newTables->retain();
 657
 658     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 659
 660         LOCKWRITENOTIFY();
 661         set = (OSSet *) copyExistingServices( table,
 662                                                 kIOServiceRegisteredState,
 663                                                 kIOServiceExistingSet);
 664         UNLOCKNOTIFY();
 665         if( set) {
 666
 667 #if IOMATCHDEBUG
 668             count += set->getCount();
 669 #endif
 670             if (allSet) {
 671                 allSet->merge((const OSSet *) set);
 672                 set->release();
 673             }
 674             else
 675                 allSet = set;
 676         }
 677
 678 #if IOMATCHDEBUG
 679         if( getDebugFlags( table ) & kIOLogMatch)
 680             LOG("Matching service count = %ld\n", (long)count);
 681 #endif
 682         newTables->removeObject(table);
 683     }
 684
 685     if (allSet) {
 686         while( (service = (IOService *) allSet->getAnyObject())) {
 687             service->startMatching(kIOServiceAsynchronous);
 688             allSet->removeObject(service);
 689         }
 690         allSet->release();
 691     }
 692
 693     newTables->release();
 694
 695     return( kIOReturnSuccess );
 696 }
 697
 698  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 699                                                 IOOptionBits options )
 700 {
 701     _IOServiceJob *     job;
 702
 703     job = new _IOServiceJob;
 704     if( job && !job->init()) {
 705         job->release();
 706         job = 0;
 707     }
 708
 709     if( job) {
 710         job->type       = type;
 711         job->nub        = nub;
 712         job->options    = options;
 713         nub->retain();                  // thread will release()
 714         pingConfig( job );
 715     }
 716
 717     return( job );
 718 }
 719
 720 /*
 721  * Called on a registered service to see if it matches
 722  * a property table.
 723  */
 724
 725 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 726 {
 727     return( matchPropertyTable(table) );
 728 }
 729
 730 bool IOService::matchPropertyTable( OSDictionary * table )
 731 {
 732     return( true );
 733 }
 734
 735 /*
 736  * Called on a matched service to allocate resources
 737  * before first driver is attached.
 738  */
 739
 740 IOReturn IOService::getResources( void )
 741 {
 742     return( kIOReturnSuccess);
 743 }
 744
 745 /*
 746  * Client/provider accessors
 747  */
 748
 749 IOService * IOService::getProvider( void ) const
 750 {
 751     IOService * self = (IOService *) this;
 752     IOService * parent;
 753     SInt32      generation;
 754
 755     parent = __provider;
 756     generation = getGenerationCount();
 757     if( __providerGeneration == generation)
 758         return( parent );
 759
 760     parent = (IOService *) getParentEntry( gIOServicePlane);
 761     if( parent == IORegistryEntry::getRegistryRoot())
 762         /* root is not an IOService */
 763         parent = 0;
 764
 765     self->__provider = parent;
 766     // save the count before getParentEntry()
 767     self->__providerGeneration = generation;
 768
 769     return( parent );
 770 }
 771
 772 IOWorkLoop * IOService::getWorkLoop() const
 773 {
 774     IOService *provider = getProvider();
 775
 776     if (provider)
 777         return provider->getWorkLoop();
 778     else
 779         return 0;
 780 }
 781
 782 OSIterator * IOService::getProviderIterator( void ) const
 783 {
 784     return( getParentIterator( gIOServicePlane));
 785 }
 786
 787 IOService * IOService::getClient( void ) const
 788 {
 789     return( (IOService *) getChildEntry( gIOServicePlane));
 790 }
 791
 792 OSIterator * IOService::getClientIterator( void ) const
 793 {
 794     return( getChildIterator( gIOServicePlane));
 795 }
 796
 797 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
 798                                                 const IOService * client,
 799                                                 const IOService * provider )
 800 {
 801     _IOOpenServiceIterator * inst;
 802
 803     if( !_iter)
 804         return( 0 );
 805
 806     inst = new _IOOpenServiceIterator;
 807
 808     if( inst && !inst->init()) {
 809         inst->release();
 810         inst = 0;
 811     }
 812     if( inst) {
 813         inst->iter = _iter;
 814         inst->client = client;
 815         inst->provider = provider;
 816     }
 817
 818     return( inst );
 819 }
 820
 821 void _IOOpenServiceIterator::free()
 822 {
 823     iter->release();
 824     if( last)
 825         last->unlockForArbitration();
 826     OSIterator::free();
 827 }
 828
 829 OSObject * _IOOpenServiceIterator::getNextObject()
 830 {
 831     IOService * next;
 832
 833     if( last)
 834         last->unlockForArbitration();
 835
 836     while( (next = (IOService *) iter->getNextObject())) {
 837
 838         next->lockForArbitration();
 839         if( (client && (next->isOpen( client )))
 840          || (provider && (provider->isOpen( next ))) )
 841             break;
 842         next->unlockForArbitration();
 843     }
 844
 845     last = next;
 846
 847     return( next );
 848 }
 849
 850 bool _IOOpenServiceIterator::isValid()
 851 {
 852     return( iter->isValid() );
 853 }
 854
 855 void _IOOpenServiceIterator::reset()
 856 {
 857     if( last) {
 858         last->unlockForArbitration();
 859         last = 0;
 860     }
 861     iter->reset();
 862 }
 863
 864 OSIterator * IOService::getOpenProviderIterator( void ) const
 865 {
 866     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
 867 }
 868
 869 OSIterator * IOService::getOpenClientIterator( void ) const
 870 {
 871     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
 872 }
 873
 874
 875 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
 876                                           bool waitForFunction,
 877                                           void *param1, void *param2,
 878                                           void *param3, void *param4 )
 879 {
 880   IOReturn  result = kIOReturnUnsupported;
 881   IOService *provider;
 882
 883   if (gIOPlatformFunctionHandlerSet == functionName)
 884   {
 885 #if defined(__i386__) || defined(__x86_64__)
 886     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
 887     IOService *      target              = (IOService *) param2;
 888     bool             enable              = (param3 != 0);
 889
 890     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
 891         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
 892     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
 893         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
 894 #endif /* defined(__i386__) || defined(__x86_64__) */
 895   }
 896
 897   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
 898     result = provider->callPlatformFunction(functionName, waitForFunction,
 899                                             param1, param2, param3, param4);
 900   }
 901
 902   return result;
 903 }
 904
 905 IOReturn IOService::callPlatformFunction( const char * functionName,
 906                                           bool waitForFunction,
 907                                           void *param1, void *param2,
 908                                           void *param3, void *param4 )
 909 {
 910   IOReturn result = kIOReturnNoMemory;
 911   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
 912
 913   if (functionSymbol != 0) {
 914     result = callPlatformFunction(functionSymbol, waitForFunction,
 915                                   param1, param2, param3, param4);
 916     functionSymbol->release();
 917   }
 918
 919   return result;
 920 }
 921
 922
 923 /*
 924  * Accessors for global services
 925  */
 926
 927 IOPlatformExpert * IOService::getPlatform( void )
 928 {
 929     return( gIOPlatform);
 930 }
 931
 932 class IOPMrootDomain * IOService::getPMRootDomain( void )
 933 {
 934     return( gIOPMRootDomain);
 935 }
 936
 937 IOService * IOService::getResourceService( void )
 938 {
 939     return( gIOResources );
 940 }
 941
 942 void IOService::setPlatform( IOPlatformExpert * platform)
 943 {
 944     gIOPlatform = platform;
 945     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
 946 }
 947
 948 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
 949 {
 950     gIOPMRootDomain = rootDomain;
 951     publishResource("IOKit");
 952 }
 953
 954 /*
 955  * Stacking change
 956  */
 957
 958 bool IOService::lockForArbitration( bool isSuccessRequired )
 959 {
 960     bool                          found;
 961     bool                          success;
 962     ArbitrationLockQueueElement * element;
 963     ArbitrationLockQueueElement * active;
 964     ArbitrationLockQueueElement * waiting;
 965
 966     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
 967
 968     // lock global access
 969     IOTakeLock( gArbitrationLockQueueLock );
 970
 971     // obtain an unused queue element
 972     if( !queue_empty( &gArbitrationLockQueueFree )) {
 973         queue_remove_first( &gArbitrationLockQueueFree,
 974                             element,
 975                             ArbitrationLockQueueElement *,
 976                             link );
 977     } else {
 978         element = IONew( ArbitrationLockQueueElement, 1 );
 979         assert( element );
 980     }
 981
 982     // prepare the queue element
 983     element->thread   = IOThreadSelf();
 984     element->service  = this;
 985     element->count    = 1;
 986     element->required = isSuccessRequired;
 987     element->aborted  = false;
 988
 989     // determine whether this object is already locked (ie. on active queue)
 990     found = false;
 991     queue_iterate( &gArbitrationLockQueueActive,
 992                     active,
 993                     ArbitrationLockQueueElement *,
 994                     link )
 995     {
 996         if( active->service == element->service ) {
 997             found = true;
 998             break;
 999         }
1000     }
1001
1002     if( found ) { // this object is already locked
1003
1004         // determine whether it is the same or a different thread trying to lock
1005         if( active->thread != element->thread ) { // it is a different thread
1006
1007             ArbitrationLockQueueElement * victim = 0;
1008
1009             // before placing this new thread on the waiting queue, we look for
1010             // a deadlock cycle...
1011
1012             while( 1 ) {
1013                 // determine whether the active thread holding the object we
1014                 // want is waiting for another object to be unlocked
1015                 found = false;
1016                 queue_iterate( &gArbitrationLockQueueWaiting,
1017                                waiting,
1018                                ArbitrationLockQueueElement *,
1019                                link )
1020                 {
1021                     if( waiting->thread == active->thread ) {
1022                         assert( false == waiting->aborted );
1023                         found = true;
1024                         break;
1025                     }
1026                 }
1027
1028                 if( found ) { // yes, active thread waiting for another object
1029
1030                     // this may be a candidate for rejection if the required
1031                     // flag is not set, should we detect a deadlock later on
1032                     if( false == waiting->required )
1033                         victim = waiting;
1034
1035                     // find the thread that is holding this other object, that
1036                     // is blocking the active thread from proceeding (fun :-)
1037                     found = false;
1038                     queue_iterate( &gArbitrationLockQueueActive,
1039                                    active,      // (reuse active queue element)
1040                                    ArbitrationLockQueueElement *,
1041                                    link )
1042                     {
1043                         if( active->service == waiting->service ) {
1044                             found = true;
1045                             break;
1046                         }
1047                     }
1048
1049                     // someone must be holding it or it wouldn't be waiting
1050                     assert( found );
1051
1052                     if( active->thread == element->thread ) {
1053
1054                         // doh, it's waiting for the thread that originated
1055                         // this whole lock (ie. current thread) -> deadlock
1056                         if( false == element->required ) { // willing to fail?
1057
1058                             // the originating thread doesn't have the required
1059                             // flag, so it can fail
1060                             success = false; // (fail originating lock request)
1061                             break; // (out of while)
1062
1063                         } else { // originating thread is not willing to fail
1064
1065                             // see if we came across a waiting thread that did
1066                             // not have the 'required' flag set: we'll fail it
1067                             if( victim ) {
1068
1069                                 // we do have a willing victim, fail it's lock
1070                                 victim->aborted = true;
1071
1072                                 // take the victim off the waiting queue
1073                                 queue_remove( &gArbitrationLockQueueWaiting,
1074                                               victim,
1075                                               ArbitrationLockQueueElement *,
1076                                               link );
1077
1078                                 // wake the victim
1079                                 IOLockWakeup( gArbitrationLockQueueLock,
1080                                               victim,
1081                                               /* one thread */ true );
1082
1083                                 // allow this thread to proceed (ie. wait)
1084                                 success = true; // (put request on wait queue)
1085                                 break; // (out of while)
1086                             } else {
1087
1088                                 // all the waiting threads we came across in
1089                                 // finding this loop had the 'required' flag
1090                                 // set, so we've got a deadlock we can't avoid
1091                                 panic("I/O Kit: Unrecoverable deadlock.");
1092                             }
1093                         }
1094                     } else {
1095                         // repeat while loop, redefining active thread to be the
1096                         // thread holding "this other object" (see above), and
1097                         // looking for threads waiting on it; note the active
1098                         // variable points to "this other object" already... so
1099                         // there nothing to do in this else clause.
1100                     }
1101                 } else { // no, active thread is not waiting for another object
1102
1103                     success = true; // (put request on wait queue)
1104                     break; // (out of while)
1105                 }
1106             } // while forever
1107
1108             if( success ) { // put the request on the waiting queue?
1109                 kern_return_t wait_result;
1110
1111                 // place this thread on the waiting queue and put it to sleep;
1112                 // we place it at the tail of the queue...
1113                 queue_enter( &gArbitrationLockQueueWaiting,
1114                              element,
1115                              ArbitrationLockQueueElement *,
1116                              link );
1117
1118                 // declare that this thread will wait for a given event
1119 restart_sleep:  wait_result = assert_wait( element,
1120                                            element->required ? THREAD_UNINT
1121                                            : THREAD_INTERRUPTIBLE );
1122
1123                 // unlock global access
1124                 IOUnlock( gArbitrationLockQueueLock );
1125
1126                 // put thread to sleep, waiting for our event to fire...
1127                 if (wait_result == THREAD_WAITING)
1128                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1129
1130
1131                 // ...and we've been woken up; we might be in one of two states:
1132                 // (a) we've been aborted and our queue element is not on
1133                 //     any of the three queues, but is floating around
1134                 // (b) we're allowed to proceed with the lock and we have
1135                 //     already been moved from the waiting queue to the
1136                 //     active queue.
1137                 // ...plus a 3rd state, should the thread have been interrupted:
1138                 // (c) we're still on the waiting queue
1139
1140                 // determine whether we were interrupted out of our sleep
1141                 if( THREAD_INTERRUPTED == wait_result ) {
1142
1143                     // re-lock global access
1144                     IOTakeLock( gArbitrationLockQueueLock );
1145
1146                     // determine whether we're still on the waiting queue
1147                     found = false;
1148                     queue_iterate( &gArbitrationLockQueueWaiting,
1149                                    waiting,     // (reuse waiting queue element)
1150                                    ArbitrationLockQueueElement *,
1151                                    link )
1152                     {
1153                         if( waiting == element ) {
1154                             found = true;
1155                             break;
1156                         }
1157                     }
1158
1159                     if( found ) { // yes, we're still on the waiting queue
1160
1161                         // determine whether we're willing to fail
1162                         if( false == element->required ) {
1163
1164                             // mark us as aborted
1165                             element->aborted = true;
1166
1167                             // take us off the waiting queue
1168                             queue_remove( &gArbitrationLockQueueWaiting,
1169                                           element,
1170                                           ArbitrationLockQueueElement *,
1171                                           link );
1172                         } else { // we are not willing to fail
1173
1174                             // ignore interruption, go back to sleep
1175                             goto restart_sleep;
1176                         }
1177                     }
1178
1179                     // unlock global access
1180                     IOUnlock( gArbitrationLockQueueLock );
1181
1182                     // proceed as though this were a normal wake up
1183                     wait_result = THREAD_AWAKENED;
1184                 }
1185
1186                 assert( THREAD_AWAKENED == wait_result );
1187
1188                 // determine whether we've been aborted while we were asleep
1189                 if( element->aborted ) {
1190                     assert( false == element->required );
1191
1192                     // re-lock global access
1193                     IOTakeLock( gArbitrationLockQueueLock );
1194
1195                     action = kPutOnFreeQueue;
1196                     success = false;
1197                 } else { // we weren't aborted, so we must be ready to go :-)
1198
1199                     // we've already been moved from waiting to active queue
1200                     return true;
1201                 }
1202
1203             } else { // the lock request is to be failed
1204
1205                 // return unused queue element to queue
1206                 action = kPutOnFreeQueue;
1207             }
1208         } else { // it is the same thread, recursive access is allowed
1209
1210             // add one level of recursion
1211             active->count++;
1212
1213             // return unused queue element to queue
1214             action = kPutOnFreeQueue;
1215             success = true;
1216         }
1217     } else { // this object is not already locked, so let this thread through
1218         action = kPutOnActiveQueue;
1219         success = true;
1220     }
1221
1222     // put the new element on a queue
1223     if( kPutOnActiveQueue == action ) {
1224         queue_enter( &gArbitrationLockQueueActive,
1225                      element,
1226                      ArbitrationLockQueueElement *,
1227                      link );
1228     } else if( kPutOnFreeQueue == action ) {
1229         queue_enter( &gArbitrationLockQueueFree,
1230                      element,
1231                      ArbitrationLockQueueElement *,
1232                      link );
1233     } else {
1234         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1235     }
1236
1237     // unlock global access
1238     IOUnlock( gArbitrationLockQueueLock );
1239
1240     return( success );
1241 }
1242
1243 void IOService::unlockForArbitration( void )
1244 {
1245     bool                          found;
1246     ArbitrationLockQueueElement * element;
1247
1248     // lock global access
1249     IOTakeLock( gArbitrationLockQueueLock );
1250
1251     // find the lock element for this object (ie. on active queue)
1252     found = false;
1253     queue_iterate( &gArbitrationLockQueueActive,
1254                     element,
1255                     ArbitrationLockQueueElement *,
1256                     link )
1257     {
1258         if( element->service == this ) {
1259             found = true;
1260             break;
1261         }
1262     }
1263
1264     assert( found );
1265
1266     // determine whether the lock has been taken recursively
1267     if( element->count > 1 ) {
1268         // undo one level of recursion
1269         element->count--;
1270
1271     } else {
1272
1273         // remove it from the active queue
1274         queue_remove( &gArbitrationLockQueueActive,
1275                       element,
1276                       ArbitrationLockQueueElement *,
1277                       link );
1278
1279         // put it on the free queue
1280         queue_enter( &gArbitrationLockQueueFree,
1281                      element,
1282                      ArbitrationLockQueueElement *,
1283                      link );
1284
1285         // determine whether a thread is waiting for object (head to tail scan)
1286         found = false;
1287         queue_iterate( &gArbitrationLockQueueWaiting,
1288                        element,
1289                        ArbitrationLockQueueElement *,
1290                        link )
1291         {
1292             if( element->service == this ) {
1293                 found = true;
1294                 break;
1295             }
1296         }
1297
1298         if ( found ) { // we found an interested thread on waiting queue
1299
1300             // remove it from the waiting queue
1301             queue_remove( &gArbitrationLockQueueWaiting,
1302                           element,
1303                           ArbitrationLockQueueElement *,
1304                           link );
1305
1306             // put it on the active queue
1307             queue_enter( &gArbitrationLockQueueActive,
1308                          element,
1309                          ArbitrationLockQueueElement *,
1310                          link );
1311
1312             // wake the waiting thread
1313             IOLockWakeup( gArbitrationLockQueueLock,
1314                           element,
1315                           /* one thread */ true );
1316         }
1317     }
1318
1319     // unlock global access
1320     IOUnlock( gArbitrationLockQueueLock );
1321 }
1322
1323 void IOService::applyToProviders( IOServiceApplierFunction applier,
1324                                   void * context )
1325 {
1326     applyToParents( (IORegistryEntryApplierFunction) applier,
1327                     context, gIOServicePlane );
1328 }
1329
1330 void IOService::applyToClients( IOServiceApplierFunction applier,
1331                                 void * context )
1332 {
1333     applyToChildren( (IORegistryEntryApplierFunction) applier,
1334                      context, gIOServicePlane );
1335 }
1336
1337
1338 /*
1339  * Client messages
1340  */
1341
1342
1343 // send a message to a client or interested party of this service
1344 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1345                                    void * argument, vm_size_t argSize )
1346 {
1347     IOReturn                            ret;
1348     IOService *                         service;
1349     _IOServiceInterestNotifier *        notify;
1350
1351     if( (service = OSDynamicCast( IOService, client)))
1352         ret = service->message( type, this, argument );
1353
1354     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1355
1356         _IOServiceNotifierInvocation invocation;
1357         bool                     willNotify;
1358
1359         invocation.thread = current_thread();
1360
1361         LOCKWRITENOTIFY();
1362         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1363
1364         if( willNotify) {
1365             queue_enter( &notify->handlerInvocations, &invocation,
1366                             _IOServiceNotifierInvocation *, link );
1367         }
1368         UNLOCKNOTIFY();
1369
1370         if( willNotify) {
1371
1372             ret = (*notify->handler)( notify->target, notify->ref,
1373                                           type, this, argument, argSize );
1374
1375             LOCKWRITENOTIFY();
1376             queue_remove( &notify->handlerInvocations, &invocation,
1377                             _IOServiceNotifierInvocation *, link );
1378             if( kIOServiceNotifyWaiter & notify->state) {
1379                 notify->state &= ~kIOServiceNotifyWaiter;
1380                 WAKEUPNOTIFY( notify );
1381             }
1382             UNLOCKNOTIFY();
1383
1384         } else
1385             ret = kIOReturnSuccess;
1386
1387     } else
1388         ret = kIOReturnBadArgument;
1389
1390     return( ret );
1391 }
1392
1393 static void
1394 applyToInterestNotifiers(const IORegistryEntry *target,
1395                          const OSSymbol * typeOfInterest,
1396                          OSObjectApplierFunction applier,
1397                          void * context )
1398 {
1399     OSArray *           copyArray = 0;
1400
1401     LOCKREADNOTIFY();
1402
1403     IOCommand *notifyList =
1404         OSDynamicCast( IOCommand, target->getProperty( typeOfInterest ));
1405
1406     if( notifyList) {
1407         copyArray = OSArray::withCapacity(1);
1408
1409         // iterate over queue, entry is set to each element in the list
1410         iterqueue(&notifyList->fCommandChain, entry) {
1411             _IOServiceInterestNotifier * notify;
1412
1413             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1414             copyArray->setObject(notify);
1415         }
1416     }
1417     UNLOCKNOTIFY();
1418
1419     if( copyArray) {
1420         unsigned int    index;
1421         OSObject *      next;
1422
1423         for( index = 0; (next = copyArray->getObject( index )); index++)
1424             (*applier)(next, context);
1425         copyArray->release();
1426     }
1427 }
1428
1429 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1430                                    OSObjectApplierFunction applier,
1431                                    void * context )
1432 {
1433     if (gIOGeneralInterest == typeOfInterest)
1434         applyToClients( (IOServiceApplierFunction) applier, context );
1435     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1436 }
1437
1438 struct MessageClientsContext {
1439     IOService * service;
1440     UInt32      type;
1441     void *      argument;
1442     vm_size_t   argSize;
1443     IOReturn    ret;
1444 };
1445
1446 static void messageClientsApplier( OSObject * object, void * ctx )
1447 {
1448     IOReturn                ret;
1449     MessageClientsContext * context = (MessageClientsContext *) ctx;
1450
1451     ret = context->service->messageClient( context->type,
1452                                            object, context->argument, context->argSize );
1453     if( kIOReturnSuccess != ret)
1454         context->ret = ret;
1455 }
1456
1457 // send a message to all clients
1458 IOReturn IOService::messageClients( UInt32 type,
1459                                     void * argument, vm_size_t argSize )
1460 {
1461     MessageClientsContext       context;
1462
1463     context.service     = this;
1464     context.type        = type;
1465     context.argument    = argument;
1466     context.argSize     = argSize;
1467     context.ret         = kIOReturnSuccess;
1468
1469     applyToInterested( gIOGeneralInterest,
1470                        &messageClientsApplier, &context );
1471
1472     return( context.ret );
1473 }
1474
1475 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1476                                               IOOptionBits response )
1477 {
1478     return( kIOReturnUnsupported );
1479 }
1480
1481 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1482                   IOServiceInterestHandler handler, void * target, void * ref )
1483 {
1484     _IOServiceInterestNotifier * notify = 0;
1485
1486     if( (typeOfInterest != gIOGeneralInterest)
1487      && (typeOfInterest != gIOBusyInterest)
1488      && (typeOfInterest != gIOAppPowerStateInterest)
1489      && (typeOfInterest != gIOConsoleSecurityInterest)
1490      && (typeOfInterest != gIOPriorityPowerStateInterest))
1491         return( 0 );
1492
1493     lockForArbitration();
1494     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1495
1496         notify = new _IOServiceInterestNotifier;
1497         if( notify && !notify->init()) {
1498             notify->release();
1499             notify = 0;
1500         }
1501
1502         if( notify) {
1503             notify->handler = handler;
1504             notify->target = target;
1505             notify->ref = ref;
1506             notify->state = kIOServiceNotifyEnable;
1507             queue_init( &notify->handlerInvocations );
1508
1509             ////// queue
1510
1511             LOCKWRITENOTIFY();
1512
1513             // Get the head of the notifier linked list
1514             IOCommand *notifyList = (IOCommand *) getProperty( typeOfInterest );
1515             if (!notifyList || !OSDynamicCast(IOCommand, notifyList)) {
1516                 notifyList = OSTypeAlloc(IOCommand);
1517                 if (notifyList) {
1518                     notifyList->init();
1519                     setProperty( typeOfInterest, notifyList);
1520                     notifyList->release();
1521                 }
1522             }
1523
1524             if (notifyList) {
1525                 enqueue(&notifyList->fCommandChain, &notify->chain);
1526                 notify->retain();       // ref'ed while in list
1527             }
1528
1529             UNLOCKNOTIFY();
1530         }
1531     }
1532     unlockForArbitration();
1533
1534     return( notify );
1535 }
1536
1537 static void cleanInterestList( OSObject * head )
1538 {
1539     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1540     if (!notifyHead)
1541         return;
1542
1543     LOCKWRITENOTIFY();
1544     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1545         queue_next(entry) = queue_prev(entry) = 0;
1546
1547         _IOServiceInterestNotifier * notify;
1548
1549         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1550         notify->release();
1551     }
1552     UNLOCKNOTIFY();
1553 }
1554
1555 void IOService::unregisterAllInterest( void )
1556 {
1557     cleanInterestList( getProperty( gIOGeneralInterest ));
1558     cleanInterestList( getProperty( gIOBusyInterest ));
1559     cleanInterestList( getProperty( gIOAppPowerStateInterest ));
1560     cleanInterestList( getProperty( gIOPriorityPowerStateInterest ));
1561     cleanInterestList( getProperty( gIOConsoleSecurityInterest ));
1562 }
1563
1564 /*
1565  * _IOServiceInterestNotifier
1566  */
1567
1568 // wait for all threads, other than the current one,
1569 //  to exit the handler
1570
1571 void _IOServiceInterestNotifier::wait()
1572 {
1573     _IOServiceNotifierInvocation * next;
1574     bool doWait;
1575
1576     do {
1577         doWait = false;
1578         queue_iterate( &handlerInvocations, next,
1579                         _IOServiceNotifierInvocation *, link) {
1580             if( next->thread != current_thread() ) {
1581                 doWait = true;
1582                 break;
1583             }
1584         }
1585         if( doWait) {
1586             state |= kIOServiceNotifyWaiter;
1587                SLEEPNOTIFY(this);
1588         }
1589
1590     } while( doWait );
1591 }
1592
1593 void _IOServiceInterestNotifier::free()
1594 {
1595     assert( queue_empty( &handlerInvocations ));
1596     OSObject::free();
1597 }
1598
1599 void _IOServiceInterestNotifier::remove()
1600 {
1601     LOCKWRITENOTIFY();
1602
1603     if( queue_next( &chain )) {
1604         remqueue(&chain);
1605         queue_next( &chain) = queue_prev( &chain) = 0;
1606         release();
1607     }
1608
1609     state &= ~kIOServiceNotifyEnable;
1610
1611     wait();
1612
1613     UNLOCKNOTIFY();
1614
1615     release();
1616 }
1617
1618 bool _IOServiceInterestNotifier::disable()
1619 {
1620     bool        ret;
1621
1622     LOCKWRITENOTIFY();
1623
1624     ret = (0 != (kIOServiceNotifyEnable & state));
1625     state &= ~kIOServiceNotifyEnable;
1626     if( ret)
1627         wait();
1628
1629     UNLOCKNOTIFY();
1630
1631     return( ret );
1632 }
1633
1634 void _IOServiceInterestNotifier::enable( bool was )
1635 {
1636     LOCKWRITENOTIFY();
1637     if( was)
1638         state |= kIOServiceNotifyEnable;
1639     else
1640         state &= ~kIOServiceNotifyEnable;
1641     UNLOCKNOTIFY();
1642 }
1643
1644 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1645
1646 /*
1647  * Termination
1648  */
1649
1650 #define tailQ(o)                setObject(o)
1651 #define headQ(o)                setObject(0, o)
1652 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
1653
1654 static void _workLoopAction( IOWorkLoop::Action action,
1655                              IOService * service,
1656                              void * p0 = 0, void * p1 = 0,
1657                              void * p2 = 0, void * p3 = 0 )
1658 {
1659     IOWorkLoop * wl;
1660
1661     if( (wl = service->getWorkLoop())) {
1662         wl->retain();
1663         wl->runAction( action, service, p0, p1, p2, p3 );
1664         wl->release();
1665     } else
1666         (*action)( service, p0, p1, p2, p3 );
1667 }
1668
1669 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
1670 {
1671     bool ok;
1672
1673     // if its our only provider
1674     ok = isParent( provider, gIOServicePlane, true);
1675
1676     // -- compat
1677     if( ok) {
1678         provider->terminateClient( this, options | kIOServiceRecursing );
1679         ok = (0 != (__state[1] & kIOServiceRecursing));
1680     }
1681     // --
1682
1683     return( ok );
1684 }
1685
1686 bool IOService::terminatePhase1( IOOptionBits options )
1687 {
1688     IOService *  victim;
1689     IOService *  client;
1690     OSIterator * iter;
1691     OSArray *    makeInactive;
1692         int          waitResult = THREAD_AWAKENED;
1693         bool         wait;
1694     bool                 ok;
1695     bool                 didInactive;
1696     bool                 startPhase2 = false;
1697
1698     TLOG("%s::terminatePhase1(%08llx)\n", getName(), (long long)options);
1699
1700     uint64_t regID = getRegistryEntryID();
1701     IOServiceTrace(
1702         IOSERVICE_TERMINATE_PHASE1,
1703         (uintptr_t) regID,
1704         (uintptr_t) (regID >> 32),
1705         (uintptr_t) this,
1706         (uintptr_t) options);
1707
1708     // -- compat
1709     if( options & kIOServiceRecursing) {
1710         __state[1] |= kIOServiceRecursing;
1711         return( true );
1712     }
1713     // --
1714
1715     makeInactive = OSArray::withCapacity( 16 );
1716     if( !makeInactive)
1717         return( false );
1718
1719     victim = this;
1720     victim->retain();
1721
1722     while( victim ) {
1723
1724                 didInactive = victim->lockForArbitration( true );
1725         if( didInactive) {
1726             didInactive = (0 == (victim->__state[0] & kIOServiceInactiveState));
1727             if( didInactive) {
1728                 victim->__state[0] |= kIOServiceInactiveState;
1729                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
1730                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
1731
1732                                 if (victim == this)
1733                                         victim->__state[1] |= kIOServiceTermPhase1State;
1734
1735                 victim->_adjustBusy( 1 );
1736
1737             } else if (victim != this) do {
1738
1739                                 IOLockLock(gIOServiceBusyLock);
1740                                 wait = (victim->__state[1] & kIOServiceTermPhase1State);
1741                                 if( wait) {
1742                                     TLOG("%s::waitPhase1(%s)\n", getName(), victim->getName());
1743                                         victim->__state[1] |= kIOServiceTerm1WaiterState;
1744                                         victim->unlockForArbitration();
1745                                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
1746                                 }
1747                                 IOLockUnlock(gIOServiceBusyLock);
1748                                 if( wait) {
1749                                         waitResult = thread_block(THREAD_CONTINUE_NULL);
1750                                     TLOG("%s::did waitPhase1(%s)\n", getName(), victim->getName());
1751                                         victim->lockForArbitration();
1752                                 }
1753                         } while( wait && (waitResult != THREAD_TIMED_OUT));
1754
1755                         victim->unlockForArbitration();
1756         }
1757         if( victim == this)
1758             startPhase2 = didInactive;
1759         if( didInactive) {
1760
1761             victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff );
1762             IOUserClient::destroyUserReferences( victim );
1763
1764             iter = victim->getClientIterator();
1765             if( iter) {
1766                 while( (client = (IOService *) iter->getNextObject())) {
1767                     TLOG("%s::requestTerminate(%s, %08llx)\n",
1768                             client->getName(), victim->getName(), (long long)options);
1769                     ok = client->requestTerminate( victim, options );
1770                     TLOG("%s::requestTerminate(%s, ok = %d)\n",
1771                             client->getName(), victim->getName(), ok);
1772
1773                     uint64_t regID1 = client->getRegistryEntryID();
1774                     uint64_t regID2 = victim->getRegistryEntryID();
1775                     IOServiceTrace(
1776                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
1777                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
1778                         (uintptr_t) regID1,
1779                         (uintptr_t) (regID1 >> 32),
1780                         (uintptr_t) regID2,
1781                         (uintptr_t) (regID2 >> 32));
1782
1783                     if( ok)
1784                         makeInactive->setObject( client );
1785                 }
1786                 iter->release();
1787             }
1788         }
1789         victim->release();
1790         victim = (IOService *) makeInactive->getObject(0);
1791         if( victim) {
1792             victim->retain();
1793             makeInactive->removeObject(0);
1794         }
1795     }
1796
1797     makeInactive->release();
1798
1799     if( startPhase2)
1800     {
1801                 lockForArbitration();
1802                 __state[1] &= ~kIOServiceTermPhase1State;
1803                 if (kIOServiceTerm1WaiterState & __state[1])
1804                 {
1805                         __state[1] &= ~kIOServiceTerm1WaiterState;
1806                         TLOG("%s::wakePhase1\n", getName());
1807                         IOLockLock( gIOServiceBusyLock );
1808                         thread_wakeup( (event_t) &__state[1]);
1809                         IOLockUnlock( gIOServiceBusyLock );
1810                 }
1811                 unlockForArbitration();
1812
1813         scheduleTerminatePhase2( options );
1814     }
1815     return( true );
1816 }
1817
1818 void IOService::scheduleTerminatePhase2( IOOptionBits options )
1819 {
1820     AbsoluteTime        deadline;
1821     int                 waitResult = THREAD_AWAKENED;
1822     bool                wait, haveDeadline = false;
1823
1824     options |= kIOServiceRequired;
1825
1826     retain();
1827
1828     IOLockLock( gJobsLock );
1829
1830     if( (options & kIOServiceSynchronous)
1831         && (current_thread() != gIOTerminateThread)) {
1832
1833         do {
1834             wait = (gIOTerminateThread != 0);
1835             if( wait) {
1836                 // wait to become the terminate thread
1837                 IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT);
1838             }
1839         } while( wait );
1840
1841         gIOTerminateThread = current_thread();
1842         gIOTerminatePhase2List->setObject( this );
1843         gIOTerminateWork++;
1844
1845         do {
1846             while( gIOTerminateWork )
1847                 terminateWorker( options );
1848             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
1849             if( wait) {
1850                 // wait for the victim to go non-busy
1851                 if( !haveDeadline) {
1852                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
1853                     haveDeadline = true;
1854                 }
1855                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
1856                                                   deadline, THREAD_UNINT );
1857                 if( waitResult == THREAD_TIMED_OUT) {
1858                     IOLog("%s::terminate(kIOServiceSynchronous) timeout\n", getName());
1859                 }
1860             }
1861         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
1862
1863         gIOTerminateThread = 0;
1864         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
1865
1866     } else {
1867         // ! kIOServiceSynchronous
1868
1869         gIOTerminatePhase2List->setObject( this );
1870         if( 0 == gIOTerminateWork++) {
1871             if( !gIOTerminateThread)
1872                 kernel_thread_start(&terminateThread, (void *) options, &gIOTerminateThread);
1873             else
1874                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
1875         }
1876     }
1877
1878     IOLockUnlock( gJobsLock );
1879
1880     release();
1881 }
1882
1883 void IOService::terminateThread( void * arg, wait_result_t waitResult )
1884 {
1885     IOLockLock( gJobsLock );
1886
1887     while (gIOTerminateWork)
1888         terminateWorker( (uintptr_t) arg );
1889
1890     thread_deallocate(gIOTerminateThread);
1891     gIOTerminateThread = 0;
1892     IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
1893
1894     IOLockUnlock( gJobsLock );
1895 }
1896
1897 void IOService::scheduleStop( IOService * provider )
1898 {
1899     TLOG("%s::scheduleStop(%s)\n", getName(), provider->getName());
1900
1901     uint64_t regID1 = getRegistryEntryID();
1902     uint64_t regID2 = provider->getRegistryEntryID();
1903     IOServiceTrace(
1904         IOSERVICE_TERMINATE_SCHEDULE_STOP,
1905         (uintptr_t) regID1,
1906         (uintptr_t) (regID1 >> 32),
1907         (uintptr_t) regID2,
1908         (uintptr_t) (regID2 >> 32));
1909
1910     IOLockLock( gJobsLock );
1911     gIOStopList->tailQ( this );
1912     gIOStopProviderList->tailQ( provider );
1913
1914     if( 0 == gIOTerminateWork++) {
1915         if( !gIOTerminateThread)
1916             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
1917         else
1918             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
1919     }
1920
1921     IOLockUnlock( gJobsLock );
1922 }
1923
1924 void IOService::scheduleFinalize( void )
1925 {
1926     TLOG("%s::scheduleFinalize\n", getName());
1927
1928     uint64_t regID1 = getRegistryEntryID();
1929     IOServiceTrace(
1930         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
1931         (uintptr_t) regID1,
1932         (uintptr_t) (regID1 >> 32),
1933         0, 0);
1934
1935     IOLockLock( gJobsLock );
1936     gIOFinalizeList->tailQ( this );
1937
1938     if( 0 == gIOTerminateWork++) {
1939         if( !gIOTerminateThread)
1940             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
1941         else
1942             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
1943     }
1944
1945     IOLockUnlock( gJobsLock );
1946 }
1947
1948 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
1949 {
1950     return( true );
1951 }
1952
1953 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
1954 {
1955     if( false == *defer) {
1956
1957         if( lockForArbitration( true )) {
1958             if( false == provider->handleIsOpen( this ))
1959                 scheduleStop( provider );
1960             // -- compat
1961             else {
1962                 message( kIOMessageServiceIsRequestingClose, provider, (void *) options );
1963                 if( false == provider->handleIsOpen( this ))
1964                     scheduleStop( provider );
1965             }
1966             // --
1967             unlockForArbitration();
1968         }
1969     }
1970
1971     return( true );
1972 }
1973
1974 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
1975                                      OSArray * doPhase2List,
1976                                      void *unused2 __unused,
1977                                      void *unused3 __unused  )
1978 {
1979     OSIterator * iter;
1980     IOService *  client;
1981     bool         ok;
1982
1983     iter = victim->getClientIterator();
1984     if( iter) {
1985         while( (client = (IOService *) iter->getNextObject())) {
1986             TLOG("%s::willTerminate(%s, %08llx)\n",
1987                     client->getName(), victim->getName(), (long long)options);
1988
1989             uint64_t regID1 = client->getRegistryEntryID();
1990             uint64_t regID2 = victim->getRegistryEntryID();
1991             IOServiceTrace(
1992                 IOSERVICE_TERMINATE_WILL,
1993                 (uintptr_t) regID1,
1994                 (uintptr_t) (regID1 >> 32),
1995                 (uintptr_t) regID2,
1996                 (uintptr_t) (regID2 >> 32));
1997
1998             ok = client->willTerminate( victim, options );
1999             doPhase2List->tailQ( client );
2000         }
2001         iter->release();
2002     }
2003 }
2004
2005 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2006                             void *unused1 __unused, void *unused2 __unused,
2007                             void *unused3 __unused )
2008 {
2009     OSIterator * iter;
2010     IOService *  client;
2011     bool defer = false;
2012
2013     victim->messageClients( kIOMessageServiceIsTerminated, (void *) options );
2014
2015     iter = victim->getClientIterator();
2016     if( iter) {
2017         while( (client = (IOService *) iter->getNextObject())) {
2018             TLOG("%s::didTerminate(%s, %08llx)\n",
2019                     client->getName(), victim->getName(), (long long)options);
2020             client->didTerminate( victim, options, &defer );
2021
2022             uint64_t regID1 = client->getRegistryEntryID();
2023             uint64_t regID2 = victim->getRegistryEntryID();
2024             IOServiceTrace(
2025                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2026                        : IOSERVICE_TERMINATE_DID),
2027                 (uintptr_t) regID1,
2028                 (uintptr_t) (regID1 >> 32),
2029                 (uintptr_t) regID2,
2030                 (uintptr_t) (regID2 >> 32));
2031
2032             TLOG("%s::didTerminate(%s, defer %d)\n",
2033                     client->getName(), victim->getName(), defer);
2034         }
2035         iter->release();
2036     }
2037 }
2038
2039 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2040                             void *unused1 __unused, void *unused2 __unused,
2041                             void *unused3 __unused )
2042 {
2043     TLOG("%s::finalize(%08llx)\n", victim->getName(), (long long)options);
2044
2045     uint64_t regID1 = victim->getRegistryEntryID();
2046     IOServiceTrace(
2047         IOSERVICE_TERMINATE_FINALIZE,
2048         (uintptr_t) regID1,
2049         (uintptr_t) (regID1 >> 32),
2050         0, 0);
2051
2052     victim->finalize( options );
2053 }
2054
2055 void IOService::actionStop( IOService * provider, IOService * client,
2056                             void *unused1 __unused, void *unused2 __unused,
2057                             void *unused3 __unused )
2058 {
2059     TLOG("%s::stop(%s)\n", client->getName(), provider->getName());
2060
2061     uint64_t regID1 = provider->getRegistryEntryID();
2062     uint64_t regID2 = client->getRegistryEntryID();
2063     IOServiceTrace(
2064         IOSERVICE_TERMINATE_STOP,
2065         (uintptr_t) regID1,
2066         (uintptr_t) (regID1 >> 32),
2067         (uintptr_t) regID2,
2068         (uintptr_t) (regID2 >> 32));
2069
2070     client->stop( provider );
2071     if( provider->isOpen( client ))
2072         provider->close( client );
2073     TLOG("%s::detach(%s)\n", client->getName(), provider->getName());
2074     client->detach( provider );
2075 }
2076
2077 void IOService::terminateWorker( IOOptionBits options )
2078 {
2079     OSArray *           doPhase2List;
2080     OSArray *           didPhase2List;
2081     OSSet *             freeList;
2082     UInt32              workDone;
2083     IOService *         victim;
2084     IOService *         client;
2085     IOService *         provider;
2086     unsigned int        idx;
2087     bool                moreToDo;
2088     bool                doPhase2;
2089     bool                doPhase3;
2090
2091     options |= kIOServiceRequired;
2092
2093     doPhase2List  = OSArray::withCapacity( 16 );
2094     didPhase2List = OSArray::withCapacity( 16 );
2095     freeList      = OSSet::withCapacity( 16 );
2096     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2097         return;
2098
2099     do {
2100         workDone = gIOTerminateWork;
2101
2102         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2103
2104             victim->retain();
2105             gIOTerminatePhase2List->removeObject(0);
2106             IOLockUnlock( gJobsLock );
2107
2108             while( victim ) {
2109
2110                 doPhase2 = victim->lockForArbitration( true );
2111                 if( doPhase2) {
2112                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2113                     if( doPhase2) {
2114                         doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State))
2115                                 && (0 == (victim->__state[1] & kIOServiceConfigState));
2116                         if( doPhase2)
2117                             victim->__state[1] |= kIOServiceTermPhase2State;
2118                     }
2119                     victim->unlockForArbitration();
2120                 }
2121                 if( doPhase2) {
2122                     if( 0 == victim->getClient()) {
2123                         // no clients - will go to finalize
2124                         IOLockLock( gJobsLock );
2125                         gIOFinalizeList->tailQ( victim );
2126                         IOLockUnlock( gJobsLock );
2127                     } else {
2128                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2129                                             victim, (void *) options, (void *) doPhase2List );
2130                     }
2131                     didPhase2List->headQ( victim );
2132                 }
2133                 victim->release();
2134                 victim = (IOService *) doPhase2List->getObject(0);
2135                 if( victim) {
2136                     victim->retain();
2137                     doPhase2List->removeObject(0);
2138                 }
2139             }
2140
2141             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2142
2143                 if( victim->lockForArbitration( true )) {
2144                     victim->__state[1] |= kIOServiceTermPhase3State;
2145                     victim->unlockForArbitration();
2146                 }
2147                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2148                                     victim, (void *) options );
2149                 didPhase2List->removeObject(0);
2150             }
2151             IOLockLock( gJobsLock );
2152         }
2153
2154         // phase 3
2155         do {
2156             doPhase3 = false;
2157             // finalize leaves
2158             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2159
2160                 IOLockUnlock( gJobsLock );
2161                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2162                                     victim, (void *) options );
2163                 IOLockLock( gJobsLock );
2164                 // hold off free
2165                 freeList->setObject( victim );
2166                 // safe if finalize list is append only
2167                 gIOFinalizeList->removeObject(0);
2168             }
2169
2170             for( idx = 0;
2171                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2172
2173                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2174                 assert( provider );
2175
2176                 if( !provider->isChild( client, gIOServicePlane )) {
2177                     // may be multiply queued - nop it
2178                     TLOG("%s::nop stop(%s)\n", client->getName(), provider->getName());
2179
2180                     uint64_t regID1 = provider->getRegistryEntryID();
2181                     uint64_t regID2 = client->getRegistryEntryID();
2182                     IOServiceTrace(
2183                         IOSERVICE_TERMINATE_STOP_NOP,
2184                         (uintptr_t) regID1,
2185                         (uintptr_t) (regID1 >> 32),
2186                         (uintptr_t) regID2,
2187                         (uintptr_t) (regID2 >> 32));
2188
2189                 } else {
2190                     // a terminated client is not ready for stop if it has clients, skip it
2191                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2192                         TLOG("%s::defer stop(%s)\n", client->getName(), provider->getName());
2193
2194                         uint64_t regID1 = provider->getRegistryEntryID();
2195                         uint64_t regID2 = client->getRegistryEntryID();
2196                         IOServiceTrace(
2197                             IOSERVICE_TERMINATE_STOP_DEFER,
2198                             (uintptr_t) regID1,
2199                             (uintptr_t) (regID1 >> 32),
2200                             (uintptr_t) regID2,
2201                             (uintptr_t) (regID2 >> 32));
2202
2203                         idx++;
2204                         continue;
2205                     }
2206
2207                     IOLockUnlock( gJobsLock );
2208                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2209                                      provider, (void *) client );
2210                     IOLockLock( gJobsLock );
2211                     // check the finalize list now
2212                     doPhase3 = true;
2213                 }
2214                 // hold off free
2215                 freeList->setObject( client );
2216                 freeList->setObject( provider );
2217
2218                 // safe if stop list is append only
2219                 gIOStopList->removeObject( idx );
2220                 gIOStopProviderList->removeObject( idx );
2221                 idx = 0;
2222             }
2223
2224         } while( doPhase3 );
2225
2226         gIOTerminateWork -= workDone;
2227         moreToDo = (gIOTerminateWork != 0);
2228
2229         if( !moreToDo) {
2230             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2231             IOServiceTrace(
2232                 IOSERVICE_TERMINATE_DONE,
2233                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2234         }
2235
2236     } while( moreToDo );
2237
2238     IOLockUnlock( gJobsLock );
2239
2240     freeList->release();
2241     doPhase2List->release();
2242     didPhase2List->release();
2243
2244     IOLockLock( gJobsLock );
2245 }
2246
2247 bool IOService::finalize( IOOptionBits options )
2248 {
2249     OSIterator *        iter;
2250     IOService *         provider;
2251
2252     iter = getProviderIterator();
2253     assert( iter );
2254
2255     if( iter) {
2256         while( (provider = (IOService *) iter->getNextObject())) {
2257
2258             // -- compat
2259             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2260                 /* we come down here on programmatic terminate */
2261                 stop( provider );
2262                 if( provider->isOpen( this ))
2263                     provider->close( this );
2264                 detach( provider );
2265             } else {
2266             //--
2267                 if( provider->lockForArbitration( true )) {
2268                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2269                         scheduleStop( provider );
2270                     provider->unlockForArbitration();
2271                 }
2272             }
2273         }
2274         iter->release();
2275     }
2276
2277     return( true );
2278 }
2279
2280 #undef tailQ
2281 #undef headQ
2282
2283 /*
2284  * Terminate
2285  */
2286
2287 void IOService::doServiceTerminate( IOOptionBits options )
2288 {
2289 }
2290
2291 // a method in case someone needs to override it
2292 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2293 {
2294     bool ok;
2295
2296     if( client->isParent( this, gIOServicePlane, true))
2297         // we are the clients only provider
2298         ok = client->terminate( options );
2299     else
2300         ok = true;
2301
2302     return( ok );
2303 }
2304
2305 bool IOService::terminate( IOOptionBits options )
2306 {
2307     options |= kIOServiceTerminate;
2308
2309     return( terminatePhase1( options ));
2310 }
2311
2312 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2313
2314 /*
2315  * Open & close
2316  */
2317
2318 struct ServiceOpenMessageContext
2319 {
2320     IOService *  service;
2321     UInt32       type;
2322     IOService *  excludeClient;
2323     IOOptionBits options;
2324 };
2325
2326 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2327 {
2328     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2329
2330     if( object != context->excludeClient)
2331         context->service->messageClient( context->type, object, (void *) context->options );
2332 }
2333
2334 bool IOService::open(   IOService *     forClient,
2335                         IOOptionBits    options,
2336                         void *          arg )
2337 {
2338     bool                        ok;
2339     ServiceOpenMessageContext   context;
2340
2341     context.service             = this;
2342     context.type                = kIOMessageServiceIsAttemptingOpen;
2343     context.excludeClient       = forClient;
2344     context.options             = options;
2345
2346     applyToInterested( gIOGeneralInterest,
2347                         &serviceOpenMessageApplier, &context );
2348
2349     if( false == lockForArbitration(false) )
2350         return false;
2351
2352     ok = (0 == (__state[0] & kIOServiceInactiveState));
2353     if( ok)
2354         ok = handleOpen( forClient, options, arg );
2355
2356     unlockForArbitration();
2357
2358     return( ok );
2359 }
2360
2361 void IOService::close(  IOService *     forClient,
2362                         IOOptionBits    options )
2363 {
2364     bool                wasClosed;
2365     bool                last = false;
2366
2367     lockForArbitration();
2368
2369     wasClosed = handleIsOpen( forClient );
2370     if( wasClosed) {
2371         handleClose( forClient, options );
2372         last = (__state[1] & kIOServiceTermPhase3State);
2373     }
2374
2375     unlockForArbitration();
2376
2377     if( last)
2378         forClient->scheduleStop( this );
2379
2380     else if( wasClosed) {
2381
2382         ServiceOpenMessageContext context;
2383
2384         context.service         = this;
2385         context.type            = kIOMessageServiceWasClosed;
2386         context.excludeClient   = forClient;
2387         context.options         = options;
2388
2389         applyToInterested( gIOGeneralInterest,
2390                             &serviceOpenMessageApplier, &context );
2391     }
2392 }
2393
2394 bool IOService::isOpen( const IOService * forClient ) const
2395 {
2396     IOService * self = (IOService *) this;
2397     bool ok;
2398
2399     self->lockForArbitration();
2400
2401     ok = handleIsOpen( forClient );
2402
2403     self->unlockForArbitration();
2404
2405     return( ok );
2406 }
2407
2408 bool IOService::handleOpen(     IOService *     forClient,
2409                                 IOOptionBits    options,
2410                                 void *          arg )
2411 {
2412     bool        ok;
2413
2414     ok = (0 == __owner);
2415     if( ok )
2416         __owner = forClient;
2417
2418     else if( options & kIOServiceSeize ) {
2419         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2420                                 __owner, (void *) options ));
2421         if( ok && (0 == __owner ))
2422             __owner = forClient;
2423         else
2424             ok = false;
2425     }
2426     return( ok );
2427 }
2428
2429 void IOService::handleClose(    IOService *     forClient,
2430                                 IOOptionBits    options )
2431 {
2432     if( __owner == forClient)
2433         __owner = 0;
2434 }
2435
2436 bool IOService::handleIsOpen(   const IOService * forClient ) const
2437 {
2438     if( forClient)
2439         return( __owner == forClient );
2440     else
2441         return( __owner != forClient );
2442 }
2443
2444 /*
2445  * Probing & starting
2446  */
2447 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2448 {
2449     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
2450     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
2451     SInt32             val1;
2452     SInt32             val2;
2453
2454     val1 = 0;
2455     val2 = 0;
2456
2457     if ( obj1 )
2458         val1 = obj1->priority;
2459
2460     if ( obj2 )
2461         val2 = obj2->priority;
2462
2463     return ( val1 - val2 );
2464 }
2465
2466 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
2467 {
2468     OSDictionary *      dict;
2469     IOService *         service;
2470     _IOServiceNotifier * notify;
2471     OSSymbol *          key = (OSSymbol *) ref;
2472     OSNumber *          offset;
2473
2474     if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
2475         return( notify->priority );
2476
2477     else if( (service = OSDynamicCast( IOService, entry)))
2478         offset = OSDynamicCast(OSNumber, service->getProperty( key ));
2479     else if( (dict = OSDynamicCast( OSDictionary, entry)))
2480         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
2481     else {
2482         assert( false );
2483         offset = 0;
2484     }
2485
2486     if( offset)
2487         return( (SInt32) offset->unsigned32BitValue());
2488     else
2489         return( kIODefaultProbeScore );
2490 }
2491
2492 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2493 {
2494     const OSObject *    obj1 = (const OSObject *) inObj1;
2495     const OSObject *    obj2 = (const OSObject *) inObj2;
2496     SInt32               val1;
2497     SInt32               val2;
2498
2499     val1 = 0;
2500     val2 = 0;
2501
2502     if ( obj1 )
2503         val1 = IOServiceObjectOrder( obj1, ref );
2504
2505     if ( obj2 )
2506         val2 = IOServiceObjectOrder( obj2, ref );
2507
2508     return ( val1 - val2 );
2509 }
2510
2511 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
2512 {
2513     IOService *         service = 0;
2514     OSIterator *        iter;
2515     const OSSymbol *    nextCat;
2516
2517     iter = getClientIterator();
2518     if( iter) {
2519         while( (service = (IOService *) iter->getNextObject())) {
2520             if( kIOServiceInactiveState & service->__state[0])
2521                 continue;
2522             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
2523                         service->getProperty( gIOMatchCategoryKey ));
2524             if( category == nextCat)
2525             {
2526                 service->retain();
2527                 break;
2528             }
2529         }
2530         iter->release();
2531     }
2532     return( service );
2533 }
2534
2535 IOService * IOService::getClientWithCategory( const OSSymbol * category )
2536 {
2537     IOService *
2538     service = copyClientWithCategory(category);
2539     if (service)
2540         service->release();
2541     return (service);
2542 }
2543
2544 bool IOService::invokeNotifer( _IOServiceNotifier * notify )
2545 {
2546     _IOServiceNotifierInvocation invocation;
2547     bool                         willNotify;
2548     bool                         ret = true;
2549
2550     invocation.thread = current_thread();
2551
2552     LOCKWRITENOTIFY();
2553     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2554
2555     if( willNotify) {
2556         queue_enter( &notify->handlerInvocations, &invocation,
2557                         _IOServiceNotifierInvocation *, link );
2558     }
2559     UNLOCKNOTIFY();
2560
2561     if( willNotify) {
2562
2563         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
2564
2565         LOCKWRITENOTIFY();
2566         queue_remove( &notify->handlerInvocations, &invocation,
2567                         _IOServiceNotifierInvocation *, link );
2568         if( kIOServiceNotifyWaiter & notify->state) {
2569             notify->state &= ~kIOServiceNotifyWaiter;
2570             WAKEUPNOTIFY( notify );
2571         }
2572         UNLOCKNOTIFY();
2573     }
2574
2575     return( ret );
2576 }
2577
2578 /*
2579  * Alloc and probe matching classes,
2580  * called on the provider instance
2581  */
2582
2583 void IOService::probeCandidates( OSOrderedSet * matches )
2584 {
2585     OSDictionary        *       match = 0;
2586     OSSymbol            *       symbol;
2587     IOService           *       inst;
2588     IOService           *       newInst;
2589     OSDictionary        *       props;
2590     SInt32                      score;
2591     OSNumber            *       newPri;
2592     OSOrderedSet        *       familyMatches = 0;
2593     OSOrderedSet        *       startList;
2594     OSDictionary        *       startDict = 0;
2595     const OSSymbol      *       category;
2596     OSIterator          *       iter;
2597     _IOServiceNotifier  *               notify;
2598     OSObject            *       nextMatch = 0;
2599     bool                        started;
2600     bool                        needReloc = false;
2601 #if CONFIG_MACF_KEXT
2602     OSBoolean           *       isSandbox = 0;
2603     bool                        useSandbox = false;
2604 #endif
2605 #if IOMATCHDEBUG
2606     SInt64                      debugFlags;
2607 #endif
2608     IOService * client = NULL;
2609
2610
2611     assert( matches );
2612     while( !needReloc && (nextMatch = matches->getFirstObject())) {
2613
2614         nextMatch->retain();
2615         matches->removeObject(nextMatch);
2616
2617         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
2618
2619             lockForArbitration();
2620             if( 0 == (__state[0] & kIOServiceInactiveState))
2621                 invokeNotifer( notify );
2622             unlockForArbitration();
2623             nextMatch->release();
2624             nextMatch = 0;
2625             continue;
2626
2627         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
2628             nextMatch->release();
2629             nextMatch = 0;
2630             continue;
2631         }
2632
2633         props = 0;
2634 #if IOMATCHDEBUG
2635         debugFlags = getDebugFlags( match );
2636 #endif
2637
2638         do {
2639             category = OSDynamicCast( OSSymbol,
2640                         match->getObject( gIOMatchCategoryKey ));
2641             if( 0 == category)
2642                 category = gIODefaultMatchCategoryKey;
2643
2644             if( (client = copyClientWithCategory(category)) ) {
2645 #if IOMATCHDEBUG
2646                 if( debugFlags & kIOLogMatch)
2647                     LOG("%s: match category %s exists\n", getName(),
2648                                 category->getCStringNoCopy());
2649 #endif
2650                 nextMatch->release();
2651                 nextMatch = 0;
2652
2653                 client->release();
2654                 client = NULL;
2655
2656                 continue;
2657             }
2658
2659             // create a copy now in case its modified during matching
2660             props = OSDictionary::withDictionary( match, match->getCount());
2661             if( 0 == props)
2662                 continue;
2663             props->setCapacityIncrement(1);
2664
2665             // check the nub matches
2666             if( false == passiveMatch( props, true ))
2667                 continue;
2668
2669             // Check to see if driver reloc has been loaded.
2670             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
2671             if( needReloc) {
2672 #if IOMATCHDEBUG
2673                 if( debugFlags & kIOLogCatalogue)
2674                     LOG("%s: stalling for module\n", getName());
2675 #endif
2676                 // If reloc hasn't been loaded, exit;
2677                 // reprobing will occur after reloc has been loaded.
2678                 continue;
2679             }
2680
2681             // reorder on family matchPropertyTable score.
2682             if( 0 == familyMatches)
2683                 familyMatches = OSOrderedSet::withCapacity( 1,
2684                         IOServiceOrdering, (void *) gIOProbeScoreKey );
2685             if( familyMatches)
2686                 familyMatches->setObject( props );
2687
2688         } while( false );
2689
2690         if (nextMatch) {
2691             nextMatch->release();
2692             nextMatch = 0;
2693         }
2694         if( props)
2695             props->release();
2696     }
2697     matches->release();
2698     matches = 0;
2699
2700     if( familyMatches) {
2701
2702         while( !needReloc
2703              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
2704
2705             props->retain();
2706             familyMatches->removeObject( props );
2707
2708             inst = 0;
2709             newInst = 0;
2710 #if IOMATCHDEBUG
2711             debugFlags = getDebugFlags( props );
2712 #endif
2713             do {
2714                 symbol = OSDynamicCast( OSSymbol,
2715                                 props->getObject( gIOClassKey));
2716                 if( !symbol)
2717                     continue;
2718
2719                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), symbol, props);
2720
2721                 // alloc the driver instance
2722                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
2723
2724                 if( !inst) {
2725                     IOLog("Couldn't alloc class \"%s\"\n",
2726                         symbol->getCStringNoCopy());
2727                     continue;
2728                 }
2729
2730                 // init driver instance
2731                 if( !(inst->init( props ))) {
2732 #if IOMATCHDEBUG
2733                     if( debugFlags & kIOLogStart)
2734                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
2735 #endif
2736                     continue;
2737                 }
2738                 if( __state[1] & kIOServiceSynchronousState)
2739                     inst->__state[1] |= kIOServiceSynchronousState;
2740
2741                 // give the driver the default match category if not specified
2742                 category = OSDynamicCast( OSSymbol,
2743                             props->getObject( gIOMatchCategoryKey ));
2744                 if( 0 == category)
2745                     category = gIODefaultMatchCategoryKey;
2746                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
2747 #if CONFIG_MACF_KEXT
2748                 isSandbox = OSDynamicCast(OSBoolean,
2749                             props->getObject("IOKitForceMatch"));
2750 #endif
2751                 // attach driver instance
2752                 if( !(inst->attach( this )))
2753                         continue;
2754
2755                 // pass in score from property table
2756                 score = familyMatches->orderObject( props );
2757
2758                 // & probe the new driver instance
2759 #if IOMATCHDEBUG
2760                 if( debugFlags & kIOLogProbe)
2761                     LOG("%s::probe(%s)\n",
2762                         inst->getMetaClass()->getClassName(), getName());
2763 #endif
2764
2765                 newInst = inst->probe( this, &score );
2766                 inst->detach( this );
2767 #if CONFIG_MACF_KEXT
2768                 /*
2769                  * If this is the Sandbox driver and it matched, this is a
2770                  * disallowed device; toss any drivers that were already
2771                  * matched.
2772                  */
2773                 if (isSandbox && isSandbox->isTrue() && newInst != 0) {
2774                     if (startDict != 0) {
2775                         startDict->flushCollection();
2776                         startDict->release();
2777                         startDict = 0;
2778                     }
2779                     useSandbox = true;
2780                 }
2781 #endif
2782                 if( 0 == newInst) {
2783 #if IOMATCHDEBUG
2784                     if( debugFlags & kIOLogProbe)
2785                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
2786 #endif
2787                     continue;
2788                 }
2789
2790                 // save the score
2791                 newPri = OSNumber::withNumber( score, 32 );
2792                 if( newPri) {
2793                     newInst->setProperty( gIOProbeScoreKey, newPri );
2794                     newPri->release();
2795                 }
2796
2797                 // add to start list for the match category
2798                 if( 0 == startDict)
2799                     startDict = OSDictionary::withCapacity( 1 );
2800                 assert( startDict );
2801                 startList = (OSOrderedSet *)
2802                                 startDict->getObject( category );
2803                 if( 0 == startList) {
2804                     startList = OSOrderedSet::withCapacity( 1,
2805                             IOServiceOrdering, (void *) gIOProbeScoreKey );
2806                     if( startDict && startList) {
2807                         startDict->setObject( category, startList );
2808                         startList->release();
2809                     }
2810                 }
2811                 assert( startList );
2812                 if( startList)
2813                     startList->setObject( newInst );
2814
2815             } while( false );
2816
2817             props->release();
2818             if( inst)
2819                 inst->release();
2820 #if CONFIG_MACF_KEXT
2821             /*
2822              * If we're forcing the sandbox, drop out of the loop.
2823              */
2824             if (isSandbox && isSandbox->isTrue() && useSandbox)
2825                     break;
2826 #endif
2827         }
2828         familyMatches->release();
2829         familyMatches = 0;
2830     }
2831
2832     // start the best (until success) of each category
2833
2834     iter = OSCollectionIterator::withCollection( startDict );
2835     if( iter) {
2836         while( (category = (const OSSymbol *) iter->getNextObject())) {
2837
2838             startList = (OSOrderedSet *) startDict->getObject( category );
2839             assert( startList );
2840             if( !startList)
2841                 continue;
2842
2843             started = false;
2844             while( true // (!started)
2845                    && (inst = (IOService *)startList->getFirstObject())) {
2846
2847                 inst->retain();
2848                 startList->removeObject(inst);
2849
2850 #if IOMATCHDEBUG
2851                 debugFlags = getDebugFlags( inst->getPropertyTable() );
2852
2853                 if( debugFlags & kIOLogStart) {
2854                     if( started)
2855                         LOG( "match category exists, skipping " );
2856                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
2857                          getName(), inst->getRetainCount());
2858                 }
2859 #endif
2860                 if( false == started)
2861                     started = startCandidate( inst );
2862 #if IOMATCHDEBUG
2863                 if( (debugFlags & kIOLogStart) && (false == started))
2864                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
2865                          inst->getRetainCount());
2866 #endif
2867                 inst->release();
2868             }
2869         }
2870         iter->release();
2871     }
2872
2873
2874     // adjust the busy count by +1 if matching is stalled for a module,
2875     // or -1 if a previously stalled matching is complete.
2876     lockForArbitration();
2877     SInt32 adjBusy = 0;
2878     uint64_t regID = getRegistryEntryID();
2879
2880     if( needReloc) {
2881         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
2882         if( adjBusy) {
2883
2884             IOServiceTrace(
2885                 IOSERVICE_MODULESTALL,
2886                 (uintptr_t) regID,
2887                 (uintptr_t) (regID >> 32),
2888                 (uintptr_t) this,
2889                 0);
2890
2891             __state[1] |= kIOServiceModuleStallState;
2892         }
2893
2894     } else if( __state[1] & kIOServiceModuleStallState) {
2895
2896             IOServiceTrace(
2897             IOSERVICE_MODULEUNSTALL,
2898             (uintptr_t) regID,
2899             (uintptr_t) (regID >> 32),
2900             (uintptr_t) this,
2901             0);
2902
2903         __state[1] &= ~kIOServiceModuleStallState;
2904         adjBusy = -1;
2905     }
2906     if( adjBusy)
2907         _adjustBusy( adjBusy );
2908     unlockForArbitration();
2909
2910     if( startDict)
2911         startDict->release();
2912 }
2913
2914 /*
2915  * Start a previously attached & probed instance,
2916  * called on exporting object instance
2917  */
2918
2919 bool IOService::startCandidate( IOService * service )
2920 {
2921     bool                ok;
2922
2923     ok = service->attach( this );
2924
2925     if( ok)
2926     {
2927         if (this != gIOResources)
2928         {
2929             // stall for any nub resources
2930             checkResources();
2931             // stall for any driver resources
2932             service->checkResources();
2933         }
2934
2935         AbsoluteTime startTime;
2936         AbsoluteTime endTime;
2937         UInt64       nano;
2938
2939         if (kIOLogStart & gIOKitDebug)
2940             clock_get_uptime(&startTime);
2941
2942         ok = service->start(this);
2943
2944         if (kIOLogStart & gIOKitDebug)
2945         {
2946             clock_get_uptime(&endTime);
2947
2948             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
2949             {
2950                 SUB_ABSOLUTETIME(&endTime, &startTime);
2951                 absolutetime_to_nanoseconds(endTime, &nano);
2952                 if (nano > 500000000ULL)
2953                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
2954             }
2955         }
2956         if( !ok)
2957             service->detach( this );
2958     }
2959     return( ok );
2960 }
2961
2962 void IOService::publishResource( const char * key, OSObject * value )
2963 {
2964     const OSSymbol *    sym;
2965
2966     if( (sym = OSSymbol::withCString( key))) {
2967         publishResource( sym, value);
2968         sym->release();
2969     }
2970 }
2971
2972 void IOService::publishResource( const OSSymbol * key, OSObject * value )
2973 {
2974     if( 0 == value)
2975         value = (OSObject *) gIOServiceKey;
2976
2977     gIOResources->setProperty( key, value);
2978
2979     if( IORecursiveLockHaveLock( gNotificationLock))
2980         return;
2981
2982     gIOResourceGenerationCount++;
2983     gIOResources->registerService();
2984 }
2985
2986 bool IOService::addNeededResource( const char * key )
2987 {
2988     OSObject *  resourcesProp;
2989     OSSet *     set;
2990     OSString *  newKey;
2991     bool ret;
2992
2993     resourcesProp = getProperty( gIOResourceMatchKey );
2994
2995     newKey = OSString::withCString( key );
2996     if( (0 == resourcesProp) || (0 == newKey))
2997         return( false);
2998
2999     set = OSDynamicCast( OSSet, resourcesProp );
3000     if( !set) {
3001         set = OSSet::withCapacity( 1 );
3002         if( set)
3003             set->setObject( resourcesProp );
3004     }
3005     else
3006         set->retain();
3007
3008     set->setObject( newKey );
3009     newKey->release();
3010     ret = setProperty( gIOResourceMatchKey, set );
3011     set->release();
3012
3013     return( ret );
3014 }
3015
3016 bool IOService::checkResource( OSObject * matching )
3017 {
3018     OSString *          str;
3019     OSDictionary *      table;
3020
3021     if( (str = OSDynamicCast( OSString, matching ))) {
3022         if( gIOResources->getProperty( str ))
3023             return( true );
3024     }
3025
3026     if( str)
3027         table = resourceMatching( str );
3028     else if( (table = OSDynamicCast( OSDictionary, matching )))
3029         table->retain();
3030     else {
3031         IOLog("%s: Can't match using: %s\n", getName(),
3032                 matching->getMetaClass()->getClassName());
3033         /* false would stall forever */
3034         return( true );
3035     }
3036
3037     if( gIOKitDebug & kIOLogConfig)
3038         LOG("config(%p): stalling %s\n", IOThreadSelf(), getName());
3039
3040     waitForService( table );
3041
3042     if( gIOKitDebug & kIOLogConfig)
3043         LOG("config(%p): waking\n", IOThreadSelf() );
3044
3045     return( true );
3046 }
3047
3048 bool IOService::checkResources( void )
3049 {
3050     OSObject *          resourcesProp;
3051     OSSet *             set;
3052     OSIterator *        iter;
3053     bool                ok;
3054
3055     resourcesProp = getProperty( gIOResourceMatchKey );
3056     if( 0 == resourcesProp)
3057         return( true );
3058
3059     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3060
3061         iter = OSCollectionIterator::withCollection( set );
3062         ok = (0 != iter);
3063         while( ok && (resourcesProp = iter->getNextObject()) )
3064             ok = checkResource( resourcesProp );
3065         if( iter)
3066             iter->release();
3067
3068     } else
3069         ok = checkResource( resourcesProp );
3070
3071     return( ok );
3072 }
3073
3074
3075 void _IOConfigThread::configThread( void )
3076 {
3077     _IOConfigThread *   inst;
3078
3079     do {
3080         if( !(inst = new _IOConfigThread))
3081             continue;
3082         if( !inst->init())
3083             continue;
3084         thread_t unused;
3085         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3086             continue;
3087
3088         return;
3089
3090     } while( false);
3091
3092     if( inst)
3093         inst->release();
3094
3095     return;
3096 }
3097
3098 void _IOConfigThread::free( void )
3099 {
3100     thread_deallocate(current_thread());
3101     OSObject::free();
3102 }
3103
3104 void IOService::doServiceMatch( IOOptionBits options )
3105 {
3106     _IOServiceNotifier * notify;
3107     OSIterator *        iter;
3108     OSOrderedSet *      matches;
3109     SInt32              catalogGeneration;
3110     bool                keepGuessing = true;
3111     bool                reRegistered = true;
3112
3113 //    job->nub->deliverNotification( gIOPublishNotification,
3114 //                              kIOServiceRegisteredState, 0xffffffff );
3115
3116     while( keepGuessing ) {
3117
3118         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3119         // the matches list should always be created by findDrivers()
3120         if( matches) {
3121
3122             lockForArbitration();
3123             if( 0 == (__state[0] & kIOServiceFirstPublishState))
3124                 deliverNotification( gIOFirstPublishNotification,
3125                                      kIOServiceFirstPublishState, 0xffffffff );
3126             LOCKREADNOTIFY();
3127             __state[1] &= ~kIOServiceNeedConfigState;
3128             __state[1] |= kIOServiceConfigState;
3129             __state[0] |= kIOServiceRegisteredState;
3130
3131             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3132             if (reRegistered && keepGuessing) {
3133                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3134                         gNotifications->getObject( gIOPublishNotification ) );
3135                 if( iter) {
3136                     while((notify = (_IOServiceNotifier *)
3137                            iter->getNextObject())) {
3138
3139                         if( passiveMatch( notify->matching )
3140                          && (kIOServiceNotifyEnable & notify->state))
3141                             matches->setObject( notify );
3142                     }
3143                     iter->release();
3144                 }
3145             }
3146
3147             UNLOCKNOTIFY();
3148             unlockForArbitration();
3149
3150             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3151                 probeCandidates( matches );
3152             else
3153                 matches->release();
3154         }
3155
3156         lockForArbitration();
3157         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3158         keepGuessing =
3159                    (reRegistered || (catalogGeneration !=
3160                                         gIOCatalogue->getGenerationCount()))
3161                 && (0 == (__state[0] & kIOServiceInactiveState));
3162
3163         if( keepGuessing)
3164             unlockForArbitration();
3165     }
3166
3167     if( (0 == (__state[0] & kIOServiceInactiveState))
3168      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3169         deliverNotification( gIOMatchedNotification,
3170                 kIOServiceMatchedState, 0xffffffff );
3171         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3172             deliverNotification( gIOFirstMatchNotification,
3173                 kIOServiceFirstMatchState, 0xffffffff );
3174     }
3175
3176     __state[1] &= ~kIOServiceConfigState;
3177     if( __state[0] & kIOServiceInactiveState)
3178         scheduleTerminatePhase2();
3179
3180     _adjustBusy( -1 );
3181     unlockForArbitration();
3182 }
3183
3184 UInt32 IOService::_adjustBusy( SInt32 delta )
3185 {
3186     IOService * next;
3187     UInt32      count;
3188     UInt32      result;
3189     bool        wasQuiet, nowQuiet, needWake;
3190
3191     next = this;
3192     result = __state[1] & kIOServiceBusyStateMask;
3193
3194     if( delta) do {
3195         if( next != this)
3196             next->lockForArbitration();
3197         count = next->__state[1] & kIOServiceBusyStateMask;
3198         wasQuiet = (0 == count);
3199         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3200             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3201         else
3202             count += delta;
3203         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3204         nowQuiet = (0 == count);
3205         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3206
3207         if( needWake) {
3208             next->__state[1] &= ~kIOServiceBusyWaiterState;
3209             IOLockLock( gIOServiceBusyLock );
3210             thread_wakeup( (event_t) next);
3211             IOLockUnlock( gIOServiceBusyLock );
3212         }
3213         if( next != this)
3214             next->unlockForArbitration();
3215
3216         if( (wasQuiet || nowQuiet) ) {
3217             uint64_t regID = next->getRegistryEntryID();
3218
3219                 IOServiceTrace(
3220                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3221                 (uintptr_t) regID,
3222                 (uintptr_t) (regID >> 32),
3223                 (uintptr_t) next,
3224                 0);
3225
3226             if (wasQuiet)
3227             {
3228                 next->__timeBusy = mach_absolute_time();
3229             }
3230             else
3231             {
3232                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3233                 next->__timeBusy = 0;
3234             }
3235
3236             MessageClientsContext context;
3237
3238             context.service  = next;
3239             context.type     = kIOMessageServiceBusyStateChange;
3240             context.argument = (void *) wasQuiet;       /*nowBusy*/
3241             context.argSize  = 0;
3242
3243             applyToInterestNotifiers( next, gIOBusyInterest,
3244                                      &messageClientsApplier, &context );
3245
3246 #if !NO_KEXTD
3247             if( nowQuiet && (next == gIOServiceRoot)) {
3248                 OSKext::considerUnloads();
3249                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3250             }
3251 #endif
3252         }
3253
3254         delta = nowQuiet ? -1 : +1;
3255
3256     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3257
3258     return( result );
3259 }
3260
3261 void IOService::adjustBusy( SInt32 delta )
3262 {
3263     lockForArbitration();
3264     _adjustBusy( delta );
3265     unlockForArbitration();
3266 }
3267
3268 uint64_t IOService::getAccumulatedBusyTime( void )
3269 {
3270     uint64_t accumBusy = __accumBusy;
3271     uint64_t timeBusy = __timeBusy;
3272     uint64_t nano;
3273
3274     do
3275     {
3276         accumBusy = __accumBusy;
3277         timeBusy  = __timeBusy;
3278         if (timeBusy)
3279             accumBusy += mach_absolute_time() - timeBusy;
3280     }
3281     while (timeBusy != __timeBusy);
3282
3283     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3284
3285     return (nano);
3286 }
3287
3288 UInt32 IOService::getBusyState( void )
3289 {
3290     return( __state[1] & kIOServiceBusyStateMask );
3291 }
3292
3293 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3294                                   mach_timespec_t * timeout )
3295 {
3296     panic("waitForState");
3297     return (kIOReturnUnsupported);
3298 }
3299
3300 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3301                                    uint64_t timeout )
3302 {
3303     bool            wait;
3304     int             waitResult = THREAD_AWAKENED;
3305     bool            computeDeadline = true;
3306     AbsoluteTime    abstime;
3307
3308     do {
3309         lockForArbitration();
3310         IOLockLock( gIOServiceBusyLock );
3311         wait = (value != (__state[1] & mask));
3312         if( wait) {
3313             __state[1] |= kIOServiceBusyWaiterState;
3314             unlockForArbitration();
3315             if( timeout != UINT64_MAX ) {
3316                 if( computeDeadline ) {
3317                     AbsoluteTime  nsinterval;
3318                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3319                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3320                     computeDeadline = false;
3321                 }
3322                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3323             }
3324             else
3325                 assert_wait((event_t)this, THREAD_UNINT );
3326         } else
3327             unlockForArbitration();
3328         IOLockUnlock( gIOServiceBusyLock );
3329         if( wait)
3330             waitResult = thread_block(THREAD_CONTINUE_NULL);
3331
3332     } while( wait && (waitResult != THREAD_TIMED_OUT));
3333
3334     if( waitResult == THREAD_TIMED_OUT)
3335         return( kIOReturnTimeout );
3336     else
3337         return( kIOReturnSuccess );
3338 }
3339
3340 IOReturn IOService::waitQuiet( uint64_t timeout )
3341 {
3342     return( waitForState( kIOServiceBusyStateMask, 0, timeout ));
3343 }
3344
3345 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
3346 {
3347     uint64_t    timeoutNS;
3348
3349     if (timeout)
3350     {
3351         timeoutNS = timeout->tv_sec;
3352         timeoutNS *= kSecondScale;
3353         timeoutNS += timeout->tv_nsec;
3354     }
3355     else
3356         timeoutNS = UINT64_MAX;
3357
3358     return( waitForState( kIOServiceBusyStateMask, 0, timeoutNS ));
3359 }
3360
3361 bool IOService::serializeProperties( OSSerialize * s ) const
3362 {
3363 #if 0
3364     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
3365                 sizeof( __state), "__state");
3366 #endif
3367     return( super::serializeProperties(s) );
3368 }
3369
3370
3371 void _IOConfigThread::main(void * arg, wait_result_t result)
3372 {
3373     _IOConfigThread * self = (_IOConfigThread *) arg;
3374     _IOServiceJob * job;
3375     IOService   *   nub;
3376     bool            alive = true;
3377     kern_return_t   kr;
3378     thread_precedence_policy_data_t precedence = { -1 };
3379
3380     kr = thread_policy_set(current_thread(),
3381                             THREAD_PRECEDENCE_POLICY,
3382                             (thread_policy_t) &precedence,
3383                             THREAD_PRECEDENCE_POLICY_COUNT);
3384     if (KERN_SUCCESS != kr)
3385         IOLog("thread_policy_set(%d)\n", kr);
3386
3387     do {
3388
3389 //      randomDelay();
3390
3391         semaphore_wait( gJobsSemaphore );
3392
3393         IOTakeLock( gJobsLock );
3394         job = (_IOServiceJob *) gJobs->getFirstObject();
3395         job->retain();
3396         gJobs->removeObject(job);
3397         if( job) {
3398             gOutstandingJobs--;
3399 //          gNumConfigThreads--;        // we're out of service
3400             gNumWaitingThreads--;       // we're out of service
3401         }
3402         IOUnlock( gJobsLock );
3403
3404         if( job) {
3405
3406           nub = job->nub;
3407
3408           if( gIOKitDebug & kIOLogConfig)
3409             LOG("config(%p): starting on %s, %d\n",
3410                         IOThreadSelf(), job->nub->getName(), job->type);
3411
3412           switch( job->type) {
3413
3414             case kMatchNubJob:
3415                 nub->doServiceMatch( job->options );
3416                 break;
3417
3418             default:
3419                 LOG("config(%p): strange type (%d)\n",
3420                         IOThreadSelf(), job->type );
3421                 break;
3422             }
3423
3424             nub->release();
3425             job->release();
3426
3427             IOTakeLock( gJobsLock );
3428             alive = (gOutstandingJobs > gNumWaitingThreads);
3429             if( alive)
3430                 gNumWaitingThreads++;   // back in service
3431 //              gNumConfigThreads++;
3432             else {
3433                 if( 0 == --gNumConfigThreads) {
3434 //                    IOLog("MATCH IDLE\n");
3435                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
3436                 }
3437             }
3438             IOUnlock( gJobsLock );
3439         }
3440
3441     } while( alive );
3442
3443     if( gIOKitDebug & kIOLogConfig)
3444         LOG("config(%p): terminating\n", IOThreadSelf() );
3445
3446     self->release();
3447 }
3448
3449 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
3450 {
3451     bool            wait;
3452     int             waitResult = THREAD_AWAKENED;
3453     bool            computeDeadline = true;
3454     AbsoluteTime    deadline;
3455
3456     IOLockLock( gJobsLock );
3457     do {
3458         wait = (0 != gNumConfigThreads);
3459         if( wait) {
3460             if( msToWait) {
3461                 if( computeDeadline ) {
3462                     clock_interval_to_deadline(
3463                           msToWait, kMillisecondScale, &deadline );
3464                     computeDeadline = false;
3465                 }
3466                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
3467                                                                 deadline, THREAD_UNINT );
3468                    } else {
3469                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
3470                                                                 THREAD_UNINT );
3471                 }
3472         }
3473     } while( wait && (waitResult != THREAD_TIMED_OUT));
3474         IOLockUnlock( gJobsLock );
3475
3476     if( waitResult == THREAD_TIMED_OUT)
3477         return( kIOReturnTimeout );
3478     else
3479         return( kIOReturnSuccess );
3480 }
3481
3482 void _IOServiceJob::pingConfig( _IOServiceJob * job )
3483 {
3484     int         count;
3485     bool        create;
3486
3487     assert( job );
3488
3489     IOTakeLock( gJobsLock );
3490
3491     gOutstandingJobs++;
3492     gJobs->setLastObject( job );
3493
3494     count = gNumWaitingThreads;
3495 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
3496
3497     create = (  (gOutstandingJobs > count)
3498                 && (gNumConfigThreads < kMaxConfigThreads) );
3499     if( create) {
3500         gNumConfigThreads++;
3501         gNumWaitingThreads++;
3502     }
3503
3504     IOUnlock( gJobsLock );
3505
3506     job->release();
3507
3508     if( create) {
3509         if( gIOKitDebug & kIOLogConfig)
3510             LOG("config(%d): creating\n", gNumConfigThreads - 1);
3511         _IOConfigThread::configThread();
3512     }
3513
3514     semaphore_signal( gJobsSemaphore );
3515 }
3516
3517 // internal - call with gNotificationLock
3518 OSObject * IOService::copyExistingServices( OSDictionary * matching,
3519                  IOOptionBits inState, IOOptionBits options )
3520 {
3521     OSObject *          current = 0;
3522     OSIterator *        iter;
3523     IOService *         service;
3524     OSObject *          obj;
3525
3526     if( !matching)
3527         return( 0 );
3528
3529     if(true
3530       && (obj = matching->getObject(gIOProviderClassKey))
3531       && gIOResourcesKey
3532       && gIOResourcesKey->isEqualTo(obj)
3533       && (service = gIOResources))
3534     {
3535         if( (inState == (service->__state[0] & inState))
3536           && (0 == (service->__state[0] & kIOServiceInactiveState))
3537           &&  service->passiveMatch( matching ))
3538         {
3539             if( options & kIONotifyOnce)
3540             {
3541                 service->retain();
3542                 current = service;
3543             }
3544             else
3545                 current = OSSet::withObjects(
3546                                 (const OSObject **) &service, 1, 1 );
3547         }
3548     }
3549     else
3550     {
3551         iter = IORegistryIterator::iterateOver( gIOServicePlane,
3552                                             kIORegistryIterateRecursively );
3553         if( iter) {
3554             do {
3555                 iter->reset();
3556                 while( (service = (IOService *) iter->getNextObject())) {
3557                     if( (inState == (service->__state[0] & inState))
3558                     && (0 == (service->__state[0] & kIOServiceInactiveState))
3559                     &&  service->passiveMatch( matching )) {
3560
3561                         if( options & kIONotifyOnce) {
3562                             service->retain();
3563                             current = service;
3564                             break;
3565                         }
3566                         if( current)
3567                             ((OSSet *)current)->setObject( service );
3568                         else
3569                             current = OSSet::withObjects(
3570                                             (const OSObject **) &service, 1, 1 );
3571                     }
3572                 }
3573             } while( !service && !iter->isValid());
3574             iter->release();
3575         }
3576     }
3577
3578     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
3579         iter = OSCollectionIterator::withCollection( (OSSet *)current );
3580         current->release();
3581         current = iter;
3582     }
3583
3584     return( current );
3585 }
3586
3587 // public version
3588 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
3589 {
3590     OSIterator *        iter;
3591
3592     // is a lock even needed?
3593     LOCKWRITENOTIFY();
3594
3595     iter = (OSIterator *) copyExistingServices( matching,
3596                                                 kIOServiceMatchedState );
3597
3598     UNLOCKNOTIFY();
3599
3600     return( iter );
3601 }
3602
3603 struct _IOServiceMatchingNotificationHandlerRef
3604 {
3605     IOServiceNotificationHandler handler;
3606     void * ref;
3607 };
3608
3609 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
3610                                                    IOService * newService,
3611                                                    IONotifier * notifier )
3612 {
3613     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
3614 }
3615
3616 // internal - call with gNotificationLock
3617 IONotifier * IOService::setNotification(
3618             const OSSymbol * type, OSDictionary * matching,
3619             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
3620             SInt32 priority )
3621 {
3622     _IOServiceNotifier * notify = 0;
3623     OSOrderedSet *      set;
3624
3625     if( !matching)
3626         return( 0 );
3627
3628     notify = new _IOServiceNotifier;
3629     if( notify && !notify->init()) {
3630         notify->release();
3631         notify = 0;
3632     }
3633
3634     if( notify) {
3635         notify->handler = handler;
3636         notify->target = target;
3637         notify->matching = matching;
3638         matching->retain();
3639         if (handler == &_IOServiceMatchingNotificationHandler)
3640         {
3641             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
3642             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
3643         }
3644         else
3645             notify->ref = ref;
3646         notify->priority = priority;
3647         notify->state = kIOServiceNotifyEnable;
3648         queue_init( &notify->handlerInvocations );
3649
3650         ////// queue
3651
3652         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
3653             set = OSOrderedSet::withCapacity( 1,
3654                         IONotifyOrdering, 0 );
3655             if( set) {
3656                 gNotifications->setObject( type, set );
3657                 set->release();
3658             }
3659         }
3660         notify->whence = set;
3661         if( set)
3662             set->setObject( notify );
3663     }
3664
3665     return( notify );
3666 }
3667
3668 // internal - call with gNotificationLock
3669 IONotifier * IOService::doInstallNotification(
3670                         const OSSymbol * type, OSDictionary * matching,
3671                         IOServiceMatchingNotificationHandler handler,
3672                         void * target, void * ref,
3673                         SInt32 priority, OSIterator ** existing )
3674 {
3675     OSIterator *        exist;
3676     IONotifier *        notify;
3677     IOOptionBits        inState;
3678
3679     if( !matching)
3680         return( 0 );
3681
3682     if( type == gIOPublishNotification)
3683         inState = kIOServiceRegisteredState;
3684
3685     else if( type == gIOFirstPublishNotification)
3686         inState = kIOServiceFirstPublishState;
3687
3688     else if( (type == gIOMatchedNotification)
3689           || (type == gIOFirstMatchNotification))
3690         inState = kIOServiceMatchedState;
3691     else if( type == gIOTerminatedNotification)
3692         inState = 0;
3693     else
3694         return( 0 );
3695
3696     notify = setNotification( type, matching, handler, target, ref, priority );
3697
3698     if( inState)
3699         // get the current set
3700         exist = (OSIterator *) copyExistingServices( matching, inState );
3701     else
3702         exist = 0;
3703
3704     *existing = exist;
3705
3706     return( notify );
3707 }
3708
3709 #if !defined(__LP64__)
3710 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
3711                                         IOServiceNotificationHandler handler,
3712                                         void * target, void * refCon,
3713                                         SInt32 priority, OSIterator ** existing )
3714 {
3715     IONotifier * result;
3716     _IOServiceMatchingNotificationHandlerRef ref;
3717     ref.handler = handler;
3718     ref.ref     = refCon;
3719
3720     result = (_IOServiceNotifier *) installNotification( type, matching,
3721                          &_IOServiceMatchingNotificationHandler,
3722                         target, &ref, priority, existing );
3723     if (result)
3724         matching->release();
3725
3726     return (result);
3727 }
3728 #endif /* !defined(__LP64__) */
3729
3730
3731 IONotifier * IOService::installNotification(
3732                         const OSSymbol * type, OSDictionary * matching,
3733                         IOServiceMatchingNotificationHandler handler,
3734                         void * target, void * ref,
3735                         SInt32 priority, OSIterator ** existing )
3736 {
3737     IONotifier * notify;
3738
3739     LOCKWRITENOTIFY();
3740
3741     notify = doInstallNotification( type, matching, handler, target, ref,
3742                 priority, existing );
3743
3744     UNLOCKNOTIFY();
3745
3746     return( notify );
3747 }
3748
3749 IONotifier * IOService::addNotification(
3750                         const OSSymbol * type, OSDictionary * matching,
3751                         IOServiceNotificationHandler handler,
3752                         void * target, void * refCon,
3753                         SInt32 priority )
3754 {
3755     IONotifier * result;
3756     _IOServiceMatchingNotificationHandlerRef ref;
3757
3758     ref.handler = handler;
3759     ref.ref     = refCon;
3760
3761     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
3762                             target, &ref, priority);
3763
3764     if (result)
3765         matching->release();
3766
3767     return (result);
3768 }
3769
3770 IONotifier * IOService::addMatchingNotification(
3771                         const OSSymbol * type, OSDictionary * matching,
3772                         IOServiceMatchingNotificationHandler handler,
3773                         void * target, void * ref,
3774                         SInt32 priority )
3775 {
3776     OSIterator *                existing = NULL;
3777     _IOServiceNotifier *        notify;
3778     IOService *                 next;
3779
3780     notify = (_IOServiceNotifier *) installNotification( type, matching,
3781                 handler, target, ref, priority, &existing );
3782
3783     // send notifications for existing set
3784     if( existing) {
3785
3786         notify->retain();               // in case handler remove()s
3787         while( (next = (IOService *) existing->getNextObject())) {
3788
3789             next->lockForArbitration();
3790             if( 0 == (next->__state[0] & kIOServiceInactiveState))
3791                 next->invokeNotifer( notify );
3792             next->unlockForArbitration();
3793         }
3794         notify->release();
3795         existing->release();
3796     }
3797
3798     return( notify );
3799 }
3800
3801 bool IOService::syncNotificationHandler(
3802                         void * /* target */, void * ref,
3803                         IOService * newService,
3804                         IONotifier * notifier )
3805 {
3806
3807     LOCKWRITENOTIFY();
3808     if (!*((IOService **) ref))
3809     {
3810         newService->retain();
3811         (*(IOService **) ref) = newService;
3812         WAKEUPNOTIFY(ref);
3813     }
3814     UNLOCKNOTIFY();
3815
3816     return( false );
3817 }
3818
3819 IOService * IOService::waitForMatchingService( OSDictionary * matching,
3820                                                 uint64_t timeout)
3821 {
3822     IONotifier *        notify = 0;
3823     // priority doesn't help us much since we need a thread wakeup
3824     SInt32              priority = 0;
3825     IOService *         result;
3826
3827     if (!matching)
3828         return( 0 );
3829
3830     result = NULL;
3831
3832     LOCKWRITENOTIFY();
3833     do
3834     {
3835         result = (IOService *) copyExistingServices( matching,
3836                             kIOServiceMatchedState, kIONotifyOnce );
3837         if (result)
3838             break;
3839         notify = IOService::setNotification( gIOMatchedNotification, matching,
3840                     &IOService::syncNotificationHandler, (void *) 0,
3841                     &result, priority );
3842          if (!notify)
3843             break;
3844         if (UINT64_MAX != timeout)
3845         {
3846             AbsoluteTime deadline;
3847             nanoseconds_to_absolutetime(timeout, &deadline);
3848             clock_absolutetime_interval_to_deadline(deadline, &deadline);
3849             SLEEPNOTIFYTO(&result, deadline);
3850         }
3851         else
3852         {
3853             SLEEPNOTIFY(&result);
3854         }
3855     }
3856     while( false );
3857
3858     UNLOCKNOTIFY();
3859
3860     if (notify)
3861         notify->remove();       // dequeues
3862
3863     return( result );
3864 }
3865
3866 IOService * IOService::waitForService( OSDictionary * matching,
3867                                         mach_timespec_t * timeout )
3868 {
3869     IOService * result;
3870     uint64_t    timeoutNS;
3871
3872     if (timeout)
3873     {
3874         timeoutNS = timeout->tv_sec;
3875         timeoutNS *= kSecondScale;
3876         timeoutNS += timeout->tv_nsec;
3877     }
3878     else
3879         timeoutNS = UINT64_MAX;
3880
3881     result = waitForMatchingService(matching, timeoutNS);
3882
3883     matching->release();
3884     if (result)
3885         result->release();
3886
3887     return (result);
3888 }
3889
3890 void IOService::deliverNotification( const OSSymbol * type,
3891                             IOOptionBits orNewState, IOOptionBits andNewState )
3892 {
3893     _IOServiceNotifier * notify;
3894     OSIterator *         iter;
3895     OSArray *            willSend = 0;
3896
3897     lockForArbitration();
3898
3899     if( (0 == (__state[0] & kIOServiceInactiveState))
3900      || (type == gIOTerminatedNotification)) {
3901
3902         LOCKREADNOTIFY();
3903
3904         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3905                     gNotifications->getObject( type ) );
3906
3907         if( iter) {
3908             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
3909
3910                 if( passiveMatch( notify->matching)
3911                   && (kIOServiceNotifyEnable & notify->state)) {
3912                     if( 0 == willSend)
3913                         willSend = OSArray::withCapacity(8);
3914                     if( willSend)
3915                         willSend->setObject( notify );
3916                 }
3917             }
3918             iter->release();
3919         }
3920
3921         __state[0] = (__state[0] | orNewState) & andNewState;
3922
3923         UNLOCKNOTIFY();
3924     }
3925
3926     if( willSend) {
3927         for( unsigned int idx = 0;
3928              (notify = (_IOServiceNotifier *) willSend->getObject(idx));
3929              idx++) {
3930             invokeNotifer( notify );
3931         }
3932         willSend->release();
3933     }
3934     unlockForArbitration();
3935 }
3936
3937 IOOptionBits IOService::getState( void ) const
3938 {
3939     return( __state[0] );
3940 }
3941
3942 /*
3943  * Helpers to make matching objects for simple cases
3944  */
3945
3946 OSDictionary * IOService::serviceMatching( const OSString * name,
3947                         OSDictionary * table )
3948 {
3949     if( !table)
3950         table = OSDictionary::withCapacity( 2 );
3951     if( table)
3952         table->setObject(gIOProviderClassKey, (OSObject *)name );
3953
3954     return( table );
3955 }
3956
3957 OSDictionary * IOService::serviceMatching( const char * name,
3958                         OSDictionary * table )
3959 {
3960     const OSString *    str;
3961
3962     str = OSSymbol::withCString( name );
3963     if( !str)
3964         return( 0 );
3965
3966     table = serviceMatching( str, table );
3967     str->release();
3968     return( table );
3969 }
3970
3971 OSDictionary * IOService::nameMatching( const OSString * name,
3972                         OSDictionary * table )
3973 {
3974     if( !table)
3975         table = OSDictionary::withCapacity( 2 );
3976     if( table)
3977         table->setObject( gIONameMatchKey, (OSObject *)name );
3978
3979     return( table );
3980 }
3981
3982 OSDictionary * IOService::nameMatching( const char * name,
3983                         OSDictionary * table )
3984 {
3985     const OSString *    str;
3986
3987     str = OSSymbol::withCString( name );
3988     if( !str)
3989         return( 0 );
3990
3991     table = nameMatching( str, table );
3992     str->release();
3993     return( table );
3994 }
3995
3996 OSDictionary * IOService::resourceMatching( const OSString * str,
3997                         OSDictionary * table )
3998 {
3999     table = serviceMatching( gIOResourcesKey, table );
4000     if( table)
4001         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4002
4003     return( table );
4004 }
4005
4006 OSDictionary * IOService::resourceMatching( const char * name,
4007                         OSDictionary * table )
4008 {
4009     const OSSymbol *    str;
4010
4011     str = OSSymbol::withCString( name );
4012     if( !str)
4013         return( 0 );
4014
4015     table = resourceMatching( str, table );
4016     str->release();
4017
4018     return( table );
4019 }
4020
4021 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4022                         OSDictionary * table )
4023 {
4024     OSDictionary * properties;
4025
4026     properties = OSDictionary::withCapacity( 2 );
4027     if( !properties)
4028         return( 0 );
4029     properties->setObject( key, value );
4030
4031     if( !table)
4032         table = OSDictionary::withCapacity( 2 );
4033     if( table)
4034         table->setObject( gIOPropertyMatchKey, properties );
4035
4036     properties->release();
4037
4038     return( table );
4039 }
4040
4041 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4042                         OSDictionary * table )
4043 {
4044     OSNumber *     num;
4045
4046     num = OSNumber::withNumber( entryID, 64 );
4047     if( !num)
4048         return( 0 );
4049
4050     if( !table)
4051         table = OSDictionary::withCapacity( 2 );
4052     if( table)
4053         table->setObject( gIORegistryEntryIDKey, num );
4054
4055     if (num)
4056         num->release();
4057
4058     return( table );
4059 }
4060
4061
4062 /*
4063  * _IOServiceNotifier
4064  */
4065
4066 // wait for all threads, other than the current one,
4067 //  to exit the handler
4068
4069 void _IOServiceNotifier::wait()
4070 {
4071     _IOServiceNotifierInvocation * next;
4072     bool doWait;
4073
4074     do {
4075         doWait = false;
4076         queue_iterate( &handlerInvocations, next,
4077                         _IOServiceNotifierInvocation *, link) {
4078             if( next->thread != current_thread() ) {
4079                 doWait = true;
4080                 break;
4081             }
4082         }
4083         if( doWait) {
4084             state |= kIOServiceNotifyWaiter;
4085             SLEEPNOTIFY(this);
4086         }
4087
4088     } while( doWait );
4089 }
4090
4091 void _IOServiceNotifier::free()
4092 {
4093     assert( queue_empty( &handlerInvocations ));
4094     OSObject::free();
4095 }
4096
4097 void _IOServiceNotifier::remove()
4098 {
4099     LOCKWRITENOTIFY();
4100
4101     if( whence) {
4102         whence->removeObject( (OSObject *) this );
4103         whence = 0;
4104     }
4105     if( matching) {
4106         matching->release();
4107         matching = 0;
4108     }
4109
4110     state &= ~kIOServiceNotifyEnable;
4111
4112     wait();
4113
4114     UNLOCKNOTIFY();
4115
4116     release();
4117 }
4118
4119 bool _IOServiceNotifier::disable()
4120 {
4121     bool        ret;
4122
4123     LOCKWRITENOTIFY();
4124
4125     ret = (0 != (kIOServiceNotifyEnable & state));
4126     state &= ~kIOServiceNotifyEnable;
4127     if( ret)
4128         wait();
4129
4130     UNLOCKNOTIFY();
4131
4132     return( ret );
4133 }
4134
4135 void _IOServiceNotifier::enable( bool was )
4136 {
4137     LOCKWRITENOTIFY();
4138     if( was)
4139         state |= kIOServiceNotifyEnable;
4140     else
4141         state &= ~kIOServiceNotifyEnable;
4142     UNLOCKNOTIFY();
4143 }
4144
4145 /*
4146  * IOResources
4147  */
4148
4149 IOService * IOResources::resources( void )
4150 {
4151     IOResources *       inst;
4152
4153     inst = new IOResources;
4154     if( inst && !inst->init()) {
4155         inst->release();
4156         inst = 0;
4157     }
4158
4159     return( inst );
4160 }
4161
4162 bool IOResources::init( OSDictionary * dictionary )
4163 {
4164     // Do super init first
4165     if ( !super::init() )
4166         return false;
4167
4168     // Allow PAL layer to publish a value
4169     const char *property_name;
4170     int property_value;
4171
4172     pal_get_resource_property( &property_name, &property_value );
4173
4174     if( property_name ) {
4175         OSNumber *num;
4176         const OSSymbol *        sym;
4177
4178         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
4179             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
4180                 this->setProperty( sym, num );
4181                 sym->release();
4182             }
4183             num->release();
4184         }
4185     }
4186
4187     return true;
4188 }
4189
4190 IOWorkLoop * IOResources::getWorkLoop() const
4191 {
4192     // If we are the resource root
4193     // then use the platform's workloop
4194     if (this == (IOResources *) gIOResources)
4195         return getPlatform()->getWorkLoop();
4196     else
4197         return IOService::getWorkLoop();
4198 }
4199
4200 bool IOResources::matchPropertyTable( OSDictionary * table )
4201 {
4202     OSObject *          prop;
4203     OSString *          str;
4204     OSSet *             set;
4205     OSIterator *        iter;
4206     bool                ok = false;
4207
4208     prop = table->getObject( gIOResourceMatchKey );
4209     str = OSDynamicCast( OSString, prop );
4210     if( str)
4211         ok = (0 != getProperty( str ));
4212
4213     else if( (set = OSDynamicCast( OSSet, prop))) {
4214
4215         iter = OSCollectionIterator::withCollection( set );
4216         ok = (iter != 0);
4217         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
4218             ok = (0 != getProperty( str ));
4219
4220         if( iter)
4221             iter->release();
4222     }
4223
4224     return( ok );
4225 }
4226
4227 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
4228 {
4229     IOService::updateConsoleUsers(NULL, 0);
4230 }
4231
4232 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
4233 {
4234     IORegistryEntry * regEntry;
4235     OSObject *        locked = kOSBooleanFalse;
4236     uint32_t          idx;
4237     bool              publish;
4238     OSDictionary *    user;
4239     static IOMessage  sSystemPower;
4240
4241     regEntry = IORegistryEntry::getRegistryRoot();
4242
4243     IOLockLock(gIOConsoleUsersLock);
4244
4245     if (systemMessage)
4246     {
4247         sSystemPower = systemMessage;
4248     }
4249     if (consoleUsers)
4250     {
4251         OSNumber * num = 0;
4252         for (idx = 0;
4253               (!num) && (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
4254               idx++)
4255         {
4256             num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
4257         }
4258         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
4259     }
4260
4261     if (gIOConsoleLockTime)
4262     {
4263         if (kIOMessageSystemWillSleep == sSystemPower)
4264             locked = kOSBooleanTrue;
4265         else
4266         {
4267             clock_sec_t  now;
4268             clock_usec_t microsecs;
4269
4270             clock_get_calendar_microtime(&now, &microsecs);
4271             if (gIOConsoleLockTime > now)
4272             {
4273                 AbsoluteTime deadline;
4274                 clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
4275                 thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
4276             }
4277             else
4278             {
4279                 locked = kOSBooleanTrue;
4280             }
4281         }
4282     }
4283
4284     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
4285     if (publish)
4286     {
4287         regEntry->setProperty(gIOConsoleLockedKey, locked);
4288         if (consoleUsers)
4289         {
4290             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
4291         }
4292         OSIncrementAtomic( &gIOConsoleUsersSeed );
4293     }
4294
4295     IOLockUnlock(gIOConsoleUsersLock);
4296
4297     if (publish)
4298     {
4299         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
4300
4301         MessageClientsContext context;
4302
4303         context.service  = getServiceRoot();
4304         context.type     = kIOMessageConsoleSecurityChange;
4305         context.argument = (void *) regEntry;
4306         context.argSize  = 0;
4307
4308         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
4309                                  &messageClientsApplier, &context );
4310     }
4311 }
4312
4313 IOReturn IOResources::setProperties( OSObject * properties )
4314 {
4315     IOReturn                    err;
4316     const OSSymbol *            key;
4317     OSDictionary *              dict;
4318     OSCollectionIterator *      iter;
4319
4320     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
4321     if ( kIOReturnSuccess != err)
4322         return( err );
4323
4324     dict = OSDynamicCast(OSDictionary, properties);
4325     if( 0 == dict)
4326         return( kIOReturnBadArgument);
4327
4328     iter = OSCollectionIterator::withCollection( dict);
4329     if( 0 == iter)
4330         return( kIOReturnBadArgument);
4331
4332     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
4333     {
4334         if (gIOConsoleUsersKey == key) do
4335         {
4336             OSArray * consoleUsers;
4337             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
4338             if (!consoleUsers)
4339                 continue;
4340             IOService::updateConsoleUsers(consoleUsers, 0);
4341         }
4342         while (false);
4343
4344         publishResource( key, dict->getObject(key) );
4345     }
4346
4347     iter->release();
4348
4349     return( kIOReturnSuccess );
4350 }
4351
4352 /*
4353  * Helpers for matching dictionaries.
4354  * Keys existing in matching are checked in properties.
4355  * Keys may be a string or OSCollection of IOStrings
4356  */
4357
4358 bool IOService::compareProperty( OSDictionary * matching,
4359                                  const char *   key )
4360 {
4361     OSObject *  value;
4362     bool        ok;
4363
4364     value = matching->getObject( key );
4365     if( value)
4366         ok = value->isEqualTo( getProperty( key ));
4367     else
4368         ok = true;
4369
4370     return( ok );
4371 }
4372
4373
4374 bool IOService::compareProperty( OSDictionary *   matching,
4375                                  const OSString * key )
4376 {
4377     OSObject *  value;
4378     bool        ok;
4379
4380     value = matching->getObject( key );
4381     if( value)
4382         ok = value->isEqualTo( getProperty( key ));
4383     else
4384         ok = true;
4385
4386     return( ok );
4387 }
4388
4389 bool IOService::compareProperties( OSDictionary * matching,
4390                                    OSCollection * keys )
4391 {
4392     OSCollectionIterator *      iter;
4393     const OSString *            key;
4394     bool                        ok = true;
4395
4396     if( !matching || !keys)
4397         return( false );
4398
4399     iter = OSCollectionIterator::withCollection( keys );
4400
4401     if( iter) {
4402         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
4403             ok = compareProperty( matching, key );
4404
4405         iter->release();
4406     }
4407     keys->release();    // !! consume a ref !!
4408
4409     return( ok );
4410 }
4411
4412 /* Helper to add a location matching dict to the table */
4413
4414 OSDictionary * IOService::addLocation( OSDictionary * table )
4415 {
4416     OSDictionary *      dict;
4417
4418     if( !table)
4419         return( 0 );
4420
4421     dict = OSDictionary::withCapacity( 1 );
4422     if( dict) {
4423         table->setObject( gIOLocationMatchKey, dict );
4424         dict->release();
4425     }
4426
4427     return( dict );
4428 }
4429
4430 /*
4431  * Go looking for a provider to match a location dict.
4432  */
4433
4434 IOService * IOService::matchLocation( IOService * /* client */ )
4435 {
4436     IOService * parent;
4437
4438     parent = getProvider();
4439
4440     if( parent)
4441         parent = parent->matchLocation( this );
4442
4443     return( parent );
4444 }
4445
4446 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
4447 {
4448     IOService *         where;
4449     OSString *          matched;
4450     OSObject *          obj;
4451     OSString *          str;
4452     IORegistryEntry *   entry;
4453     OSNumber *          num;
4454     SInt32              score;
4455     OSNumber *          newPri;
4456     bool                match = true;
4457     bool                matchParent = false;
4458     UInt32              done;
4459
4460     assert( table );
4461
4462     where = this;
4463
4464     do {
4465         do {
4466             done = 0;
4467
4468             str = OSDynamicCast( OSString, table->getObject( gIOProviderClassKey));
4469             if( str) {
4470                 done++;
4471                 match = (0 != where->metaCast( str ));
4472                 if( !match)
4473                     break;
4474             }
4475
4476             obj = table->getObject( gIONameMatchKey );
4477             if( obj) {
4478                 done++;
4479                 match = where->compareNames( obj, changesOK ? &matched : 0 );
4480                 if( !match)
4481                     break;
4482                 if( changesOK && matched) {
4483                     // leave a hint as to which name matched
4484                     table->setObject( gIONameMatchedKey, matched );
4485                     matched->release();
4486                 }
4487             }
4488
4489             str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
4490             if( str) {
4491
4492                 const OSSymbol * sym;
4493
4494                 done++;
4495                 match = false;
4496                 sym = where->copyLocation();
4497                 if( sym) {
4498                     match = sym->isEqualTo( str );
4499                     sym->release();
4500                 }
4501                 if( !match)
4502                     break;
4503             }
4504
4505             obj = table->getObject( gIOPropertyMatchKey );
4506             if( obj) {
4507
4508                 OSDictionary * dict;
4509                 OSDictionary * nextDict;
4510                 OSIterator *   iter;
4511
4512                 done++;
4513                 match = false;
4514                 dict = where->dictionaryWithProperties();
4515                 if( dict) {
4516                     nextDict = OSDynamicCast( OSDictionary, obj);
4517                     if( nextDict)
4518                         iter = 0;
4519                     else
4520                         iter = OSCollectionIterator::withCollection(
4521                                     OSDynamicCast(OSCollection, obj));
4522
4523                     while( nextDict
4524                         || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
4525                                                 iter->getNextObject()))))) {
4526                         match = dict->isEqualTo( nextDict, nextDict);
4527                         if( match)
4528                             break;
4529                         nextDict = 0;
4530                     }
4531                     dict->release();
4532                     if( iter)
4533                         iter->release();
4534                 }
4535                 if( !match)
4536                     break;
4537             }
4538
4539             str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
4540             if( str) {
4541                 done++;
4542                 entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
4543                 match = (where == entry);
4544                 if( entry)
4545                     entry->release();
4546                 if( !match)
4547                     break;
4548             }
4549
4550             num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
4551             if( num) {
4552                 done++;
4553                 match = (getRegistryEntryID() == num->unsigned64BitValue());
4554             }
4555
4556             num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
4557             if( num) {
4558
4559                 OSIterator *    iter;
4560                 IOService *             service = 0;
4561                 UInt32          serviceCount = 0;
4562
4563                 done++;
4564                 iter = where->getClientIterator();
4565                 if( iter) {
4566                     while( (service = (IOService *) iter->getNextObject())) {
4567                         if( kIOServiceInactiveState & service->__state[0])
4568                             continue;
4569                         if( 0 == service->getProperty( gIOMatchCategoryKey ))
4570                             continue;
4571                         ++serviceCount;
4572                     }
4573                     iter->release();
4574                 }
4575                 match = (serviceCount == num->unsigned32BitValue());
4576                 if( !match)
4577                     break;
4578             }
4579
4580             if( done == table->getCount()) {
4581                 // don't call family if we've done all the entries in the table
4582                 matchParent = false;
4583                 break;
4584             }
4585
4586             // pass in score from property table
4587             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
4588
4589             // do family specific matching
4590             match = where->matchPropertyTable( table, &score );
4591
4592             if( !match) {
4593 #if IOMATCHDEBUG
4594                 if( kIOLogMatch & getDebugFlags( table ))
4595                     LOG("%s: family specific matching fails\n", where->getName());
4596 #endif
4597                 break;
4598             }
4599
4600             if( changesOK) {
4601                 // save the score
4602                 newPri = OSNumber::withNumber( score, 32 );
4603                 if( newPri) {
4604                     table->setObject( gIOProbeScoreKey, newPri );
4605                     newPri->release();
4606                 }
4607             }
4608
4609             if( !(match = where->compareProperty( table, kIOBSDNameKey )))
4610                 break;
4611             if( !(match = where->compareProperty( table, kIOBSDMajorKey )))
4612                 break;
4613             if( !(match = where->compareProperty( table, kIOBSDMinorKey )))
4614                 break;
4615             if( !(match = where->compareProperty( table, kIOBSDUnitKey )))
4616                 break;
4617
4618             matchParent = false;
4619
4620             obj = OSDynamicCast( OSDictionary,
4621                   table->getObject( gIOParentMatchKey ));
4622             if( obj) {
4623                 match = false;
4624                 matchParent = true;
4625                 table = (OSDictionary *) obj;
4626                 break;
4627             }
4628
4629             table = OSDynamicCast( OSDictionary,
4630                     table->getObject( gIOLocationMatchKey ));
4631             if( table) {
4632                 match = false;
4633                 where = where->getProvider();
4634                 if( where)
4635                     where = where->matchLocation( where );
4636             }
4637
4638         } while( table && where );
4639
4640     } while( matchParent && (where = where->getProvider()) );
4641
4642     if( kIOLogMatch & gIOKitDebug)
4643         if( where && (where != this) )
4644             LOG("match parent @ %s = %d\n",
4645                         where->getName(), match );
4646
4647     return( match );
4648 }
4649
4650
4651 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
4652                                     UInt32 type,  OSDictionary * properties,
4653                                     IOUserClient ** handler )
4654 {
4655     const OSSymbol *userClientClass = 0;
4656     IOUserClient *client;
4657     OSObject *temp;
4658
4659     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
4660         return kIOReturnSuccess;
4661
4662     // First try my own properties for a user client class name
4663     temp = getProperty(gIOUserClientClassKey);
4664     if (temp) {
4665         if (OSDynamicCast(OSSymbol, temp))
4666             userClientClass = (const OSSymbol *) temp;
4667         else if (OSDynamicCast(OSString, temp)) {
4668             userClientClass = OSSymbol::withString((OSString *) temp);
4669             if (userClientClass)
4670                 setProperty(kIOUserClientClassKey,
4671                             (OSObject *) userClientClass);
4672         }
4673     }
4674
4675     // Didn't find one so lets just bomb out now without further ado.
4676     if (!userClientClass)
4677         return kIOReturnUnsupported;
4678
4679     // This reference is consumed by the IOServiceOpen call
4680     temp = OSMetaClass::allocClassWithName(userClientClass);
4681     if (!temp)
4682         return kIOReturnNoMemory;
4683
4684     if (OSDynamicCast(IOUserClient, temp))
4685         client = (IOUserClient *) temp;
4686     else {
4687         temp->release();
4688         return kIOReturnUnsupported;
4689     }
4690
4691     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
4692         client->release();
4693         return kIOReturnBadArgument;
4694     }
4695
4696     if ( !client->attach(this) ) {
4697         client->release();
4698         return kIOReturnUnsupported;
4699     }
4700
4701     if ( !client->start(this) ) {
4702         client->detach(this);
4703         client->release();
4704         return kIOReturnUnsupported;
4705     }
4706
4707     *handler = client;
4708     return kIOReturnSuccess;
4709 }
4710
4711 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
4712                                     UInt32 type, IOUserClient ** handler )
4713 {
4714     return( kIOReturnUnsupported );
4715 }
4716
4717 IOReturn IOService::requestProbe( IOOptionBits options )
4718 {
4719     return( kIOReturnUnsupported);
4720 }
4721
4722 /*
4723  * Convert an IOReturn to text. Subclasses which add additional
4724  * IOReturn's should override this method and call
4725  * super::stringFromReturn if the desired value is not found.
4726  */
4727
4728 const char * IOService::stringFromReturn( IOReturn rtn )
4729 {
4730     static const IONamedValue IOReturn_values[] = {
4731         {kIOReturnSuccess,          "success"                           },
4732         {kIOReturnError,            "general error"                     },
4733         {kIOReturnNoMemory,         "memory allocation error"           },
4734         {kIOReturnNoResources,      "resource shortage"                 },
4735         {kIOReturnIPCError,         "Mach IPC failure"                  },
4736         {kIOReturnNoDevice,         "no such device"                    },
4737         {kIOReturnNotPrivileged,    "privilege violation"               },
4738         {kIOReturnBadArgument,      "invalid argument"                  },
4739         {kIOReturnLockedRead,       "device is read locked"             },
4740         {kIOReturnLockedWrite,      "device is write locked"            },
4741         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
4742         {kIOReturnBadMessageID,     "bad IPC message ID"                },
4743         {kIOReturnUnsupported,      "unsupported function"              },
4744         {kIOReturnVMError,          "virtual memory error"              },
4745         {kIOReturnInternalError,    "internal driver error"             },
4746         {kIOReturnIOError,          "I/O error"                         },
4747         {kIOReturnCannotLock,       "cannot acquire lock"               },
4748         {kIOReturnNotOpen,          "device is not open"                },
4749         {kIOReturnNotReadable,      "device is not readable"            },
4750         {kIOReturnNotWritable,      "device is not writeable"           },
4751         {kIOReturnNotAligned,       "alignment error"                   },
4752         {kIOReturnBadMedia,         "media error"                       },
4753         {kIOReturnStillOpen,        "device is still open"              },
4754         {kIOReturnRLDError,         "rld failure"                       },
4755         {kIOReturnDMAError,         "DMA failure"                       },
4756         {kIOReturnBusy,             "device is busy"                    },
4757         {kIOReturnTimeout,          "I/O timeout"                       },
4758         {kIOReturnOffline,          "device is offline"                 },
4759         {kIOReturnNotReady,         "device is not ready"               },
4760         {kIOReturnNotAttached,      "device/channel is not attached"    },
4761         {kIOReturnNoChannels,       "no DMA channels available"         },
4762         {kIOReturnNoSpace,          "no space for data"                 },
4763         {kIOReturnPortExists,       "device port already exists"        },
4764         {kIOReturnCannotWire,       "cannot wire physical memory"       },
4765         {kIOReturnNoInterrupt,      "no interrupt attached"             },
4766         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
4767         {kIOReturnMessageTooLarge,  "message is too large"              },
4768         {kIOReturnNotPermitted,     "operation is not permitted"        },
4769         {kIOReturnNoPower,          "device is without power"           },
4770         {kIOReturnNoMedia,          "media is not present"              },
4771         {kIOReturnUnformattedMedia, "media is not formatted"            },
4772         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
4773         {kIOReturnUnderrun,         "data underrun"                     },
4774         {kIOReturnOverrun,          "data overrun"                      },
4775         {kIOReturnDeviceError,      "device error"                      },
4776         {kIOReturnNoCompletion,     "no completion routine"             },
4777         {kIOReturnAborted,          "operation was aborted"             },
4778         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
4779         {kIOReturnNotResponding,    "device is not responding"          },
4780         {kIOReturnInvalid,          "unanticipated driver error"        },
4781         {0,                         NULL                                }
4782     };
4783
4784     return IOFindNameForValue(rtn, IOReturn_values);
4785 }
4786
4787 /*
4788  * Convert an IOReturn to an errno.
4789  */
4790 int IOService::errnoFromReturn( IOReturn rtn )
4791 {
4792     switch(rtn) {
4793         // (obvious match)
4794         case kIOReturnSuccess:
4795             return(0);
4796         case kIOReturnNoMemory:
4797             return(ENOMEM);
4798         case kIOReturnNoDevice:
4799             return(ENXIO);
4800         case kIOReturnVMError:
4801             return(EFAULT);
4802         case kIOReturnNotPermitted:
4803             return(EPERM);
4804         case kIOReturnNotPrivileged:
4805             return(EACCES);
4806         case kIOReturnIOError:
4807             return(EIO);
4808         case kIOReturnNotWritable:
4809             return(EROFS);
4810         case kIOReturnBadArgument:
4811             return(EINVAL);
4812         case kIOReturnUnsupported:
4813             return(ENOTSUP);
4814         case kIOReturnBusy:
4815             return(EBUSY);
4816         case kIOReturnNoPower:
4817             return(EPWROFF);
4818         case kIOReturnDeviceError:
4819             return(EDEVERR);
4820         case kIOReturnTimeout:
4821             return(ETIMEDOUT);
4822         case kIOReturnMessageTooLarge:
4823             return(EMSGSIZE);
4824         case kIOReturnNoSpace:
4825             return(ENOSPC);
4826         case kIOReturnCannotLock:
4827             return(ENOLCK);
4828
4829         // (best match)
4830         case kIOReturnBadMessageID:
4831         case kIOReturnNoCompletion:
4832         case kIOReturnNotAligned:
4833             return(EINVAL);
4834         case kIOReturnNotReady:
4835             return(EBUSY);
4836         case kIOReturnRLDError:
4837             return(EBADMACHO);
4838         case kIOReturnPortExists:
4839         case kIOReturnStillOpen:
4840             return(EEXIST);
4841         case kIOReturnExclusiveAccess:
4842         case kIOReturnLockedRead:
4843         case kIOReturnLockedWrite:
4844         case kIOReturnNotOpen:
4845         case kIOReturnNotReadable:
4846             return(EACCES);
4847         case kIOReturnCannotWire:
4848         case kIOReturnNoResources:
4849             return(ENOMEM);
4850         case kIOReturnAborted:
4851         case kIOReturnOffline:
4852         case kIOReturnNotResponding:
4853             return(EBUSY);
4854         case kIOReturnBadMedia:
4855         case kIOReturnNoMedia:
4856         case kIOReturnNotAttached:
4857         case kIOReturnUnformattedMedia:
4858             return(ENXIO); // (media error)
4859         case kIOReturnDMAError:
4860         case kIOReturnOverrun:
4861         case kIOReturnUnderrun:
4862             return(EIO); // (transfer error)
4863         case kIOReturnNoBandwidth:
4864         case kIOReturnNoChannels:
4865         case kIOReturnNoFrames:
4866         case kIOReturnNoInterrupt:
4867             return(EIO); // (hardware error)
4868         case kIOReturnError:
4869         case kIOReturnInternalError:
4870         case kIOReturnInvalid:
4871             return(EIO); // (generic error)
4872         case kIOReturnIPCError:
4873             return(EIO); // (ipc error)
4874         default:
4875             return(EIO); // (all other errors)
4876     }
4877 }
4878
4879 IOReturn IOService::message( UInt32 type, IOService * provider,
4880                                 void * argument )
4881 {
4882     /*
4883      * Generic entry point for calls from the provider.  A return value of
4884      * kIOReturnSuccess indicates that the message was received, and where
4885      * applicable, that it was successful.
4886      */
4887
4888     return kIOReturnUnsupported;
4889 }
4890
4891 /*
4892  * Device memory
4893  */
4894
4895 IOItemCount IOService::getDeviceMemoryCount( void )
4896 {
4897     OSArray *           array;
4898     IOItemCount         count;
4899
4900     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
4901     if( array)
4902         count = array->getCount();
4903     else
4904         count = 0;
4905
4906     return( count);
4907 }
4908
4909 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
4910 {
4911     OSArray *           array;
4912     IODeviceMemory *    range;
4913
4914     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
4915     if( array)
4916         range = (IODeviceMemory *) array->getObject( index );
4917     else
4918         range = 0;
4919
4920     return( range);
4921 }
4922
4923 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
4924                                                 IOOptionBits options )
4925 {
4926     IODeviceMemory *    range;
4927     IOMemoryMap *       map;
4928
4929     range = getDeviceMemoryWithIndex( index );
4930     if( range)
4931         map = range->map( options );
4932     else
4933         map = 0;
4934
4935     return( map );
4936 }
4937
4938 OSArray * IOService::getDeviceMemory( void )
4939 {
4940     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
4941 }
4942
4943
4944 void IOService::setDeviceMemory( OSArray * array )
4945 {
4946     setProperty( gIODeviceMemoryKey, array);
4947 }
4948
4949 /*
4950  * For machines where the transfers on an I/O bus can stall because
4951  * the CPU is in an idle mode, These APIs allow a driver to specify
4952  * the maximum bus stall that they can handle.  0 indicates no limit.
4953  */
4954 void IOService::
4955 setCPUSnoopDelay(UInt32 __unused ns)
4956 {
4957 #if defined(__i386__) || defined(__x86_64__)
4958     ml_set_maxsnoop(ns);
4959 #endif /* defined(__i386__) || defined(__x86_64__) */
4960 }
4961
4962 UInt32 IOService::
4963 getCPUSnoopDelay()
4964 {
4965 #if defined(__i386__) || defined(__x86_64__)
4966     return ml_get_maxsnoop();
4967 #else
4968     return 0;
4969 #endif /* defined(__i386__) || defined(__x86_64__) */
4970 }
4971
4972 #if defined(__i386__) || defined(__x86_64__)
4973 static void
4974 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
4975 {
4976     static const UInt kNoReplace = -1U; // Must be an illegal index
4977     UInt replace = kNoReplace;
4978     bool setCpuDelay = false;
4979
4980     IORecursiveLockLock(sCpuDelayLock);
4981
4982     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
4983     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
4984     IOService * holder = NULL;
4985
4986     if (ns) {
4987         const CpuDelayEntry ne = {service, ns, delayType};
4988         holder = service;
4989         // Set maximum delay.
4990         for (UInt i = 0; i < count; i++) {
4991             IOService *thisService = entries[i].fService;
4992             bool sameType = (delayType == entries[i].fDelayType);
4993             if ((service == thisService) && sameType)
4994                 replace = i;
4995             else if (!thisService) {
4996                 if (kNoReplace == replace)
4997                     replace = i;
4998             }
4999             else if (sameType) {
5000                 const UInt32 thisMax = entries[i].fMaxDelay;
5001                 if (thisMax < ns)
5002                 {
5003                     ns = thisMax;
5004                     holder = thisService;
5005                 }
5006             }
5007         }
5008
5009         setCpuDelay = true;
5010         if (kNoReplace == replace)
5011             sCpuDelayData->appendBytes(&ne, sizeof(ne));
5012         else
5013             entries[replace] = ne;
5014     }
5015     else {
5016         ns = -1U;       // Set to max unsigned, i.e. no restriction
5017
5018         for (UInt i = 0; i < count; i++) {
5019             // Clear a maximum delay.
5020             IOService *thisService = entries[i].fService;
5021             if (thisService && (delayType == entries[i].fDelayType)) {
5022                 UInt32 thisMax = entries[i].fMaxDelay;
5023                 if (service == thisService)
5024                     replace = i;
5025                 else if (thisMax < ns) {
5026                     ns = thisMax;
5027                     holder = thisService;
5028                 }
5029             }
5030         }
5031
5032         // Check if entry found
5033         if (kNoReplace != replace) {
5034             entries[replace].fService = 0;      // Null the entry
5035             setCpuDelay = true;
5036         }
5037     }
5038
5039     if (setCpuDelay)
5040     {
5041         // Must be safe to call from locked context
5042         if (delayType == kCpuDelayBusStall)
5043         {
5044             ml_set_maxbusdelay(ns);
5045         }
5046         else if (delayType == kCpuDelayInterrupt)
5047         {
5048             ml_set_maxintdelay(ns);
5049         }
5050
5051         OSArray * handlers = sCpuLatencyHandlers[delayType];
5052         IOService * target;
5053         if (handlers) for (unsigned int idx = 0;
5054                             (target = (IOService *) handlers->getObject(idx));
5055                             idx++)
5056         {
5057             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5058                                             (void *) (uintptr_t) ns, holder,
5059                                             NULL, NULL);
5060         }
5061     }
5062
5063     IORecursiveLockUnlock(sCpuDelayLock);
5064 }
5065
5066 static IOReturn
5067 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
5068 {
5069     IOReturn result = kIOReturnNotFound;
5070     OSArray * array;
5071     unsigned int idx;
5072
5073     IORecursiveLockLock(sCpuDelayLock);
5074
5075     do
5076     {
5077         if (enable && !sCpuLatencyHandlers[delayType])
5078             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
5079         array = sCpuLatencyHandlers[delayType];
5080         if (!array)
5081             break;
5082         idx = array->getNextIndexOfObject(target, 0);
5083         if (!enable)
5084         {
5085             if (-1U != idx)
5086             {
5087                 array->removeObject(idx);
5088                 result = kIOReturnSuccess;
5089             }
5090         }
5091         else
5092         {
5093             if (-1U != idx) {
5094                 result = kIOReturnExclusiveAccess;
5095                 break;
5096             }
5097             array->setObject(target);
5098
5099             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5100             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5101             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
5102             IOService * holder = NULL;
5103
5104             for (UInt i = 0; i < count; i++) {
5105                 if (entries[i].fService
5106                   && (delayType == entries[i].fDelayType)
5107                   && (entries[i].fMaxDelay < ns)) {
5108                     ns = entries[i].fMaxDelay;
5109                     holder = entries[i].fService;
5110                 }
5111             }
5112             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5113                                             (void *) (uintptr_t) ns, holder,
5114                                             NULL, NULL);
5115             result = kIOReturnSuccess;
5116         }
5117     }
5118     while (false);
5119
5120     IORecursiveLockUnlock(sCpuDelayLock);
5121
5122     return (result);
5123 }
5124
5125 #endif /* defined(__i386__) || defined(__x86_64__) */
5126
5127 void IOService::
5128 requireMaxBusStall(UInt32 __unused ns)
5129 {
5130 #if defined(__i386__) || defined(__x86_64__)
5131     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
5132 #endif
5133 }
5134
5135 void IOService::
5136 requireMaxInterruptDelay(uint32_t __unused ns)
5137 {
5138 #if defined(__i386__) || defined(__x86_64__)
5139     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
5140 #endif
5141 }
5142
5143 /*
5144  * Device interrupts
5145  */
5146
5147 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
5148 {
5149   IOInterruptController *interruptController;
5150   OSArray               *array;
5151   OSData                *data;
5152   OSSymbol              *interruptControllerName;
5153   long                  numSources;
5154   IOInterruptSource     *interruptSources;
5155
5156   // Get the parents list from the nub.
5157   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
5158   if (array == 0) return kIOReturnNoResources;
5159
5160   // Allocate space for the IOInterruptSources if needed... then return early.
5161   if (nub->_interruptSources == 0) {
5162     numSources = array->getCount();
5163     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
5164     if (interruptSources == 0) return kIOReturnNoMemory;
5165
5166     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
5167
5168     nub->_numInterruptSources = numSources;
5169     nub->_interruptSources = interruptSources;
5170     return kIOReturnSuccess;
5171   }
5172
5173   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
5174   if (interruptControllerName == 0) return kIOReturnNoResources;
5175
5176   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
5177   if (interruptController == 0) return kIOReturnNoResources;
5178
5179   // Get the interrupt numbers from the nub.
5180   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
5181   if (array == 0) return kIOReturnNoResources;
5182   data = OSDynamicCast(OSData, array->getObject(source));
5183   if (data == 0) return kIOReturnNoResources;
5184
5185   // Set the interruptController and interruptSource in the nub's table.
5186   interruptSources = nub->_interruptSources;
5187   interruptSources[source].interruptController = interruptController;
5188   interruptSources[source].vectorData = data;
5189
5190   return kIOReturnSuccess;
5191 }
5192
5193 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
5194 {
5195   IOReturn ret;
5196
5197   /* Make sure the _interruptSources are set */
5198   if (_interruptSources == 0) {
5199     ret = resolveInterrupt(this, source);
5200     if (ret != kIOReturnSuccess) return ret;
5201   }
5202
5203   /* Make sure the local source number is valid */
5204   if ((source < 0) || (source >= _numInterruptSources))
5205     return kIOReturnNoInterrupt;
5206
5207   /* Look up the contoller for the local source */
5208   *interruptController = _interruptSources[source].interruptController;
5209
5210   if (*interruptController == NULL) {
5211     if (!resolve) return kIOReturnNoInterrupt;
5212
5213     /* Try to reslove the interrupt */
5214     ret = resolveInterrupt(this, source);
5215     if (ret != kIOReturnSuccess) return ret;
5216
5217     *interruptController = _interruptSources[source].interruptController;
5218   }
5219
5220   return kIOReturnSuccess;
5221 }
5222
5223 IOReturn IOService::registerInterrupt(int source, OSObject *target,
5224                                       IOInterruptAction handler,
5225                                       void *refCon)
5226 {
5227   IOInterruptController *interruptController;
5228   IOReturn              ret;
5229
5230   ret = lookupInterrupt(source, true, &interruptController);
5231   if (ret != kIOReturnSuccess) return ret;
5232
5233   /* Register the source */
5234   return interruptController->registerInterrupt(this, source, target,
5235                                                 (IOInterruptHandler)handler,
5236                                                 refCon);
5237 }
5238
5239 IOReturn IOService::unregisterInterrupt(int source)
5240 {
5241   IOInterruptController *interruptController;
5242   IOReturn              ret;
5243
5244   ret = lookupInterrupt(source, false, &interruptController);
5245   if (ret != kIOReturnSuccess) return ret;
5246
5247   /* Unregister the source */
5248   return interruptController->unregisterInterrupt(this, source);
5249 }
5250
5251 IOReturn IOService::getInterruptType(int source, int *interruptType)
5252 {
5253   IOInterruptController *interruptController;
5254   IOReturn              ret;
5255
5256   ret = lookupInterrupt(source, true, &interruptController);
5257   if (ret != kIOReturnSuccess) return ret;
5258
5259   /* Return the type */
5260   return interruptController->getInterruptType(this, source, interruptType);
5261 }
5262
5263 IOReturn IOService::enableInterrupt(int source)
5264 {
5265   IOInterruptController *interruptController;
5266   IOReturn              ret;
5267
5268   ret = lookupInterrupt(source, false, &interruptController);
5269   if (ret != kIOReturnSuccess) return ret;
5270
5271   /* Enable the source */
5272   return interruptController->enableInterrupt(this, source);
5273 }
5274
5275 IOReturn IOService::disableInterrupt(int source)
5276 {
5277   IOInterruptController *interruptController;
5278   IOReturn              ret;
5279
5280   ret = lookupInterrupt(source, false, &interruptController);
5281   if (ret != kIOReturnSuccess) return ret;
5282
5283   /* Disable the source */
5284   return interruptController->disableInterrupt(this, source);
5285 }
5286
5287 IOReturn IOService::causeInterrupt(int source)
5288 {
5289   IOInterruptController *interruptController;
5290   IOReturn              ret;
5291
5292   ret = lookupInterrupt(source, false, &interruptController);
5293   if (ret != kIOReturnSuccess) return ret;
5294
5295   /* Cause an interrupt for the source */
5296   return interruptController->causeInterrupt(this, source);
5297 }
5298
5299 #if __LP64__
5300 OSMetaClassDefineReservedUnused(IOService, 0);
5301 OSMetaClassDefineReservedUnused(IOService, 1);
5302 OSMetaClassDefineReservedUnused(IOService, 2);
5303 OSMetaClassDefineReservedUnused(IOService, 3);
5304 OSMetaClassDefineReservedUnused(IOService, 4);
5305 OSMetaClassDefineReservedUnused(IOService, 5);
5306 #else
5307 OSMetaClassDefineReservedUsed(IOService, 0);
5308 OSMetaClassDefineReservedUsed(IOService, 1);
5309 OSMetaClassDefineReservedUsed(IOService, 2);
5310 OSMetaClassDefineReservedUsed(IOService, 3);
5311 OSMetaClassDefineReservedUsed(IOService, 4);
5312 OSMetaClassDefineReservedUsed(IOService, 5);
5313 #endif
5314 OSMetaClassDefineReservedUnused(IOService, 6);
5315 OSMetaClassDefineReservedUnused(IOService, 7);
5316 OSMetaClassDefineReservedUnused(IOService, 8);
5317 OSMetaClassDefineReservedUnused(IOService, 9);
5318 OSMetaClassDefineReservedUnused(IOService, 10);
5319 OSMetaClassDefineReservedUnused(IOService, 11);
5320 OSMetaClassDefineReservedUnused(IOService, 12);
5321 OSMetaClassDefineReservedUnused(IOService, 13);
5322 OSMetaClassDefineReservedUnused(IOService, 14);
5323 OSMetaClassDefineReservedUnused(IOService, 15);
5324 OSMetaClassDefineReservedUnused(IOService, 16);
5325 OSMetaClassDefineReservedUnused(IOService, 17);
5326 OSMetaClassDefineReservedUnused(IOService, 18);
5327 OSMetaClassDefineReservedUnused(IOService, 19);
5328 OSMetaClassDefineReservedUnused(IOService, 20);
5329 OSMetaClassDefineReservedUnused(IOService, 21);
5330 OSMetaClassDefineReservedUnused(IOService, 22);
5331 OSMetaClassDefineReservedUnused(IOService, 23);
5332 OSMetaClassDefineReservedUnused(IOService, 24);
5333 OSMetaClassDefineReservedUnused(IOService, 25);
5334 OSMetaClassDefineReservedUnused(IOService, 26);
5335 OSMetaClassDefineReservedUnused(IOService, 27);
5336 OSMetaClassDefineReservedUnused(IOService, 28);
5337 OSMetaClassDefineReservedUnused(IOService, 29);
5338 OSMetaClassDefineReservedUnused(IOService, 30);
5339 OSMetaClassDefineReservedUnused(IOService, 31);
5340 OSMetaClassDefineReservedUnused(IOService, 32);
5341 OSMetaClassDefineReservedUnused(IOService, 33);
5342 OSMetaClassDefineReservedUnused(IOService, 34);
5343 OSMetaClassDefineReservedUnused(IOService, 35);
5344 OSMetaClassDefineReservedUnused(IOService, 36);
5345 OSMetaClassDefineReservedUnused(IOService, 37);
5346 OSMetaClassDefineReservedUnused(IOService, 38);
5347 OSMetaClassDefineReservedUnused(IOService, 39);
5348 OSMetaClassDefineReservedUnused(IOService, 40);
5349 OSMetaClassDefineReservedUnused(IOService, 41);
5350 OSMetaClassDefineReservedUnused(IOService, 42);
5351 OSMetaClassDefineReservedUnused(IOService, 43);
5352 OSMetaClassDefineReservedUnused(IOService, 44);
5353 OSMetaClassDefineReservedUnused(IOService, 45);
5354 OSMetaClassDefineReservedUnused(IOService, 46);
5355 OSMetaClassDefineReservedUnused(IOService, 47);