iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <mach/sync_policy.h>
  54 #include <IOKit/assert.h>
  55 #include <sys/errno.h>
  56
  57 #include <machine/pal_routines.h>
  58
  59 #define LOG kprintf
  60 //#define LOG IOLog
  61 #define MATCH_DEBUG     0
  62 #define OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  63
  64 #include "IOServicePrivate.h"
  65 #include "IOKitKernelInternal.h"
  66
  67 // take lockForArbitration before LOCKNOTIFY
  68
  69 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  70
  71 #define super IORegistryEntry
  72
  73 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  74
  75 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  76
  77 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  78
  79 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  80
  81 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  82
  83 OSDefineMetaClassAndStructors(IOResources, IOService)
  84
  85 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  86
  87 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  88
  89 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  90
  91 static IOPlatformExpert *       gIOPlatform;
  92 static class IOPMrootDomain *   gIOPMRootDomain;
  93 const IORegistryPlane *         gIOServicePlane;
  94 const IORegistryPlane *         gIOPowerPlane;
  95 const OSSymbol *                gIODeviceMemoryKey;
  96 const OSSymbol *                gIOInterruptControllersKey;
  97 const OSSymbol *                gIOInterruptSpecifiersKey;
  98
  99 const OSSymbol *                gIOResourcesKey;
 100 const OSSymbol *                gIOResourceMatchKey;
 101 const OSSymbol *                gIOProviderClassKey;
 102 const OSSymbol *                gIONameMatchKey;
 103 const OSSymbol *                gIONameMatchedKey;
 104 const OSSymbol *                gIOPropertyMatchKey;
 105 const OSSymbol *                gIOLocationMatchKey;
 106 const OSSymbol *                gIOParentMatchKey;
 107 const OSSymbol *                gIOPathMatchKey;
 108 const OSSymbol *                gIOMatchCategoryKey;
 109 const OSSymbol *                gIODefaultMatchCategoryKey;
 110 const OSSymbol *                gIOMatchedServiceCountKey;
 111
 112 const OSSymbol *                gIOMapperIDKey;
 113 const OSSymbol *                gIOUserClientClassKey;
 114 const OSSymbol *                gIOKitDebugKey;
 115
 116 const OSSymbol *                gIOCommandPoolSizeKey;
 117
 118 const OSSymbol *                gIOConsoleLockedKey;
 119 const OSSymbol *                gIOConsoleUsersKey;
 120 const OSSymbol *                gIOConsoleSessionUIDKey;
 121 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 122 const OSSymbol *                gIOConsoleUsersSeedKey;
 123 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 124 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 125 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 126 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 127
 128 clock_sec_t                     gIOConsoleLockTime;
 129 static bool                     gIOConsoleLoggedIn;
 130 #if HIBERNATION
 131 static uint32_t                 gIOScreenLockState;
 132 #endif
 133 static IORegistryEntry *        gIOChosenEntry;
 134
 135 static int                      gIOResourceGenerationCount;
 136
 137 const OSSymbol *                gIOServiceKey;
 138 const OSSymbol *                gIOPublishNotification;
 139 const OSSymbol *                gIOFirstPublishNotification;
 140 const OSSymbol *                gIOMatchedNotification;
 141 const OSSymbol *                gIOFirstMatchNotification;
 142 const OSSymbol *                gIOTerminatedNotification;
 143
 144 const OSSymbol *                gIOGeneralInterest;
 145 const OSSymbol *                gIOBusyInterest;
 146 const OSSymbol *                gIOAppPowerStateInterest;
 147 const OSSymbol *                gIOPriorityPowerStateInterest;
 148 const OSSymbol *                gIOConsoleSecurityInterest;
 149
 150 static OSDictionary *           gNotifications;
 151 static IORecursiveLock *        gNotificationLock;
 152
 153 static IOService *              gIOResources;
 154 static IOService *              gIOServiceRoot;
 155
 156 static OSOrderedSet *           gJobs;
 157 static semaphore_port_t         gJobsSemaphore;
 158 static IOLock *                 gJobsLock;
 159 static int                      gOutstandingJobs;
 160 static int                      gNumConfigThreads;
 161 static int                      gNumWaitingThreads;
 162 static IOLock *                 gIOServiceBusyLock;
 163
 164 static thread_t                 gIOTerminateThread;
 165 static UInt32                   gIOTerminateWork;
 166 static OSArray *                gIOTerminatePhase2List;
 167 static OSArray *                gIOStopList;
 168 static OSArray *                gIOStopProviderList;
 169 static OSArray *                gIOFinalizeList;
 170
 171 static SInt32                   gIOConsoleUsersSeed;
 172 static OSData *                 gIOConsoleUsersSeedValue;
 173
 174 extern const OSSymbol *         gIODTPHandleKey;
 175
 176 const OSSymbol *                gIOPlatformSleepActionKey;
 177 const OSSymbol *                gIOPlatformWakeActionKey;
 178 const OSSymbol *                gIOPlatformQuiesceActionKey;
 179 const OSSymbol *                gIOPlatformActiveActionKey;
 180 const OSSymbol *                gIOPlatformHaltRestartActionKey;
 181
 182 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 183
 184 static IOLock *                 gIOConsoleUsersLock;
 185 static thread_call_t            gIOConsoleLockCallout;
 186
 187 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 188
 189 #define LOCKREADNOTIFY()        \
 190     IORecursiveLockLock( gNotificationLock )
 191 #define LOCKWRITENOTIFY()       \
 192     IORecursiveLockLock( gNotificationLock )
 193 #define LOCKWRITE2READNOTIFY()
 194 #define UNLOCKNOTIFY()          \
 195     IORecursiveLockUnlock( gNotificationLock )
 196 #define SLEEPNOTIFY(event) \
 197     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 198 #define SLEEPNOTIFYTO(event, deadline) \
 199     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 200 #define WAKEUPNOTIFY(event) \
 201         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 202
 203 #define randomDelay()   \
 204         int del = read_processor_clock();                               \
 205         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 206         IOSleep( del );
 207
 208 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 209
 210 #define queue_element(entry, element, type, field) do { \
 211         vm_address_t __ele = (vm_address_t) (entry);    \
 212         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 213         (element) = (type) __ele;                       \
 214     } while(0)
 215
 216 #define iterqueue(que, elt)                             \
 217         for (queue_entry_t elt = queue_first(que);      \
 218              !queue_end(que, elt);                      \
 219              elt = queue_next(elt))
 220
 221 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 222
 223 struct IOInterruptAccountingReporter {
 224     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 225     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 226 };
 227
 228 struct ArbitrationLockQueueElement {
 229     queue_chain_t link;
 230     IOThread      thread;
 231     IOService *   service;
 232     unsigned      count;
 233     bool          required;
 234     bool          aborted;
 235 };
 236
 237 static queue_head_t gArbitrationLockQueueActive;
 238 static queue_head_t gArbitrationLockQueueWaiting;
 239 static queue_head_t gArbitrationLockQueueFree;
 240 static IOLock *     gArbitrationLockQueueLock;
 241
 242 bool IOService::isInactive( void ) const
 243     { return( 0 != (kIOServiceInactiveState & getState())); }
 244
 245 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 246
 247 #if defined(__i386__) || defined(__x86_64__)
 248
 249 // Only used by the intel implementation of
 250 //     IOService::requireMaxBusStall(UInt32 ns)
 251 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 252 struct CpuDelayEntry
 253 {
 254     IOService * fService;
 255     UInt32      fMaxDelay;
 256     UInt32      fDelayType;
 257 };
 258
 259 enum {
 260     kCpuDelayBusStall, kCpuDelayInterrupt,
 261     kCpuNumDelayTypes
 262 };
 263
 264 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 265 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 266 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 267 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 268 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 269 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 270
 271 static void
 272 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 273 static IOReturn
 274 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 275
 276 #endif /* defined(__i386__) || defined(__x86_64__) */
 277
 278 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 279
 280 void IOService::initialize( void )
 281 {
 282     kern_return_t       err;
 283
 284     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 285     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 286
 287     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 288     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 289     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 290     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 291     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 292     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 293     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 294
 295     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 296     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 297                                         kIODefaultMatchCategoryKey );
 298     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 299                                         kIOMatchedServiceCountKey );
 300
 301     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 302
 303     gIOResourcesKey     = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 304     gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 305
 306     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 307     gIOInterruptControllersKey
 308         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 309     gIOInterruptSpecifiersKey
 310         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 311
 312     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 313
 314     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 315
 316     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 317
 318     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 319     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 320     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 321     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 322     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 323
 324     gNotifications              = OSDictionary::withCapacity( 1 );
 325     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 326                                                  kIOPublishNotification );
 327     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 328                                                  kIOFirstPublishNotification );
 329     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 330                                                  kIOMatchedNotification );
 331     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 332                                                  kIOFirstMatchNotification );
 333     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 334                                                  kIOTerminatedNotification );
 335     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 336
 337     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 338     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 339     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 340     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 341
 342     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 343     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 344     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 345     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 346     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 347
 348     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 349
 350     gIOPlatformSleepActionKey       = OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey);
 351     gIOPlatformWakeActionKey        = OSSymbol::withCStringNoCopy(kIOPlatformWakeActionKey);
 352     gIOPlatformQuiesceActionKey     = OSSymbol::withCStringNoCopy(kIOPlatformQuiesceActionKey);
 353     gIOPlatformActiveActionKey      = OSSymbol::withCStringNoCopy(kIOPlatformActiveActionKey);
 354     gIOPlatformHaltRestartActionKey = OSSymbol::withCStringNoCopy(kIOPlatformHaltRestartActionKey);
 355
 356     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 357 #if defined(__i386__) || defined(__x86_64__)
 358     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 359     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 360     uint32_t  idx;
 361     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 362     {
 363         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 364         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 365         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 366     }
 367 #endif
 368     gNotificationLock           = IORecursiveLockAlloc();
 369
 370     assert( gIOServicePlane && gIODeviceMemoryKey
 371         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 372         && gIOResourcesKey && gNotifications && gNotificationLock
 373         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 374         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 375         && gIOPublishNotification && gIOMatchedNotification
 376         && gIOTerminatedNotification && gIOServiceKey
 377         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 378     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 379         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 380
 381     gJobsLock   = IOLockAlloc();
 382     gJobs       = OSOrderedSet::withCapacity( 10 );
 383
 384     gIOServiceBusyLock = IOLockAlloc();
 385
 386     gIOConsoleUsersLock = IOLockAlloc();
 387
 388     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 389
 390     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 391
 392     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 393
 394     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 395                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 396
 397     gIOResources = IOResources::resources();
 398     assert( gIOResources );
 399
 400     gArbitrationLockQueueLock = IOLockAlloc();
 401     queue_init(&gArbitrationLockQueueActive);
 402     queue_init(&gArbitrationLockQueueWaiting);
 403     queue_init(&gArbitrationLockQueueFree);
 404
 405     assert( gArbitrationLockQueueLock );
 406
 407     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 408     gIOStopList            = OSArray::withCapacity( 16 );
 409     gIOStopProviderList    = OSArray::withCapacity( 16 );
 410     gIOFinalizeList        = OSArray::withCapacity( 16 );
 411     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 412 }
 413
 414 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 415
 416 #if IOMATCHDEBUG
 417 static UInt64 getDebugFlags( OSDictionary * props )
 418 {
 419     OSNumber *  debugProp;
 420     UInt64      debugFlags;
 421
 422     debugProp = OSDynamicCast( OSNumber,
 423                 props->getObject( gIOKitDebugKey ));
 424     if( debugProp)
 425         debugFlags = debugProp->unsigned64BitValue();
 426     else
 427         debugFlags = gIOKitDebug;
 428
 429     return( debugFlags );
 430 }
 431 #endif
 432
 433 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 434
 435 // Probe a matched service and return an instance to be started.
 436 // The default score is from the property table, & may be altered
 437 // during probe to change the start order.
 438
 439 IOService * IOService::probe(   IOService * provider,
 440                                 SInt32    * score )
 441 {
 442     return( this );
 443 }
 444
 445 bool IOService::start( IOService * provider )
 446 {
 447     return( true );
 448 }
 449
 450 void IOService::stop( IOService * provider )
 451 {
 452 }
 453
 454 bool IOService::init( OSDictionary * dictionary )
 455 {
 456     bool ret = false;
 457
 458     ret = super::init(dictionary);
 459
 460     if (!ret)
 461         goto done;
 462
 463     reserved = IONew(ExpansionData, 1);
 464
 465     if (!reserved) {
 466         ret = false;
 467         goto done;
 468     }
 469
 470     bzero(reserved, sizeof(*reserved));
 471
 472     /*
 473      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 474      * lock based on the presence of specifiers ran into issues as some
 475      * platforms set up the specifiers after IOService initialization.
 476      *
 477      * We may be able to get away with a global lock, as this should only be
 478      * contended by IOReporting clients and driver start/stop (unless a
 479      * driver wants to remove/add handlers in the course of normal operation,
 480      * which should be unlikely).
 481      */
 482     reserved->interruptStatisticsLock = IOLockAlloc();
 483
 484     if (!reserved->interruptStatisticsLock) {
 485         ret = false;
 486         goto done;
 487     }
 488
 489 done:
 490     return ret;
 491 }
 492
 493 bool IOService::init( IORegistryEntry * from,
 494                       const IORegistryPlane * inPlane )
 495 {
 496     bool ret = false;
 497
 498     ret = super::init(from, inPlane);
 499
 500     if (!ret)
 501         goto done;
 502
 503     reserved = IONew(ExpansionData, 1);
 504
 505     if (!reserved) {
 506         ret = false;
 507         goto done;
 508     }
 509
 510     bzero(reserved, sizeof(*reserved));
 511
 512     /*
 513      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 514      * lock based on the presence of specifiers ran into issues as some
 515      * platforms set up the specifiers after IOService initialization.
 516      *
 517      * We may be able to get away with a global lock, as this should only be
 518      * contended by IOReporting clients and driver start/stop (unless a
 519      * driver wants to remove/add handlers in the course of normal operation,
 520      * which should be unlikely).
 521      */
 522     reserved->interruptStatisticsLock = IOLockAlloc();
 523
 524     if (!reserved->interruptStatisticsLock) {
 525         ret = false;
 526         goto done;
 527     }
 528
 529 done:
 530     return ret;
 531 }
 532
 533 void IOService::free( void )
 534 {
 535     int i = 0;
 536     requireMaxBusStall(0);
 537     requireMaxInterruptDelay(0);
 538     if( getPropertyTable())
 539         unregisterAllInterest();
 540     PMfree();
 541
 542     if (reserved) {
 543         if (reserved->interruptStatisticsArray) {
 544             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 545                 if (reserved->interruptStatisticsArray[i].reporter)
 546                     reserved->interruptStatisticsArray[i].reporter->release();
 547             }
 548
 549             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 550         }
 551
 552         if (reserved->interruptStatisticsLock)
 553             IOLockFree(reserved->interruptStatisticsLock);
 554         IODelete(reserved, ExpansionData, 1);
 555     }
 556
 557     super::free();
 558 }
 559
 560 /*
 561  * Attach in service plane
 562  */
 563 bool IOService::attach( IOService * provider )
 564 {
 565     bool        ok;
 566
 567     if( provider) {
 568
 569         if( gIOKitDebug & kIOLogAttach)
 570             LOG( "%s::attach(%s)\n", getName(),
 571                     provider->getName());
 572
 573         provider->lockForArbitration();
 574         if( provider->__state[0] & kIOServiceInactiveState)
 575             ok = false;
 576         else
 577             ok = attachToParent( provider, gIOServicePlane);
 578         provider->unlockForArbitration();
 579
 580     } else {
 581         gIOServiceRoot = this;
 582         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 583     }
 584
 585     if (ok && !__provider) (void) getProvider();
 586
 587     return( ok );
 588 }
 589
 590 IOService * IOService::getServiceRoot( void )
 591 {
 592     return( gIOServiceRoot );
 593 }
 594
 595 void IOService::detach( IOService * provider )
 596 {
 597     IOService * newProvider = 0;
 598     SInt32      busy;
 599     bool        adjParent;
 600
 601     if( gIOKitDebug & kIOLogAttach)
 602         LOG("%s::detach(%s)\n", getName(), provider->getName());
 603
 604     lockForArbitration();
 605
 606     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 607                && (provider == getProvider()));
 608
 609     detachFromParent( provider, gIOServicePlane );
 610
 611     if( busy) {
 612         newProvider = getProvider();
 613         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 614             _adjustBusy( -busy );
 615     }
 616
 617     if (kIOServiceInactiveState & __state[0]) {
 618         getMetaClass()->removeInstance(this);
 619     }
 620
 621     unlockForArbitration();
 622
 623     if( newProvider) {
 624         newProvider->lockForArbitration();
 625         newProvider->_adjustBusy(1);
 626         newProvider->unlockForArbitration();
 627     }
 628
 629     // check for last client detach from a terminated service
 630     if( provider->lockForArbitration( true ))
 631     {
 632         if (kIOServiceStartState & __state[1])
 633         {
 634             provider->scheduleTerminatePhase2();
 635         }
 636         if( adjParent) provider->_adjustBusy( -1 );
 637         if( (provider->__state[1] & kIOServiceTermPhase3State)
 638          && (0 == provider->getClient())) {
 639             provider->scheduleFinalize();
 640         }
 641         provider->unlockForArbitration();
 642     }
 643 }
 644
 645 /*
 646  * Register instance - publish it for matching
 647  */
 648
 649 void IOService::registerService( IOOptionBits options )
 650 {
 651     char *              pathBuf;
 652     const char *        path;
 653     char *              skip;
 654     int                 len;
 655     enum { kMaxPathLen  = 256 };
 656     enum { kMaxChars    = 63 };
 657
 658     IORegistryEntry * parent = this;
 659     IORegistryEntry * root = getRegistryRoot();
 660     while( parent && (parent != root))
 661         parent = parent->getParentEntry( gIOServicePlane);
 662
 663     if( parent != root) {
 664         IOLog("%s: not registry member at registerService()\n", getName());
 665         return;
 666     }
 667
 668     // Allow the Platform Expert to adjust this node.
 669     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 670         return;
 671
 672     if( (this != gIOResources)
 673      && (kIOLogRegister & gIOKitDebug)) {
 674
 675         pathBuf = (char *) IOMalloc( kMaxPathLen );
 676
 677         IOLog( "Registering: " );
 678
 679         len = kMaxPathLen;
 680         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 681
 682             path = pathBuf;
 683             if( len > kMaxChars) {
 684                 IOLog("..");
 685                 len -= kMaxChars;
 686                 path += len;
 687                 if( (skip = strchr( path, '/')))
 688                     path = skip;
 689             }
 690         } else
 691             path = getName();
 692
 693         IOLog( "%s\n", path );
 694
 695         if( pathBuf)
 696             IOFree( pathBuf, kMaxPathLen );
 697     }
 698
 699     startMatching( options );
 700 }
 701
 702 void IOService::startMatching( IOOptionBits options )
 703 {
 704     IOService * provider;
 705     UInt32      prevBusy = 0;
 706     bool        needConfig;
 707     bool        needWake = false;
 708     bool        ok;
 709     bool        sync;
 710     bool        waitAgain;
 711
 712     lockForArbitration();
 713
 714     sync = (options & kIOServiceSynchronous)
 715         || ((provider = getProvider())
 716                 && (provider->__state[1] & kIOServiceSynchronousState));
 717
 718         if ( options & kIOServiceAsynchronous )
 719                 sync = false;
 720
 721     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState)))
 722                && (0 == (__state[0] & kIOServiceInactiveState));
 723
 724     __state[1] |= kIOServiceNeedConfigState;
 725
 726 //    __state[0] &= ~kIOServiceInactiveState;
 727
 728 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 729 //                      OSKernelStackRemaining(), getName());
 730
 731     if( needConfig) {
 732         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 733     }
 734
 735     if( sync)
 736         __state[1] |= kIOServiceSynchronousState;
 737     else
 738         __state[1] &= ~kIOServiceSynchronousState;
 739
 740     if( needConfig) prevBusy = _adjustBusy( 1 );
 741
 742     unlockForArbitration();
 743
 744     if( needConfig) {
 745
 746         if( needWake) {
 747             IOLockLock( gIOServiceBusyLock );
 748             thread_wakeup( (event_t) this/*&__state[1]*/ );
 749             IOLockUnlock( gIOServiceBusyLock );
 750
 751         } else if( !sync || (kIOServiceAsynchronous & options)) {
 752
 753             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 754
 755         } else do {
 756
 757             if( (__state[1] & kIOServiceNeedConfigState))
 758                 doServiceMatch( options );
 759
 760             lockForArbitration();
 761             IOLockLock( gIOServiceBusyLock );
 762
 763             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 764                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 765
 766             if( waitAgain)
 767                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 768             else
 769                 __state[1] &= ~kIOServiceSyncPubState;
 770
 771             unlockForArbitration();
 772
 773             if( waitAgain)
 774                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 775
 776             IOLockUnlock( gIOServiceBusyLock );
 777             if( waitAgain)
 778                 thread_block(THREAD_CONTINUE_NULL);
 779
 780         } while( waitAgain );
 781     }
 782 }
 783
 784 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 785 {
 786     OSDictionary *      table;
 787     OSSet *             set;
 788     OSSet *             allSet = 0;
 789     IOService *         service;
 790 #if IOMATCHDEBUG
 791     SInt32              count = 0;
 792 #endif
 793
 794     newTables->retain();
 795
 796     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 797
 798         LOCKWRITENOTIFY();
 799         set = (OSSet *) copyExistingServices( table,
 800                                                 kIOServiceRegisteredState,
 801                                                 kIOServiceExistingSet);
 802         UNLOCKNOTIFY();
 803         if( set) {
 804
 805 #if IOMATCHDEBUG
 806             count += set->getCount();
 807 #endif
 808             if (allSet) {
 809                 allSet->merge((const OSSet *) set);
 810                 set->release();
 811             }
 812             else
 813                 allSet = set;
 814         }
 815
 816 #if IOMATCHDEBUG
 817         if( getDebugFlags( table ) & kIOLogMatch)
 818             LOG("Matching service count = %ld\n", (long)count);
 819 #endif
 820         newTables->removeObject(table);
 821     }
 822
 823     if (allSet) {
 824         while( (service = (IOService *) allSet->getAnyObject())) {
 825             service->startMatching(kIOServiceAsynchronous);
 826             allSet->removeObject(service);
 827         }
 828         allSet->release();
 829     }
 830
 831     newTables->release();
 832
 833     return( kIOReturnSuccess );
 834 }
 835
 836  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 837                                                 IOOptionBits options )
 838 {
 839     _IOServiceJob *     job;
 840
 841     job = new _IOServiceJob;
 842     if( job && !job->init()) {
 843         job->release();
 844         job = 0;
 845     }
 846
 847     if( job) {
 848         job->type       = type;
 849         job->nub        = nub;
 850         job->options    = options;
 851         nub->retain();                  // thread will release()
 852         pingConfig( job );
 853     }
 854
 855     return( job );
 856 }
 857
 858 /*
 859  * Called on a registered service to see if it matches
 860  * a property table.
 861  */
 862
 863 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 864 {
 865     return( matchPropertyTable(table) );
 866 }
 867
 868 bool IOService::matchPropertyTable( OSDictionary * table )
 869 {
 870     return( true );
 871 }
 872
 873 /*
 874  * Called on a matched service to allocate resources
 875  * before first driver is attached.
 876  */
 877
 878 IOReturn IOService::getResources( void )
 879 {
 880     return( kIOReturnSuccess);
 881 }
 882
 883 /*
 884  * Client/provider accessors
 885  */
 886
 887 IOService * IOService::getProvider( void ) const
 888 {
 889     IOService * self = (IOService *) this;
 890     IOService * parent;
 891     SInt32      generation;
 892
 893     generation = getGenerationCount();
 894     if( __providerGeneration == generation)
 895         return( __provider );
 896
 897     parent = (IOService *) getParentEntry( gIOServicePlane);
 898     if( parent == IORegistryEntry::getRegistryRoot())
 899         /* root is not an IOService */
 900         parent = 0;
 901
 902     self->__provider = parent;
 903     OSMemoryBarrier();
 904     // save the count from before call to getParentEntry()
 905     self->__providerGeneration = generation;
 906
 907     return( parent );
 908 }
 909
 910 IOWorkLoop * IOService::getWorkLoop() const
 911 {
 912     IOService *provider = getProvider();
 913
 914     if (provider)
 915         return provider->getWorkLoop();
 916     else
 917         return 0;
 918 }
 919
 920 OSIterator * IOService::getProviderIterator( void ) const
 921 {
 922     return( getParentIterator( gIOServicePlane));
 923 }
 924
 925 IOService * IOService::getClient( void ) const
 926 {
 927     return( (IOService *) getChildEntry( gIOServicePlane));
 928 }
 929
 930 OSIterator * IOService::getClientIterator( void ) const
 931 {
 932     return( getChildIterator( gIOServicePlane));
 933 }
 934
 935 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
 936                                                 const IOService * client,
 937                                                 const IOService * provider )
 938 {
 939     _IOOpenServiceIterator * inst;
 940
 941     if( !_iter)
 942         return( 0 );
 943
 944     inst = new _IOOpenServiceIterator;
 945
 946     if( inst && !inst->init()) {
 947         inst->release();
 948         inst = 0;
 949     }
 950     if( inst) {
 951         inst->iter = _iter;
 952         inst->client = client;
 953         inst->provider = provider;
 954     }
 955
 956     return( inst );
 957 }
 958
 959 void _IOOpenServiceIterator::free()
 960 {
 961     iter->release();
 962     if( last)
 963         last->unlockForArbitration();
 964     OSIterator::free();
 965 }
 966
 967 OSObject * _IOOpenServiceIterator::getNextObject()
 968 {
 969     IOService * next;
 970
 971     if( last)
 972         last->unlockForArbitration();
 973
 974     while( (next = (IOService *) iter->getNextObject())) {
 975
 976         next->lockForArbitration();
 977         if( (client && (next->isOpen( client )))
 978          || (provider && (provider->isOpen( next ))) )
 979             break;
 980         next->unlockForArbitration();
 981     }
 982
 983     last = next;
 984
 985     return( next );
 986 }
 987
 988 bool _IOOpenServiceIterator::isValid()
 989 {
 990     return( iter->isValid() );
 991 }
 992
 993 void _IOOpenServiceIterator::reset()
 994 {
 995     if( last) {
 996         last->unlockForArbitration();
 997         last = 0;
 998     }
 999     iter->reset();
1000 }
1001
1002 OSIterator * IOService::getOpenProviderIterator( void ) const
1003 {
1004     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1005 }
1006
1007 OSIterator * IOService::getOpenClientIterator( void ) const
1008 {
1009     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1010 }
1011
1012
1013 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1014                                           bool waitForFunction,
1015                                           void *param1, void *param2,
1016                                           void *param3, void *param4 )
1017 {
1018   IOReturn  result = kIOReturnUnsupported;
1019   IOService *provider;
1020
1021   if (gIOPlatformFunctionHandlerSet == functionName)
1022   {
1023 #if defined(__i386__) || defined(__x86_64__)
1024     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1025     IOService *      target              = (IOService *) param2;
1026     bool             enable              = (param3 != 0);
1027
1028     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1029         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1030     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1031         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1032 #endif /* defined(__i386__) || defined(__x86_64__) */
1033   }
1034
1035   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1036     result = provider->callPlatformFunction(functionName, waitForFunction,
1037                                             param1, param2, param3, param4);
1038   }
1039
1040   return result;
1041 }
1042
1043 IOReturn IOService::callPlatformFunction( const char * functionName,
1044                                           bool waitForFunction,
1045                                           void *param1, void *param2,
1046                                           void *param3, void *param4 )
1047 {
1048   IOReturn result = kIOReturnNoMemory;
1049   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1050
1051   if (functionSymbol != 0) {
1052     result = callPlatformFunction(functionSymbol, waitForFunction,
1053                                   param1, param2, param3, param4);
1054     functionSymbol->release();
1055   }
1056
1057   return result;
1058 }
1059
1060
1061 /*
1062  * Accessors for global services
1063  */
1064
1065 IOPlatformExpert * IOService::getPlatform( void )
1066 {
1067     return( gIOPlatform);
1068 }
1069
1070 class IOPMrootDomain * IOService::getPMRootDomain( void )
1071 {
1072     return( gIOPMRootDomain);
1073 }
1074
1075 IOService * IOService::getResourceService( void )
1076 {
1077     return( gIOResources );
1078 }
1079
1080 void IOService::setPlatform( IOPlatformExpert * platform)
1081 {
1082     gIOPlatform = platform;
1083     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1084
1085 #if defined(__i386__) || defined(__x86_64__)
1086
1087     static const char * keys[kCpuNumDelayTypes] = {
1088         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1089     const OSObject * objs[2];
1090     OSArray * array;
1091     uint32_t  idx;
1092
1093     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1094     {
1095         objs[0] = sCPULatencySet[idx];
1096         objs[1] = sCPULatencyHolder[idx];
1097         array   = OSArray::withObjects(objs, 2);
1098         if (!array) break;
1099         platform->setProperty(keys[idx], array);
1100         array->release();
1101     }
1102 #endif /* defined(__i386__) || defined(__x86_64__) */
1103 }
1104
1105 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1106 {
1107     gIOPMRootDomain = rootDomain;
1108     publishResource("IOKit");
1109 }
1110
1111 /*
1112  * Stacking change
1113  */
1114
1115 bool IOService::lockForArbitration( bool isSuccessRequired )
1116 {
1117     bool                          found;
1118     bool                          success;
1119     ArbitrationLockQueueElement * element;
1120     ArbitrationLockQueueElement * active;
1121     ArbitrationLockQueueElement * waiting;
1122
1123     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1124
1125     // lock global access
1126     IOTakeLock( gArbitrationLockQueueLock );
1127
1128     // obtain an unused queue element
1129     if( !queue_empty( &gArbitrationLockQueueFree )) {
1130         queue_remove_first( &gArbitrationLockQueueFree,
1131                             element,
1132                             ArbitrationLockQueueElement *,
1133                             link );
1134     } else {
1135         element = IONew( ArbitrationLockQueueElement, 1 );
1136         assert( element );
1137     }
1138
1139     // prepare the queue element
1140     element->thread   = IOThreadSelf();
1141     element->service  = this;
1142     element->count    = 1;
1143     element->required = isSuccessRequired;
1144     element->aborted  = false;
1145
1146     // determine whether this object is already locked (ie. on active queue)
1147     found = false;
1148     queue_iterate( &gArbitrationLockQueueActive,
1149                     active,
1150                     ArbitrationLockQueueElement *,
1151                     link )
1152     {
1153         if( active->service == element->service ) {
1154             found = true;
1155             break;
1156         }
1157     }
1158
1159     if( found ) { // this object is already locked
1160
1161         // determine whether it is the same or a different thread trying to lock
1162         if( active->thread != element->thread ) { // it is a different thread
1163
1164             ArbitrationLockQueueElement * victim = 0;
1165
1166             // before placing this new thread on the waiting queue, we look for
1167             // a deadlock cycle...
1168
1169             while( 1 ) {
1170                 // determine whether the active thread holding the object we
1171                 // want is waiting for another object to be unlocked
1172                 found = false;
1173                 queue_iterate( &gArbitrationLockQueueWaiting,
1174                                waiting,
1175                                ArbitrationLockQueueElement *,
1176                                link )
1177                 {
1178                     if( waiting->thread == active->thread ) {
1179                         assert( false == waiting->aborted );
1180                         found = true;
1181                         break;
1182                     }
1183                 }
1184
1185                 if( found ) { // yes, active thread waiting for another object
1186
1187                     // this may be a candidate for rejection if the required
1188                     // flag is not set, should we detect a deadlock later on
1189                     if( false == waiting->required )
1190                         victim = waiting;
1191
1192                     // find the thread that is holding this other object, that
1193                     // is blocking the active thread from proceeding (fun :-)
1194                     found = false;
1195                     queue_iterate( &gArbitrationLockQueueActive,
1196                                    active,      // (reuse active queue element)
1197                                    ArbitrationLockQueueElement *,
1198                                    link )
1199                     {
1200                         if( active->service == waiting->service ) {
1201                             found = true;
1202                             break;
1203                         }
1204                     }
1205
1206                     // someone must be holding it or it wouldn't be waiting
1207                     assert( found );
1208
1209                     if( active->thread == element->thread ) {
1210
1211                         // doh, it's waiting for the thread that originated
1212                         // this whole lock (ie. current thread) -> deadlock
1213                         if( false == element->required ) { // willing to fail?
1214
1215                             // the originating thread doesn't have the required
1216                             // flag, so it can fail
1217                             success = false; // (fail originating lock request)
1218                             break; // (out of while)
1219
1220                         } else { // originating thread is not willing to fail
1221
1222                             // see if we came across a waiting thread that did
1223                             // not have the 'required' flag set: we'll fail it
1224                             if( victim ) {
1225
1226                                 // we do have a willing victim, fail it's lock
1227                                 victim->aborted = true;
1228
1229                                 // take the victim off the waiting queue
1230                                 queue_remove( &gArbitrationLockQueueWaiting,
1231                                               victim,
1232                                               ArbitrationLockQueueElement *,
1233                                               link );
1234
1235                                 // wake the victim
1236                                 IOLockWakeup( gArbitrationLockQueueLock,
1237                                               victim,
1238                                               /* one thread */ true );
1239
1240                                 // allow this thread to proceed (ie. wait)
1241                                 success = true; // (put request on wait queue)
1242                                 break; // (out of while)
1243                             } else {
1244
1245                                 // all the waiting threads we came across in
1246                                 // finding this loop had the 'required' flag
1247                                 // set, so we've got a deadlock we can't avoid
1248                                 panic("I/O Kit: Unrecoverable deadlock.");
1249                             }
1250                         }
1251                     } else {
1252                         // repeat while loop, redefining active thread to be the
1253                         // thread holding "this other object" (see above), and
1254                         // looking for threads waiting on it; note the active
1255                         // variable points to "this other object" already... so
1256                         // there nothing to do in this else clause.
1257                     }
1258                 } else { // no, active thread is not waiting for another object
1259
1260                     success = true; // (put request on wait queue)
1261                     break; // (out of while)
1262                 }
1263             } // while forever
1264
1265             if( success ) { // put the request on the waiting queue?
1266                 kern_return_t wait_result;
1267
1268                 // place this thread on the waiting queue and put it to sleep;
1269                 // we place it at the tail of the queue...
1270                 queue_enter( &gArbitrationLockQueueWaiting,
1271                              element,
1272                              ArbitrationLockQueueElement *,
1273                              link );
1274
1275                 // declare that this thread will wait for a given event
1276 restart_sleep:  wait_result = assert_wait( element,
1277                                            element->required ? THREAD_UNINT
1278                                            : THREAD_INTERRUPTIBLE );
1279
1280                 // unlock global access
1281                 IOUnlock( gArbitrationLockQueueLock );
1282
1283                 // put thread to sleep, waiting for our event to fire...
1284                 if (wait_result == THREAD_WAITING)
1285                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1286
1287
1288                 // ...and we've been woken up; we might be in one of two states:
1289                 // (a) we've been aborted and our queue element is not on
1290                 //     any of the three queues, but is floating around
1291                 // (b) we're allowed to proceed with the lock and we have
1292                 //     already been moved from the waiting queue to the
1293                 //     active queue.
1294                 // ...plus a 3rd state, should the thread have been interrupted:
1295                 // (c) we're still on the waiting queue
1296
1297                 // determine whether we were interrupted out of our sleep
1298                 if( THREAD_INTERRUPTED == wait_result ) {
1299
1300                     // re-lock global access
1301                     IOTakeLock( gArbitrationLockQueueLock );
1302
1303                     // determine whether we're still on the waiting queue
1304                     found = false;
1305                     queue_iterate( &gArbitrationLockQueueWaiting,
1306                                    waiting,     // (reuse waiting queue element)
1307                                    ArbitrationLockQueueElement *,
1308                                    link )
1309                     {
1310                         if( waiting == element ) {
1311                             found = true;
1312                             break;
1313                         }
1314                     }
1315
1316                     if( found ) { // yes, we're still on the waiting queue
1317
1318                         // determine whether we're willing to fail
1319                         if( false == element->required ) {
1320
1321                             // mark us as aborted
1322                             element->aborted = true;
1323
1324                             // take us off the waiting queue
1325                             queue_remove( &gArbitrationLockQueueWaiting,
1326                                           element,
1327                                           ArbitrationLockQueueElement *,
1328                                           link );
1329                         } else { // we are not willing to fail
1330
1331                             // ignore interruption, go back to sleep
1332                             goto restart_sleep;
1333                         }
1334                     }
1335
1336                     // unlock global access
1337                     IOUnlock( gArbitrationLockQueueLock );
1338
1339                     // proceed as though this were a normal wake up
1340                     wait_result = THREAD_AWAKENED;
1341                 }
1342
1343                 assert( THREAD_AWAKENED == wait_result );
1344
1345                 // determine whether we've been aborted while we were asleep
1346                 if( element->aborted ) {
1347                     assert( false == element->required );
1348
1349                     // re-lock global access
1350                     IOTakeLock( gArbitrationLockQueueLock );
1351
1352                     action = kPutOnFreeQueue;
1353                     success = false;
1354                 } else { // we weren't aborted, so we must be ready to go :-)
1355
1356                     // we've already been moved from waiting to active queue
1357                     return true;
1358                 }
1359
1360             } else { // the lock request is to be failed
1361
1362                 // return unused queue element to queue
1363                 action = kPutOnFreeQueue;
1364             }
1365         } else { // it is the same thread, recursive access is allowed
1366
1367             // add one level of recursion
1368             active->count++;
1369
1370             // return unused queue element to queue
1371             action = kPutOnFreeQueue;
1372             success = true;
1373         }
1374     } else { // this object is not already locked, so let this thread through
1375         action = kPutOnActiveQueue;
1376         success = true;
1377     }
1378
1379     // put the new element on a queue
1380     if( kPutOnActiveQueue == action ) {
1381         queue_enter( &gArbitrationLockQueueActive,
1382                      element,
1383                      ArbitrationLockQueueElement *,
1384                      link );
1385     } else if( kPutOnFreeQueue == action ) {
1386         queue_enter( &gArbitrationLockQueueFree,
1387                      element,
1388                      ArbitrationLockQueueElement *,
1389                      link );
1390     } else {
1391         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1392     }
1393
1394     // unlock global access
1395     IOUnlock( gArbitrationLockQueueLock );
1396
1397     return( success );
1398 }
1399
1400 void IOService::unlockForArbitration( void )
1401 {
1402     bool                          found;
1403     ArbitrationLockQueueElement * element;
1404
1405     // lock global access
1406     IOTakeLock( gArbitrationLockQueueLock );
1407
1408     // find the lock element for this object (ie. on active queue)
1409     found = false;
1410     queue_iterate( &gArbitrationLockQueueActive,
1411                     element,
1412                     ArbitrationLockQueueElement *,
1413                     link )
1414     {
1415         if( element->service == this ) {
1416             found = true;
1417             break;
1418         }
1419     }
1420
1421     assert( found );
1422
1423     // determine whether the lock has been taken recursively
1424     if( element->count > 1 ) {
1425         // undo one level of recursion
1426         element->count--;
1427
1428     } else {
1429
1430         // remove it from the active queue
1431         queue_remove( &gArbitrationLockQueueActive,
1432                       element,
1433                       ArbitrationLockQueueElement *,
1434                       link );
1435
1436         // put it on the free queue
1437         queue_enter( &gArbitrationLockQueueFree,
1438                      element,
1439                      ArbitrationLockQueueElement *,
1440                      link );
1441
1442         // determine whether a thread is waiting for object (head to tail scan)
1443         found = false;
1444         queue_iterate( &gArbitrationLockQueueWaiting,
1445                        element,
1446                        ArbitrationLockQueueElement *,
1447                        link )
1448         {
1449             if( element->service == this ) {
1450                 found = true;
1451                 break;
1452             }
1453         }
1454
1455         if ( found ) { // we found an interested thread on waiting queue
1456
1457             // remove it from the waiting queue
1458             queue_remove( &gArbitrationLockQueueWaiting,
1459                           element,
1460                           ArbitrationLockQueueElement *,
1461                           link );
1462
1463             // put it on the active queue
1464             queue_enter( &gArbitrationLockQueueActive,
1465                          element,
1466                          ArbitrationLockQueueElement *,
1467                          link );
1468
1469             // wake the waiting thread
1470             IOLockWakeup( gArbitrationLockQueueLock,
1471                           element,
1472                           /* one thread */ true );
1473         }
1474     }
1475
1476     // unlock global access
1477     IOUnlock( gArbitrationLockQueueLock );
1478 }
1479
1480 void IOService::applyToProviders( IOServiceApplierFunction applier,
1481                                   void * context )
1482 {
1483     applyToParents( (IORegistryEntryApplierFunction) applier,
1484                     context, gIOServicePlane );
1485 }
1486
1487 void IOService::applyToClients( IOServiceApplierFunction applier,
1488                                 void * context )
1489 {
1490     applyToChildren( (IORegistryEntryApplierFunction) applier,
1491                      context, gIOServicePlane );
1492 }
1493
1494
1495 /*
1496  * Client messages
1497  */
1498
1499
1500 // send a message to a client or interested party of this service
1501 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1502                                    void * argument, vm_size_t argSize )
1503 {
1504     IOReturn                            ret;
1505     IOService *                         service;
1506     _IOServiceInterestNotifier *        notify;
1507
1508     if( (service = OSDynamicCast( IOService, client)))
1509         ret = service->message( type, this, argument );
1510
1511     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1512
1513         _IOServiceNotifierInvocation invocation;
1514         bool                     willNotify;
1515
1516         invocation.thread = current_thread();
1517
1518         LOCKWRITENOTIFY();
1519         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1520
1521         if( willNotify) {
1522             queue_enter( &notify->handlerInvocations, &invocation,
1523                             _IOServiceNotifierInvocation *, link );
1524         }
1525         UNLOCKNOTIFY();
1526
1527         if( willNotify) {
1528
1529             ret = (*notify->handler)( notify->target, notify->ref,
1530                                           type, this, argument, argSize );
1531
1532             LOCKWRITENOTIFY();
1533             queue_remove( &notify->handlerInvocations, &invocation,
1534                             _IOServiceNotifierInvocation *, link );
1535             if( kIOServiceNotifyWaiter & notify->state) {
1536                 notify->state &= ~kIOServiceNotifyWaiter;
1537                 WAKEUPNOTIFY( notify );
1538             }
1539             UNLOCKNOTIFY();
1540
1541         } else
1542             ret = kIOReturnSuccess;
1543
1544     } else
1545         ret = kIOReturnBadArgument;
1546
1547     return( ret );
1548 }
1549
1550 static void
1551 applyToInterestNotifiers(const IORegistryEntry *target,
1552                          const OSSymbol * typeOfInterest,
1553                          OSObjectApplierFunction applier,
1554                          void * context )
1555 {
1556     OSArray *           copyArray = 0;
1557
1558     LOCKREADNOTIFY();
1559
1560     IOCommand *notifyList =
1561         OSDynamicCast( IOCommand, target->getProperty( typeOfInterest ));
1562
1563     if( notifyList) {
1564         copyArray = OSArray::withCapacity(1);
1565
1566         // iterate over queue, entry is set to each element in the list
1567         iterqueue(&notifyList->fCommandChain, entry) {
1568             _IOServiceInterestNotifier * notify;
1569
1570             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1571             copyArray->setObject(notify);
1572         }
1573     }
1574     UNLOCKNOTIFY();
1575
1576     if( copyArray) {
1577         unsigned int    index;
1578         OSObject *      next;
1579
1580         for( index = 0; (next = copyArray->getObject( index )); index++)
1581             (*applier)(next, context);
1582         copyArray->release();
1583     }
1584 }
1585
1586 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1587                                    OSObjectApplierFunction applier,
1588                                    void * context )
1589 {
1590     if (gIOGeneralInterest == typeOfInterest)
1591         applyToClients( (IOServiceApplierFunction) applier, context );
1592     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1593 }
1594
1595 struct MessageClientsContext {
1596     IOService * service;
1597     UInt32      type;
1598     void *      argument;
1599     vm_size_t   argSize;
1600     IOReturn    ret;
1601 };
1602
1603 static void messageClientsApplier( OSObject * object, void * ctx )
1604 {
1605     IOReturn                ret;
1606     MessageClientsContext * context = (MessageClientsContext *) ctx;
1607
1608     ret = context->service->messageClient( context->type,
1609                                            object, context->argument, context->argSize );
1610     if( kIOReturnSuccess != ret)
1611         context->ret = ret;
1612 }
1613
1614 // send a message to all clients
1615 IOReturn IOService::messageClients( UInt32 type,
1616                                     void * argument, vm_size_t argSize )
1617 {
1618     MessageClientsContext       context;
1619
1620     context.service     = this;
1621     context.type        = type;
1622     context.argument    = argument;
1623     context.argSize     = argSize;
1624     context.ret         = kIOReturnSuccess;
1625
1626     applyToInterested( gIOGeneralInterest,
1627                        &messageClientsApplier, &context );
1628
1629     return( context.ret );
1630 }
1631
1632 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1633                                               IOOptionBits response )
1634 {
1635     return( kIOReturnUnsupported );
1636 }
1637
1638 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1639                   IOServiceInterestHandler handler, void * target, void * ref )
1640 {
1641     _IOServiceInterestNotifier * notify = 0;
1642     IOReturn rc = kIOReturnError;
1643
1644     notify = new _IOServiceInterestNotifier;
1645     if (!notify) return NULL;
1646
1647     if(notify->init()) {
1648         rc = registerInterestForNotifer(notify, typeOfInterest,
1649                               handler, target, ref);
1650     }
1651
1652     if (rc != kIOReturnSuccess) {
1653         notify->release();
1654         notify = 0;
1655     }
1656
1657     return( notify );
1658 }
1659
1660 IOReturn IOService::registerInterestForNotifer( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1661                   IOServiceInterestHandler handler, void * target, void * ref )
1662 {
1663     IOReturn rc = kIOReturnSuccess;
1664     _IOServiceInterestNotifier  *notify = 0;
1665
1666     if( (typeOfInterest != gIOGeneralInterest)
1667      && (typeOfInterest != gIOBusyInterest)
1668      && (typeOfInterest != gIOAppPowerStateInterest)
1669      && (typeOfInterest != gIOConsoleSecurityInterest)
1670      && (typeOfInterest != gIOPriorityPowerStateInterest))
1671         return( kIOReturnBadArgument );
1672
1673     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1674         return( kIOReturnBadArgument );
1675
1676     lockForArbitration();
1677     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1678
1679         notify->handler = handler;
1680         notify->target = target;
1681         notify->ref = ref;
1682         notify->state = kIOServiceNotifyEnable;
1683
1684         ////// queue
1685
1686         LOCKWRITENOTIFY();
1687
1688         // Get the head of the notifier linked list
1689         IOCommand * notifyList;
1690         OSObject  * obj = copyProperty( typeOfInterest );
1691         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1692             notifyList = OSTypeAlloc(IOCommand);
1693             if (notifyList) {
1694                 notifyList->init();
1695                 bool ok = setProperty( typeOfInterest, notifyList);
1696                 notifyList->release();
1697                 if (!ok) notifyList = 0;
1698             }
1699         }
1700         if (obj) obj->release();
1701
1702         if (notifyList) {
1703             enqueue(&notifyList->fCommandChain, &notify->chain);
1704             notify->retain();   // ref'ed while in list
1705         }
1706
1707         UNLOCKNOTIFY();
1708     }
1709     else {
1710         rc = kIOReturnNotReady;
1711     }
1712     unlockForArbitration();
1713
1714     return rc;
1715 }
1716
1717 static void cleanInterestList( OSObject * head )
1718 {
1719     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1720     if (!notifyHead)
1721         return;
1722
1723     LOCKWRITENOTIFY();
1724     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1725         queue_next(entry) = queue_prev(entry) = 0;
1726
1727         _IOServiceInterestNotifier * notify;
1728
1729         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1730         notify->release();
1731     }
1732     UNLOCKNOTIFY();
1733 }
1734
1735 void IOService::unregisterAllInterest( void )
1736 {
1737     cleanInterestList( getProperty( gIOGeneralInterest ));
1738     cleanInterestList( getProperty( gIOBusyInterest ));
1739     cleanInterestList( getProperty( gIOAppPowerStateInterest ));
1740     cleanInterestList( getProperty( gIOPriorityPowerStateInterest ));
1741     cleanInterestList( getProperty( gIOConsoleSecurityInterest ));
1742 }
1743
1744 /*
1745  * _IOServiceInterestNotifier
1746  */
1747
1748 // wait for all threads, other than the current one,
1749 //  to exit the handler
1750
1751 void _IOServiceInterestNotifier::wait()
1752 {
1753     _IOServiceNotifierInvocation * next;
1754     bool doWait;
1755
1756     do {
1757         doWait = false;
1758         queue_iterate( &handlerInvocations, next,
1759                         _IOServiceNotifierInvocation *, link) {
1760             if( next->thread != current_thread() ) {
1761                 doWait = true;
1762                 break;
1763             }
1764         }
1765         if( doWait) {
1766             state |= kIOServiceNotifyWaiter;
1767                SLEEPNOTIFY(this);
1768         }
1769
1770     } while( doWait );
1771 }
1772
1773 void _IOServiceInterestNotifier::free()
1774 {
1775     assert( queue_empty( &handlerInvocations ));
1776     OSObject::free();
1777 }
1778
1779 void _IOServiceInterestNotifier::remove()
1780 {
1781     LOCKWRITENOTIFY();
1782
1783     if( queue_next( &chain )) {
1784         remqueue(&chain);
1785         queue_next( &chain) = queue_prev( &chain) = 0;
1786         release();
1787     }
1788
1789     state &= ~kIOServiceNotifyEnable;
1790
1791     wait();
1792
1793     UNLOCKNOTIFY();
1794
1795     release();
1796 }
1797
1798 bool _IOServiceInterestNotifier::disable()
1799 {
1800     bool        ret;
1801
1802     LOCKWRITENOTIFY();
1803
1804     ret = (0 != (kIOServiceNotifyEnable & state));
1805     state &= ~kIOServiceNotifyEnable;
1806     if( ret)
1807         wait();
1808
1809     UNLOCKNOTIFY();
1810
1811     return( ret );
1812 }
1813
1814 void _IOServiceInterestNotifier::enable( bool was )
1815 {
1816     LOCKWRITENOTIFY();
1817     if( was)
1818         state |= kIOServiceNotifyEnable;
1819     else
1820         state &= ~kIOServiceNotifyEnable;
1821     UNLOCKNOTIFY();
1822 }
1823
1824 bool _IOServiceInterestNotifier::init()
1825 {
1826     queue_init( &handlerInvocations );
1827     return (OSObject::init());
1828 }
1829 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1830
1831 /*
1832  * Termination
1833  */
1834
1835 #define tailQ(o)                setObject(o)
1836 #define headQ(o)                setObject(0, o)
1837 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
1838
1839 static void _workLoopAction( IOWorkLoop::Action action,
1840                              IOService * service,
1841                              void * p0 = 0, void * p1 = 0,
1842                              void * p2 = 0, void * p3 = 0 )
1843 {
1844     IOWorkLoop * wl;
1845
1846     if( (wl = service->getWorkLoop())) {
1847         wl->retain();
1848         wl->runAction( action, service, p0, p1, p2, p3 );
1849         wl->release();
1850     } else
1851         (*action)( service, p0, p1, p2, p3 );
1852 }
1853
1854 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
1855 {
1856     bool ok;
1857
1858     // if its our only provider
1859     ok = isParent( provider, gIOServicePlane, true);
1860
1861     // -- compat
1862     if( ok) {
1863         provider->terminateClient( this, options | kIOServiceRecursing );
1864         ok = (0 != (__state[1] & kIOServiceRecursing));
1865     }
1866     // --
1867
1868     return( ok );
1869 }
1870
1871 bool IOService::terminatePhase1( IOOptionBits options )
1872 {
1873     IOService *  victim;
1874     IOService *  client;
1875     OSIterator * iter;
1876     OSArray *    makeInactive;
1877         int          waitResult = THREAD_AWAKENED;
1878         bool         wait;
1879     bool                 ok;
1880     bool                 didInactive;
1881     bool                 startPhase2 = false;
1882
1883     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
1884
1885     uint64_t regID = getRegistryEntryID();
1886     IOServiceTrace(
1887         IOSERVICE_TERMINATE_PHASE1,
1888         (uintptr_t) regID,
1889         (uintptr_t) (regID >> 32),
1890         (uintptr_t) this,
1891         (uintptr_t) options);
1892
1893     // -- compat
1894     if( options & kIOServiceRecursing) {
1895         lockForArbitration();
1896         __state[0] |= kIOServiceInactiveState;
1897         __state[1] |= kIOServiceRecursing;
1898         unlockForArbitration();
1899
1900         return( true );
1901     }
1902     // --
1903
1904     makeInactive = OSArray::withCapacity( 16 );
1905     if( !makeInactive)
1906         return( false );
1907
1908     victim = this;
1909     victim->retain();
1910
1911     while( victim ) {
1912
1913         didInactive = victim->lockForArbitration( true );
1914         if( didInactive) {
1915             didInactive = (0 == (victim->__state[0] & kIOServiceInactiveState))
1916                          || (victim->__state[1] & kIOServiceRecursing);
1917             if( didInactive) {
1918                 victim->__state[0] |= kIOServiceInactiveState;
1919                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
1920                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
1921                 victim->__state[1] &= ~kIOServiceRecursing;
1922
1923                 if (victim == this)
1924                 {
1925                     victim->__state[1] |= kIOServiceTermPhase1State;
1926                     if (kIOServiceTerminateNeedWillTerminate & options)
1927                     {
1928                         victim->__state[1] |= kIOServiceNeedWillTerminate;
1929                     }
1930                 }
1931
1932                 victim->_adjustBusy( 1 );
1933
1934             } else if (victim != this) do {
1935
1936                 IOLockLock(gIOServiceBusyLock);
1937                 wait = (victim->__state[1] & kIOServiceTermPhase1State);
1938                 if( wait) {
1939                     TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
1940                         getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
1941                         victim->__state[1] |= kIOServiceTerm1WaiterState;
1942                         victim->unlockForArbitration();
1943                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
1944                 }
1945                 IOLockUnlock(gIOServiceBusyLock);
1946                 if( wait) {
1947                     waitResult = thread_block(THREAD_CONTINUE_NULL);
1948                     TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
1949                         getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
1950                         victim->lockForArbitration();
1951                 }
1952             } while( wait && (waitResult != THREAD_TIMED_OUT));
1953
1954             victim->unlockForArbitration();
1955         }
1956         if( victim == this)
1957             startPhase2 = didInactive;
1958         if( didInactive) {
1959
1960             victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff );
1961             IOUserClient::destroyUserReferences( victim );
1962
1963             iter = victim->getClientIterator();
1964             if( iter) {
1965                 while( (client = (IOService *) iter->getNextObject())) {
1966                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
1967                             client->getName(), client->getRegistryEntryID(),
1968                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
1969                     ok = client->requestTerminate( victim, options );
1970                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
1971                             client->getName(), client->getRegistryEntryID(),
1972                             victim->getName(), victim->getRegistryEntryID(), ok);
1973
1974                     uint64_t regID1 = client->getRegistryEntryID();
1975                     uint64_t regID2 = victim->getRegistryEntryID();
1976                     IOServiceTrace(
1977                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
1978                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
1979                         (uintptr_t) regID1,
1980                         (uintptr_t) (regID1 >> 32),
1981                         (uintptr_t) regID2,
1982                         (uintptr_t) (regID2 >> 32));
1983
1984                     if( ok)
1985                         makeInactive->setObject( client );
1986                 }
1987                 iter->release();
1988             }
1989         }
1990         victim->release();
1991         victim = (IOService *) makeInactive->getObject(0);
1992         if( victim) {
1993             victim->retain();
1994             makeInactive->removeObject(0);
1995         }
1996     }
1997
1998     makeInactive->release();
1999
2000     if( startPhase2)
2001     {
2002         lockForArbitration();
2003         __state[1] &= ~kIOServiceTermPhase1State;
2004         if (kIOServiceTerm1WaiterState & __state[1])
2005         {
2006             __state[1] &= ~kIOServiceTerm1WaiterState;
2007             TLOG("%s[0x%qx]::wakePhase1\n", getName(), getRegistryEntryID());
2008             IOLockLock( gIOServiceBusyLock );
2009             thread_wakeup( (event_t) &__state[1]);
2010             IOLockUnlock( gIOServiceBusyLock );
2011         }
2012         unlockForArbitration();
2013         scheduleTerminatePhase2( options );
2014     }
2015
2016     return( true );
2017 }
2018
2019 void IOService::setTerminateDefer(IOService * provider, bool defer)
2020 {
2021     lockForArbitration();
2022     if (defer)  __state[1] |= kIOServiceStartState;
2023     else        __state[1] &= ~kIOServiceStartState;
2024     unlockForArbitration();
2025
2026     if (provider && !defer)
2027     {
2028         provider->lockForArbitration();
2029         provider->scheduleTerminatePhase2();
2030         provider->unlockForArbitration();
2031     }
2032 }
2033
2034 // call with lockForArbitration
2035 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2036 {
2037     AbsoluteTime        deadline;
2038     int                 waitResult = THREAD_AWAKENED;
2039     bool                wait, haveDeadline = false;
2040
2041     if (!(__state[0] & kIOServiceInactiveState)
2042       || (__state[1] & kIOServiceTermPhase1State))              return;
2043
2044     options |= kIOServiceRequired;
2045
2046     retain();
2047
2048     IOLockLock( gJobsLock );
2049
2050     if( (options & kIOServiceSynchronous)
2051         && (current_thread() != gIOTerminateThread)) {
2052
2053         do {
2054             wait = (gIOTerminateThread != 0);
2055             if( wait) {
2056                 // wait to become the terminate thread
2057                 IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2058             }
2059         } while( wait );
2060
2061         gIOTerminateThread = current_thread();
2062         gIOTerminatePhase2List->setObject( this );
2063         gIOTerminateWork++;
2064
2065         do {
2066             while( gIOTerminateWork )
2067                 terminateWorker( options );
2068             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2069             if( wait) {
2070                 // wait for the victim to go non-busy
2071                 if( !haveDeadline) {
2072                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2073                     haveDeadline = true;
2074                 }
2075                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2076                                                   deadline, THREAD_UNINT );
2077                 if( waitResult == THREAD_TIMED_OUT) {
2078                     IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2079                 }
2080             }
2081         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2082
2083         gIOTerminateThread = 0;
2084         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2085
2086     } else {
2087         // ! kIOServiceSynchronous
2088
2089         gIOTerminatePhase2List->setObject( this );
2090         if( 0 == gIOTerminateWork++) {
2091             if( !gIOTerminateThread)
2092                 kernel_thread_start(&terminateThread, (void *)(uintptr_t) options, &gIOTerminateThread);
2093             else
2094                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2095         }
2096     }
2097
2098     IOLockUnlock( gJobsLock );
2099
2100     release();
2101 }
2102
2103 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2104 {
2105     IOLockLock( gJobsLock );
2106
2107     while (gIOTerminateWork)
2108         terminateWorker( (uintptr_t) arg );
2109
2110     thread_deallocate(gIOTerminateThread);
2111     gIOTerminateThread = 0;
2112     IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2113
2114     IOLockUnlock( gJobsLock );
2115 }
2116
2117 void IOService::scheduleStop( IOService * provider )
2118 {
2119     uint64_t regID1 = getRegistryEntryID();
2120     uint64_t regID2 = provider->getRegistryEntryID();
2121
2122     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2123     IOServiceTrace(
2124         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2125         (uintptr_t) regID1,
2126         (uintptr_t) (regID1 >> 32),
2127         (uintptr_t) regID2,
2128         (uintptr_t) (regID2 >> 32));
2129
2130     IOLockLock( gJobsLock );
2131     gIOStopList->tailQ( this );
2132     gIOStopProviderList->tailQ( provider );
2133
2134     if( 0 == gIOTerminateWork++) {
2135         if( !gIOTerminateThread)
2136             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2137         else
2138             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2139     }
2140
2141     IOLockUnlock( gJobsLock );
2142 }
2143
2144 void IOService::scheduleFinalize( void )
2145 {
2146     uint64_t regID1 = getRegistryEntryID();
2147
2148     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2149     IOServiceTrace(
2150         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2151         (uintptr_t) regID1,
2152         (uintptr_t) (regID1 >> 32),
2153         0, 0);
2154
2155     IOLockLock( gJobsLock );
2156     gIOFinalizeList->tailQ( this );
2157
2158     if( 0 == gIOTerminateWork++) {
2159         if( !gIOTerminateThread)
2160             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2161         else
2162             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2163     }
2164
2165     IOLockUnlock( gJobsLock );
2166 }
2167
2168 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2169 {
2170     return( true );
2171 }
2172
2173 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2174 {
2175     if( false == *defer) {
2176
2177         if( lockForArbitration( true )) {
2178             if( false == provider->handleIsOpen( this ))
2179                 scheduleStop( provider );
2180             // -- compat
2181             else {
2182                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2183                 if( false == provider->handleIsOpen( this ))
2184                     scheduleStop( provider );
2185             }
2186             // --
2187             unlockForArbitration();
2188         }
2189     }
2190
2191     return( true );
2192 }
2193
2194 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2195                                      OSArray * doPhase2List,
2196                                      void *unused2 __unused,
2197                                      void *unused3 __unused  )
2198 {
2199     OSIterator * iter;
2200     IOService *  client;
2201     bool         ok;
2202     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2203
2204     iter = victim->getClientIterator();
2205     if( iter) {
2206         while( (client = (IOService *) iter->getNextObject())) {
2207
2208             regID1 = client->getRegistryEntryID();
2209             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2210                     client->getName(), regID1,
2211                     victim->getName(), regID2, (long long)options);
2212             IOServiceTrace(
2213                 IOSERVICE_TERMINATE_WILL,
2214                 (uintptr_t) regID1,
2215                 (uintptr_t) (regID1 >> 32),
2216                 (uintptr_t) regID2,
2217                 (uintptr_t) (regID2 >> 32));
2218
2219             ok = client->willTerminate( victim, options );
2220             doPhase2List->tailQ( client );
2221         }
2222         iter->release();
2223     }
2224 }
2225
2226 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2227                             void *unused1 __unused, void *unused2 __unused,
2228                             void *unused3 __unused )
2229 {
2230     OSIterator * iter;
2231     IOService *  client;
2232     bool defer = false;
2233     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2234
2235     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2236
2237     iter = victim->getClientIterator();
2238     if( iter) {
2239         while( (client = (IOService *) iter->getNextObject())) {
2240
2241             regID1 = client->getRegistryEntryID();
2242             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2243                     client->getName(), regID1,
2244                     victim->getName(), regID2, (long long)options);
2245             client->didTerminate( victim, options, &defer );
2246
2247             IOServiceTrace(
2248                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2249                        : IOSERVICE_TERMINATE_DID),
2250                 (uintptr_t) regID1,
2251                 (uintptr_t) (regID1 >> 32),
2252                 (uintptr_t) regID2,
2253                 (uintptr_t) (regID2 >> 32));
2254
2255             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2256                     client->getName(), regID1,
2257                     victim->getName(), regID2, defer);
2258         }
2259         iter->release();
2260     }
2261 }
2262
2263
2264 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2265                             void *unused1 __unused, void *unused2 __unused,
2266                             void *unused3 __unused )
2267 {
2268     OSIterator * iter;
2269     IOService *  provider;
2270     bool         ok;
2271     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2272
2273     iter = victim->getProviderIterator();
2274     if( iter) {
2275         while( (provider = (IOService *) iter->getNextObject())) {
2276
2277             regID1 = provider->getRegistryEntryID();
2278             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2279                     victim->getName(), regID2,
2280                     provider->getName(), regID1, (long long)options);
2281             IOServiceTrace(
2282                 IOSERVICE_TERMINATE_WILL,
2283                 (uintptr_t) regID2,
2284                 (uintptr_t) (regID2 >> 32),
2285                 (uintptr_t) regID1,
2286                 (uintptr_t) (regID1 >> 32));
2287
2288             ok = victim->willTerminate( provider, options );
2289         }
2290         iter->release();
2291     }
2292 }
2293
2294 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2295                             void *unused1 __unused, void *unused2 __unused,
2296                             void *unused3 __unused )
2297 {
2298     OSIterator * iter;
2299     IOService *  provider;
2300     bool defer = false;
2301     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2302
2303     iter = victim->getProviderIterator();
2304     if( iter) {
2305         while( (provider = (IOService *) iter->getNextObject())) {
2306
2307             regID1 = provider->getRegistryEntryID();
2308             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2309                     victim->getName(), regID2,
2310                     provider->getName(), regID1, (long long)options);
2311             victim->didTerminate( provider, options, &defer );
2312
2313             IOServiceTrace(
2314                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2315                        : IOSERVICE_TERMINATE_DID),
2316                 (uintptr_t) regID2,
2317                 (uintptr_t) (regID2 >> 32),
2318                 (uintptr_t) regID1,
2319                 (uintptr_t) (regID1 >> 32));
2320
2321             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2322                     victim->getName(), regID2,
2323                     provider->getName(), regID1, defer);
2324         }
2325         iter->release();
2326     }
2327 }
2328
2329
2330 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2331                             void *unused1 __unused, void *unused2 __unused,
2332                             void *unused3 __unused )
2333 {
2334     uint64_t regID1 = victim->getRegistryEntryID();
2335     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2336     IOServiceTrace(
2337         IOSERVICE_TERMINATE_FINALIZE,
2338         (uintptr_t) regID1,
2339         (uintptr_t) (regID1 >> 32),
2340         0, 0);
2341
2342     victim->finalize( options );
2343 }
2344
2345 void IOService::actionStop( IOService * provider, IOService * client,
2346                             void *unused1 __unused, void *unused2 __unused,
2347                             void *unused3 __unused )
2348 {
2349     uint64_t regID1 = provider->getRegistryEntryID();
2350     uint64_t regID2 = client->getRegistryEntryID();
2351
2352     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2353     IOServiceTrace(
2354         IOSERVICE_TERMINATE_STOP,
2355         (uintptr_t) regID1,
2356         (uintptr_t) (regID1 >> 32),
2357         (uintptr_t) regID2,
2358         (uintptr_t) (regID2 >> 32));
2359
2360     client->stop( provider );
2361     if( provider->isOpen( client ))
2362         provider->close( client );
2363
2364     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2365     client->detach( provider );
2366 }
2367
2368 void IOService::terminateWorker( IOOptionBits options )
2369 {
2370     OSArray *           doPhase2List;
2371     OSArray *           didPhase2List;
2372     OSSet *             freeList;
2373     OSIterator *        iter;
2374     UInt32              workDone;
2375     IOService *         victim;
2376     IOService *         client;
2377     IOService *         provider;
2378     unsigned int        idx;
2379     bool                moreToDo;
2380     bool                doPhase2;
2381     bool                doPhase3;
2382
2383     options |= kIOServiceRequired;
2384
2385     doPhase2List  = OSArray::withCapacity( 16 );
2386     didPhase2List = OSArray::withCapacity( 16 );
2387     freeList      = OSSet::withCapacity( 16 );
2388     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2389         return;
2390
2391     do {
2392         workDone = gIOTerminateWork;
2393
2394         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2395
2396             victim->retain();
2397             gIOTerminatePhase2List->removeObject(0);
2398             IOLockUnlock( gJobsLock );
2399
2400             while( victim ) {
2401
2402                 doPhase2 = victim->lockForArbitration( true );
2403                 if( doPhase2) {
2404                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2405                     if( doPhase2) {
2406                         doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State))
2407                                 && (0 == (victim->__state[1] & kIOServiceConfigState));
2408
2409                         if (doPhase2 && (iter = victim->getClientIterator())) {
2410                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2411                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2412
2413                                 if (!doPhase2) TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2414                                                victim->getName(), victim->getRegistryEntryID(),
2415                                                client->getName(), client->getRegistryEntryID());
2416                             }
2417                             iter->release();
2418                         }
2419                         if( doPhase2)
2420                             victim->__state[1] |= kIOServiceTermPhase2State;
2421                     }
2422                     victim->unlockForArbitration();
2423                 }
2424                 if( doPhase2) {
2425
2426                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2427                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2428                                             victim, (void *)(uintptr_t) options, NULL );
2429                     }
2430
2431                     if( 0 == victim->getClient()) {
2432                         // no clients - will go to finalize
2433                         IOLockLock( gJobsLock );
2434                         gIOFinalizeList->tailQ( victim );
2435                         IOLockUnlock( gJobsLock );
2436                     } else {
2437                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2438                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2439                     }
2440                     didPhase2List->headQ( victim );
2441                 }
2442                 victim->release();
2443                 victim = (IOService *) doPhase2List->getObject(0);
2444                 if( victim) {
2445                     victim->retain();
2446                     doPhase2List->removeObject(0);
2447                 }
2448             }
2449
2450             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2451
2452                 if( victim->lockForArbitration( true )) {
2453                     victim->__state[1] |= kIOServiceTermPhase3State;
2454                     victim->unlockForArbitration();
2455                 }
2456                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2457                                     victim, (void *)(uintptr_t) options );
2458                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2459                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2460                                         victim, (void *)(uintptr_t) options, NULL );
2461                 }
2462                 didPhase2List->removeObject(0);
2463             }
2464             IOLockLock( gJobsLock );
2465         }
2466
2467         // phase 3
2468         do {
2469             doPhase3 = false;
2470             // finalize leaves
2471             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2472
2473                 IOLockUnlock( gJobsLock );
2474                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2475                                     victim, (void *)(uintptr_t) options );
2476                 IOLockLock( gJobsLock );
2477                 // hold off free
2478                 freeList->setObject( victim );
2479                 // safe if finalize list is append only
2480                 gIOFinalizeList->removeObject(0);
2481             }
2482
2483             for( idx = 0;
2484                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2485
2486                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2487                 assert( provider );
2488
2489                 uint64_t regID1 = provider->getRegistryEntryID();
2490                 uint64_t regID2 = client->getRegistryEntryID();
2491
2492                 if( !provider->isChild( client, gIOServicePlane )) {
2493                     // may be multiply queued - nop it
2494                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2495                     IOServiceTrace(
2496                         IOSERVICE_TERMINATE_STOP_NOP,
2497                         (uintptr_t) regID1,
2498                         (uintptr_t) (regID1 >> 32),
2499                         (uintptr_t) regID2,
2500                         (uintptr_t) (regID2 >> 32));
2501
2502                 } else {
2503                     // a terminated client is not ready for stop if it has clients, skip it
2504                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2505                         TLOG("%s[0x%qx]::defer stop(%s[0x%qx])\n",
2506                                 client->getName(), regID2,
2507                                 client->getClient()->getName(), client->getClient()->getRegistryEntryID());
2508                         IOServiceTrace(
2509                             IOSERVICE_TERMINATE_STOP_DEFER,
2510                             (uintptr_t) regID1,
2511                             (uintptr_t) (regID1 >> 32),
2512                             (uintptr_t) regID2,
2513                             (uintptr_t) (regID2 >> 32));
2514
2515                         idx++;
2516                         continue;
2517                     }
2518
2519                     IOLockUnlock( gJobsLock );
2520                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2521                                      provider, (void *) client );
2522                     IOLockLock( gJobsLock );
2523                     // check the finalize list now
2524                     doPhase3 = true;
2525                 }
2526                 // hold off free
2527                 freeList->setObject( client );
2528                 freeList->setObject( provider );
2529
2530                 // safe if stop list is append only
2531                 gIOStopList->removeObject( idx );
2532                 gIOStopProviderList->removeObject( idx );
2533                 idx = 0;
2534             }
2535
2536         } while( doPhase3 );
2537
2538         gIOTerminateWork -= workDone;
2539         moreToDo = (gIOTerminateWork != 0);
2540
2541         if( !moreToDo) {
2542             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2543             IOServiceTrace(
2544                 IOSERVICE_TERMINATE_DONE,
2545                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2546         }
2547
2548     } while( moreToDo );
2549
2550     IOLockUnlock( gJobsLock );
2551
2552     freeList->release();
2553     doPhase2List->release();
2554     didPhase2List->release();
2555
2556     IOLockLock( gJobsLock );
2557 }
2558
2559 bool IOService::finalize( IOOptionBits options )
2560 {
2561     OSIterator *  iter;
2562     IOService *   provider;
2563     uint64_t      regID1, regID2 = getRegistryEntryID();
2564
2565     iter = getProviderIterator();
2566     assert( iter );
2567
2568     if( iter) {
2569         while( (provider = (IOService *) iter->getNextObject())) {
2570
2571             // -- compat
2572             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2573                 /* we come down here on programmatic terminate */
2574
2575                 regID1 = provider->getRegistryEntryID();
2576                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2577                 IOServiceTrace(
2578                     IOSERVICE_TERMINATE_STOP,
2579                     (uintptr_t) regID1,
2580                     (uintptr_t) (regID1 >> 32),
2581                     (uintptr_t) regID2,
2582                     (uintptr_t) (regID2 >> 32));
2583
2584                 stop( provider );
2585                 if( provider->isOpen( this ))
2586                     provider->close( this );
2587                 detach( provider );
2588             } else {
2589             //--
2590                 if( provider->lockForArbitration( true )) {
2591                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2592                         scheduleStop( provider );
2593                     provider->unlockForArbitration();
2594                 }
2595             }
2596         }
2597         iter->release();
2598     }
2599
2600     return( true );
2601 }
2602
2603 #undef tailQ
2604 #undef headQ
2605
2606 /*
2607  * Terminate
2608  */
2609
2610 void IOService::doServiceTerminate( IOOptionBits options )
2611 {
2612 }
2613
2614 // a method in case someone needs to override it
2615 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2616 {
2617     bool ok;
2618
2619     if( client->isParent( this, gIOServicePlane, true))
2620         // we are the clients only provider
2621         ok = client->terminate( options );
2622     else
2623         ok = true;
2624
2625     return( ok );
2626 }
2627
2628 bool IOService::terminate( IOOptionBits options )
2629 {
2630     options |= kIOServiceTerminate;
2631
2632     return( terminatePhase1( options ));
2633 }
2634
2635 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2636
2637 /*
2638  * Open & close
2639  */
2640
2641 struct ServiceOpenMessageContext
2642 {
2643     IOService *  service;
2644     UInt32       type;
2645     IOService *  excludeClient;
2646     IOOptionBits options;
2647 };
2648
2649 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2650 {
2651     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2652
2653     if( object != context->excludeClient)
2654         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2655 }
2656
2657 bool IOService::open(   IOService *     forClient,
2658                         IOOptionBits    options,
2659                         void *          arg )
2660 {
2661     bool                        ok;
2662     ServiceOpenMessageContext   context;
2663
2664     context.service             = this;
2665     context.type                = kIOMessageServiceIsAttemptingOpen;
2666     context.excludeClient       = forClient;
2667     context.options             = options;
2668
2669     applyToInterested( gIOGeneralInterest,
2670                         &serviceOpenMessageApplier, &context );
2671
2672     if( false == lockForArbitration(false) )
2673         return false;
2674
2675     ok = (0 == (__state[0] & kIOServiceInactiveState));
2676     if( ok)
2677         ok = handleOpen( forClient, options, arg );
2678
2679     unlockForArbitration();
2680
2681     return( ok );
2682 }
2683
2684 void IOService::close(  IOService *     forClient,
2685                         IOOptionBits    options )
2686 {
2687     bool                wasClosed;
2688     bool                last = false;
2689
2690     lockForArbitration();
2691
2692     wasClosed = handleIsOpen( forClient );
2693     if( wasClosed) {
2694         handleClose( forClient, options );
2695         last = (__state[1] & kIOServiceTermPhase3State);
2696     }
2697
2698     unlockForArbitration();
2699
2700     if( last)
2701         forClient->scheduleStop( this );
2702
2703     else if( wasClosed) {
2704
2705         ServiceOpenMessageContext context;
2706
2707         context.service         = this;
2708         context.type            = kIOMessageServiceWasClosed;
2709         context.excludeClient   = forClient;
2710         context.options         = options;
2711
2712         applyToInterested( gIOGeneralInterest,
2713                             &serviceOpenMessageApplier, &context );
2714     }
2715 }
2716
2717 bool IOService::isOpen( const IOService * forClient ) const
2718 {
2719     IOService * self = (IOService *) this;
2720     bool ok;
2721
2722     self->lockForArbitration();
2723
2724     ok = handleIsOpen( forClient );
2725
2726     self->unlockForArbitration();
2727
2728     return( ok );
2729 }
2730
2731 bool IOService::handleOpen(     IOService *     forClient,
2732                                 IOOptionBits    options,
2733                                 void *          arg )
2734 {
2735     bool        ok;
2736
2737     ok = (0 == __owner);
2738     if( ok )
2739         __owner = forClient;
2740
2741     else if( options & kIOServiceSeize ) {
2742         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2743                                 __owner, (void *)(uintptr_t) options ));
2744         if( ok && (0 == __owner ))
2745             __owner = forClient;
2746         else
2747             ok = false;
2748     }
2749     return( ok );
2750 }
2751
2752 void IOService::handleClose(    IOService *     forClient,
2753                                 IOOptionBits    options )
2754 {
2755     if( __owner == forClient)
2756         __owner = 0;
2757 }
2758
2759 bool IOService::handleIsOpen(   const IOService * forClient ) const
2760 {
2761     if( forClient)
2762         return( __owner == forClient );
2763     else
2764         return( __owner != forClient );
2765 }
2766
2767 /*
2768  * Probing & starting
2769  */
2770 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2771 {
2772     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
2773     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
2774     SInt32             val1;
2775     SInt32             val2;
2776
2777     val1 = 0;
2778     val2 = 0;
2779
2780     if ( obj1 )
2781         val1 = obj1->priority;
2782
2783     if ( obj2 )
2784         val2 = obj2->priority;
2785
2786     return ( val1 - val2 );
2787 }
2788
2789 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
2790 {
2791     OSDictionary *      dict;
2792     IOService *         service;
2793     _IOServiceNotifier * notify;
2794     OSSymbol *          key = (OSSymbol *) ref;
2795     OSNumber *          offset;
2796
2797     if( (dict = OSDynamicCast( OSDictionary, entry)))
2798         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
2799     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
2800         return( notify->priority );
2801
2802     else if( (service = OSDynamicCast( IOService, entry)))
2803         offset = OSDynamicCast(OSNumber, service->getProperty( key ));
2804     else {
2805         assert( false );
2806         offset = 0;
2807     }
2808
2809     if( offset)
2810         return( (SInt32) offset->unsigned32BitValue());
2811     else
2812         return( kIODefaultProbeScore );
2813 }
2814
2815 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2816 {
2817     const OSObject *    obj1 = (const OSObject *) inObj1;
2818     const OSObject *    obj2 = (const OSObject *) inObj2;
2819     SInt32               val1;
2820     SInt32               val2;
2821
2822     val1 = 0;
2823     val2 = 0;
2824
2825     if ( obj1 )
2826         val1 = IOServiceObjectOrder( obj1, ref );
2827
2828     if ( obj2 )
2829         val2 = IOServiceObjectOrder( obj2, ref );
2830
2831     return ( val1 - val2 );
2832 }
2833
2834 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
2835 {
2836     IOService *         service = 0;
2837     OSIterator *        iter;
2838     const OSSymbol *    nextCat;
2839
2840     iter = getClientIterator();
2841     if( iter) {
2842         while( (service = (IOService *) iter->getNextObject())) {
2843             if( kIOServiceInactiveState & service->__state[0])
2844                 continue;
2845             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
2846                         service->getProperty( gIOMatchCategoryKey ));
2847             if( category == nextCat)
2848             {
2849                 service->retain();
2850                 break;
2851             }
2852         }
2853         iter->release();
2854     }
2855     return( service );
2856 }
2857
2858 IOService * IOService::getClientWithCategory( const OSSymbol * category )
2859 {
2860     IOService *
2861     service = copyClientWithCategory(category);
2862     if (service)
2863         service->release();
2864     return (service);
2865 }
2866
2867 bool IOService::invokeNotifer( _IOServiceNotifier * notify )
2868 {
2869     _IOServiceNotifierInvocation invocation;
2870     bool                         willNotify;
2871     bool                         ret = true;
2872
2873     invocation.thread = current_thread();
2874
2875     LOCKWRITENOTIFY();
2876     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2877
2878     if( willNotify) {
2879         queue_enter( &notify->handlerInvocations, &invocation,
2880                         _IOServiceNotifierInvocation *, link );
2881     }
2882     UNLOCKNOTIFY();
2883
2884     if( willNotify) {
2885
2886         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
2887
2888         LOCKWRITENOTIFY();
2889         queue_remove( &notify->handlerInvocations, &invocation,
2890                         _IOServiceNotifierInvocation *, link );
2891         if( kIOServiceNotifyWaiter & notify->state) {
2892             notify->state &= ~kIOServiceNotifyWaiter;
2893             WAKEUPNOTIFY( notify );
2894         }
2895         UNLOCKNOTIFY();
2896     }
2897
2898     return( ret );
2899 }
2900
2901 /*
2902  * Alloc and probe matching classes,
2903  * called on the provider instance
2904  */
2905
2906 void IOService::probeCandidates( OSOrderedSet * matches )
2907 {
2908     OSDictionary        *       match = 0;
2909     OSSymbol            *       symbol;
2910     IOService           *       inst;
2911     IOService           *       newInst;
2912     OSDictionary        *       props;
2913     SInt32                      score;
2914     OSNumber            *       newPri;
2915     OSOrderedSet        *       familyMatches = 0;
2916     OSOrderedSet        *       startList;
2917     OSDictionary        *       startDict = 0;
2918     const OSSymbol      *       category;
2919     OSIterator          *       iter;
2920     _IOServiceNotifier  *               notify;
2921     OSObject            *       nextMatch = 0;
2922     bool                        started;
2923     bool                        needReloc = false;
2924 #if IOMATCHDEBUG
2925     SInt64                      debugFlags;
2926 #endif
2927     IOService * client = NULL;
2928
2929
2930     assert( matches );
2931     while( !needReloc && (nextMatch = matches->getFirstObject())) {
2932
2933         nextMatch->retain();
2934         matches->removeObject(nextMatch);
2935
2936         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
2937
2938             lockForArbitration();
2939             if( 0 == (__state[0] & kIOServiceInactiveState))
2940                 invokeNotifer( notify );
2941             unlockForArbitration();
2942             nextMatch->release();
2943             nextMatch = 0;
2944             continue;
2945
2946         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
2947             nextMatch->release();
2948             nextMatch = 0;
2949             continue;
2950         }
2951
2952         props = 0;
2953 #if IOMATCHDEBUG
2954         debugFlags = getDebugFlags( match );
2955 #endif
2956
2957         do {
2958             category = OSDynamicCast( OSSymbol,
2959                         match->getObject( gIOMatchCategoryKey ));
2960             if( 0 == category)
2961                 category = gIODefaultMatchCategoryKey;
2962
2963             if( (client = copyClientWithCategory(category)) ) {
2964 #if IOMATCHDEBUG
2965                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
2966                     LOG("%s: match category %s exists\n", getName(),
2967                                 category->getCStringNoCopy());
2968 #endif
2969                 nextMatch->release();
2970                 nextMatch = 0;
2971
2972                 client->release();
2973                 client = NULL;
2974
2975                 continue;
2976             }
2977
2978             // create a copy now in case its modified during matching
2979             props = OSDictionary::withDictionary( match, match->getCount());
2980             if( 0 == props)
2981                 continue;
2982             props->setCapacityIncrement(1);
2983
2984             // check the nub matches
2985             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
2986                 continue;
2987
2988             // Check to see if driver reloc has been loaded.
2989             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
2990             if( needReloc) {
2991 #if IOMATCHDEBUG
2992                 if( debugFlags & kIOLogCatalogue)
2993                     LOG("%s: stalling for module\n", getName());
2994 #endif
2995                 // If reloc hasn't been loaded, exit;
2996                 // reprobing will occur after reloc has been loaded.
2997                 continue;
2998             }
2999
3000             // reorder on family matchPropertyTable score.
3001             if( 0 == familyMatches)
3002                 familyMatches = OSOrderedSet::withCapacity( 1,
3003                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3004             if( familyMatches)
3005                 familyMatches->setObject( props );
3006
3007         } while( false );
3008
3009         if (nextMatch) {
3010             nextMatch->release();
3011             nextMatch = 0;
3012         }
3013         if( props)
3014             props->release();
3015     }
3016     matches->release();
3017     matches = 0;
3018
3019     if( familyMatches) {
3020
3021         while( !needReloc
3022              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3023
3024             props->retain();
3025             familyMatches->removeObject( props );
3026
3027             inst = 0;
3028             newInst = 0;
3029 #if IOMATCHDEBUG
3030             debugFlags = getDebugFlags( props );
3031 #endif
3032             do {
3033                 symbol = OSDynamicCast( OSSymbol,
3034                                 props->getObject( gIOClassKey));
3035                 if( !symbol)
3036                     continue;
3037
3038                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), OBFUSCATE(symbol), OBFUSCATE(props));
3039
3040                 // alloc the driver instance
3041                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3042
3043                 if( !inst) {
3044                     IOLog("Couldn't alloc class \"%s\"\n",
3045                         symbol->getCStringNoCopy());
3046                     continue;
3047                 }
3048
3049                 // init driver instance
3050                 if( !(inst->init( props ))) {
3051 #if IOMATCHDEBUG
3052                     if( debugFlags & kIOLogStart)
3053                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3054 #endif
3055                     continue;
3056                 }
3057                 if( __state[1] & kIOServiceSynchronousState)
3058                     inst->__state[1] |= kIOServiceSynchronousState;
3059
3060                 // give the driver the default match category if not specified
3061                 category = OSDynamicCast( OSSymbol,
3062                             props->getObject( gIOMatchCategoryKey ));
3063                 if( 0 == category)
3064                     category = gIODefaultMatchCategoryKey;
3065                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3066                 // attach driver instance
3067                 if( !(inst->attach( this )))
3068                         continue;
3069
3070                 // pass in score from property table
3071                 score = familyMatches->orderObject( props );
3072
3073                 // & probe the new driver instance
3074 #if IOMATCHDEBUG
3075                 if( debugFlags & kIOLogProbe)
3076                     LOG("%s::probe(%s)\n",
3077                         inst->getMetaClass()->getClassName(), getName());
3078 #endif
3079
3080                 newInst = inst->probe( this, &score );
3081                 inst->detach( this );
3082                 if( 0 == newInst) {
3083 #if IOMATCHDEBUG
3084                     if( debugFlags & kIOLogProbe)
3085                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3086 #endif
3087                     continue;
3088                 }
3089
3090                 // save the score
3091                 newPri = OSNumber::withNumber( score, 32 );
3092                 if( newPri) {
3093                     newInst->setProperty( gIOProbeScoreKey, newPri );
3094                     newPri->release();
3095                 }
3096
3097                 // add to start list for the match category
3098                 if( 0 == startDict)
3099                     startDict = OSDictionary::withCapacity( 1 );
3100                 assert( startDict );
3101                 startList = (OSOrderedSet *)
3102                                 startDict->getObject( category );
3103                 if( 0 == startList) {
3104                     startList = OSOrderedSet::withCapacity( 1,
3105                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3106                     if( startDict && startList) {
3107                         startDict->setObject( category, startList );
3108                         startList->release();
3109                     }
3110                 }
3111                 assert( startList );
3112                 if( startList)
3113                     startList->setObject( newInst );
3114
3115             } while( false );
3116
3117             props->release();
3118             if( inst)
3119                 inst->release();
3120         }
3121         familyMatches->release();
3122         familyMatches = 0;
3123     }
3124
3125     // start the best (until success) of each category
3126
3127     iter = OSCollectionIterator::withCollection( startDict );
3128     if( iter) {
3129         while( (category = (const OSSymbol *) iter->getNextObject())) {
3130
3131             startList = (OSOrderedSet *) startDict->getObject( category );
3132             assert( startList );
3133             if( !startList)
3134                 continue;
3135
3136             started = false;
3137             while( true // (!started)
3138                    && (inst = (IOService *)startList->getFirstObject())) {
3139
3140                 inst->retain();
3141                 startList->removeObject(inst);
3142
3143 #if IOMATCHDEBUG
3144                 debugFlags = getDebugFlags( inst->getPropertyTable() );
3145
3146                 if( debugFlags & kIOLogStart) {
3147                     if( started)
3148                         LOG( "match category exists, skipping " );
3149                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3150                          getName(), inst->getRetainCount());
3151                 }
3152 #endif
3153                 if( false == started)
3154                     started = startCandidate( inst );
3155 #if IOMATCHDEBUG
3156                 if( (debugFlags & kIOLogStart) && (false == started))
3157                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3158                          inst->getRetainCount());
3159 #endif
3160                 inst->release();
3161             }
3162         }
3163         iter->release();
3164     }
3165
3166
3167     // adjust the busy count by +1 if matching is stalled for a module,
3168     // or -1 if a previously stalled matching is complete.
3169     lockForArbitration();
3170     SInt32 adjBusy = 0;
3171     uint64_t regID = getRegistryEntryID();
3172
3173     if( needReloc) {
3174         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3175         if( adjBusy) {
3176
3177             IOServiceTrace(
3178                 IOSERVICE_MODULESTALL,
3179                 (uintptr_t) regID,
3180                 (uintptr_t) (regID >> 32),
3181                 (uintptr_t) this,
3182                 0);
3183
3184             __state[1] |= kIOServiceModuleStallState;
3185         }
3186
3187     } else if( __state[1] & kIOServiceModuleStallState) {
3188
3189             IOServiceTrace(
3190             IOSERVICE_MODULEUNSTALL,
3191             (uintptr_t) regID,
3192             (uintptr_t) (regID >> 32),
3193             (uintptr_t) this,
3194             0);
3195
3196         __state[1] &= ~kIOServiceModuleStallState;
3197         adjBusy = -1;
3198     }
3199     if( adjBusy)
3200         _adjustBusy( adjBusy );
3201     unlockForArbitration();
3202
3203     if( startDict)
3204         startDict->release();
3205 }
3206
3207 /*
3208  * Start a previously attached & probed instance,
3209  * called on exporting object instance
3210  */
3211
3212 bool IOService::startCandidate( IOService * service )
3213 {
3214     bool                ok;
3215
3216     ok = service->attach( this );
3217
3218     if( ok)
3219     {
3220         if (this != gIOResources)
3221         {
3222             // stall for any nub resources
3223             checkResources();
3224             // stall for any driver resources
3225             service->checkResources();
3226         }
3227
3228         AbsoluteTime startTime;
3229         AbsoluteTime endTime;
3230         UInt64       nano;
3231
3232         if (kIOLogStart & gIOKitDebug)
3233             clock_get_uptime(&startTime);
3234
3235         ok = service->start(this);
3236
3237         if (kIOLogStart & gIOKitDebug)
3238         {
3239             clock_get_uptime(&endTime);
3240
3241             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3242             {
3243                 SUB_ABSOLUTETIME(&endTime, &startTime);
3244                 absolutetime_to_nanoseconds(endTime, &nano);
3245                 if (nano > 500000000ULL)
3246                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3247             }
3248         }
3249         if( !ok)
3250             service->detach( this );
3251     }
3252     return( ok );
3253 }
3254
3255 void IOService::publishResource( const char * key, OSObject * value )
3256 {
3257     const OSSymbol *    sym;
3258
3259     if( (sym = OSSymbol::withCString( key))) {
3260         publishResource( sym, value);
3261         sym->release();
3262     }
3263 }
3264
3265 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3266 {
3267     if( 0 == value)
3268         value = (OSObject *) gIOServiceKey;
3269
3270     gIOResources->setProperty( key, value);
3271
3272     if( IORecursiveLockHaveLock( gNotificationLock))
3273         return;
3274
3275     gIOResourceGenerationCount++;
3276     gIOResources->registerService();
3277 }
3278
3279 bool IOService::addNeededResource( const char * key )
3280 {
3281     OSObject *  resourcesProp;
3282     OSSet *     set;
3283     OSString *  newKey;
3284     bool ret;
3285
3286     resourcesProp = getProperty( gIOResourceMatchKey );
3287
3288     newKey = OSString::withCString( key );
3289     if( (0 == resourcesProp) || (0 == newKey))
3290         return( false);
3291
3292     set = OSDynamicCast( OSSet, resourcesProp );
3293     if( !set) {
3294         set = OSSet::withCapacity( 1 );
3295         if( set)
3296             set->setObject( resourcesProp );
3297     }
3298     else
3299         set->retain();
3300
3301     set->setObject( newKey );
3302     newKey->release();
3303     ret = setProperty( gIOResourceMatchKey, set );
3304     set->release();
3305
3306     return( ret );
3307 }
3308
3309 bool IOService::checkResource( OSObject * matching )
3310 {
3311     OSString *          str;
3312     OSDictionary *      table;
3313
3314     if( (str = OSDynamicCast( OSString, matching ))) {
3315         if( gIOResources->getProperty( str ))
3316             return( true );
3317     }
3318
3319     if( str)
3320         table = resourceMatching( str );
3321     else if( (table = OSDynamicCast( OSDictionary, matching )))
3322         table->retain();
3323     else {
3324         IOLog("%s: Can't match using: %s\n", getName(),
3325                 matching->getMetaClass()->getClassName());
3326         /* false would stall forever */
3327         return( true );
3328     }
3329
3330     if( gIOKitDebug & kIOLogConfig)
3331         LOG("config(%p): stalling %s\n", OBFUSCATE(IOThreadSelf()), getName());
3332
3333     waitForService( table );
3334
3335     if( gIOKitDebug & kIOLogConfig)
3336         LOG("config(%p): waking\n", OBFUSCATE(IOThreadSelf()) );
3337
3338     return( true );
3339 }
3340
3341 bool IOService::checkResources( void )
3342 {
3343     OSObject *          resourcesProp;
3344     OSSet *             set;
3345     OSIterator *        iter;
3346     bool                ok;
3347
3348     resourcesProp = getProperty( gIOResourceMatchKey );
3349     if( 0 == resourcesProp)
3350         return( true );
3351
3352     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3353
3354         iter = OSCollectionIterator::withCollection( set );
3355         ok = (0 != iter);
3356         while( ok && (resourcesProp = iter->getNextObject()) )
3357             ok = checkResource( resourcesProp );
3358         if( iter)
3359             iter->release();
3360
3361     } else
3362         ok = checkResource( resourcesProp );
3363
3364     return( ok );
3365 }
3366
3367
3368 void _IOConfigThread::configThread( void )
3369 {
3370     _IOConfigThread *   inst;
3371
3372     do {
3373         if( !(inst = new _IOConfigThread))
3374             continue;
3375         if( !inst->init())
3376             continue;
3377         thread_t unused;
3378         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3379             continue;
3380
3381         return;
3382
3383     } while( false);
3384
3385     if( inst)
3386         inst->release();
3387
3388     return;
3389 }
3390
3391 void _IOConfigThread::free( void )
3392 {
3393     thread_deallocate(current_thread());
3394     OSObject::free();
3395 }
3396
3397 void IOService::doServiceMatch( IOOptionBits options )
3398 {
3399     _IOServiceNotifier * notify;
3400     OSIterator *        iter;
3401     OSOrderedSet *      matches;
3402     SInt32              catalogGeneration;
3403     bool                keepGuessing = true;
3404     bool                reRegistered = true;
3405     bool                didRegister;
3406
3407 //    job->nub->deliverNotification( gIOPublishNotification,
3408 //                              kIOServiceRegisteredState, 0xffffffff );
3409
3410     while( keepGuessing ) {
3411
3412         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3413         // the matches list should always be created by findDrivers()
3414         if( matches) {
3415
3416             lockForArbitration();
3417             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3418                 getMetaClass()->addInstance(this);
3419                 deliverNotification( gIOFirstPublishNotification,
3420                                      kIOServiceFirstPublishState, 0xffffffff );
3421             }
3422             LOCKREADNOTIFY();
3423             __state[1] &= ~kIOServiceNeedConfigState;
3424             __state[1] |= kIOServiceConfigState;
3425             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3426             __state[0] |= kIOServiceRegisteredState;
3427
3428             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3429             if (reRegistered && keepGuessing) {
3430                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3431                         gNotifications->getObject( gIOPublishNotification ) );
3432                 if( iter) {
3433                     while((notify = (_IOServiceNotifier *)
3434                            iter->getNextObject())) {
3435
3436                         if( matchPassive(notify->matching, 0)
3437                          && (kIOServiceNotifyEnable & notify->state))
3438                             matches->setObject( notify );
3439                     }
3440                     iter->release();
3441                 }
3442             }
3443
3444             UNLOCKNOTIFY();
3445             unlockForArbitration();
3446
3447             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3448                 probeCandidates( matches );
3449             else
3450                 matches->release();
3451         }
3452
3453         lockForArbitration();
3454         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3455         keepGuessing =
3456                    (reRegistered || (catalogGeneration !=
3457                                         gIOCatalogue->getGenerationCount()))
3458                 && (0 == (__state[0] & kIOServiceInactiveState));
3459
3460         if( keepGuessing)
3461             unlockForArbitration();
3462     }
3463
3464     if( (0 == (__state[0] & kIOServiceInactiveState))
3465      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3466         deliverNotification( gIOMatchedNotification,
3467                 kIOServiceMatchedState, 0xffffffff );
3468         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3469             deliverNotification( gIOFirstMatchNotification,
3470                 kIOServiceFirstMatchState, 0xffffffff );
3471     }
3472
3473     __state[1] &= ~kIOServiceConfigState;
3474     scheduleTerminatePhase2();
3475
3476     _adjustBusy( -1 );
3477     unlockForArbitration();
3478 }
3479
3480 UInt32 IOService::_adjustBusy( SInt32 delta )
3481 {
3482     IOService * next;
3483     UInt32      count;
3484     UInt32      result;
3485     bool        wasQuiet, nowQuiet, needWake;
3486
3487     next = this;
3488     result = __state[1] & kIOServiceBusyStateMask;
3489
3490     if( delta) do {
3491         if( next != this)
3492             next->lockForArbitration();
3493         count = next->__state[1] & kIOServiceBusyStateMask;
3494         wasQuiet = (0 == count);
3495         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3496             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3497         else
3498             count += delta;
3499         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3500         nowQuiet = (0 == count);
3501         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3502
3503         if( needWake) {
3504             next->__state[1] &= ~kIOServiceBusyWaiterState;
3505             IOLockLock( gIOServiceBusyLock );
3506             thread_wakeup( (event_t) next);
3507             IOLockUnlock( gIOServiceBusyLock );
3508         }
3509         if( next != this)
3510             next->unlockForArbitration();
3511
3512         if( (wasQuiet || nowQuiet) ) {
3513             uint64_t regID = next->getRegistryEntryID();
3514
3515                 IOServiceTrace(
3516                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3517                 (uintptr_t) regID,
3518                 (uintptr_t) (regID >> 32),
3519                 (uintptr_t) next,
3520                 0);
3521
3522             if (wasQuiet)
3523             {
3524                 next->__timeBusy = mach_absolute_time();
3525             }
3526             else
3527             {
3528                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3529                 next->__timeBusy = 0;
3530             }
3531
3532             MessageClientsContext context;
3533
3534             context.service  = next;
3535             context.type     = kIOMessageServiceBusyStateChange;
3536             context.argument = (void *) wasQuiet;       /*nowBusy*/
3537             context.argSize  = 0;
3538
3539             applyToInterestNotifiers( next, gIOBusyInterest,
3540                                      &messageClientsApplier, &context );
3541
3542 #if !NO_KEXTD
3543             if( nowQuiet && (next == gIOServiceRoot)) {
3544                 OSKext::considerUnloads();
3545                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3546             }
3547 #endif
3548         }
3549
3550         delta = nowQuiet ? -1 : +1;
3551
3552     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3553
3554     return( result );
3555 }
3556
3557 void IOService::adjustBusy( SInt32 delta )
3558 {
3559     lockForArbitration();
3560     _adjustBusy( delta );
3561     unlockForArbitration();
3562 }
3563
3564 uint64_t IOService::getAccumulatedBusyTime( void )
3565 {
3566     uint64_t accumBusy = __accumBusy;
3567     uint64_t timeBusy = __timeBusy;
3568     uint64_t nano;
3569
3570     do
3571     {
3572         accumBusy = __accumBusy;
3573         timeBusy  = __timeBusy;
3574         if (timeBusy)
3575             accumBusy += mach_absolute_time() - timeBusy;
3576     }
3577     while (timeBusy != __timeBusy);
3578
3579     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3580
3581     return (nano);
3582 }
3583
3584 UInt32 IOService::getBusyState( void )
3585 {
3586     return( __state[1] & kIOServiceBusyStateMask );
3587 }
3588
3589 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3590                                   mach_timespec_t * timeout )
3591 {
3592     panic("waitForState");
3593     return (kIOReturnUnsupported);
3594 }
3595
3596 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3597                                    uint64_t timeout )
3598 {
3599     bool            wait;
3600     int             waitResult = THREAD_AWAKENED;
3601     bool            computeDeadline = true;
3602     AbsoluteTime    abstime;
3603
3604     do {
3605         lockForArbitration();
3606         IOLockLock( gIOServiceBusyLock );
3607         wait = (value != (__state[1] & mask));
3608         if( wait) {
3609             __state[1] |= kIOServiceBusyWaiterState;
3610             unlockForArbitration();
3611             if( timeout != UINT64_MAX ) {
3612                 if( computeDeadline ) {
3613                     AbsoluteTime  nsinterval;
3614                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3615                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3616                     computeDeadline = false;
3617                 }
3618                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3619             }
3620             else
3621                 assert_wait((event_t)this, THREAD_UNINT );
3622         } else
3623             unlockForArbitration();
3624         IOLockUnlock( gIOServiceBusyLock );
3625         if( wait)
3626             waitResult = thread_block(THREAD_CONTINUE_NULL);
3627
3628     } while( wait && (waitResult != THREAD_TIMED_OUT));
3629
3630     if( waitResult == THREAD_TIMED_OUT)
3631         return( kIOReturnTimeout );
3632     else
3633         return( kIOReturnSuccess );
3634 }
3635
3636 IOReturn IOService::waitQuiet( uint64_t timeout )
3637 {
3638         IOReturn ret;
3639     ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3640         if ((kIOReturnTimeout == ret) && (timeout >= 30000000000) && (kIOWaitQuietPanics & gIOKitDebug))
3641         {
3642                 panic("IOService 0x%llx (%s) busy timeout", getRegistryEntryID(), getName());
3643         }
3644         return (ret);
3645 }
3646
3647 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
3648 {
3649     uint64_t    timeoutNS;
3650
3651     if (timeout)
3652     {
3653         timeoutNS = timeout->tv_sec;
3654         timeoutNS *= kSecondScale;
3655         timeoutNS += timeout->tv_nsec;
3656     }
3657     else
3658         timeoutNS = UINT64_MAX;
3659
3660     return (waitQuiet(timeoutNS));
3661 }
3662
3663 bool IOService::serializeProperties( OSSerialize * s ) const
3664 {
3665 #if 0
3666     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
3667                 sizeof( __state), "__state");
3668 #endif
3669     return( super::serializeProperties(s) );
3670 }
3671
3672
3673 void _IOConfigThread::main(void * arg, wait_result_t result)
3674 {
3675     _IOConfigThread * self = (_IOConfigThread *) arg;
3676     _IOServiceJob * job;
3677     IOService   *   nub;
3678     bool            alive = true;
3679     kern_return_t   kr;
3680     thread_precedence_policy_data_t precedence = { -1 };
3681
3682     kr = thread_policy_set(current_thread(),
3683                             THREAD_PRECEDENCE_POLICY,
3684                             (thread_policy_t) &precedence,
3685                             THREAD_PRECEDENCE_POLICY_COUNT);
3686     if (KERN_SUCCESS != kr)
3687         IOLog("thread_policy_set(%d)\n", kr);
3688
3689     do {
3690
3691 //      randomDelay();
3692
3693         semaphore_wait( gJobsSemaphore );
3694
3695         IOTakeLock( gJobsLock );
3696         job = (_IOServiceJob *) gJobs->getFirstObject();
3697         job->retain();
3698         gJobs->removeObject(job);
3699         if( job) {
3700             gOutstandingJobs--;
3701 //          gNumConfigThreads--;        // we're out of service
3702             gNumWaitingThreads--;       // we're out of service
3703         }
3704         IOUnlock( gJobsLock );
3705
3706         if( job) {
3707
3708           nub = job->nub;
3709
3710           if( gIOKitDebug & kIOLogConfig)
3711             LOG("config(%p): starting on %s, %d\n",
3712                         OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
3713
3714           switch( job->type) {
3715
3716             case kMatchNubJob:
3717                 nub->doServiceMatch( job->options );
3718                 break;
3719
3720             default:
3721                 LOG("config(%p): strange type (%d)\n",
3722                         OBFUSCATE(IOThreadSelf()), job->type );
3723                 break;
3724             }
3725
3726             nub->release();
3727             job->release();
3728
3729             IOTakeLock( gJobsLock );
3730             alive = (gOutstandingJobs > gNumWaitingThreads);
3731             if( alive)
3732                 gNumWaitingThreads++;   // back in service
3733 //              gNumConfigThreads++;
3734             else {
3735                 if( 0 == --gNumConfigThreads) {
3736 //                    IOLog("MATCH IDLE\n");
3737                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
3738                 }
3739             }
3740             IOUnlock( gJobsLock );
3741         }
3742
3743     } while( alive );
3744
3745     if( gIOKitDebug & kIOLogConfig)
3746         LOG("config(%p): terminating\n", OBFUSCATE(IOThreadSelf()) );
3747
3748     self->release();
3749 }
3750
3751 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
3752 {
3753     bool            wait;
3754     int             waitResult = THREAD_AWAKENED;
3755     bool            computeDeadline = true;
3756     AbsoluteTime    deadline;
3757
3758     IOLockLock( gJobsLock );
3759     do {
3760         wait = (0 != gNumConfigThreads);
3761         if( wait) {
3762             if( msToWait) {
3763                 if( computeDeadline ) {
3764                     clock_interval_to_deadline(
3765                           msToWait, kMillisecondScale, &deadline );
3766                     computeDeadline = false;
3767                 }
3768                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
3769                                                                 deadline, THREAD_UNINT );
3770                    } else {
3771                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
3772                                                                 THREAD_UNINT );
3773                 }
3774         }
3775     } while( wait && (waitResult != THREAD_TIMED_OUT));
3776         IOLockUnlock( gJobsLock );
3777
3778     if( waitResult == THREAD_TIMED_OUT)
3779         return( kIOReturnTimeout );
3780     else
3781         return( kIOReturnSuccess );
3782 }
3783
3784 void _IOServiceJob::pingConfig( _IOServiceJob * job )
3785 {
3786     int         count;
3787     bool        create;
3788
3789     assert( job );
3790
3791     IOTakeLock( gJobsLock );
3792
3793     gOutstandingJobs++;
3794     gJobs->setLastObject( job );
3795
3796     count = gNumWaitingThreads;
3797 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
3798
3799     create = (  (gOutstandingJobs > count)
3800                 && (gNumConfigThreads < kMaxConfigThreads) );
3801     if( create) {
3802         gNumConfigThreads++;
3803         gNumWaitingThreads++;
3804     }
3805
3806     IOUnlock( gJobsLock );
3807
3808     job->release();
3809
3810     if( create) {
3811         if( gIOKitDebug & kIOLogConfig)
3812             LOG("config(%d): creating\n", gNumConfigThreads - 1);
3813         _IOConfigThread::configThread();
3814     }
3815
3816     semaphore_signal( gJobsSemaphore );
3817 }
3818
3819 struct IOServiceMatchContext
3820 {
3821     OSDictionary * table;
3822     OSObject *     result;
3823     uint32_t       options;
3824     uint32_t       state;
3825     uint32_t       count;
3826     uint32_t       done;
3827 };
3828
3829 bool IOService::instanceMatch(const OSObject * entry, void * context)
3830 {
3831     IOServiceMatchContext * ctx = (typeof(ctx)) context;
3832     IOService *    service = (typeof(service)) entry;
3833     OSDictionary * table   = ctx->table;
3834     uint32_t       options = ctx->options;
3835     uint32_t       state   = ctx->state;
3836     uint32_t       done;
3837     bool           match;
3838
3839     done = 0;
3840     do
3841     {
3842         match = ((state == (state & service->__state[0]))
3843                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
3844         if (!match) break;
3845         ctx->count += table->getCount();
3846         match = service->matchInternal(table, options, &done);
3847         ctx->done += done;
3848     }
3849     while (false);
3850     if (!match)
3851         return (false);
3852
3853     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
3854     {
3855         service->retain();
3856         ctx->result = service;
3857         return (true);
3858     }
3859     else if (!ctx->result)
3860     {
3861         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
3862     }
3863     else
3864     {
3865         ((OSSet *)ctx->result)->setObject(service);
3866     }
3867     return (false);
3868 }
3869
3870 // internal - call with gNotificationLock
3871 OSObject * IOService::copyExistingServices( OSDictionary * matching,
3872                  IOOptionBits inState, IOOptionBits options )
3873 {
3874     OSObject *   current = 0;
3875     OSIterator * iter;
3876     IOService *  service;
3877     OSObject *   obj;
3878     OSString *   str;
3879
3880     if( !matching)
3881         return( 0 );
3882
3883 #if MATCH_DEBUG
3884     OSSerialize * s = OSSerialize::withCapacity(128);
3885     matching->serialize(s);
3886 #endif
3887
3888     if((obj = matching->getObject(gIOProviderClassKey))
3889       && gIOResourcesKey
3890       && gIOResourcesKey->isEqualTo(obj)
3891       && (service = gIOResources))
3892     {
3893         if( (inState == (service->__state[0] & inState))
3894           && (0 == (service->__state[0] & kIOServiceInactiveState))
3895           &&  service->matchPassive(matching, options))
3896         {
3897             if( options & kIONotifyOnce)
3898             {
3899                 service->retain();
3900                 current = service;
3901             }
3902             else
3903                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
3904         }
3905     }
3906     else
3907     {
3908         IOServiceMatchContext ctx;
3909         ctx.table   = matching;
3910         ctx.state   = inState;
3911         ctx.count   = 0;
3912         ctx.done    = 0;
3913         ctx.options = options;
3914         ctx.result  = 0;
3915
3916         if ((str = OSDynamicCast(OSString, obj)))
3917         {
3918             const OSSymbol * sym = OSSymbol::withString(str);
3919             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
3920             sym->release();
3921         }
3922         else
3923         {
3924             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
3925         }
3926
3927
3928         current = ctx.result;
3929
3930         options |= kIOServiceInternalDone | kIOServiceClassDone;
3931         if (current && (ctx.done != ctx.count))
3932         {
3933             OSSet *
3934             source = OSDynamicCast(OSSet, current);
3935             current = 0;
3936             while ((service = (IOService *) source->getAnyObject()))
3937             {
3938                 if (service->matchPassive(matching, options))
3939                 {
3940                     if( options & kIONotifyOnce)
3941                     {
3942                         service->retain();
3943                         current = service;
3944                         break;
3945                     }
3946                     if( current)
3947                     {
3948                         ((OSSet *)current)->setObject( service );
3949                     }
3950                     else
3951                     {
3952                         current = OSSet::withObjects(
3953                                         (const OSObject **) &service, 1, 1 );
3954                     }
3955                 }
3956                 source->removeObject(service);
3957             }
3958             source->release();
3959         }
3960     }
3961
3962 #if MATCH_DEBUG
3963     {
3964         OSObject * _current = 0;
3965
3966         iter = IORegistryIterator::iterateOver( gIOServicePlane,
3967                                             kIORegistryIterateRecursively );
3968         if( iter) {
3969             do {
3970                 iter->reset();
3971                 while( (service = (IOService *) iter->getNextObject())) {
3972                     if( (inState == (service->__state[0] & inState))
3973                     && (0 == (service->__state[0] & kIOServiceInactiveState))
3974                     &&  service->matchPassive(matching, 0)) {
3975
3976                         if( options & kIONotifyOnce) {
3977                             service->retain();
3978                             _current = service;
3979                             break;
3980                         }
3981                         if( _current)
3982                             ((OSSet *)_current)->setObject( service );
3983                         else
3984                             _current = OSSet::withObjects(
3985                                             (const OSObject **) &service, 1, 1 );
3986                     }
3987                 }
3988             } while( !service && !iter->isValid());
3989             iter->release();
3990         }
3991
3992
3993         if ( ((current != 0) != (_current != 0))
3994         || (current && _current && !current->isEqualTo(_current)))
3995         {
3996             OSSerialize * s1 = OSSerialize::withCapacity(128);
3997             OSSerialize * s2 = OSSerialize::withCapacity(128);
3998             current->serialize(s1);
3999             _current->serialize(s2);
4000             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", OBFUSCATE(current),
4001                 OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4002             s1->release();
4003             s2->release();
4004         }
4005
4006         if (_current) _current->release();
4007     }
4008
4009     s->release();
4010 #endif
4011
4012     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4013         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4014         current->release();
4015         current = iter;
4016     }
4017
4018     return( current );
4019 }
4020
4021 // public version
4022 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4023 {
4024     OSIterator *        iter;
4025
4026     // is a lock even needed?
4027     LOCKWRITENOTIFY();
4028
4029     iter = (OSIterator *) copyExistingServices( matching,
4030                                                 kIOServiceMatchedState );
4031
4032     UNLOCKNOTIFY();
4033
4034     return( iter );
4035 }
4036
4037 IOService * IOService::copyMatchingService( OSDictionary * matching )
4038 {
4039     IOService * service;
4040
4041     // is a lock even needed?
4042     LOCKWRITENOTIFY();
4043
4044     service = (IOService *) copyExistingServices( matching,
4045                                                 kIOServiceMatchedState, kIONotifyOnce );
4046
4047     UNLOCKNOTIFY();
4048
4049     return( service );
4050 }
4051
4052 struct _IOServiceMatchingNotificationHandlerRef
4053 {
4054     IOServiceNotificationHandler handler;
4055     void * ref;
4056 };
4057
4058 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4059                                                    IOService * newService,
4060                                                    IONotifier * notifier )
4061 {
4062     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4063 }
4064
4065 // internal - call with gNotificationLock
4066 IONotifier * IOService::setNotification(
4067             const OSSymbol * type, OSDictionary * matching,
4068             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4069             SInt32 priority )
4070 {
4071     _IOServiceNotifier * notify = 0;
4072     OSOrderedSet *      set;
4073
4074     if( !matching)
4075         return( 0 );
4076
4077     notify = new _IOServiceNotifier;
4078     if( notify && !notify->init()) {
4079         notify->release();
4080         notify = 0;
4081     }
4082
4083     if( notify) {
4084         notify->handler = handler;
4085         notify->target = target;
4086         notify->matching = matching;
4087         matching->retain();
4088         if (handler == &_IOServiceMatchingNotificationHandler)
4089         {
4090             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4091             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4092         }
4093         else
4094             notify->ref = ref;
4095         notify->priority = priority;
4096         notify->state = kIOServiceNotifyEnable;
4097         queue_init( &notify->handlerInvocations );
4098
4099         ////// queue
4100
4101         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4102             set = OSOrderedSet::withCapacity( 1,
4103                         IONotifyOrdering, 0 );
4104             if( set) {
4105                 gNotifications->setObject( type, set );
4106                 set->release();
4107             }
4108         }
4109         notify->whence = set;
4110         if( set)
4111             set->setObject( notify );
4112     }
4113
4114     return( notify );
4115 }
4116
4117 // internal - call with gNotificationLock
4118 IONotifier * IOService::doInstallNotification(
4119                         const OSSymbol * type, OSDictionary * matching,
4120                         IOServiceMatchingNotificationHandler handler,
4121                         void * target, void * ref,
4122                         SInt32 priority, OSIterator ** existing )
4123 {
4124     OSIterator *        exist;
4125     IONotifier *        notify;
4126     IOOptionBits        inState;
4127
4128     if( !matching)
4129         return( 0 );
4130
4131     if( type == gIOPublishNotification)
4132         inState = kIOServiceRegisteredState;
4133
4134     else if( type == gIOFirstPublishNotification)
4135         inState = kIOServiceFirstPublishState;
4136
4137     else if( (type == gIOMatchedNotification)
4138           || (type == gIOFirstMatchNotification))
4139         inState = kIOServiceMatchedState;
4140     else if( type == gIOTerminatedNotification)
4141         inState = 0;
4142     else
4143         return( 0 );
4144
4145     notify = setNotification( type, matching, handler, target, ref, priority );
4146
4147     if( inState)
4148         // get the current set
4149         exist = (OSIterator *) copyExistingServices( matching, inState );
4150     else
4151         exist = 0;
4152
4153     *existing = exist;
4154
4155     return( notify );
4156 }
4157
4158 #if !defined(__LP64__)
4159 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4160                                         IOServiceNotificationHandler handler,
4161                                         void * target, void * refCon,
4162                                         SInt32 priority, OSIterator ** existing )
4163 {
4164     IONotifier * result;
4165     _IOServiceMatchingNotificationHandlerRef ref;
4166     ref.handler = handler;
4167     ref.ref     = refCon;
4168
4169     result = (_IOServiceNotifier *) installNotification( type, matching,
4170                          &_IOServiceMatchingNotificationHandler,
4171                         target, &ref, priority, existing );
4172     if (result)
4173         matching->release();
4174
4175     return (result);
4176 }
4177 #endif /* !defined(__LP64__) */
4178
4179
4180 IONotifier * IOService::installNotification(
4181                         const OSSymbol * type, OSDictionary * matching,
4182                         IOServiceMatchingNotificationHandler handler,
4183                         void * target, void * ref,
4184                         SInt32 priority, OSIterator ** existing )
4185 {
4186     IONotifier * notify;
4187
4188     LOCKWRITENOTIFY();
4189
4190     notify = doInstallNotification( type, matching, handler, target, ref,
4191                 priority, existing );
4192
4193     UNLOCKNOTIFY();
4194
4195     return( notify );
4196 }
4197
4198 IONotifier * IOService::addNotification(
4199                         const OSSymbol * type, OSDictionary * matching,
4200                         IOServiceNotificationHandler handler,
4201                         void * target, void * refCon,
4202                         SInt32 priority )
4203 {
4204     IONotifier * result;
4205     _IOServiceMatchingNotificationHandlerRef ref;
4206
4207     ref.handler = handler;
4208     ref.ref     = refCon;
4209
4210     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4211                             target, &ref, priority);
4212
4213     if (result)
4214         matching->release();
4215
4216     return (result);
4217 }
4218
4219 IONotifier * IOService::addMatchingNotification(
4220                         const OSSymbol * type, OSDictionary * matching,
4221                         IOServiceMatchingNotificationHandler handler,
4222                         void * target, void * ref,
4223                         SInt32 priority )
4224 {
4225     OSIterator *                existing = NULL;
4226     _IOServiceNotifier *        notify;
4227     IOService *                 next;
4228
4229     notify = (_IOServiceNotifier *) installNotification( type, matching,
4230                 handler, target, ref, priority, &existing );
4231
4232     // send notifications for existing set
4233     if( existing) {
4234
4235         notify->retain();               // in case handler remove()s
4236         while( (next = (IOService *) existing->getNextObject())) {
4237
4238             next->lockForArbitration();
4239             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4240                 next->invokeNotifer( notify );
4241             next->unlockForArbitration();
4242         }
4243         notify->release();
4244         existing->release();
4245     }
4246
4247     return( notify );
4248 }
4249
4250 bool IOService::syncNotificationHandler(
4251                         void * /* target */, void * ref,
4252                         IOService * newService,
4253                         IONotifier * notifier )
4254 {
4255
4256     LOCKWRITENOTIFY();
4257     if (!*((IOService **) ref))
4258     {
4259         newService->retain();
4260         (*(IOService **) ref) = newService;
4261         WAKEUPNOTIFY(ref);
4262     }
4263     UNLOCKNOTIFY();
4264
4265     return( false );
4266 }
4267
4268 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4269                                                 uint64_t timeout)
4270 {
4271     IONotifier *        notify = 0;
4272     // priority doesn't help us much since we need a thread wakeup
4273     SInt32              priority = 0;
4274     IOService *         result;
4275
4276     if (!matching)
4277         return( 0 );
4278
4279     result = NULL;
4280
4281     LOCKWRITENOTIFY();
4282     do
4283     {
4284         result = (IOService *) copyExistingServices( matching,
4285                             kIOServiceMatchedState, kIONotifyOnce );
4286         if (result)
4287             break;
4288         notify = IOService::setNotification( gIOMatchedNotification, matching,
4289                     &IOService::syncNotificationHandler, (void *) 0,
4290                     &result, priority );
4291          if (!notify)
4292             break;
4293         if (UINT64_MAX != timeout)
4294         {
4295             AbsoluteTime deadline;
4296             nanoseconds_to_absolutetime(timeout, &deadline);
4297             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4298             SLEEPNOTIFYTO(&result, deadline);
4299         }
4300         else
4301         {
4302             SLEEPNOTIFY(&result);
4303         }
4304     }
4305     while( false );
4306
4307     UNLOCKNOTIFY();
4308
4309     if (notify)
4310         notify->remove();       // dequeues
4311
4312     return( result );
4313 }
4314
4315 IOService * IOService::waitForService( OSDictionary * matching,
4316                                         mach_timespec_t * timeout )
4317 {
4318     IOService * result;
4319     uint64_t    timeoutNS;
4320
4321     if (timeout)
4322     {
4323         timeoutNS = timeout->tv_sec;
4324         timeoutNS *= kSecondScale;
4325         timeoutNS += timeout->tv_nsec;
4326     }
4327     else
4328         timeoutNS = UINT64_MAX;
4329
4330     result = waitForMatchingService(matching, timeoutNS);
4331
4332     matching->release();
4333     if (result)
4334         result->release();
4335
4336     return (result);
4337 }
4338
4339 void IOService::deliverNotification( const OSSymbol * type,
4340                             IOOptionBits orNewState, IOOptionBits andNewState )
4341 {
4342     _IOServiceNotifier * notify;
4343     OSIterator *         iter;
4344     OSArray *            willSend = 0;
4345
4346     lockForArbitration();
4347
4348     if( (0 == (__state[0] & kIOServiceInactiveState))
4349      || (type == gIOTerminatedNotification)) {
4350
4351         LOCKREADNOTIFY();
4352
4353         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4354                     gNotifications->getObject( type ) );
4355
4356         if( iter) {
4357             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4358
4359                 if( matchPassive(notify->matching, 0)
4360                   && (kIOServiceNotifyEnable & notify->state)) {
4361                     if( 0 == willSend)
4362                         willSend = OSArray::withCapacity(8);
4363                     if( willSend)
4364                         willSend->setObject( notify );
4365                 }
4366             }
4367             iter->release();
4368         }
4369
4370         __state[0] = (__state[0] | orNewState) & andNewState;
4371
4372         UNLOCKNOTIFY();
4373     }
4374
4375     if( willSend) {
4376         for( unsigned int idx = 0;
4377              (notify = (_IOServiceNotifier *) willSend->getObject(idx));
4378              idx++) {
4379             invokeNotifer( notify );
4380         }
4381         willSend->release();
4382     }
4383     unlockForArbitration();
4384 }
4385
4386 IOOptionBits IOService::getState( void ) const
4387 {
4388     return( __state[0] );
4389 }
4390
4391 /*
4392  * Helpers to make matching objects for simple cases
4393  */
4394
4395 OSDictionary * IOService::serviceMatching( const OSString * name,
4396                         OSDictionary * table )
4397 {
4398
4399     const OSString *    str;
4400
4401     str = OSSymbol::withString(name);
4402     if( !str)
4403         return( 0 );
4404
4405     if( !table)
4406         table = OSDictionary::withCapacity( 2 );
4407     if( table)
4408         table->setObject(gIOProviderClassKey, (OSObject *)str );
4409     str->release();
4410
4411     return( table );
4412 }
4413
4414 OSDictionary * IOService::serviceMatching( const char * name,
4415                         OSDictionary * table )
4416 {
4417     const OSString *    str;
4418
4419     str = OSSymbol::withCString( name );
4420     if( !str)
4421         return( 0 );
4422
4423     table = serviceMatching( str, table );
4424     str->release();
4425     return( table );
4426 }
4427
4428 OSDictionary * IOService::nameMatching( const OSString * name,
4429                         OSDictionary * table )
4430 {
4431     if( !table)
4432         table = OSDictionary::withCapacity( 2 );
4433     if( table)
4434         table->setObject( gIONameMatchKey, (OSObject *)name );
4435
4436     return( table );
4437 }
4438
4439 OSDictionary * IOService::nameMatching( const char * name,
4440                         OSDictionary * table )
4441 {
4442     const OSString *    str;
4443
4444     str = OSSymbol::withCString( name );
4445     if( !str)
4446         return( 0 );
4447
4448     table = nameMatching( str, table );
4449     str->release();
4450     return( table );
4451 }
4452
4453 OSDictionary * IOService::resourceMatching( const OSString * str,
4454                         OSDictionary * table )
4455 {
4456     table = serviceMatching( gIOResourcesKey, table );
4457     if( table)
4458         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4459
4460     return( table );
4461 }
4462
4463 OSDictionary * IOService::resourceMatching( const char * name,
4464                         OSDictionary * table )
4465 {
4466     const OSSymbol *    str;
4467
4468     str = OSSymbol::withCString( name );
4469     if( !str)
4470         return( 0 );
4471
4472     table = resourceMatching( str, table );
4473     str->release();
4474
4475     return( table );
4476 }
4477
4478 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4479                         OSDictionary * table )
4480 {
4481     OSDictionary * properties;
4482
4483     properties = OSDictionary::withCapacity( 2 );
4484     if( !properties)
4485         return( 0 );
4486     properties->setObject( key, value );
4487
4488     if( !table)
4489         table = OSDictionary::withCapacity( 2 );
4490     if( table)
4491         table->setObject( gIOPropertyMatchKey, properties );
4492
4493     properties->release();
4494
4495     return( table );
4496 }
4497
4498 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4499                         OSDictionary * table )
4500 {
4501     OSNumber *     num;
4502
4503     num = OSNumber::withNumber( entryID, 64 );
4504     if( !num)
4505         return( 0 );
4506
4507     if( !table)
4508         table = OSDictionary::withCapacity( 2 );
4509     if( table)
4510         table->setObject( gIORegistryEntryIDKey, num );
4511
4512     if (num)
4513         num->release();
4514
4515     return( table );
4516 }
4517
4518
4519 /*
4520  * _IOServiceNotifier
4521  */
4522
4523 // wait for all threads, other than the current one,
4524 //  to exit the handler
4525
4526 void _IOServiceNotifier::wait()
4527 {
4528     _IOServiceNotifierInvocation * next;
4529     bool doWait;
4530
4531     do {
4532         doWait = false;
4533         queue_iterate( &handlerInvocations, next,
4534                         _IOServiceNotifierInvocation *, link) {
4535             if( next->thread != current_thread() ) {
4536                 doWait = true;
4537                 break;
4538             }
4539         }
4540         if( doWait) {
4541             state |= kIOServiceNotifyWaiter;
4542             SLEEPNOTIFY(this);
4543         }
4544
4545     } while( doWait );
4546 }
4547
4548 void _IOServiceNotifier::free()
4549 {
4550     assert( queue_empty( &handlerInvocations ));
4551     OSObject::free();
4552 }
4553
4554 void _IOServiceNotifier::remove()
4555 {
4556     LOCKWRITENOTIFY();
4557
4558     if( whence) {
4559         whence->removeObject( (OSObject *) this );
4560         whence = 0;
4561     }
4562     if( matching) {
4563         matching->release();
4564         matching = 0;
4565     }
4566
4567     state &= ~kIOServiceNotifyEnable;
4568
4569     wait();
4570
4571     UNLOCKNOTIFY();
4572
4573     release();
4574 }
4575
4576 bool _IOServiceNotifier::disable()
4577 {
4578     bool        ret;
4579
4580     LOCKWRITENOTIFY();
4581
4582     ret = (0 != (kIOServiceNotifyEnable & state));
4583     state &= ~kIOServiceNotifyEnable;
4584     if( ret)
4585         wait();
4586
4587     UNLOCKNOTIFY();
4588
4589     return( ret );
4590 }
4591
4592 void _IOServiceNotifier::enable( bool was )
4593 {
4594     LOCKWRITENOTIFY();
4595     if( was)
4596         state |= kIOServiceNotifyEnable;
4597     else
4598         state &= ~kIOServiceNotifyEnable;
4599     UNLOCKNOTIFY();
4600 }
4601
4602 /*
4603  * IOResources
4604  */
4605
4606 IOService * IOResources::resources( void )
4607 {
4608     IOResources *       inst;
4609
4610     inst = new IOResources;
4611     if( inst && !inst->init()) {
4612         inst->release();
4613         inst = 0;
4614     }
4615
4616     return( inst );
4617 }
4618
4619 bool IOResources::init( OSDictionary * dictionary )
4620 {
4621     // Do super init first
4622     if ( !super::init() )
4623         return false;
4624
4625     // Allow PAL layer to publish a value
4626     const char *property_name;
4627     int property_value;
4628
4629     pal_get_resource_property( &property_name, &property_value );
4630
4631     if( property_name ) {
4632         OSNumber *num;
4633         const OSSymbol *        sym;
4634
4635         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
4636             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
4637                 this->setProperty( sym, num );
4638                 sym->release();
4639             }
4640             num->release();
4641         }
4642     }
4643
4644     return true;
4645 }
4646
4647 IOWorkLoop * IOResources::getWorkLoop() const
4648 {
4649     // If we are the resource root
4650     // then use the platform's workloop
4651     if (this == (IOResources *) gIOResources)
4652         return getPlatform()->getWorkLoop();
4653     else
4654         return IOService::getWorkLoop();
4655 }
4656
4657 bool IOResources::matchPropertyTable( OSDictionary * table )
4658 {
4659     OSObject *          prop;
4660     OSString *          str;
4661     OSSet *             set;
4662     OSIterator *        iter;
4663     bool                ok = true;
4664
4665     prop = table->getObject( gIOResourceMatchKey );
4666     str = OSDynamicCast( OSString, prop );
4667     if( str)
4668         ok = (0 != getProperty( str ));
4669
4670     else if( (set = OSDynamicCast( OSSet, prop))) {
4671
4672         iter = OSCollectionIterator::withCollection( set );
4673         ok = (iter != 0);
4674         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
4675             ok = (0 != getProperty( str ));
4676
4677         if( iter)
4678             iter->release();
4679     }
4680
4681     return( ok );
4682 }
4683
4684 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
4685 {
4686     IOService::updateConsoleUsers(NULL, 0);
4687 }
4688
4689 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
4690 {
4691     IORegistryEntry * regEntry;
4692     OSObject *        locked = kOSBooleanFalse;
4693     uint32_t          idx;
4694     bool              publish;
4695     OSDictionary *    user;
4696     static IOMessage  sSystemPower;
4697
4698     regEntry = IORegistryEntry::getRegistryRoot();
4699
4700     if (!gIOChosenEntry)
4701         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
4702
4703     IOLockLock(gIOConsoleUsersLock);
4704
4705     if (systemMessage)
4706     {
4707         sSystemPower = systemMessage;
4708 #if HIBERNATION
4709         if ((kIOMessageSystemHasPoweredOn == systemMessage) && IOHibernateWasScreenLocked())
4710         {
4711             locked = kOSBooleanTrue;
4712         }
4713 #endif /* HIBERNATION */
4714     }
4715
4716     if (consoleUsers)
4717     {
4718         OSNumber * num = 0;
4719         gIOConsoleLoggedIn = false;
4720         for (idx = 0;
4721               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
4722               idx++)
4723         {
4724             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
4725                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
4726             if (!num)
4727             {
4728                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
4729             }
4730         }
4731         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
4732     }
4733
4734     if (!gIOConsoleLoggedIn
4735      || (kIOMessageSystemWillSleep == sSystemPower)
4736      || (kIOMessageSystemPagingOff == sSystemPower))
4737     {
4738         locked = kOSBooleanTrue;
4739     }
4740     else if (gIOConsoleLockTime)
4741     {
4742         clock_sec_t  now;
4743         clock_usec_t microsecs;
4744
4745         clock_get_calendar_microtime(&now, &microsecs);
4746         if (gIOConsoleLockTime > now)
4747         {
4748             AbsoluteTime deadline;
4749             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
4750             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
4751         }
4752         else
4753         {
4754             locked = kOSBooleanTrue;
4755         }
4756     }
4757
4758     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
4759     if (publish)
4760     {
4761         regEntry->setProperty(gIOConsoleLockedKey, locked);
4762         if (consoleUsers)
4763         {
4764             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
4765         }
4766         OSIncrementAtomic( &gIOConsoleUsersSeed );
4767     }
4768
4769 #if HIBERNATION
4770     if (gIOChosenEntry)
4771     {
4772         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
4773         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
4774         else                          gIOScreenLockState = kIOScreenLockNoLock;
4775         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
4776     }
4777 #endif /* HIBERNATION */
4778
4779     IOLockUnlock(gIOConsoleUsersLock);
4780
4781     if (publish)
4782     {
4783         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
4784
4785         MessageClientsContext context;
4786
4787         context.service  = getServiceRoot();
4788         context.type     = kIOMessageConsoleSecurityChange;
4789         context.argument = (void *) regEntry;
4790         context.argSize  = 0;
4791
4792         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
4793                                  &messageClientsApplier, &context );
4794     }
4795 }
4796
4797 IOReturn IOResources::setProperties( OSObject * properties )
4798 {
4799     IOReturn                    err;
4800     const OSSymbol *            key;
4801     OSDictionary *              dict;
4802     OSCollectionIterator *      iter;
4803
4804     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
4805     if ( kIOReturnSuccess != err)
4806         return( err );
4807
4808     dict = OSDynamicCast(OSDictionary, properties);
4809     if( 0 == dict)
4810         return( kIOReturnBadArgument);
4811
4812     iter = OSCollectionIterator::withCollection( dict);
4813     if( 0 == iter)
4814         return( kIOReturnBadArgument);
4815
4816     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
4817     {
4818         if (gIOConsoleUsersKey == key) do
4819         {
4820             OSArray * consoleUsers;
4821             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
4822             if (!consoleUsers)
4823                 continue;
4824             IOService::updateConsoleUsers(consoleUsers, 0);
4825         }
4826         while (false);
4827
4828         publishResource( key, dict->getObject(key) );
4829     }
4830
4831     iter->release();
4832
4833     return( kIOReturnSuccess );
4834 }
4835
4836 /*
4837  * Helpers for matching dictionaries.
4838  * Keys existing in matching are checked in properties.
4839  * Keys may be a string or OSCollection of IOStrings
4840  */
4841
4842 bool IOService::compareProperty( OSDictionary * matching,
4843                                  const char *   key )
4844 {
4845     OSObject *  value;
4846     bool        ok;
4847
4848     value = matching->getObject( key );
4849     if( value)
4850         ok = value->isEqualTo( getProperty( key ));
4851     else
4852         ok = true;
4853
4854     return( ok );
4855 }
4856
4857
4858 bool IOService::compareProperty( OSDictionary *   matching,
4859                                  const OSString * key )
4860 {
4861     OSObject *  value;
4862     bool        ok;
4863
4864     value = matching->getObject( key );
4865     if( value)
4866         ok = value->isEqualTo( getProperty( key ));
4867     else
4868         ok = true;
4869
4870     return( ok );
4871 }
4872
4873 bool IOService::compareProperties( OSDictionary * matching,
4874                                    OSCollection * keys )
4875 {
4876     OSCollectionIterator *      iter;
4877     const OSString *            key;
4878     bool                        ok = true;
4879
4880     if( !matching || !keys)
4881         return( false );
4882
4883     iter = OSCollectionIterator::withCollection( keys );
4884
4885     if( iter) {
4886         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
4887             ok = compareProperty( matching, key );
4888
4889         iter->release();
4890     }
4891     keys->release();    // !! consume a ref !!
4892
4893     return( ok );
4894 }
4895
4896 /* Helper to add a location matching dict to the table */
4897
4898 OSDictionary * IOService::addLocation( OSDictionary * table )
4899 {
4900     OSDictionary *      dict;
4901
4902     if( !table)
4903         return( 0 );
4904
4905     dict = OSDictionary::withCapacity( 1 );
4906     if( dict) {
4907         table->setObject( gIOLocationMatchKey, dict );
4908         dict->release();
4909     }
4910
4911     return( dict );
4912 }
4913
4914 /*
4915  * Go looking for a provider to match a location dict.
4916  */
4917
4918 IOService * IOService::matchLocation( IOService * /* client */ )
4919 {
4920     IOService * parent;
4921
4922     parent = getProvider();
4923
4924     if( parent)
4925         parent = parent->matchLocation( this );
4926
4927     return( parent );
4928 }
4929
4930 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
4931 {
4932     OSString *          matched;
4933     OSObject *          obj;
4934     OSString *          str;
4935     IORegistryEntry *   entry;
4936     OSNumber *          num;
4937     bool                match = true;
4938     bool                changesOK = (0 != (kIOServiceChangesOK & options));
4939     uint32_t            count;
4940     uint32_t            done;
4941
4942     do
4943     {
4944         count = table->getCount();
4945         done = 0;
4946         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
4947         if (str) {
4948             done++;
4949             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
4950 #if MATCH_DEBUG
4951             match = (0 != metaCast( str ));
4952             if ((kIOServiceClassDone & options) && !match) panic("classDone");
4953 #endif
4954             if ((!match) || (done == count)) break;
4955         }
4956
4957         obj = table->getObject( gIONameMatchKey );
4958         if( obj) {
4959             done++;
4960             match = compareNames( obj, changesOK ? &matched : 0 );
4961             if (!match) break;
4962             if( changesOK && matched) {
4963                 // leave a hint as to which name matched
4964                 table->setObject( gIONameMatchedKey, matched );
4965                 matched->release();
4966             }
4967             if (done == count) break;
4968         }
4969
4970         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
4971         if (str)
4972         {
4973             const OSSymbol * sym;
4974             done++;
4975             match = false;
4976             sym = copyLocation();
4977             if (sym) {
4978                 match = sym->isEqualTo( str );
4979                 sym->release();
4980             }
4981             if ((!match) || (done == count)) break;
4982         }
4983
4984         obj = table->getObject( gIOPropertyMatchKey );
4985         if( obj)
4986         {
4987             OSDictionary * dict;
4988             OSDictionary * nextDict;
4989             OSIterator *   iter;
4990             done++;
4991             match = false;
4992             dict = dictionaryWithProperties();
4993             if( dict) {
4994                 nextDict = OSDynamicCast( OSDictionary, obj);
4995                 if( nextDict)
4996                     iter = 0;
4997                 else
4998                     iter = OSCollectionIterator::withCollection(
4999                                 OSDynamicCast(OSCollection, obj));
5000
5001                 while( nextDict
5002                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5003                                             iter->getNextObject()))))) {
5004                     match = dict->isEqualTo( nextDict, nextDict);
5005                     if( match)
5006                         break;
5007                     nextDict = 0;
5008                 }
5009                 dict->release();
5010                 if( iter)
5011                     iter->release();
5012             }
5013             if ((!match) || (done == count)) break;
5014         }
5015
5016         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5017         if( str) {
5018             done++;
5019             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5020             match = (this == entry);
5021             if( entry)
5022                 entry->release();
5023             if ((!match) || (done == count)) break;
5024         }
5025
5026         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5027         if (num) {
5028             done++;
5029             match = (getRegistryEntryID() == num->unsigned64BitValue());
5030             if ((!match) || (done == count)) break;
5031         }
5032
5033         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5034         if( num)
5035         {
5036             OSIterator *        iter;
5037             IOService *         service = 0;
5038             UInt32              serviceCount = 0;
5039
5040             done++;
5041             iter = getClientIterator();
5042             if( iter) {
5043                 while( (service = (IOService *) iter->getNextObject())) {
5044                     if( kIOServiceInactiveState & service->__state[0])
5045                         continue;
5046                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5047                         continue;
5048                     ++serviceCount;
5049                 }
5050                 iter->release();
5051             }
5052             match = (serviceCount == num->unsigned32BitValue());
5053             if ((!match) || (done == count)) break;
5054         }
5055
5056 #define propMatch(key)                                  \
5057         obj = table->getObject(key);                    \
5058         if (obj)                                        \
5059         {                                               \
5060             OSObject * prop;                            \
5061             done++;                                     \
5062             prop = copyProperty(key);                   \
5063             match = obj->isEqualTo(prop);               \
5064             if (prop) prop->release();                  \
5065             if ((!match) || (done == count)) break;     \
5066         }
5067         propMatch(kIOBSDNameKey)
5068         propMatch(kIOBSDMajorKey)
5069         propMatch(kIOBSDMinorKey)
5070         propMatch(kIOBSDUnitKey)
5071 #undef propMatch
5072     }
5073     while (false);
5074
5075     if (did) *did = done;
5076     return (match);
5077 }
5078
5079 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5080 {
5081     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5082 }
5083
5084 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5085 {
5086     IOService *         where;
5087     OSDictionary *      nextTable;
5088     SInt32              score;
5089     OSNumber *          newPri;
5090     bool                match = true;
5091     bool                matchParent = false;
5092     uint32_t            count;
5093     uint32_t            done;
5094
5095     assert( table );
5096
5097 #if MATCH_DEBUG
5098     OSDictionary * root = table;
5099 #endif
5100
5101     where = this;
5102     do
5103     {
5104         do
5105         {
5106             count = table->getCount();
5107             if (!(kIOServiceInternalDone & options))
5108             {
5109                 match = where->matchInternal(table, options, &done);
5110                 // don't call family if we've done all the entries in the table
5111                 if ((!match) || (done == count)) break;
5112             }
5113
5114             // pass in score from property table
5115             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5116
5117             // do family specific matching
5118             match = where->matchPropertyTable( table, &score );
5119
5120             if( !match) {
5121 #if IOMATCHDEBUG
5122                 if( kIOLogMatch & getDebugFlags( table ))
5123                     LOG("%s: family specific matching fails\n", where->getName());
5124 #endif
5125                 break;
5126             }
5127
5128             if (kIOServiceChangesOK & options) {
5129                 // save the score
5130                 newPri = OSNumber::withNumber( score, 32 );
5131                 if( newPri) {
5132                     table->setObject( gIOProbeScoreKey, newPri );
5133                     newPri->release();
5134                 }
5135             }
5136
5137             options = 0;
5138             matchParent = false;
5139
5140             nextTable = OSDynamicCast(OSDictionary,
5141                   table->getObject( gIOParentMatchKey ));
5142             if( nextTable) {
5143                 // look for a matching entry anywhere up to root
5144                 match = false;
5145                 matchParent = true;
5146                 table = nextTable;
5147                 break;
5148             }
5149
5150             table = OSDynamicCast(OSDictionary,
5151                     table->getObject( gIOLocationMatchKey ));
5152             if (table) {
5153                 // look for a matching entry at matchLocation()
5154                 match = false;
5155                 where = where->getProvider();
5156                 if (where && (where = where->matchLocation(where))) continue;
5157             }
5158             break;
5159         }
5160         while (true);
5161     }
5162     while( matchParent && (!match) && (where = where->getProvider()) );
5163
5164 #if MATCH_DEBUG
5165     if (where != this)
5166     {
5167         OSSerialize * s = OSSerialize::withCapacity(128);
5168         root->serialize(s);
5169         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5170         s->release();
5171     }
5172 #endif
5173
5174     return( match );
5175 }
5176
5177
5178 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5179                                     UInt32 type,  OSDictionary * properties,
5180                                     IOUserClient ** handler )
5181 {
5182     const OSSymbol *userClientClass = 0;
5183     IOUserClient *client;
5184     OSObject *temp;
5185
5186     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5187         return kIOReturnSuccess;
5188
5189     // First try my own properties for a user client class name
5190     temp = getProperty(gIOUserClientClassKey);
5191     if (temp) {
5192         if (OSDynamicCast(OSSymbol, temp))
5193             userClientClass = (const OSSymbol *) temp;
5194         else if (OSDynamicCast(OSString, temp)) {
5195             userClientClass = OSSymbol::withString((OSString *) temp);
5196             if (userClientClass)
5197                 setProperty(kIOUserClientClassKey,
5198                             (OSObject *) userClientClass);
5199         }
5200     }
5201
5202     // Didn't find one so lets just bomb out now without further ado.
5203     if (!userClientClass)
5204         return kIOReturnUnsupported;
5205
5206     // This reference is consumed by the IOServiceOpen call
5207     temp = OSMetaClass::allocClassWithName(userClientClass);
5208     if (!temp)
5209         return kIOReturnNoMemory;
5210
5211     if (OSDynamicCast(IOUserClient, temp))
5212         client = (IOUserClient *) temp;
5213     else {
5214         temp->release();
5215         return kIOReturnUnsupported;
5216     }
5217
5218     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5219         client->release();
5220         return kIOReturnBadArgument;
5221     }
5222
5223     if ( !client->attach(this) ) {
5224         client->release();
5225         return kIOReturnUnsupported;
5226     }
5227
5228     if ( !client->start(this) ) {
5229         client->detach(this);
5230         client->release();
5231         return kIOReturnUnsupported;
5232     }
5233
5234     *handler = client;
5235     return kIOReturnSuccess;
5236 }
5237
5238 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5239                                     UInt32 type, IOUserClient ** handler )
5240 {
5241     return( kIOReturnUnsupported );
5242 }
5243
5244 IOReturn IOService::requestProbe( IOOptionBits options )
5245 {
5246     return( kIOReturnUnsupported);
5247 }
5248
5249 /*
5250  * Convert an IOReturn to text. Subclasses which add additional
5251  * IOReturn's should override this method and call
5252  * super::stringFromReturn if the desired value is not found.
5253  */
5254
5255 const char * IOService::stringFromReturn( IOReturn rtn )
5256 {
5257     static const IONamedValue IOReturn_values[] = {
5258         {kIOReturnSuccess,          "success"                           },
5259         {kIOReturnError,            "general error"                     },
5260         {kIOReturnNoMemory,         "memory allocation error"           },
5261         {kIOReturnNoResources,      "resource shortage"                 },
5262         {kIOReturnIPCError,         "Mach IPC failure"                  },
5263         {kIOReturnNoDevice,         "no such device"                    },
5264         {kIOReturnNotPrivileged,    "privilege violation"               },
5265         {kIOReturnBadArgument,      "invalid argument"                  },
5266         {kIOReturnLockedRead,       "device is read locked"             },
5267         {kIOReturnLockedWrite,      "device is write locked"            },
5268         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5269         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5270         {kIOReturnUnsupported,      "unsupported function"              },
5271         {kIOReturnVMError,          "virtual memory error"              },
5272         {kIOReturnInternalError,    "internal driver error"             },
5273         {kIOReturnIOError,          "I/O error"                         },
5274         {kIOReturnCannotLock,       "cannot acquire lock"               },
5275         {kIOReturnNotOpen,          "device is not open"                },
5276         {kIOReturnNotReadable,      "device is not readable"            },
5277         {kIOReturnNotWritable,      "device is not writeable"           },
5278         {kIOReturnNotAligned,       "alignment error"                   },
5279         {kIOReturnBadMedia,         "media error"                       },
5280         {kIOReturnStillOpen,        "device is still open"              },
5281         {kIOReturnRLDError,         "rld failure"                       },
5282         {kIOReturnDMAError,         "DMA failure"                       },
5283         {kIOReturnBusy,             "device is busy"                    },
5284         {kIOReturnTimeout,          "I/O timeout"                       },
5285         {kIOReturnOffline,          "device is offline"                 },
5286         {kIOReturnNotReady,         "device is not ready"               },
5287         {kIOReturnNotAttached,      "device/channel is not attached"    },
5288         {kIOReturnNoChannels,       "no DMA channels available"         },
5289         {kIOReturnNoSpace,          "no space for data"                 },
5290         {kIOReturnPortExists,       "device port already exists"        },
5291         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5292         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5293         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5294         {kIOReturnMessageTooLarge,  "message is too large"              },
5295         {kIOReturnNotPermitted,     "operation is not permitted"        },
5296         {kIOReturnNoPower,          "device is without power"           },
5297         {kIOReturnNoMedia,          "media is not present"              },
5298         {kIOReturnUnformattedMedia, "media is not formatted"            },
5299         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5300         {kIOReturnUnderrun,         "data underrun"                     },
5301         {kIOReturnOverrun,          "data overrun"                      },
5302         {kIOReturnDeviceError,      "device error"                      },
5303         {kIOReturnNoCompletion,     "no completion routine"             },
5304         {kIOReturnAborted,          "operation was aborted"             },
5305         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5306         {kIOReturnNotResponding,    "device is not responding"          },
5307         {kIOReturnInvalid,          "unanticipated driver error"        },
5308         {0,                         NULL                                }
5309     };
5310
5311     return IOFindNameForValue(rtn, IOReturn_values);
5312 }
5313
5314 /*
5315  * Convert an IOReturn to an errno.
5316  */
5317 int IOService::errnoFromReturn( IOReturn rtn )
5318 {
5319     if (unix_err(err_get_code(rtn)) == rtn)
5320         return err_get_code(rtn);
5321
5322     switch(rtn) {
5323         // (obvious match)
5324         case kIOReturnSuccess:
5325             return(0);
5326         case kIOReturnNoMemory:
5327             return(ENOMEM);
5328         case kIOReturnNoDevice:
5329             return(ENXIO);
5330         case kIOReturnVMError:
5331             return(EFAULT);
5332         case kIOReturnNotPermitted:
5333             return(EPERM);
5334         case kIOReturnNotPrivileged:
5335             return(EACCES);
5336         case kIOReturnIOError:
5337             return(EIO);
5338         case kIOReturnNotWritable:
5339             return(EROFS);
5340         case kIOReturnBadArgument:
5341             return(EINVAL);
5342         case kIOReturnUnsupported:
5343             return(ENOTSUP);
5344         case kIOReturnBusy:
5345             return(EBUSY);
5346         case kIOReturnNoPower:
5347             return(EPWROFF);
5348         case kIOReturnDeviceError:
5349             return(EDEVERR);
5350         case kIOReturnTimeout:
5351             return(ETIMEDOUT);
5352         case kIOReturnMessageTooLarge:
5353             return(EMSGSIZE);
5354         case kIOReturnNoSpace:
5355             return(ENOSPC);
5356         case kIOReturnCannotLock:
5357             return(ENOLCK);
5358
5359         // (best match)
5360         case kIOReturnBadMessageID:
5361         case kIOReturnNoCompletion:
5362         case kIOReturnNotAligned:
5363             return(EINVAL);
5364         case kIOReturnNotReady:
5365             return(EBUSY);
5366         case kIOReturnRLDError:
5367             return(EBADMACHO);
5368         case kIOReturnPortExists:
5369         case kIOReturnStillOpen:
5370             return(EEXIST);
5371         case kIOReturnExclusiveAccess:
5372         case kIOReturnLockedRead:
5373         case kIOReturnLockedWrite:
5374         case kIOReturnNotOpen:
5375         case kIOReturnNotReadable:
5376             return(EACCES);
5377         case kIOReturnCannotWire:
5378         case kIOReturnNoResources:
5379             return(ENOMEM);
5380         case kIOReturnAborted:
5381         case kIOReturnOffline:
5382         case kIOReturnNotResponding:
5383             return(EBUSY);
5384         case kIOReturnBadMedia:
5385         case kIOReturnNoMedia:
5386         case kIOReturnNotAttached:
5387         case kIOReturnUnformattedMedia:
5388             return(ENXIO); // (media error)
5389         case kIOReturnDMAError:
5390         case kIOReturnOverrun:
5391         case kIOReturnUnderrun:
5392             return(EIO); // (transfer error)
5393         case kIOReturnNoBandwidth:
5394         case kIOReturnNoChannels:
5395         case kIOReturnNoFrames:
5396         case kIOReturnNoInterrupt:
5397             return(EIO); // (hardware error)
5398         case kIOReturnError:
5399         case kIOReturnInternalError:
5400         case kIOReturnInvalid:
5401             return(EIO); // (generic error)
5402         case kIOReturnIPCError:
5403             return(EIO); // (ipc error)
5404         default:
5405             return(EIO); // (all other errors)
5406     }
5407 }
5408
5409 IOReturn IOService::message( UInt32 type, IOService * provider,
5410                                 void * argument )
5411 {
5412     /*
5413      * Generic entry point for calls from the provider.  A return value of
5414      * kIOReturnSuccess indicates that the message was received, and where
5415      * applicable, that it was successful.
5416      */
5417
5418     return kIOReturnUnsupported;
5419 }
5420
5421 /*
5422  * Device memory
5423  */
5424
5425 IOItemCount IOService::getDeviceMemoryCount( void )
5426 {
5427     OSArray *           array;
5428     IOItemCount         count;
5429
5430     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5431     if( array)
5432         count = array->getCount();
5433     else
5434         count = 0;
5435
5436     return( count);
5437 }
5438
5439 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
5440 {
5441     OSArray *           array;
5442     IODeviceMemory *    range;
5443
5444     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5445     if( array)
5446         range = (IODeviceMemory *) array->getObject( index );
5447     else
5448         range = 0;
5449
5450     return( range);
5451 }
5452
5453 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
5454                                                 IOOptionBits options )
5455 {
5456     IODeviceMemory *    range;
5457     IOMemoryMap *       map;
5458
5459     range = getDeviceMemoryWithIndex( index );
5460     if( range)
5461         map = range->map( options );
5462     else
5463         map = 0;
5464
5465     return( map );
5466 }
5467
5468 OSArray * IOService::getDeviceMemory( void )
5469 {
5470     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
5471 }
5472
5473
5474 void IOService::setDeviceMemory( OSArray * array )
5475 {
5476     setProperty( gIODeviceMemoryKey, array);
5477 }
5478
5479 /*
5480  * For machines where the transfers on an I/O bus can stall because
5481  * the CPU is in an idle mode, These APIs allow a driver to specify
5482  * the maximum bus stall that they can handle.  0 indicates no limit.
5483  */
5484 void IOService::
5485 setCPUSnoopDelay(UInt32 __unused ns)
5486 {
5487 #if defined(__i386__) || defined(__x86_64__)
5488     ml_set_maxsnoop(ns);
5489 #endif /* defined(__i386__) || defined(__x86_64__) */
5490 }
5491
5492 UInt32 IOService::
5493 getCPUSnoopDelay()
5494 {
5495 #if defined(__i386__) || defined(__x86_64__)
5496     return ml_get_maxsnoop();
5497 #else
5498     return 0;
5499 #endif /* defined(__i386__) || defined(__x86_64__) */
5500 }
5501
5502 #if defined(__i386__) || defined(__x86_64__)
5503 static void
5504 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
5505 {
5506     static const UInt kNoReplace = -1U; // Must be an illegal index
5507     UInt replace = kNoReplace;
5508     bool setCpuDelay = false;
5509
5510     IORecursiveLockLock(sCpuDelayLock);
5511
5512     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5513     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5514     IOService * holder = NULL;
5515
5516     if (ns) {
5517         const CpuDelayEntry ne = {service, ns, delayType};
5518         holder = service;
5519         // Set maximum delay.
5520         for (UInt i = 0; i < count; i++) {
5521             IOService *thisService = entries[i].fService;
5522             bool sameType = (delayType == entries[i].fDelayType);
5523             if ((service == thisService) && sameType)
5524                 replace = i;
5525             else if (!thisService) {
5526                 if (kNoReplace == replace)
5527                     replace = i;
5528             }
5529             else if (sameType) {
5530                 const UInt32 thisMax = entries[i].fMaxDelay;
5531                 if (thisMax < ns)
5532                 {
5533                     ns = thisMax;
5534                     holder = thisService;
5535                 }
5536             }
5537         }
5538
5539         setCpuDelay = true;
5540         if (kNoReplace == replace)
5541             sCpuDelayData->appendBytes(&ne, sizeof(ne));
5542         else
5543             entries[replace] = ne;
5544     }
5545     else {
5546         ns = -1U;       // Set to max unsigned, i.e. no restriction
5547
5548         for (UInt i = 0; i < count; i++) {
5549             // Clear a maximum delay.
5550             IOService *thisService = entries[i].fService;
5551             if (thisService && (delayType == entries[i].fDelayType)) {
5552                 UInt32 thisMax = entries[i].fMaxDelay;
5553                 if (service == thisService)
5554                     replace = i;
5555                 else if (thisMax < ns) {
5556                     ns = thisMax;
5557                     holder = thisService;
5558                 }
5559             }
5560         }
5561
5562         // Check if entry found
5563         if (kNoReplace != replace) {
5564             entries[replace].fService = 0;      // Null the entry
5565             setCpuDelay = true;
5566         }
5567     }
5568
5569     if (setCpuDelay)
5570     {
5571         // Must be safe to call from locked context
5572         if (delayType == kCpuDelayBusStall)
5573         {
5574             ml_set_maxbusdelay(ns);
5575         }
5576         else if (delayType == kCpuDelayInterrupt)
5577         {
5578             ml_set_maxintdelay(ns);
5579         }
5580         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
5581         sCPULatencySet   [delayType]->setValue(ns);
5582
5583         OSArray * handlers = sCpuLatencyHandlers[delayType];
5584         IOService * target;
5585         if (handlers) for (unsigned int idx = 0;
5586                             (target = (IOService *) handlers->getObject(idx));
5587                             idx++)
5588         {
5589             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5590                                             (void *) (uintptr_t) ns, holder,
5591                                             NULL, NULL);
5592         }
5593     }
5594
5595     IORecursiveLockUnlock(sCpuDelayLock);
5596 }
5597
5598 static IOReturn
5599 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
5600 {
5601     IOReturn result = kIOReturnNotFound;
5602     OSArray * array;
5603     unsigned int idx;
5604
5605     IORecursiveLockLock(sCpuDelayLock);
5606
5607     do
5608     {
5609         if (enable && !sCpuLatencyHandlers[delayType])
5610             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
5611         array = sCpuLatencyHandlers[delayType];
5612         if (!array)
5613             break;
5614         idx = array->getNextIndexOfObject(target, 0);
5615         if (!enable)
5616         {
5617             if (-1U != idx)
5618             {
5619                 array->removeObject(idx);
5620                 result = kIOReturnSuccess;
5621             }
5622         }
5623         else
5624         {
5625             if (-1U != idx) {
5626                 result = kIOReturnExclusiveAccess;
5627                 break;
5628             }
5629             array->setObject(target);
5630
5631             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5632             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5633             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
5634             IOService * holder = NULL;
5635
5636             for (UInt i = 0; i < count; i++) {
5637                 if (entries[i].fService
5638                   && (delayType == entries[i].fDelayType)
5639                   && (entries[i].fMaxDelay < ns)) {
5640                     ns = entries[i].fMaxDelay;
5641                     holder = entries[i].fService;
5642                 }
5643             }
5644             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5645                                             (void *) (uintptr_t) ns, holder,
5646                                             NULL, NULL);
5647             result = kIOReturnSuccess;
5648         }
5649     }
5650     while (false);
5651
5652     IORecursiveLockUnlock(sCpuDelayLock);
5653
5654     return (result);
5655 }
5656
5657 #endif /* defined(__i386__) || defined(__x86_64__) */
5658
5659 void IOService::
5660 requireMaxBusStall(UInt32 __unused ns)
5661 {
5662 #if defined(__i386__) || defined(__x86_64__)
5663     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
5664 #endif
5665 }
5666
5667 void IOService::
5668 requireMaxInterruptDelay(uint32_t __unused ns)
5669 {
5670 #if defined(__i386__) || defined(__x86_64__)
5671     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
5672 #endif
5673 }
5674
5675 /*
5676  * Device interrupts
5677  */
5678
5679 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
5680 {
5681   IOInterruptController *interruptController;
5682   OSArray               *array;
5683   OSData                *data;
5684   OSSymbol              *interruptControllerName;
5685   long                  numSources;
5686   IOInterruptSource     *interruptSources;
5687
5688   // Get the parents list from the nub.
5689   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
5690   if (array == 0) return kIOReturnNoResources;
5691
5692   // Allocate space for the IOInterruptSources if needed... then return early.
5693   if (nub->_interruptSources == 0) {
5694     numSources = array->getCount();
5695     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
5696     if (interruptSources == 0) return kIOReturnNoMemory;
5697
5698     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
5699
5700     nub->_numInterruptSources = numSources;
5701     nub->_interruptSources = interruptSources;
5702     return kIOReturnSuccess;
5703   }
5704
5705   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
5706   if (interruptControllerName == 0) return kIOReturnNoResources;
5707
5708   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
5709   if (interruptController == 0) return kIOReturnNoResources;
5710
5711   // Get the interrupt numbers from the nub.
5712   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
5713   if (array == 0) return kIOReturnNoResources;
5714   data = OSDynamicCast(OSData, array->getObject(source));
5715   if (data == 0) return kIOReturnNoResources;
5716
5717   // Set the interruptController and interruptSource in the nub's table.
5718   interruptSources = nub->_interruptSources;
5719   interruptSources[source].interruptController = interruptController;
5720   interruptSources[source].vectorData = data;
5721
5722   return kIOReturnSuccess;
5723 }
5724
5725 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
5726 {
5727   IOReturn ret;
5728
5729   /* Make sure the _interruptSources are set */
5730   if (_interruptSources == 0) {
5731     ret = resolveInterrupt(this, source);
5732     if (ret != kIOReturnSuccess) return ret;
5733   }
5734
5735   /* Make sure the local source number is valid */
5736   if ((source < 0) || (source >= _numInterruptSources))
5737     return kIOReturnNoInterrupt;
5738
5739   /* Look up the contoller for the local source */
5740   *interruptController = _interruptSources[source].interruptController;
5741
5742   if (*interruptController == NULL) {
5743     if (!resolve) return kIOReturnNoInterrupt;
5744
5745     /* Try to reslove the interrupt */
5746     ret = resolveInterrupt(this, source);
5747     if (ret != kIOReturnSuccess) return ret;
5748
5749     *interruptController = _interruptSources[source].interruptController;
5750   }
5751
5752   return kIOReturnSuccess;
5753 }
5754
5755 IOReturn IOService::registerInterrupt(int source, OSObject *target,
5756                                       IOInterruptAction handler,
5757                                       void *refCon)
5758 {
5759   IOInterruptController *interruptController;
5760   IOReturn              ret;
5761
5762   ret = lookupInterrupt(source, true, &interruptController);
5763   if (ret != kIOReturnSuccess) return ret;
5764
5765   /* Register the source */
5766   return interruptController->registerInterrupt(this, source, target,
5767                                                 (IOInterruptHandler)handler,
5768                                                 refCon);
5769 }
5770
5771 IOReturn IOService::unregisterInterrupt(int source)
5772 {
5773   IOInterruptController *interruptController;
5774   IOReturn              ret;
5775
5776   ret = lookupInterrupt(source, false, &interruptController);
5777   if (ret != kIOReturnSuccess) return ret;
5778
5779   /* Unregister the source */
5780   return interruptController->unregisterInterrupt(this, source);
5781 }
5782
5783 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
5784 {
5785   IOReportLegend * legend = NULL;
5786   IOInterruptAccountingData * oldValue = NULL;
5787   IOInterruptAccountingReporter * newArray = NULL;
5788   int newArraySize = 0;
5789   int i = 0;
5790
5791   if (source < 0) {
5792     return kIOReturnBadArgument;
5793   }
5794
5795   /*
5796    * We support statistics on a maximum of 256 interrupts per nub; if a nub
5797    * has more than 256 interrupt specifiers associated with it, and tries
5798    * to register a high interrupt index with interrupt accounting, panic.
5799    * Having more than 256 interrupts associated with a single nub is
5800    * probably a sign that something fishy is going on.
5801    */
5802   if (source > IA_INDEX_MAX) {
5803     panic("addInterruptStatistics called for an excessively large index (%d)", source);
5804   }
5805
5806   /*
5807    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
5808    * leaving it as is because the likelihood of contention where we are
5809    * actually growing the array is minimal (we would realistically need
5810    * to be starting a driver for the first time, with an IOReporting
5811    * client already in place).  Nonetheless, cleanup that can be done
5812    * to adhere to best practices; it'll make the code more complicated,
5813    * unfortunately.
5814    */
5815   IOLockLock(reserved->interruptStatisticsLock);
5816
5817   /*
5818    * Lazily allocate the statistics array.
5819    */
5820   if (!reserved->interruptStatisticsArray) {
5821     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
5822     assert(reserved->interruptStatisticsArray);
5823     reserved->interruptStatisticsArrayCount = 1;
5824     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
5825   }
5826
5827   if (source >= reserved->interruptStatisticsArrayCount) {
5828     /*
5829      * We're still within the range of supported indices, but we are out
5830      * of space in the current array.  Do a nasty realloc (because
5831      * IORealloc isn't a thing) here.  We'll double the size with each
5832      * reallocation.
5833      *
5834      * Yes, the "next power of 2" could be more efficient; but this will
5835      * be invoked incredibly rarely.  Who cares.
5836      */
5837     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
5838
5839     while (newArraySize <= source)
5840       newArraySize = (newArraySize << 1);
5841     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
5842
5843     assert(newArray);
5844
5845     /*
5846      * TODO: This even zeroes the memory it is about to overwrite.
5847      * Shameful; fix it.  Not particularly high impact, however.
5848      */
5849     bzero(newArray, newArraySize * sizeof(*newArray));
5850     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
5851     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
5852     reserved->interruptStatisticsArray = newArray;
5853     reserved->interruptStatisticsArrayCount = newArraySize;
5854   }
5855
5856   if (!reserved->interruptStatisticsArray[source].reporter) {
5857     /*
5858      * We don't have a reporter associated with this index yet, so we
5859      * need to create one.
5860      */
5861     /*
5862      * TODO: Some statistics do in fact have common units (time); should this be
5863      * split into separate reporters to communicate this?
5864      */
5865      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryInterrupt, kIOReportUnitNone);
5866
5867     /*
5868      * Each statistic is given an identifier based on the interrupt index (which
5869      * should be unique relative to any single nub) and the statistic involved.
5870      * We should now have a sane (small and positive) index, so start
5871      * constructing the channels for statistics.
5872      */
5873     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
5874       /*
5875        * TODO: Currently, this does not add channels for disabled statistics.
5876        * Will this be confusing for clients?  If so, we should just add the
5877        * channels; we can avoid updating the channels even if they exist.
5878        */
5879       if (IA_GET_STATISTIC_ENABLED(i))
5880         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
5881     }
5882
5883     /*
5884      * We now need to add the legend for this reporter to the registry.
5885      */
5886     legend = IOReportLegend::with(OSDynamicCast(OSArray, getProperty(kIOReportLegendKey)));
5887
5888     if ((source >= IA_MAX_SUBGROUP_NAME) || (source < 0)) {
5889       /*
5890        * Either we're using a nonsensical index (should never happen), or the
5891        * index is larger than anticipated (may happen, almost certainly won't).
5892        * This may move to live generation of the names in the future, but for
5893        * now, point both cases to a generic subgroup name (this will confuse
5894        * clients, unfortunately).
5895        */
5896       legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, kInterruptAccountingGenericSubgroupName);
5897     } else {
5898       legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, kInterruptAccountingSubgroupNames[source]);
5899     }
5900
5901     setProperty(kIOReportLegendKey, legend->getLegend());
5902     legend->release();
5903
5904     /*
5905      * TODO: Is this a good idea?  Probably not; my assumption is it opts
5906      * all entities who register interrupts into public disclosure of all
5907      * IOReporting channels.  Unfortunately, this appears to be as fine
5908      * grain as it gets.
5909      */
5910     setProperty(kIOReportLegendPublicKey, true);
5911   }
5912
5913   /*
5914    * Don't stomp existing entries.  If we are about to, panic; this
5915    * probably means we failed to tear down our old interrupt source
5916    * correctly.
5917    */
5918   oldValue = reserved->interruptStatisticsArray[source].statistics;
5919
5920   if (oldValue) {
5921     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
5922   }
5923
5924   reserved->interruptStatisticsArray[source].statistics = statistics;
5925
5926   /*
5927    * Inherit the reporter values for each statistic.  The target may
5928    * be torn down as part of the runtime of the service (especially
5929    * for sleep/wake), so we inherit in order to avoid having values
5930    * reset for no apparent reason.  Our statistics are ultimately
5931    * tied to the index and the sevice, not to an individual target,
5932    * so we should maintain them accordingly.
5933    */
5934   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
5935
5936   IOLockUnlock(reserved->interruptStatisticsLock);
5937
5938   return kIOReturnSuccess;
5939 }
5940
5941 IOReturn IOService::removeInterruptStatistics(int source)
5942 {
5943   IOInterruptAccountingData * value = NULL;
5944
5945   if (source < 0) {
5946     return kIOReturnBadArgument;
5947   }
5948
5949   IOLockLock(reserved->interruptStatisticsLock);
5950
5951   /*
5952    * We dynamically grow the statistics array, so an excessively
5953    * large index value has NEVER been registered.  This either
5954    * means our cap on the array size is too small (unlikely), or
5955    * that we have been passed a corrupt index (this must be passed
5956    * the plain index into the interrupt specifier list).
5957    */
5958   if (source >= reserved->interruptStatisticsArrayCount) {
5959     panic("removeInterruptStatistics called for index %d, which was never registered", source);
5960   }
5961
5962   assert(reserved->interruptStatisticsArray);
5963
5964   /*
5965    * If there is no existing entry, we are most likely trying to
5966    * free an interrupt owner twice, or we have corrupted the
5967    * index value.
5968    */
5969   value = reserved->interruptStatisticsArray[source].statistics;
5970
5971   if (!value) {
5972     panic("removeInterruptStatistics called for empty index %d", source);
5973   }
5974
5975   /*
5976    * We update the statistics, so that any delta with the reporter
5977    * state is not lost.
5978    */
5979   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
5980   reserved->interruptStatisticsArray[source].statistics = NULL;
5981   IOLockUnlock(reserved->interruptStatisticsLock);
5982
5983   return kIOReturnSuccess;
5984 }
5985
5986 IOReturn IOService::getInterruptType(int source, int *interruptType)
5987 {
5988   IOInterruptController *interruptController;
5989   IOReturn              ret;
5990
5991   ret = lookupInterrupt(source, true, &interruptController);
5992   if (ret != kIOReturnSuccess) return ret;
5993
5994   /* Return the type */
5995   return interruptController->getInterruptType(this, source, interruptType);
5996 }
5997
5998 IOReturn IOService::enableInterrupt(int source)
5999 {
6000   IOInterruptController *interruptController;
6001   IOReturn              ret;
6002
6003   ret = lookupInterrupt(source, false, &interruptController);
6004   if (ret != kIOReturnSuccess) return ret;
6005
6006   /* Enable the source */
6007   return interruptController->enableInterrupt(this, source);
6008 }
6009
6010 IOReturn IOService::disableInterrupt(int source)
6011 {
6012   IOInterruptController *interruptController;
6013   IOReturn              ret;
6014
6015   ret = lookupInterrupt(source, false, &interruptController);
6016   if (ret != kIOReturnSuccess) return ret;
6017
6018   /* Disable the source */
6019   return interruptController->disableInterrupt(this, source);
6020 }
6021
6022 IOReturn IOService::causeInterrupt(int source)
6023 {
6024   IOInterruptController *interruptController;
6025   IOReturn              ret;
6026
6027   ret = lookupInterrupt(source, false, &interruptController);
6028   if (ret != kIOReturnSuccess) return ret;
6029
6030   /* Cause an interrupt for the source */
6031   return interruptController->causeInterrupt(this, source);
6032 }
6033
6034 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6035                                     IOReportConfigureAction action,
6036                                     void                   *result,
6037                                     void                   *destination)
6038 {
6039     unsigned cnt;
6040
6041     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6042         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6043             if (pwrMgt) configurePowerStatesReport(action, result);
6044             else  return kIOReturnUnsupported;
6045         }
6046         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6047             if (pwrMgt) configureSimplePowerReport(action, result);
6048             else  return kIOReturnUnsupported;
6049         }
6050     }
6051
6052     IOLockLock(reserved->interruptStatisticsLock);
6053
6054     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6055     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6056         if (reserved->interruptStatisticsArray[cnt].reporter) {
6057             /*
6058              * If the reporter is currently associated with the statistics
6059              * for an event source, we may need to update the reporter.
6060              */
6061             if (reserved->interruptStatisticsArray[cnt].statistics)
6062                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6063
6064             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6065         }
6066     }
6067
6068     IOLockUnlock(reserved->interruptStatisticsLock);
6069
6070     return kIOReturnSuccess;
6071 }
6072
6073 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6074                                  IOReportUpdateAction      action,
6075                                  void                     *result,
6076                                  void                     *destination)
6077 {
6078     unsigned cnt;
6079
6080     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6081         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6082             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6083             else return kIOReturnUnsupported;
6084         }
6085         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6086             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6087             else return kIOReturnUnsupported;
6088         }
6089     }
6090
6091     IOLockLock(reserved->interruptStatisticsLock);
6092
6093     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6094     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6095         if (reserved->interruptStatisticsArray[cnt].reporter) {
6096             /*
6097              * If the reporter is currently associated with the statistics
6098              * for an event source, we need to update the reporter.
6099              */
6100             if (reserved->interruptStatisticsArray[cnt].statistics)
6101                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6102
6103             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6104         }
6105     }
6106
6107     IOLockUnlock(reserved->interruptStatisticsLock);
6108
6109     return kIOReturnSuccess;
6110 }
6111
6112 uint64_t IOService::getAuthorizationID( void )
6113 {
6114     return reserved->authorizationID;
6115 }
6116
6117 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6118 {
6119     OSObject * entitlement;
6120     IOReturn status;
6121
6122     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6123
6124     if ( entitlement )
6125     {
6126         if ( entitlement == kOSBooleanTrue )
6127         {
6128             reserved->authorizationID = authorizationID;
6129
6130             status = kIOReturnSuccess;
6131         }
6132         else
6133         {
6134             status = kIOReturnNotPrivileged;
6135         }
6136
6137         entitlement->release( );
6138     }
6139     else
6140     {
6141         status = kIOReturnNotPrivileged;
6142     }
6143
6144     return status;
6145 }
6146
6147 #if __LP64__
6148 OSMetaClassDefineReservedUsed(IOService, 0);
6149 OSMetaClassDefineReservedUsed(IOService, 1);
6150 OSMetaClassDefineReservedUnused(IOService, 2);
6151 OSMetaClassDefineReservedUnused(IOService, 3);
6152 OSMetaClassDefineReservedUnused(IOService, 4);
6153 OSMetaClassDefineReservedUnused(IOService, 5);
6154 OSMetaClassDefineReservedUnused(IOService, 6);
6155 OSMetaClassDefineReservedUnused(IOService, 7);
6156 #else
6157 OSMetaClassDefineReservedUsed(IOService, 0);
6158 OSMetaClassDefineReservedUsed(IOService, 1);
6159 OSMetaClassDefineReservedUsed(IOService, 2);
6160 OSMetaClassDefineReservedUsed(IOService, 3);
6161 OSMetaClassDefineReservedUsed(IOService, 4);
6162 OSMetaClassDefineReservedUsed(IOService, 5);
6163 OSMetaClassDefineReservedUsed(IOService, 6);
6164 OSMetaClassDefineReservedUsed(IOService, 7);
6165 #endif
6166 OSMetaClassDefineReservedUnused(IOService, 8);
6167 OSMetaClassDefineReservedUnused(IOService, 9);
6168 OSMetaClassDefineReservedUnused(IOService, 10);
6169 OSMetaClassDefineReservedUnused(IOService, 11);
6170 OSMetaClassDefineReservedUnused(IOService, 12);
6171 OSMetaClassDefineReservedUnused(IOService, 13);
6172 OSMetaClassDefineReservedUnused(IOService, 14);
6173 OSMetaClassDefineReservedUnused(IOService, 15);
6174 OSMetaClassDefineReservedUnused(IOService, 16);
6175 OSMetaClassDefineReservedUnused(IOService, 17);
6176 OSMetaClassDefineReservedUnused(IOService, 18);
6177 OSMetaClassDefineReservedUnused(IOService, 19);
6178 OSMetaClassDefineReservedUnused(IOService, 20);
6179 OSMetaClassDefineReservedUnused(IOService, 21);
6180 OSMetaClassDefineReservedUnused(IOService, 22);
6181 OSMetaClassDefineReservedUnused(IOService, 23);
6182 OSMetaClassDefineReservedUnused(IOService, 24);
6183 OSMetaClassDefineReservedUnused(IOService, 25);
6184 OSMetaClassDefineReservedUnused(IOService, 26);
6185 OSMetaClassDefineReservedUnused(IOService, 27);
6186 OSMetaClassDefineReservedUnused(IOService, 28);
6187 OSMetaClassDefineReservedUnused(IOService, 29);
6188 OSMetaClassDefineReservedUnused(IOService, 30);
6189 OSMetaClassDefineReservedUnused(IOService, 31);
6190 OSMetaClassDefineReservedUnused(IOService, 32);
6191 OSMetaClassDefineReservedUnused(IOService, 33);
6192 OSMetaClassDefineReservedUnused(IOService, 34);
6193 OSMetaClassDefineReservedUnused(IOService, 35);
6194 OSMetaClassDefineReservedUnused(IOService, 36);
6195 OSMetaClassDefineReservedUnused(IOService, 37);
6196 OSMetaClassDefineReservedUnused(IOService, 38);
6197 OSMetaClassDefineReservedUnused(IOService, 39);
6198 OSMetaClassDefineReservedUnused(IOService, 40);
6199 OSMetaClassDefineReservedUnused(IOService, 41);
6200 OSMetaClassDefineReservedUnused(IOService, 42);
6201 OSMetaClassDefineReservedUnused(IOService, 43);
6202 OSMetaClassDefineReservedUnused(IOService, 44);
6203 OSMetaClassDefineReservedUnused(IOService, 45);
6204 OSMetaClassDefineReservedUnused(IOService, 46);
6205 OSMetaClassDefineReservedUnused(IOService, 47);