iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <IOKit/AppleKeyStoreInterface.h>
  54 #include <IOKit/pwr_mgt/RootDomain.h>
  55 #include <IOKit/IOCPU.h>
  56 #include <mach/sync_policy.h>
  57 #include <IOKit/assert.h>
  58 #include <sys/errno.h>
  59 #include <sys/kdebug.h>
  60 #include <string.h>
  61
  62 #include <machine/pal_routines.h>
  63
  64 #define LOG kprintf
  65 //#define LOG IOLog
  66 #define MATCH_DEBUG     0
  67 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  68
  69 // disabled since lockForArbitration() can be held externally
  70 #define DEBUG_NOTIFIER_LOCKED   0
  71
  72 #include "IOServicePrivate.h"
  73 #include "IOKitKernelInternal.h"
  74
  75 // take lockForArbitration before LOCKNOTIFY
  76
  77 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  78
  79 #define super IORegistryEntry
  80
  81 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  82
  83 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  84 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  85
  86 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  87
  88 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  89
  90 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  91
  92 OSDefineMetaClassAndStructors(IOResources, IOService)
  93
  94 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  95
  96 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  97
  98 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  99
 100 static IOPlatformExpert *       gIOPlatform;
 101 static class IOPMrootDomain *   gIOPMRootDomain;
 102 const IORegistryPlane *         gIOServicePlane;
 103 const IORegistryPlane *         gIOPowerPlane;
 104 const OSSymbol *                gIODeviceMemoryKey;
 105 const OSSymbol *                gIOInterruptControllersKey;
 106 const OSSymbol *                gIOInterruptSpecifiersKey;
 107
 108 const OSSymbol *                gIOResourcesKey;
 109 const OSSymbol *                gIOResourceMatchKey;
 110 const OSSymbol *                gIOResourceMatchedKey;
 111 const OSSymbol *                gIOProviderClassKey;
 112 const OSSymbol *                gIONameMatchKey;
 113 const OSSymbol *                gIONameMatchedKey;
 114 const OSSymbol *                gIOPropertyMatchKey;
 115 const OSSymbol *                gIOPropertyExistsMatchKey;
 116 const OSSymbol *                gIOLocationMatchKey;
 117 const OSSymbol *                gIOParentMatchKey;
 118 const OSSymbol *                gIOPathMatchKey;
 119 const OSSymbol *                gIOMatchCategoryKey;
 120 const OSSymbol *                gIODefaultMatchCategoryKey;
 121 const OSSymbol *                gIOMatchedServiceCountKey;
 122 #if !CONFIG_EMBEDDED
 123 const OSSymbol *                gIOServiceLegacyMatchingRegistryIDKey;
 124 #endif
 125
 126 const OSSymbol *                gIOMapperIDKey;
 127 const OSSymbol *                gIOUserClientClassKey;
 128 const OSSymbol *                gIOKitDebugKey;
 129
 130 const OSSymbol *                gIOCommandPoolSizeKey;
 131
 132 const OSSymbol *                gIOConsoleLockedKey;
 133 const OSSymbol *                gIOConsoleUsersKey;
 134 const OSSymbol *                gIOConsoleSessionUIDKey;
 135 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 136 const OSSymbol *                gIOConsoleUsersSeedKey;
 137 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 138 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 139 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 140 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 141 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 142 clock_sec_t                     gIOConsoleLockTime;
 143 static bool                     gIOConsoleLoggedIn;
 144 #if HIBERNATION
 145 static OSBoolean *              gIOConsoleBooterLockState;
 146 static uint32_t                 gIOScreenLockState;
 147 #endif
 148 static IORegistryEntry *        gIOChosenEntry;
 149
 150 static int                      gIOResourceGenerationCount;
 151
 152 const OSSymbol *                gIOServiceKey;
 153 const OSSymbol *                gIOPublishNotification;
 154 const OSSymbol *                gIOFirstPublishNotification;
 155 const OSSymbol *                gIOMatchedNotification;
 156 const OSSymbol *                gIOFirstMatchNotification;
 157 const OSSymbol *                gIOTerminatedNotification;
 158 const OSSymbol *                gIOWillTerminateNotification;
 159
 160 const OSSymbol *                gIOGeneralInterest;
 161 const OSSymbol *                gIOBusyInterest;
 162 const OSSymbol *                gIOAppPowerStateInterest;
 163 const OSSymbol *                gIOPriorityPowerStateInterest;
 164 const OSSymbol *                gIOConsoleSecurityInterest;
 165
 166 const OSSymbol *                gIOBSDKey;
 167 const OSSymbol *                gIOBSDNameKey;
 168 const OSSymbol *                gIOBSDMajorKey;
 169 const OSSymbol *                gIOBSDMinorKey;
 170 const OSSymbol *                gIOBSDUnitKey;
 171
 172 const  OSSymbol *               gAKSGetKey;
 173 #if defined(__i386__) || defined(__x86_64__)
 174 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 175 #endif
 176
 177 static OSDictionary *           gNotifications;
 178 static IORecursiveLock *        gNotificationLock;
 179
 180 static IOService *              gIOResources;
 181 static IOService *              gIOServiceRoot;
 182
 183 static OSOrderedSet *           gJobs;
 184 static semaphore_port_t         gJobsSemaphore;
 185 static IOLock *                 gJobsLock;
 186 static int                      gOutstandingJobs;
 187 static int                      gNumConfigThreads;
 188 static int                      gNumWaitingThreads;
 189 static IOLock *                 gIOServiceBusyLock;
 190 bool                            gCPUsRunning;
 191
 192 static thread_t                 gIOTerminateThread;
 193 static thread_t                 gIOTerminateWorkerThread;
 194 static UInt32                   gIOTerminateWork;
 195 static OSArray *                gIOTerminatePhase2List;
 196 static OSArray *                gIOStopList;
 197 static OSArray *                gIOStopProviderList;
 198 static OSArray *                gIOFinalizeList;
 199
 200 static SInt32                   gIOConsoleUsersSeed;
 201 static OSData *                 gIOConsoleUsersSeedValue;
 202
 203 extern const OSSymbol *         gIODTPHandleKey;
 204
 205 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 206
 207 static IOLock *                 gIOConsoleUsersLock;
 208 static thread_call_t            gIOConsoleLockCallout;
 209 static IONotifier *             gIOServiceNullNotifier;
 210
 211 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 212
 213 #define LOCKREADNOTIFY()        \
 214     IORecursiveLockLock( gNotificationLock )
 215 #define LOCKWRITENOTIFY()       \
 216     IORecursiveLockLock( gNotificationLock )
 217 #define LOCKWRITE2READNOTIFY()
 218 #define UNLOCKNOTIFY()          \
 219     IORecursiveLockUnlock( gNotificationLock )
 220 #define SLEEPNOTIFY(event) \
 221     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 222 #define SLEEPNOTIFYTO(event, deadline) \
 223     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 224 #define WAKEUPNOTIFY(event) \
 225         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 226
 227 #define randomDelay()   \
 228         int del = read_processor_clock();                               \
 229         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 230         IOSleep( del );
 231
 232 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 233
 234 #define queue_element(entry, element, type, field) do { \
 235         vm_address_t __ele = (vm_address_t) (entry);    \
 236         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 237         (element) = (type) __ele;                       \
 238     } while(0)
 239
 240 #define iterqueue(que, elt)                             \
 241         for (queue_entry_t elt = queue_first(que);      \
 242              !queue_end(que, elt);                      \
 243              elt = queue_next(elt))
 244
 245 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 246
 247 struct IOInterruptAccountingReporter {
 248     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 249     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 250 };
 251
 252 struct ArbitrationLockQueueElement {
 253     queue_chain_t link;
 254     IOThread      thread;
 255     IOService *   service;
 256     unsigned      count;
 257     bool          required;
 258     bool          aborted;
 259 };
 260
 261 static queue_head_t gArbitrationLockQueueActive;
 262 static queue_head_t gArbitrationLockQueueWaiting;
 263 static queue_head_t gArbitrationLockQueueFree;
 264 static IOLock *     gArbitrationLockQueueLock;
 265
 266 bool IOService::isInactive( void ) const
 267     { return( 0 != (kIOServiceInactiveState & getState())); }
 268
 269 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 270
 271 #if defined(__i386__) || defined(__x86_64__)
 272
 273 // Only used by the intel implementation of
 274 //     IOService::requireMaxBusStall(UInt32 ns)
 275 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 276 struct CpuDelayEntry
 277 {
 278     IOService * fService;
 279     UInt32      fMaxDelay;
 280     UInt32      fDelayType;
 281 };
 282
 283 enum {
 284     kCpuDelayBusStall, kCpuDelayInterrupt,
 285     kCpuNumDelayTypes
 286 };
 287
 288 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 289 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 290 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 291 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 292 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 293 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 294 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 295
 296 static void
 297 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 298 static IOReturn
 299 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 300
 301 #endif /* defined(__i386__) || defined(__x86_64__) */
 302
 303 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 304
 305 void IOService::initialize( void )
 306 {
 307     kern_return_t       err;
 308
 309     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 310     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 311
 312     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 313     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 314     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 315     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 316     gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 317     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 318     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 319     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 320
 321     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 322     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 323                                         kIODefaultMatchCategoryKey );
 324     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 325                                         kIOMatchedServiceCountKey );
 326 #if !CONFIG_EMBEDDED
 327     gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
 328                                         kIOServiceLegacyMatchingRegistryIDKey );
 329 #endif
 330
 331     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 332
 333     gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 334     gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 335     gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 336
 337     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 338     gIOInterruptControllersKey
 339         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 340     gIOInterruptSpecifiersKey
 341         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 342
 343     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 344
 345     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 346
 347     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 348
 349     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 350     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 351     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 352     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 353     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 354
 355     gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 356     gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 357     gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 358     gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 359     gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 360
 361     gNotifications              = OSDictionary::withCapacity( 1 );
 362     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 363                                                  kIOPublishNotification );
 364     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 365                                                  kIOFirstPublishNotification );
 366     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 367                                                  kIOMatchedNotification );
 368     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 369                                                  kIOFirstMatchNotification );
 370     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 371                                                  kIOTerminatedNotification );
 372     gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
 373                                                  kIOWillTerminateNotification );
 374     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 375
 376     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 377     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 378     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 379     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 380
 381     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 382     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 383     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 384     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 385     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 386     gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 387
 388     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 389
 390     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 391 #if defined(__i386__) || defined(__x86_64__)
 392     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 393     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 394     uint32_t  idx;
 395     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 396     {
 397         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 398         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 399         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 400     }
 401     gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 402 #endif
 403     gNotificationLock           = IORecursiveLockAlloc();
 404
 405     gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 406
 407     assert( gIOServicePlane && gIODeviceMemoryKey
 408         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 409         && gIOResourcesKey && gNotifications && gNotificationLock
 410         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 411         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 412         && gIOPublishNotification && gIOMatchedNotification
 413         && gIOTerminatedNotification && gIOServiceKey
 414         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 415     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 416         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 417
 418     gJobsLock   = IOLockAlloc();
 419     gJobs       = OSOrderedSet::withCapacity( 10 );
 420
 421     gIOServiceBusyLock = IOLockAlloc();
 422
 423     gIOConsoleUsersLock = IOLockAlloc();
 424
 425     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 426
 427     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 428
 429     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 430
 431     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 432                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 433
 434     gIOResources = IOResources::resources();
 435     assert( gIOResources );
 436
 437     gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 438     assert(gIOServiceNullNotifier);
 439
 440     gArbitrationLockQueueLock = IOLockAlloc();
 441     queue_init(&gArbitrationLockQueueActive);
 442     queue_init(&gArbitrationLockQueueWaiting);
 443     queue_init(&gArbitrationLockQueueFree);
 444
 445     assert( gArbitrationLockQueueLock );
 446
 447     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 448     gIOStopList            = OSArray::withCapacity( 16 );
 449     gIOStopProviderList    = OSArray::withCapacity( 16 );
 450     gIOFinalizeList        = OSArray::withCapacity( 16 );
 451     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 452
 453     // worker thread that is responsible for terminating / cleaning up threads
 454     kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
 455     assert(gIOTerminateWorkerThread);
 456     thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
 457 }
 458
 459 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 460
 461 #if defined(__i386__) || defined(__x86_64__)
 462 extern "C" {
 463
 464 const char *getCpuDelayBusStallHolderName(void);
 465 const char *getCpuDelayBusStallHolderName(void) {
 466     return sCPULatencyHolderName[kCpuDelayBusStall];
 467 }
 468
 469 }
 470 #endif
 471
 472 #if IOMATCHDEBUG
 473 static UInt64 getDebugFlags( OSDictionary * props )
 474 {
 475     OSNumber *  debugProp;
 476     UInt64      debugFlags;
 477
 478     debugProp = OSDynamicCast( OSNumber,
 479                 props->getObject( gIOKitDebugKey ));
 480     if( debugProp)
 481         debugFlags = debugProp->unsigned64BitValue();
 482     else
 483         debugFlags = gIOKitDebug;
 484
 485     return( debugFlags );
 486 }
 487
 488 static UInt64 getDebugFlags( IOService * inst )
 489 {
 490     OSObject *  prop;
 491     OSNumber *  debugProp;
 492     UInt64      debugFlags;
 493
 494     prop = inst->copyProperty(gIOKitDebugKey);
 495     debugProp = OSDynamicCast(OSNumber, prop);
 496     if( debugProp)
 497         debugFlags = debugProp->unsigned64BitValue();
 498     else
 499         debugFlags = gIOKitDebug;
 500
 501     OSSafeReleaseNULL(prop);
 502
 503     return( debugFlags );
 504 }
 505 #endif
 506
 507 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 508
 509 // Probe a matched service and return an instance to be started.
 510 // The default score is from the property table, & may be altered
 511 // during probe to change the start order.
 512
 513 IOService * IOService::probe(   IOService * provider,
 514                                 SInt32    * score )
 515 {
 516     return( this );
 517 }
 518
 519 bool IOService::start( IOService * provider )
 520 {
 521     return( true );
 522 }
 523
 524 void IOService::stop( IOService * provider )
 525 {
 526 }
 527
 528 bool IOService::init( OSDictionary * dictionary )
 529 {
 530     bool ret;
 531
 532     ret = super::init(dictionary);
 533     if (!ret)     return (false);
 534     if (reserved) return (true);
 535
 536     reserved = IONew(ExpansionData, 1);
 537     if (!reserved) return (false);
 538     bzero(reserved, sizeof(*reserved));
 539
 540     /*
 541      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 542      * lock based on the presence of specifiers ran into issues as some
 543      * platforms set up the specifiers after IOService initialization.
 544      *
 545      * We may be able to get away with a global lock, as this should only be
 546      * contended by IOReporting clients and driver start/stop (unless a
 547      * driver wants to remove/add handlers in the course of normal operation,
 548      * which should be unlikely).
 549      */
 550     reserved->interruptStatisticsLock = IOLockAlloc();
 551     if (!reserved->interruptStatisticsLock) return (false);
 552
 553     return (true);
 554 }
 555
 556 bool IOService::init( IORegistryEntry * from,
 557                       const IORegistryPlane * inPlane )
 558 {
 559     bool ret;
 560
 561     ret = super::init(from, inPlane);
 562     if (!ret)     return (false);
 563     if (reserved) return (true);
 564
 565     reserved = IONew(ExpansionData, 1);
 566     if (!reserved) return (false);
 567     bzero(reserved, sizeof(*reserved));
 568
 569     /*
 570      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 571      * lock based on the presence of specifiers ran into issues as some
 572      * platforms set up the specifiers after IOService initialization.
 573      *
 574      * We may be able to get away with a global lock, as this should only be
 575      * contended by IOReporting clients and driver start/stop (unless a
 576      * driver wants to remove/add handlers in the course of normal operation,
 577      * which should be unlikely).
 578      */
 579     reserved->interruptStatisticsLock = IOLockAlloc();
 580     if (!reserved->interruptStatisticsLock) return (false);
 581
 582     return (true);
 583 }
 584
 585 void IOService::free( void )
 586 {
 587     int i = 0;
 588     requireMaxBusStall(0);
 589     requireMaxInterruptDelay(0);
 590     if( getPropertyTable())
 591         unregisterAllInterest();
 592     PMfree();
 593
 594     if (reserved) {
 595         if (reserved->interruptStatisticsArray) {
 596             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 597                 if (reserved->interruptStatisticsArray[i].reporter)
 598                     reserved->interruptStatisticsArray[i].reporter->release();
 599             }
 600
 601             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 602         }
 603
 604         if (reserved->interruptStatisticsLock)
 605             IOLockFree(reserved->interruptStatisticsLock);
 606         IODelete(reserved, ExpansionData, 1);
 607     }
 608
 609     if (_numInterruptSources && _interruptSources)
 610     {
 611         IOFree(_interruptSources, _numInterruptSources * sizeof(IOInterruptSource));
 612         _interruptSources = 0;
 613     }
 614
 615     super::free();
 616 }
 617
 618 /*
 619  * Attach in service plane
 620  */
 621 bool IOService::attach( IOService * provider )
 622 {
 623     bool         ok;
 624     uint32_t     count;
 625     AbsoluteTime deadline;
 626     int          waitResult = THREAD_AWAKENED;
 627     bool         wait, computeDeadline = true;
 628
 629     if( provider) {
 630
 631         if( gIOKitDebug & kIOLogAttach)
 632             LOG( "%s::attach(%s)\n", getName(),
 633                     provider->getName());
 634
 635         ok   = false;
 636         do
 637         {
 638             wait = false;
 639             provider->lockForArbitration();
 640             if (provider->__state[0] & kIOServiceInactiveState) ok = false;
 641             else
 642             {
 643                 count = provider->getChildCount(gIOServicePlane);
 644                 wait = (count > (kIOServiceBusyMax - 4));
 645                 if (!wait) ok = attachToParent(provider, gIOServicePlane);
 646                 else
 647                 {
 648                     IOLog("stalling for detach from %s\n", provider->getName());
 649                     IOLockLock( gIOServiceBusyLock );
 650                     provider->__state[1] |= kIOServiceWaitDetachState;
 651                 }
 652             }
 653             provider->unlockForArbitration();
 654             if (wait)
 655             {
 656                 if (computeDeadline)
 657                 {
 658                     clock_interval_to_deadline(15, kSecondScale, &deadline);
 659                     computeDeadline = false;
 660                 }
 661                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 662                 IOLockUnlock( gIOServiceBusyLock );
 663                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 664                 wait = (waitResult != THREAD_TIMED_OUT);
 665             }
 666         }
 667         while (wait);
 668
 669     } else {
 670         gIOServiceRoot = this;
 671         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 672     }
 673
 674     if (ok && !__provider) (void) getProvider();
 675
 676     return( ok );
 677 }
 678
 679 IOService * IOService::getServiceRoot( void )
 680 {
 681     return( gIOServiceRoot );
 682 }
 683
 684 void IOService::detach( IOService * provider )
 685 {
 686     IOService * newProvider = 0;
 687     SInt32      busy;
 688     bool        adjParent;
 689
 690     if( gIOKitDebug & kIOLogAttach)
 691         LOG("%s::detach(%s)\n", getName(), provider->getName());
 692
 693     lockForArbitration();
 694
 695     uint64_t regID1 = provider->getRegistryEntryID();
 696     uint64_t regID2 = getRegistryEntryID();
 697     IOServiceTrace(
 698         IOSERVICE_DETACH,
 699         (uintptr_t) regID1,
 700         (uintptr_t) (regID1 >> 32),
 701         (uintptr_t) regID2,
 702         (uintptr_t) (regID2 >> 32));
 703
 704     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 705                && (provider == getProvider()));
 706
 707     detachFromParent( provider, gIOServicePlane );
 708
 709     if( busy) {
 710         newProvider = getProvider();
 711         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 712             _adjustBusy( -busy );
 713     }
 714
 715     if (kIOServiceInactiveState & __state[0]) {
 716         getMetaClass()->removeInstance(this);
 717         IORemoveServicePlatformActions(this);
 718     }
 719
 720     unlockForArbitration();
 721
 722     if( newProvider) {
 723         newProvider->lockForArbitration();
 724         newProvider->_adjustBusy(1);
 725         newProvider->unlockForArbitration();
 726     }
 727
 728     // check for last client detach from a terminated service
 729     if( provider->lockForArbitration( true ))
 730     {
 731         if (kIOServiceStartState & __state[1])
 732         {
 733             provider->scheduleTerminatePhase2();
 734         }
 735         if( adjParent) provider->_adjustBusy( -1 );
 736         if( (provider->__state[1] & kIOServiceTermPhase3State)
 737          && (0 == provider->getClient())) {
 738             provider->scheduleFinalize(false);
 739         }
 740
 741         IOLockLock( gIOServiceBusyLock );
 742         if (kIOServiceWaitDetachState & provider->__state[1])
 743         {
 744             provider->__state[1] &= ~kIOServiceWaitDetachState;
 745             thread_wakeup(&provider->__provider);
 746         }
 747         IOLockUnlock( gIOServiceBusyLock );
 748
 749         provider->unlockForArbitration();
 750     }
 751 }
 752
 753 /*
 754  * Register instance - publish it for matching
 755  */
 756
 757 void IOService::registerService( IOOptionBits options )
 758 {
 759     char *              pathBuf;
 760     const char *        path;
 761     char *              skip;
 762     int                 len;
 763     enum { kMaxPathLen  = 256 };
 764     enum { kMaxChars    = 63 };
 765
 766     IORegistryEntry * parent = this;
 767     IORegistryEntry * root = getRegistryRoot();
 768     while( parent && (parent != root))
 769         parent = parent->getParentEntry( gIOServicePlane);
 770
 771     if( parent != root) {
 772         IOLog("%s: not registry member at registerService()\n", getName());
 773         return;
 774     }
 775
 776     // Allow the Platform Expert to adjust this node.
 777     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 778         return;
 779
 780     IOInstallServicePlatformActions(this);
 781
 782     if( (this != gIOResources)
 783      && (kIOLogRegister & gIOKitDebug)) {
 784
 785         pathBuf = (char *) IOMalloc( kMaxPathLen );
 786
 787         IOLog( "Registering: " );
 788
 789         len = kMaxPathLen;
 790         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 791
 792             path = pathBuf;
 793             if( len > kMaxChars) {
 794                 IOLog("..");
 795                 len -= kMaxChars;
 796                 path += len;
 797                 if( (skip = strchr( path, '/')))
 798                     path = skip;
 799             }
 800         } else
 801             path = getName();
 802
 803         IOLog( "%s\n", path );
 804
 805         if( pathBuf)
 806             IOFree( pathBuf, kMaxPathLen );
 807     }
 808
 809     startMatching( options );
 810 }
 811
 812 void IOService::startMatching( IOOptionBits options )
 813 {
 814     IOService * provider;
 815     UInt32      prevBusy = 0;
 816     bool        needConfig;
 817     bool        needWake = false;
 818     bool        ok;
 819     bool        sync;
 820     bool        waitAgain;
 821
 822     lockForArbitration();
 823
 824     sync = (options & kIOServiceSynchronous)
 825         || ((provider = getProvider())
 826                 && (provider->__state[1] & kIOServiceSynchronousState));
 827
 828         if ( options & kIOServiceAsynchronous )
 829                 sync = false;
 830
 831     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
 832                && (0 == (__state[0] & kIOServiceInactiveState));
 833
 834     __state[1] |= kIOServiceNeedConfigState;
 835
 836 //    __state[0] &= ~kIOServiceInactiveState;
 837
 838 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 839 //                      OSKernelStackRemaining(), getName());
 840
 841     if( needConfig) {
 842         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 843     }
 844
 845     if( sync)
 846         __state[1] |= kIOServiceSynchronousState;
 847     else
 848         __state[1] &= ~kIOServiceSynchronousState;
 849
 850     if( needConfig) prevBusy = _adjustBusy( 1 );
 851
 852     unlockForArbitration();
 853
 854     if( needConfig) {
 855
 856         if( needWake) {
 857             IOLockLock( gIOServiceBusyLock );
 858             thread_wakeup( (event_t) this/*&__state[1]*/ );
 859             IOLockUnlock( gIOServiceBusyLock );
 860
 861         } else if( !sync || (kIOServiceAsynchronous & options)) {
 862
 863             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 864
 865         } else do {
 866
 867             if( (__state[1] & kIOServiceNeedConfigState))
 868                 doServiceMatch( options );
 869
 870             lockForArbitration();
 871             IOLockLock( gIOServiceBusyLock );
 872
 873             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 874                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 875
 876             if( waitAgain)
 877                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 878             else
 879                 __state[1] &= ~kIOServiceSyncPubState;
 880
 881             unlockForArbitration();
 882
 883             if( waitAgain)
 884                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 885
 886             IOLockUnlock( gIOServiceBusyLock );
 887             if( waitAgain)
 888                 thread_block(THREAD_CONTINUE_NULL);
 889
 890         } while( waitAgain );
 891     }
 892 }
 893
 894 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 895 {
 896     OSDictionary *      table;
 897     OSSet *             set;
 898     OSSet *             allSet = 0;
 899     IOService *         service;
 900 #if IOMATCHDEBUG
 901     SInt32              count = 0;
 902 #endif
 903
 904     newTables->retain();
 905
 906     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 907
 908         LOCKWRITENOTIFY();
 909         set = (OSSet *) copyExistingServices( table,
 910                                                 kIOServiceRegisteredState,
 911                                                 kIOServiceExistingSet);
 912         UNLOCKNOTIFY();
 913         if( set) {
 914
 915 #if IOMATCHDEBUG
 916             count += set->getCount();
 917 #endif
 918             if (allSet) {
 919                 allSet->merge((const OSSet *) set);
 920                 set->release();
 921             }
 922             else
 923                 allSet = set;
 924         }
 925
 926 #if IOMATCHDEBUG
 927         if( getDebugFlags( table ) & kIOLogMatch)
 928             LOG("Matching service count = %ld\n", (long)count);
 929 #endif
 930         newTables->removeObject(table);
 931     }
 932
 933     if (allSet) {
 934         while( (service = (IOService *) allSet->getAnyObject())) {
 935             service->startMatching(kIOServiceAsynchronous);
 936             allSet->removeObject(service);
 937         }
 938         allSet->release();
 939     }
 940
 941     newTables->release();
 942
 943     return( kIOReturnSuccess );
 944 }
 945
 946  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 947                                                 IOOptionBits options )
 948 {
 949     _IOServiceJob *     job;
 950
 951     job = new _IOServiceJob;
 952     if( job && !job->init()) {
 953         job->release();
 954         job = 0;
 955     }
 956
 957     if( job) {
 958         job->type       = type;
 959         job->nub        = nub;
 960         job->options    = options;
 961         nub->retain();                  // thread will release()
 962         pingConfig( job );
 963     }
 964
 965     return( job );
 966 }
 967
 968 /*
 969  * Called on a registered service to see if it matches
 970  * a property table.
 971  */
 972
 973 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 974 {
 975     return( matchPropertyTable(table) );
 976 }
 977
 978 bool IOService::matchPropertyTable( OSDictionary * table )
 979 {
 980     return( true );
 981 }
 982
 983 /*
 984  * Called on a matched service to allocate resources
 985  * before first driver is attached.
 986  */
 987
 988 IOReturn IOService::getResources( void )
 989 {
 990     return( kIOReturnSuccess);
 991 }
 992
 993 /*
 994  * Client/provider accessors
 995  */
 996
 997 IOService * IOService::getProvider( void ) const
 998 {
 999     IOService * self = (IOService *) this;
1000     IOService * parent;
1001     SInt32      generation;
1002
1003     generation = getRegistryEntryGenerationCount();
1004     if( __providerGeneration == generation)
1005         return( __provider );
1006
1007     parent = (IOService *) getParentEntry( gIOServicePlane);
1008     if( parent == IORegistryEntry::getRegistryRoot())
1009         /* root is not an IOService */
1010         parent = 0;
1011
1012     self->__provider = parent;
1013     OSMemoryBarrier();
1014     // save the count from before call to getParentEntry()
1015     self->__providerGeneration = generation;
1016
1017     return( parent );
1018 }
1019
1020 IOWorkLoop * IOService::getWorkLoop() const
1021 {
1022     IOService *provider = getProvider();
1023
1024     if (provider)
1025         return provider->getWorkLoop();
1026     else
1027         return 0;
1028 }
1029
1030 OSIterator * IOService::getProviderIterator( void ) const
1031 {
1032     return( getParentIterator( gIOServicePlane));
1033 }
1034
1035 IOService * IOService::getClient( void ) const
1036 {
1037     return( (IOService *) getChildEntry( gIOServicePlane));
1038 }
1039
1040 OSIterator * IOService::getClientIterator( void ) const
1041 {
1042     return( getChildIterator( gIOServicePlane));
1043 }
1044
1045 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
1046                                                 const IOService * client,
1047                                                 const IOService * provider )
1048 {
1049     _IOOpenServiceIterator * inst;
1050
1051     if( !_iter)
1052         return( 0 );
1053
1054     inst = new _IOOpenServiceIterator;
1055
1056     if( inst && !inst->init()) {
1057         inst->release();
1058         inst = 0;
1059     }
1060     if( inst) {
1061         inst->iter = _iter;
1062         inst->client = client;
1063         inst->provider = provider;
1064     }
1065
1066     return( inst );
1067 }
1068
1069 void _IOOpenServiceIterator::free()
1070 {
1071     iter->release();
1072     if( last)
1073         last->unlockForArbitration();
1074     OSIterator::free();
1075 }
1076
1077 OSObject * _IOOpenServiceIterator::getNextObject()
1078 {
1079     IOService * next;
1080
1081     if( last)
1082         last->unlockForArbitration();
1083
1084     while( (next = (IOService *) iter->getNextObject())) {
1085
1086         next->lockForArbitration();
1087         if( (client && (next->isOpen( client )))
1088          || (provider && (provider->isOpen( next ))) )
1089             break;
1090         next->unlockForArbitration();
1091     }
1092
1093     last = next;
1094
1095     return( next );
1096 }
1097
1098 bool _IOOpenServiceIterator::isValid()
1099 {
1100     return( iter->isValid() );
1101 }
1102
1103 void _IOOpenServiceIterator::reset()
1104 {
1105     if( last) {
1106         last->unlockForArbitration();
1107         last = 0;
1108     }
1109     iter->reset();
1110 }
1111
1112 OSIterator * IOService::getOpenProviderIterator( void ) const
1113 {
1114     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1115 }
1116
1117 OSIterator * IOService::getOpenClientIterator( void ) const
1118 {
1119     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1120 }
1121
1122
1123 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1124                                           bool waitForFunction,
1125                                           void *param1, void *param2,
1126                                           void *param3, void *param4 )
1127 {
1128   IOReturn  result = kIOReturnUnsupported;
1129   IOService *provider;
1130
1131   if (gIOPlatformFunctionHandlerSet == functionName)
1132   {
1133 #if defined(__i386__) || defined(__x86_64__)
1134     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1135     IOService *      target              = (IOService *) param2;
1136     bool             enable              = (param3 != 0);
1137
1138     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1139         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1140     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1141         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1142 #endif /* defined(__i386__) || defined(__x86_64__) */
1143   }
1144
1145   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1146     result = provider->callPlatformFunction(functionName, waitForFunction,
1147                                             param1, param2, param3, param4);
1148   }
1149
1150   return result;
1151 }
1152
1153 IOReturn IOService::callPlatformFunction( const char * functionName,
1154                                           bool waitForFunction,
1155                                           void *param1, void *param2,
1156                                           void *param3, void *param4 )
1157 {
1158   IOReturn result = kIOReturnNoMemory;
1159   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1160
1161   if (functionSymbol != 0) {
1162     result = callPlatformFunction(functionSymbol, waitForFunction,
1163                                   param1, param2, param3, param4);
1164     functionSymbol->release();
1165   }
1166
1167   return result;
1168 }
1169
1170
1171 /*
1172  * Accessors for global services
1173  */
1174
1175 IOPlatformExpert * IOService::getPlatform( void )
1176 {
1177     return( gIOPlatform);
1178 }
1179
1180 class IOPMrootDomain * IOService::getPMRootDomain( void )
1181 {
1182     return( gIOPMRootDomain);
1183 }
1184
1185 IOService * IOService::getResourceService( void )
1186 {
1187     return( gIOResources );
1188 }
1189
1190 void IOService::setPlatform( IOPlatformExpert * platform)
1191 {
1192     gIOPlatform = platform;
1193     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1194
1195 #if defined(__i386__) || defined(__x86_64__)
1196
1197     static const char * keys[kCpuNumDelayTypes] = {
1198         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1199     const OSObject * objs[2];
1200     OSArray * array;
1201     uint32_t  idx;
1202
1203     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1204     {
1205         objs[0] = sCPULatencySet[idx];
1206         objs[1] = sCPULatencyHolder[idx];
1207         array   = OSArray::withObjects(objs, 2);
1208         if (!array) break;
1209         platform->setProperty(keys[idx], array);
1210         array->release();
1211     }
1212 #endif /* defined(__i386__) || defined(__x86_64__) */
1213 }
1214
1215 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1216 {
1217     gIOPMRootDomain = rootDomain;
1218     publishResource("IOKit");
1219 }
1220
1221 /*
1222  * Stacking change
1223  */
1224
1225 bool IOService::lockForArbitration( bool isSuccessRequired )
1226 {
1227     bool                          found;
1228     bool                          success;
1229     ArbitrationLockQueueElement * element;
1230     ArbitrationLockQueueElement * active;
1231     ArbitrationLockQueueElement * waiting;
1232
1233     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1234
1235     // lock global access
1236     IOTakeLock( gArbitrationLockQueueLock );
1237
1238     // obtain an unused queue element
1239     if( !queue_empty( &gArbitrationLockQueueFree )) {
1240         queue_remove_first( &gArbitrationLockQueueFree,
1241                             element,
1242                             ArbitrationLockQueueElement *,
1243                             link );
1244     } else {
1245         element = IONew( ArbitrationLockQueueElement, 1 );
1246         assert( element );
1247     }
1248
1249     // prepare the queue element
1250     element->thread   = IOThreadSelf();
1251     element->service  = this;
1252     element->count    = 1;
1253     element->required = isSuccessRequired;
1254     element->aborted  = false;
1255
1256     // determine whether this object is already locked (ie. on active queue)
1257     found = false;
1258     queue_iterate( &gArbitrationLockQueueActive,
1259                     active,
1260                     ArbitrationLockQueueElement *,
1261                     link )
1262     {
1263         if( active->service == element->service ) {
1264             found = true;
1265             break;
1266         }
1267     }
1268
1269     if( found ) { // this object is already locked
1270
1271         // determine whether it is the same or a different thread trying to lock
1272         if( active->thread != element->thread ) { // it is a different thread
1273
1274             ArbitrationLockQueueElement * victim = 0;
1275
1276             // before placing this new thread on the waiting queue, we look for
1277             // a deadlock cycle...
1278
1279             while( 1 ) {
1280                 // determine whether the active thread holding the object we
1281                 // want is waiting for another object to be unlocked
1282                 found = false;
1283                 queue_iterate( &gArbitrationLockQueueWaiting,
1284                                waiting,
1285                                ArbitrationLockQueueElement *,
1286                                link )
1287                 {
1288                     if( waiting->thread == active->thread ) {
1289                         assert( false == waiting->aborted );
1290                         found = true;
1291                         break;
1292                     }
1293                 }
1294
1295                 if( found ) { // yes, active thread waiting for another object
1296
1297                     // this may be a candidate for rejection if the required
1298                     // flag is not set, should we detect a deadlock later on
1299                     if( false == waiting->required )
1300                         victim = waiting;
1301
1302                     // find the thread that is holding this other object, that
1303                     // is blocking the active thread from proceeding (fun :-)
1304                     found = false;
1305                     queue_iterate( &gArbitrationLockQueueActive,
1306                                    active,      // (reuse active queue element)
1307                                    ArbitrationLockQueueElement *,
1308                                    link )
1309                     {
1310                         if( active->service == waiting->service ) {
1311                             found = true;
1312                             break;
1313                         }
1314                     }
1315
1316                     // someone must be holding it or it wouldn't be waiting
1317                     assert( found );
1318
1319                     if( active->thread == element->thread ) {
1320
1321                         // doh, it's waiting for the thread that originated
1322                         // this whole lock (ie. current thread) -> deadlock
1323                         if( false == element->required ) { // willing to fail?
1324
1325                             // the originating thread doesn't have the required
1326                             // flag, so it can fail
1327                             success = false; // (fail originating lock request)
1328                             break; // (out of while)
1329
1330                         } else { // originating thread is not willing to fail
1331
1332                             // see if we came across a waiting thread that did
1333                             // not have the 'required' flag set: we'll fail it
1334                             if( victim ) {
1335
1336                                 // we do have a willing victim, fail it's lock
1337                                 victim->aborted = true;
1338
1339                                 // take the victim off the waiting queue
1340                                 queue_remove( &gArbitrationLockQueueWaiting,
1341                                               victim,
1342                                               ArbitrationLockQueueElement *,
1343                                               link );
1344
1345                                 // wake the victim
1346                                 IOLockWakeup( gArbitrationLockQueueLock,
1347                                               victim,
1348                                               /* one thread */ true );
1349
1350                                 // allow this thread to proceed (ie. wait)
1351                                 success = true; // (put request on wait queue)
1352                                 break; // (out of while)
1353                             } else {
1354
1355                                 // all the waiting threads we came across in
1356                                 // finding this loop had the 'required' flag
1357                                 // set, so we've got a deadlock we can't avoid
1358                                 panic("I/O Kit: Unrecoverable deadlock.");
1359                             }
1360                         }
1361                     } else {
1362                         // repeat while loop, redefining active thread to be the
1363                         // thread holding "this other object" (see above), and
1364                         // looking for threads waiting on it; note the active
1365                         // variable points to "this other object" already... so
1366                         // there nothing to do in this else clause.
1367                     }
1368                 } else { // no, active thread is not waiting for another object
1369
1370                     success = true; // (put request on wait queue)
1371                     break; // (out of while)
1372                 }
1373             } // while forever
1374
1375             if( success ) { // put the request on the waiting queue?
1376                 kern_return_t wait_result;
1377
1378                 // place this thread on the waiting queue and put it to sleep;
1379                 // we place it at the tail of the queue...
1380                 queue_enter( &gArbitrationLockQueueWaiting,
1381                              element,
1382                              ArbitrationLockQueueElement *,
1383                              link );
1384
1385                 // declare that this thread will wait for a given event
1386 restart_sleep:  wait_result = assert_wait( element,
1387                                            element->required ? THREAD_UNINT
1388                                            : THREAD_INTERRUPTIBLE );
1389
1390                 // unlock global access
1391                 IOUnlock( gArbitrationLockQueueLock );
1392
1393                 // put thread to sleep, waiting for our event to fire...
1394                 if (wait_result == THREAD_WAITING)
1395                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1396
1397
1398                 // ...and we've been woken up; we might be in one of two states:
1399                 // (a) we've been aborted and our queue element is not on
1400                 //     any of the three queues, but is floating around
1401                 // (b) we're allowed to proceed with the lock and we have
1402                 //     already been moved from the waiting queue to the
1403                 //     active queue.
1404                 // ...plus a 3rd state, should the thread have been interrupted:
1405                 // (c) we're still on the waiting queue
1406
1407                 // determine whether we were interrupted out of our sleep
1408                 if( THREAD_INTERRUPTED == wait_result ) {
1409
1410                     // re-lock global access
1411                     IOTakeLock( gArbitrationLockQueueLock );
1412
1413                     // determine whether we're still on the waiting queue
1414                     found = false;
1415                     queue_iterate( &gArbitrationLockQueueWaiting,
1416                                    waiting,     // (reuse waiting queue element)
1417                                    ArbitrationLockQueueElement *,
1418                                    link )
1419                     {
1420                         if( waiting == element ) {
1421                             found = true;
1422                             break;
1423                         }
1424                     }
1425
1426                     if( found ) { // yes, we're still on the waiting queue
1427
1428                         // determine whether we're willing to fail
1429                         if( false == element->required ) {
1430
1431                             // mark us as aborted
1432                             element->aborted = true;
1433
1434                             // take us off the waiting queue
1435                             queue_remove( &gArbitrationLockQueueWaiting,
1436                                           element,
1437                                           ArbitrationLockQueueElement *,
1438                                           link );
1439                         } else { // we are not willing to fail
1440
1441                             // ignore interruption, go back to sleep
1442                             goto restart_sleep;
1443                         }
1444                     }
1445
1446                     // unlock global access
1447                     IOUnlock( gArbitrationLockQueueLock );
1448
1449                     // proceed as though this were a normal wake up
1450                     wait_result = THREAD_AWAKENED;
1451                 }
1452
1453                 assert( THREAD_AWAKENED == wait_result );
1454
1455                 // determine whether we've been aborted while we were asleep
1456                 if( element->aborted ) {
1457                     assert( false == element->required );
1458
1459                     // re-lock global access
1460                     IOTakeLock( gArbitrationLockQueueLock );
1461
1462                     action = kPutOnFreeQueue;
1463                     success = false;
1464                 } else { // we weren't aborted, so we must be ready to go :-)
1465
1466                     // we've already been moved from waiting to active queue
1467                     return true;
1468                 }
1469
1470             } else { // the lock request is to be failed
1471
1472                 // return unused queue element to queue
1473                 action = kPutOnFreeQueue;
1474             }
1475         } else { // it is the same thread, recursive access is allowed
1476
1477             // add one level of recursion
1478             active->count++;
1479
1480             // return unused queue element to queue
1481             action = kPutOnFreeQueue;
1482             success = true;
1483         }
1484     } else { // this object is not already locked, so let this thread through
1485         action = kPutOnActiveQueue;
1486         success = true;
1487     }
1488
1489     // put the new element on a queue
1490     if( kPutOnActiveQueue == action ) {
1491         queue_enter( &gArbitrationLockQueueActive,
1492                      element,
1493                      ArbitrationLockQueueElement *,
1494                      link );
1495     } else if( kPutOnFreeQueue == action ) {
1496         queue_enter( &gArbitrationLockQueueFree,
1497                      element,
1498                      ArbitrationLockQueueElement *,
1499                      link );
1500     } else {
1501         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1502     }
1503
1504     // unlock global access
1505     IOUnlock( gArbitrationLockQueueLock );
1506
1507     return( success );
1508 }
1509
1510 void IOService::unlockForArbitration( void )
1511 {
1512     bool                          found;
1513     ArbitrationLockQueueElement * element;
1514
1515     // lock global access
1516     IOTakeLock( gArbitrationLockQueueLock );
1517
1518     // find the lock element for this object (ie. on active queue)
1519     found = false;
1520     queue_iterate( &gArbitrationLockQueueActive,
1521                     element,
1522                     ArbitrationLockQueueElement *,
1523                     link )
1524     {
1525         if( element->service == this ) {
1526             found = true;
1527             break;
1528         }
1529     }
1530
1531     assert( found );
1532
1533     // determine whether the lock has been taken recursively
1534     if( element->count > 1 ) {
1535         // undo one level of recursion
1536         element->count--;
1537
1538     } else {
1539
1540         // remove it from the active queue
1541         queue_remove( &gArbitrationLockQueueActive,
1542                       element,
1543                       ArbitrationLockQueueElement *,
1544                       link );
1545
1546         // put it on the free queue
1547         queue_enter( &gArbitrationLockQueueFree,
1548                      element,
1549                      ArbitrationLockQueueElement *,
1550                      link );
1551
1552         // determine whether a thread is waiting for object (head to tail scan)
1553         found = false;
1554         queue_iterate( &gArbitrationLockQueueWaiting,
1555                        element,
1556                        ArbitrationLockQueueElement *,
1557                        link )
1558         {
1559             if( element->service == this ) {
1560                 found = true;
1561                 break;
1562             }
1563         }
1564
1565         if ( found ) { // we found an interested thread on waiting queue
1566
1567             // remove it from the waiting queue
1568             queue_remove( &gArbitrationLockQueueWaiting,
1569                           element,
1570                           ArbitrationLockQueueElement *,
1571                           link );
1572
1573             // put it on the active queue
1574             queue_enter( &gArbitrationLockQueueActive,
1575                          element,
1576                          ArbitrationLockQueueElement *,
1577                          link );
1578
1579             // wake the waiting thread
1580             IOLockWakeup( gArbitrationLockQueueLock,
1581                           element,
1582                           /* one thread */ true );
1583         }
1584     }
1585
1586     // unlock global access
1587     IOUnlock( gArbitrationLockQueueLock );
1588 }
1589
1590 uint32_t IOService::isLockedForArbitration(IOService * service)
1591 {
1592 #if DEBUG_NOTIFIER_LOCKED
1593     uint32_t                      count;
1594     ArbitrationLockQueueElement * active;
1595
1596     // lock global access
1597     IOLockLock(gArbitrationLockQueueLock);
1598
1599     // determine whether this object is already locked (ie. on active queue)
1600     count = 0;
1601     queue_iterate(&gArbitrationLockQueueActive,
1602                   active,
1603                   ArbitrationLockQueueElement *,
1604                   link)
1605     {
1606         if ((active->thread == IOThreadSelf())
1607             && (!service || (active->service == service)))
1608         {
1609             count += 0x10000;
1610             count += active->count;
1611         }
1612     }
1613
1614     IOLockUnlock(gArbitrationLockQueueLock);
1615
1616     return (count);
1617
1618 #else /* DEBUG_NOTIFIER_LOCKED */
1619
1620     return (0);
1621
1622 #endif /* DEBUG_NOTIFIER_LOCKED */
1623 }
1624
1625 void IOService::applyToProviders( IOServiceApplierFunction applier,
1626                                   void * context )
1627 {
1628     applyToParents( (IORegistryEntryApplierFunction) applier,
1629                     context, gIOServicePlane );
1630 }
1631
1632 void IOService::applyToClients( IOServiceApplierFunction applier,
1633                                 void * context )
1634 {
1635     applyToChildren( (IORegistryEntryApplierFunction) applier,
1636                      context, gIOServicePlane );
1637 }
1638
1639
1640 /*
1641  * Client messages
1642  */
1643
1644
1645 // send a message to a client or interested party of this service
1646 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1647                                    void * argument, vm_size_t argSize )
1648 {
1649     IOReturn                            ret;
1650     IOService *                         service;
1651     _IOServiceInterestNotifier *        notify;
1652
1653     if( (service = OSDynamicCast( IOService, client)))
1654         ret = service->message( type, this, argument );
1655
1656     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1657
1658         _IOServiceNotifierInvocation invocation;
1659         bool                     willNotify;
1660
1661         invocation.thread = current_thread();
1662
1663         LOCKWRITENOTIFY();
1664         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1665
1666         if( willNotify) {
1667             queue_enter( &notify->handlerInvocations, &invocation,
1668                             _IOServiceNotifierInvocation *, link );
1669         }
1670         UNLOCKNOTIFY();
1671
1672         if( willNotify) {
1673
1674             ret = (*notify->handler)( notify->target, notify->ref,
1675                                           type, this, argument, argSize );
1676
1677             LOCKWRITENOTIFY();
1678             queue_remove( &notify->handlerInvocations, &invocation,
1679                             _IOServiceNotifierInvocation *, link );
1680             if( kIOServiceNotifyWaiter & notify->state) {
1681                 notify->state &= ~kIOServiceNotifyWaiter;
1682                 WAKEUPNOTIFY( notify );
1683             }
1684             UNLOCKNOTIFY();
1685
1686         } else
1687             ret = kIOReturnSuccess;
1688
1689     } else
1690         ret = kIOReturnBadArgument;
1691
1692     return( ret );
1693 }
1694
1695 static void
1696 applyToInterestNotifiers(const IORegistryEntry *target,
1697                          const OSSymbol * typeOfInterest,
1698                          OSObjectApplierFunction applier,
1699                          void * context )
1700 {
1701     OSArray *  copyArray = 0;
1702     OSObject * prop;
1703
1704     LOCKREADNOTIFY();
1705
1706     prop = target->copyProperty(typeOfInterest);
1707     IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
1708
1709     if( notifyList) {
1710         copyArray = OSArray::withCapacity(1);
1711
1712         // iterate over queue, entry is set to each element in the list
1713         iterqueue(&notifyList->fCommandChain, entry) {
1714             _IOServiceInterestNotifier * notify;
1715
1716             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1717             copyArray->setObject(notify);
1718         }
1719     }
1720     UNLOCKNOTIFY();
1721
1722     if( copyArray) {
1723         unsigned int    index;
1724         OSObject *      next;
1725
1726         for( index = 0; (next = copyArray->getObject( index )); index++)
1727             (*applier)(next, context);
1728         copyArray->release();
1729     }
1730
1731     OSSafeReleaseNULL(prop);
1732 }
1733
1734 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1735                                    OSObjectApplierFunction applier,
1736                                    void * context )
1737 {
1738     if (gIOGeneralInterest == typeOfInterest)
1739         applyToClients( (IOServiceApplierFunction) applier, context );
1740     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1741 }
1742
1743 struct MessageClientsContext {
1744     IOService * service;
1745     UInt32      type;
1746     void *      argument;
1747     vm_size_t   argSize;
1748     IOReturn    ret;
1749 };
1750
1751 static void messageClientsApplier( OSObject * object, void * ctx )
1752 {
1753     IOReturn                ret;
1754     MessageClientsContext * context = (MessageClientsContext *) ctx;
1755
1756     ret = context->service->messageClient( context->type,
1757                                            object, context->argument, context->argSize );
1758     if( kIOReturnSuccess != ret)
1759         context->ret = ret;
1760 }
1761
1762 // send a message to all clients
1763 IOReturn IOService::messageClients( UInt32 type,
1764                                     void * argument, vm_size_t argSize )
1765 {
1766     MessageClientsContext       context;
1767
1768     context.service     = this;
1769     context.type        = type;
1770     context.argument    = argument;
1771     context.argSize     = argSize;
1772     context.ret         = kIOReturnSuccess;
1773
1774     applyToInterested( gIOGeneralInterest,
1775                        &messageClientsApplier, &context );
1776
1777     return( context.ret );
1778 }
1779
1780 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1781                                               IOOptionBits response )
1782 {
1783     return( kIOReturnUnsupported );
1784 }
1785
1786 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1787                   IOServiceInterestHandler handler, void * target, void * ref )
1788 {
1789     _IOServiceInterestNotifier * notify = 0;
1790     IOReturn rc = kIOReturnError;
1791
1792     notify = new _IOServiceInterestNotifier;
1793     if (!notify) return NULL;
1794
1795     if(notify->init()) {
1796         rc = registerInterestForNotifier(notify, typeOfInterest,
1797                               handler, target, ref);
1798     }
1799
1800     if (rc != kIOReturnSuccess) {
1801         notify->release();
1802         notify = 0;
1803     }
1804
1805     return( notify );
1806 }
1807
1808 IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1809                   IOServiceInterestHandler handler, void * target, void * ref )
1810 {
1811     IOReturn rc = kIOReturnSuccess;
1812     _IOServiceInterestNotifier  *notify = 0;
1813
1814     if( (typeOfInterest != gIOGeneralInterest)
1815      && (typeOfInterest != gIOBusyInterest)
1816      && (typeOfInterest != gIOAppPowerStateInterest)
1817      && (typeOfInterest != gIOConsoleSecurityInterest)
1818      && (typeOfInterest != gIOPriorityPowerStateInterest))
1819         return( kIOReturnBadArgument );
1820
1821     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1822         return( kIOReturnBadArgument );
1823
1824     lockForArbitration();
1825     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1826
1827         notify->handler = handler;
1828         notify->target = target;
1829         notify->ref = ref;
1830         notify->state = kIOServiceNotifyEnable;
1831
1832         ////// queue
1833
1834         LOCKWRITENOTIFY();
1835
1836         // Get the head of the notifier linked list
1837         IOCommand * notifyList;
1838         OSObject  * obj = copyProperty( typeOfInterest );
1839         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1840             notifyList = OSTypeAlloc(IOCommand);
1841             if (notifyList) {
1842                 notifyList->init();
1843                 bool ok = setProperty( typeOfInterest, notifyList);
1844                 notifyList->release();
1845                 if (!ok) notifyList = 0;
1846             }
1847         }
1848         if (obj) obj->release();
1849
1850         if (notifyList) {
1851             enqueue(&notifyList->fCommandChain, &notify->chain);
1852             notify->retain();   // ref'ed while in list
1853         }
1854
1855         UNLOCKNOTIFY();
1856     }
1857     else {
1858         rc = kIOReturnNotReady;
1859     }
1860     unlockForArbitration();
1861
1862     return rc;
1863 }
1864
1865 static void cleanInterestList( OSObject * head )
1866 {
1867     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1868     if (!notifyHead)
1869         return;
1870
1871     LOCKWRITENOTIFY();
1872     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1873         queue_next(entry) = queue_prev(entry) = 0;
1874
1875         _IOServiceInterestNotifier * notify;
1876
1877         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1878         notify->release();
1879     }
1880     UNLOCKNOTIFY();
1881 }
1882
1883 void IOService::unregisterAllInterest( void )
1884 {
1885     OSObject * prop;
1886
1887     prop = copyProperty(gIOGeneralInterest);
1888     cleanInterestList(prop);
1889     OSSafeReleaseNULL(prop);
1890
1891     prop = copyProperty(gIOBusyInterest);
1892     cleanInterestList(prop);
1893     OSSafeReleaseNULL(prop);
1894
1895     prop = copyProperty(gIOAppPowerStateInterest);
1896     cleanInterestList(prop);
1897     OSSafeReleaseNULL(prop);
1898
1899     prop = copyProperty(gIOPriorityPowerStateInterest);
1900     cleanInterestList(prop);
1901     OSSafeReleaseNULL(prop);
1902
1903     prop = copyProperty(gIOConsoleSecurityInterest);
1904     cleanInterestList(prop);
1905     OSSafeReleaseNULL(prop);
1906 }
1907
1908 /*
1909  * _IOServiceInterestNotifier
1910  */
1911
1912 // wait for all threads, other than the current one,
1913 //  to exit the handler
1914
1915 void _IOServiceInterestNotifier::wait()
1916 {
1917     _IOServiceNotifierInvocation * next;
1918     bool doWait;
1919
1920     do {
1921         doWait = false;
1922         queue_iterate( &handlerInvocations, next,
1923                         _IOServiceNotifierInvocation *, link) {
1924             if( next->thread != current_thread() ) {
1925                 doWait = true;
1926                 break;
1927             }
1928         }
1929         if( doWait) {
1930             state |= kIOServiceNotifyWaiter;
1931                SLEEPNOTIFY(this);
1932         }
1933
1934     } while( doWait );
1935 }
1936
1937 void _IOServiceInterestNotifier::free()
1938 {
1939     assert( queue_empty( &handlerInvocations ));
1940     OSObject::free();
1941 }
1942
1943 void _IOServiceInterestNotifier::remove()
1944 {
1945     LOCKWRITENOTIFY();
1946
1947     if( queue_next( &chain )) {
1948         remqueue(&chain);
1949         queue_next( &chain) = queue_prev( &chain) = 0;
1950         release();
1951     }
1952
1953     state &= ~kIOServiceNotifyEnable;
1954
1955     wait();
1956
1957     UNLOCKNOTIFY();
1958
1959     release();
1960 }
1961
1962 bool _IOServiceInterestNotifier::disable()
1963 {
1964     bool        ret;
1965
1966     LOCKWRITENOTIFY();
1967
1968     ret = (0 != (kIOServiceNotifyEnable & state));
1969     state &= ~kIOServiceNotifyEnable;
1970     if( ret)
1971         wait();
1972
1973     UNLOCKNOTIFY();
1974
1975     return( ret );
1976 }
1977
1978 void _IOServiceInterestNotifier::enable( bool was )
1979 {
1980     LOCKWRITENOTIFY();
1981     if( was)
1982         state |= kIOServiceNotifyEnable;
1983     else
1984         state &= ~kIOServiceNotifyEnable;
1985     UNLOCKNOTIFY();
1986 }
1987
1988 bool _IOServiceInterestNotifier::init()
1989 {
1990     queue_init( &handlerInvocations );
1991     return (OSObject::init());
1992 }
1993 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1994
1995 /*
1996  * Termination
1997  */
1998
1999 #define tailQ(o)                setObject(o)
2000 #define headQ(o)                setObject(0, o)
2001 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2002
2003 static void _workLoopAction( IOWorkLoop::Action action,
2004                              IOService * service,
2005                              void * p0 = 0, void * p1 = 0,
2006                              void * p2 = 0, void * p3 = 0 )
2007 {
2008     IOWorkLoop * wl;
2009
2010     if( (wl = service->getWorkLoop())) {
2011         wl->retain();
2012         wl->runAction( action, service, p0, p1, p2, p3 );
2013         wl->release();
2014     } else
2015         (*action)( service, p0, p1, p2, p3 );
2016 }
2017
2018 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
2019 {
2020     bool ok;
2021
2022     // if its our only provider
2023     ok = isParent( provider, gIOServicePlane, true);
2024
2025     // -- compat
2026     if( ok) {
2027         provider->terminateClient( this, options | kIOServiceRecursing );
2028         ok = (0 != (kIOServiceInactiveState & __state[0]));
2029     }
2030     // --
2031
2032     return( ok );
2033 }
2034
2035 bool IOService::terminatePhase1( IOOptionBits options )
2036 {
2037     IOService *  victim;
2038     IOService *  client;
2039     OSIterator * iter;
2040     OSArray *    makeInactive;
2041     OSArray *    waitingInactive;
2042     int          waitResult = THREAD_AWAKENED;
2043     bool         wait;
2044     bool                 ok;
2045     bool                 didInactive;
2046     bool                 startPhase2 = false;
2047
2048     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2049
2050     uint64_t regID = getRegistryEntryID();
2051     IOServiceTrace(
2052         IOSERVICE_TERMINATE_PHASE1,
2053         (uintptr_t) regID,
2054         (uintptr_t) (regID >> 32),
2055         (uintptr_t) this,
2056         (uintptr_t) options);
2057
2058     // -- compat
2059     if( options & kIOServiceRecursing) {
2060         lockForArbitration();
2061         if (0 == (kIOServiceInactiveState & __state[0]))
2062         {
2063             __state[0] |= kIOServiceInactiveState;
2064             __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2065         }
2066         unlockForArbitration();
2067
2068         return( true );
2069     }
2070     // --
2071
2072     makeInactive    = OSArray::withCapacity( 16 );
2073     waitingInactive = OSArray::withCapacity( 16 );
2074     if(!makeInactive || !waitingInactive) return( false );
2075
2076     victim = this;
2077     victim->retain();
2078
2079     while( victim )
2080     {
2081         didInactive = victim->lockForArbitration( true );
2082         if( didInactive)
2083         {
2084             uint64_t regID1 = victim->getRegistryEntryID();
2085             IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2086                 (uintptr_t) regID1,
2087                 (uintptr_t) (regID1 >> 32),
2088                 (uintptr_t) victim->__state[1],
2089                 (uintptr_t) 0);
2090
2091             enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2092             didInactive = (kRP1 == (victim->__state[1] & kRP1))
2093                         || (0 == (victim->__state[0] & kIOServiceInactiveState));
2094
2095             if (!didInactive)
2096             {
2097                 // a multiply attached IOService can be visited twice
2098                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do
2099                 {
2100                     IOLockLock(gIOServiceBusyLock);
2101                     wait = (victim->__state[1] & kIOServiceTermPhase1State);
2102                     if( wait) {
2103                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2104                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2105                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2106                         victim->unlockForArbitration();
2107                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2108                     }
2109                     IOLockUnlock(gIOServiceBusyLock);
2110                     if( wait) {
2111                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2112                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2113                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2114                         victim->lockForArbitration();
2115                     }
2116                 }
2117                 while (wait && (waitResult != THREAD_TIMED_OUT));
2118             }
2119             else
2120             {
2121                 victim->__state[0] |= kIOServiceInactiveState;
2122                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2123                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2124                 victim->__state[1] &= ~kIOServiceRecursing;
2125                 victim->__state[1] |= kIOServiceTermPhase1State;
2126                 waitingInactive->headQ(victim);
2127                 if (victim == this)
2128                 {
2129                     if (kIOServiceTerminateNeedWillTerminate & options)
2130                     {
2131                         victim->__state[1] |= kIOServiceNeedWillTerminate;
2132                     }
2133                 }
2134                 victim->_adjustBusy( 1 );
2135             }
2136             victim->unlockForArbitration();
2137         }
2138         if( victim == this) startPhase2 = didInactive;
2139         if (didInactive)
2140         {
2141             OSArray * notifiers;
2142             notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2143             victim->invokeNotifiers(&notifiers);
2144
2145             IOUserClient::destroyUserReferences( victim );
2146
2147             iter = victim->getClientIterator();
2148             if( iter) {
2149                 while( (client = (IOService *) iter->getNextObject())) {
2150                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2151                             client->getName(), client->getRegistryEntryID(),
2152                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2153                     ok = client->requestTerminate( victim, options );
2154                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2155                             client->getName(), client->getRegistryEntryID(),
2156                             victim->getName(), victim->getRegistryEntryID(), ok);
2157
2158                     uint64_t regID1 = client->getRegistryEntryID();
2159                     uint64_t regID2 = victim->getRegistryEntryID();
2160                     IOServiceTrace(
2161                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2162                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
2163                         (uintptr_t) regID1,
2164                         (uintptr_t) (regID1 >> 32),
2165                         (uintptr_t) regID2,
2166                         (uintptr_t) (regID2 >> 32));
2167
2168                     if( ok)
2169                         makeInactive->setObject( client );
2170                 }
2171                 iter->release();
2172             }
2173         }
2174         victim->release();
2175         victim = (IOService *) makeInactive->getObject(0);
2176         if( victim) {
2177             victim->retain();
2178             makeInactive->removeObject(0);
2179         }
2180     }
2181     makeInactive->release();
2182
2183     while ((victim = (IOService *) waitingInactive->getObject(0)))
2184     {
2185         victim->retain();
2186         waitingInactive->removeObject(0);
2187
2188         victim->lockForArbitration();
2189         victim->__state[1] &= ~kIOServiceTermPhase1State;
2190         if (kIOServiceTerm1WaiterState & victim->__state[1])
2191         {
2192             victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2193             TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2194             IOLockLock( gIOServiceBusyLock );
2195             thread_wakeup( (event_t) &victim->__state[1]);
2196             IOLockUnlock( gIOServiceBusyLock );
2197         }
2198         victim->unlockForArbitration();
2199         victim->release();
2200     }
2201     waitingInactive->release();
2202
2203     if( startPhase2)
2204     {
2205         retain();
2206         lockForArbitration();
2207         scheduleTerminatePhase2(options);
2208         unlockForArbitration();
2209         release();
2210     }
2211
2212     return( true );
2213 }
2214
2215 void IOService::setTerminateDefer(IOService * provider, bool defer)
2216 {
2217     lockForArbitration();
2218     if (defer)  __state[1] |= kIOServiceStartState;
2219     else        __state[1] &= ~kIOServiceStartState;
2220     unlockForArbitration();
2221
2222     if (provider && !defer)
2223     {
2224         provider->lockForArbitration();
2225         provider->scheduleTerminatePhase2();
2226         provider->unlockForArbitration();
2227     }
2228 }
2229
2230 // Must call this while holding gJobsLock
2231 void IOService::waitToBecomeTerminateThread(void)
2232 {
2233     IOLockAssert(gJobsLock, kIOLockAssertOwned);
2234     bool wait;
2235     do {
2236         wait = (gIOTerminateThread != THREAD_NULL);
2237         if (wait) {
2238             IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2239         }
2240     } while (wait);
2241     gIOTerminateThread = current_thread();
2242 }
2243
2244 // call with lockForArbitration
2245 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2246 {
2247     AbsoluteTime        deadline;
2248     uint64_t            regID1;
2249     int                 waitResult = THREAD_AWAKENED;
2250     bool                wait = false, haveDeadline = false;
2251
2252     if (!(__state[0] & kIOServiceInactiveState)) return;
2253
2254     regID1 = getRegistryEntryID();
2255     IOServiceTrace(
2256         IOSERVICE_TERM_SCHED_PHASE2,
2257         (uintptr_t) regID1,
2258         (uintptr_t) (regID1 >> 32),
2259         (uintptr_t) __state[1],
2260         (uintptr_t) options);
2261
2262     if (__state[1] & kIOServiceTermPhase1State)         return;
2263
2264     retain();
2265     unlockForArbitration();
2266     options |= kIOServiceRequired;
2267     IOLockLock( gJobsLock );
2268
2269     if( (options & kIOServiceSynchronous)
2270         && (current_thread() != gIOTerminateThread)) {
2271
2272         waitToBecomeTerminateThread();
2273         gIOTerminatePhase2List->setObject( this );
2274         gIOTerminateWork++;
2275
2276         do {
2277             while( gIOTerminateWork )
2278                 terminateWorker( options );
2279             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2280             if( wait) {
2281                 // wait for the victim to go non-busy
2282                 if( !haveDeadline) {
2283                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2284                     haveDeadline = true;
2285                 }
2286                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2287                                                   deadline, THREAD_UNINT );
2288                 if(__improbable(waitResult == THREAD_TIMED_OUT)) {
2289                     panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2290                 }
2291             }
2292         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2293
2294         gIOTerminateThread = 0;
2295         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2296
2297     } else {
2298         // ! kIOServiceSynchronous
2299
2300         gIOTerminatePhase2List->setObject( this );
2301         if( 0 == gIOTerminateWork++) {
2302             assert(gIOTerminateWorkerThread);
2303             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2304         }
2305     }
2306
2307     IOLockUnlock( gJobsLock );
2308     lockForArbitration();
2309     release();
2310 }
2311
2312 __attribute__((__noreturn__))
2313 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2314 {
2315     // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2316     IOLockLock(gJobsLock);
2317     while (true) {
2318         if (gIOTerminateThread != gIOTerminateWorkerThread) {
2319             waitToBecomeTerminateThread();
2320         }
2321
2322         while (gIOTerminateWork)
2323             terminateWorker( (uintptr_t)arg );
2324
2325         gIOTerminateThread = 0;
2326         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2327         IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2328     }
2329 }
2330
2331 void IOService::scheduleStop( IOService * provider )
2332 {
2333     uint64_t regID1 = getRegistryEntryID();
2334     uint64_t regID2 = provider->getRegistryEntryID();
2335
2336     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2337     IOServiceTrace(
2338         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2339         (uintptr_t) regID1,
2340         (uintptr_t) (regID1 >> 32),
2341         (uintptr_t) regID2,
2342         (uintptr_t) (regID2 >> 32));
2343
2344     IOLockLock( gJobsLock );
2345     gIOStopList->tailQ( this );
2346     gIOStopProviderList->tailQ( provider );
2347
2348     if( 0 == gIOTerminateWork++) {
2349         assert(gIOTerminateWorkerThread);
2350         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2351     }
2352
2353     IOLockUnlock( gJobsLock );
2354 }
2355
2356 void IOService::scheduleFinalize(bool now)
2357 {
2358     uint64_t regID1 = getRegistryEntryID();
2359
2360     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2361     IOServiceTrace(
2362         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2363         (uintptr_t) regID1,
2364         (uintptr_t) (regID1 >> 32),
2365         0, 0);
2366
2367     if (now || IOUserClient::finalizeUserReferences(this))
2368     {
2369         IOLockLock( gJobsLock );
2370         gIOFinalizeList->tailQ(this);
2371         if( 0 == gIOTerminateWork++) {
2372             assert(gIOTerminateWorkerThread);
2373             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2374         }
2375         IOLockUnlock( gJobsLock );
2376     }
2377 }
2378
2379 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2380 {
2381     return( true );
2382 }
2383
2384 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2385 {
2386     if( false == *defer) {
2387
2388         if( lockForArbitration( true )) {
2389             if( false == provider->handleIsOpen( this ))
2390                 scheduleStop( provider );
2391             // -- compat
2392             else {
2393                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2394                 if( false == provider->handleIsOpen( this ))
2395                     scheduleStop( provider );
2396             }
2397             // --
2398             unlockForArbitration();
2399         }
2400     }
2401
2402     return( true );
2403 }
2404
2405 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2406                                      OSArray * doPhase2List,
2407                                      void *unused2 __unused,
2408                                      void *unused3 __unused  )
2409 {
2410     OSIterator * iter;
2411     IOService *  client;
2412     bool         ok;
2413     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2414
2415     iter = victim->getClientIterator();
2416     if( iter) {
2417         while( (client = (IOService *) iter->getNextObject())) {
2418
2419             regID1 = client->getRegistryEntryID();
2420             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2421                     client->getName(), regID1,
2422                     victim->getName(), regID2, (long long)options);
2423             IOServiceTrace(
2424                 IOSERVICE_TERMINATE_WILL,
2425                 (uintptr_t) regID1,
2426                 (uintptr_t) (regID1 >> 32),
2427                 (uintptr_t) regID2,
2428                 (uintptr_t) (regID2 >> 32));
2429
2430             ok = client->willTerminate( victim, options );
2431             doPhase2List->tailQ( client );
2432         }
2433         iter->release();
2434     }
2435 }
2436
2437 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2438                             void *unused1 __unused, void *unused2 __unused,
2439                             void *unused3 __unused )
2440 {
2441     OSIterator * iter;
2442     IOService *  client;
2443     bool         defer;
2444     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2445
2446     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2447
2448     iter = victim->getClientIterator();
2449     if( iter) {
2450         while( (client = (IOService *) iter->getNextObject())) {
2451
2452             regID1 = client->getRegistryEntryID();
2453             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2454                     client->getName(), regID1,
2455                     victim->getName(), regID2, (long long)options);
2456             defer = false;
2457             client->didTerminate( victim, options, &defer );
2458
2459             IOServiceTrace(
2460                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2461                        : IOSERVICE_TERMINATE_DID),
2462                 (uintptr_t) regID1,
2463                 (uintptr_t) (regID1 >> 32),
2464                 (uintptr_t) regID2,
2465                 (uintptr_t) (regID2 >> 32));
2466
2467             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2468                     client->getName(), regID1,
2469                     victim->getName(), regID2, defer);
2470         }
2471         iter->release();
2472     }
2473 }
2474
2475
2476 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2477                             void *unused1 __unused, void *unused2 __unused,
2478                             void *unused3 __unused )
2479 {
2480     OSIterator * iter;
2481     IOService *  provider;
2482     bool         ok;
2483     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2484
2485     iter = victim->getProviderIterator();
2486     if( iter) {
2487         while( (provider = (IOService *) iter->getNextObject())) {
2488
2489             regID1 = provider->getRegistryEntryID();
2490             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2491                     victim->getName(), regID2,
2492                     provider->getName(), regID1, (long long)options);
2493             IOServiceTrace(
2494                 IOSERVICE_TERMINATE_WILL,
2495                 (uintptr_t) regID2,
2496                 (uintptr_t) (regID2 >> 32),
2497                 (uintptr_t) regID1,
2498                 (uintptr_t) (regID1 >> 32));
2499
2500             ok = victim->willTerminate( provider, options );
2501         }
2502         iter->release();
2503     }
2504 }
2505
2506 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2507                             void *unused1 __unused, void *unused2 __unused,
2508                             void *unused3 __unused )
2509 {
2510     OSIterator * iter;
2511     IOService *  provider;
2512     bool defer = false;
2513     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2514
2515     iter = victim->getProviderIterator();
2516     if( iter) {
2517         while( (provider = (IOService *) iter->getNextObject())) {
2518
2519             regID1 = provider->getRegistryEntryID();
2520             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2521                     victim->getName(), regID2,
2522                     provider->getName(), regID1, (long long)options);
2523             victim->didTerminate( provider, options, &defer );
2524
2525             IOServiceTrace(
2526                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2527                        : IOSERVICE_TERMINATE_DID),
2528                 (uintptr_t) regID2,
2529                 (uintptr_t) (regID2 >> 32),
2530                 (uintptr_t) regID1,
2531                 (uintptr_t) (regID1 >> 32));
2532
2533             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2534                     victim->getName(), regID2,
2535                     provider->getName(), regID1, defer);
2536         }
2537         iter->release();
2538     }
2539 }
2540
2541
2542 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2543                             void *unused1 __unused, void *unused2 __unused,
2544                             void *unused3 __unused )
2545 {
2546     uint64_t regID1 = victim->getRegistryEntryID();
2547     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2548     IOServiceTrace(
2549         IOSERVICE_TERMINATE_FINALIZE,
2550         (uintptr_t) regID1,
2551         (uintptr_t) (regID1 >> 32),
2552         0, 0);
2553
2554     victim->finalize( options );
2555 }
2556
2557 void IOService::actionStop( IOService * provider, IOService * client,
2558                             void *unused1 __unused, void *unused2 __unused,
2559                             void *unused3 __unused )
2560 {
2561     uint64_t regID1 = provider->getRegistryEntryID();
2562     uint64_t regID2 = client->getRegistryEntryID();
2563
2564     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2565     IOServiceTrace(
2566         IOSERVICE_TERMINATE_STOP,
2567         (uintptr_t) regID1,
2568         (uintptr_t) (regID1 >> 32),
2569         (uintptr_t) regID2,
2570         (uintptr_t) (regID2 >> 32));
2571
2572     client->stop( provider );
2573     if( provider->isOpen( client ))
2574         provider->close( client );
2575
2576     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2577     client->detach( provider );
2578 }
2579
2580 void IOService::terminateWorker( IOOptionBits options )
2581 {
2582     OSArray *           doPhase2List;
2583     OSArray *           didPhase2List;
2584     OSSet *             freeList;
2585     OSIterator *        iter;
2586     UInt32              workDone;
2587     IOService *         victim;
2588     IOService *         client;
2589     IOService *         provider;
2590     unsigned int        idx;
2591     bool                moreToDo;
2592     bool                doPhase2;
2593     bool                doPhase3;
2594
2595     options |= kIOServiceRequired;
2596
2597     doPhase2List  = OSArray::withCapacity( 16 );
2598     didPhase2List = OSArray::withCapacity( 16 );
2599     freeList      = OSSet::withCapacity( 16 );
2600     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2601         return;
2602
2603     do {
2604         workDone = gIOTerminateWork;
2605
2606         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2607
2608             victim->retain();
2609             gIOTerminatePhase2List->removeObject(0);
2610             IOLockUnlock( gJobsLock );
2611
2612             uint64_t regID1 = victim->getRegistryEntryID();
2613             IOServiceTrace(
2614                 IOSERVICE_TERM_START_PHASE2,
2615                 (uintptr_t) regID1,
2616                 (uintptr_t) (regID1 >> 32),
2617                 (uintptr_t) 0,
2618                 (uintptr_t) 0);
2619
2620             while( victim ) {
2621
2622                 doPhase2 = victim->lockForArbitration( true );
2623                 if( doPhase2) {
2624                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2625                     if( doPhase2) {
2626
2627                         uint64_t regID1 = victim->getRegistryEntryID();
2628                         IOServiceTrace(
2629                             IOSERVICE_TERM_TRY_PHASE2,
2630                             (uintptr_t) regID1,
2631                             (uintptr_t) (regID1 >> 32),
2632                             (uintptr_t) victim->__state[1],
2633                             (uintptr_t) 0);
2634
2635                         doPhase2 = (0 == (victim->__state[1] &
2636                                     (kIOServiceTermPhase1State
2637                                     | kIOServiceTermPhase2State
2638                                     | kIOServiceConfigState)));
2639
2640                         if (doPhase2 && (iter = victim->getClientIterator())) {
2641                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2642                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2643                                 if (!doPhase2)
2644                                 {
2645                                     uint64_t regID1 = client->getRegistryEntryID();
2646                                     IOServiceTrace(
2647                                         IOSERVICE_TERM_UC_DEFER,
2648                                         (uintptr_t) regID1,
2649                                         (uintptr_t) (regID1 >> 32),
2650                                         (uintptr_t) client->__state[1],
2651                                         (uintptr_t) 0);
2652                                     TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2653                                            victim->getName(), victim->getRegistryEntryID(),
2654                                            client->getName(), client->getRegistryEntryID());
2655                                 }
2656                             }
2657                             iter->release();
2658                         }
2659                         if( doPhase2)
2660                             victim->__state[1] |= kIOServiceTermPhase2State;
2661                     }
2662                     victim->unlockForArbitration();
2663                 }
2664                 if( doPhase2) {
2665
2666                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2667                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2668                                             victim, (void *)(uintptr_t) options, NULL );
2669                     }
2670
2671                     OSArray * notifiers;
2672                     notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
2673                     victim->invokeNotifiers(&notifiers);
2674
2675                     if( 0 == victim->getClient()) {
2676
2677                         // no clients - will go to finalize
2678                         victim->scheduleFinalize(false);
2679
2680                     } else {
2681                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2682                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2683                     }
2684                     didPhase2List->headQ( victim );
2685                 }
2686                 victim->release();
2687                 victim = (IOService *) doPhase2List->getObject(0);
2688                 if( victim) {
2689                     victim->retain();
2690                     doPhase2List->removeObject(0);
2691                 }
2692             }
2693
2694             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2695
2696                 if( victim->lockForArbitration( true )) {
2697                     victim->__state[1] |= kIOServiceTermPhase3State;
2698                     victim->unlockForArbitration();
2699                 }
2700                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2701                                     victim, (void *)(uintptr_t) options );
2702                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2703                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2704                                         victim, (void *)(uintptr_t) options, NULL );
2705                 }
2706                 didPhase2List->removeObject(0);
2707             }
2708             IOLockLock( gJobsLock );
2709         }
2710
2711         // phase 3
2712         do {
2713             doPhase3 = false;
2714             // finalize leaves
2715             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2716
2717                 IOLockUnlock( gJobsLock );
2718                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2719                                     victim, (void *)(uintptr_t) options );
2720                 IOLockLock( gJobsLock );
2721                 // hold off free
2722                 freeList->setObject( victim );
2723                 // safe if finalize list is append only
2724                 gIOFinalizeList->removeObject(0);
2725             }
2726
2727             for( idx = 0;
2728                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2729
2730                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2731                 assert( provider );
2732
2733                 uint64_t regID1 = provider->getRegistryEntryID();
2734                 uint64_t regID2 = client->getRegistryEntryID();
2735
2736                 if( !provider->isChild( client, gIOServicePlane )) {
2737                     // may be multiply queued - nop it
2738                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2739                     IOServiceTrace(
2740                         IOSERVICE_TERMINATE_STOP_NOP,
2741                         (uintptr_t) regID1,
2742                         (uintptr_t) (regID1 >> 32),
2743                         (uintptr_t) regID2,
2744                         (uintptr_t) (regID2 >> 32));
2745
2746                 } else {
2747                     // a terminated client is not ready for stop if it has clients, skip it
2748                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2749                         TLOG("%s[0x%qx]::defer stop(%s[0x%qx])\n",
2750                                 client->getName(), regID2,
2751                                 client->getClient()->getName(), client->getClient()->getRegistryEntryID());
2752                         IOServiceTrace(
2753                             IOSERVICE_TERMINATE_STOP_DEFER,
2754                             (uintptr_t) regID1,
2755                             (uintptr_t) (regID1 >> 32),
2756                             (uintptr_t) regID2,
2757                             (uintptr_t) (regID2 >> 32));
2758
2759                         idx++;
2760                         continue;
2761                     }
2762
2763                     IOLockUnlock( gJobsLock );
2764                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2765                                      provider, (void *) client );
2766                     IOLockLock( gJobsLock );
2767                     // check the finalize list now
2768                     doPhase3 = true;
2769                 }
2770                 // hold off free
2771                 freeList->setObject( client );
2772                 freeList->setObject( provider );
2773
2774                 // safe if stop list is append only
2775                 gIOStopList->removeObject( idx );
2776                 gIOStopProviderList->removeObject( idx );
2777                 idx = 0;
2778             }
2779
2780         } while( doPhase3 );
2781
2782         gIOTerminateWork -= workDone;
2783         moreToDo = (gIOTerminateWork != 0);
2784
2785         if( !moreToDo) {
2786             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2787             IOServiceTrace(
2788                 IOSERVICE_TERMINATE_DONE,
2789                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2790         }
2791
2792     } while( moreToDo );
2793
2794     IOLockUnlock( gJobsLock );
2795
2796     freeList->release();
2797     doPhase2List->release();
2798     didPhase2List->release();
2799
2800     IOLockLock( gJobsLock );
2801 }
2802
2803 bool IOService::finalize( IOOptionBits options )
2804 {
2805     OSIterator *  iter;
2806     IOService *   provider;
2807     uint64_t      regID1, regID2 = getRegistryEntryID();
2808
2809     iter = getProviderIterator();
2810     assert( iter );
2811
2812     if( iter) {
2813         while( (provider = (IOService *) iter->getNextObject())) {
2814
2815             // -- compat
2816             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2817                 /* we come down here on programmatic terminate */
2818
2819                 regID1 = provider->getRegistryEntryID();
2820                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2821                 IOServiceTrace(
2822                     IOSERVICE_TERMINATE_STOP,
2823                     (uintptr_t) regID1,
2824                     (uintptr_t) (regID1 >> 32),
2825                     (uintptr_t) regID2,
2826                     (uintptr_t) (regID2 >> 32));
2827
2828                 stop( provider );
2829                 if( provider->isOpen( this ))
2830                     provider->close( this );
2831                 detach( provider );
2832             } else {
2833             //--
2834                 if( provider->lockForArbitration( true )) {
2835                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2836                         scheduleStop( provider );
2837                     provider->unlockForArbitration();
2838                 }
2839             }
2840         }
2841         iter->release();
2842     }
2843
2844     return( true );
2845 }
2846
2847 #undef tailQ
2848 #undef headQ
2849
2850 /*
2851  * Terminate
2852  */
2853
2854 void IOService::doServiceTerminate( IOOptionBits options )
2855 {
2856 }
2857
2858 // a method in case someone needs to override it
2859 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2860 {
2861     bool ok;
2862
2863     if( client->isParent( this, gIOServicePlane, true))
2864         // we are the clients only provider
2865         ok = client->terminate( options );
2866     else
2867         ok = true;
2868
2869     return( ok );
2870 }
2871
2872 bool IOService::terminate( IOOptionBits options )
2873 {
2874     options |= kIOServiceTerminate;
2875
2876     return( terminatePhase1( options ));
2877 }
2878
2879 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2880
2881 /*
2882  * Open & close
2883  */
2884
2885 struct ServiceOpenMessageContext
2886 {
2887     IOService *  service;
2888     UInt32       type;
2889     IOService *  excludeClient;
2890     IOOptionBits options;
2891 };
2892
2893 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2894 {
2895     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2896
2897     if( object != context->excludeClient)
2898         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2899 }
2900
2901 bool IOService::open(   IOService *     forClient,
2902                         IOOptionBits    options,
2903                         void *          arg )
2904 {
2905     bool                        ok;
2906     ServiceOpenMessageContext   context;
2907
2908     context.service             = this;
2909     context.type                = kIOMessageServiceIsAttemptingOpen;
2910     context.excludeClient       = forClient;
2911     context.options             = options;
2912
2913     applyToInterested( gIOGeneralInterest,
2914                         &serviceOpenMessageApplier, &context );
2915
2916     if( false == lockForArbitration(false) )
2917         return false;
2918
2919     ok = (0 == (__state[0] & kIOServiceInactiveState));
2920     if( ok)
2921         ok = handleOpen( forClient, options, arg );
2922
2923     unlockForArbitration();
2924
2925     return( ok );
2926 }
2927
2928 void IOService::close(  IOService *     forClient,
2929                         IOOptionBits    options )
2930 {
2931     bool                wasClosed;
2932     bool                last = false;
2933
2934     lockForArbitration();
2935
2936     wasClosed = handleIsOpen( forClient );
2937     if( wasClosed) {
2938         handleClose( forClient, options );
2939         last = (__state[1] & kIOServiceTermPhase3State);
2940     }
2941
2942     unlockForArbitration();
2943
2944     if( last)
2945         forClient->scheduleStop( this );
2946
2947     else if( wasClosed) {
2948
2949         ServiceOpenMessageContext context;
2950
2951         context.service         = this;
2952         context.type            = kIOMessageServiceWasClosed;
2953         context.excludeClient   = forClient;
2954         context.options         = options;
2955
2956         applyToInterested( gIOGeneralInterest,
2957                             &serviceOpenMessageApplier, &context );
2958     }
2959 }
2960
2961 bool IOService::isOpen( const IOService * forClient ) const
2962 {
2963     IOService * self = (IOService *) this;
2964     bool ok;
2965
2966     self->lockForArbitration();
2967
2968     ok = handleIsOpen( forClient );
2969
2970     self->unlockForArbitration();
2971
2972     return( ok );
2973 }
2974
2975 bool IOService::handleOpen(     IOService *     forClient,
2976                                 IOOptionBits    options,
2977                                 void *          arg )
2978 {
2979     bool        ok;
2980
2981     ok = (0 == __owner);
2982     if( ok )
2983         __owner = forClient;
2984
2985     else if( options & kIOServiceSeize ) {
2986         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2987                                 __owner, (void *)(uintptr_t) options ));
2988         if( ok && (0 == __owner ))
2989             __owner = forClient;
2990         else
2991             ok = false;
2992     }
2993     return( ok );
2994 }
2995
2996 void IOService::handleClose(    IOService *     forClient,
2997                                 IOOptionBits    options )
2998 {
2999     if( __owner == forClient)
3000         __owner = 0;
3001 }
3002
3003 bool IOService::handleIsOpen(   const IOService * forClient ) const
3004 {
3005     if( forClient)
3006         return( __owner == forClient );
3007     else
3008         return( __owner != forClient );
3009 }
3010
3011 /*
3012  * Probing & starting
3013  */
3014 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3015 {
3016     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3017     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3018     SInt32             val1;
3019     SInt32             val2;
3020
3021     val1 = 0;
3022     val2 = 0;
3023
3024     if ( obj1 )
3025         val1 = obj1->priority;
3026
3027     if ( obj2 )
3028         val2 = obj2->priority;
3029
3030     return ( val1 - val2 );
3031 }
3032
3033 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
3034 {
3035     OSDictionary *      dict;
3036     IOService *         service;
3037     _IOServiceNotifier * notify;
3038     OSSymbol *          key = (OSSymbol *) ref;
3039     OSNumber *          offset;
3040     OSObject *          prop;
3041     SInt32              result;
3042
3043     prop = 0;
3044     result = kIODefaultProbeScore;
3045     if( (dict = OSDynamicCast( OSDictionary, entry)))
3046         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3047     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
3048         return( notify->priority );
3049     else if( (service = OSDynamicCast( IOService, entry)))
3050     {
3051         prop = service->copyProperty(key);
3052         offset = OSDynamicCast(OSNumber, prop);
3053     }
3054     else {
3055         assert( false );
3056         offset = 0;
3057     }
3058
3059     if (offset) result = offset->unsigned32BitValue();
3060
3061     OSSafeReleaseNULL(prop);
3062
3063     return (result);
3064 }
3065
3066 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3067 {
3068     const OSObject *    obj1 = (const OSObject *) inObj1;
3069     const OSObject *    obj2 = (const OSObject *) inObj2;
3070     SInt32               val1;
3071     SInt32               val2;
3072
3073     val1 = 0;
3074     val2 = 0;
3075
3076     if ( obj1 )
3077         val1 = IOServiceObjectOrder( obj1, ref );
3078
3079     if ( obj2 )
3080         val2 = IOServiceObjectOrder( obj2, ref );
3081
3082     return ( val1 - val2 );
3083 }
3084
3085 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
3086 {
3087     IOService *         service = 0;
3088     OSIterator *        iter;
3089     const OSSymbol *    nextCat;
3090
3091     iter = getClientIterator();
3092     if( iter) {
3093         while( (service = (IOService *) iter->getNextObject())) {
3094             if( kIOServiceInactiveState & service->__state[0])
3095                 continue;
3096             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3097                         service->getProperty( gIOMatchCategoryKey ));
3098             if( category == nextCat)
3099             {
3100                 service->retain();
3101                 break;
3102             }
3103         }
3104         iter->release();
3105     }
3106     return( service );
3107 }
3108
3109 IOService * IOService::getClientWithCategory( const OSSymbol * category )
3110 {
3111     IOService *
3112     service = copyClientWithCategory(category);
3113     if (service)
3114         service->release();
3115     return (service);
3116 }
3117
3118 bool IOService::invokeNotifier( _IOServiceNotifier * notify )
3119 {
3120     _IOServiceNotifierInvocation invocation;
3121     bool                         willNotify;
3122     bool                         ret = true;
3123     invocation.thread = current_thread();
3124
3125 #if DEBUG_NOTIFIER_LOCKED
3126     uint32_t count;
3127     if ((count = isLockedForArbitration(0)))
3128     {
3129         IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3130         panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3131     }
3132 #endif /* DEBUG_NOTIFIER_LOCKED */
3133
3134     LOCKWRITENOTIFY();
3135     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3136
3137     if( willNotify) {
3138         queue_enter( &notify->handlerInvocations, &invocation,
3139                         _IOServiceNotifierInvocation *, link );
3140     }
3141     UNLOCKNOTIFY();
3142
3143     if( willNotify) {
3144
3145         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3146
3147         LOCKWRITENOTIFY();
3148         queue_remove( &notify->handlerInvocations, &invocation,
3149                         _IOServiceNotifierInvocation *, link );
3150         if( kIOServiceNotifyWaiter & notify->state) {
3151             notify->state &= ~kIOServiceNotifyWaiter;
3152             WAKEUPNOTIFY( notify );
3153         }
3154         UNLOCKNOTIFY();
3155     }
3156
3157     return( ret );
3158 }
3159
3160 bool IOService::invokeNotifiers(OSArray ** willSend)
3161 {
3162     OSArray *            array;
3163     _IOServiceNotifier * notify;
3164     bool                 ret = true;
3165
3166     array = *willSend;
3167     if (!array) return (true);
3168     *willSend = 0;
3169
3170     for( unsigned int idx = 0;
3171          (notify = (_IOServiceNotifier *) array->getObject(idx));
3172          idx++) {
3173         ret &= invokeNotifier(notify);
3174     }
3175     array->release();
3176
3177     return (ret);
3178 }
3179
3180
3181 /*
3182  * Alloc and probe matching classes,
3183  * called on the provider instance
3184  */
3185
3186 void IOService::probeCandidates( OSOrderedSet * matches )
3187 {
3188     OSDictionary        *       match = 0;
3189     OSSymbol            *       symbol;
3190     IOService           *       inst;
3191     IOService           *       newInst;
3192     OSDictionary        *       props;
3193     SInt32                      score;
3194     OSNumber            *       newPri;
3195     OSOrderedSet        *       familyMatches = 0;
3196     OSOrderedSet        *       startList;
3197     OSDictionary        *       startDict = 0;
3198     const OSSymbol      *       category;
3199     OSIterator          *       iter;
3200     _IOServiceNotifier  *               notify;
3201     OSObject            *       nextMatch = 0;
3202     bool                        started;
3203     bool                        needReloc = false;
3204 #if IOMATCHDEBUG
3205     SInt64                      debugFlags;
3206 #endif
3207     IOService * client = NULL;
3208
3209
3210     assert( matches );
3211     while( !needReloc && (nextMatch = matches->getFirstObject())) {
3212
3213         nextMatch->retain();
3214         matches->removeObject(nextMatch);
3215
3216         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3217
3218             if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify );
3219             nextMatch->release();
3220             nextMatch = 0;
3221             continue;
3222
3223         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3224             nextMatch->release();
3225             nextMatch = 0;
3226             continue;
3227         }
3228
3229         props = 0;
3230 #if IOMATCHDEBUG
3231         debugFlags = getDebugFlags( match );
3232 #endif
3233
3234         do {
3235             category = OSDynamicCast( OSSymbol,
3236                         match->getObject( gIOMatchCategoryKey ));
3237             if( 0 == category)
3238                 category = gIODefaultMatchCategoryKey;
3239
3240             if( (client = copyClientWithCategory(category)) ) {
3241 #if IOMATCHDEBUG
3242                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
3243                     LOG("%s: match category %s exists\n", getName(),
3244                                 category->getCStringNoCopy());
3245 #endif
3246                 nextMatch->release();
3247                 nextMatch = 0;
3248
3249                 client->release();
3250                 client = NULL;
3251
3252                 continue;
3253             }
3254
3255             // create a copy now in case its modified during matching
3256             props = OSDictionary::withDictionary( match, match->getCount());
3257             if( 0 == props)
3258                 continue;
3259             props->setCapacityIncrement(1);
3260
3261             // check the nub matches
3262             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
3263                 continue;
3264
3265             // Check to see if driver reloc has been loaded.
3266             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
3267             if( needReloc) {
3268 #if IOMATCHDEBUG
3269                 if( debugFlags & kIOLogCatalogue)
3270                     LOG("%s: stalling for module\n", getName());
3271 #endif
3272                 // If reloc hasn't been loaded, exit;
3273                 // reprobing will occur after reloc has been loaded.
3274                 continue;
3275             }
3276
3277             // reorder on family matchPropertyTable score.
3278             if( 0 == familyMatches)
3279                 familyMatches = OSOrderedSet::withCapacity( 1,
3280                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3281             if( familyMatches)
3282                 familyMatches->setObject( props );
3283
3284         } while( false );
3285
3286         if (nextMatch) {
3287             nextMatch->release();
3288             nextMatch = 0;
3289         }
3290         if( props)
3291             props->release();
3292     }
3293     matches->release();
3294     matches = 0;
3295
3296     if( familyMatches) {
3297
3298         while( !needReloc
3299              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3300
3301             props->retain();
3302             familyMatches->removeObject( props );
3303
3304             inst = 0;
3305             newInst = 0;
3306 #if IOMATCHDEBUG
3307             debugFlags = getDebugFlags( props );
3308 #endif
3309             do {
3310                 symbol = OSDynamicCast( OSSymbol,
3311                                 props->getObject( gIOClassKey));
3312                 if( !symbol)
3313                     continue;
3314
3315                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
3316
3317                 // alloc the driver instance
3318                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3319
3320                 if( !inst || !OSDynamicCast(IOService, inst)) {
3321                     IOLog("Couldn't alloc class \"%s\"\n",
3322                         symbol->getCStringNoCopy());
3323                     continue;
3324                 }
3325
3326                 // init driver instance
3327                 if( !(inst->init( props ))) {
3328 #if IOMATCHDEBUG
3329                     if( debugFlags & kIOLogStart)
3330                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3331 #endif
3332                     continue;
3333                 }
3334                 if( __state[1] & kIOServiceSynchronousState)
3335                     inst->__state[1] |= kIOServiceSynchronousState;
3336
3337                 // give the driver the default match category if not specified
3338                 category = OSDynamicCast( OSSymbol,
3339                             props->getObject( gIOMatchCategoryKey ));
3340                 if( 0 == category)
3341                     category = gIODefaultMatchCategoryKey;
3342                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3343                 // attach driver instance
3344                 if( !(inst->attach( this )))
3345                         continue;
3346
3347                 // pass in score from property table
3348                 score = familyMatches->orderObject( props );
3349
3350                 // & probe the new driver instance
3351 #if IOMATCHDEBUG
3352                 if( debugFlags & kIOLogProbe)
3353                     LOG("%s::probe(%s)\n",
3354                         inst->getMetaClass()->getClassName(), getName());
3355 #endif
3356
3357                 newInst = inst->probe( this, &score );
3358                 inst->detach( this );
3359                 if( 0 == newInst) {
3360 #if IOMATCHDEBUG
3361                     if( debugFlags & kIOLogProbe)
3362                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3363 #endif
3364                     continue;
3365                 }
3366
3367                 // save the score
3368                 newPri = OSNumber::withNumber( score, 32 );
3369                 if( newPri) {
3370                     newInst->setProperty( gIOProbeScoreKey, newPri );
3371                     newPri->release();
3372                 }
3373
3374                 // add to start list for the match category
3375                 if( 0 == startDict)
3376                     startDict = OSDictionary::withCapacity( 1 );
3377                 assert( startDict );
3378                 startList = (OSOrderedSet *)
3379                                 startDict->getObject( category );
3380                 if( 0 == startList) {
3381                     startList = OSOrderedSet::withCapacity( 1,
3382                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3383                     if( startDict && startList) {
3384                         startDict->setObject( category, startList );
3385                         startList->release();
3386                     }
3387                 }
3388                 assert( startList );
3389                 if( startList)
3390                     startList->setObject( newInst );
3391
3392             } while( false );
3393
3394             props->release();
3395             if( inst)
3396                 inst->release();
3397         }
3398         familyMatches->release();
3399         familyMatches = 0;
3400     }
3401
3402     // start the best (until success) of each category
3403
3404     iter = OSCollectionIterator::withCollection( startDict );
3405     if( iter) {
3406         while( (category = (const OSSymbol *) iter->getNextObject())) {
3407
3408             startList = (OSOrderedSet *) startDict->getObject( category );
3409             assert( startList );
3410             if( !startList)
3411                 continue;
3412
3413             started = false;
3414             while( true // (!started)
3415                    && (inst = (IOService *)startList->getFirstObject())) {
3416
3417                 inst->retain();
3418                 startList->removeObject(inst);
3419
3420 #if IOMATCHDEBUG
3421                 debugFlags = getDebugFlags( inst );
3422
3423                 if( debugFlags & kIOLogStart) {
3424                     if( started)
3425                         LOG( "match category exists, skipping " );
3426                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3427                          getName(), inst->getRetainCount());
3428                 }
3429 #endif
3430                 if( false == started)
3431                     started = startCandidate( inst );
3432 #if IOMATCHDEBUG
3433                 if( (debugFlags & kIOLogStart) && (false == started))
3434                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3435                          inst->getRetainCount());
3436 #endif
3437                 inst->release();
3438             }
3439         }
3440         iter->release();
3441     }
3442
3443
3444     // adjust the busy count by +1 if matching is stalled for a module,
3445     // or -1 if a previously stalled matching is complete.
3446     lockForArbitration();
3447     SInt32 adjBusy = 0;
3448     uint64_t regID = getRegistryEntryID();
3449
3450     if( needReloc) {
3451         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3452         if( adjBusy) {
3453
3454             IOServiceTrace(
3455                 IOSERVICE_MODULESTALL,
3456                 (uintptr_t) regID,
3457                 (uintptr_t) (regID >> 32),
3458                 (uintptr_t) this,
3459                 0);
3460
3461             __state[1] |= kIOServiceModuleStallState;
3462         }
3463
3464     } else if( __state[1] & kIOServiceModuleStallState) {
3465
3466             IOServiceTrace(
3467             IOSERVICE_MODULEUNSTALL,
3468             (uintptr_t) regID,
3469             (uintptr_t) (regID >> 32),
3470             (uintptr_t) this,
3471             0);
3472
3473         __state[1] &= ~kIOServiceModuleStallState;
3474         adjBusy = -1;
3475     }
3476     if( adjBusy)
3477         _adjustBusy( adjBusy );
3478     unlockForArbitration();
3479
3480     if( startDict)
3481         startDict->release();
3482 }
3483
3484 /*
3485  * Start a previously attached & probed instance,
3486  * called on exporting object instance
3487  */
3488
3489 bool IOService::startCandidate( IOService * service )
3490 {
3491     bool                ok;
3492
3493     ok = service->attach( this );
3494
3495     if( ok)
3496     {
3497         if (this != gIOResources)
3498         {
3499             // stall for any nub resources
3500             checkResources();
3501             // stall for any driver resources
3502             service->checkResources();
3503         }
3504
3505         AbsoluteTime startTime;
3506         AbsoluteTime endTime;
3507         UInt64       nano;
3508
3509         if (kIOLogStart & gIOKitDebug)
3510             clock_get_uptime(&startTime);
3511
3512         ok = service->start(this);
3513
3514         if (kIOLogStart & gIOKitDebug)
3515         {
3516             clock_get_uptime(&endTime);
3517
3518             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3519             {
3520                 SUB_ABSOLUTETIME(&endTime, &startTime);
3521                 absolutetime_to_nanoseconds(endTime, &nano);
3522                 if (nano > 500000000ULL)
3523                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3524             }
3525         }
3526         if( !ok)
3527             service->detach( this );
3528     }
3529     return( ok );
3530 }
3531
3532 void IOService::publishResource( const char * key, OSObject * value )
3533 {
3534     const OSSymbol *    sym;
3535
3536     if( (sym = OSSymbol::withCString( key))) {
3537         publishResource( sym, value);
3538         sym->release();
3539     }
3540 }
3541
3542 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3543 {
3544     if( 0 == value)
3545         value = (OSObject *) gIOServiceKey;
3546
3547     gIOResources->setProperty( key, value);
3548
3549     if( IORecursiveLockHaveLock( gNotificationLock))
3550         return;
3551
3552     gIOResourceGenerationCount++;
3553     gIOResources->registerService();
3554 }
3555
3556 bool IOService::addNeededResource( const char * key )
3557 {
3558     OSObject *  resourcesProp;
3559     OSSet *     set;
3560     OSString *  newKey;
3561     bool ret;
3562
3563     resourcesProp = copyProperty( gIOResourceMatchKey );
3564     if (!resourcesProp) return(false);
3565
3566     newKey = OSString::withCString( key );
3567     if (!newKey)
3568     {
3569         resourcesProp->release();
3570         return( false);
3571     }
3572
3573     set = OSDynamicCast( OSSet, resourcesProp );
3574     if( !set) {
3575         set = OSSet::withCapacity( 1 );
3576         if( set)
3577             set->setObject( resourcesProp );
3578     }
3579     else
3580         set->retain();
3581
3582     set->setObject( newKey );
3583     newKey->release();
3584     ret = setProperty( gIOResourceMatchKey, set );
3585     set->release();
3586     resourcesProp->release();
3587
3588     return( ret );
3589 }
3590
3591 bool IOService::checkResource( OSObject * matching )
3592 {
3593     OSString *          str;
3594     OSDictionary *      table;
3595
3596     if( (str = OSDynamicCast( OSString, matching ))) {
3597         if( gIOResources->getProperty( str ))
3598             return( true );
3599     }
3600
3601     if( str)
3602         table = resourceMatching( str );
3603     else if( (table = OSDynamicCast( OSDictionary, matching )))
3604         table->retain();
3605     else {
3606         IOLog("%s: Can't match using: %s\n", getName(),
3607                 matching->getMetaClass()->getClassName());
3608         /* false would stall forever */
3609         return( true );
3610     }
3611
3612     if( gIOKitDebug & kIOLogConfig)
3613         LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
3614
3615     waitForService( table );
3616
3617     if( gIOKitDebug & kIOLogConfig)
3618         LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
3619
3620     return( true );
3621 }
3622
3623 bool IOService::checkResources( void )
3624 {
3625     OSObject *          resourcesProp;
3626     OSSet *             set;
3627     OSIterator *        iter;
3628     bool                ok;
3629
3630     resourcesProp = copyProperty( gIOResourceMatchKey );
3631     if( 0 == resourcesProp)
3632         return( true );
3633
3634     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3635
3636         iter = OSCollectionIterator::withCollection( set );
3637         ok = (0 != iter);
3638         while( ok && (resourcesProp = iter->getNextObject()) )
3639             ok = checkResource( resourcesProp );
3640         if( iter)
3641             iter->release();
3642
3643     } else
3644         ok = checkResource( resourcesProp );
3645
3646     OSSafeReleaseNULL(resourcesProp);
3647
3648     return( ok );
3649 }
3650
3651
3652 void _IOConfigThread::configThread( void )
3653 {
3654     _IOConfigThread *   inst;
3655
3656     do {
3657         if( !(inst = new _IOConfigThread))
3658             continue;
3659         if( !inst->init())
3660             continue;
3661         thread_t unused;
3662         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3663             continue;
3664
3665         return;
3666
3667     } while( false);
3668
3669     if( inst)
3670         inst->release();
3671
3672     return;
3673 }
3674
3675 void _IOConfigThread::free( void )
3676 {
3677     thread_deallocate(current_thread());
3678     OSObject::free();
3679 }
3680
3681 void IOService::doServiceMatch( IOOptionBits options )
3682 {
3683     _IOServiceNotifier * notify;
3684     OSIterator *        iter;
3685     OSOrderedSet *      matches;
3686     OSArray *           resourceKeys = 0;
3687     SInt32              catalogGeneration;
3688     bool                keepGuessing = true;
3689     bool                reRegistered = true;
3690     bool                didRegister;
3691     OSArray *           notifiers[2] = {0};
3692
3693 //    job->nub->deliverNotification( gIOPublishNotification,
3694 //                              kIOServiceRegisteredState, 0xffffffff );
3695
3696     while( keepGuessing ) {
3697
3698         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3699         // the matches list should always be created by findDrivers()
3700         if( matches) {
3701
3702             lockForArbitration();
3703             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3704                 getMetaClass()->addInstance(this);
3705                 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
3706                                      kIOServiceFirstPublishState, 0xffffffff );
3707             }
3708             LOCKREADNOTIFY();
3709             __state[1] &= ~kIOServiceNeedConfigState;
3710             __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
3711             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3712             __state[0] |= kIOServiceRegisteredState;
3713
3714             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3715             if (reRegistered && keepGuessing) {
3716                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3717                         gNotifications->getObject( gIOPublishNotification ) );
3718                 if( iter) {
3719                     while((notify = (_IOServiceNotifier *)
3720                            iter->getNextObject())) {
3721
3722                         if( matchPassive(notify->matching, 0)
3723                          && (kIOServiceNotifyEnable & notify->state))
3724                             matches->setObject( notify );
3725                     }
3726                     iter->release();
3727                 }
3728             }
3729
3730             UNLOCKNOTIFY();
3731             unlockForArbitration();
3732             invokeNotifiers(&notifiers[0]);
3733
3734             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3735             {
3736                 if (this == gIOResources)
3737                 {
3738                    if (resourceKeys) resourceKeys->release();
3739                    resourceKeys = copyPropertyKeys();
3740                 }
3741                 probeCandidates( matches );
3742             }
3743             else
3744                 matches->release();
3745         }
3746
3747         lockForArbitration();
3748         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3749         keepGuessing =
3750                    (reRegistered || (catalogGeneration !=
3751                                         gIOCatalogue->getGenerationCount()))
3752                 && (0 == (__state[0] & kIOServiceInactiveState));
3753
3754         if( keepGuessing)
3755             unlockForArbitration();
3756     }
3757
3758     if( (0 == (__state[0] & kIOServiceInactiveState))
3759      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3760
3761         if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys);
3762
3763         notifiers[0] = copyNotifiers(gIOMatchedNotification,
3764                 kIOServiceMatchedState, 0xffffffff);
3765         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3766             notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
3767                 kIOServiceFirstMatchState, 0xffffffff);
3768     }
3769
3770     __state[1] &= ~kIOServiceConfigRunning;
3771     unlockForArbitration();
3772
3773     if (resourceKeys) resourceKeys->release();
3774
3775     invokeNotifiers(&notifiers[0]);
3776     invokeNotifiers(&notifiers[1]);
3777
3778     lockForArbitration();
3779     __state[1] &= ~kIOServiceConfigState;
3780     scheduleTerminatePhase2();
3781
3782     _adjustBusy( -1 );
3783     unlockForArbitration();
3784 }
3785
3786 UInt32 IOService::_adjustBusy( SInt32 delta )
3787 {
3788     IOService * next;
3789     UInt32      count;
3790     UInt32      result;
3791     bool        wasQuiet, nowQuiet, needWake;
3792
3793     next = this;
3794     result = __state[1] & kIOServiceBusyStateMask;
3795
3796     if( delta) do {
3797         if( next != this)
3798             next->lockForArbitration();
3799         count = next->__state[1] & kIOServiceBusyStateMask;
3800         wasQuiet = (0 == count);
3801         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3802             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3803         else
3804             count += delta;
3805         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3806         nowQuiet = (0 == count);
3807         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3808
3809         if( needWake) {
3810             next->__state[1] &= ~kIOServiceBusyWaiterState;
3811             IOLockLock( gIOServiceBusyLock );
3812             thread_wakeup( (event_t) next);
3813             IOLockUnlock( gIOServiceBusyLock );
3814         }
3815         if( next != this)
3816             next->unlockForArbitration();
3817
3818         if( (wasQuiet || nowQuiet) ) {
3819
3820             uint64_t regID = next->getRegistryEntryID();
3821             IOServiceTrace(
3822                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3823                 (uintptr_t) regID,
3824                 (uintptr_t) (regID >> 32),
3825                 (uintptr_t) next,
3826                 0);
3827
3828             if (wasQuiet)
3829             {
3830                 next->__timeBusy = mach_absolute_time();
3831             }
3832             else
3833             {
3834                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3835                 next->__timeBusy = 0;
3836             }
3837
3838             MessageClientsContext context;
3839
3840             context.service  = next;
3841             context.type     = kIOMessageServiceBusyStateChange;
3842             context.argument = (void *) wasQuiet;       /*nowBusy*/
3843             context.argSize  = 0;
3844
3845             applyToInterestNotifiers( next, gIOBusyInterest,
3846                                      &messageClientsApplier, &context );
3847
3848 #if !NO_KEXTD
3849             if( nowQuiet && (next == gIOServiceRoot)) {
3850                 OSKext::considerUnloads();
3851                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3852             }
3853 #endif
3854         }
3855
3856         delta = nowQuiet ? -1 : +1;
3857
3858     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3859
3860     return( result );
3861 }
3862
3863 void IOService::adjustBusy( SInt32 delta )
3864 {
3865     lockForArbitration();
3866     _adjustBusy( delta );
3867     unlockForArbitration();
3868 }
3869
3870 uint64_t IOService::getAccumulatedBusyTime( void )
3871 {
3872     uint64_t accumBusy = __accumBusy;
3873     uint64_t timeBusy = __timeBusy;
3874     uint64_t nano;
3875
3876     do
3877     {
3878         accumBusy = __accumBusy;
3879         timeBusy  = __timeBusy;
3880         if (timeBusy)
3881             accumBusy += mach_absolute_time() - timeBusy;
3882     }
3883     while (timeBusy != __timeBusy);
3884
3885     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3886
3887     return (nano);
3888 }
3889
3890 UInt32 IOService::getBusyState( void )
3891 {
3892     return( __state[1] & kIOServiceBusyStateMask );
3893 }
3894
3895 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3896                                   mach_timespec_t * timeout )
3897 {
3898     panic("waitForState");
3899     return (kIOReturnUnsupported);
3900 }
3901
3902 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3903                                    uint64_t timeout )
3904 {
3905     bool            wait;
3906     int             waitResult = THREAD_AWAKENED;
3907     bool            computeDeadline = true;
3908     AbsoluteTime    abstime;
3909
3910     do {
3911         lockForArbitration();
3912         IOLockLock( gIOServiceBusyLock );
3913         wait = (value != (__state[1] & mask));
3914         if( wait) {
3915             __state[1] |= kIOServiceBusyWaiterState;
3916             unlockForArbitration();
3917             if( timeout != UINT64_MAX ) {
3918                 if( computeDeadline ) {
3919                     AbsoluteTime  nsinterval;
3920                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3921                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3922                     computeDeadline = false;
3923                 }
3924                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3925             }
3926             else
3927                 assert_wait((event_t)this, THREAD_UNINT );
3928         } else
3929             unlockForArbitration();
3930         IOLockUnlock( gIOServiceBusyLock );
3931         if( wait)
3932             waitResult = thread_block(THREAD_CONTINUE_NULL);
3933
3934     } while( wait && (waitResult != THREAD_TIMED_OUT));
3935
3936     if( waitResult == THREAD_TIMED_OUT)
3937         return( kIOReturnTimeout );
3938     else
3939         return( kIOReturnSuccess );
3940 }
3941
3942 IOReturn IOService::waitQuiet( uint64_t timeout )
3943 {
3944     IOReturn ret;
3945     uint32_t loops;
3946     char *   string = NULL;
3947     char *   panicString = NULL;
3948     size_t   len;
3949     size_t   panicStringLen;
3950     uint64_t time;
3951     uint64_t nano;
3952     bool     kextdWait;
3953     bool     dopanic;
3954
3955     enum { kTimeoutExtensions = 4 };
3956
3957     time = mach_absolute_time();
3958     kextdWait = false;
3959     for (loops = 0; loops < kTimeoutExtensions; loops++)
3960     {
3961         ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3962
3963         if (loops && (kIOReturnSuccess == ret))
3964         {
3965             time = mach_absolute_time() - time;
3966             absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
3967             IOLog("busy extended ok[%d], (%llds, %llds)\n",
3968                   loops, timeout / 1000000000ULL, nano / 1000000000ULL);
3969             break;
3970         }
3971         else if (kIOReturnTimeout != ret) break;
3972         else if (timeout < 41000000000)   break;
3973
3974         {
3975             IORegistryIterator * iter;
3976             OSOrderedSet       * set;
3977             OSOrderedSet       * leaves;
3978             IOService          * next;
3979             IOService          * nextParent;
3980             char               * s;
3981             size_t               l;
3982
3983             len = 256;
3984             panicStringLen = 256;
3985             if (!string)      string      = IONew(char, len);
3986             if (!panicString) panicString = IONew(char, panicStringLen);
3987             set = NULL;
3988             kextdWait = OSKext::isWaitingKextd();
3989             iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
3990             leaves = OSOrderedSet::withCapacity(4);
3991             if (iter) set = iter->iterateAll();
3992             if (string && panicString && leaves && set)
3993             {
3994                 string[0] = panicString[0] = 0;
3995                 set->setObject(this);
3996                 while ((next = (IOService *) set->getLastObject()))
3997                 {
3998                     if (next->getBusyState())
3999                     {
4000                         if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true;
4001                         leaves->setObject(next);
4002                         nextParent = next;
4003                         while ((nextParent = nextParent->getProvider()))
4004                         {
4005                             set->removeObject(nextParent);
4006                             leaves->removeObject(nextParent);
4007                         }
4008                     }
4009                     set->removeObject(next);
4010                 }
4011                 s = string;
4012                 while ((next = (IOService *) leaves->getLastObject()))
4013                 {
4014                     l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
4015                     if (l >= len) break;
4016                     s += l;
4017                     len -= l;
4018                     leaves->removeObject(next);
4019                 }
4020             }
4021             OSSafeReleaseNULL(leaves);
4022             OSSafeReleaseNULL(set);
4023             OSSafeReleaseNULL(iter);
4024         }
4025
4026         dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug));
4027         snprintf(panicString, panicStringLen,
4028                  "%s[%d], (%llds): %s",
4029                  kextdWait ? "kextd stall" : "busy timeout",
4030                  loops, timeout / 1000000000ULL,
4031                  string ? string : "");
4032         IOLog("%s\n", panicString);
4033         if (dopanic)     panic("%s", panicString);
4034         else if (!loops) getPMRootDomain()->startSpinDump(1);
4035     }
4036
4037     if (string)      IODelete(string, char, 256);
4038     if (panicString) IODelete(panicString, char, panicStringLen);
4039
4040     return (ret);
4041 }
4042
4043 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
4044 {
4045     uint64_t    timeoutNS;
4046
4047     if (timeout)
4048     {
4049         timeoutNS = timeout->tv_sec;
4050         timeoutNS *= kSecondScale;
4051         timeoutNS += timeout->tv_nsec;
4052     }
4053     else
4054         timeoutNS = UINT64_MAX;
4055
4056     return (waitQuiet(timeoutNS));
4057 }
4058
4059 bool IOService::serializeProperties( OSSerialize * s ) const
4060 {
4061 #if 0
4062     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
4063                 sizeof( __state), "__state");
4064 #endif
4065     return( super::serializeProperties(s) );
4066 }
4067
4068
4069 void _IOConfigThread::main(void * arg, wait_result_t result)
4070 {
4071     _IOConfigThread * self = (_IOConfigThread *) arg;
4072     _IOServiceJob * job;
4073     IOService   *   nub;
4074     bool            alive = true;
4075     kern_return_t   kr;
4076     thread_precedence_policy_data_t precedence = { -1 };
4077
4078     kr = thread_policy_set(current_thread(),
4079                             THREAD_PRECEDENCE_POLICY,
4080                             (thread_policy_t) &precedence,
4081                             THREAD_PRECEDENCE_POLICY_COUNT);
4082     if (KERN_SUCCESS != kr)
4083         IOLog("thread_policy_set(%d)\n", kr);
4084
4085     do {
4086
4087 //      randomDelay();
4088
4089         semaphore_wait( gJobsSemaphore );
4090
4091         IOTakeLock( gJobsLock );
4092         job = (_IOServiceJob *) gJobs->getFirstObject();
4093         job->retain();
4094         gJobs->removeObject(job);
4095         if( job) {
4096             gOutstandingJobs--;
4097 //          gNumConfigThreads--;        // we're out of service
4098             gNumWaitingThreads--;       // we're out of service
4099         }
4100         IOUnlock( gJobsLock );
4101
4102         if( job) {
4103
4104           nub = job->nub;
4105
4106           if( gIOKitDebug & kIOLogConfig)
4107             LOG("config(%p): starting on %s, %d\n",
4108                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
4109
4110           switch( job->type) {
4111
4112             case kMatchNubJob:
4113                 nub->doServiceMatch( job->options );
4114                 break;
4115
4116             default:
4117                 LOG("config(%p): strange type (%d)\n",
4118                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
4119                 break;
4120             }
4121
4122             nub->release();
4123             job->release();
4124
4125             IOTakeLock( gJobsLock );
4126             alive = (gOutstandingJobs > gNumWaitingThreads);
4127             if( alive)
4128                 gNumWaitingThreads++;   // back in service
4129 //              gNumConfigThreads++;
4130             else {
4131                 if( 0 == --gNumConfigThreads) {
4132 //                    IOLog("MATCH IDLE\n");
4133                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
4134                 }
4135             }
4136             IOUnlock( gJobsLock );
4137         }
4138
4139     } while( alive );
4140
4141     if( gIOKitDebug & kIOLogConfig)
4142         LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
4143
4144     self->release();
4145 }
4146
4147 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
4148 {
4149     bool            wait;
4150     int             waitResult = THREAD_AWAKENED;
4151     bool            computeDeadline = true;
4152     AbsoluteTime    deadline;
4153
4154     IOLockLock( gJobsLock );
4155     do {
4156         wait = (0 != gNumConfigThreads);
4157         if( wait) {
4158             if( msToWait) {
4159                 if( computeDeadline ) {
4160                     clock_interval_to_deadline(
4161                           msToWait, kMillisecondScale, &deadline );
4162                     computeDeadline = false;
4163                 }
4164                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
4165                                                                 deadline, THREAD_UNINT );
4166                    } else {
4167                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
4168                                                                 THREAD_UNINT );
4169                 }
4170         }
4171     } while( wait && (waitResult != THREAD_TIMED_OUT));
4172         IOLockUnlock( gJobsLock );
4173
4174     if( waitResult == THREAD_TIMED_OUT)
4175         return( kIOReturnTimeout );
4176     else
4177         return( kIOReturnSuccess );
4178 }
4179
4180 void IOService::cpusRunning(void)
4181 {
4182     gCPUsRunning = true;
4183 }
4184
4185 void _IOServiceJob::pingConfig( _IOServiceJob * job )
4186 {
4187     int         count;
4188     bool        create;
4189
4190     assert( job );
4191
4192     IOTakeLock( gJobsLock );
4193
4194     gOutstandingJobs++;
4195     gJobs->setLastObject( job );
4196
4197     count = gNumWaitingThreads;
4198 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
4199
4200     create = (  (gOutstandingJobs > count)
4201                 && ((gNumConfigThreads < kMaxConfigThreads)
4202             || (job->nub == gIOResources)
4203             || !gCPUsRunning));
4204     if( create) {
4205         gNumConfigThreads++;
4206         gNumWaitingThreads++;
4207     }
4208
4209     IOUnlock( gJobsLock );
4210
4211     job->release();
4212
4213     if( create) {
4214         if( gIOKitDebug & kIOLogConfig)
4215             LOG("config(%d): creating\n", gNumConfigThreads - 1);
4216         _IOConfigThread::configThread();
4217     }
4218
4219     semaphore_signal( gJobsSemaphore );
4220 }
4221
4222 struct IOServiceMatchContext
4223 {
4224     OSDictionary * table;
4225     OSObject *     result;
4226     uint32_t       options;
4227     uint32_t       state;
4228     uint32_t       count;
4229     uint32_t       done;
4230 };
4231
4232 bool IOService::instanceMatch(const OSObject * entry, void * context)
4233 {
4234     IOServiceMatchContext * ctx = (typeof(ctx)) context;
4235     IOService *    service = (typeof(service)) entry;
4236     OSDictionary * table   = ctx->table;
4237     uint32_t       options = ctx->options;
4238     uint32_t       state   = ctx->state;
4239     uint32_t       done;
4240     bool           match;
4241
4242     done = 0;
4243     do
4244     {
4245         match = ((state == (state & service->__state[0]))
4246                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
4247         if (!match) break;
4248         ctx->count += table->getCount();
4249         match = service->matchInternal(table, options, &done);
4250         ctx->done += done;
4251     }
4252     while (false);
4253     if (!match)
4254         return (false);
4255
4256     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
4257     {
4258         service->retain();
4259         ctx->result = service;
4260         return (true);
4261     }
4262     else if (!ctx->result)
4263     {
4264         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
4265     }
4266     else
4267     {
4268         ((OSSet *)ctx->result)->setObject(service);
4269     }
4270     return (false);
4271 }
4272
4273 // internal - call with gNotificationLock
4274 OSObject * IOService::copyExistingServices( OSDictionary * matching,
4275                  IOOptionBits inState, IOOptionBits options )
4276 {
4277     OSObject *   current = 0;
4278     OSIterator * iter;
4279     IOService *  service;
4280     OSObject *   obj;
4281     OSString *   str;
4282
4283     if( !matching)
4284         return( 0 );
4285
4286 #if MATCH_DEBUG
4287     OSSerialize * s = OSSerialize::withCapacity(128);
4288     matching->serialize(s);
4289 #endif
4290
4291     if((obj = matching->getObject(gIOProviderClassKey))
4292       && gIOResourcesKey
4293       && gIOResourcesKey->isEqualTo(obj)
4294       && (service = gIOResources))
4295     {
4296         if( (inState == (service->__state[0] & inState))
4297           && (0 == (service->__state[0] & kIOServiceInactiveState))
4298           &&  service->matchPassive(matching, options))
4299         {
4300             if( options & kIONotifyOnce)
4301             {
4302                 service->retain();
4303                 current = service;
4304             }
4305             else
4306                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
4307         }
4308     }
4309     else
4310     {
4311         IOServiceMatchContext ctx;
4312         ctx.table   = matching;
4313         ctx.state   = inState;
4314         ctx.count   = 0;
4315         ctx.done    = 0;
4316         ctx.options = options;
4317         ctx.result  = 0;
4318
4319         if ((str = OSDynamicCast(OSString, obj)))
4320         {
4321             const OSSymbol * sym = OSSymbol::withString(str);
4322             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
4323             sym->release();
4324         }
4325         else
4326         {
4327             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
4328         }
4329
4330
4331         current = ctx.result;
4332
4333         options |= kIOServiceInternalDone | kIOServiceClassDone;
4334         if (current && (ctx.done != ctx.count))
4335         {
4336             OSSet *
4337             source = OSDynamicCast(OSSet, current);
4338             current = 0;
4339             while ((service = (IOService *) source->getAnyObject()))
4340             {
4341                 if (service->matchPassive(matching, options))
4342                 {
4343                     if( options & kIONotifyOnce)
4344                     {
4345                         service->retain();
4346                         current = service;
4347                         break;
4348                     }
4349                     if( current)
4350                     {
4351                         ((OSSet *)current)->setObject( service );
4352                     }
4353                     else
4354                     {
4355                         current = OSSet::withObjects(
4356                                         (const OSObject **) &service, 1, 1 );
4357                     }
4358                 }
4359                 source->removeObject(service);
4360             }
4361             source->release();
4362         }
4363     }
4364
4365 #if MATCH_DEBUG
4366     {
4367         OSObject * _current = 0;
4368
4369         iter = IORegistryIterator::iterateOver( gIOServicePlane,
4370                                             kIORegistryIterateRecursively );
4371         if( iter) {
4372             do {
4373                 iter->reset();
4374                 while( (service = (IOService *) iter->getNextObject())) {
4375                     if( (inState == (service->__state[0] & inState))
4376                     && (0 == (service->__state[0] & kIOServiceInactiveState))
4377                     &&  service->matchPassive(matching, 0)) {
4378
4379                         if( options & kIONotifyOnce) {
4380                             service->retain();
4381                             _current = service;
4382                             break;
4383                         }
4384                         if( _current)
4385                             ((OSSet *)_current)->setObject( service );
4386                         else
4387                             _current = OSSet::withObjects(
4388                                             (const OSObject **) &service, 1, 1 );
4389                     }
4390                 }
4391             } while( !service && !iter->isValid());
4392             iter->release();
4393         }
4394
4395
4396         if ( ((current != 0) != (_current != 0))
4397         || (current && _current && !current->isEqualTo(_current)))
4398         {
4399             OSSerialize * s1 = OSSerialize::withCapacity(128);
4400             OSSerialize * s2 = OSSerialize::withCapacity(128);
4401             current->serialize(s1);
4402             _current->serialize(s2);
4403             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
4404                 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4405             s1->release();
4406             s2->release();
4407         }
4408
4409         if (_current) _current->release();
4410     }
4411
4412     s->release();
4413 #endif
4414
4415     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4416         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4417         current->release();
4418         current = iter;
4419     }
4420
4421     return( current );
4422 }
4423
4424 // public version
4425 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4426 {
4427     OSIterator *        iter;
4428
4429     // is a lock even needed?
4430     LOCKWRITENOTIFY();
4431
4432     iter = (OSIterator *) copyExistingServices( matching,
4433                                                 kIOServiceMatchedState );
4434
4435     UNLOCKNOTIFY();
4436
4437     return( iter );
4438 }
4439
4440 IOService * IOService::copyMatchingService( OSDictionary * matching )
4441 {
4442     IOService * service;
4443
4444     // is a lock even needed?
4445     LOCKWRITENOTIFY();
4446
4447     service = (IOService *) copyExistingServices( matching,
4448                                                 kIOServiceMatchedState, kIONotifyOnce );
4449
4450     UNLOCKNOTIFY();
4451
4452     return( service );
4453 }
4454
4455 struct _IOServiceMatchingNotificationHandlerRef
4456 {
4457     IOServiceNotificationHandler handler;
4458     void * ref;
4459 };
4460
4461 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4462                                                    IOService * newService,
4463                                                    IONotifier * notifier )
4464 {
4465     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4466 }
4467
4468 // internal - call with gNotificationLock
4469 IONotifier * IOService::setNotification(
4470             const OSSymbol * type, OSDictionary * matching,
4471             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4472             SInt32 priority )
4473 {
4474     _IOServiceNotifier * notify = 0;
4475     OSOrderedSet *      set;
4476
4477     if( !matching)
4478         return( 0 );
4479
4480     notify = new _IOServiceNotifier;
4481     if( notify && !notify->init()) {
4482         notify->release();
4483         notify = 0;
4484     }
4485
4486     if( notify) {
4487         notify->handler = handler;
4488         notify->target = target;
4489         notify->type = type;
4490         notify->matching = matching;
4491         matching->retain();
4492         if (handler == &_IOServiceMatchingNotificationHandler)
4493         {
4494             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4495             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4496         }
4497         else
4498             notify->ref = ref;
4499         notify->priority = priority;
4500         notify->state = kIOServiceNotifyEnable;
4501         queue_init( &notify->handlerInvocations );
4502
4503         ////// queue
4504
4505         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4506             set = OSOrderedSet::withCapacity( 1,
4507                         IONotifyOrdering, 0 );
4508             if( set) {
4509                 gNotifications->setObject( type, set );
4510                 set->release();
4511             }
4512         }
4513         notify->whence = set;
4514         if( set)
4515             set->setObject( notify );
4516     }
4517
4518     return( notify );
4519 }
4520
4521 // internal - call with gNotificationLock
4522 IONotifier * IOService::doInstallNotification(
4523                         const OSSymbol * type, OSDictionary * matching,
4524                         IOServiceMatchingNotificationHandler handler,
4525                         void * target, void * ref,
4526                         SInt32 priority, OSIterator ** existing )
4527 {
4528     OSIterator *        exist;
4529     IONotifier *        notify;
4530     IOOptionBits        inState;
4531
4532     if( !matching)
4533         return( 0 );
4534
4535     if( type == gIOPublishNotification)
4536         inState = kIOServiceRegisteredState;
4537
4538     else if( type == gIOFirstPublishNotification)
4539         inState = kIOServiceFirstPublishState;
4540
4541     else if (type == gIOMatchedNotification)
4542         inState = kIOServiceMatchedState;
4543
4544     else if (type == gIOFirstMatchNotification)
4545         inState = kIOServiceFirstMatchState;
4546
4547     else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification))
4548         inState = 0;
4549     else
4550         return( 0 );
4551
4552     notify = setNotification( type, matching, handler, target, ref, priority );
4553
4554     if( inState)
4555         // get the current set
4556         exist = (OSIterator *) copyExistingServices( matching, inState );
4557     else
4558         exist = 0;
4559
4560     *existing = exist;
4561
4562     return( notify );
4563 }
4564
4565 #if !defined(__LP64__)
4566 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4567                                         IOServiceNotificationHandler handler,
4568                                         void * target, void * refCon,
4569                                         SInt32 priority, OSIterator ** existing )
4570 {
4571     IONotifier * result;
4572     _IOServiceMatchingNotificationHandlerRef ref;
4573     ref.handler = handler;
4574     ref.ref     = refCon;
4575
4576     result = (_IOServiceNotifier *) installNotification( type, matching,
4577                          &_IOServiceMatchingNotificationHandler,
4578                         target, &ref, priority, existing );
4579     if (result)
4580         matching->release();
4581
4582     return (result);
4583 }
4584 #endif /* !defined(__LP64__) */
4585
4586
4587 IONotifier * IOService::installNotification(
4588                         const OSSymbol * type, OSDictionary * matching,
4589                         IOServiceMatchingNotificationHandler handler,
4590                         void * target, void * ref,
4591                         SInt32 priority, OSIterator ** existing )
4592 {
4593     IONotifier * notify;
4594
4595     LOCKWRITENOTIFY();
4596
4597     notify = doInstallNotification( type, matching, handler, target, ref,
4598                 priority, existing );
4599
4600     // in case handler remove()s
4601     if (notify) notify->retain();
4602
4603     UNLOCKNOTIFY();
4604
4605     return( notify );
4606 }
4607
4608 IONotifier * IOService::addNotification(
4609                         const OSSymbol * type, OSDictionary * matching,
4610                         IOServiceNotificationHandler handler,
4611                         void * target, void * refCon,
4612                         SInt32 priority )
4613 {
4614     IONotifier * result;
4615     _IOServiceMatchingNotificationHandlerRef ref;
4616
4617     ref.handler = handler;
4618     ref.ref     = refCon;
4619
4620     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4621                             target, &ref, priority);
4622
4623     if (result)
4624         matching->release();
4625
4626     return (result);
4627 }
4628
4629 IONotifier * IOService::addMatchingNotification(
4630                         const OSSymbol * type, OSDictionary * matching,
4631                         IOServiceMatchingNotificationHandler handler,
4632                         void * target, void * ref,
4633                         SInt32 priority )
4634 {
4635     OSIterator *                existing = NULL;
4636     IONotifier *                ret;
4637     _IOServiceNotifier *        notify;
4638     IOService *                 next;
4639
4640     ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
4641                 handler, target, ref, priority, &existing );
4642     if (!ret) return (0);
4643
4644     // send notifications for existing set
4645     if (existing)
4646     {
4647         while( (next = (IOService *) existing->getNextObject()))
4648         {
4649             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4650             {
4651                 next->invokeNotifier( notify );
4652             }
4653         }
4654         existing->release();
4655     }
4656
4657     LOCKWRITENOTIFY();
4658     bool removed = (0 == notify->whence);
4659     notify->release();
4660     if (removed) ret = gIOServiceNullNotifier;
4661     UNLOCKNOTIFY();
4662
4663     return( ret );
4664 }
4665
4666 bool IOService::syncNotificationHandler(
4667                         void * /* target */, void * ref,
4668                         IOService * newService,
4669                         IONotifier * notifier )
4670 {
4671
4672     LOCKWRITENOTIFY();
4673     if (!*((IOService **) ref))
4674     {
4675         newService->retain();
4676         (*(IOService **) ref) = newService;
4677         WAKEUPNOTIFY(ref);
4678     }
4679     UNLOCKNOTIFY();
4680
4681     return( false );
4682 }
4683
4684 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4685                                                 uint64_t timeout)
4686 {
4687     IONotifier *        notify = 0;
4688     // priority doesn't help us much since we need a thread wakeup
4689     SInt32              priority = 0;
4690     IOService *         result;
4691
4692     if (!matching)
4693         return( 0 );
4694
4695     result = NULL;
4696
4697     LOCKWRITENOTIFY();
4698     do
4699     {
4700         result = (IOService *) copyExistingServices( matching,
4701                             kIOServiceMatchedState, kIONotifyOnce );
4702         if (result)
4703             break;
4704         notify = IOService::setNotification( gIOMatchedNotification, matching,
4705                     &IOService::syncNotificationHandler, (void *) 0,
4706                     &result, priority );
4707          if (!notify)
4708             break;
4709         if (UINT64_MAX != timeout)
4710         {
4711             AbsoluteTime deadline;
4712             nanoseconds_to_absolutetime(timeout, &deadline);
4713             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4714             SLEEPNOTIFYTO(&result, deadline);
4715         }
4716         else
4717         {
4718             SLEEPNOTIFY(&result);
4719         }
4720     }
4721     while( false );
4722
4723     UNLOCKNOTIFY();
4724
4725     if (notify)
4726         notify->remove();       // dequeues
4727
4728     return( result );
4729 }
4730
4731 IOService * IOService::waitForService( OSDictionary * matching,
4732                                         mach_timespec_t * timeout )
4733 {
4734     IOService * result;
4735     uint64_t    timeoutNS;
4736
4737     if (timeout)
4738     {
4739         timeoutNS = timeout->tv_sec;
4740         timeoutNS *= kSecondScale;
4741         timeoutNS += timeout->tv_nsec;
4742     }
4743     else
4744         timeoutNS = UINT64_MAX;
4745
4746     result = waitForMatchingService(matching, timeoutNS);
4747
4748     matching->release();
4749     if (result)
4750         result->release();
4751
4752     return (result);
4753 }
4754
4755 void IOService::deliverNotification( const OSSymbol * type,
4756                             IOOptionBits orNewState, IOOptionBits andNewState )
4757 {
4758     panic("deliverNotification");
4759 }
4760
4761 OSArray * IOService::copyNotifiers(const OSSymbol * type,
4762                                    IOOptionBits orNewState, IOOptionBits andNewState )
4763 {
4764     _IOServiceNotifier * notify;
4765     OSIterator *         iter;
4766     OSArray *            willSend = 0;
4767
4768     lockForArbitration();
4769
4770     if( (0 == (__state[0] & kIOServiceInactiveState))
4771      || (type == gIOTerminatedNotification)
4772      || (type == gIOWillTerminateNotification)) {
4773
4774         LOCKREADNOTIFY();
4775
4776         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4777                     gNotifications->getObject( type ) );
4778
4779         if( iter) {
4780             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4781
4782                 if( matchPassive(notify->matching, 0)
4783                   && (kIOServiceNotifyEnable & notify->state)) {
4784                     if( 0 == willSend)
4785                         willSend = OSArray::withCapacity(8);
4786                     if( willSend)
4787                         willSend->setObject( notify );
4788                 }
4789             }
4790             iter->release();
4791         }
4792         __state[0] = (__state[0] | orNewState) & andNewState;
4793         UNLOCKNOTIFY();
4794     }
4795
4796     unlockForArbitration();
4797
4798     return (willSend);
4799
4800 }
4801
4802 IOOptionBits IOService::getState( void ) const
4803 {
4804     return( __state[0] );
4805 }
4806
4807 /*
4808  * Helpers to make matching objects for simple cases
4809  */
4810
4811 OSDictionary * IOService::serviceMatching( const OSString * name,
4812                         OSDictionary * table )
4813 {
4814
4815     const OSString *    str;
4816
4817     str = OSSymbol::withString(name);
4818     if( !str)
4819         return( 0 );
4820
4821     if( !table)
4822         table = OSDictionary::withCapacity( 2 );
4823     if( table)
4824         table->setObject(gIOProviderClassKey, (OSObject *)str );
4825     str->release();
4826
4827     return( table );
4828 }
4829
4830 OSDictionary * IOService::serviceMatching( const char * name,
4831                         OSDictionary * table )
4832 {
4833     const OSString *    str;
4834
4835     str = OSSymbol::withCString( name );
4836     if( !str)
4837         return( 0 );
4838
4839     table = serviceMatching( str, table );
4840     str->release();
4841     return( table );
4842 }
4843
4844 OSDictionary * IOService::nameMatching( const OSString * name,
4845                         OSDictionary * table )
4846 {
4847     if( !table)
4848         table = OSDictionary::withCapacity( 2 );
4849     if( table)
4850         table->setObject( gIONameMatchKey, (OSObject *)name );
4851
4852     return( table );
4853 }
4854
4855 OSDictionary * IOService::nameMatching( const char * name,
4856                         OSDictionary * table )
4857 {
4858     const OSString *    str;
4859
4860     str = OSSymbol::withCString( name );
4861     if( !str)
4862         return( 0 );
4863
4864     table = nameMatching( str, table );
4865     str->release();
4866     return( table );
4867 }
4868
4869 OSDictionary * IOService::resourceMatching( const OSString * str,
4870                         OSDictionary * table )
4871 {
4872     table = serviceMatching( gIOResourcesKey, table );
4873     if( table)
4874         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4875
4876     return( table );
4877 }
4878
4879 OSDictionary * IOService::resourceMatching( const char * name,
4880                         OSDictionary * table )
4881 {
4882     const OSSymbol *    str;
4883
4884     str = OSSymbol::withCString( name );
4885     if( !str)
4886         return( 0 );
4887
4888     table = resourceMatching( str, table );
4889     str->release();
4890
4891     return( table );
4892 }
4893
4894 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4895                         OSDictionary * table )
4896 {
4897     OSDictionary * properties;
4898
4899     properties = OSDictionary::withCapacity( 2 );
4900     if( !properties)
4901         return( 0 );
4902     properties->setObject( key, value );
4903
4904     if( !table)
4905         table = OSDictionary::withCapacity( 2 );
4906     if( table)
4907         table->setObject( gIOPropertyMatchKey, properties );
4908
4909     properties->release();
4910
4911     return( table );
4912 }
4913
4914 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4915                         OSDictionary * table )
4916 {
4917     OSNumber *     num;
4918
4919     num = OSNumber::withNumber( entryID, 64 );
4920     if( !num)
4921         return( 0 );
4922
4923     if( !table)
4924         table = OSDictionary::withCapacity( 2 );
4925     if( table)
4926         table->setObject( gIORegistryEntryIDKey, num );
4927
4928     if (num)
4929         num->release();
4930
4931     return( table );
4932 }
4933
4934
4935 /*
4936  * _IOServiceNotifier
4937  */
4938
4939 // wait for all threads, other than the current one,
4940 //  to exit the handler
4941
4942 void _IOServiceNotifier::wait()
4943 {
4944     _IOServiceNotifierInvocation * next;
4945     bool doWait;
4946
4947     do {
4948         doWait = false;
4949         queue_iterate( &handlerInvocations, next,
4950                         _IOServiceNotifierInvocation *, link) {
4951             if( next->thread != current_thread() ) {
4952                 doWait = true;
4953                 break;
4954             }
4955         }
4956         if( doWait) {
4957             state |= kIOServiceNotifyWaiter;
4958             SLEEPNOTIFY(this);
4959         }
4960
4961     } while( doWait );
4962 }
4963
4964 void _IOServiceNotifier::free()
4965 {
4966     assert( queue_empty( &handlerInvocations ));
4967     OSObject::free();
4968 }
4969
4970 void _IOServiceNotifier::remove()
4971 {
4972     LOCKWRITENOTIFY();
4973
4974     if( whence) {
4975         whence->removeObject( (OSObject *) this );
4976         whence = 0;
4977     }
4978     if( matching) {
4979         matching->release();
4980         matching = 0;
4981     }
4982
4983     state &= ~kIOServiceNotifyEnable;
4984
4985     wait();
4986
4987     UNLOCKNOTIFY();
4988
4989     release();
4990 }
4991
4992 bool _IOServiceNotifier::disable()
4993 {
4994     bool        ret;
4995
4996     LOCKWRITENOTIFY();
4997
4998     ret = (0 != (kIOServiceNotifyEnable & state));
4999     state &= ~kIOServiceNotifyEnable;
5000     if( ret)
5001         wait();
5002
5003     UNLOCKNOTIFY();
5004
5005     return( ret );
5006 }
5007
5008 void _IOServiceNotifier::enable( bool was )
5009 {
5010     LOCKWRITENOTIFY();
5011     if( was)
5012         state |= kIOServiceNotifyEnable;
5013     else
5014         state &= ~kIOServiceNotifyEnable;
5015     UNLOCKNOTIFY();
5016 }
5017
5018
5019 /*
5020  * _IOServiceNullNotifier
5021  */
5022
5023 void _IOServiceNullNotifier::taggedRetain(const void *tag) const                      {}
5024 void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const     {}
5025 void _IOServiceNullNotifier::free()                                                   {}
5026 void _IOServiceNullNotifier::wait()                                                   {}
5027 void _IOServiceNullNotifier::remove()                                                 {}
5028 void _IOServiceNullNotifier::enable(bool was)                                         {}
5029 bool _IOServiceNullNotifier::disable()                                { return(false); }
5030
5031 /*
5032  * IOResources
5033  */
5034
5035 IOService * IOResources::resources( void )
5036 {
5037     IOResources *       inst;
5038
5039     inst = new IOResources;
5040     if( inst && !inst->init()) {
5041         inst->release();
5042         inst = 0;
5043     }
5044
5045     return( inst );
5046 }
5047
5048 bool IOResources::init( OSDictionary * dictionary )
5049 {
5050     // Do super init first
5051     if ( !IOService::init() )
5052         return false;
5053
5054     // Allow PAL layer to publish a value
5055     const char *property_name;
5056     int property_value;
5057
5058     pal_get_resource_property( &property_name, &property_value );
5059
5060     if( property_name ) {
5061         OSNumber *num;
5062         const OSSymbol *        sym;
5063
5064         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
5065             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
5066                 this->setProperty( sym, num );
5067                 sym->release();
5068             }
5069             num->release();
5070         }
5071     }
5072
5073     return true;
5074 }
5075
5076 IOReturn IOResources::newUserClient(task_t owningTask, void * securityID,
5077                                     UInt32 type,  OSDictionary * properties,
5078                                     IOUserClient ** handler)
5079 {
5080     return( kIOReturnUnsupported );
5081 }
5082
5083 IOWorkLoop * IOResources::getWorkLoop() const
5084 {
5085     // If we are the resource root
5086     // then use the platform's workloop
5087     if (this == (IOResources *) gIOResources)
5088         return getPlatform()->getWorkLoop();
5089     else
5090         return IOService::getWorkLoop();
5091 }
5092
5093 bool IOResources::matchPropertyTable( OSDictionary * table )
5094 {
5095     OSObject *          prop;
5096     OSString *          str;
5097     OSSet *             set;
5098     OSIterator *        iter;
5099     OSObject *          obj;
5100     OSArray *           keys;
5101     bool                ok = true;
5102
5103     prop = table->getObject( gIOResourceMatchKey );
5104     str = OSDynamicCast( OSString, prop );
5105     if( str)
5106         ok = (0 != getProperty( str ));
5107
5108     else if( (set = OSDynamicCast( OSSet, prop))) {
5109
5110         iter = OSCollectionIterator::withCollection( set );
5111         ok = (iter != 0);
5112         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
5113             ok = (0 != getProperty( str ));
5114
5115         if( iter)
5116             iter->release();
5117     }
5118     else if ((prop = table->getObject(gIOResourceMatchedKey)))
5119     {
5120         obj = copyProperty(gIOResourceMatchedKey);
5121         keys = OSDynamicCast(OSArray, obj);
5122         ok = false;
5123         if (keys)
5124         {
5125             // assuming OSSymbol
5126             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
5127         }
5128         OSSafeReleaseNULL(obj);
5129     }
5130
5131     return( ok );
5132 }
5133
5134 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
5135 {
5136     IOService::updateConsoleUsers(NULL, 0);
5137 }
5138
5139 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
5140 {
5141     IORegistryEntry * regEntry;
5142     OSObject *        locked = kOSBooleanFalse;
5143     uint32_t          idx;
5144     bool              publish;
5145     OSDictionary *    user;
5146     static IOMessage  sSystemPower;
5147     clock_sec_t       now = 0;
5148     clock_usec_t      microsecs;
5149
5150     regEntry = IORegistryEntry::getRegistryRoot();
5151
5152     if (!gIOChosenEntry)
5153         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
5154
5155     IOLockLock(gIOConsoleUsersLock);
5156
5157     if (systemMessage)
5158     {
5159         sSystemPower = systemMessage;
5160 #if HIBERNATION
5161         if (kIOMessageSystemHasPoweredOn == systemMessage)
5162         {
5163             uint32_t lockState = IOHibernateWasScreenLocked();
5164             switch (lockState)
5165             {
5166                 case 0:
5167                     break;
5168                 case kIOScreenLockLocked:
5169                 case kIOScreenLockFileVaultDialog:
5170                     gIOConsoleBooterLockState = kOSBooleanTrue;
5171                     break;
5172                 case kIOScreenLockNoLock:
5173                     gIOConsoleBooterLockState = 0;
5174                     break;
5175                 case kIOScreenLockUnlocked:
5176                 default:
5177                     gIOConsoleBooterLockState = kOSBooleanFalse;
5178                     break;
5179             }
5180         }
5181 #endif /* HIBERNATION */
5182     }
5183
5184     if (consoleUsers)
5185     {
5186         OSNumber * num = 0;
5187         bool       loginLocked = true;
5188
5189         gIOConsoleLoggedIn = false;
5190         for (idx = 0;
5191               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
5192               idx++)
5193         {
5194             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
5195                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
5196
5197             loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
5198             if (!num)
5199             {
5200                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
5201             }
5202         }
5203 #if HIBERNATION
5204         if (!loginLocked) gIOConsoleBooterLockState = 0;
5205         IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
5206                 (num != 0), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
5207                 gIOConsoleLoggedIn, loginLocked);
5208 #endif /* HIBERNATION */
5209         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
5210     }
5211
5212     if (!gIOConsoleLoggedIn
5213      || (kIOMessageSystemWillSleep == sSystemPower)
5214      || (kIOMessageSystemPagingOff == sSystemPower))
5215     {
5216         locked = kOSBooleanTrue;
5217     }
5218 #if HIBERNATION
5219     else if (gIOConsoleBooterLockState)
5220     {
5221         locked = gIOConsoleBooterLockState;
5222     }
5223 #endif /* HIBERNATION */
5224     else if (gIOConsoleLockTime)
5225     {
5226         clock_get_calendar_microtime(&now, &microsecs);
5227         if (gIOConsoleLockTime > now)
5228         {
5229             AbsoluteTime deadline;
5230             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
5231             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
5232         }
5233         else
5234         {
5235             locked = kOSBooleanTrue;
5236         }
5237     }
5238
5239     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
5240     if (publish)
5241     {
5242         regEntry->setProperty(gIOConsoleLockedKey, locked);
5243         if (consoleUsers)
5244         {
5245             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
5246         }
5247         OSIncrementAtomic( &gIOConsoleUsersSeed );
5248     }
5249
5250 #if HIBERNATION
5251     if (gIOChosenEntry)
5252     {
5253         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
5254         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
5255         else                          gIOScreenLockState = kIOScreenLockNoLock;
5256         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
5257
5258         IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
5259                 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != 0), now, systemMessage);
5260     }
5261 #endif /* HIBERNATION */
5262
5263     IOLockUnlock(gIOConsoleUsersLock);
5264
5265     if (publish)
5266     {
5267         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
5268
5269         MessageClientsContext context;
5270
5271         context.service  = getServiceRoot();
5272         context.type     = kIOMessageConsoleSecurityChange;
5273         context.argument = (void *) regEntry;
5274         context.argSize  = 0;
5275
5276         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
5277                                  &messageClientsApplier, &context );
5278     }
5279 }
5280
5281 IOReturn IOResources::setProperties( OSObject * properties )
5282 {
5283     IOReturn                    err;
5284     const OSSymbol *            key;
5285     OSDictionary *              dict;
5286     OSCollectionIterator *      iter;
5287
5288     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
5289     if ( kIOReturnSuccess != err)
5290         return( err );
5291
5292     dict = OSDynamicCast(OSDictionary, properties);
5293     if( 0 == dict)
5294         return( kIOReturnBadArgument);
5295
5296     iter = OSCollectionIterator::withCollection( dict);
5297     if( 0 == iter)
5298         return( kIOReturnBadArgument);
5299
5300     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
5301     {
5302         if (gIOConsoleUsersKey == key) do
5303         {
5304             OSArray * consoleUsers;
5305             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
5306             if (!consoleUsers)
5307                 continue;
5308             IOService::updateConsoleUsers(consoleUsers, 0);
5309         }
5310         while (false);
5311
5312         publishResource( key, dict->getObject(key) );
5313     }
5314
5315     iter->release();
5316
5317     return( kIOReturnSuccess );
5318 }
5319
5320 /*
5321  * Helpers for matching dictionaries.
5322  * Keys existing in matching are checked in properties.
5323  * Keys may be a string or OSCollection of IOStrings
5324  */
5325
5326 bool IOService::compareProperty( OSDictionary * matching,
5327                                  const char *   key )
5328 {
5329     OSObject *  value;
5330     OSObject *  prop;
5331     bool        ok;
5332
5333     value = matching->getObject( key );
5334     if( value)
5335     {
5336         prop = copyProperty(key);
5337         ok = value->isEqualTo(prop);
5338         if (prop) prop->release();
5339     }
5340     else
5341         ok = true;
5342
5343     return( ok );
5344 }
5345
5346
5347 bool IOService::compareProperty( OSDictionary *   matching,
5348                                  const OSString * key )
5349 {
5350     OSObject *  value;
5351     OSObject *  prop;
5352     bool        ok;
5353
5354     value = matching->getObject( key );
5355     if( value)
5356     {
5357         prop = copyProperty(key);
5358         ok = value->isEqualTo(prop);
5359         if (prop) prop->release();
5360     }
5361     else
5362         ok = true;
5363
5364     return( ok );
5365 }
5366
5367 bool IOService::compareProperties( OSDictionary * matching,
5368                                    OSCollection * keys )
5369 {
5370     OSCollectionIterator *      iter;
5371     const OSString *            key;
5372     bool                        ok = true;
5373
5374     if( !matching || !keys)
5375         return( false );
5376
5377     iter = OSCollectionIterator::withCollection( keys );
5378
5379     if( iter) {
5380         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
5381             ok = compareProperty( matching, key );
5382
5383         iter->release();
5384     }
5385     keys->release();    // !! consume a ref !!
5386
5387     return( ok );
5388 }
5389
5390 /* Helper to add a location matching dict to the table */
5391
5392 OSDictionary * IOService::addLocation( OSDictionary * table )
5393 {
5394     OSDictionary *      dict;
5395
5396     if( !table)
5397         return( 0 );
5398
5399     dict = OSDictionary::withCapacity( 1 );
5400     if( dict) {
5401         table->setObject( gIOLocationMatchKey, dict );
5402         dict->release();
5403     }
5404
5405     return( dict );
5406 }
5407
5408 /*
5409  * Go looking for a provider to match a location dict.
5410  */
5411
5412 IOService * IOService::matchLocation( IOService * /* client */ )
5413 {
5414     IOService * parent;
5415
5416     parent = getProvider();
5417
5418     if( parent)
5419         parent = parent->matchLocation( this );
5420
5421     return( parent );
5422 }
5423
5424 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
5425 {
5426     OSString *          matched;
5427     OSObject *          obj;
5428     OSString *          str;
5429     IORegistryEntry *   entry;
5430     OSNumber *          num;
5431     bool                match = true;
5432     bool                changesOK = (0 != (kIOServiceChangesOK & options));
5433     uint32_t            count;
5434     uint32_t            done;
5435
5436     do
5437     {
5438         count = table->getCount();
5439         done = 0;
5440
5441         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
5442         if (str) {
5443             done++;
5444             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
5445 #if MATCH_DEBUG
5446             match = (0 != metaCast( str ));
5447             if ((kIOServiceClassDone & options) && !match) panic("classDone");
5448 #endif
5449             if ((!match) || (done == count)) break;
5450         }
5451
5452         obj = table->getObject( gIONameMatchKey );
5453         if( obj) {
5454             done++;
5455             match = compareNames( obj, changesOK ? &matched : 0 );
5456             if (!match) break;
5457             if( changesOK && matched) {
5458                 // leave a hint as to which name matched
5459                 table->setObject( gIONameMatchedKey, matched );
5460                 matched->release();
5461             }
5462             if (done == count) break;
5463         }
5464
5465         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
5466         if (str)
5467         {
5468             const OSSymbol * sym;
5469             done++;
5470             match = false;
5471             sym = copyLocation();
5472             if (sym) {
5473                 match = sym->isEqualTo( str );
5474                 sym->release();
5475             }
5476             if ((!match) || (done == count)) break;
5477         }
5478
5479         obj = table->getObject( gIOPropertyMatchKey );
5480         if( obj)
5481         {
5482             OSDictionary * dict;
5483             OSDictionary * nextDict;
5484             OSIterator *   iter;
5485             done++;
5486             match = false;
5487             dict = dictionaryWithProperties();
5488             if( dict) {
5489                 nextDict = OSDynamicCast( OSDictionary, obj);
5490                 if( nextDict)
5491                     iter = 0;
5492                 else
5493                     iter = OSCollectionIterator::withCollection(
5494                                 OSDynamicCast(OSCollection, obj));
5495
5496                 while( nextDict
5497                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5498                                             iter->getNextObject()))))) {
5499                     match = dict->isEqualTo( nextDict, nextDict);
5500                     if( match)
5501                         break;
5502                     nextDict = 0;
5503                 }
5504                 dict->release();
5505                 if( iter)
5506                     iter->release();
5507             }
5508             if ((!match) || (done == count)) break;
5509         }
5510
5511         obj = table->getObject( gIOPropertyExistsMatchKey );
5512         if( obj)
5513         {
5514             OSDictionary * dict;
5515             OSString     * nextKey;
5516             OSIterator *   iter;
5517             done++;
5518             match = false;
5519             dict = dictionaryWithProperties();
5520             if( dict) {
5521                 nextKey = OSDynamicCast( OSString, obj);
5522                 if( nextKey)
5523                     iter = 0;
5524                 else
5525                     iter = OSCollectionIterator::withCollection(
5526                                 OSDynamicCast(OSCollection, obj));
5527
5528                 while( nextKey
5529                     || (iter && (0 != (nextKey = OSDynamicCast(OSString,
5530                                             iter->getNextObject()))))) {
5531                     match = (0 != dict->getObject(nextKey));
5532                     if( match)
5533                         break;
5534                     nextKey = 0;
5535                 }
5536                 dict->release();
5537                 if( iter)
5538                     iter->release();
5539             }
5540             if ((!match) || (done == count)) break;
5541         }
5542
5543         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5544         if( str) {
5545             done++;
5546             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5547             match = (this == entry);
5548             if( entry)
5549                 entry->release();
5550             if ((!match) || (done == count)) break;
5551         }
5552
5553         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5554         if (num) {
5555             done++;
5556             match = (getRegistryEntryID() == num->unsigned64BitValue());
5557             if ((!match) || (done == count)) break;
5558         }
5559
5560         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5561         if( num)
5562         {
5563             OSIterator *        iter;
5564             IOService *         service = 0;
5565             UInt32              serviceCount = 0;
5566
5567             done++;
5568             iter = getClientIterator();
5569             if( iter) {
5570                 while( (service = (IOService *) iter->getNextObject())) {
5571                     if( kIOServiceInactiveState & service->__state[0])
5572                         continue;
5573                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5574                         continue;
5575                     ++serviceCount;
5576                 }
5577                 iter->release();
5578             }
5579             match = (serviceCount == num->unsigned32BitValue());
5580             if ((!match) || (done == count)) break;
5581         }
5582
5583 #define propMatch(key)                                  \
5584         obj = table->getObject(key);                    \
5585         if (obj)                                        \
5586         {                                               \
5587             OSObject * prop;                            \
5588             done++;                                     \
5589             prop = copyProperty(key);                   \
5590             match = obj->isEqualTo(prop);               \
5591             if (prop) prop->release();                  \
5592             if ((!match) || (done == count)) break;     \
5593         }
5594         propMatch(gIOBSDNameKey)
5595         propMatch(gIOBSDMajorKey)
5596         propMatch(gIOBSDMinorKey)
5597         propMatch(gIOBSDUnitKey)
5598 #undef propMatch
5599     }
5600     while (false);
5601
5602     if (did) *did = done;
5603     return (match);
5604 }
5605
5606 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5607 {
5608     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5609 }
5610
5611 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5612 {
5613     IOService *         where;
5614     OSDictionary *      nextTable;
5615     SInt32              score;
5616     OSNumber *          newPri;
5617     bool                match = true;
5618     bool                matchParent = false;
5619     uint32_t            count;
5620     uint32_t            done;
5621
5622     assert( table );
5623
5624 #if !CONFIG_EMBEDDED
5625     OSArray* aliasServiceRegIds = NULL;
5626     IOService* foundAlternateService = NULL;
5627 #endif
5628
5629 #if MATCH_DEBUG
5630     OSDictionary * root = table;
5631 #endif
5632
5633     where = this;
5634     do
5635     {
5636         do
5637         {
5638             count = table->getCount();
5639             if (!(kIOServiceInternalDone & options))
5640             {
5641                 match = where->matchInternal(table, options, &done);
5642                 // don't call family if we've done all the entries in the table
5643                 if ((!match) || (done == count)) break;
5644             }
5645
5646             // pass in score from property table
5647             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5648
5649             // do family specific matching
5650             match = where->matchPropertyTable( table, &score );
5651
5652             if( !match) {
5653 #if IOMATCHDEBUG
5654                 if( kIOLogMatch & getDebugFlags( table ))
5655                     LOG("%s: family specific matching fails\n", where->getName());
5656 #endif
5657                 break;
5658             }
5659
5660             if (kIOServiceChangesOK & options) {
5661                 // save the score
5662                 newPri = OSNumber::withNumber( score, 32 );
5663                 if( newPri) {
5664                     table->setObject( gIOProbeScoreKey, newPri );
5665                     newPri->release();
5666                 }
5667             }
5668
5669             options = 0;
5670             matchParent = false;
5671
5672             nextTable = OSDynamicCast(OSDictionary,
5673                   table->getObject( gIOParentMatchKey ));
5674             if( nextTable) {
5675                 // look for a matching entry anywhere up to root
5676                 match = false;
5677                 matchParent = true;
5678                 table = nextTable;
5679                 break;
5680             }
5681
5682             table = OSDynamicCast(OSDictionary,
5683                     table->getObject( gIOLocationMatchKey ));
5684             if (table) {
5685                 // look for a matching entry at matchLocation()
5686                 match = false;
5687                 where = where->getProvider();
5688                 if (where && (where = where->matchLocation(where))) continue;
5689             }
5690             break;
5691         }
5692         while (true);
5693
5694         if(match == true) {
5695             break;
5696         }
5697
5698         if(matchParent == true) {
5699 #if !CONFIG_EMBEDDED
5700             // check if service has an alias to search its other "parents" if a parent match isn't found
5701             OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
5702             OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
5703             if(alternateRegistryID != NULL) {
5704                 if(aliasServiceRegIds == NULL)
5705                 {
5706                     aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
5707                 }
5708                 aliasServiceRegIds->setObject(alternateRegistryID);
5709             }
5710             OSSafeReleaseNULL(prop);
5711 #endif
5712         }
5713         else {
5714             break;
5715         }
5716
5717         where = where->getProvider();
5718 #if !CONFIG_EMBEDDED
5719         if(where == NULL) {
5720             // there were no matching parent services, check to see if there are aliased services that have a matching parent
5721             if(aliasServiceRegIds != NULL) {
5722                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
5723                 if(numAliasedServices != 0) {
5724                     OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
5725                     if(alternateRegistryID != NULL) {
5726                         OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
5727                         aliasServiceRegIds->removeObject(numAliasedServices - 1);
5728                         if(alternateMatchingDict != NULL) {
5729                             OSSafeReleaseNULL(foundAlternateService);
5730                             foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
5731                             alternateMatchingDict->release();
5732                             if(foundAlternateService != NULL) {
5733                                 where = foundAlternateService;
5734                             }
5735                         }
5736                     }
5737                 }
5738             }
5739         }
5740 #endif
5741     }
5742     while( where != NULL );
5743
5744 #if !CONFIG_EMBEDDED
5745     OSSafeReleaseNULL(foundAlternateService);
5746     OSSafeReleaseNULL(aliasServiceRegIds);
5747 #endif
5748
5749 #if MATCH_DEBUG
5750     if (where != this)
5751     {
5752         OSSerialize * s = OSSerialize::withCapacity(128);
5753         root->serialize(s);
5754         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5755         s->release();
5756     }
5757 #endif
5758
5759     return( match );
5760 }
5761
5762
5763 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5764                                     UInt32 type,  OSDictionary * properties,
5765                                     IOUserClient ** handler )
5766 {
5767     const OSSymbol *userClientClass = 0;
5768     IOUserClient *client;
5769     OSObject *prop;
5770     OSObject *temp;
5771
5772     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5773         return kIOReturnSuccess;
5774
5775     // First try my own properties for a user client class name
5776     prop = copyProperty(gIOUserClientClassKey);
5777     if (prop) {
5778         if (OSDynamicCast(OSSymbol, prop))
5779             userClientClass = (const OSSymbol *) prop;
5780         else if (OSDynamicCast(OSString, prop)) {
5781             userClientClass = OSSymbol::withString((OSString *) prop);
5782             if (userClientClass)
5783                 setProperty(gIOUserClientClassKey,
5784                             (OSObject *) userClientClass);
5785         }
5786     }
5787
5788     // Didn't find one so lets just bomb out now without further ado.
5789     if (!userClientClass)
5790     {
5791         OSSafeReleaseNULL(prop);
5792         return kIOReturnUnsupported;
5793     }
5794
5795     // This reference is consumed by the IOServiceOpen call
5796     temp = OSMetaClass::allocClassWithName(userClientClass);
5797     OSSafeReleaseNULL(prop);
5798     if (!temp)
5799         return kIOReturnNoMemory;
5800
5801     if (OSDynamicCast(IOUserClient, temp))
5802         client = (IOUserClient *) temp;
5803     else {
5804         temp->release();
5805         return kIOReturnUnsupported;
5806     }
5807
5808     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5809         client->release();
5810         return kIOReturnBadArgument;
5811     }
5812
5813     if ( !client->attach(this) ) {
5814         client->release();
5815         return kIOReturnUnsupported;
5816     }
5817
5818     if ( !client->start(this) ) {
5819         client->detach(this);
5820         client->release();
5821         return kIOReturnUnsupported;
5822     }
5823
5824     *handler = client;
5825     return kIOReturnSuccess;
5826 }
5827
5828 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5829                                     UInt32 type, IOUserClient ** handler )
5830 {
5831     return( kIOReturnUnsupported );
5832 }
5833
5834 IOReturn IOService::requestProbe( IOOptionBits options )
5835 {
5836     return( kIOReturnUnsupported);
5837 }
5838
5839 /*
5840  * Convert an IOReturn to text. Subclasses which add additional
5841  * IOReturn's should override this method and call
5842  * super::stringFromReturn if the desired value is not found.
5843  */
5844
5845 const char * IOService::stringFromReturn( IOReturn rtn )
5846 {
5847     static const IONamedValue IOReturn_values[] = {
5848         {kIOReturnSuccess,          "success"                           },
5849         {kIOReturnError,            "general error"                     },
5850         {kIOReturnNoMemory,         "memory allocation error"           },
5851         {kIOReturnNoResources,      "resource shortage"                 },
5852         {kIOReturnIPCError,         "Mach IPC failure"                  },
5853         {kIOReturnNoDevice,         "no such device"                    },
5854         {kIOReturnNotPrivileged,    "privilege violation"               },
5855         {kIOReturnBadArgument,      "invalid argument"                  },
5856         {kIOReturnLockedRead,       "device is read locked"             },
5857         {kIOReturnLockedWrite,      "device is write locked"            },
5858         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5859         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5860         {kIOReturnUnsupported,      "unsupported function"              },
5861         {kIOReturnVMError,          "virtual memory error"              },
5862         {kIOReturnInternalError,    "internal driver error"             },
5863         {kIOReturnIOError,          "I/O error"                         },
5864         {kIOReturnCannotLock,       "cannot acquire lock"               },
5865         {kIOReturnNotOpen,          "device is not open"                },
5866         {kIOReturnNotReadable,      "device is not readable"            },
5867         {kIOReturnNotWritable,      "device is not writeable"           },
5868         {kIOReturnNotAligned,       "alignment error"                   },
5869         {kIOReturnBadMedia,         "media error"                       },
5870         {kIOReturnStillOpen,        "device is still open"              },
5871         {kIOReturnRLDError,         "rld failure"                       },
5872         {kIOReturnDMAError,         "DMA failure"                       },
5873         {kIOReturnBusy,             "device is busy"                    },
5874         {kIOReturnTimeout,          "I/O timeout"                       },
5875         {kIOReturnOffline,          "device is offline"                 },
5876         {kIOReturnNotReady,         "device is not ready"               },
5877         {kIOReturnNotAttached,      "device/channel is not attached"    },
5878         {kIOReturnNoChannels,       "no DMA channels available"         },
5879         {kIOReturnNoSpace,          "no space for data"                 },
5880         {kIOReturnPortExists,       "device port already exists"        },
5881         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5882         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5883         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5884         {kIOReturnMessageTooLarge,  "message is too large"              },
5885         {kIOReturnNotPermitted,     "operation is not permitted"        },
5886         {kIOReturnNoPower,          "device is without power"           },
5887         {kIOReturnNoMedia,          "media is not present"              },
5888         {kIOReturnUnformattedMedia, "media is not formatted"            },
5889         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5890         {kIOReturnUnderrun,         "data underrun"                     },
5891         {kIOReturnOverrun,          "data overrun"                      },
5892         {kIOReturnDeviceError,      "device error"                      },
5893         {kIOReturnNoCompletion,     "no completion routine"             },
5894         {kIOReturnAborted,          "operation was aborted"             },
5895         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5896         {kIOReturnNotResponding,    "device is not responding"          },
5897         {kIOReturnInvalid,          "unanticipated driver error"        },
5898         {0,                         NULL                                }
5899     };
5900
5901     return IOFindNameForValue(rtn, IOReturn_values);
5902 }
5903
5904 /*
5905  * Convert an IOReturn to an errno.
5906  */
5907 int IOService::errnoFromReturn( IOReturn rtn )
5908 {
5909     if (unix_err(err_get_code(rtn)) == rtn)
5910         return err_get_code(rtn);
5911
5912     switch(rtn) {
5913         // (obvious match)
5914         case kIOReturnSuccess:
5915             return(0);
5916         case kIOReturnNoMemory:
5917             return(ENOMEM);
5918         case kIOReturnNoDevice:
5919             return(ENXIO);
5920         case kIOReturnVMError:
5921             return(EFAULT);
5922         case kIOReturnNotPermitted:
5923             return(EPERM);
5924         case kIOReturnNotPrivileged:
5925             return(EACCES);
5926         case kIOReturnIOError:
5927             return(EIO);
5928         case kIOReturnNotWritable:
5929             return(EROFS);
5930         case kIOReturnBadArgument:
5931             return(EINVAL);
5932         case kIOReturnUnsupported:
5933             return(ENOTSUP);
5934         case kIOReturnBusy:
5935             return(EBUSY);
5936         case kIOReturnNoPower:
5937             return(EPWROFF);
5938         case kIOReturnDeviceError:
5939             return(EDEVERR);
5940         case kIOReturnTimeout:
5941             return(ETIMEDOUT);
5942         case kIOReturnMessageTooLarge:
5943             return(EMSGSIZE);
5944         case kIOReturnNoSpace:
5945             return(ENOSPC);
5946         case kIOReturnCannotLock:
5947             return(ENOLCK);
5948
5949         // (best match)
5950         case kIOReturnBadMessageID:
5951         case kIOReturnNoCompletion:
5952         case kIOReturnNotAligned:
5953             return(EINVAL);
5954         case kIOReturnNotReady:
5955             return(EBUSY);
5956         case kIOReturnRLDError:
5957             return(EBADMACHO);
5958         case kIOReturnPortExists:
5959         case kIOReturnStillOpen:
5960             return(EEXIST);
5961         case kIOReturnExclusiveAccess:
5962         case kIOReturnLockedRead:
5963         case kIOReturnLockedWrite:
5964         case kIOReturnNotOpen:
5965         case kIOReturnNotReadable:
5966             return(EACCES);
5967         case kIOReturnCannotWire:
5968         case kIOReturnNoResources:
5969             return(ENOMEM);
5970         case kIOReturnAborted:
5971         case kIOReturnOffline:
5972         case kIOReturnNotResponding:
5973             return(EBUSY);
5974         case kIOReturnBadMedia:
5975         case kIOReturnNoMedia:
5976         case kIOReturnNotAttached:
5977         case kIOReturnUnformattedMedia:
5978             return(ENXIO); // (media error)
5979         case kIOReturnDMAError:
5980         case kIOReturnOverrun:
5981         case kIOReturnUnderrun:
5982             return(EIO); // (transfer error)
5983         case kIOReturnNoBandwidth:
5984         case kIOReturnNoChannels:
5985         case kIOReturnNoFrames:
5986         case kIOReturnNoInterrupt:
5987             return(EIO); // (hardware error)
5988         case kIOReturnError:
5989         case kIOReturnInternalError:
5990         case kIOReturnInvalid:
5991             return(EIO); // (generic error)
5992         case kIOReturnIPCError:
5993             return(EIO); // (ipc error)
5994         default:
5995             return(EIO); // (all other errors)
5996     }
5997 }
5998
5999 IOReturn IOService::message( UInt32 type, IOService * provider,
6000                                 void * argument )
6001 {
6002     /*
6003      * Generic entry point for calls from the provider.  A return value of
6004      * kIOReturnSuccess indicates that the message was received, and where
6005      * applicable, that it was successful.
6006      */
6007
6008     return kIOReturnUnsupported;
6009 }
6010
6011 /*
6012  * Device memory
6013  */
6014
6015 IOItemCount IOService::getDeviceMemoryCount( void )
6016 {
6017     OSArray *           array;
6018     IOItemCount         count;
6019
6020     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6021     if( array)
6022         count = array->getCount();
6023     else
6024         count = 0;
6025
6026     return( count);
6027 }
6028
6029 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
6030 {
6031     OSArray *           array;
6032     IODeviceMemory *    range;
6033
6034     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6035     if( array)
6036         range = (IODeviceMemory *) array->getObject( index );
6037     else
6038         range = 0;
6039
6040     return( range);
6041 }
6042
6043 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
6044                                                 IOOptionBits options )
6045 {
6046     IODeviceMemory *    range;
6047     IOMemoryMap *       map;
6048
6049     range = getDeviceMemoryWithIndex( index );
6050     if( range)
6051         map = range->map( options );
6052     else
6053         map = 0;
6054
6055     return( map );
6056 }
6057
6058 OSArray * IOService::getDeviceMemory( void )
6059 {
6060     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
6061 }
6062
6063
6064 void IOService::setDeviceMemory( OSArray * array )
6065 {
6066     setProperty( gIODeviceMemoryKey, array);
6067 }
6068
6069 /*
6070  * For machines where the transfers on an I/O bus can stall because
6071  * the CPU is in an idle mode, These APIs allow a driver to specify
6072  * the maximum bus stall that they can handle.  0 indicates no limit.
6073  */
6074 void IOService::
6075 setCPUSnoopDelay(UInt32 __unused ns)
6076 {
6077 #if defined(__i386__) || defined(__x86_64__)
6078     ml_set_maxsnoop(ns);
6079 #endif /* defined(__i386__) || defined(__x86_64__) */
6080 }
6081
6082 UInt32 IOService::
6083 getCPUSnoopDelay()
6084 {
6085 #if defined(__i386__) || defined(__x86_64__)
6086     return ml_get_maxsnoop();
6087 #else
6088     return 0;
6089 #endif /* defined(__i386__) || defined(__x86_64__) */
6090 }
6091
6092 #if defined(__i386__) || defined(__x86_64__)
6093 static void
6094 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
6095 {
6096     static const UInt kNoReplace = -1U; // Must be an illegal index
6097     UInt replace = kNoReplace;
6098     bool setCpuDelay = false;
6099
6100     IORecursiveLockLock(sCpuDelayLock);
6101
6102     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6103     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6104     IOService * holder = NULL;
6105
6106     if (ns) {
6107         const CpuDelayEntry ne = {service, ns, delayType};
6108         holder = service;
6109         // Set maximum delay.
6110         for (UInt i = 0; i < count; i++) {
6111             IOService *thisService = entries[i].fService;
6112             bool sameType = (delayType == entries[i].fDelayType);
6113             if ((service == thisService) && sameType)
6114                 replace = i;
6115             else if (!thisService) {
6116                 if (kNoReplace == replace)
6117                     replace = i;
6118             }
6119             else if (sameType) {
6120                 const UInt32 thisMax = entries[i].fMaxDelay;
6121                 if (thisMax < ns)
6122                 {
6123                     ns = thisMax;
6124                     holder = thisService;
6125                 }
6126             }
6127         }
6128
6129         setCpuDelay = true;
6130         if (kNoReplace == replace)
6131             sCpuDelayData->appendBytes(&ne, sizeof(ne));
6132         else
6133             entries[replace] = ne;
6134     }
6135     else {
6136         ns = -1U;       // Set to max unsigned, i.e. no restriction
6137
6138         for (UInt i = 0; i < count; i++) {
6139             // Clear a maximum delay.
6140             IOService *thisService = entries[i].fService;
6141             if (thisService && (delayType == entries[i].fDelayType)) {
6142                 UInt32 thisMax = entries[i].fMaxDelay;
6143                 if (service == thisService)
6144                     replace = i;
6145                 else if (thisMax < ns) {
6146                     ns = thisMax;
6147                     holder = thisService;
6148                 }
6149             }
6150         }
6151
6152         // Check if entry found
6153         if (kNoReplace != replace) {
6154             entries[replace].fService = 0;      // Null the entry
6155             setCpuDelay = true;
6156         }
6157     }
6158
6159     if (setCpuDelay)
6160     {
6161         if (holder && debug_boot_arg) {
6162             strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
6163         }
6164
6165         // Must be safe to call from locked context
6166         if (delayType == kCpuDelayBusStall)
6167         {
6168             ml_set_maxbusdelay(ns);
6169         }
6170         else if (delayType == kCpuDelayInterrupt)
6171         {
6172             ml_set_maxintdelay(ns);
6173         }
6174         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
6175         sCPULatencySet   [delayType]->setValue(ns);
6176
6177         OSArray * handlers = sCpuLatencyHandlers[delayType];
6178         IOService * target;
6179         if (handlers) for (unsigned int idx = 0;
6180                             (target = (IOService *) handlers->getObject(idx));
6181                             idx++)
6182         {
6183             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6184                                             (void *) (uintptr_t) ns, holder,
6185                                             NULL, NULL);
6186         }
6187     }
6188
6189     IORecursiveLockUnlock(sCpuDelayLock);
6190 }
6191
6192 static IOReturn
6193 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
6194 {
6195     IOReturn result = kIOReturnNotFound;
6196     OSArray * array;
6197     unsigned int idx;
6198
6199     IORecursiveLockLock(sCpuDelayLock);
6200
6201     do
6202     {
6203         if (enable && !sCpuLatencyHandlers[delayType])
6204             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
6205         array = sCpuLatencyHandlers[delayType];
6206         if (!array)
6207             break;
6208         idx = array->getNextIndexOfObject(target, 0);
6209         if (!enable)
6210         {
6211             if (-1U != idx)
6212             {
6213                 array->removeObject(idx);
6214                 result = kIOReturnSuccess;
6215             }
6216         }
6217         else
6218         {
6219             if (-1U != idx) {
6220                 result = kIOReturnExclusiveAccess;
6221                 break;
6222             }
6223             array->setObject(target);
6224
6225             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6226             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6227             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
6228             IOService * holder = NULL;
6229
6230             for (UInt i = 0; i < count; i++) {
6231                 if (entries[i].fService
6232                   && (delayType == entries[i].fDelayType)
6233                   && (entries[i].fMaxDelay < ns)) {
6234                     ns = entries[i].fMaxDelay;
6235                     holder = entries[i].fService;
6236                 }
6237             }
6238             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6239                                             (void *) (uintptr_t) ns, holder,
6240                                             NULL, NULL);
6241             result = kIOReturnSuccess;
6242         }
6243     }
6244     while (false);
6245
6246     IORecursiveLockUnlock(sCpuDelayLock);
6247
6248     return (result);
6249 }
6250
6251 #endif /* defined(__i386__) || defined(__x86_64__) */
6252
6253 void IOService::
6254 requireMaxBusStall(UInt32 __unused ns)
6255 {
6256 #if defined(__i386__) || defined(__x86_64__)
6257     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
6258 #endif
6259 }
6260
6261 void IOService::
6262 requireMaxInterruptDelay(uint32_t __unused ns)
6263 {
6264 #if defined(__i386__) || defined(__x86_64__)
6265     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
6266 #endif
6267 }
6268
6269 /*
6270  * Device interrupts
6271  */
6272
6273 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
6274 {
6275   IOInterruptController *interruptController;
6276   OSArray               *array;
6277   OSData                *data;
6278   OSSymbol              *interruptControllerName;
6279   long                  numSources;
6280   IOInterruptSource     *interruptSources;
6281
6282   // Get the parents list from the nub.
6283   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
6284   if (array == 0) return kIOReturnNoResources;
6285
6286   // Allocate space for the IOInterruptSources if needed... then return early.
6287   if (nub->_interruptSources == 0) {
6288     numSources = array->getCount();
6289     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
6290     if (interruptSources == 0) return kIOReturnNoMemory;
6291
6292     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
6293
6294     nub->_numInterruptSources = numSources;
6295     nub->_interruptSources = interruptSources;
6296     return kIOReturnSuccess;
6297   }
6298
6299   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
6300   if (interruptControllerName == 0) return kIOReturnNoResources;
6301
6302   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
6303   if (interruptController == 0) return kIOReturnNoResources;
6304
6305   // Get the interrupt numbers from the nub.
6306   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
6307   if (array == 0) return kIOReturnNoResources;
6308   data = OSDynamicCast(OSData, array->getObject(source));
6309   if (data == 0) return kIOReturnNoResources;
6310
6311   // Set the interruptController and interruptSource in the nub's table.
6312   interruptSources = nub->_interruptSources;
6313   interruptSources[source].interruptController = interruptController;
6314   interruptSources[source].vectorData = data;
6315
6316   return kIOReturnSuccess;
6317 }
6318
6319 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
6320 {
6321   IOReturn ret;
6322
6323   /* Make sure the _interruptSources are set */
6324   if (_interruptSources == 0) {
6325     ret = resolveInterrupt(this, source);
6326     if (ret != kIOReturnSuccess) return ret;
6327   }
6328
6329   /* Make sure the local source number is valid */
6330   if ((source < 0) || (source >= _numInterruptSources))
6331     return kIOReturnNoInterrupt;
6332
6333   /* Look up the contoller for the local source */
6334   *interruptController = _interruptSources[source].interruptController;
6335
6336   if (*interruptController == NULL) {
6337     if (!resolve) return kIOReturnNoInterrupt;
6338
6339     /* Try to reslove the interrupt */
6340     ret = resolveInterrupt(this, source);
6341     if (ret != kIOReturnSuccess) return ret;
6342
6343     *interruptController = _interruptSources[source].interruptController;
6344   }
6345
6346   return kIOReturnSuccess;
6347 }
6348
6349 IOReturn IOService::registerInterrupt(int source, OSObject *target,
6350                                       IOInterruptAction handler,
6351                                       void *refCon)
6352 {
6353   IOInterruptController *interruptController;
6354   IOReturn              ret;
6355
6356   ret = lookupInterrupt(source, true, &interruptController);
6357   if (ret != kIOReturnSuccess) return ret;
6358
6359   /* Register the source */
6360   return interruptController->registerInterrupt(this, source, target,
6361                                                 (IOInterruptHandler)handler,
6362                                                 refCon);
6363 }
6364
6365 IOReturn IOService::unregisterInterrupt(int source)
6366 {
6367   IOInterruptController *interruptController;
6368   IOReturn              ret;
6369
6370   ret = lookupInterrupt(source, false, &interruptController);
6371   if (ret != kIOReturnSuccess) return ret;
6372
6373   /* Unregister the source */
6374   return interruptController->unregisterInterrupt(this, source);
6375 }
6376
6377 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
6378 {
6379   IOReportLegend * legend = NULL;
6380   IOInterruptAccountingData * oldValue = NULL;
6381   IOInterruptAccountingReporter * newArray = NULL;
6382   char subgroupName[64];
6383   int newArraySize = 0;
6384   int i = 0;
6385
6386   if (source < 0) {
6387     return kIOReturnBadArgument;
6388   }
6389
6390   /*
6391    * We support statistics on a maximum of 256 interrupts per nub; if a nub
6392    * has more than 256 interrupt specifiers associated with it, and tries
6393    * to register a high interrupt index with interrupt accounting, panic.
6394    * Having more than 256 interrupts associated with a single nub is
6395    * probably a sign that something fishy is going on.
6396    */
6397   if (source > IA_INDEX_MAX) {
6398     panic("addInterruptStatistics called for an excessively large index (%d)", source);
6399   }
6400
6401   /*
6402    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
6403    * leaving it as is because the likelihood of contention where we are
6404    * actually growing the array is minimal (we would realistically need
6405    * to be starting a driver for the first time, with an IOReporting
6406    * client already in place).  Nonetheless, cleanup that can be done
6407    * to adhere to best practices; it'll make the code more complicated,
6408    * unfortunately.
6409    */
6410   IOLockLock(reserved->interruptStatisticsLock);
6411
6412   /*
6413    * Lazily allocate the statistics array.
6414    */
6415   if (!reserved->interruptStatisticsArray) {
6416     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
6417     assert(reserved->interruptStatisticsArray);
6418     reserved->interruptStatisticsArrayCount = 1;
6419     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
6420   }
6421
6422   if (source >= reserved->interruptStatisticsArrayCount) {
6423     /*
6424      * We're still within the range of supported indices, but we are out
6425      * of space in the current array.  Do a nasty realloc (because
6426      * IORealloc isn't a thing) here.  We'll double the size with each
6427      * reallocation.
6428      *
6429      * Yes, the "next power of 2" could be more efficient; but this will
6430      * be invoked incredibly rarely.  Who cares.
6431      */
6432     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
6433
6434     while (newArraySize <= source)
6435       newArraySize = (newArraySize << 1);
6436     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
6437
6438     assert(newArray);
6439
6440     /*
6441      * TODO: This even zeroes the memory it is about to overwrite.
6442      * Shameful; fix it.  Not particularly high impact, however.
6443      */
6444     bzero(newArray, newArraySize * sizeof(*newArray));
6445     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
6446     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
6447     reserved->interruptStatisticsArray = newArray;
6448     reserved->interruptStatisticsArrayCount = newArraySize;
6449   }
6450
6451   if (!reserved->interruptStatisticsArray[source].reporter) {
6452     /*
6453      * We don't have a reporter associated with this index yet, so we
6454      * need to create one.
6455      */
6456     /*
6457      * TODO: Some statistics do in fact have common units (time); should this be
6458      * split into separate reporters to communicate this?
6459      */
6460      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
6461
6462     /*
6463      * Each statistic is given an identifier based on the interrupt index (which
6464      * should be unique relative to any single nub) and the statistic involved.
6465      * We should now have a sane (small and positive) index, so start
6466      * constructing the channels for statistics.
6467      */
6468     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
6469       /*
6470        * TODO: Currently, this does not add channels for disabled statistics.
6471        * Will this be confusing for clients?  If so, we should just add the
6472        * channels; we can avoid updating the channels even if they exist.
6473        */
6474       if (IA_GET_STATISTIC_ENABLED(i))
6475         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
6476     }
6477
6478     /*
6479      * We now need to add the legend for this reporter to the registry.
6480      */
6481     OSObject * prop = copyProperty(kIOReportLegendKey);
6482     legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
6483     OSSafeReleaseNULL(prop);
6484
6485     /*
6486      * Note that while we compose the subgroup name, we do not need to
6487      * manage its lifecycle (the reporter will handle this).
6488      */
6489     snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
6490     subgroupName[sizeof(subgroupName) - 1] = 0;
6491     legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
6492     setProperty(kIOReportLegendKey, legend->getLegend());
6493     legend->release();
6494
6495     /*
6496      * TODO: Is this a good idea?  Probably not; my assumption is it opts
6497      * all entities who register interrupts into public disclosure of all
6498      * IOReporting channels.  Unfortunately, this appears to be as fine
6499      * grain as it gets.
6500      */
6501     setProperty(kIOReportLegendPublicKey, true);
6502   }
6503
6504   /*
6505    * Don't stomp existing entries.  If we are about to, panic; this
6506    * probably means we failed to tear down our old interrupt source
6507    * correctly.
6508    */
6509   oldValue = reserved->interruptStatisticsArray[source].statistics;
6510
6511   if (oldValue) {
6512     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
6513   }
6514
6515   reserved->interruptStatisticsArray[source].statistics = statistics;
6516
6517   /*
6518    * Inherit the reporter values for each statistic.  The target may
6519    * be torn down as part of the runtime of the service (especially
6520    * for sleep/wake), so we inherit in order to avoid having values
6521    * reset for no apparent reason.  Our statistics are ultimately
6522    * tied to the index and the sevice, not to an individual target,
6523    * so we should maintain them accordingly.
6524    */
6525   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6526
6527   IOLockUnlock(reserved->interruptStatisticsLock);
6528
6529   return kIOReturnSuccess;
6530 }
6531
6532 IOReturn IOService::removeInterruptStatistics(int source)
6533 {
6534   IOInterruptAccountingData * value = NULL;
6535
6536   if (source < 0) {
6537     return kIOReturnBadArgument;
6538   }
6539
6540   IOLockLock(reserved->interruptStatisticsLock);
6541
6542   /*
6543    * We dynamically grow the statistics array, so an excessively
6544    * large index value has NEVER been registered.  This either
6545    * means our cap on the array size is too small (unlikely), or
6546    * that we have been passed a corrupt index (this must be passed
6547    * the plain index into the interrupt specifier list).
6548    */
6549   if (source >= reserved->interruptStatisticsArrayCount) {
6550     panic("removeInterruptStatistics called for index %d, which was never registered", source);
6551   }
6552
6553   assert(reserved->interruptStatisticsArray);
6554
6555   /*
6556    * If there is no existing entry, we are most likely trying to
6557    * free an interrupt owner twice, or we have corrupted the
6558    * index value.
6559    */
6560   value = reserved->interruptStatisticsArray[source].statistics;
6561
6562   if (!value) {
6563     panic("removeInterruptStatistics called for empty index %d", source);
6564   }
6565
6566   /*
6567    * We update the statistics, so that any delta with the reporter
6568    * state is not lost.
6569    */
6570   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6571   reserved->interruptStatisticsArray[source].statistics = NULL;
6572   IOLockUnlock(reserved->interruptStatisticsLock);
6573
6574   return kIOReturnSuccess;
6575 }
6576
6577 IOReturn IOService::getInterruptType(int source, int *interruptType)
6578 {
6579   IOInterruptController *interruptController;
6580   IOReturn              ret;
6581
6582   ret = lookupInterrupt(source, true, &interruptController);
6583   if (ret != kIOReturnSuccess) return ret;
6584
6585   /* Return the type */
6586   return interruptController->getInterruptType(this, source, interruptType);
6587 }
6588
6589 IOReturn IOService::enableInterrupt(int source)
6590 {
6591   IOInterruptController *interruptController;
6592   IOReturn              ret;
6593
6594   ret = lookupInterrupt(source, false, &interruptController);
6595   if (ret != kIOReturnSuccess) return ret;
6596
6597   /* Enable the source */
6598   return interruptController->enableInterrupt(this, source);
6599 }
6600
6601 IOReturn IOService::disableInterrupt(int source)
6602 {
6603   IOInterruptController *interruptController;
6604   IOReturn              ret;
6605
6606   ret = lookupInterrupt(source, false, &interruptController);
6607   if (ret != kIOReturnSuccess) return ret;
6608
6609   /* Disable the source */
6610   return interruptController->disableInterrupt(this, source);
6611 }
6612
6613 IOReturn IOService::causeInterrupt(int source)
6614 {
6615   IOInterruptController *interruptController;
6616   IOReturn              ret;
6617
6618   ret = lookupInterrupt(source, false, &interruptController);
6619   if (ret != kIOReturnSuccess) return ret;
6620
6621   /* Cause an interrupt for the source */
6622   return interruptController->causeInterrupt(this, source);
6623 }
6624
6625 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6626                                     IOReportConfigureAction action,
6627                                     void                   *result,
6628                                     void                   *destination)
6629 {
6630     unsigned cnt;
6631
6632     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6633         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6634             if (pwrMgt) configurePowerStatesReport(action, result);
6635             else  return kIOReturnUnsupported;
6636         }
6637         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6638             if (pwrMgt) configureSimplePowerReport(action, result);
6639             else  return kIOReturnUnsupported;
6640         }
6641     }
6642
6643     IOLockLock(reserved->interruptStatisticsLock);
6644
6645     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6646     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6647         if (reserved->interruptStatisticsArray[cnt].reporter) {
6648             /*
6649              * If the reporter is currently associated with the statistics
6650              * for an event source, we may need to update the reporter.
6651              */
6652             if (reserved->interruptStatisticsArray[cnt].statistics)
6653                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6654
6655             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6656         }
6657     }
6658
6659     IOLockUnlock(reserved->interruptStatisticsLock);
6660
6661     return kIOReturnSuccess;
6662 }
6663
6664 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6665                                  IOReportUpdateAction      action,
6666                                  void                     *result,
6667                                  void                     *destination)
6668 {
6669     unsigned cnt;
6670
6671     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6672         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6673             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6674             else return kIOReturnUnsupported;
6675         }
6676         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6677             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6678             else return kIOReturnUnsupported;
6679         }
6680     }
6681
6682     IOLockLock(reserved->interruptStatisticsLock);
6683
6684     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6685     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6686         if (reserved->interruptStatisticsArray[cnt].reporter) {
6687             /*
6688              * If the reporter is currently associated with the statistics
6689              * for an event source, we need to update the reporter.
6690              */
6691             if (reserved->interruptStatisticsArray[cnt].statistics)
6692                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6693
6694             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6695         }
6696     }
6697
6698     IOLockUnlock(reserved->interruptStatisticsLock);
6699
6700     return kIOReturnSuccess;
6701 }
6702
6703 uint64_t IOService::getAuthorizationID( void )
6704 {
6705     return reserved->authorizationID;
6706 }
6707
6708 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6709 {
6710     OSObject * entitlement;
6711     IOReturn status;
6712
6713     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6714
6715     if ( entitlement )
6716     {
6717         if ( entitlement == kOSBooleanTrue )
6718         {
6719             reserved->authorizationID = authorizationID;
6720
6721             status = kIOReturnSuccess;
6722         }
6723         else
6724         {
6725             status = kIOReturnNotPrivileged;
6726         }
6727
6728         entitlement->release( );
6729     }
6730     else
6731     {
6732         status = kIOReturnNotPrivileged;
6733     }
6734
6735     return status;
6736 }
6737
6738 #if __LP64__
6739 OSMetaClassDefineReservedUsed(IOService, 0);
6740 OSMetaClassDefineReservedUsed(IOService, 1);
6741 OSMetaClassDefineReservedUnused(IOService, 2);
6742 OSMetaClassDefineReservedUnused(IOService, 3);
6743 OSMetaClassDefineReservedUnused(IOService, 4);
6744 OSMetaClassDefineReservedUnused(IOService, 5);
6745 OSMetaClassDefineReservedUnused(IOService, 6);
6746 OSMetaClassDefineReservedUnused(IOService, 7);
6747 #else
6748 OSMetaClassDefineReservedUsed(IOService, 0);
6749 OSMetaClassDefineReservedUsed(IOService, 1);
6750 OSMetaClassDefineReservedUsed(IOService, 2);
6751 OSMetaClassDefineReservedUsed(IOService, 3);
6752 OSMetaClassDefineReservedUsed(IOService, 4);
6753 OSMetaClassDefineReservedUsed(IOService, 5);
6754 OSMetaClassDefineReservedUsed(IOService, 6);
6755 OSMetaClassDefineReservedUsed(IOService, 7);
6756 #endif
6757 OSMetaClassDefineReservedUnused(IOService, 8);
6758 OSMetaClassDefineReservedUnused(IOService, 9);
6759 OSMetaClassDefineReservedUnused(IOService, 10);
6760 OSMetaClassDefineReservedUnused(IOService, 11);
6761 OSMetaClassDefineReservedUnused(IOService, 12);
6762 OSMetaClassDefineReservedUnused(IOService, 13);
6763 OSMetaClassDefineReservedUnused(IOService, 14);
6764 OSMetaClassDefineReservedUnused(IOService, 15);
6765 OSMetaClassDefineReservedUnused(IOService, 16);
6766 OSMetaClassDefineReservedUnused(IOService, 17);
6767 OSMetaClassDefineReservedUnused(IOService, 18);
6768 OSMetaClassDefineReservedUnused(IOService, 19);
6769 OSMetaClassDefineReservedUnused(IOService, 20);
6770 OSMetaClassDefineReservedUnused(IOService, 21);
6771 OSMetaClassDefineReservedUnused(IOService, 22);
6772 OSMetaClassDefineReservedUnused(IOService, 23);
6773 OSMetaClassDefineReservedUnused(IOService, 24);
6774 OSMetaClassDefineReservedUnused(IOService, 25);
6775 OSMetaClassDefineReservedUnused(IOService, 26);
6776 OSMetaClassDefineReservedUnused(IOService, 27);
6777 OSMetaClassDefineReservedUnused(IOService, 28);
6778 OSMetaClassDefineReservedUnused(IOService, 29);
6779 OSMetaClassDefineReservedUnused(IOService, 30);
6780 OSMetaClassDefineReservedUnused(IOService, 31);
6781 OSMetaClassDefineReservedUnused(IOService, 32);
6782 OSMetaClassDefineReservedUnused(IOService, 33);
6783 OSMetaClassDefineReservedUnused(IOService, 34);
6784 OSMetaClassDefineReservedUnused(IOService, 35);
6785 OSMetaClassDefineReservedUnused(IOService, 36);
6786 OSMetaClassDefineReservedUnused(IOService, 37);
6787 OSMetaClassDefineReservedUnused(IOService, 38);
6788 OSMetaClassDefineReservedUnused(IOService, 39);
6789 OSMetaClassDefineReservedUnused(IOService, 40);
6790 OSMetaClassDefineReservedUnused(IOService, 41);
6791 OSMetaClassDefineReservedUnused(IOService, 42);
6792 OSMetaClassDefineReservedUnused(IOService, 43);
6793 OSMetaClassDefineReservedUnused(IOService, 44);
6794 OSMetaClassDefineReservedUnused(IOService, 45);
6795 OSMetaClassDefineReservedUnused(IOService, 46);
6796 OSMetaClassDefineReservedUnused(IOService, 47);