iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <IOKit/AppleKeyStoreInterface.h>
  54 #include <IOKit/pwr_mgt/RootDomain.h>
  55 #include <IOKit/IOCPU.h>
  56 #include <mach/sync_policy.h>
  57 #include <IOKit/assert.h>
  58 #include <sys/errno.h>
  59 #include <sys/kdebug.h>
  60 #include <string.h>
  61
  62 #include <machine/pal_routines.h>
  63
  64 #define LOG kprintf
  65 //#define LOG IOLog
  66 #define MATCH_DEBUG     0
  67 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  68
  69 // disabled since lockForArbitration() can be held externally
  70 #define DEBUG_NOTIFIER_LOCKED   0
  71
  72 #include "IOServicePrivate.h"
  73 #include "IOKitKernelInternal.h"
  74
  75 // take lockForArbitration before LOCKNOTIFY
  76
  77 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  78
  79 #define super IORegistryEntry
  80
  81 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  82
  83 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  84 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  85
  86 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  87
  88 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  89
  90 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  91
  92 OSDefineMetaClassAndStructors(IOResources, IOService)
  93
  94 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  95
  96 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  97
  98 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  99
 100 static IOPlatformExpert *       gIOPlatform;
 101 static class IOPMrootDomain *   gIOPMRootDomain;
 102 const IORegistryPlane *         gIOServicePlane;
 103 const IORegistryPlane *         gIOPowerPlane;
 104 const OSSymbol *                gIODeviceMemoryKey;
 105 const OSSymbol *                gIOInterruptControllersKey;
 106 const OSSymbol *                gIOInterruptSpecifiersKey;
 107
 108 const OSSymbol *                gIOResourcesKey;
 109 const OSSymbol *                gIOResourceMatchKey;
 110 const OSSymbol *                gIOResourceMatchedKey;
 111 const OSSymbol *                gIOProviderClassKey;
 112 const OSSymbol *                gIONameMatchKey;
 113 const OSSymbol *                gIONameMatchedKey;
 114 const OSSymbol *                gIOPropertyMatchKey;
 115 const OSSymbol *                gIOPropertyExistsMatchKey;
 116 const OSSymbol *                gIOLocationMatchKey;
 117 const OSSymbol *                gIOParentMatchKey;
 118 const OSSymbol *                gIOPathMatchKey;
 119 const OSSymbol *                gIOMatchCategoryKey;
 120 const OSSymbol *                gIODefaultMatchCategoryKey;
 121 const OSSymbol *                gIOMatchedServiceCountKey;
 122 #if !CONFIG_EMBEDDED
 123 const OSSymbol *                gIOServiceLegacyMatchingRegistryIDKey;
 124 #endif
 125
 126 const OSSymbol *                gIOMapperIDKey;
 127 const OSSymbol *                gIOUserClientClassKey;
 128 const OSSymbol *                gIOKitDebugKey;
 129
 130 const OSSymbol *                gIOCommandPoolSizeKey;
 131
 132 const OSSymbol *                gIOConsoleLockedKey;
 133 const OSSymbol *                gIOConsoleUsersKey;
 134 const OSSymbol *                gIOConsoleSessionUIDKey;
 135 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 136 const OSSymbol *                gIOConsoleUsersSeedKey;
 137 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 138 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 139 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 140 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 141 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 142 clock_sec_t                     gIOConsoleLockTime;
 143 static bool                     gIOConsoleLoggedIn;
 144 #if HIBERNATION
 145 static OSBoolean *              gIOConsoleBooterLockState;
 146 static uint32_t                 gIOScreenLockState;
 147 #endif
 148 static IORegistryEntry *        gIOChosenEntry;
 149
 150 static int                      gIOResourceGenerationCount;
 151
 152 const OSSymbol *                gIOServiceKey;
 153 const OSSymbol *                gIOPublishNotification;
 154 const OSSymbol *                gIOFirstPublishNotification;
 155 const OSSymbol *                gIOMatchedNotification;
 156 const OSSymbol *                gIOFirstMatchNotification;
 157 const OSSymbol *                gIOTerminatedNotification;
 158 const OSSymbol *                gIOWillTerminateNotification;
 159
 160 const OSSymbol *                gIOGeneralInterest;
 161 const OSSymbol *                gIOBusyInterest;
 162 const OSSymbol *                gIOAppPowerStateInterest;
 163 const OSSymbol *                gIOPriorityPowerStateInterest;
 164 const OSSymbol *                gIOConsoleSecurityInterest;
 165
 166 const OSSymbol *                gIOBSDKey;
 167 const OSSymbol *                gIOBSDNameKey;
 168 const OSSymbol *                gIOBSDMajorKey;
 169 const OSSymbol *                gIOBSDMinorKey;
 170 const OSSymbol *                gIOBSDUnitKey;
 171
 172 const  OSSymbol *               gAKSGetKey;
 173 #if defined(__i386__) || defined(__x86_64__)
 174 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 175 #endif
 176
 177 static OSDictionary *           gNotifications;
 178 static IORecursiveLock *        gNotificationLock;
 179
 180 static IOService *              gIOResources;
 181 static IOService *              gIOServiceRoot;
 182
 183 static OSOrderedSet *           gJobs;
 184 static semaphore_port_t         gJobsSemaphore;
 185 static IOLock *                 gJobsLock;
 186 static int                      gOutstandingJobs;
 187 static int                      gNumConfigThreads;
 188 static int                      gNumWaitingThreads;
 189 static IOLock *                 gIOServiceBusyLock;
 190 bool                            gCPUsRunning;
 191
 192 static thread_t                 gIOTerminateThread;
 193 static thread_t                 gIOTerminateWorkerThread;
 194 static UInt32                   gIOTerminateWork;
 195 static OSArray *                gIOTerminatePhase2List;
 196 static OSArray *                gIOStopList;
 197 static OSArray *                gIOStopProviderList;
 198 static OSArray *                gIOFinalizeList;
 199
 200 static SInt32                   gIOConsoleUsersSeed;
 201 static OSData *                 gIOConsoleUsersSeedValue;
 202
 203 extern const OSSymbol *         gIODTPHandleKey;
 204
 205 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 206
 207 static IOLock *                 gIOConsoleUsersLock;
 208 static thread_call_t            gIOConsoleLockCallout;
 209 static IONotifier *             gIOServiceNullNotifier;
 210
 211 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 212
 213 #define LOCKREADNOTIFY()        \
 214     IORecursiveLockLock( gNotificationLock )
 215 #define LOCKWRITENOTIFY()       \
 216     IORecursiveLockLock( gNotificationLock )
 217 #define LOCKWRITE2READNOTIFY()
 218 #define UNLOCKNOTIFY()          \
 219     IORecursiveLockUnlock( gNotificationLock )
 220 #define SLEEPNOTIFY(event) \
 221     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 222 #define SLEEPNOTIFYTO(event, deadline) \
 223     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 224 #define WAKEUPNOTIFY(event) \
 225         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 226
 227 #define randomDelay()   \
 228         int del = read_processor_clock();                               \
 229         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 230         IOSleep( del );
 231
 232 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 233
 234 #define queue_element(entry, element, type, field) do { \
 235         vm_address_t __ele = (vm_address_t) (entry);    \
 236         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 237         (element) = (type) __ele;                       \
 238     } while(0)
 239
 240 #define iterqueue(que, elt)                             \
 241         for (queue_entry_t elt = queue_first(que);      \
 242              !queue_end(que, elt);                      \
 243              elt = queue_next(elt))
 244
 245 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 246
 247 struct IOInterruptAccountingReporter {
 248     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 249     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 250 };
 251
 252 struct ArbitrationLockQueueElement {
 253     queue_chain_t link;
 254     IOThread      thread;
 255     IOService *   service;
 256     unsigned      count;
 257     bool          required;
 258     bool          aborted;
 259 };
 260
 261 static queue_head_t gArbitrationLockQueueActive;
 262 static queue_head_t gArbitrationLockQueueWaiting;
 263 static queue_head_t gArbitrationLockQueueFree;
 264 static IOLock *     gArbitrationLockQueueLock;
 265
 266 bool IOService::isInactive( void ) const
 267     { return( 0 != (kIOServiceInactiveState & getState())); }
 268
 269 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 270
 271 #if defined(__i386__) || defined(__x86_64__)
 272
 273 // Only used by the intel implementation of
 274 //     IOService::requireMaxBusStall(UInt32 ns)
 275 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 276 struct CpuDelayEntry
 277 {
 278     IOService * fService;
 279     UInt32      fMaxDelay;
 280     UInt32      fDelayType;
 281 };
 282
 283 enum {
 284     kCpuDelayBusStall, kCpuDelayInterrupt,
 285     kCpuNumDelayTypes
 286 };
 287
 288 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 289 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 290 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 291 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 292 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 293 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 294 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 295
 296 static void
 297 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 298 static IOReturn
 299 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 300
 301 #endif /* defined(__i386__) || defined(__x86_64__) */
 302
 303 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 304
 305 void IOService::initialize( void )
 306 {
 307     kern_return_t       err;
 308
 309     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 310     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 311
 312     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 313     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 314     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 315     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 316     gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 317     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 318     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 319     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 320
 321     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 322     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 323                                         kIODefaultMatchCategoryKey );
 324     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 325                                         kIOMatchedServiceCountKey );
 326 #if !CONFIG_EMBEDDED
 327     gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
 328                                         kIOServiceLegacyMatchingRegistryIDKey );
 329 #endif
 330
 331     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 332
 333     gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 334     gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 335     gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 336
 337     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 338     gIOInterruptControllersKey
 339         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 340     gIOInterruptSpecifiersKey
 341         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 342
 343     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 344
 345     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 346
 347     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 348
 349     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 350     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 351     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 352     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 353     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 354
 355     gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 356     gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 357     gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 358     gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 359     gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 360
 361     gNotifications              = OSDictionary::withCapacity( 1 );
 362     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 363                                                  kIOPublishNotification );
 364     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 365                                                  kIOFirstPublishNotification );
 366     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 367                                                  kIOMatchedNotification );
 368     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 369                                                  kIOFirstMatchNotification );
 370     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 371                                                  kIOTerminatedNotification );
 372     gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
 373                                                  kIOWillTerminateNotification );
 374     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 375
 376     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 377     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 378     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 379     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 380
 381     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 382     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 383     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 384     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 385     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 386     gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 387
 388     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 389
 390     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 391 #if defined(__i386__) || defined(__x86_64__)
 392     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 393     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 394     uint32_t  idx;
 395     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 396     {
 397         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 398         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 399         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 400     }
 401     gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 402 #endif
 403     gNotificationLock           = IORecursiveLockAlloc();
 404
 405     gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 406
 407     assert( gIOServicePlane && gIODeviceMemoryKey
 408         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 409         && gIOResourcesKey && gNotifications && gNotificationLock
 410         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 411         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 412         && gIOPublishNotification && gIOMatchedNotification
 413         && gIOTerminatedNotification && gIOServiceKey
 414         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 415     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 416         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 417
 418     gJobsLock   = IOLockAlloc();
 419     gJobs       = OSOrderedSet::withCapacity( 10 );
 420
 421     gIOServiceBusyLock = IOLockAlloc();
 422
 423     gIOConsoleUsersLock = IOLockAlloc();
 424
 425     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 426
 427     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 428
 429     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 430
 431     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 432                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 433
 434     gIOResources = IOResources::resources();
 435     assert( gIOResources );
 436
 437     gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 438     assert(gIOServiceNullNotifier);
 439
 440     gArbitrationLockQueueLock = IOLockAlloc();
 441     queue_init(&gArbitrationLockQueueActive);
 442     queue_init(&gArbitrationLockQueueWaiting);
 443     queue_init(&gArbitrationLockQueueFree);
 444
 445     assert( gArbitrationLockQueueLock );
 446
 447     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 448     gIOStopList            = OSArray::withCapacity( 16 );
 449     gIOStopProviderList    = OSArray::withCapacity( 16 );
 450     gIOFinalizeList        = OSArray::withCapacity( 16 );
 451     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 452
 453     // worker thread that is responsible for terminating / cleaning up threads
 454     kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
 455     assert(gIOTerminateWorkerThread);
 456     thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
 457 }
 458
 459 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 460
 461 #if defined(__i386__) || defined(__x86_64__)
 462 extern "C" {
 463
 464 const char *getCpuDelayBusStallHolderName(void);
 465 const char *getCpuDelayBusStallHolderName(void) {
 466     return sCPULatencyHolderName[kCpuDelayBusStall];
 467 }
 468
 469 }
 470 #endif
 471
 472 #if IOMATCHDEBUG
 473 static UInt64 getDebugFlags( OSDictionary * props )
 474 {
 475     OSNumber *  debugProp;
 476     UInt64      debugFlags;
 477
 478     debugProp = OSDynamicCast( OSNumber,
 479                 props->getObject( gIOKitDebugKey ));
 480     if( debugProp)
 481         debugFlags = debugProp->unsigned64BitValue();
 482     else
 483         debugFlags = gIOKitDebug;
 484
 485     return( debugFlags );
 486 }
 487
 488 static UInt64 getDebugFlags( IOService * inst )
 489 {
 490     OSObject *  prop;
 491     OSNumber *  debugProp;
 492     UInt64      debugFlags;
 493
 494     prop = inst->copyProperty(gIOKitDebugKey);
 495     debugProp = OSDynamicCast(OSNumber, prop);
 496     if( debugProp)
 497         debugFlags = debugProp->unsigned64BitValue();
 498     else
 499         debugFlags = gIOKitDebug;
 500
 501     OSSafeReleaseNULL(prop);
 502
 503     return( debugFlags );
 504 }
 505 #endif
 506
 507 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 508
 509 // Probe a matched service and return an instance to be started.
 510 // The default score is from the property table, & may be altered
 511 // during probe to change the start order.
 512
 513 IOService * IOService::probe(   IOService * provider,
 514                                 SInt32    * score )
 515 {
 516     return( this );
 517 }
 518
 519 bool IOService::start( IOService * provider )
 520 {
 521     return( true );
 522 }
 523
 524 void IOService::stop( IOService * provider )
 525 {
 526 }
 527
 528 bool IOService::init( OSDictionary * dictionary )
 529 {
 530     bool ret;
 531
 532     ret = super::init(dictionary);
 533     if (!ret)     return (false);
 534     if (reserved) return (true);
 535
 536     reserved = IONew(ExpansionData, 1);
 537     if (!reserved) return (false);
 538     bzero(reserved, sizeof(*reserved));
 539
 540     /*
 541      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 542      * lock based on the presence of specifiers ran into issues as some
 543      * platforms set up the specifiers after IOService initialization.
 544      *
 545      * We may be able to get away with a global lock, as this should only be
 546      * contended by IOReporting clients and driver start/stop (unless a
 547      * driver wants to remove/add handlers in the course of normal operation,
 548      * which should be unlikely).
 549      */
 550     reserved->interruptStatisticsLock = IOLockAlloc();
 551     if (!reserved->interruptStatisticsLock) return (false);
 552
 553     return (true);
 554 }
 555
 556 bool IOService::init( IORegistryEntry * from,
 557                       const IORegistryPlane * inPlane )
 558 {
 559     bool ret;
 560
 561     ret = super::init(from, inPlane);
 562     if (!ret)     return (false);
 563     if (reserved) return (true);
 564
 565     reserved = IONew(ExpansionData, 1);
 566     if (!reserved) return (false);
 567     bzero(reserved, sizeof(*reserved));
 568
 569     /*
 570      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 571      * lock based on the presence of specifiers ran into issues as some
 572      * platforms set up the specifiers after IOService initialization.
 573      *
 574      * We may be able to get away with a global lock, as this should only be
 575      * contended by IOReporting clients and driver start/stop (unless a
 576      * driver wants to remove/add handlers in the course of normal operation,
 577      * which should be unlikely).
 578      */
 579     reserved->interruptStatisticsLock = IOLockAlloc();
 580     if (!reserved->interruptStatisticsLock) return (false);
 581
 582     return (true);
 583 }
 584
 585 void IOService::free( void )
 586 {
 587     int i = 0;
 588     requireMaxBusStall(0);
 589     requireMaxInterruptDelay(0);
 590     if( getPropertyTable())
 591         unregisterAllInterest();
 592     PMfree();
 593
 594     if (reserved) {
 595         if (reserved->interruptStatisticsArray) {
 596             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 597                 if (reserved->interruptStatisticsArray[i].reporter)
 598                     reserved->interruptStatisticsArray[i].reporter->release();
 599             }
 600
 601             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 602         }
 603
 604         if (reserved->interruptStatisticsLock)
 605             IOLockFree(reserved->interruptStatisticsLock);
 606         IODelete(reserved, ExpansionData, 1);
 607     }
 608
 609     if (_numInterruptSources && _interruptSources)
 610     {
 611         IOFree(_interruptSources, _numInterruptSources * sizeof(IOInterruptSource));
 612         _interruptSources = 0;
 613     }
 614
 615     super::free();
 616 }
 617
 618 /*
 619  * Attach in service plane
 620  */
 621 bool IOService::attach( IOService * provider )
 622 {
 623     bool         ok;
 624     uint32_t     count;
 625     AbsoluteTime deadline;
 626     int          waitResult = THREAD_AWAKENED;
 627     bool         wait, computeDeadline = true;
 628
 629     if( provider) {
 630
 631         if( gIOKitDebug & kIOLogAttach)
 632             LOG( "%s::attach(%s)\n", getName(),
 633                     provider->getName());
 634
 635         ok   = false;
 636         do
 637         {
 638             wait = false;
 639             provider->lockForArbitration();
 640             if (provider->__state[0] & kIOServiceInactiveState) ok = false;
 641             else
 642             {
 643                 count = provider->getChildCount(gIOServicePlane);
 644                 wait = (count > (kIOServiceBusyMax - 4));
 645                 if (!wait) ok = attachToParent(provider, gIOServicePlane);
 646                 else
 647                 {
 648                     IOLog("stalling for detach from %s\n", provider->getName());
 649                     IOLockLock( gIOServiceBusyLock );
 650                     provider->__state[1] |= kIOServiceWaitDetachState;
 651                 }
 652             }
 653             provider->unlockForArbitration();
 654             if (wait)
 655             {
 656                 if (computeDeadline)
 657                 {
 658                     clock_interval_to_deadline(15, kSecondScale, &deadline);
 659                     computeDeadline = false;
 660                 }
 661                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 662                 IOLockUnlock( gIOServiceBusyLock );
 663                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 664                 wait = (waitResult != THREAD_TIMED_OUT);
 665             }
 666         }
 667         while (wait);
 668
 669     } else {
 670         gIOServiceRoot = this;
 671         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 672     }
 673
 674     if (ok && !__provider) (void) getProvider();
 675
 676     return( ok );
 677 }
 678
 679 IOService * IOService::getServiceRoot( void )
 680 {
 681     return( gIOServiceRoot );
 682 }
 683
 684 void IOService::detach( IOService * provider )
 685 {
 686     IOService * newProvider = 0;
 687     SInt32      busy;
 688     bool        adjParent;
 689
 690     if( gIOKitDebug & kIOLogAttach)
 691         LOG("%s::detach(%s)\n", getName(), provider->getName());
 692
 693     lockForArbitration();
 694
 695     uint64_t regID1 = provider->getRegistryEntryID();
 696     uint64_t regID2 = getRegistryEntryID();
 697     IOServiceTrace(
 698         IOSERVICE_DETACH,
 699         (uintptr_t) regID1,
 700         (uintptr_t) (regID1 >> 32),
 701         (uintptr_t) regID2,
 702         (uintptr_t) (regID2 >> 32));
 703
 704     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 705                && (provider == getProvider()));
 706
 707     detachFromParent( provider, gIOServicePlane );
 708
 709     if( busy) {
 710         newProvider = getProvider();
 711         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 712             _adjustBusy( -busy );
 713     }
 714
 715     if (kIOServiceInactiveState & __state[0]) {
 716         getMetaClass()->removeInstance(this);
 717         IORemoveServicePlatformActions(this);
 718     }
 719
 720     unlockForArbitration();
 721
 722     if( newProvider) {
 723         newProvider->lockForArbitration();
 724         newProvider->_adjustBusy(1);
 725         newProvider->unlockForArbitration();
 726     }
 727
 728     // check for last client detach from a terminated service
 729     if( provider->lockForArbitration( true ))
 730     {
 731         if (kIOServiceStartState & __state[1])
 732         {
 733             provider->scheduleTerminatePhase2();
 734         }
 735         if( adjParent) provider->_adjustBusy( -1 );
 736         if( (provider->__state[1] & kIOServiceTermPhase3State)
 737          && (0 == provider->getClient())) {
 738             provider->scheduleFinalize(false);
 739         }
 740
 741         IOLockLock( gIOServiceBusyLock );
 742         if (kIOServiceWaitDetachState & provider->__state[1])
 743         {
 744             provider->__state[1] &= ~kIOServiceWaitDetachState;
 745             thread_wakeup(&provider->__provider);
 746         }
 747         IOLockUnlock( gIOServiceBusyLock );
 748
 749         provider->unlockForArbitration();
 750     }
 751 }
 752
 753 /*
 754  * Register instance - publish it for matching
 755  */
 756
 757 void IOService::registerService( IOOptionBits options )
 758 {
 759     char *              pathBuf;
 760     const char *        path;
 761     char *              skip;
 762     int                 len;
 763     enum { kMaxPathLen  = 256 };
 764     enum { kMaxChars    = 63 };
 765
 766     IORegistryEntry * parent = this;
 767     IORegistryEntry * root = getRegistryRoot();
 768     while( parent && (parent != root))
 769         parent = parent->getParentEntry( gIOServicePlane);
 770
 771     if( parent != root) {
 772         IOLog("%s: not registry member at registerService()\n", getName());
 773         return;
 774     }
 775
 776     // Allow the Platform Expert to adjust this node.
 777     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 778         return;
 779
 780     IOInstallServicePlatformActions(this);
 781
 782     if( (this != gIOResources)
 783      && (kIOLogRegister & gIOKitDebug)) {
 784
 785         pathBuf = (char *) IOMalloc( kMaxPathLen );
 786
 787         IOLog( "Registering: " );
 788
 789         len = kMaxPathLen;
 790         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 791
 792             path = pathBuf;
 793             if( len > kMaxChars) {
 794                 IOLog("..");
 795                 len -= kMaxChars;
 796                 path += len;
 797                 if( (skip = strchr( path, '/')))
 798                     path = skip;
 799             }
 800         } else
 801             path = getName();
 802
 803         IOLog( "%s\n", path );
 804
 805         if( pathBuf)
 806             IOFree( pathBuf, kMaxPathLen );
 807     }
 808
 809     startMatching( options );
 810 }
 811
 812 void IOService::startMatching( IOOptionBits options )
 813 {
 814     IOService * provider;
 815     UInt32      prevBusy = 0;
 816     bool        needConfig;
 817     bool        needWake = false;
 818     bool        ok;
 819     bool        sync;
 820     bool        waitAgain;
 821
 822     lockForArbitration();
 823
 824     sync = (options & kIOServiceSynchronous)
 825         || ((provider = getProvider())
 826                 && (provider->__state[1] & kIOServiceSynchronousState));
 827
 828         if ( options & kIOServiceAsynchronous )
 829                 sync = false;
 830
 831     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
 832                && (0 == (__state[0] & kIOServiceInactiveState));
 833
 834     __state[1] |= kIOServiceNeedConfigState;
 835
 836 //    __state[0] &= ~kIOServiceInactiveState;
 837
 838 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 839 //                      OSKernelStackRemaining(), getName());
 840
 841     if( needConfig) {
 842         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 843     }
 844
 845     if( sync)
 846         __state[1] |= kIOServiceSynchronousState;
 847     else
 848         __state[1] &= ~kIOServiceSynchronousState;
 849
 850     if( needConfig) prevBusy = _adjustBusy( 1 );
 851
 852     unlockForArbitration();
 853
 854     if( needConfig) {
 855
 856         if( needWake) {
 857             IOLockLock( gIOServiceBusyLock );
 858             thread_wakeup( (event_t) this/*&__state[1]*/ );
 859             IOLockUnlock( gIOServiceBusyLock );
 860
 861         } else if( !sync || (kIOServiceAsynchronous & options)) {
 862
 863             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 864
 865         } else do {
 866
 867             if( (__state[1] & kIOServiceNeedConfigState))
 868                 doServiceMatch( options );
 869
 870             lockForArbitration();
 871             IOLockLock( gIOServiceBusyLock );
 872
 873             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 874                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 875
 876             if( waitAgain)
 877                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 878             else
 879                 __state[1] &= ~kIOServiceSyncPubState;
 880
 881             unlockForArbitration();
 882
 883             if( waitAgain)
 884                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 885
 886             IOLockUnlock( gIOServiceBusyLock );
 887             if( waitAgain)
 888                 thread_block(THREAD_CONTINUE_NULL);
 889
 890         } while( waitAgain );
 891     }
 892 }
 893
 894 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 895 {
 896     OSDictionary *      table;
 897     OSSet *             set;
 898     OSSet *             allSet = 0;
 899     IOService *         service;
 900 #if IOMATCHDEBUG
 901     SInt32              count = 0;
 902 #endif
 903
 904     newTables->retain();
 905
 906     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 907
 908         LOCKWRITENOTIFY();
 909         set = (OSSet *) copyExistingServices( table,
 910                                                 kIOServiceRegisteredState,
 911                                                 kIOServiceExistingSet);
 912         UNLOCKNOTIFY();
 913         if( set) {
 914
 915 #if IOMATCHDEBUG
 916             count += set->getCount();
 917 #endif
 918             if (allSet) {
 919                 allSet->merge((const OSSet *) set);
 920                 set->release();
 921             }
 922             else
 923                 allSet = set;
 924         }
 925
 926 #if IOMATCHDEBUG
 927         if( getDebugFlags( table ) & kIOLogMatch)
 928             LOG("Matching service count = %ld\n", (long)count);
 929 #endif
 930         newTables->removeObject(table);
 931     }
 932
 933     if (allSet) {
 934         while( (service = (IOService *) allSet->getAnyObject())) {
 935             service->startMatching(kIOServiceAsynchronous);
 936             allSet->removeObject(service);
 937         }
 938         allSet->release();
 939     }
 940
 941     newTables->release();
 942
 943     return( kIOReturnSuccess );
 944 }
 945
 946  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 947                                                 IOOptionBits options )
 948 {
 949     _IOServiceJob *     job;
 950
 951     job = new _IOServiceJob;
 952     if( job && !job->init()) {
 953         job->release();
 954         job = 0;
 955     }
 956
 957     if( job) {
 958         job->type       = type;
 959         job->nub        = nub;
 960         job->options    = options;
 961         nub->retain();                  // thread will release()
 962         pingConfig( job );
 963     }
 964
 965     return( job );
 966 }
 967
 968 /*
 969  * Called on a registered service to see if it matches
 970  * a property table.
 971  */
 972
 973 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 974 {
 975     return( matchPropertyTable(table) );
 976 }
 977
 978 bool IOService::matchPropertyTable( OSDictionary * table )
 979 {
 980     return( true );
 981 }
 982
 983 /*
 984  * Called on a matched service to allocate resources
 985  * before first driver is attached.
 986  */
 987
 988 IOReturn IOService::getResources( void )
 989 {
 990     return( kIOReturnSuccess);
 991 }
 992
 993 /*
 994  * Client/provider accessors
 995  */
 996
 997 IOService * IOService::getProvider( void ) const
 998 {
 999     IOService * self = (IOService *) this;
1000     IOService * parent;
1001     SInt32      generation;
1002
1003     generation = getRegistryEntryGenerationCount();
1004     if( __providerGeneration == generation)
1005         return( __provider );
1006
1007     parent = (IOService *) getParentEntry( gIOServicePlane);
1008     if( parent == IORegistryEntry::getRegistryRoot())
1009         /* root is not an IOService */
1010         parent = 0;
1011
1012     self->__provider = parent;
1013     OSMemoryBarrier();
1014     // save the count from before call to getParentEntry()
1015     self->__providerGeneration = generation;
1016
1017     return( parent );
1018 }
1019
1020 IOWorkLoop * IOService::getWorkLoop() const
1021 {
1022     IOService *provider = getProvider();
1023
1024     if (provider)
1025         return provider->getWorkLoop();
1026     else
1027         return 0;
1028 }
1029
1030 OSIterator * IOService::getProviderIterator( void ) const
1031 {
1032     return( getParentIterator( gIOServicePlane));
1033 }
1034
1035 IOService * IOService::getClient( void ) const
1036 {
1037     return( (IOService *) getChildEntry( gIOServicePlane));
1038 }
1039
1040 OSIterator * IOService::getClientIterator( void ) const
1041 {
1042     return( getChildIterator( gIOServicePlane));
1043 }
1044
1045 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
1046                                                 const IOService * client,
1047                                                 const IOService * provider )
1048 {
1049     _IOOpenServiceIterator * inst;
1050
1051     if( !_iter)
1052         return( 0 );
1053
1054     inst = new _IOOpenServiceIterator;
1055
1056     if( inst && !inst->init()) {
1057         inst->release();
1058         inst = 0;
1059     }
1060     if( inst) {
1061         inst->iter = _iter;
1062         inst->client = client;
1063         inst->provider = provider;
1064     }
1065
1066     return( inst );
1067 }
1068
1069 void _IOOpenServiceIterator::free()
1070 {
1071     iter->release();
1072     if( last)
1073         last->unlockForArbitration();
1074     OSIterator::free();
1075 }
1076
1077 OSObject * _IOOpenServiceIterator::getNextObject()
1078 {
1079     IOService * next;
1080
1081     if( last)
1082         last->unlockForArbitration();
1083
1084     while( (next = (IOService *) iter->getNextObject())) {
1085
1086         next->lockForArbitration();
1087         if( (client && (next->isOpen( client )))
1088          || (provider && (provider->isOpen( next ))) )
1089             break;
1090         next->unlockForArbitration();
1091     }
1092
1093     last = next;
1094
1095     return( next );
1096 }
1097
1098 bool _IOOpenServiceIterator::isValid()
1099 {
1100     return( iter->isValid() );
1101 }
1102
1103 void _IOOpenServiceIterator::reset()
1104 {
1105     if( last) {
1106         last->unlockForArbitration();
1107         last = 0;
1108     }
1109     iter->reset();
1110 }
1111
1112 OSIterator * IOService::getOpenProviderIterator( void ) const
1113 {
1114     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1115 }
1116
1117 OSIterator * IOService::getOpenClientIterator( void ) const
1118 {
1119     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1120 }
1121
1122
1123 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1124                                           bool waitForFunction,
1125                                           void *param1, void *param2,
1126                                           void *param3, void *param4 )
1127 {
1128   IOReturn  result = kIOReturnUnsupported;
1129   IOService *provider;
1130
1131   if (gIOPlatformFunctionHandlerSet == functionName)
1132   {
1133 #if defined(__i386__) || defined(__x86_64__)
1134     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1135     IOService *      target              = (IOService *) param2;
1136     bool             enable              = (param3 != 0);
1137
1138     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1139         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1140     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1141         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1142 #endif /* defined(__i386__) || defined(__x86_64__) */
1143   }
1144
1145   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1146     result = provider->callPlatformFunction(functionName, waitForFunction,
1147                                             param1, param2, param3, param4);
1148   }
1149
1150   return result;
1151 }
1152
1153 IOReturn IOService::callPlatformFunction( const char * functionName,
1154                                           bool waitForFunction,
1155                                           void *param1, void *param2,
1156                                           void *param3, void *param4 )
1157 {
1158   IOReturn result = kIOReturnNoMemory;
1159   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1160
1161   if (functionSymbol != 0) {
1162     result = callPlatformFunction(functionSymbol, waitForFunction,
1163                                   param1, param2, param3, param4);
1164     functionSymbol->release();
1165   }
1166
1167   return result;
1168 }
1169
1170
1171 /*
1172  * Accessors for global services
1173  */
1174
1175 IOPlatformExpert * IOService::getPlatform( void )
1176 {
1177     return( gIOPlatform);
1178 }
1179
1180 class IOPMrootDomain * IOService::getPMRootDomain( void )
1181 {
1182     return( gIOPMRootDomain);
1183 }
1184
1185 IOService * IOService::getResourceService( void )
1186 {
1187     return( gIOResources );
1188 }
1189
1190 void IOService::setPlatform( IOPlatformExpert * platform)
1191 {
1192     gIOPlatform = platform;
1193     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1194
1195 #if defined(__i386__) || defined(__x86_64__)
1196
1197     static const char * keys[kCpuNumDelayTypes] = {
1198         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1199     const OSObject * objs[2];
1200     OSArray * array;
1201     uint32_t  idx;
1202
1203     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1204     {
1205         objs[0] = sCPULatencySet[idx];
1206         objs[1] = sCPULatencyHolder[idx];
1207         array   = OSArray::withObjects(objs, 2);
1208         if (!array) break;
1209         platform->setProperty(keys[idx], array);
1210         array->release();
1211     }
1212 #endif /* defined(__i386__) || defined(__x86_64__) */
1213 }
1214
1215 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1216 {
1217     gIOPMRootDomain = rootDomain;
1218     publishResource("IOKit");
1219 }
1220
1221 /*
1222  * Stacking change
1223  */
1224
1225 bool IOService::lockForArbitration( bool isSuccessRequired )
1226 {
1227     bool                          found;
1228     bool                          success;
1229     ArbitrationLockQueueElement * element;
1230     ArbitrationLockQueueElement * active;
1231     ArbitrationLockQueueElement * waiting;
1232
1233     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1234
1235     // lock global access
1236     IOTakeLock( gArbitrationLockQueueLock );
1237
1238     // obtain an unused queue element
1239     if( !queue_empty( &gArbitrationLockQueueFree )) {
1240         queue_remove_first( &gArbitrationLockQueueFree,
1241                             element,
1242                             ArbitrationLockQueueElement *,
1243                             link );
1244     } else {
1245         element = IONew( ArbitrationLockQueueElement, 1 );
1246         assert( element );
1247     }
1248
1249     // prepare the queue element
1250     element->thread   = IOThreadSelf();
1251     element->service  = this;
1252     element->count    = 1;
1253     element->required = isSuccessRequired;
1254     element->aborted  = false;
1255
1256     // determine whether this object is already locked (ie. on active queue)
1257     found = false;
1258     queue_iterate( &gArbitrationLockQueueActive,
1259                     active,
1260                     ArbitrationLockQueueElement *,
1261                     link )
1262     {
1263         if( active->service == element->service ) {
1264             found = true;
1265             break;
1266         }
1267     }
1268
1269     if( found ) { // this object is already locked
1270
1271         // determine whether it is the same or a different thread trying to lock
1272         if( active->thread != element->thread ) { // it is a different thread
1273
1274             ArbitrationLockQueueElement * victim = 0;
1275
1276             // before placing this new thread on the waiting queue, we look for
1277             // a deadlock cycle...
1278
1279             while( 1 ) {
1280                 // determine whether the active thread holding the object we
1281                 // want is waiting for another object to be unlocked
1282                 found = false;
1283                 queue_iterate( &gArbitrationLockQueueWaiting,
1284                                waiting,
1285                                ArbitrationLockQueueElement *,
1286                                link )
1287                 {
1288                     if( waiting->thread == active->thread ) {
1289                         assert( false == waiting->aborted );
1290                         found = true;
1291                         break;
1292                     }
1293                 }
1294
1295                 if( found ) { // yes, active thread waiting for another object
1296
1297                     // this may be a candidate for rejection if the required
1298                     // flag is not set, should we detect a deadlock later on
1299                     if( false == waiting->required )
1300                         victim = waiting;
1301
1302                     // find the thread that is holding this other object, that
1303                     // is blocking the active thread from proceeding (fun :-)
1304                     found = false;
1305                     queue_iterate( &gArbitrationLockQueueActive,
1306                                    active,      // (reuse active queue element)
1307                                    ArbitrationLockQueueElement *,
1308                                    link )
1309                     {
1310                         if( active->service == waiting->service ) {
1311                             found = true;
1312                             break;
1313                         }
1314                     }
1315
1316                     // someone must be holding it or it wouldn't be waiting
1317                     assert( found );
1318
1319                     if( active->thread == element->thread ) {
1320
1321                         // doh, it's waiting for the thread that originated
1322                         // this whole lock (ie. current thread) -> deadlock
1323                         if( false == element->required ) { // willing to fail?
1324
1325                             // the originating thread doesn't have the required
1326                             // flag, so it can fail
1327                             success = false; // (fail originating lock request)
1328                             break; // (out of while)
1329
1330                         } else { // originating thread is not willing to fail
1331
1332                             // see if we came across a waiting thread that did
1333                             // not have the 'required' flag set: we'll fail it
1334                             if( victim ) {
1335
1336                                 // we do have a willing victim, fail it's lock
1337                                 victim->aborted = true;
1338
1339                                 // take the victim off the waiting queue
1340                                 queue_remove( &gArbitrationLockQueueWaiting,
1341                                               victim,
1342                                               ArbitrationLockQueueElement *,
1343                                               link );
1344
1345                                 // wake the victim
1346                                 IOLockWakeup( gArbitrationLockQueueLock,
1347                                               victim,
1348                                               /* one thread */ true );
1349
1350                                 // allow this thread to proceed (ie. wait)
1351                                 success = true; // (put request on wait queue)
1352                                 break; // (out of while)
1353                             } else {
1354
1355                                 // all the waiting threads we came across in
1356                                 // finding this loop had the 'required' flag
1357                                 // set, so we've got a deadlock we can't avoid
1358                                 panic("I/O Kit: Unrecoverable deadlock.");
1359                             }
1360                         }
1361                     } else {
1362                         // repeat while loop, redefining active thread to be the
1363                         // thread holding "this other object" (see above), and
1364                         // looking for threads waiting on it; note the active
1365                         // variable points to "this other object" already... so
1366                         // there nothing to do in this else clause.
1367                     }
1368                 } else { // no, active thread is not waiting for another object
1369
1370                     success = true; // (put request on wait queue)
1371                     break; // (out of while)
1372                 }
1373             } // while forever
1374
1375             if( success ) { // put the request on the waiting queue?
1376                 kern_return_t wait_result;
1377
1378                 // place this thread on the waiting queue and put it to sleep;
1379                 // we place it at the tail of the queue...
1380                 queue_enter( &gArbitrationLockQueueWaiting,
1381                              element,
1382                              ArbitrationLockQueueElement *,
1383                              link );
1384
1385                 // declare that this thread will wait for a given event
1386 restart_sleep:  wait_result = assert_wait( element,
1387                                            element->required ? THREAD_UNINT
1388                                            : THREAD_INTERRUPTIBLE );
1389
1390                 // unlock global access
1391                 IOUnlock( gArbitrationLockQueueLock );
1392
1393                 // put thread to sleep, waiting for our event to fire...
1394                 if (wait_result == THREAD_WAITING)
1395                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1396
1397
1398                 // ...and we've been woken up; we might be in one of two states:
1399                 // (a) we've been aborted and our queue element is not on
1400                 //     any of the three queues, but is floating around
1401                 // (b) we're allowed to proceed with the lock and we have
1402                 //     already been moved from the waiting queue to the
1403                 //     active queue.
1404                 // ...plus a 3rd state, should the thread have been interrupted:
1405                 // (c) we're still on the waiting queue
1406
1407                 // determine whether we were interrupted out of our sleep
1408                 if( THREAD_INTERRUPTED == wait_result ) {
1409
1410                     // re-lock global access
1411                     IOTakeLock( gArbitrationLockQueueLock );
1412
1413                     // determine whether we're still on the waiting queue
1414                     found = false;
1415                     queue_iterate( &gArbitrationLockQueueWaiting,
1416                                    waiting,     // (reuse waiting queue element)
1417                                    ArbitrationLockQueueElement *,
1418                                    link )
1419                     {
1420                         if( waiting == element ) {
1421                             found = true;
1422                             break;
1423                         }
1424                     }
1425
1426                     if( found ) { // yes, we're still on the waiting queue
1427
1428                         // determine whether we're willing to fail
1429                         if( false == element->required ) {
1430
1431                             // mark us as aborted
1432                             element->aborted = true;
1433
1434                             // take us off the waiting queue
1435                             queue_remove( &gArbitrationLockQueueWaiting,
1436                                           element,
1437                                           ArbitrationLockQueueElement *,
1438                                           link );
1439                         } else { // we are not willing to fail
1440
1441                             // ignore interruption, go back to sleep
1442                             goto restart_sleep;
1443                         }
1444                     }
1445
1446                     // unlock global access
1447                     IOUnlock( gArbitrationLockQueueLock );
1448
1449                     // proceed as though this were a normal wake up
1450                     wait_result = THREAD_AWAKENED;
1451                 }
1452
1453                 assert( THREAD_AWAKENED == wait_result );
1454
1455                 // determine whether we've been aborted while we were asleep
1456                 if( element->aborted ) {
1457                     assert( false == element->required );
1458
1459                     // re-lock global access
1460                     IOTakeLock( gArbitrationLockQueueLock );
1461
1462                     action = kPutOnFreeQueue;
1463                     success = false;
1464                 } else { // we weren't aborted, so we must be ready to go :-)
1465
1466                     // we've already been moved from waiting to active queue
1467                     return true;
1468                 }
1469
1470             } else { // the lock request is to be failed
1471
1472                 // return unused queue element to queue
1473                 action = kPutOnFreeQueue;
1474             }
1475         } else { // it is the same thread, recursive access is allowed
1476
1477             // add one level of recursion
1478             active->count++;
1479
1480             // return unused queue element to queue
1481             action = kPutOnFreeQueue;
1482             success = true;
1483         }
1484     } else { // this object is not already locked, so let this thread through
1485         action = kPutOnActiveQueue;
1486         success = true;
1487     }
1488
1489     // put the new element on a queue
1490     if( kPutOnActiveQueue == action ) {
1491         queue_enter( &gArbitrationLockQueueActive,
1492                      element,
1493                      ArbitrationLockQueueElement *,
1494                      link );
1495     } else if( kPutOnFreeQueue == action ) {
1496         queue_enter( &gArbitrationLockQueueFree,
1497                      element,
1498                      ArbitrationLockQueueElement *,
1499                      link );
1500     } else {
1501         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1502     }
1503
1504     // unlock global access
1505     IOUnlock( gArbitrationLockQueueLock );
1506
1507     return( success );
1508 }
1509
1510 void IOService::unlockForArbitration( void )
1511 {
1512     bool                          found;
1513     ArbitrationLockQueueElement * element;
1514
1515     // lock global access
1516     IOTakeLock( gArbitrationLockQueueLock );
1517
1518     // find the lock element for this object (ie. on active queue)
1519     found = false;
1520     queue_iterate( &gArbitrationLockQueueActive,
1521                     element,
1522                     ArbitrationLockQueueElement *,
1523                     link )
1524     {
1525         if( element->service == this ) {
1526             found = true;
1527             break;
1528         }
1529     }
1530
1531     assert( found );
1532
1533     // determine whether the lock has been taken recursively
1534     if( element->count > 1 ) {
1535         // undo one level of recursion
1536         element->count--;
1537
1538     } else {
1539
1540         // remove it from the active queue
1541         queue_remove( &gArbitrationLockQueueActive,
1542                       element,
1543                       ArbitrationLockQueueElement *,
1544                       link );
1545
1546         // put it on the free queue
1547         queue_enter( &gArbitrationLockQueueFree,
1548                      element,
1549                      ArbitrationLockQueueElement *,
1550                      link );
1551
1552         // determine whether a thread is waiting for object (head to tail scan)
1553         found = false;
1554         queue_iterate( &gArbitrationLockQueueWaiting,
1555                        element,
1556                        ArbitrationLockQueueElement *,
1557                        link )
1558         {
1559             if( element->service == this ) {
1560                 found = true;
1561                 break;
1562             }
1563         }
1564
1565         if ( found ) { // we found an interested thread on waiting queue
1566
1567             // remove it from the waiting queue
1568             queue_remove( &gArbitrationLockQueueWaiting,
1569                           element,
1570                           ArbitrationLockQueueElement *,
1571                           link );
1572
1573             // put it on the active queue
1574             queue_enter( &gArbitrationLockQueueActive,
1575                          element,
1576                          ArbitrationLockQueueElement *,
1577                          link );
1578
1579             // wake the waiting thread
1580             IOLockWakeup( gArbitrationLockQueueLock,
1581                           element,
1582                           /* one thread */ true );
1583         }
1584     }
1585
1586     // unlock global access
1587     IOUnlock( gArbitrationLockQueueLock );
1588 }
1589
1590 uint32_t IOService::isLockedForArbitration(IOService * service)
1591 {
1592 #if DEBUG_NOTIFIER_LOCKED
1593     uint32_t                      count;
1594     ArbitrationLockQueueElement * active;
1595
1596     // lock global access
1597     IOLockLock(gArbitrationLockQueueLock);
1598
1599     // determine whether this object is already locked (ie. on active queue)
1600     count = 0;
1601     queue_iterate(&gArbitrationLockQueueActive,
1602                   active,
1603                   ArbitrationLockQueueElement *,
1604                   link)
1605     {
1606         if ((active->thread == IOThreadSelf())
1607             && (!service || (active->service == service)))
1608         {
1609             count += 0x10000;
1610             count += active->count;
1611         }
1612     }
1613
1614     IOLockUnlock(gArbitrationLockQueueLock);
1615
1616     return (count);
1617
1618 #else /* DEBUG_NOTIFIER_LOCKED */
1619
1620     return (0);
1621
1622 #endif /* DEBUG_NOTIFIER_LOCKED */
1623 }
1624
1625 void IOService::applyToProviders( IOServiceApplierFunction applier,
1626                                   void * context )
1627 {
1628     applyToParents( (IORegistryEntryApplierFunction) applier,
1629                     context, gIOServicePlane );
1630 }
1631
1632 void IOService::applyToClients( IOServiceApplierFunction applier,
1633                                 void * context )
1634 {
1635     applyToChildren( (IORegistryEntryApplierFunction) applier,
1636                      context, gIOServicePlane );
1637 }
1638
1639
1640 /*
1641  * Client messages
1642  */
1643
1644
1645 // send a message to a client or interested party of this service
1646 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1647                                    void * argument, vm_size_t argSize )
1648 {
1649     IOReturn                            ret;
1650     IOService *                         service;
1651     _IOServiceInterestNotifier *        notify;
1652
1653     if( (service = OSDynamicCast( IOService, client)))
1654         ret = service->message( type, this, argument );
1655
1656     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1657
1658         _IOServiceNotifierInvocation invocation;
1659         bool                     willNotify;
1660
1661         invocation.thread = current_thread();
1662
1663         LOCKWRITENOTIFY();
1664         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1665
1666         if( willNotify) {
1667             queue_enter( &notify->handlerInvocations, &invocation,
1668                             _IOServiceNotifierInvocation *, link );
1669         }
1670         UNLOCKNOTIFY();
1671
1672         if( willNotify) {
1673
1674             ret = (*notify->handler)( notify->target, notify->ref,
1675                                           type, this, argument, argSize );
1676
1677             LOCKWRITENOTIFY();
1678             queue_remove( &notify->handlerInvocations, &invocation,
1679                             _IOServiceNotifierInvocation *, link );
1680             if( kIOServiceNotifyWaiter & notify->state) {
1681                 notify->state &= ~kIOServiceNotifyWaiter;
1682                 WAKEUPNOTIFY( notify );
1683             }
1684             UNLOCKNOTIFY();
1685
1686         } else
1687             ret = kIOReturnSuccess;
1688
1689     } else
1690         ret = kIOReturnBadArgument;
1691
1692     return( ret );
1693 }
1694
1695 static void
1696 applyToInterestNotifiers(const IORegistryEntry *target,
1697                          const OSSymbol * typeOfInterest,
1698                          OSObjectApplierFunction applier,
1699                          void * context )
1700 {
1701     OSArray *  copyArray = 0;
1702     OSObject * prop;
1703
1704     LOCKREADNOTIFY();
1705
1706     prop = target->copyProperty(typeOfInterest);
1707     IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
1708
1709     if( notifyList) {
1710         copyArray = OSArray::withCapacity(1);
1711
1712         // iterate over queue, entry is set to each element in the list
1713         iterqueue(&notifyList->fCommandChain, entry) {
1714             _IOServiceInterestNotifier * notify;
1715
1716             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1717             copyArray->setObject(notify);
1718         }
1719     }
1720     UNLOCKNOTIFY();
1721
1722     if( copyArray) {
1723         unsigned int    index;
1724         OSObject *      next;
1725
1726         for( index = 0; (next = copyArray->getObject( index )); index++)
1727             (*applier)(next, context);
1728         copyArray->release();
1729     }
1730
1731     OSSafeReleaseNULL(prop);
1732 }
1733
1734 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1735                                    OSObjectApplierFunction applier,
1736                                    void * context )
1737 {
1738     if (gIOGeneralInterest == typeOfInterest)
1739         applyToClients( (IOServiceApplierFunction) applier, context );
1740     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1741 }
1742
1743 struct MessageClientsContext {
1744     IOService * service;
1745     UInt32      type;
1746     void *      argument;
1747     vm_size_t   argSize;
1748     IOReturn    ret;
1749 };
1750
1751 static void messageClientsApplier( OSObject * object, void * ctx )
1752 {
1753     IOReturn                ret;
1754     MessageClientsContext * context = (MessageClientsContext *) ctx;
1755
1756     ret = context->service->messageClient( context->type,
1757                                            object, context->argument, context->argSize );
1758     if( kIOReturnSuccess != ret)
1759         context->ret = ret;
1760 }
1761
1762 // send a message to all clients
1763 IOReturn IOService::messageClients( UInt32 type,
1764                                     void * argument, vm_size_t argSize )
1765 {
1766     MessageClientsContext       context;
1767
1768     context.service     = this;
1769     context.type        = type;
1770     context.argument    = argument;
1771     context.argSize     = argSize;
1772     context.ret         = kIOReturnSuccess;
1773
1774     applyToInterested( gIOGeneralInterest,
1775                        &messageClientsApplier, &context );
1776
1777     return( context.ret );
1778 }
1779
1780 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1781                                               IOOptionBits response )
1782 {
1783     return( kIOReturnUnsupported );
1784 }
1785
1786 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1787                   IOServiceInterestHandler handler, void * target, void * ref )
1788 {
1789     _IOServiceInterestNotifier * notify = 0;
1790     IOReturn rc = kIOReturnError;
1791
1792     notify = new _IOServiceInterestNotifier;
1793     if (!notify) return NULL;
1794
1795     if(notify->init()) {
1796         rc = registerInterestForNotifier(notify, typeOfInterest,
1797                               handler, target, ref);
1798     }
1799
1800     if (rc != kIOReturnSuccess) {
1801         notify->release();
1802         notify = 0;
1803     }
1804
1805     return( notify );
1806 }
1807
1808 IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1809                   IOServiceInterestHandler handler, void * target, void * ref )
1810 {
1811     IOReturn rc = kIOReturnSuccess;
1812     _IOServiceInterestNotifier  *notify = 0;
1813
1814     if( (typeOfInterest != gIOGeneralInterest)
1815      && (typeOfInterest != gIOBusyInterest)
1816      && (typeOfInterest != gIOAppPowerStateInterest)
1817      && (typeOfInterest != gIOConsoleSecurityInterest)
1818      && (typeOfInterest != gIOPriorityPowerStateInterest))
1819         return( kIOReturnBadArgument );
1820
1821     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1822         return( kIOReturnBadArgument );
1823
1824     lockForArbitration();
1825     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1826
1827         notify->handler = handler;
1828         notify->target = target;
1829         notify->ref = ref;
1830         notify->state = kIOServiceNotifyEnable;
1831
1832         ////// queue
1833
1834         LOCKWRITENOTIFY();
1835
1836         // Get the head of the notifier linked list
1837         IOCommand * notifyList;
1838         OSObject  * obj = copyProperty( typeOfInterest );
1839         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1840             notifyList = OSTypeAlloc(IOCommand);
1841             if (notifyList) {
1842                 notifyList->init();
1843                 bool ok = setProperty( typeOfInterest, notifyList);
1844                 notifyList->release();
1845                 if (!ok) notifyList = 0;
1846             }
1847         }
1848         if (obj) obj->release();
1849
1850         if (notifyList) {
1851             enqueue(&notifyList->fCommandChain, &notify->chain);
1852             notify->retain();   // ref'ed while in list
1853         }
1854
1855         UNLOCKNOTIFY();
1856     }
1857     else {
1858         rc = kIOReturnNotReady;
1859     }
1860     unlockForArbitration();
1861
1862     return rc;
1863 }
1864
1865 static void cleanInterestList( OSObject * head )
1866 {
1867     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1868     if (!notifyHead)
1869         return;
1870
1871     LOCKWRITENOTIFY();
1872     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1873         queue_next(entry) = queue_prev(entry) = 0;
1874
1875         _IOServiceInterestNotifier * notify;
1876
1877         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1878         notify->release();
1879     }
1880     UNLOCKNOTIFY();
1881 }
1882
1883 void IOService::unregisterAllInterest( void )
1884 {
1885     OSObject * prop;
1886
1887     prop = copyProperty(gIOGeneralInterest);
1888     cleanInterestList(prop);
1889     OSSafeReleaseNULL(prop);
1890
1891     prop = copyProperty(gIOBusyInterest);
1892     cleanInterestList(prop);
1893     OSSafeReleaseNULL(prop);
1894
1895     prop = copyProperty(gIOAppPowerStateInterest);
1896     cleanInterestList(prop);
1897     OSSafeReleaseNULL(prop);
1898
1899     prop = copyProperty(gIOPriorityPowerStateInterest);
1900     cleanInterestList(prop);
1901     OSSafeReleaseNULL(prop);
1902
1903     prop = copyProperty(gIOConsoleSecurityInterest);
1904     cleanInterestList(prop);
1905     OSSafeReleaseNULL(prop);
1906 }
1907
1908 /*
1909  * _IOServiceInterestNotifier
1910  */
1911
1912 // wait for all threads, other than the current one,
1913 //  to exit the handler
1914
1915 void _IOServiceInterestNotifier::wait()
1916 {
1917     _IOServiceNotifierInvocation * next;
1918     bool doWait;
1919
1920     do {
1921         doWait = false;
1922         queue_iterate( &handlerInvocations, next,
1923                         _IOServiceNotifierInvocation *, link) {
1924             if( next->thread != current_thread() ) {
1925                 doWait = true;
1926                 break;
1927             }
1928         }
1929         if( doWait) {
1930             state |= kIOServiceNotifyWaiter;
1931                SLEEPNOTIFY(this);
1932         }
1933
1934     } while( doWait );
1935 }
1936
1937 void _IOServiceInterestNotifier::free()
1938 {
1939     assert( queue_empty( &handlerInvocations ));
1940     OSObject::free();
1941 }
1942
1943 void _IOServiceInterestNotifier::remove()
1944 {
1945     LOCKWRITENOTIFY();
1946
1947     if( queue_next( &chain )) {
1948         remqueue(&chain);
1949         queue_next( &chain) = queue_prev( &chain) = 0;
1950         release();
1951     }
1952
1953     state &= ~kIOServiceNotifyEnable;
1954
1955     wait();
1956
1957     UNLOCKNOTIFY();
1958
1959     release();
1960 }
1961
1962 bool _IOServiceInterestNotifier::disable()
1963 {
1964     bool        ret;
1965
1966     LOCKWRITENOTIFY();
1967
1968     ret = (0 != (kIOServiceNotifyEnable & state));
1969     state &= ~kIOServiceNotifyEnable;
1970     if( ret)
1971         wait();
1972
1973     UNLOCKNOTIFY();
1974
1975     return( ret );
1976 }
1977
1978 void _IOServiceInterestNotifier::enable( bool was )
1979 {
1980     LOCKWRITENOTIFY();
1981     if( was)
1982         state |= kIOServiceNotifyEnable;
1983     else
1984         state &= ~kIOServiceNotifyEnable;
1985     UNLOCKNOTIFY();
1986 }
1987
1988 bool _IOServiceInterestNotifier::init()
1989 {
1990     queue_init( &handlerInvocations );
1991     return (OSObject::init());
1992 }
1993 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1994
1995 /*
1996  * Termination
1997  */
1998
1999 #define tailQ(o)                setObject(o)
2000 #define headQ(o)                setObject(0, o)
2001 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2002
2003 static void _workLoopAction( IOWorkLoop::Action action,
2004                              IOService * service,
2005                              void * p0 = 0, void * p1 = 0,
2006                              void * p2 = 0, void * p3 = 0 )
2007 {
2008     IOWorkLoop * wl;
2009
2010     if( (wl = service->getWorkLoop())) {
2011         wl->retain();
2012         wl->runAction( action, service, p0, p1, p2, p3 );
2013         wl->release();
2014     } else
2015         (*action)( service, p0, p1, p2, p3 );
2016 }
2017
2018 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
2019 {
2020     bool ok;
2021
2022     // if its our only provider
2023     ok = isParent( provider, gIOServicePlane, true);
2024
2025     // -- compat
2026     if( ok) {
2027         provider->terminateClient( this, options | kIOServiceRecursing );
2028         ok = (0 != (kIOServiceInactiveState & __state[0]));
2029     }
2030     // --
2031
2032     return( ok );
2033 }
2034
2035 bool IOService::terminatePhase1( IOOptionBits options )
2036 {
2037     IOService *  victim;
2038     IOService *  client;
2039     OSIterator * iter;
2040     OSArray *    makeInactive;
2041     OSArray *    waitingInactive;
2042     int          waitResult = THREAD_AWAKENED;
2043     bool         wait;
2044     bool                 ok;
2045     bool                 didInactive;
2046     bool                 startPhase2 = false;
2047
2048     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2049
2050     uint64_t regID = getRegistryEntryID();
2051     IOServiceTrace(
2052         IOSERVICE_TERMINATE_PHASE1,
2053         (uintptr_t) regID,
2054         (uintptr_t) (regID >> 32),
2055         (uintptr_t) this,
2056         (uintptr_t) options);
2057
2058     // -- compat
2059     if( options & kIOServiceRecursing) {
2060         lockForArbitration();
2061         if (0 == (kIOServiceInactiveState & __state[0]))
2062         {
2063             __state[0] |= kIOServiceInactiveState;
2064             __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2065         }
2066         unlockForArbitration();
2067
2068         return( true );
2069     }
2070     // --
2071
2072     makeInactive    = OSArray::withCapacity( 16 );
2073     waitingInactive = OSArray::withCapacity( 16 );
2074     if(!makeInactive || !waitingInactive) return( false );
2075
2076     victim = this;
2077     victim->retain();
2078
2079     while( victim )
2080     {
2081         didInactive = victim->lockForArbitration( true );
2082         if( didInactive)
2083         {
2084             uint64_t regID1 = victim->getRegistryEntryID();
2085             IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2086                 (uintptr_t) regID1,
2087                 (uintptr_t) (regID1 >> 32),
2088                 (uintptr_t) victim->__state[1],
2089                 (uintptr_t) 0);
2090
2091             enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2092             didInactive = (kRP1 == (victim->__state[1] & kRP1))
2093                         || (0 == (victim->__state[0] & kIOServiceInactiveState));
2094
2095             if (!didInactive)
2096             {
2097                 // a multiply attached IOService can be visited twice
2098                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do
2099                 {
2100                     IOLockLock(gIOServiceBusyLock);
2101                     wait = (victim->__state[1] & kIOServiceTermPhase1State);
2102                     if( wait) {
2103                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2104                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2105                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2106                         victim->unlockForArbitration();
2107                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2108                     }
2109                     IOLockUnlock(gIOServiceBusyLock);
2110                     if( wait) {
2111                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2112                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2113                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2114                         victim->lockForArbitration();
2115                     }
2116                 }
2117                 while (wait && (waitResult != THREAD_TIMED_OUT));
2118             }
2119             else
2120             {
2121                 victim->__state[0] |= kIOServiceInactiveState;
2122                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2123                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2124                 victim->__state[1] &= ~kIOServiceRecursing;
2125                 victim->__state[1] |= kIOServiceTermPhase1State;
2126                 waitingInactive->headQ(victim);
2127                 if (victim == this)
2128                 {
2129                     if (kIOServiceTerminateNeedWillTerminate & options)
2130                     {
2131                         victim->__state[1] |= kIOServiceNeedWillTerminate;
2132                     }
2133                 }
2134                 victim->_adjustBusy( 1 );
2135             }
2136             victim->unlockForArbitration();
2137         }
2138         if( victim == this) startPhase2 = didInactive;
2139         if (didInactive)
2140         {
2141             OSArray * notifiers;
2142             notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2143             victim->invokeNotifiers(&notifiers);
2144
2145             IOUserClient::destroyUserReferences( victim );
2146
2147             iter = victim->getClientIterator();
2148             if( iter) {
2149                 while( (client = (IOService *) iter->getNextObject())) {
2150                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2151                             client->getName(), client->getRegistryEntryID(),
2152                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2153                     ok = client->requestTerminate( victim, options );
2154                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2155                             client->getName(), client->getRegistryEntryID(),
2156                             victim->getName(), victim->getRegistryEntryID(), ok);
2157
2158                     uint64_t regID1 = client->getRegistryEntryID();
2159                     uint64_t regID2 = victim->getRegistryEntryID();
2160                     IOServiceTrace(
2161                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2162                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
2163                         (uintptr_t) regID1,
2164                         (uintptr_t) (regID1 >> 32),
2165                         (uintptr_t) regID2,
2166                         (uintptr_t) (regID2 >> 32));
2167
2168                     if( ok)
2169                         makeInactive->setObject( client );
2170                 }
2171                 iter->release();
2172             }
2173         }
2174         victim->release();
2175         victim = (IOService *) makeInactive->getObject(0);
2176         if( victim) {
2177             victim->retain();
2178             makeInactive->removeObject(0);
2179         }
2180     }
2181     makeInactive->release();
2182
2183     while ((victim = (IOService *) waitingInactive->getObject(0)))
2184     {
2185         victim->retain();
2186         waitingInactive->removeObject(0);
2187
2188         victim->lockForArbitration();
2189         victim->__state[1] &= ~kIOServiceTermPhase1State;
2190         if (kIOServiceTerm1WaiterState & victim->__state[1])
2191         {
2192             victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2193             TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2194             IOLockLock( gIOServiceBusyLock );
2195             thread_wakeup( (event_t) &victim->__state[1]);
2196             IOLockUnlock( gIOServiceBusyLock );
2197         }
2198         victim->unlockForArbitration();
2199         victim->release();
2200     }
2201     waitingInactive->release();
2202
2203     if( startPhase2)
2204     {
2205         retain();
2206         lockForArbitration();
2207         scheduleTerminatePhase2(options);
2208         unlockForArbitration();
2209         release();
2210     }
2211
2212     return( true );
2213 }
2214
2215 void IOService::setTerminateDefer(IOService * provider, bool defer)
2216 {
2217     lockForArbitration();
2218     if (defer)  __state[1] |= kIOServiceStartState;
2219     else        __state[1] &= ~kIOServiceStartState;
2220     unlockForArbitration();
2221
2222     if (provider && !defer)
2223     {
2224         provider->lockForArbitration();
2225         provider->scheduleTerminatePhase2();
2226         provider->unlockForArbitration();
2227     }
2228 }
2229
2230 // Must call this while holding gJobsLock
2231 void IOService::waitToBecomeTerminateThread(void)
2232 {
2233     IOLockAssert(gJobsLock, kIOLockAssertOwned);
2234     bool wait;
2235     do {
2236         wait = (gIOTerminateThread != THREAD_NULL);
2237         if (wait) {
2238             IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2239         }
2240     } while (wait);
2241     gIOTerminateThread = current_thread();
2242 }
2243
2244 // call with lockForArbitration
2245 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2246 {
2247     AbsoluteTime        deadline;
2248     uint64_t            regID1;
2249     int                 waitResult = THREAD_AWAKENED;
2250     bool                wait = false, haveDeadline = false;
2251
2252     if (!(__state[0] & kIOServiceInactiveState)) return;
2253
2254     regID1 = getRegistryEntryID();
2255     IOServiceTrace(
2256         IOSERVICE_TERM_SCHED_PHASE2,
2257         (uintptr_t) regID1,
2258         (uintptr_t) (regID1 >> 32),
2259         (uintptr_t) __state[1],
2260         (uintptr_t) options);
2261
2262     if (__state[1] & kIOServiceTermPhase1State)         return;
2263
2264     retain();
2265     unlockForArbitration();
2266     options |= kIOServiceRequired;
2267     IOLockLock( gJobsLock );
2268
2269     if( (options & kIOServiceSynchronous)
2270         && (current_thread() != gIOTerminateThread)) {
2271
2272         waitToBecomeTerminateThread();
2273         gIOTerminatePhase2List->setObject( this );
2274         gIOTerminateWork++;
2275
2276         do {
2277             while( gIOTerminateWork )
2278                 terminateWorker( options );
2279             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2280             if( wait) {
2281                 /* wait for the victim to go non-busy */
2282                 if( !haveDeadline) {
2283                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2284                     haveDeadline = true;
2285                 }
2286                 /* let others do work while we wait */
2287                 gIOTerminateThread = 0;
2288                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2289                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2290                                                   deadline, THREAD_UNINT );
2291                 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2292                     panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2293                 }
2294                 waitToBecomeTerminateThread();
2295             }
2296         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2297
2298         gIOTerminateThread = 0;
2299         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2300
2301     } else {
2302         // ! kIOServiceSynchronous
2303
2304         gIOTerminatePhase2List->setObject( this );
2305         if( 0 == gIOTerminateWork++) {
2306             assert(gIOTerminateWorkerThread);
2307             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2308         }
2309     }
2310
2311     IOLockUnlock( gJobsLock );
2312     lockForArbitration();
2313     release();
2314 }
2315
2316 __attribute__((__noreturn__))
2317 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2318 {
2319     // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2320     IOLockLock(gJobsLock);
2321     while (true) {
2322         if (gIOTerminateThread != gIOTerminateWorkerThread) {
2323             waitToBecomeTerminateThread();
2324         }
2325
2326         while (gIOTerminateWork)
2327             terminateWorker( (uintptr_t)arg );
2328
2329         gIOTerminateThread = 0;
2330         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2331         IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2332     }
2333 }
2334
2335 void IOService::scheduleStop( IOService * provider )
2336 {
2337     uint64_t regID1 = getRegistryEntryID();
2338     uint64_t regID2 = provider->getRegistryEntryID();
2339
2340     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2341     IOServiceTrace(
2342         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2343         (uintptr_t) regID1,
2344         (uintptr_t) (regID1 >> 32),
2345         (uintptr_t) regID2,
2346         (uintptr_t) (regID2 >> 32));
2347
2348     IOLockLock( gJobsLock );
2349     gIOStopList->tailQ( this );
2350     gIOStopProviderList->tailQ( provider );
2351
2352     if( 0 == gIOTerminateWork++) {
2353         assert(gIOTerminateWorkerThread);
2354         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2355     }
2356
2357     IOLockUnlock( gJobsLock );
2358 }
2359
2360 void IOService::scheduleFinalize(bool now)
2361 {
2362     uint64_t regID1 = getRegistryEntryID();
2363
2364     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2365     IOServiceTrace(
2366         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2367         (uintptr_t) regID1,
2368         (uintptr_t) (regID1 >> 32),
2369         0, 0);
2370
2371     if (now || IOUserClient::finalizeUserReferences(this))
2372     {
2373         IOLockLock( gJobsLock );
2374         gIOFinalizeList->tailQ(this);
2375         if( 0 == gIOTerminateWork++) {
2376             assert(gIOTerminateWorkerThread);
2377             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2378         }
2379         IOLockUnlock( gJobsLock );
2380     }
2381 }
2382
2383 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2384 {
2385     return( true );
2386 }
2387
2388 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2389 {
2390     if( false == *defer) {
2391
2392         if( lockForArbitration( true )) {
2393             if( false == provider->handleIsOpen( this ))
2394                 scheduleStop( provider );
2395             // -- compat
2396             else {
2397                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2398                 if( false == provider->handleIsOpen( this ))
2399                     scheduleStop( provider );
2400             }
2401             // --
2402             unlockForArbitration();
2403         }
2404     }
2405
2406     return( true );
2407 }
2408
2409 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2410                                      OSArray * doPhase2List,
2411                                      void *unused2 __unused,
2412                                      void *unused3 __unused  )
2413 {
2414     OSIterator * iter;
2415     IOService *  client;
2416     bool         ok;
2417     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2418
2419     iter = victim->getClientIterator();
2420     if( iter) {
2421         while( (client = (IOService *) iter->getNextObject())) {
2422
2423             regID1 = client->getRegistryEntryID();
2424             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2425                     client->getName(), regID1,
2426                     victim->getName(), regID2, (long long)options);
2427             IOServiceTrace(
2428                 IOSERVICE_TERMINATE_WILL,
2429                 (uintptr_t) regID1,
2430                 (uintptr_t) (regID1 >> 32),
2431                 (uintptr_t) regID2,
2432                 (uintptr_t) (regID2 >> 32));
2433
2434             ok = client->willTerminate( victim, options );
2435             doPhase2List->tailQ( client );
2436         }
2437         iter->release();
2438     }
2439 }
2440
2441 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2442                             void *unused1 __unused, void *unused2 __unused,
2443                             void *unused3 __unused )
2444 {
2445     OSIterator * iter;
2446     IOService *  client;
2447     bool         defer;
2448     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2449
2450     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2451
2452     iter = victim->getClientIterator();
2453     if( iter) {
2454         while( (client = (IOService *) iter->getNextObject())) {
2455
2456             regID1 = client->getRegistryEntryID();
2457             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2458                     client->getName(), regID1,
2459                     victim->getName(), regID2, (long long)options);
2460             defer = false;
2461             client->didTerminate( victim, options, &defer );
2462
2463             IOServiceTrace(
2464                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2465                        : IOSERVICE_TERMINATE_DID),
2466                 (uintptr_t) regID1,
2467                 (uintptr_t) (regID1 >> 32),
2468                 (uintptr_t) regID2,
2469                 (uintptr_t) (regID2 >> 32));
2470
2471             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2472                     client->getName(), regID1,
2473                     victim->getName(), regID2, defer);
2474         }
2475         iter->release();
2476     }
2477 }
2478
2479
2480 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2481                             void *unused1 __unused, void *unused2 __unused,
2482                             void *unused3 __unused )
2483 {
2484     OSIterator * iter;
2485     IOService *  provider;
2486     bool         ok;
2487     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2488
2489     iter = victim->getProviderIterator();
2490     if( iter) {
2491         while( (provider = (IOService *) iter->getNextObject())) {
2492
2493             regID1 = provider->getRegistryEntryID();
2494             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2495                     victim->getName(), regID2,
2496                     provider->getName(), regID1, (long long)options);
2497             IOServiceTrace(
2498                 IOSERVICE_TERMINATE_WILL,
2499                 (uintptr_t) regID2,
2500                 (uintptr_t) (regID2 >> 32),
2501                 (uintptr_t) regID1,
2502                 (uintptr_t) (regID1 >> 32));
2503
2504             ok = victim->willTerminate( provider, options );
2505         }
2506         iter->release();
2507     }
2508 }
2509
2510 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2511                             void *unused1 __unused, void *unused2 __unused,
2512                             void *unused3 __unused )
2513 {
2514     OSIterator * iter;
2515     IOService *  provider;
2516     bool defer = false;
2517     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2518
2519     iter = victim->getProviderIterator();
2520     if( iter) {
2521         while( (provider = (IOService *) iter->getNextObject())) {
2522
2523             regID1 = provider->getRegistryEntryID();
2524             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2525                     victim->getName(), regID2,
2526                     provider->getName(), regID1, (long long)options);
2527             victim->didTerminate( provider, options, &defer );
2528
2529             IOServiceTrace(
2530                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2531                        : IOSERVICE_TERMINATE_DID),
2532                 (uintptr_t) regID2,
2533                 (uintptr_t) (regID2 >> 32),
2534                 (uintptr_t) regID1,
2535                 (uintptr_t) (regID1 >> 32));
2536
2537             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2538                     victim->getName(), regID2,
2539                     provider->getName(), regID1, defer);
2540         }
2541         iter->release();
2542     }
2543 }
2544
2545
2546 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2547                             void *unused1 __unused, void *unused2 __unused,
2548                             void *unused3 __unused )
2549 {
2550     uint64_t regID1 = victim->getRegistryEntryID();
2551     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2552     IOServiceTrace(
2553         IOSERVICE_TERMINATE_FINALIZE,
2554         (uintptr_t) regID1,
2555         (uintptr_t) (regID1 >> 32),
2556         0, 0);
2557
2558     victim->finalize( options );
2559 }
2560
2561 void IOService::actionStop( IOService * provider, IOService * client,
2562                             void *unused1 __unused, void *unused2 __unused,
2563                             void *unused3 __unused )
2564 {
2565     uint64_t regID1 = provider->getRegistryEntryID();
2566     uint64_t regID2 = client->getRegistryEntryID();
2567
2568     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2569     IOServiceTrace(
2570         IOSERVICE_TERMINATE_STOP,
2571         (uintptr_t) regID1,
2572         (uintptr_t) (regID1 >> 32),
2573         (uintptr_t) regID2,
2574         (uintptr_t) (regID2 >> 32));
2575
2576     client->stop( provider );
2577     if( provider->isOpen( client ))
2578         provider->close( client );
2579
2580     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2581     client->detach( provider );
2582 }
2583
2584 void IOService::terminateWorker( IOOptionBits options )
2585 {
2586     OSArray *           doPhase2List;
2587     OSArray *           didPhase2List;
2588     OSSet *             freeList;
2589     OSIterator *        iter;
2590     UInt32              workDone;
2591     IOService *         victim;
2592     IOService *         client;
2593     IOService *         provider;
2594     unsigned int        idx;
2595     bool                moreToDo;
2596     bool                doPhase2;
2597     bool                doPhase3;
2598
2599     options |= kIOServiceRequired;
2600
2601     doPhase2List  = OSArray::withCapacity( 16 );
2602     didPhase2List = OSArray::withCapacity( 16 );
2603     freeList      = OSSet::withCapacity( 16 );
2604     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2605         return;
2606
2607     do {
2608         workDone = gIOTerminateWork;
2609
2610         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2611
2612             victim->retain();
2613             gIOTerminatePhase2List->removeObject(0);
2614             IOLockUnlock( gJobsLock );
2615
2616             uint64_t regID1 = victim->getRegistryEntryID();
2617             IOServiceTrace(
2618                 IOSERVICE_TERM_START_PHASE2,
2619                 (uintptr_t) regID1,
2620                 (uintptr_t) (regID1 >> 32),
2621                 (uintptr_t) 0,
2622                 (uintptr_t) 0);
2623
2624             while( victim ) {
2625
2626                 doPhase2 = victim->lockForArbitration( true );
2627                 if( doPhase2) {
2628                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2629                     if( doPhase2) {
2630
2631                         uint64_t regID1 = victim->getRegistryEntryID();
2632                         IOServiceTrace(
2633                             IOSERVICE_TERM_TRY_PHASE2,
2634                             (uintptr_t) regID1,
2635                             (uintptr_t) (regID1 >> 32),
2636                             (uintptr_t) victim->__state[1],
2637                             (uintptr_t) 0);
2638
2639                         doPhase2 = (0 == (victim->__state[1] &
2640                                     (kIOServiceTermPhase1State
2641                                     | kIOServiceTermPhase2State
2642                                     | kIOServiceConfigState)));
2643
2644                         if (doPhase2 && (iter = victim->getClientIterator())) {
2645                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2646                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2647                                 if (!doPhase2)
2648                                 {
2649                                     uint64_t regID1 = client->getRegistryEntryID();
2650                                     IOServiceTrace(
2651                                         IOSERVICE_TERM_UC_DEFER,
2652                                         (uintptr_t) regID1,
2653                                         (uintptr_t) (regID1 >> 32),
2654                                         (uintptr_t) client->__state[1],
2655                                         (uintptr_t) 0);
2656                                     TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2657                                            victim->getName(), victim->getRegistryEntryID(),
2658                                            client->getName(), client->getRegistryEntryID());
2659                                 }
2660                             }
2661                             iter->release();
2662                         }
2663                         if( doPhase2)
2664                             victim->__state[1] |= kIOServiceTermPhase2State;
2665                     }
2666                     victim->unlockForArbitration();
2667                 }
2668                 if( doPhase2) {
2669
2670                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2671                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2672                                             victim, (void *)(uintptr_t) options, NULL );
2673                     }
2674
2675                     OSArray * notifiers;
2676                     notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
2677                     victim->invokeNotifiers(&notifiers);
2678
2679                     if( 0 == victim->getClient()) {
2680
2681                         // no clients - will go to finalize
2682                         victim->scheduleFinalize(false);
2683
2684                     } else {
2685                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2686                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2687                     }
2688                     didPhase2List->headQ( victim );
2689                 }
2690                 victim->release();
2691                 victim = (IOService *) doPhase2List->getObject(0);
2692                 if( victim) {
2693                     victim->retain();
2694                     doPhase2List->removeObject(0);
2695                 }
2696             }
2697
2698             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2699
2700                 if( victim->lockForArbitration( true )) {
2701                     victim->__state[1] |= kIOServiceTermPhase3State;
2702                     victim->unlockForArbitration();
2703                 }
2704                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2705                                     victim, (void *)(uintptr_t) options );
2706                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2707                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2708                                         victim, (void *)(uintptr_t) options, NULL );
2709                 }
2710                 didPhase2List->removeObject(0);
2711             }
2712             IOLockLock( gJobsLock );
2713         }
2714
2715         // phase 3
2716         do {
2717             doPhase3 = false;
2718             // finalize leaves
2719             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2720
2721                 IOLockUnlock( gJobsLock );
2722                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2723                                     victim, (void *)(uintptr_t) options );
2724                 IOLockLock( gJobsLock );
2725                 // hold off free
2726                 freeList->setObject( victim );
2727                 // safe if finalize list is append only
2728                 gIOFinalizeList->removeObject(0);
2729             }
2730
2731             for( idx = 0;
2732                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2733
2734                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2735                 assert( provider );
2736
2737                 uint64_t regID1 = provider->getRegistryEntryID();
2738                 uint64_t regID2 = client->getRegistryEntryID();
2739
2740                 if( !provider->isChild( client, gIOServicePlane )) {
2741                     // may be multiply queued - nop it
2742                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2743                     IOServiceTrace(
2744                         IOSERVICE_TERMINATE_STOP_NOP,
2745                         (uintptr_t) regID1,
2746                         (uintptr_t) (regID1 >> 32),
2747                         (uintptr_t) regID2,
2748                         (uintptr_t) (regID2 >> 32));
2749
2750                 } else {
2751                     // a terminated client is not ready for stop if it has clients, skip it
2752                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2753                         TLOG("%s[0x%qx]::defer stop(%s[0x%qx])\n",
2754                                 client->getName(), regID2,
2755                                 client->getClient()->getName(), client->getClient()->getRegistryEntryID());
2756                         IOServiceTrace(
2757                             IOSERVICE_TERMINATE_STOP_DEFER,
2758                             (uintptr_t) regID1,
2759                             (uintptr_t) (regID1 >> 32),
2760                             (uintptr_t) regID2,
2761                             (uintptr_t) (regID2 >> 32));
2762
2763                         idx++;
2764                         continue;
2765                     }
2766
2767                     IOLockUnlock( gJobsLock );
2768                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2769                                      provider, (void *) client );
2770                     IOLockLock( gJobsLock );
2771                     // check the finalize list now
2772                     doPhase3 = true;
2773                 }
2774                 // hold off free
2775                 freeList->setObject( client );
2776                 freeList->setObject( provider );
2777
2778                 // safe if stop list is append only
2779                 gIOStopList->removeObject( idx );
2780                 gIOStopProviderList->removeObject( idx );
2781                 idx = 0;
2782             }
2783
2784         } while( doPhase3 );
2785
2786         gIOTerminateWork -= workDone;
2787         moreToDo = (gIOTerminateWork != 0);
2788
2789         if( !moreToDo) {
2790             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2791             IOServiceTrace(
2792                 IOSERVICE_TERMINATE_DONE,
2793                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2794         }
2795
2796     } while( moreToDo );
2797
2798     IOLockUnlock( gJobsLock );
2799
2800     freeList->release();
2801     doPhase2List->release();
2802     didPhase2List->release();
2803
2804     IOLockLock( gJobsLock );
2805 }
2806
2807 bool IOService::finalize( IOOptionBits options )
2808 {
2809     OSIterator *  iter;
2810     IOService *   provider;
2811     uint64_t      regID1, regID2 = getRegistryEntryID();
2812
2813     iter = getProviderIterator();
2814     assert( iter );
2815
2816     if( iter) {
2817         while( (provider = (IOService *) iter->getNextObject())) {
2818
2819             // -- compat
2820             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2821                 /* we come down here on programmatic terminate */
2822
2823                 regID1 = provider->getRegistryEntryID();
2824                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2825                 IOServiceTrace(
2826                     IOSERVICE_TERMINATE_STOP,
2827                     (uintptr_t) regID1,
2828                     (uintptr_t) (regID1 >> 32),
2829                     (uintptr_t) regID2,
2830                     (uintptr_t) (regID2 >> 32));
2831
2832                 stop( provider );
2833                 if( provider->isOpen( this ))
2834                     provider->close( this );
2835                 detach( provider );
2836             } else {
2837             //--
2838                 if( provider->lockForArbitration( true )) {
2839                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2840                         scheduleStop( provider );
2841                     provider->unlockForArbitration();
2842                 }
2843             }
2844         }
2845         iter->release();
2846     }
2847
2848     return( true );
2849 }
2850
2851 #undef tailQ
2852 #undef headQ
2853
2854 /*
2855  * Terminate
2856  */
2857
2858 void IOService::doServiceTerminate( IOOptionBits options )
2859 {
2860 }
2861
2862 // a method in case someone needs to override it
2863 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2864 {
2865     bool ok;
2866
2867     if( client->isParent( this, gIOServicePlane, true))
2868         // we are the clients only provider
2869         ok = client->terminate( options );
2870     else
2871         ok = true;
2872
2873     return( ok );
2874 }
2875
2876 bool IOService::terminate( IOOptionBits options )
2877 {
2878     options |= kIOServiceTerminate;
2879
2880     return( terminatePhase1( options ));
2881 }
2882
2883 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2884
2885 /*
2886  * Open & close
2887  */
2888
2889 struct ServiceOpenMessageContext
2890 {
2891     IOService *  service;
2892     UInt32       type;
2893     IOService *  excludeClient;
2894     IOOptionBits options;
2895 };
2896
2897 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2898 {
2899     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2900
2901     if( object != context->excludeClient)
2902         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2903 }
2904
2905 bool IOService::open(   IOService *     forClient,
2906                         IOOptionBits    options,
2907                         void *          arg )
2908 {
2909     bool                        ok;
2910     ServiceOpenMessageContext   context;
2911
2912     context.service             = this;
2913     context.type                = kIOMessageServiceIsAttemptingOpen;
2914     context.excludeClient       = forClient;
2915     context.options             = options;
2916
2917     applyToInterested( gIOGeneralInterest,
2918                         &serviceOpenMessageApplier, &context );
2919
2920     if( false == lockForArbitration(false) )
2921         return false;
2922
2923     ok = (0 == (__state[0] & kIOServiceInactiveState));
2924     if( ok)
2925         ok = handleOpen( forClient, options, arg );
2926
2927     unlockForArbitration();
2928
2929     return( ok );
2930 }
2931
2932 void IOService::close(  IOService *     forClient,
2933                         IOOptionBits    options )
2934 {
2935     bool                wasClosed;
2936     bool                last = false;
2937
2938     lockForArbitration();
2939
2940     wasClosed = handleIsOpen( forClient );
2941     if( wasClosed) {
2942         handleClose( forClient, options );
2943         last = (__state[1] & kIOServiceTermPhase3State);
2944     }
2945
2946     unlockForArbitration();
2947
2948     if( last)
2949         forClient->scheduleStop( this );
2950
2951     else if( wasClosed) {
2952
2953         ServiceOpenMessageContext context;
2954
2955         context.service         = this;
2956         context.type            = kIOMessageServiceWasClosed;
2957         context.excludeClient   = forClient;
2958         context.options         = options;
2959
2960         applyToInterested( gIOGeneralInterest,
2961                             &serviceOpenMessageApplier, &context );
2962     }
2963 }
2964
2965 bool IOService::isOpen( const IOService * forClient ) const
2966 {
2967     IOService * self = (IOService *) this;
2968     bool ok;
2969
2970     self->lockForArbitration();
2971
2972     ok = handleIsOpen( forClient );
2973
2974     self->unlockForArbitration();
2975
2976     return( ok );
2977 }
2978
2979 bool IOService::handleOpen(     IOService *     forClient,
2980                                 IOOptionBits    options,
2981                                 void *          arg )
2982 {
2983     bool        ok;
2984
2985     ok = (0 == __owner);
2986     if( ok )
2987         __owner = forClient;
2988
2989     else if( options & kIOServiceSeize ) {
2990         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2991                                 __owner, (void *)(uintptr_t) options ));
2992         if( ok && (0 == __owner ))
2993             __owner = forClient;
2994         else
2995             ok = false;
2996     }
2997     return( ok );
2998 }
2999
3000 void IOService::handleClose(    IOService *     forClient,
3001                                 IOOptionBits    options )
3002 {
3003     if( __owner == forClient)
3004         __owner = 0;
3005 }
3006
3007 bool IOService::handleIsOpen(   const IOService * forClient ) const
3008 {
3009     if( forClient)
3010         return( __owner == forClient );
3011     else
3012         return( __owner != forClient );
3013 }
3014
3015 /*
3016  * Probing & starting
3017  */
3018 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3019 {
3020     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3021     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3022     SInt32             val1;
3023     SInt32             val2;
3024
3025     val1 = 0;
3026     val2 = 0;
3027
3028     if ( obj1 )
3029         val1 = obj1->priority;
3030
3031     if ( obj2 )
3032         val2 = obj2->priority;
3033
3034     return ( val1 - val2 );
3035 }
3036
3037 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
3038 {
3039     OSDictionary *      dict;
3040     IOService *         service;
3041     _IOServiceNotifier * notify;
3042     OSSymbol *          key = (OSSymbol *) ref;
3043     OSNumber *          offset;
3044     OSObject *          prop;
3045     SInt32              result;
3046
3047     prop = 0;
3048     result = kIODefaultProbeScore;
3049     if( (dict = OSDynamicCast( OSDictionary, entry)))
3050         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3051     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
3052         return( notify->priority );
3053     else if( (service = OSDynamicCast( IOService, entry)))
3054     {
3055         prop = service->copyProperty(key);
3056         offset = OSDynamicCast(OSNumber, prop);
3057     }
3058     else {
3059         assert( false );
3060         offset = 0;
3061     }
3062
3063     if (offset) result = offset->unsigned32BitValue();
3064
3065     OSSafeReleaseNULL(prop);
3066
3067     return (result);
3068 }
3069
3070 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3071 {
3072     const OSObject *    obj1 = (const OSObject *) inObj1;
3073     const OSObject *    obj2 = (const OSObject *) inObj2;
3074     SInt32               val1;
3075     SInt32               val2;
3076
3077     val1 = 0;
3078     val2 = 0;
3079
3080     if ( obj1 )
3081         val1 = IOServiceObjectOrder( obj1, ref );
3082
3083     if ( obj2 )
3084         val2 = IOServiceObjectOrder( obj2, ref );
3085
3086     return ( val1 - val2 );
3087 }
3088
3089 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
3090 {
3091     IOService *         service = 0;
3092     OSIterator *        iter;
3093     const OSSymbol *    nextCat;
3094
3095     iter = getClientIterator();
3096     if( iter) {
3097         while( (service = (IOService *) iter->getNextObject())) {
3098             if( kIOServiceInactiveState & service->__state[0])
3099                 continue;
3100             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3101                         service->getProperty( gIOMatchCategoryKey ));
3102             if( category == nextCat)
3103             {
3104                 service->retain();
3105                 break;
3106             }
3107         }
3108         iter->release();
3109     }
3110     return( service );
3111 }
3112
3113 IOService * IOService::getClientWithCategory( const OSSymbol * category )
3114 {
3115     IOService *
3116     service = copyClientWithCategory(category);
3117     if (service)
3118         service->release();
3119     return (service);
3120 }
3121
3122 bool IOService::invokeNotifier( _IOServiceNotifier * notify )
3123 {
3124     _IOServiceNotifierInvocation invocation;
3125     bool                         willNotify;
3126     bool                         ret = true;
3127     invocation.thread = current_thread();
3128
3129 #if DEBUG_NOTIFIER_LOCKED
3130     uint32_t count;
3131     if ((count = isLockedForArbitration(0)))
3132     {
3133         IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3134         panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3135     }
3136 #endif /* DEBUG_NOTIFIER_LOCKED */
3137
3138     LOCKWRITENOTIFY();
3139     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3140
3141     if( willNotify) {
3142         queue_enter( &notify->handlerInvocations, &invocation,
3143                         _IOServiceNotifierInvocation *, link );
3144     }
3145     UNLOCKNOTIFY();
3146
3147     if( willNotify) {
3148
3149         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3150
3151         LOCKWRITENOTIFY();
3152         queue_remove( &notify->handlerInvocations, &invocation,
3153                         _IOServiceNotifierInvocation *, link );
3154         if( kIOServiceNotifyWaiter & notify->state) {
3155             notify->state &= ~kIOServiceNotifyWaiter;
3156             WAKEUPNOTIFY( notify );
3157         }
3158         UNLOCKNOTIFY();
3159     }
3160
3161     return( ret );
3162 }
3163
3164 bool IOService::invokeNotifiers(OSArray ** willSend)
3165 {
3166     OSArray *            array;
3167     _IOServiceNotifier * notify;
3168     bool                 ret = true;
3169
3170     array = *willSend;
3171     if (!array) return (true);
3172     *willSend = 0;
3173
3174     for( unsigned int idx = 0;
3175          (notify = (_IOServiceNotifier *) array->getObject(idx));
3176          idx++) {
3177         ret &= invokeNotifier(notify);
3178     }
3179     array->release();
3180
3181     return (ret);
3182 }
3183
3184
3185 /*
3186  * Alloc and probe matching classes,
3187  * called on the provider instance
3188  */
3189
3190 void IOService::probeCandidates( OSOrderedSet * matches )
3191 {
3192     OSDictionary        *       match = 0;
3193     OSSymbol            *       symbol;
3194     IOService           *       inst;
3195     IOService           *       newInst;
3196     OSDictionary        *       props;
3197     SInt32                      score;
3198     OSNumber            *       newPri;
3199     OSOrderedSet        *       familyMatches = 0;
3200     OSOrderedSet        *       startList;
3201     OSDictionary        *       startDict = 0;
3202     const OSSymbol      *       category;
3203     OSIterator          *       iter;
3204     _IOServiceNotifier  *               notify;
3205     OSObject            *       nextMatch = 0;
3206     bool                        started;
3207     bool                        needReloc = false;
3208 #if IOMATCHDEBUG
3209     SInt64                      debugFlags;
3210 #endif
3211     IOService * client = NULL;
3212
3213
3214     assert( matches );
3215     while( !needReloc && (nextMatch = matches->getFirstObject())) {
3216
3217         nextMatch->retain();
3218         matches->removeObject(nextMatch);
3219
3220         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3221
3222             if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify );
3223             nextMatch->release();
3224             nextMatch = 0;
3225             continue;
3226
3227         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3228             nextMatch->release();
3229             nextMatch = 0;
3230             continue;
3231         }
3232
3233         props = 0;
3234 #if IOMATCHDEBUG
3235         debugFlags = getDebugFlags( match );
3236 #endif
3237
3238         do {
3239             category = OSDynamicCast( OSSymbol,
3240                         match->getObject( gIOMatchCategoryKey ));
3241             if( 0 == category)
3242                 category = gIODefaultMatchCategoryKey;
3243
3244             if( (client = copyClientWithCategory(category)) ) {
3245 #if IOMATCHDEBUG
3246                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
3247                     LOG("%s: match category %s exists\n", getName(),
3248                                 category->getCStringNoCopy());
3249 #endif
3250                 nextMatch->release();
3251                 nextMatch = 0;
3252
3253                 client->release();
3254                 client = NULL;
3255
3256                 continue;
3257             }
3258
3259             // create a copy now in case its modified during matching
3260             props = OSDictionary::withDictionary( match, match->getCount());
3261             if( 0 == props)
3262                 continue;
3263             props->setCapacityIncrement(1);
3264
3265             // check the nub matches
3266             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
3267                 continue;
3268
3269             // Check to see if driver reloc has been loaded.
3270             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
3271             if( needReloc) {
3272 #if IOMATCHDEBUG
3273                 if( debugFlags & kIOLogCatalogue)
3274                     LOG("%s: stalling for module\n", getName());
3275 #endif
3276                 // If reloc hasn't been loaded, exit;
3277                 // reprobing will occur after reloc has been loaded.
3278                 continue;
3279             }
3280
3281             // reorder on family matchPropertyTable score.
3282             if( 0 == familyMatches)
3283                 familyMatches = OSOrderedSet::withCapacity( 1,
3284                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3285             if( familyMatches)
3286                 familyMatches->setObject( props );
3287
3288         } while( false );
3289
3290         if (nextMatch) {
3291             nextMatch->release();
3292             nextMatch = 0;
3293         }
3294         if( props)
3295             props->release();
3296     }
3297     matches->release();
3298     matches = 0;
3299
3300     if( familyMatches) {
3301
3302         while( !needReloc
3303              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3304
3305             props->retain();
3306             familyMatches->removeObject( props );
3307
3308             inst = 0;
3309             newInst = 0;
3310 #if IOMATCHDEBUG
3311             debugFlags = getDebugFlags( props );
3312 #endif
3313             do {
3314                 symbol = OSDynamicCast( OSSymbol,
3315                                 props->getObject( gIOClassKey));
3316                 if( !symbol)
3317                     continue;
3318
3319                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
3320
3321                 // alloc the driver instance
3322                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3323
3324                 if( !inst || !OSDynamicCast(IOService, inst)) {
3325                     IOLog("Couldn't alloc class \"%s\"\n",
3326                         symbol->getCStringNoCopy());
3327                     continue;
3328                 }
3329
3330                 // init driver instance
3331                 if( !(inst->init( props ))) {
3332 #if IOMATCHDEBUG
3333                     if( debugFlags & kIOLogStart)
3334                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3335 #endif
3336                     continue;
3337                 }
3338                 if( __state[1] & kIOServiceSynchronousState)
3339                     inst->__state[1] |= kIOServiceSynchronousState;
3340
3341                 // give the driver the default match category if not specified
3342                 category = OSDynamicCast( OSSymbol,
3343                             props->getObject( gIOMatchCategoryKey ));
3344                 if( 0 == category)
3345                     category = gIODefaultMatchCategoryKey;
3346                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3347                 // attach driver instance
3348                 if( !(inst->attach( this )))
3349                         continue;
3350
3351                 // pass in score from property table
3352                 score = familyMatches->orderObject( props );
3353
3354                 // & probe the new driver instance
3355 #if IOMATCHDEBUG
3356                 if( debugFlags & kIOLogProbe)
3357                     LOG("%s::probe(%s)\n",
3358                         inst->getMetaClass()->getClassName(), getName());
3359 #endif
3360
3361                 newInst = inst->probe( this, &score );
3362                 inst->detach( this );
3363                 if( 0 == newInst) {
3364 #if IOMATCHDEBUG
3365                     if( debugFlags & kIOLogProbe)
3366                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3367 #endif
3368                     continue;
3369                 }
3370
3371                 // save the score
3372                 newPri = OSNumber::withNumber( score, 32 );
3373                 if( newPri) {
3374                     newInst->setProperty( gIOProbeScoreKey, newPri );
3375                     newPri->release();
3376                 }
3377
3378                 // add to start list for the match category
3379                 if( 0 == startDict)
3380                     startDict = OSDictionary::withCapacity( 1 );
3381                 assert( startDict );
3382                 startList = (OSOrderedSet *)
3383                                 startDict->getObject( category );
3384                 if( 0 == startList) {
3385                     startList = OSOrderedSet::withCapacity( 1,
3386                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3387                     if( startDict && startList) {
3388                         startDict->setObject( category, startList );
3389                         startList->release();
3390                     }
3391                 }
3392                 assert( startList );
3393                 if( startList)
3394                     startList->setObject( newInst );
3395
3396             } while( false );
3397
3398             props->release();
3399             if( inst)
3400                 inst->release();
3401         }
3402         familyMatches->release();
3403         familyMatches = 0;
3404     }
3405
3406     // start the best (until success) of each category
3407
3408     iter = OSCollectionIterator::withCollection( startDict );
3409     if( iter) {
3410         while( (category = (const OSSymbol *) iter->getNextObject())) {
3411
3412             startList = (OSOrderedSet *) startDict->getObject( category );
3413             assert( startList );
3414             if( !startList)
3415                 continue;
3416
3417             started = false;
3418             while( true // (!started)
3419                    && (inst = (IOService *)startList->getFirstObject())) {
3420
3421                 inst->retain();
3422                 startList->removeObject(inst);
3423
3424 #if IOMATCHDEBUG
3425                 debugFlags = getDebugFlags( inst );
3426
3427                 if( debugFlags & kIOLogStart) {
3428                     if( started)
3429                         LOG( "match category exists, skipping " );
3430                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3431                          getName(), inst->getRetainCount());
3432                 }
3433 #endif
3434                 if( false == started)
3435                     started = startCandidate( inst );
3436 #if IOMATCHDEBUG
3437                 if( (debugFlags & kIOLogStart) && (false == started))
3438                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3439                          inst->getRetainCount());
3440 #endif
3441                 inst->release();
3442             }
3443         }
3444         iter->release();
3445     }
3446
3447
3448     // adjust the busy count by +1 if matching is stalled for a module,
3449     // or -1 if a previously stalled matching is complete.
3450     lockForArbitration();
3451     SInt32 adjBusy = 0;
3452     uint64_t regID = getRegistryEntryID();
3453
3454     if( needReloc) {
3455         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3456         if( adjBusy) {
3457
3458             IOServiceTrace(
3459                 IOSERVICE_MODULESTALL,
3460                 (uintptr_t) regID,
3461                 (uintptr_t) (regID >> 32),
3462                 (uintptr_t) this,
3463                 0);
3464
3465             __state[1] |= kIOServiceModuleStallState;
3466         }
3467
3468     } else if( __state[1] & kIOServiceModuleStallState) {
3469
3470             IOServiceTrace(
3471             IOSERVICE_MODULEUNSTALL,
3472             (uintptr_t) regID,
3473             (uintptr_t) (regID >> 32),
3474             (uintptr_t) this,
3475             0);
3476
3477         __state[1] &= ~kIOServiceModuleStallState;
3478         adjBusy = -1;
3479     }
3480     if( adjBusy)
3481         _adjustBusy( adjBusy );
3482     unlockForArbitration();
3483
3484     if( startDict)
3485         startDict->release();
3486 }
3487
3488 /*
3489  * Start a previously attached & probed instance,
3490  * called on exporting object instance
3491  */
3492
3493 bool IOService::startCandidate( IOService * service )
3494 {
3495     bool                ok;
3496
3497     ok = service->attach( this );
3498
3499     if( ok)
3500     {
3501         if (this != gIOResources)
3502         {
3503             // stall for any nub resources
3504             checkResources();
3505             // stall for any driver resources
3506             service->checkResources();
3507         }
3508
3509         AbsoluteTime startTime;
3510         AbsoluteTime endTime;
3511         UInt64       nano;
3512
3513         if (kIOLogStart & gIOKitDebug)
3514             clock_get_uptime(&startTime);
3515
3516         ok = service->start(this);
3517
3518         if (kIOLogStart & gIOKitDebug)
3519         {
3520             clock_get_uptime(&endTime);
3521
3522             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3523             {
3524                 SUB_ABSOLUTETIME(&endTime, &startTime);
3525                 absolutetime_to_nanoseconds(endTime, &nano);
3526                 if (nano > 500000000ULL)
3527                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3528             }
3529         }
3530         if( !ok)
3531             service->detach( this );
3532     }
3533     return( ok );
3534 }
3535
3536 void IOService::publishResource( const char * key, OSObject * value )
3537 {
3538     const OSSymbol *    sym;
3539
3540     if( (sym = OSSymbol::withCString( key))) {
3541         publishResource( sym, value);
3542         sym->release();
3543     }
3544 }
3545
3546 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3547 {
3548     if( 0 == value)
3549         value = (OSObject *) gIOServiceKey;
3550
3551     gIOResources->setProperty( key, value);
3552
3553     if( IORecursiveLockHaveLock( gNotificationLock))
3554         return;
3555
3556     gIOResourceGenerationCount++;
3557     gIOResources->registerService();
3558 }
3559
3560 bool IOService::addNeededResource( const char * key )
3561 {
3562     OSObject *  resourcesProp;
3563     OSSet *     set;
3564     OSString *  newKey;
3565     bool ret;
3566
3567     resourcesProp = copyProperty( gIOResourceMatchKey );
3568     if (!resourcesProp) return(false);
3569
3570     newKey = OSString::withCString( key );
3571     if (!newKey)
3572     {
3573         resourcesProp->release();
3574         return( false);
3575     }
3576
3577     set = OSDynamicCast( OSSet, resourcesProp );
3578     if( !set) {
3579         set = OSSet::withCapacity( 1 );
3580         if( set)
3581             set->setObject( resourcesProp );
3582     }
3583     else
3584         set->retain();
3585
3586     set->setObject( newKey );
3587     newKey->release();
3588     ret = setProperty( gIOResourceMatchKey, set );
3589     set->release();
3590     resourcesProp->release();
3591
3592     return( ret );
3593 }
3594
3595 bool IOService::checkResource( OSObject * matching )
3596 {
3597     OSString *          str;
3598     OSDictionary *      table;
3599
3600     if( (str = OSDynamicCast( OSString, matching ))) {
3601         if( gIOResources->getProperty( str ))
3602             return( true );
3603     }
3604
3605     if( str)
3606         table = resourceMatching( str );
3607     else if( (table = OSDynamicCast( OSDictionary, matching )))
3608         table->retain();
3609     else {
3610         IOLog("%s: Can't match using: %s\n", getName(),
3611                 matching->getMetaClass()->getClassName());
3612         /* false would stall forever */
3613         return( true );
3614     }
3615
3616     if( gIOKitDebug & kIOLogConfig)
3617         LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
3618
3619     waitForService( table );
3620
3621     if( gIOKitDebug & kIOLogConfig)
3622         LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
3623
3624     return( true );
3625 }
3626
3627 bool IOService::checkResources( void )
3628 {
3629     OSObject *          resourcesProp;
3630     OSSet *             set;
3631     OSIterator *        iter;
3632     bool                ok;
3633
3634     resourcesProp = copyProperty( gIOResourceMatchKey );
3635     if( 0 == resourcesProp)
3636         return( true );
3637
3638     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3639
3640         iter = OSCollectionIterator::withCollection( set );
3641         ok = (0 != iter);
3642         while( ok && (resourcesProp = iter->getNextObject()) )
3643             ok = checkResource( resourcesProp );
3644         if( iter)
3645             iter->release();
3646
3647     } else
3648         ok = checkResource( resourcesProp );
3649
3650     OSSafeReleaseNULL(resourcesProp);
3651
3652     return( ok );
3653 }
3654
3655
3656 void _IOConfigThread::configThread( void )
3657 {
3658     _IOConfigThread *   inst;
3659
3660     do {
3661         if( !(inst = new _IOConfigThread))
3662             continue;
3663         if( !inst->init())
3664             continue;
3665         thread_t unused;
3666         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3667             continue;
3668
3669         return;
3670
3671     } while( false);
3672
3673     if( inst)
3674         inst->release();
3675
3676     return;
3677 }
3678
3679 void _IOConfigThread::free( void )
3680 {
3681     thread_deallocate(current_thread());
3682     OSObject::free();
3683 }
3684
3685 void IOService::doServiceMatch( IOOptionBits options )
3686 {
3687     _IOServiceNotifier * notify;
3688     OSIterator *        iter;
3689     OSOrderedSet *      matches;
3690     OSArray *           resourceKeys = 0;
3691     SInt32              catalogGeneration;
3692     bool                keepGuessing = true;
3693     bool                reRegistered = true;
3694     bool                didRegister;
3695     OSArray *           notifiers[2] = {0};
3696
3697 //    job->nub->deliverNotification( gIOPublishNotification,
3698 //                              kIOServiceRegisteredState, 0xffffffff );
3699
3700     while( keepGuessing ) {
3701
3702         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3703         // the matches list should always be created by findDrivers()
3704         if( matches) {
3705
3706             lockForArbitration();
3707             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3708                 getMetaClass()->addInstance(this);
3709                 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
3710                                      kIOServiceFirstPublishState, 0xffffffff );
3711             }
3712             LOCKREADNOTIFY();
3713             __state[1] &= ~kIOServiceNeedConfigState;
3714             __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
3715             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3716             __state[0] |= kIOServiceRegisteredState;
3717
3718             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3719             if (reRegistered && keepGuessing) {
3720                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3721                         gNotifications->getObject( gIOPublishNotification ) );
3722                 if( iter) {
3723                     while((notify = (_IOServiceNotifier *)
3724                            iter->getNextObject())) {
3725
3726                         if( matchPassive(notify->matching, 0)
3727                          && (kIOServiceNotifyEnable & notify->state))
3728                             matches->setObject( notify );
3729                     }
3730                     iter->release();
3731                 }
3732             }
3733
3734             UNLOCKNOTIFY();
3735             unlockForArbitration();
3736             invokeNotifiers(&notifiers[0]);
3737
3738             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3739             {
3740                 if (this == gIOResources)
3741                 {
3742                    if (resourceKeys) resourceKeys->release();
3743                    resourceKeys = copyPropertyKeys();
3744                 }
3745                 probeCandidates( matches );
3746             }
3747             else
3748                 matches->release();
3749         }
3750
3751         lockForArbitration();
3752         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3753         keepGuessing =
3754                    (reRegistered || (catalogGeneration !=
3755                                         gIOCatalogue->getGenerationCount()))
3756                 && (0 == (__state[0] & kIOServiceInactiveState));
3757
3758         if( keepGuessing)
3759             unlockForArbitration();
3760     }
3761
3762     if( (0 == (__state[0] & kIOServiceInactiveState))
3763      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3764
3765         if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys);
3766
3767         notifiers[0] = copyNotifiers(gIOMatchedNotification,
3768                 kIOServiceMatchedState, 0xffffffff);
3769         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3770             notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
3771                 kIOServiceFirstMatchState, 0xffffffff);
3772     }
3773
3774     __state[1] &= ~kIOServiceConfigRunning;
3775     unlockForArbitration();
3776
3777     if (resourceKeys) resourceKeys->release();
3778
3779     invokeNotifiers(&notifiers[0]);
3780     invokeNotifiers(&notifiers[1]);
3781
3782     lockForArbitration();
3783     __state[1] &= ~kIOServiceConfigState;
3784     scheduleTerminatePhase2();
3785
3786     _adjustBusy( -1 );
3787     unlockForArbitration();
3788 }
3789
3790 UInt32 IOService::_adjustBusy( SInt32 delta )
3791 {
3792     IOService * next;
3793     UInt32      count;
3794     UInt32      result;
3795     bool        wasQuiet, nowQuiet, needWake;
3796
3797     next = this;
3798     result = __state[1] & kIOServiceBusyStateMask;
3799
3800     if( delta) do {
3801         if( next != this)
3802             next->lockForArbitration();
3803         count = next->__state[1] & kIOServiceBusyStateMask;
3804         wasQuiet = (0 == count);
3805         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3806             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3807         else
3808             count += delta;
3809         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3810         nowQuiet = (0 == count);
3811         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3812
3813         if( needWake) {
3814             next->__state[1] &= ~kIOServiceBusyWaiterState;
3815             IOLockLock( gIOServiceBusyLock );
3816             thread_wakeup( (event_t) next);
3817             IOLockUnlock( gIOServiceBusyLock );
3818         }
3819         if( next != this)
3820             next->unlockForArbitration();
3821
3822         if( (wasQuiet || nowQuiet) ) {
3823
3824             uint64_t regID = next->getRegistryEntryID();
3825             IOServiceTrace(
3826                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3827                 (uintptr_t) regID,
3828                 (uintptr_t) (regID >> 32),
3829                 (uintptr_t) next,
3830                 0);
3831
3832             if (wasQuiet)
3833             {
3834                 next->__timeBusy = mach_absolute_time();
3835             }
3836             else
3837             {
3838                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3839                 next->__timeBusy = 0;
3840             }
3841
3842             MessageClientsContext context;
3843
3844             context.service  = next;
3845             context.type     = kIOMessageServiceBusyStateChange;
3846             context.argument = (void *) wasQuiet;       /*nowBusy*/
3847             context.argSize  = 0;
3848
3849             applyToInterestNotifiers( next, gIOBusyInterest,
3850                                      &messageClientsApplier, &context );
3851
3852 #if !NO_KEXTD
3853             if( nowQuiet && (next == gIOServiceRoot)) {
3854                 OSKext::considerUnloads();
3855                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3856             }
3857 #endif
3858         }
3859
3860         delta = nowQuiet ? -1 : +1;
3861
3862     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3863
3864     return( result );
3865 }
3866
3867 void IOService::adjustBusy( SInt32 delta )
3868 {
3869     lockForArbitration();
3870     _adjustBusy( delta );
3871     unlockForArbitration();
3872 }
3873
3874 uint64_t IOService::getAccumulatedBusyTime( void )
3875 {
3876     uint64_t accumBusy = __accumBusy;
3877     uint64_t timeBusy = __timeBusy;
3878     uint64_t nano;
3879
3880     do
3881     {
3882         accumBusy = __accumBusy;
3883         timeBusy  = __timeBusy;
3884         if (timeBusy)
3885             accumBusy += mach_absolute_time() - timeBusy;
3886     }
3887     while (timeBusy != __timeBusy);
3888
3889     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3890
3891     return (nano);
3892 }
3893
3894 UInt32 IOService::getBusyState( void )
3895 {
3896     return( __state[1] & kIOServiceBusyStateMask );
3897 }
3898
3899 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3900                                   mach_timespec_t * timeout )
3901 {
3902     panic("waitForState");
3903     return (kIOReturnUnsupported);
3904 }
3905
3906 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3907                                    uint64_t timeout )
3908 {
3909     bool            wait;
3910     int             waitResult = THREAD_AWAKENED;
3911     bool            computeDeadline = true;
3912     AbsoluteTime    abstime;
3913
3914     do {
3915         lockForArbitration();
3916         IOLockLock( gIOServiceBusyLock );
3917         wait = (value != (__state[1] & mask));
3918         if( wait) {
3919             __state[1] |= kIOServiceBusyWaiterState;
3920             unlockForArbitration();
3921             if( timeout != UINT64_MAX ) {
3922                 if( computeDeadline ) {
3923                     AbsoluteTime  nsinterval;
3924                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3925                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3926                     computeDeadline = false;
3927                 }
3928                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3929             }
3930             else
3931                 assert_wait((event_t)this, THREAD_UNINT );
3932         } else
3933             unlockForArbitration();
3934         IOLockUnlock( gIOServiceBusyLock );
3935         if( wait)
3936             waitResult = thread_block(THREAD_CONTINUE_NULL);
3937
3938     } while( wait && (waitResult != THREAD_TIMED_OUT));
3939
3940     if( waitResult == THREAD_TIMED_OUT)
3941         return( kIOReturnTimeout );
3942     else
3943         return( kIOReturnSuccess );
3944 }
3945
3946 IOReturn IOService::waitQuiet( uint64_t timeout )
3947 {
3948     IOReturn ret;
3949     uint32_t loops;
3950     char *   string = NULL;
3951     char *   panicString = NULL;
3952     size_t   len;
3953     size_t   panicStringLen;
3954     uint64_t time;
3955     uint64_t nano;
3956     bool     kextdWait;
3957     bool     dopanic;
3958
3959     enum { kTimeoutExtensions = 4 };
3960
3961     time = mach_absolute_time();
3962     kextdWait = false;
3963     for (loops = 0; loops < kTimeoutExtensions; loops++)
3964     {
3965         ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3966
3967         if (loops && (kIOReturnSuccess == ret))
3968         {
3969             time = mach_absolute_time() - time;
3970             absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
3971             IOLog("busy extended ok[%d], (%llds, %llds)\n",
3972                   loops, timeout / 1000000000ULL, nano / 1000000000ULL);
3973             break;
3974         }
3975         else if (kIOReturnTimeout != ret) break;
3976         else if (timeout < 41000000000)   break;
3977
3978         {
3979             IORegistryIterator * iter;
3980             OSOrderedSet       * set;
3981             OSOrderedSet       * leaves;
3982             IOService          * next;
3983             IOService          * nextParent;
3984             char               * s;
3985             size_t               l;
3986
3987             len = 256;
3988             panicStringLen = 256;
3989             if (!string)      string      = IONew(char, len);
3990             if (!panicString) panicString = IONew(char, panicStringLen);
3991             set = NULL;
3992             kextdWait = OSKext::isWaitingKextd();
3993             iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
3994             leaves = OSOrderedSet::withCapacity(4);
3995             if (iter) set = iter->iterateAll();
3996             if (string && panicString && leaves && set)
3997             {
3998                 string[0] = panicString[0] = 0;
3999                 set->setObject(this);
4000                 while ((next = (IOService *) set->getLastObject()))
4001                 {
4002                     if (next->getBusyState())
4003                     {
4004                         if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true;
4005                         leaves->setObject(next);
4006                         nextParent = next;
4007                         while ((nextParent = nextParent->getProvider()))
4008                         {
4009                             set->removeObject(nextParent);
4010                             leaves->removeObject(nextParent);
4011                         }
4012                     }
4013                     set->removeObject(next);
4014                 }
4015                 s = string;
4016                 while ((next = (IOService *) leaves->getLastObject()))
4017                 {
4018                     l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
4019                     if (l >= len) break;
4020                     s += l;
4021                     len -= l;
4022                     leaves->removeObject(next);
4023                 }
4024             }
4025             OSSafeReleaseNULL(leaves);
4026             OSSafeReleaseNULL(set);
4027             OSSafeReleaseNULL(iter);
4028         }
4029
4030         dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug));
4031         snprintf(panicString, panicStringLen,
4032                  "%s[%d], (%llds): %s",
4033                  kextdWait ? "kextd stall" : "busy timeout",
4034                  loops, timeout / 1000000000ULL,
4035                  string ? string : "");
4036         IOLog("%s\n", panicString);
4037         if (dopanic)     panic("%s", panicString);
4038         else if (!loops) getPMRootDomain()->startSpinDump(1);
4039     }
4040
4041     if (string)      IODelete(string, char, 256);
4042     if (panicString) IODelete(panicString, char, panicStringLen);
4043
4044     return (ret);
4045 }
4046
4047 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
4048 {
4049     uint64_t    timeoutNS;
4050
4051     if (timeout)
4052     {
4053         timeoutNS = timeout->tv_sec;
4054         timeoutNS *= kSecondScale;
4055         timeoutNS += timeout->tv_nsec;
4056     }
4057     else
4058         timeoutNS = UINT64_MAX;
4059
4060     return (waitQuiet(timeoutNS));
4061 }
4062
4063 bool IOService::serializeProperties( OSSerialize * s ) const
4064 {
4065 #if 0
4066     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
4067                 sizeof( __state), "__state");
4068 #endif
4069     return( super::serializeProperties(s) );
4070 }
4071
4072
4073 void _IOConfigThread::main(void * arg, wait_result_t result)
4074 {
4075     _IOConfigThread * self = (_IOConfigThread *) arg;
4076     _IOServiceJob * job;
4077     IOService   *   nub;
4078     bool            alive = true;
4079     kern_return_t   kr;
4080     thread_precedence_policy_data_t precedence = { -1 };
4081
4082     kr = thread_policy_set(current_thread(),
4083                             THREAD_PRECEDENCE_POLICY,
4084                             (thread_policy_t) &precedence,
4085                             THREAD_PRECEDENCE_POLICY_COUNT);
4086     if (KERN_SUCCESS != kr)
4087         IOLog("thread_policy_set(%d)\n", kr);
4088
4089     do {
4090
4091 //      randomDelay();
4092
4093         semaphore_wait( gJobsSemaphore );
4094
4095         IOTakeLock( gJobsLock );
4096         job = (_IOServiceJob *) gJobs->getFirstObject();
4097         job->retain();
4098         gJobs->removeObject(job);
4099         if( job) {
4100             gOutstandingJobs--;
4101 //          gNumConfigThreads--;        // we're out of service
4102             gNumWaitingThreads--;       // we're out of service
4103         }
4104         IOUnlock( gJobsLock );
4105
4106         if( job) {
4107
4108           nub = job->nub;
4109
4110           if( gIOKitDebug & kIOLogConfig)
4111             LOG("config(%p): starting on %s, %d\n",
4112                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
4113
4114           switch( job->type) {
4115
4116             case kMatchNubJob:
4117                 nub->doServiceMatch( job->options );
4118                 break;
4119
4120             default:
4121                 LOG("config(%p): strange type (%d)\n",
4122                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
4123                 break;
4124             }
4125
4126             nub->release();
4127             job->release();
4128
4129             IOTakeLock( gJobsLock );
4130             alive = (gOutstandingJobs > gNumWaitingThreads);
4131             if( alive)
4132                 gNumWaitingThreads++;   // back in service
4133 //              gNumConfigThreads++;
4134             else {
4135                 if( 0 == --gNumConfigThreads) {
4136 //                    IOLog("MATCH IDLE\n");
4137                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
4138                 }
4139             }
4140             IOUnlock( gJobsLock );
4141         }
4142
4143     } while( alive );
4144
4145     if( gIOKitDebug & kIOLogConfig)
4146         LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
4147
4148     self->release();
4149 }
4150
4151 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
4152 {
4153     bool            wait;
4154     int             waitResult = THREAD_AWAKENED;
4155     bool            computeDeadline = true;
4156     AbsoluteTime    deadline;
4157
4158     IOLockLock( gJobsLock );
4159     do {
4160         wait = (0 != gNumConfigThreads);
4161         if( wait) {
4162             if( msToWait) {
4163                 if( computeDeadline ) {
4164                     clock_interval_to_deadline(
4165                           msToWait, kMillisecondScale, &deadline );
4166                     computeDeadline = false;
4167                 }
4168                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
4169                                                                 deadline, THREAD_UNINT );
4170                    } else {
4171                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
4172                                                                 THREAD_UNINT );
4173                 }
4174         }
4175     } while( wait && (waitResult != THREAD_TIMED_OUT));
4176         IOLockUnlock( gJobsLock );
4177
4178     if( waitResult == THREAD_TIMED_OUT)
4179         return( kIOReturnTimeout );
4180     else
4181         return( kIOReturnSuccess );
4182 }
4183
4184 void IOService::cpusRunning(void)
4185 {
4186     gCPUsRunning = true;
4187 }
4188
4189 void _IOServiceJob::pingConfig( _IOServiceJob * job )
4190 {
4191     int         count;
4192     bool        create;
4193
4194     assert( job );
4195
4196     IOTakeLock( gJobsLock );
4197
4198     gOutstandingJobs++;
4199     gJobs->setLastObject( job );
4200
4201     count = gNumWaitingThreads;
4202 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
4203
4204     create = (  (gOutstandingJobs > count)
4205                 && ((gNumConfigThreads < kMaxConfigThreads)
4206             || (job->nub == gIOResources)
4207             || !gCPUsRunning));
4208     if( create) {
4209         gNumConfigThreads++;
4210         gNumWaitingThreads++;
4211     }
4212
4213     IOUnlock( gJobsLock );
4214
4215     job->release();
4216
4217     if( create) {
4218         if( gIOKitDebug & kIOLogConfig)
4219             LOG("config(%d): creating\n", gNumConfigThreads - 1);
4220         _IOConfigThread::configThread();
4221     }
4222
4223     semaphore_signal( gJobsSemaphore );
4224 }
4225
4226 struct IOServiceMatchContext
4227 {
4228     OSDictionary * table;
4229     OSObject *     result;
4230     uint32_t       options;
4231     uint32_t       state;
4232     uint32_t       count;
4233     uint32_t       done;
4234 };
4235
4236 bool IOService::instanceMatch(const OSObject * entry, void * context)
4237 {
4238     IOServiceMatchContext * ctx = (typeof(ctx)) context;
4239     IOService *    service = (typeof(service)) entry;
4240     OSDictionary * table   = ctx->table;
4241     uint32_t       options = ctx->options;
4242     uint32_t       state   = ctx->state;
4243     uint32_t       done;
4244     bool           match;
4245
4246     done = 0;
4247     do
4248     {
4249         match = ((state == (state & service->__state[0]))
4250                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
4251         if (!match) break;
4252         ctx->count += table->getCount();
4253         match = service->matchInternal(table, options, &done);
4254         ctx->done += done;
4255     }
4256     while (false);
4257     if (!match)
4258         return (false);
4259
4260     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
4261     {
4262         service->retain();
4263         ctx->result = service;
4264         return (true);
4265     }
4266     else if (!ctx->result)
4267     {
4268         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
4269     }
4270     else
4271     {
4272         ((OSSet *)ctx->result)->setObject(service);
4273     }
4274     return (false);
4275 }
4276
4277 // internal - call with gNotificationLock
4278 OSObject * IOService::copyExistingServices( OSDictionary * matching,
4279                  IOOptionBits inState, IOOptionBits options )
4280 {
4281     OSObject *   current = 0;
4282     OSIterator * iter;
4283     IOService *  service;
4284     OSObject *   obj;
4285     OSString *   str;
4286
4287     if( !matching)
4288         return( 0 );
4289
4290 #if MATCH_DEBUG
4291     OSSerialize * s = OSSerialize::withCapacity(128);
4292     matching->serialize(s);
4293 #endif
4294
4295     if((obj = matching->getObject(gIOProviderClassKey))
4296       && gIOResourcesKey
4297       && gIOResourcesKey->isEqualTo(obj)
4298       && (service = gIOResources))
4299     {
4300         if( (inState == (service->__state[0] & inState))
4301           && (0 == (service->__state[0] & kIOServiceInactiveState))
4302           &&  service->matchPassive(matching, options))
4303         {
4304             if( options & kIONotifyOnce)
4305             {
4306                 service->retain();
4307                 current = service;
4308             }
4309             else
4310                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
4311         }
4312     }
4313     else
4314     {
4315         IOServiceMatchContext ctx;
4316         ctx.table   = matching;
4317         ctx.state   = inState;
4318         ctx.count   = 0;
4319         ctx.done    = 0;
4320         ctx.options = options;
4321         ctx.result  = 0;
4322
4323         if ((str = OSDynamicCast(OSString, obj)))
4324         {
4325             const OSSymbol * sym = OSSymbol::withString(str);
4326             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
4327             sym->release();
4328         }
4329         else
4330         {
4331             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
4332         }
4333
4334
4335         current = ctx.result;
4336
4337         options |= kIOServiceInternalDone | kIOServiceClassDone;
4338         if (current && (ctx.done != ctx.count))
4339         {
4340             OSSet *
4341             source = OSDynamicCast(OSSet, current);
4342             current = 0;
4343             while ((service = (IOService *) source->getAnyObject()))
4344             {
4345                 if (service->matchPassive(matching, options))
4346                 {
4347                     if( options & kIONotifyOnce)
4348                     {
4349                         service->retain();
4350                         current = service;
4351                         break;
4352                     }
4353                     if( current)
4354                     {
4355                         ((OSSet *)current)->setObject( service );
4356                     }
4357                     else
4358                     {
4359                         current = OSSet::withObjects(
4360                                         (const OSObject **) &service, 1, 1 );
4361                     }
4362                 }
4363                 source->removeObject(service);
4364             }
4365             source->release();
4366         }
4367     }
4368
4369 #if MATCH_DEBUG
4370     {
4371         OSObject * _current = 0;
4372
4373         iter = IORegistryIterator::iterateOver( gIOServicePlane,
4374                                             kIORegistryIterateRecursively );
4375         if( iter) {
4376             do {
4377                 iter->reset();
4378                 while( (service = (IOService *) iter->getNextObject())) {
4379                     if( (inState == (service->__state[0] & inState))
4380                     && (0 == (service->__state[0] & kIOServiceInactiveState))
4381                     &&  service->matchPassive(matching, 0)) {
4382
4383                         if( options & kIONotifyOnce) {
4384                             service->retain();
4385                             _current = service;
4386                             break;
4387                         }
4388                         if( _current)
4389                             ((OSSet *)_current)->setObject( service );
4390                         else
4391                             _current = OSSet::withObjects(
4392                                             (const OSObject **) &service, 1, 1 );
4393                     }
4394                 }
4395             } while( !service && !iter->isValid());
4396             iter->release();
4397         }
4398
4399
4400         if ( ((current != 0) != (_current != 0))
4401         || (current && _current && !current->isEqualTo(_current)))
4402         {
4403             OSSerialize * s1 = OSSerialize::withCapacity(128);
4404             OSSerialize * s2 = OSSerialize::withCapacity(128);
4405             current->serialize(s1);
4406             _current->serialize(s2);
4407             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
4408                 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4409             s1->release();
4410             s2->release();
4411         }
4412
4413         if (_current) _current->release();
4414     }
4415
4416     s->release();
4417 #endif
4418
4419     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4420         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4421         current->release();
4422         current = iter;
4423     }
4424
4425     return( current );
4426 }
4427
4428 // public version
4429 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4430 {
4431     OSIterator *        iter;
4432
4433     // is a lock even needed?
4434     LOCKWRITENOTIFY();
4435
4436     iter = (OSIterator *) copyExistingServices( matching,
4437                                                 kIOServiceMatchedState );
4438
4439     UNLOCKNOTIFY();
4440
4441     return( iter );
4442 }
4443
4444 IOService * IOService::copyMatchingService( OSDictionary * matching )
4445 {
4446     IOService * service;
4447
4448     // is a lock even needed?
4449     LOCKWRITENOTIFY();
4450
4451     service = (IOService *) copyExistingServices( matching,
4452                                                 kIOServiceMatchedState, kIONotifyOnce );
4453
4454     UNLOCKNOTIFY();
4455
4456     return( service );
4457 }
4458
4459 struct _IOServiceMatchingNotificationHandlerRef
4460 {
4461     IOServiceNotificationHandler handler;
4462     void * ref;
4463 };
4464
4465 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4466                                                    IOService * newService,
4467                                                    IONotifier * notifier )
4468 {
4469     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4470 }
4471
4472 // internal - call with gNotificationLock
4473 IONotifier * IOService::setNotification(
4474             const OSSymbol * type, OSDictionary * matching,
4475             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4476             SInt32 priority )
4477 {
4478     _IOServiceNotifier * notify = 0;
4479     OSOrderedSet *      set;
4480
4481     if( !matching)
4482         return( 0 );
4483
4484     notify = new _IOServiceNotifier;
4485     if( notify && !notify->init()) {
4486         notify->release();
4487         notify = 0;
4488     }
4489
4490     if( notify) {
4491         notify->handler = handler;
4492         notify->target = target;
4493         notify->type = type;
4494         notify->matching = matching;
4495         matching->retain();
4496         if (handler == &_IOServiceMatchingNotificationHandler)
4497         {
4498             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4499             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4500         }
4501         else
4502             notify->ref = ref;
4503         notify->priority = priority;
4504         notify->state = kIOServiceNotifyEnable;
4505         queue_init( &notify->handlerInvocations );
4506
4507         ////// queue
4508
4509         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4510             set = OSOrderedSet::withCapacity( 1,
4511                         IONotifyOrdering, 0 );
4512             if( set) {
4513                 gNotifications->setObject( type, set );
4514                 set->release();
4515             }
4516         }
4517         notify->whence = set;
4518         if( set)
4519             set->setObject( notify );
4520     }
4521
4522     return( notify );
4523 }
4524
4525 // internal - call with gNotificationLock
4526 IONotifier * IOService::doInstallNotification(
4527                         const OSSymbol * type, OSDictionary * matching,
4528                         IOServiceMatchingNotificationHandler handler,
4529                         void * target, void * ref,
4530                         SInt32 priority, OSIterator ** existing )
4531 {
4532     OSIterator *        exist;
4533     IONotifier *        notify;
4534     IOOptionBits        inState;
4535
4536     if( !matching)
4537         return( 0 );
4538
4539     if( type == gIOPublishNotification)
4540         inState = kIOServiceRegisteredState;
4541
4542     else if( type == gIOFirstPublishNotification)
4543         inState = kIOServiceFirstPublishState;
4544
4545     else if (type == gIOMatchedNotification)
4546         inState = kIOServiceMatchedState;
4547
4548     else if (type == gIOFirstMatchNotification)
4549         inState = kIOServiceFirstMatchState;
4550
4551     else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification))
4552         inState = 0;
4553     else
4554         return( 0 );
4555
4556     notify = setNotification( type, matching, handler, target, ref, priority );
4557
4558     if( inState)
4559         // get the current set
4560         exist = (OSIterator *) copyExistingServices( matching, inState );
4561     else
4562         exist = 0;
4563
4564     *existing = exist;
4565
4566     return( notify );
4567 }
4568
4569 #if !defined(__LP64__)
4570 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4571                                         IOServiceNotificationHandler handler,
4572                                         void * target, void * refCon,
4573                                         SInt32 priority, OSIterator ** existing )
4574 {
4575     IONotifier * result;
4576     _IOServiceMatchingNotificationHandlerRef ref;
4577     ref.handler = handler;
4578     ref.ref     = refCon;
4579
4580     result = (_IOServiceNotifier *) installNotification( type, matching,
4581                          &_IOServiceMatchingNotificationHandler,
4582                         target, &ref, priority, existing );
4583     if (result)
4584         matching->release();
4585
4586     return (result);
4587 }
4588 #endif /* !defined(__LP64__) */
4589
4590
4591 IONotifier * IOService::installNotification(
4592                         const OSSymbol * type, OSDictionary * matching,
4593                         IOServiceMatchingNotificationHandler handler,
4594                         void * target, void * ref,
4595                         SInt32 priority, OSIterator ** existing )
4596 {
4597     IONotifier * notify;
4598
4599     LOCKWRITENOTIFY();
4600
4601     notify = doInstallNotification( type, matching, handler, target, ref,
4602                 priority, existing );
4603
4604     // in case handler remove()s
4605     if (notify) notify->retain();
4606
4607     UNLOCKNOTIFY();
4608
4609     return( notify );
4610 }
4611
4612 IONotifier * IOService::addNotification(
4613                         const OSSymbol * type, OSDictionary * matching,
4614                         IOServiceNotificationHandler handler,
4615                         void * target, void * refCon,
4616                         SInt32 priority )
4617 {
4618     IONotifier * result;
4619     _IOServiceMatchingNotificationHandlerRef ref;
4620
4621     ref.handler = handler;
4622     ref.ref     = refCon;
4623
4624     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4625                             target, &ref, priority);
4626
4627     if (result)
4628         matching->release();
4629
4630     return (result);
4631 }
4632
4633 IONotifier * IOService::addMatchingNotification(
4634                         const OSSymbol * type, OSDictionary * matching,
4635                         IOServiceMatchingNotificationHandler handler,
4636                         void * target, void * ref,
4637                         SInt32 priority )
4638 {
4639     OSIterator *                existing = NULL;
4640     IONotifier *                ret;
4641     _IOServiceNotifier *        notify;
4642     IOService *                 next;
4643
4644     ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
4645                 handler, target, ref, priority, &existing );
4646     if (!ret) return (0);
4647
4648     // send notifications for existing set
4649     if (existing)
4650     {
4651         while( (next = (IOService *) existing->getNextObject()))
4652         {
4653             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4654             {
4655                 next->invokeNotifier( notify );
4656             }
4657         }
4658         existing->release();
4659     }
4660
4661     LOCKWRITENOTIFY();
4662     bool removed = (0 == notify->whence);
4663     notify->release();
4664     if (removed) ret = gIOServiceNullNotifier;
4665     UNLOCKNOTIFY();
4666
4667     return( ret );
4668 }
4669
4670 bool IOService::syncNotificationHandler(
4671                         void * /* target */, void * ref,
4672                         IOService * newService,
4673                         IONotifier * notifier )
4674 {
4675
4676     LOCKWRITENOTIFY();
4677     if (!*((IOService **) ref))
4678     {
4679         newService->retain();
4680         (*(IOService **) ref) = newService;
4681         WAKEUPNOTIFY(ref);
4682     }
4683     UNLOCKNOTIFY();
4684
4685     return( false );
4686 }
4687
4688 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4689                                                 uint64_t timeout)
4690 {
4691     IONotifier *        notify = 0;
4692     // priority doesn't help us much since we need a thread wakeup
4693     SInt32              priority = 0;
4694     IOService *         result;
4695
4696     if (!matching)
4697         return( 0 );
4698
4699     result = NULL;
4700
4701     LOCKWRITENOTIFY();
4702     do
4703     {
4704         result = (IOService *) copyExistingServices( matching,
4705                             kIOServiceMatchedState, kIONotifyOnce );
4706         if (result)
4707             break;
4708         notify = IOService::setNotification( gIOMatchedNotification, matching,
4709                     &IOService::syncNotificationHandler, (void *) 0,
4710                     &result, priority );
4711          if (!notify)
4712             break;
4713         if (UINT64_MAX != timeout)
4714         {
4715             AbsoluteTime deadline;
4716             nanoseconds_to_absolutetime(timeout, &deadline);
4717             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4718             SLEEPNOTIFYTO(&result, deadline);
4719         }
4720         else
4721         {
4722             SLEEPNOTIFY(&result);
4723         }
4724     }
4725     while( false );
4726
4727     UNLOCKNOTIFY();
4728
4729     if (notify)
4730         notify->remove();       // dequeues
4731
4732     return( result );
4733 }
4734
4735 IOService * IOService::waitForService( OSDictionary * matching,
4736                                         mach_timespec_t * timeout )
4737 {
4738     IOService * result;
4739     uint64_t    timeoutNS;
4740
4741     if (timeout)
4742     {
4743         timeoutNS = timeout->tv_sec;
4744         timeoutNS *= kSecondScale;
4745         timeoutNS += timeout->tv_nsec;
4746     }
4747     else
4748         timeoutNS = UINT64_MAX;
4749
4750     result = waitForMatchingService(matching, timeoutNS);
4751
4752     matching->release();
4753     if (result)
4754         result->release();
4755
4756     return (result);
4757 }
4758
4759 void IOService::deliverNotification( const OSSymbol * type,
4760                             IOOptionBits orNewState, IOOptionBits andNewState )
4761 {
4762     panic("deliverNotification");
4763 }
4764
4765 OSArray * IOService::copyNotifiers(const OSSymbol * type,
4766                                    IOOptionBits orNewState, IOOptionBits andNewState )
4767 {
4768     _IOServiceNotifier * notify;
4769     OSIterator *         iter;
4770     OSArray *            willSend = 0;
4771
4772     lockForArbitration();
4773
4774     if( (0 == (__state[0] & kIOServiceInactiveState))
4775      || (type == gIOTerminatedNotification)
4776      || (type == gIOWillTerminateNotification)) {
4777
4778         LOCKREADNOTIFY();
4779
4780         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4781                     gNotifications->getObject( type ) );
4782
4783         if( iter) {
4784             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4785
4786                 if( matchPassive(notify->matching, 0)
4787                   && (kIOServiceNotifyEnable & notify->state)) {
4788                     if( 0 == willSend)
4789                         willSend = OSArray::withCapacity(8);
4790                     if( willSend)
4791                         willSend->setObject( notify );
4792                 }
4793             }
4794             iter->release();
4795         }
4796         __state[0] = (__state[0] | orNewState) & andNewState;
4797         UNLOCKNOTIFY();
4798     }
4799
4800     unlockForArbitration();
4801
4802     return (willSend);
4803
4804 }
4805
4806 IOOptionBits IOService::getState( void ) const
4807 {
4808     return( __state[0] );
4809 }
4810
4811 /*
4812  * Helpers to make matching objects for simple cases
4813  */
4814
4815 OSDictionary * IOService::serviceMatching( const OSString * name,
4816                         OSDictionary * table )
4817 {
4818
4819     const OSString *    str;
4820
4821     str = OSSymbol::withString(name);
4822     if( !str)
4823         return( 0 );
4824
4825     if( !table)
4826         table = OSDictionary::withCapacity( 2 );
4827     if( table)
4828         table->setObject(gIOProviderClassKey, (OSObject *)str );
4829     str->release();
4830
4831     return( table );
4832 }
4833
4834 OSDictionary * IOService::serviceMatching( const char * name,
4835                         OSDictionary * table )
4836 {
4837     const OSString *    str;
4838
4839     str = OSSymbol::withCString( name );
4840     if( !str)
4841         return( 0 );
4842
4843     table = serviceMatching( str, table );
4844     str->release();
4845     return( table );
4846 }
4847
4848 OSDictionary * IOService::nameMatching( const OSString * name,
4849                         OSDictionary * table )
4850 {
4851     if( !table)
4852         table = OSDictionary::withCapacity( 2 );
4853     if( table)
4854         table->setObject( gIONameMatchKey, (OSObject *)name );
4855
4856     return( table );
4857 }
4858
4859 OSDictionary * IOService::nameMatching( const char * name,
4860                         OSDictionary * table )
4861 {
4862     const OSString *    str;
4863
4864     str = OSSymbol::withCString( name );
4865     if( !str)
4866         return( 0 );
4867
4868     table = nameMatching( str, table );
4869     str->release();
4870     return( table );
4871 }
4872
4873 OSDictionary * IOService::resourceMatching( const OSString * str,
4874                         OSDictionary * table )
4875 {
4876     table = serviceMatching( gIOResourcesKey, table );
4877     if( table)
4878         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4879
4880     return( table );
4881 }
4882
4883 OSDictionary * IOService::resourceMatching( const char * name,
4884                         OSDictionary * table )
4885 {
4886     const OSSymbol *    str;
4887
4888     str = OSSymbol::withCString( name );
4889     if( !str)
4890         return( 0 );
4891
4892     table = resourceMatching( str, table );
4893     str->release();
4894
4895     return( table );
4896 }
4897
4898 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4899                         OSDictionary * table )
4900 {
4901     OSDictionary * properties;
4902
4903     properties = OSDictionary::withCapacity( 2 );
4904     if( !properties)
4905         return( 0 );
4906     properties->setObject( key, value );
4907
4908     if( !table)
4909         table = OSDictionary::withCapacity( 2 );
4910     if( table)
4911         table->setObject( gIOPropertyMatchKey, properties );
4912
4913     properties->release();
4914
4915     return( table );
4916 }
4917
4918 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4919                         OSDictionary * table )
4920 {
4921     OSNumber *     num;
4922
4923     num = OSNumber::withNumber( entryID, 64 );
4924     if( !num)
4925         return( 0 );
4926
4927     if( !table)
4928         table = OSDictionary::withCapacity( 2 );
4929     if( table)
4930         table->setObject( gIORegistryEntryIDKey, num );
4931
4932     if (num)
4933         num->release();
4934
4935     return( table );
4936 }
4937
4938
4939 /*
4940  * _IOServiceNotifier
4941  */
4942
4943 // wait for all threads, other than the current one,
4944 //  to exit the handler
4945
4946 void _IOServiceNotifier::wait()
4947 {
4948     _IOServiceNotifierInvocation * next;
4949     bool doWait;
4950
4951     do {
4952         doWait = false;
4953         queue_iterate( &handlerInvocations, next,
4954                         _IOServiceNotifierInvocation *, link) {
4955             if( next->thread != current_thread() ) {
4956                 doWait = true;
4957                 break;
4958             }
4959         }
4960         if( doWait) {
4961             state |= kIOServiceNotifyWaiter;
4962             SLEEPNOTIFY(this);
4963         }
4964
4965     } while( doWait );
4966 }
4967
4968 void _IOServiceNotifier::free()
4969 {
4970     assert( queue_empty( &handlerInvocations ));
4971     OSObject::free();
4972 }
4973
4974 void _IOServiceNotifier::remove()
4975 {
4976     LOCKWRITENOTIFY();
4977
4978     if( whence) {
4979         whence->removeObject( (OSObject *) this );
4980         whence = 0;
4981     }
4982     if( matching) {
4983         matching->release();
4984         matching = 0;
4985     }
4986
4987     state &= ~kIOServiceNotifyEnable;
4988
4989     wait();
4990
4991     UNLOCKNOTIFY();
4992
4993     release();
4994 }
4995
4996 bool _IOServiceNotifier::disable()
4997 {
4998     bool        ret;
4999
5000     LOCKWRITENOTIFY();
5001
5002     ret = (0 != (kIOServiceNotifyEnable & state));
5003     state &= ~kIOServiceNotifyEnable;
5004     if( ret)
5005         wait();
5006
5007     UNLOCKNOTIFY();
5008
5009     return( ret );
5010 }
5011
5012 void _IOServiceNotifier::enable( bool was )
5013 {
5014     LOCKWRITENOTIFY();
5015     if( was)
5016         state |= kIOServiceNotifyEnable;
5017     else
5018         state &= ~kIOServiceNotifyEnable;
5019     UNLOCKNOTIFY();
5020 }
5021
5022
5023 /*
5024  * _IOServiceNullNotifier
5025  */
5026
5027 void _IOServiceNullNotifier::taggedRetain(const void *tag) const                      {}
5028 void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const     {}
5029 void _IOServiceNullNotifier::free()                                                   {}
5030 void _IOServiceNullNotifier::wait()                                                   {}
5031 void _IOServiceNullNotifier::remove()                                                 {}
5032 void _IOServiceNullNotifier::enable(bool was)                                         {}
5033 bool _IOServiceNullNotifier::disable()                                { return(false); }
5034
5035 /*
5036  * IOResources
5037  */
5038
5039 IOService * IOResources::resources( void )
5040 {
5041     IOResources *       inst;
5042
5043     inst = new IOResources;
5044     if( inst && !inst->init()) {
5045         inst->release();
5046         inst = 0;
5047     }
5048
5049     return( inst );
5050 }
5051
5052 bool IOResources::init( OSDictionary * dictionary )
5053 {
5054     // Do super init first
5055     if ( !IOService::init() )
5056         return false;
5057
5058     // Allow PAL layer to publish a value
5059     const char *property_name;
5060     int property_value;
5061
5062     pal_get_resource_property( &property_name, &property_value );
5063
5064     if( property_name ) {
5065         OSNumber *num;
5066         const OSSymbol *        sym;
5067
5068         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
5069             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
5070                 this->setProperty( sym, num );
5071                 sym->release();
5072             }
5073             num->release();
5074         }
5075     }
5076
5077     return true;
5078 }
5079
5080 IOReturn IOResources::newUserClient(task_t owningTask, void * securityID,
5081                                     UInt32 type,  OSDictionary * properties,
5082                                     IOUserClient ** handler)
5083 {
5084     return( kIOReturnUnsupported );
5085 }
5086
5087 IOWorkLoop * IOResources::getWorkLoop() const
5088 {
5089     // If we are the resource root
5090     // then use the platform's workloop
5091     if (this == (IOResources *) gIOResources)
5092         return getPlatform()->getWorkLoop();
5093     else
5094         return IOService::getWorkLoop();
5095 }
5096
5097 bool IOResources::matchPropertyTable( OSDictionary * table )
5098 {
5099     OSObject *          prop;
5100     OSString *          str;
5101     OSSet *             set;
5102     OSIterator *        iter;
5103     OSObject *          obj;
5104     OSArray *           keys;
5105     bool                ok = true;
5106
5107     prop = table->getObject( gIOResourceMatchKey );
5108     str = OSDynamicCast( OSString, prop );
5109     if( str)
5110         ok = (0 != getProperty( str ));
5111
5112     else if( (set = OSDynamicCast( OSSet, prop))) {
5113
5114         iter = OSCollectionIterator::withCollection( set );
5115         ok = (iter != 0);
5116         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
5117             ok = (0 != getProperty( str ));
5118
5119         if( iter)
5120             iter->release();
5121     }
5122     else if ((prop = table->getObject(gIOResourceMatchedKey)))
5123     {
5124         obj = copyProperty(gIOResourceMatchedKey);
5125         keys = OSDynamicCast(OSArray, obj);
5126         ok = false;
5127         if (keys)
5128         {
5129             // assuming OSSymbol
5130             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
5131         }
5132         OSSafeReleaseNULL(obj);
5133     }
5134
5135     return( ok );
5136 }
5137
5138 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
5139 {
5140     IOService::updateConsoleUsers(NULL, 0);
5141 }
5142
5143 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
5144 {
5145     IORegistryEntry * regEntry;
5146     OSObject *        locked = kOSBooleanFalse;
5147     uint32_t          idx;
5148     bool              publish;
5149     OSDictionary *    user;
5150     static IOMessage  sSystemPower;
5151     clock_sec_t       now = 0;
5152     clock_usec_t      microsecs;
5153
5154     regEntry = IORegistryEntry::getRegistryRoot();
5155
5156     if (!gIOChosenEntry)
5157         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
5158
5159     IOLockLock(gIOConsoleUsersLock);
5160
5161     if (systemMessage)
5162     {
5163         sSystemPower = systemMessage;
5164 #if HIBERNATION
5165         if (kIOMessageSystemHasPoweredOn == systemMessage)
5166         {
5167             uint32_t lockState = IOHibernateWasScreenLocked();
5168             switch (lockState)
5169             {
5170                 case 0:
5171                     break;
5172                 case kIOScreenLockLocked:
5173                 case kIOScreenLockFileVaultDialog:
5174                     gIOConsoleBooterLockState = kOSBooleanTrue;
5175                     break;
5176                 case kIOScreenLockNoLock:
5177                     gIOConsoleBooterLockState = 0;
5178                     break;
5179                 case kIOScreenLockUnlocked:
5180                 default:
5181                     gIOConsoleBooterLockState = kOSBooleanFalse;
5182                     break;
5183             }
5184         }
5185 #endif /* HIBERNATION */
5186     }
5187
5188     if (consoleUsers)
5189     {
5190         OSNumber * num = 0;
5191         bool       loginLocked = true;
5192
5193         gIOConsoleLoggedIn = false;
5194         for (idx = 0;
5195               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
5196               idx++)
5197         {
5198             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
5199                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
5200
5201             loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
5202             if (!num)
5203             {
5204                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
5205             }
5206         }
5207 #if HIBERNATION
5208         if (!loginLocked) gIOConsoleBooterLockState = 0;
5209         IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
5210                 (num != 0), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
5211                 gIOConsoleLoggedIn, loginLocked);
5212 #endif /* HIBERNATION */
5213         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
5214     }
5215
5216     if (!gIOConsoleLoggedIn
5217      || (kIOMessageSystemWillSleep == sSystemPower)
5218      || (kIOMessageSystemPagingOff == sSystemPower))
5219     {
5220         locked = kOSBooleanTrue;
5221     }
5222 #if HIBERNATION
5223     else if (gIOConsoleBooterLockState)
5224     {
5225         locked = gIOConsoleBooterLockState;
5226     }
5227 #endif /* HIBERNATION */
5228     else if (gIOConsoleLockTime)
5229     {
5230         clock_get_calendar_microtime(&now, &microsecs);
5231         if (gIOConsoleLockTime > now)
5232         {
5233             AbsoluteTime deadline;
5234             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
5235             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
5236         }
5237         else
5238         {
5239             locked = kOSBooleanTrue;
5240         }
5241     }
5242
5243     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
5244     if (publish)
5245     {
5246         regEntry->setProperty(gIOConsoleLockedKey, locked);
5247         if (consoleUsers)
5248         {
5249             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
5250         }
5251         OSIncrementAtomic( &gIOConsoleUsersSeed );
5252     }
5253
5254 #if HIBERNATION
5255     if (gIOChosenEntry)
5256     {
5257         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
5258         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
5259         else                          gIOScreenLockState = kIOScreenLockNoLock;
5260         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
5261
5262         IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
5263                 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != 0), now, systemMessage);
5264     }
5265 #endif /* HIBERNATION */
5266
5267     IOLockUnlock(gIOConsoleUsersLock);
5268
5269     if (publish)
5270     {
5271         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
5272
5273         MessageClientsContext context;
5274
5275         context.service  = getServiceRoot();
5276         context.type     = kIOMessageConsoleSecurityChange;
5277         context.argument = (void *) regEntry;
5278         context.argSize  = 0;
5279
5280         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
5281                                  &messageClientsApplier, &context );
5282     }
5283 }
5284
5285 IOReturn IOResources::setProperties( OSObject * properties )
5286 {
5287     IOReturn                    err;
5288     const OSSymbol *            key;
5289     OSDictionary *              dict;
5290     OSCollectionIterator *      iter;
5291
5292     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
5293     if ( kIOReturnSuccess != err)
5294         return( err );
5295
5296     dict = OSDynamicCast(OSDictionary, properties);
5297     if( 0 == dict)
5298         return( kIOReturnBadArgument);
5299
5300     iter = OSCollectionIterator::withCollection( dict);
5301     if( 0 == iter)
5302         return( kIOReturnBadArgument);
5303
5304     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
5305     {
5306         if (gIOConsoleUsersKey == key) do
5307         {
5308             OSArray * consoleUsers;
5309             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
5310             if (!consoleUsers)
5311                 continue;
5312             IOService::updateConsoleUsers(consoleUsers, 0);
5313         }
5314         while (false);
5315
5316         publishResource( key, dict->getObject(key) );
5317     }
5318
5319     iter->release();
5320
5321     return( kIOReturnSuccess );
5322 }
5323
5324 /*
5325  * Helpers for matching dictionaries.
5326  * Keys existing in matching are checked in properties.
5327  * Keys may be a string or OSCollection of IOStrings
5328  */
5329
5330 bool IOService::compareProperty( OSDictionary * matching,
5331                                  const char *   key )
5332 {
5333     OSObject *  value;
5334     OSObject *  prop;
5335     bool        ok;
5336
5337     value = matching->getObject( key );
5338     if( value)
5339     {
5340         prop = copyProperty(key);
5341         ok = value->isEqualTo(prop);
5342         if (prop) prop->release();
5343     }
5344     else
5345         ok = true;
5346
5347     return( ok );
5348 }
5349
5350
5351 bool IOService::compareProperty( OSDictionary *   matching,
5352                                  const OSString * key )
5353 {
5354     OSObject *  value;
5355     OSObject *  prop;
5356     bool        ok;
5357
5358     value = matching->getObject( key );
5359     if( value)
5360     {
5361         prop = copyProperty(key);
5362         ok = value->isEqualTo(prop);
5363         if (prop) prop->release();
5364     }
5365     else
5366         ok = true;
5367
5368     return( ok );
5369 }
5370
5371 bool IOService::compareProperties( OSDictionary * matching,
5372                                    OSCollection * keys )
5373 {
5374     OSCollectionIterator *      iter;
5375     const OSString *            key;
5376     bool                        ok = true;
5377
5378     if( !matching || !keys)
5379         return( false );
5380
5381     iter = OSCollectionIterator::withCollection( keys );
5382
5383     if( iter) {
5384         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
5385             ok = compareProperty( matching, key );
5386
5387         iter->release();
5388     }
5389     keys->release();    // !! consume a ref !!
5390
5391     return( ok );
5392 }
5393
5394 /* Helper to add a location matching dict to the table */
5395
5396 OSDictionary * IOService::addLocation( OSDictionary * table )
5397 {
5398     OSDictionary *      dict;
5399
5400     if( !table)
5401         return( 0 );
5402
5403     dict = OSDictionary::withCapacity( 1 );
5404     if( dict) {
5405         table->setObject( gIOLocationMatchKey, dict );
5406         dict->release();
5407     }
5408
5409     return( dict );
5410 }
5411
5412 /*
5413  * Go looking for a provider to match a location dict.
5414  */
5415
5416 IOService * IOService::matchLocation( IOService * /* client */ )
5417 {
5418     IOService * parent;
5419
5420     parent = getProvider();
5421
5422     if( parent)
5423         parent = parent->matchLocation( this );
5424
5425     return( parent );
5426 }
5427
5428 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
5429 {
5430     OSString *          matched;
5431     OSObject *          obj;
5432     OSString *          str;
5433     IORegistryEntry *   entry;
5434     OSNumber *          num;
5435     bool                match = true;
5436     bool                changesOK = (0 != (kIOServiceChangesOK & options));
5437     uint32_t            count;
5438     uint32_t            done;
5439
5440     do
5441     {
5442         count = table->getCount();
5443         done = 0;
5444
5445         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
5446         if (str) {
5447             done++;
5448             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
5449 #if MATCH_DEBUG
5450             match = (0 != metaCast( str ));
5451             if ((kIOServiceClassDone & options) && !match) panic("classDone");
5452 #endif
5453             if ((!match) || (done == count)) break;
5454         }
5455
5456         obj = table->getObject( gIONameMatchKey );
5457         if( obj) {
5458             done++;
5459             match = compareNames( obj, changesOK ? &matched : 0 );
5460             if (!match) break;
5461             if( changesOK && matched) {
5462                 // leave a hint as to which name matched
5463                 table->setObject( gIONameMatchedKey, matched );
5464                 matched->release();
5465             }
5466             if (done == count) break;
5467         }
5468
5469         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
5470         if (str)
5471         {
5472             const OSSymbol * sym;
5473             done++;
5474             match = false;
5475             sym = copyLocation();
5476             if (sym) {
5477                 match = sym->isEqualTo( str );
5478                 sym->release();
5479             }
5480             if ((!match) || (done == count)) break;
5481         }
5482
5483         obj = table->getObject( gIOPropertyMatchKey );
5484         if( obj)
5485         {
5486             OSDictionary * dict;
5487             OSDictionary * nextDict;
5488             OSIterator *   iter;
5489             done++;
5490             match = false;
5491             dict = dictionaryWithProperties();
5492             if( dict) {
5493                 nextDict = OSDynamicCast( OSDictionary, obj);
5494                 if( nextDict)
5495                     iter = 0;
5496                 else
5497                     iter = OSCollectionIterator::withCollection(
5498                                 OSDynamicCast(OSCollection, obj));
5499
5500                 while( nextDict
5501                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5502                                             iter->getNextObject()))))) {
5503                     match = dict->isEqualTo( nextDict, nextDict);
5504                     if( match)
5505                         break;
5506                     nextDict = 0;
5507                 }
5508                 dict->release();
5509                 if( iter)
5510                     iter->release();
5511             }
5512             if ((!match) || (done == count)) break;
5513         }
5514
5515         obj = table->getObject( gIOPropertyExistsMatchKey );
5516         if( obj)
5517         {
5518             OSDictionary * dict;
5519             OSString     * nextKey;
5520             OSIterator *   iter;
5521             done++;
5522             match = false;
5523             dict = dictionaryWithProperties();
5524             if( dict) {
5525                 nextKey = OSDynamicCast( OSString, obj);
5526                 if( nextKey)
5527                     iter = 0;
5528                 else
5529                     iter = OSCollectionIterator::withCollection(
5530                                 OSDynamicCast(OSCollection, obj));
5531
5532                 while( nextKey
5533                     || (iter && (0 != (nextKey = OSDynamicCast(OSString,
5534                                             iter->getNextObject()))))) {
5535                     match = (0 != dict->getObject(nextKey));
5536                     if( match)
5537                         break;
5538                     nextKey = 0;
5539                 }
5540                 dict->release();
5541                 if( iter)
5542                     iter->release();
5543             }
5544             if ((!match) || (done == count)) break;
5545         }
5546
5547         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5548         if( str) {
5549             done++;
5550             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5551             match = (this == entry);
5552             if( entry)
5553                 entry->release();
5554             if ((!match) || (done == count)) break;
5555         }
5556
5557         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5558         if (num) {
5559             done++;
5560             match = (getRegistryEntryID() == num->unsigned64BitValue());
5561             if ((!match) || (done == count)) break;
5562         }
5563
5564         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5565         if( num)
5566         {
5567             OSIterator *        iter;
5568             IOService *         service = 0;
5569             UInt32              serviceCount = 0;
5570
5571             done++;
5572             iter = getClientIterator();
5573             if( iter) {
5574                 while( (service = (IOService *) iter->getNextObject())) {
5575                     if( kIOServiceInactiveState & service->__state[0])
5576                         continue;
5577                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5578                         continue;
5579                     ++serviceCount;
5580                 }
5581                 iter->release();
5582             }
5583             match = (serviceCount == num->unsigned32BitValue());
5584             if ((!match) || (done == count)) break;
5585         }
5586
5587 #define propMatch(key)                                  \
5588         obj = table->getObject(key);                    \
5589         if (obj)                                        \
5590         {                                               \
5591             OSObject * prop;                            \
5592             done++;                                     \
5593             prop = copyProperty(key);                   \
5594             match = obj->isEqualTo(prop);               \
5595             if (prop) prop->release();                  \
5596             if ((!match) || (done == count)) break;     \
5597         }
5598         propMatch(gIOBSDNameKey)
5599         propMatch(gIOBSDMajorKey)
5600         propMatch(gIOBSDMinorKey)
5601         propMatch(gIOBSDUnitKey)
5602 #undef propMatch
5603     }
5604     while (false);
5605
5606     if (did) *did = done;
5607     return (match);
5608 }
5609
5610 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5611 {
5612     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5613 }
5614
5615 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5616 {
5617     IOService *         where;
5618     OSDictionary *      nextTable;
5619     SInt32              score;
5620     OSNumber *          newPri;
5621     bool                match = true;
5622     bool                matchParent = false;
5623     uint32_t            count;
5624     uint32_t            done;
5625
5626     assert( table );
5627
5628 #if !CONFIG_EMBEDDED
5629     OSArray* aliasServiceRegIds = NULL;
5630     IOService* foundAlternateService = NULL;
5631 #endif
5632
5633 #if MATCH_DEBUG
5634     OSDictionary * root = table;
5635 #endif
5636
5637     where = this;
5638     do
5639     {
5640         do
5641         {
5642             count = table->getCount();
5643             if (!(kIOServiceInternalDone & options))
5644             {
5645                 match = where->matchInternal(table, options, &done);
5646                 // don't call family if we've done all the entries in the table
5647                 if ((!match) || (done == count)) break;
5648             }
5649
5650             // pass in score from property table
5651             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5652
5653             // do family specific matching
5654             match = where->matchPropertyTable( table, &score );
5655
5656             if( !match) {
5657 #if IOMATCHDEBUG
5658                 if( kIOLogMatch & getDebugFlags( table ))
5659                     LOG("%s: family specific matching fails\n", where->getName());
5660 #endif
5661                 break;
5662             }
5663
5664             if (kIOServiceChangesOK & options) {
5665                 // save the score
5666                 newPri = OSNumber::withNumber( score, 32 );
5667                 if( newPri) {
5668                     table->setObject( gIOProbeScoreKey, newPri );
5669                     newPri->release();
5670                 }
5671             }
5672
5673             options = 0;
5674             matchParent = false;
5675
5676             nextTable = OSDynamicCast(OSDictionary,
5677                   table->getObject( gIOParentMatchKey ));
5678             if( nextTable) {
5679                 // look for a matching entry anywhere up to root
5680                 match = false;
5681                 matchParent = true;
5682                 table = nextTable;
5683                 break;
5684             }
5685
5686             table = OSDynamicCast(OSDictionary,
5687                     table->getObject( gIOLocationMatchKey ));
5688             if (table) {
5689                 // look for a matching entry at matchLocation()
5690                 match = false;
5691                 where = where->getProvider();
5692                 if (where && (where = where->matchLocation(where))) continue;
5693             }
5694             break;
5695         }
5696         while (true);
5697
5698         if(match == true) {
5699             break;
5700         }
5701
5702         if(matchParent == true) {
5703 #if !CONFIG_EMBEDDED
5704             // check if service has an alias to search its other "parents" if a parent match isn't found
5705             OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
5706             OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
5707             if(alternateRegistryID != NULL) {
5708                 if(aliasServiceRegIds == NULL)
5709                 {
5710                     aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
5711                 }
5712                 aliasServiceRegIds->setObject(alternateRegistryID);
5713             }
5714             OSSafeReleaseNULL(prop);
5715 #endif
5716         }
5717         else {
5718             break;
5719         }
5720
5721         where = where->getProvider();
5722 #if !CONFIG_EMBEDDED
5723         if(where == NULL) {
5724             // there were no matching parent services, check to see if there are aliased services that have a matching parent
5725             if(aliasServiceRegIds != NULL) {
5726                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
5727                 if(numAliasedServices != 0) {
5728                     OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
5729                     if(alternateRegistryID != NULL) {
5730                         OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
5731                         aliasServiceRegIds->removeObject(numAliasedServices - 1);
5732                         if(alternateMatchingDict != NULL) {
5733                             OSSafeReleaseNULL(foundAlternateService);
5734                             foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
5735                             alternateMatchingDict->release();
5736                             if(foundAlternateService != NULL) {
5737                                 where = foundAlternateService;
5738                             }
5739                         }
5740                     }
5741                 }
5742             }
5743         }
5744 #endif
5745     }
5746     while( where != NULL );
5747
5748 #if !CONFIG_EMBEDDED
5749     OSSafeReleaseNULL(foundAlternateService);
5750     OSSafeReleaseNULL(aliasServiceRegIds);
5751 #endif
5752
5753 #if MATCH_DEBUG
5754     if (where != this)
5755     {
5756         OSSerialize * s = OSSerialize::withCapacity(128);
5757         root->serialize(s);
5758         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5759         s->release();
5760     }
5761 #endif
5762
5763     return( match );
5764 }
5765
5766
5767 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5768                                     UInt32 type,  OSDictionary * properties,
5769                                     IOUserClient ** handler )
5770 {
5771     const OSSymbol *userClientClass = 0;
5772     IOUserClient *client;
5773     OSObject *prop;
5774     OSObject *temp;
5775
5776     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5777         return kIOReturnSuccess;
5778
5779     // First try my own properties for a user client class name
5780     prop = copyProperty(gIOUserClientClassKey);
5781     if (prop) {
5782         if (OSDynamicCast(OSSymbol, prop))
5783             userClientClass = (const OSSymbol *) prop;
5784         else if (OSDynamicCast(OSString, prop)) {
5785             userClientClass = OSSymbol::withString((OSString *) prop);
5786             if (userClientClass)
5787                 setProperty(gIOUserClientClassKey,
5788                             (OSObject *) userClientClass);
5789         }
5790     }
5791
5792     // Didn't find one so lets just bomb out now without further ado.
5793     if (!userClientClass)
5794     {
5795         OSSafeReleaseNULL(prop);
5796         return kIOReturnUnsupported;
5797     }
5798
5799     // This reference is consumed by the IOServiceOpen call
5800     temp = OSMetaClass::allocClassWithName(userClientClass);
5801     OSSafeReleaseNULL(prop);
5802     if (!temp)
5803         return kIOReturnNoMemory;
5804
5805     if (OSDynamicCast(IOUserClient, temp))
5806         client = (IOUserClient *) temp;
5807     else {
5808         temp->release();
5809         return kIOReturnUnsupported;
5810     }
5811
5812     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5813         client->release();
5814         return kIOReturnBadArgument;
5815     }
5816
5817     if ( !client->attach(this) ) {
5818         client->release();
5819         return kIOReturnUnsupported;
5820     }
5821
5822     if ( !client->start(this) ) {
5823         client->detach(this);
5824         client->release();
5825         return kIOReturnUnsupported;
5826     }
5827
5828     *handler = client;
5829     return kIOReturnSuccess;
5830 }
5831
5832 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5833                                     UInt32 type, IOUserClient ** handler )
5834 {
5835     return( kIOReturnUnsupported );
5836 }
5837
5838 IOReturn IOService::requestProbe( IOOptionBits options )
5839 {
5840     return( kIOReturnUnsupported);
5841 }
5842
5843 /*
5844  * Convert an IOReturn to text. Subclasses which add additional
5845  * IOReturn's should override this method and call
5846  * super::stringFromReturn if the desired value is not found.
5847  */
5848
5849 const char * IOService::stringFromReturn( IOReturn rtn )
5850 {
5851     static const IONamedValue IOReturn_values[] = {
5852         {kIOReturnSuccess,          "success"                           },
5853         {kIOReturnError,            "general error"                     },
5854         {kIOReturnNoMemory,         "memory allocation error"           },
5855         {kIOReturnNoResources,      "resource shortage"                 },
5856         {kIOReturnIPCError,         "Mach IPC failure"                  },
5857         {kIOReturnNoDevice,         "no such device"                    },
5858         {kIOReturnNotPrivileged,    "privilege violation"               },
5859         {kIOReturnBadArgument,      "invalid argument"                  },
5860         {kIOReturnLockedRead,       "device is read locked"             },
5861         {kIOReturnLockedWrite,      "device is write locked"            },
5862         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5863         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5864         {kIOReturnUnsupported,      "unsupported function"              },
5865         {kIOReturnVMError,          "virtual memory error"              },
5866         {kIOReturnInternalError,    "internal driver error"             },
5867         {kIOReturnIOError,          "I/O error"                         },
5868         {kIOReturnCannotLock,       "cannot acquire lock"               },
5869         {kIOReturnNotOpen,          "device is not open"                },
5870         {kIOReturnNotReadable,      "device is not readable"            },
5871         {kIOReturnNotWritable,      "device is not writeable"           },
5872         {kIOReturnNotAligned,       "alignment error"                   },
5873         {kIOReturnBadMedia,         "media error"                       },
5874         {kIOReturnStillOpen,        "device is still open"              },
5875         {kIOReturnRLDError,         "rld failure"                       },
5876         {kIOReturnDMAError,         "DMA failure"                       },
5877         {kIOReturnBusy,             "device is busy"                    },
5878         {kIOReturnTimeout,          "I/O timeout"                       },
5879         {kIOReturnOffline,          "device is offline"                 },
5880         {kIOReturnNotReady,         "device is not ready"               },
5881         {kIOReturnNotAttached,      "device/channel is not attached"    },
5882         {kIOReturnNoChannels,       "no DMA channels available"         },
5883         {kIOReturnNoSpace,          "no space for data"                 },
5884         {kIOReturnPortExists,       "device port already exists"        },
5885         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5886         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5887         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5888         {kIOReturnMessageTooLarge,  "message is too large"              },
5889         {kIOReturnNotPermitted,     "operation is not permitted"        },
5890         {kIOReturnNoPower,          "device is without power"           },
5891         {kIOReturnNoMedia,          "media is not present"              },
5892         {kIOReturnUnformattedMedia, "media is not formatted"            },
5893         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5894         {kIOReturnUnderrun,         "data underrun"                     },
5895         {kIOReturnOverrun,          "data overrun"                      },
5896         {kIOReturnDeviceError,      "device error"                      },
5897         {kIOReturnNoCompletion,     "no completion routine"             },
5898         {kIOReturnAborted,          "operation was aborted"             },
5899         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5900         {kIOReturnNotResponding,    "device is not responding"          },
5901         {kIOReturnInvalid,          "unanticipated driver error"        },
5902         {0,                         NULL                                }
5903     };
5904
5905     return IOFindNameForValue(rtn, IOReturn_values);
5906 }
5907
5908 /*
5909  * Convert an IOReturn to an errno.
5910  */
5911 int IOService::errnoFromReturn( IOReturn rtn )
5912 {
5913     if (unix_err(err_get_code(rtn)) == rtn)
5914         return err_get_code(rtn);
5915
5916     switch(rtn) {
5917         // (obvious match)
5918         case kIOReturnSuccess:
5919             return(0);
5920         case kIOReturnNoMemory:
5921             return(ENOMEM);
5922         case kIOReturnNoDevice:
5923             return(ENXIO);
5924         case kIOReturnVMError:
5925             return(EFAULT);
5926         case kIOReturnNotPermitted:
5927             return(EPERM);
5928         case kIOReturnNotPrivileged:
5929             return(EACCES);
5930         case kIOReturnIOError:
5931             return(EIO);
5932         case kIOReturnNotWritable:
5933             return(EROFS);
5934         case kIOReturnBadArgument:
5935             return(EINVAL);
5936         case kIOReturnUnsupported:
5937             return(ENOTSUP);
5938         case kIOReturnBusy:
5939             return(EBUSY);
5940         case kIOReturnNoPower:
5941             return(EPWROFF);
5942         case kIOReturnDeviceError:
5943             return(EDEVERR);
5944         case kIOReturnTimeout:
5945             return(ETIMEDOUT);
5946         case kIOReturnMessageTooLarge:
5947             return(EMSGSIZE);
5948         case kIOReturnNoSpace:
5949             return(ENOSPC);
5950         case kIOReturnCannotLock:
5951             return(ENOLCK);
5952
5953         // (best match)
5954         case kIOReturnBadMessageID:
5955         case kIOReturnNoCompletion:
5956         case kIOReturnNotAligned:
5957             return(EINVAL);
5958         case kIOReturnNotReady:
5959             return(EBUSY);
5960         case kIOReturnRLDError:
5961             return(EBADMACHO);
5962         case kIOReturnPortExists:
5963         case kIOReturnStillOpen:
5964             return(EEXIST);
5965         case kIOReturnExclusiveAccess:
5966         case kIOReturnLockedRead:
5967         case kIOReturnLockedWrite:
5968         case kIOReturnNotOpen:
5969         case kIOReturnNotReadable:
5970             return(EACCES);
5971         case kIOReturnCannotWire:
5972         case kIOReturnNoResources:
5973             return(ENOMEM);
5974         case kIOReturnAborted:
5975         case kIOReturnOffline:
5976         case kIOReturnNotResponding:
5977             return(EBUSY);
5978         case kIOReturnBadMedia:
5979         case kIOReturnNoMedia:
5980         case kIOReturnNotAttached:
5981         case kIOReturnUnformattedMedia:
5982             return(ENXIO); // (media error)
5983         case kIOReturnDMAError:
5984         case kIOReturnOverrun:
5985         case kIOReturnUnderrun:
5986             return(EIO); // (transfer error)
5987         case kIOReturnNoBandwidth:
5988         case kIOReturnNoChannels:
5989         case kIOReturnNoFrames:
5990         case kIOReturnNoInterrupt:
5991             return(EIO); // (hardware error)
5992         case kIOReturnError:
5993         case kIOReturnInternalError:
5994         case kIOReturnInvalid:
5995             return(EIO); // (generic error)
5996         case kIOReturnIPCError:
5997             return(EIO); // (ipc error)
5998         default:
5999             return(EIO); // (all other errors)
6000     }
6001 }
6002
6003 IOReturn IOService::message( UInt32 type, IOService * provider,
6004                                 void * argument )
6005 {
6006     /*
6007      * Generic entry point for calls from the provider.  A return value of
6008      * kIOReturnSuccess indicates that the message was received, and where
6009      * applicable, that it was successful.
6010      */
6011
6012     return kIOReturnUnsupported;
6013 }
6014
6015 /*
6016  * Device memory
6017  */
6018
6019 IOItemCount IOService::getDeviceMemoryCount( void )
6020 {
6021     OSArray *           array;
6022     IOItemCount         count;
6023
6024     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6025     if( array)
6026         count = array->getCount();
6027     else
6028         count = 0;
6029
6030     return( count);
6031 }
6032
6033 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
6034 {
6035     OSArray *           array;
6036     IODeviceMemory *    range;
6037
6038     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6039     if( array)
6040         range = (IODeviceMemory *) array->getObject( index );
6041     else
6042         range = 0;
6043
6044     return( range);
6045 }
6046
6047 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
6048                                                 IOOptionBits options )
6049 {
6050     IODeviceMemory *    range;
6051     IOMemoryMap *       map;
6052
6053     range = getDeviceMemoryWithIndex( index );
6054     if( range)
6055         map = range->map( options );
6056     else
6057         map = 0;
6058
6059     return( map );
6060 }
6061
6062 OSArray * IOService::getDeviceMemory( void )
6063 {
6064     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
6065 }
6066
6067
6068 void IOService::setDeviceMemory( OSArray * array )
6069 {
6070     setProperty( gIODeviceMemoryKey, array);
6071 }
6072
6073 /*
6074  * For machines where the transfers on an I/O bus can stall because
6075  * the CPU is in an idle mode, These APIs allow a driver to specify
6076  * the maximum bus stall that they can handle.  0 indicates no limit.
6077  */
6078 void IOService::
6079 setCPUSnoopDelay(UInt32 __unused ns)
6080 {
6081 #if defined(__i386__) || defined(__x86_64__)
6082     ml_set_maxsnoop(ns);
6083 #endif /* defined(__i386__) || defined(__x86_64__) */
6084 }
6085
6086 UInt32 IOService::
6087 getCPUSnoopDelay()
6088 {
6089 #if defined(__i386__) || defined(__x86_64__)
6090     return ml_get_maxsnoop();
6091 #else
6092     return 0;
6093 #endif /* defined(__i386__) || defined(__x86_64__) */
6094 }
6095
6096 #if defined(__i386__) || defined(__x86_64__)
6097 static void
6098 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
6099 {
6100     static const UInt kNoReplace = -1U; // Must be an illegal index
6101     UInt replace = kNoReplace;
6102     bool setCpuDelay = false;
6103
6104     IORecursiveLockLock(sCpuDelayLock);
6105
6106     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6107     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6108     IOService * holder = NULL;
6109
6110     if (ns) {
6111         const CpuDelayEntry ne = {service, ns, delayType};
6112         holder = service;
6113         // Set maximum delay.
6114         for (UInt i = 0; i < count; i++) {
6115             IOService *thisService = entries[i].fService;
6116             bool sameType = (delayType == entries[i].fDelayType);
6117             if ((service == thisService) && sameType)
6118                 replace = i;
6119             else if (!thisService) {
6120                 if (kNoReplace == replace)
6121                     replace = i;
6122             }
6123             else if (sameType) {
6124                 const UInt32 thisMax = entries[i].fMaxDelay;
6125                 if (thisMax < ns)
6126                 {
6127                     ns = thisMax;
6128                     holder = thisService;
6129                 }
6130             }
6131         }
6132
6133         setCpuDelay = true;
6134         if (kNoReplace == replace)
6135             sCpuDelayData->appendBytes(&ne, sizeof(ne));
6136         else
6137             entries[replace] = ne;
6138     }
6139     else {
6140         ns = -1U;       // Set to max unsigned, i.e. no restriction
6141
6142         for (UInt i = 0; i < count; i++) {
6143             // Clear a maximum delay.
6144             IOService *thisService = entries[i].fService;
6145             if (thisService && (delayType == entries[i].fDelayType)) {
6146                 UInt32 thisMax = entries[i].fMaxDelay;
6147                 if (service == thisService)
6148                     replace = i;
6149                 else if (thisMax < ns) {
6150                     ns = thisMax;
6151                     holder = thisService;
6152                 }
6153             }
6154         }
6155
6156         // Check if entry found
6157         if (kNoReplace != replace) {
6158             entries[replace].fService = 0;      // Null the entry
6159             setCpuDelay = true;
6160         }
6161     }
6162
6163     if (setCpuDelay)
6164     {
6165         if (holder && debug_boot_arg) {
6166             strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
6167         }
6168
6169         // Must be safe to call from locked context
6170         if (delayType == kCpuDelayBusStall)
6171         {
6172             ml_set_maxbusdelay(ns);
6173         }
6174         else if (delayType == kCpuDelayInterrupt)
6175         {
6176             ml_set_maxintdelay(ns);
6177         }
6178         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
6179         sCPULatencySet   [delayType]->setValue(ns);
6180
6181         OSArray * handlers = sCpuLatencyHandlers[delayType];
6182         IOService * target;
6183         if (handlers) for (unsigned int idx = 0;
6184                             (target = (IOService *) handlers->getObject(idx));
6185                             idx++)
6186         {
6187             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6188                                             (void *) (uintptr_t) ns, holder,
6189                                             NULL, NULL);
6190         }
6191     }
6192
6193     IORecursiveLockUnlock(sCpuDelayLock);
6194 }
6195
6196 static IOReturn
6197 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
6198 {
6199     IOReturn result = kIOReturnNotFound;
6200     OSArray * array;
6201     unsigned int idx;
6202
6203     IORecursiveLockLock(sCpuDelayLock);
6204
6205     do
6206     {
6207         if (enable && !sCpuLatencyHandlers[delayType])
6208             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
6209         array = sCpuLatencyHandlers[delayType];
6210         if (!array)
6211             break;
6212         idx = array->getNextIndexOfObject(target, 0);
6213         if (!enable)
6214         {
6215             if (-1U != idx)
6216             {
6217                 array->removeObject(idx);
6218                 result = kIOReturnSuccess;
6219             }
6220         }
6221         else
6222         {
6223             if (-1U != idx) {
6224                 result = kIOReturnExclusiveAccess;
6225                 break;
6226             }
6227             array->setObject(target);
6228
6229             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6230             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6231             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
6232             IOService * holder = NULL;
6233
6234             for (UInt i = 0; i < count; i++) {
6235                 if (entries[i].fService
6236                   && (delayType == entries[i].fDelayType)
6237                   && (entries[i].fMaxDelay < ns)) {
6238                     ns = entries[i].fMaxDelay;
6239                     holder = entries[i].fService;
6240                 }
6241             }
6242             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6243                                             (void *) (uintptr_t) ns, holder,
6244                                             NULL, NULL);
6245             result = kIOReturnSuccess;
6246         }
6247     }
6248     while (false);
6249
6250     IORecursiveLockUnlock(sCpuDelayLock);
6251
6252     return (result);
6253 }
6254
6255 #endif /* defined(__i386__) || defined(__x86_64__) */
6256
6257 void IOService::
6258 requireMaxBusStall(UInt32 __unused ns)
6259 {
6260 #if defined(__i386__) || defined(__x86_64__)
6261     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
6262 #endif
6263 }
6264
6265 void IOService::
6266 requireMaxInterruptDelay(uint32_t __unused ns)
6267 {
6268 #if defined(__i386__) || defined(__x86_64__)
6269     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
6270 #endif
6271 }
6272
6273 /*
6274  * Device interrupts
6275  */
6276
6277 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
6278 {
6279   IOInterruptController *interruptController;
6280   OSArray               *array;
6281   OSData                *data;
6282   OSSymbol              *interruptControllerName;
6283   long                  numSources;
6284   IOInterruptSource     *interruptSources;
6285
6286   // Get the parents list from the nub.
6287   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
6288   if (array == 0) return kIOReturnNoResources;
6289
6290   // Allocate space for the IOInterruptSources if needed... then return early.
6291   if (nub->_interruptSources == 0) {
6292     numSources = array->getCount();
6293     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
6294     if (interruptSources == 0) return kIOReturnNoMemory;
6295
6296     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
6297
6298     nub->_numInterruptSources = numSources;
6299     nub->_interruptSources = interruptSources;
6300     return kIOReturnSuccess;
6301   }
6302
6303   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
6304   if (interruptControllerName == 0) return kIOReturnNoResources;
6305
6306   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
6307   if (interruptController == 0) return kIOReturnNoResources;
6308
6309   // Get the interrupt numbers from the nub.
6310   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
6311   if (array == 0) return kIOReturnNoResources;
6312   data = OSDynamicCast(OSData, array->getObject(source));
6313   if (data == 0) return kIOReturnNoResources;
6314
6315   // Set the interruptController and interruptSource in the nub's table.
6316   interruptSources = nub->_interruptSources;
6317   interruptSources[source].interruptController = interruptController;
6318   interruptSources[source].vectorData = data;
6319
6320   return kIOReturnSuccess;
6321 }
6322
6323 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
6324 {
6325   IOReturn ret;
6326
6327   /* Make sure the _interruptSources are set */
6328   if (_interruptSources == 0) {
6329     ret = resolveInterrupt(this, source);
6330     if (ret != kIOReturnSuccess) return ret;
6331   }
6332
6333   /* Make sure the local source number is valid */
6334   if ((source < 0) || (source >= _numInterruptSources))
6335     return kIOReturnNoInterrupt;
6336
6337   /* Look up the contoller for the local source */
6338   *interruptController = _interruptSources[source].interruptController;
6339
6340   if (*interruptController == NULL) {
6341     if (!resolve) return kIOReturnNoInterrupt;
6342
6343     /* Try to reslove the interrupt */
6344     ret = resolveInterrupt(this, source);
6345     if (ret != kIOReturnSuccess) return ret;
6346
6347     *interruptController = _interruptSources[source].interruptController;
6348   }
6349
6350   return kIOReturnSuccess;
6351 }
6352
6353 IOReturn IOService::registerInterrupt(int source, OSObject *target,
6354                                       IOInterruptAction handler,
6355                                       void *refCon)
6356 {
6357   IOInterruptController *interruptController;
6358   IOReturn              ret;
6359
6360   ret = lookupInterrupt(source, true, &interruptController);
6361   if (ret != kIOReturnSuccess) return ret;
6362
6363   /* Register the source */
6364   return interruptController->registerInterrupt(this, source, target,
6365                                                 (IOInterruptHandler)handler,
6366                                                 refCon);
6367 }
6368
6369 IOReturn IOService::unregisterInterrupt(int source)
6370 {
6371   IOInterruptController *interruptController;
6372   IOReturn              ret;
6373
6374   ret = lookupInterrupt(source, false, &interruptController);
6375   if (ret != kIOReturnSuccess) return ret;
6376
6377   /* Unregister the source */
6378   return interruptController->unregisterInterrupt(this, source);
6379 }
6380
6381 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
6382 {
6383   IOReportLegend * legend = NULL;
6384   IOInterruptAccountingData * oldValue = NULL;
6385   IOInterruptAccountingReporter * newArray = NULL;
6386   char subgroupName[64];
6387   int newArraySize = 0;
6388   int i = 0;
6389
6390   if (source < 0) {
6391     return kIOReturnBadArgument;
6392   }
6393
6394   /*
6395    * We support statistics on a maximum of 256 interrupts per nub; if a nub
6396    * has more than 256 interrupt specifiers associated with it, and tries
6397    * to register a high interrupt index with interrupt accounting, panic.
6398    * Having more than 256 interrupts associated with a single nub is
6399    * probably a sign that something fishy is going on.
6400    */
6401   if (source > IA_INDEX_MAX) {
6402     panic("addInterruptStatistics called for an excessively large index (%d)", source);
6403   }
6404
6405   /*
6406    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
6407    * leaving it as is because the likelihood of contention where we are
6408    * actually growing the array is minimal (we would realistically need
6409    * to be starting a driver for the first time, with an IOReporting
6410    * client already in place).  Nonetheless, cleanup that can be done
6411    * to adhere to best practices; it'll make the code more complicated,
6412    * unfortunately.
6413    */
6414   IOLockLock(reserved->interruptStatisticsLock);
6415
6416   /*
6417    * Lazily allocate the statistics array.
6418    */
6419   if (!reserved->interruptStatisticsArray) {
6420     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
6421     assert(reserved->interruptStatisticsArray);
6422     reserved->interruptStatisticsArrayCount = 1;
6423     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
6424   }
6425
6426   if (source >= reserved->interruptStatisticsArrayCount) {
6427     /*
6428      * We're still within the range of supported indices, but we are out
6429      * of space in the current array.  Do a nasty realloc (because
6430      * IORealloc isn't a thing) here.  We'll double the size with each
6431      * reallocation.
6432      *
6433      * Yes, the "next power of 2" could be more efficient; but this will
6434      * be invoked incredibly rarely.  Who cares.
6435      */
6436     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
6437
6438     while (newArraySize <= source)
6439       newArraySize = (newArraySize << 1);
6440     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
6441
6442     assert(newArray);
6443
6444     /*
6445      * TODO: This even zeroes the memory it is about to overwrite.
6446      * Shameful; fix it.  Not particularly high impact, however.
6447      */
6448     bzero(newArray, newArraySize * sizeof(*newArray));
6449     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
6450     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
6451     reserved->interruptStatisticsArray = newArray;
6452     reserved->interruptStatisticsArrayCount = newArraySize;
6453   }
6454
6455   if (!reserved->interruptStatisticsArray[source].reporter) {
6456     /*
6457      * We don't have a reporter associated with this index yet, so we
6458      * need to create one.
6459      */
6460     /*
6461      * TODO: Some statistics do in fact have common units (time); should this be
6462      * split into separate reporters to communicate this?
6463      */
6464      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
6465
6466     /*
6467      * Each statistic is given an identifier based on the interrupt index (which
6468      * should be unique relative to any single nub) and the statistic involved.
6469      * We should now have a sane (small and positive) index, so start
6470      * constructing the channels for statistics.
6471      */
6472     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
6473       /*
6474        * TODO: Currently, this does not add channels for disabled statistics.
6475        * Will this be confusing for clients?  If so, we should just add the
6476        * channels; we can avoid updating the channels even if they exist.
6477        */
6478       if (IA_GET_STATISTIC_ENABLED(i))
6479         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
6480     }
6481
6482     /*
6483      * We now need to add the legend for this reporter to the registry.
6484      */
6485     OSObject * prop = copyProperty(kIOReportLegendKey);
6486     legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
6487     OSSafeReleaseNULL(prop);
6488
6489     /*
6490      * Note that while we compose the subgroup name, we do not need to
6491      * manage its lifecycle (the reporter will handle this).
6492      */
6493     snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
6494     subgroupName[sizeof(subgroupName) - 1] = 0;
6495     legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
6496     setProperty(kIOReportLegendKey, legend->getLegend());
6497     legend->release();
6498
6499     /*
6500      * TODO: Is this a good idea?  Probably not; my assumption is it opts
6501      * all entities who register interrupts into public disclosure of all
6502      * IOReporting channels.  Unfortunately, this appears to be as fine
6503      * grain as it gets.
6504      */
6505     setProperty(kIOReportLegendPublicKey, true);
6506   }
6507
6508   /*
6509    * Don't stomp existing entries.  If we are about to, panic; this
6510    * probably means we failed to tear down our old interrupt source
6511    * correctly.
6512    */
6513   oldValue = reserved->interruptStatisticsArray[source].statistics;
6514
6515   if (oldValue) {
6516     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
6517   }
6518
6519   reserved->interruptStatisticsArray[source].statistics = statistics;
6520
6521   /*
6522    * Inherit the reporter values for each statistic.  The target may
6523    * be torn down as part of the runtime of the service (especially
6524    * for sleep/wake), so we inherit in order to avoid having values
6525    * reset for no apparent reason.  Our statistics are ultimately
6526    * tied to the index and the sevice, not to an individual target,
6527    * so we should maintain them accordingly.
6528    */
6529   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6530
6531   IOLockUnlock(reserved->interruptStatisticsLock);
6532
6533   return kIOReturnSuccess;
6534 }
6535
6536 IOReturn IOService::removeInterruptStatistics(int source)
6537 {
6538   IOInterruptAccountingData * value = NULL;
6539
6540   if (source < 0) {
6541     return kIOReturnBadArgument;
6542   }
6543
6544   IOLockLock(reserved->interruptStatisticsLock);
6545
6546   /*
6547    * We dynamically grow the statistics array, so an excessively
6548    * large index value has NEVER been registered.  This either
6549    * means our cap on the array size is too small (unlikely), or
6550    * that we have been passed a corrupt index (this must be passed
6551    * the plain index into the interrupt specifier list).
6552    */
6553   if (source >= reserved->interruptStatisticsArrayCount) {
6554     panic("removeInterruptStatistics called for index %d, which was never registered", source);
6555   }
6556
6557   assert(reserved->interruptStatisticsArray);
6558
6559   /*
6560    * If there is no existing entry, we are most likely trying to
6561    * free an interrupt owner twice, or we have corrupted the
6562    * index value.
6563    */
6564   value = reserved->interruptStatisticsArray[source].statistics;
6565
6566   if (!value) {
6567     panic("removeInterruptStatistics called for empty index %d", source);
6568   }
6569
6570   /*
6571    * We update the statistics, so that any delta with the reporter
6572    * state is not lost.
6573    */
6574   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6575   reserved->interruptStatisticsArray[source].statistics = NULL;
6576   IOLockUnlock(reserved->interruptStatisticsLock);
6577
6578   return kIOReturnSuccess;
6579 }
6580
6581 IOReturn IOService::getInterruptType(int source, int *interruptType)
6582 {
6583   IOInterruptController *interruptController;
6584   IOReturn              ret;
6585
6586   ret = lookupInterrupt(source, true, &interruptController);
6587   if (ret != kIOReturnSuccess) return ret;
6588
6589   /* Return the type */
6590   return interruptController->getInterruptType(this, source, interruptType);
6591 }
6592
6593 IOReturn IOService::enableInterrupt(int source)
6594 {
6595   IOInterruptController *interruptController;
6596   IOReturn              ret;
6597
6598   ret = lookupInterrupt(source, false, &interruptController);
6599   if (ret != kIOReturnSuccess) return ret;
6600
6601   /* Enable the source */
6602   return interruptController->enableInterrupt(this, source);
6603 }
6604
6605 IOReturn IOService::disableInterrupt(int source)
6606 {
6607   IOInterruptController *interruptController;
6608   IOReturn              ret;
6609
6610   ret = lookupInterrupt(source, false, &interruptController);
6611   if (ret != kIOReturnSuccess) return ret;
6612
6613   /* Disable the source */
6614   return interruptController->disableInterrupt(this, source);
6615 }
6616
6617 IOReturn IOService::causeInterrupt(int source)
6618 {
6619   IOInterruptController *interruptController;
6620   IOReturn              ret;
6621
6622   ret = lookupInterrupt(source, false, &interruptController);
6623   if (ret != kIOReturnSuccess) return ret;
6624
6625   /* Cause an interrupt for the source */
6626   return interruptController->causeInterrupt(this, source);
6627 }
6628
6629 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6630                                     IOReportConfigureAction action,
6631                                     void                   *result,
6632                                     void                   *destination)
6633 {
6634     unsigned cnt;
6635
6636     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6637         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6638             if (pwrMgt) configurePowerStatesReport(action, result);
6639             else  return kIOReturnUnsupported;
6640         }
6641         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6642             if (pwrMgt) configureSimplePowerReport(action, result);
6643             else  return kIOReturnUnsupported;
6644         }
6645     }
6646
6647     IOLockLock(reserved->interruptStatisticsLock);
6648
6649     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6650     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6651         if (reserved->interruptStatisticsArray[cnt].reporter) {
6652             /*
6653              * If the reporter is currently associated with the statistics
6654              * for an event source, we may need to update the reporter.
6655              */
6656             if (reserved->interruptStatisticsArray[cnt].statistics)
6657                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6658
6659             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6660         }
6661     }
6662
6663     IOLockUnlock(reserved->interruptStatisticsLock);
6664
6665     return kIOReturnSuccess;
6666 }
6667
6668 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6669                                  IOReportUpdateAction      action,
6670                                  void                     *result,
6671                                  void                     *destination)
6672 {
6673     unsigned cnt;
6674
6675     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6676         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6677             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6678             else return kIOReturnUnsupported;
6679         }
6680         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6681             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6682             else return kIOReturnUnsupported;
6683         }
6684     }
6685
6686     IOLockLock(reserved->interruptStatisticsLock);
6687
6688     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6689     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6690         if (reserved->interruptStatisticsArray[cnt].reporter) {
6691             /*
6692              * If the reporter is currently associated with the statistics
6693              * for an event source, we need to update the reporter.
6694              */
6695             if (reserved->interruptStatisticsArray[cnt].statistics)
6696                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6697
6698             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6699         }
6700     }
6701
6702     IOLockUnlock(reserved->interruptStatisticsLock);
6703
6704     return kIOReturnSuccess;
6705 }
6706
6707 uint64_t IOService::getAuthorizationID( void )
6708 {
6709     return reserved->authorizationID;
6710 }
6711
6712 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6713 {
6714     OSObject * entitlement;
6715     IOReturn status;
6716
6717     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6718
6719     if ( entitlement )
6720     {
6721         if ( entitlement == kOSBooleanTrue )
6722         {
6723             reserved->authorizationID = authorizationID;
6724
6725             status = kIOReturnSuccess;
6726         }
6727         else
6728         {
6729             status = kIOReturnNotPrivileged;
6730         }
6731
6732         entitlement->release( );
6733     }
6734     else
6735     {
6736         status = kIOReturnNotPrivileged;
6737     }
6738
6739     return status;
6740 }
6741
6742 #if __LP64__
6743 OSMetaClassDefineReservedUsed(IOService, 0);
6744 OSMetaClassDefineReservedUsed(IOService, 1);
6745 OSMetaClassDefineReservedUnused(IOService, 2);
6746 OSMetaClassDefineReservedUnused(IOService, 3);
6747 OSMetaClassDefineReservedUnused(IOService, 4);
6748 OSMetaClassDefineReservedUnused(IOService, 5);
6749 OSMetaClassDefineReservedUnused(IOService, 6);
6750 OSMetaClassDefineReservedUnused(IOService, 7);
6751 #else
6752 OSMetaClassDefineReservedUsed(IOService, 0);
6753 OSMetaClassDefineReservedUsed(IOService, 1);
6754 OSMetaClassDefineReservedUsed(IOService, 2);
6755 OSMetaClassDefineReservedUsed(IOService, 3);
6756 OSMetaClassDefineReservedUsed(IOService, 4);
6757 OSMetaClassDefineReservedUsed(IOService, 5);
6758 OSMetaClassDefineReservedUsed(IOService, 6);
6759 OSMetaClassDefineReservedUsed(IOService, 7);
6760 #endif
6761 OSMetaClassDefineReservedUnused(IOService, 8);
6762 OSMetaClassDefineReservedUnused(IOService, 9);
6763 OSMetaClassDefineReservedUnused(IOService, 10);
6764 OSMetaClassDefineReservedUnused(IOService, 11);
6765 OSMetaClassDefineReservedUnused(IOService, 12);
6766 OSMetaClassDefineReservedUnused(IOService, 13);
6767 OSMetaClassDefineReservedUnused(IOService, 14);
6768 OSMetaClassDefineReservedUnused(IOService, 15);
6769 OSMetaClassDefineReservedUnused(IOService, 16);
6770 OSMetaClassDefineReservedUnused(IOService, 17);
6771 OSMetaClassDefineReservedUnused(IOService, 18);
6772 OSMetaClassDefineReservedUnused(IOService, 19);
6773 OSMetaClassDefineReservedUnused(IOService, 20);
6774 OSMetaClassDefineReservedUnused(IOService, 21);
6775 OSMetaClassDefineReservedUnused(IOService, 22);
6776 OSMetaClassDefineReservedUnused(IOService, 23);
6777 OSMetaClassDefineReservedUnused(IOService, 24);
6778 OSMetaClassDefineReservedUnused(IOService, 25);
6779 OSMetaClassDefineReservedUnused(IOService, 26);
6780 OSMetaClassDefineReservedUnused(IOService, 27);
6781 OSMetaClassDefineReservedUnused(IOService, 28);
6782 OSMetaClassDefineReservedUnused(IOService, 29);
6783 OSMetaClassDefineReservedUnused(IOService, 30);
6784 OSMetaClassDefineReservedUnused(IOService, 31);
6785 OSMetaClassDefineReservedUnused(IOService, 32);
6786 OSMetaClassDefineReservedUnused(IOService, 33);
6787 OSMetaClassDefineReservedUnused(IOService, 34);
6788 OSMetaClassDefineReservedUnused(IOService, 35);
6789 OSMetaClassDefineReservedUnused(IOService, 36);
6790 OSMetaClassDefineReservedUnused(IOService, 37);
6791 OSMetaClassDefineReservedUnused(IOService, 38);
6792 OSMetaClassDefineReservedUnused(IOService, 39);
6793 OSMetaClassDefineReservedUnused(IOService, 40);
6794 OSMetaClassDefineReservedUnused(IOService, 41);
6795 OSMetaClassDefineReservedUnused(IOService, 42);
6796 OSMetaClassDefineReservedUnused(IOService, 43);
6797 OSMetaClassDefineReservedUnused(IOService, 44);
6798 OSMetaClassDefineReservedUnused(IOService, 45);
6799 OSMetaClassDefineReservedUnused(IOService, 46);
6800 OSMetaClassDefineReservedUnused(IOService, 47);