iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <libkern/Block.h>
  37 #include <IOKit/IOCatalogue.h>
  38 #include <IOKit/IOCommand.h>
  39 #include <IOKit/IODeviceTreeSupport.h>
  40 #include <IOKit/IODeviceMemory.h>
  41 #include <IOKit/IOInterrupts.h>
  42 #include <IOKit/IOInterruptController.h>
  43 #include <IOKit/IOPlatformExpert.h>
  44 #include <IOKit/IOMessage.h>
  45 #include <IOKit/IOLib.h>
  46 #include <IOKit/IOKitKeysPrivate.h>
  47 #include <IOKit/IOBSD.h>
  48 #include <IOKit/IOUserClient.h>
  49 #include <IOKit/IOWorkLoop.h>
  50 #include <IOKit/IOTimeStamp.h>
  51 #include <IOKit/IOHibernatePrivate.h>
  52 #include <IOKit/IOInterruptAccountingPrivate.h>
  53 #include <IOKit/IOKernelReporters.h>
  54 #include <IOKit/AppleKeyStoreInterface.h>
  55 #include <IOKit/pwr_mgt/RootDomain.h>
  56 #include <IOKit/IOCPU.h>
  57 #include <mach/sync_policy.h>
  58 #include <IOKit/assert.h>
  59 #include <sys/errno.h>
  60 #include <sys/kdebug.h>
  61 #include <string.h>
  62
  63 #include <machine/pal_routines.h>
  64
  65 #define LOG kprintf
  66 //#define LOG IOLog
  67 #define MATCH_DEBUG     0
  68 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  69
  70 // disabled since lockForArbitration() can be held externally
  71 #define DEBUG_NOTIFIER_LOCKED   0
  72
  73 #include "IOServicePrivate.h"
  74 #include "IOKitKernelInternal.h"
  75
  76 // take lockForArbitration before LOCKNOTIFY
  77
  78 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  79
  80 #define super IORegistryEntry
  81
  82 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  83
  84 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  85 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  86
  87 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  88
  89 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  90
  91 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  92
  93 OSDefineMetaClassAndStructors(IOResources, IOService)
  94
  95 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  96
  97 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  98
  99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 100
 101 static IOPlatformExpert *       gIOPlatform;
 102 static class IOPMrootDomain *   gIOPMRootDomain;
 103 const IORegistryPlane *         gIOServicePlane;
 104 const IORegistryPlane *         gIOPowerPlane;
 105 const OSSymbol *                gIODeviceMemoryKey;
 106 const OSSymbol *                gIOInterruptControllersKey;
 107 const OSSymbol *                gIOInterruptSpecifiersKey;
 108
 109 const OSSymbol *                gIOResourcesKey;
 110 const OSSymbol *                gIOResourceMatchKey;
 111 const OSSymbol *                gIOResourceMatchedKey;
 112 const OSSymbol *                gIOProviderClassKey;
 113 const OSSymbol *                gIONameMatchKey;
 114 const OSSymbol *                gIONameMatchedKey;
 115 const OSSymbol *                gIOPropertyMatchKey;
 116 const OSSymbol *                gIOPropertyExistsMatchKey;
 117 const OSSymbol *                gIOLocationMatchKey;
 118 const OSSymbol *                gIOParentMatchKey;
 119 const OSSymbol *                gIOPathMatchKey;
 120 const OSSymbol *                gIOMatchCategoryKey;
 121 const OSSymbol *                gIODefaultMatchCategoryKey;
 122 const OSSymbol *                gIOMatchedServiceCountKey;
 123 #if !CONFIG_EMBEDDED
 124 const OSSymbol *                gIOServiceLegacyMatchingRegistryIDKey;
 125 #endif
 126
 127 const OSSymbol *                gIOMapperIDKey;
 128 const OSSymbol *                gIOUserClientClassKey;
 129 const OSSymbol *                gIOKitDebugKey;
 130
 131 const OSSymbol *                gIOCommandPoolSizeKey;
 132
 133 const OSSymbol *                gIOConsoleLockedKey;
 134 const OSSymbol *                gIOConsoleUsersKey;
 135 const OSSymbol *                gIOConsoleSessionUIDKey;
 136 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 137 const OSSymbol *                gIOConsoleUsersSeedKey;
 138 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 139 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 140 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 141 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 142 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 143 clock_sec_t                     gIOConsoleLockTime;
 144 static bool                     gIOConsoleLoggedIn;
 145 #if HIBERNATION
 146 static OSBoolean *              gIOConsoleBooterLockState;
 147 static uint32_t                 gIOScreenLockState;
 148 #endif
 149 static IORegistryEntry *        gIOChosenEntry;
 150
 151 static int                      gIOResourceGenerationCount;
 152
 153 const OSSymbol *                gIOServiceKey;
 154 const OSSymbol *                gIOPublishNotification;
 155 const OSSymbol *                gIOFirstPublishNotification;
 156 const OSSymbol *                gIOMatchedNotification;
 157 const OSSymbol *                gIOFirstMatchNotification;
 158 const OSSymbol *                gIOTerminatedNotification;
 159 const OSSymbol *                gIOWillTerminateNotification;
 160
 161 const OSSymbol *                gIOGeneralInterest;
 162 const OSSymbol *                gIOBusyInterest;
 163 const OSSymbol *                gIOAppPowerStateInterest;
 164 const OSSymbol *                gIOPriorityPowerStateInterest;
 165 const OSSymbol *                gIOConsoleSecurityInterest;
 166
 167 const OSSymbol *                gIOBSDKey;
 168 const OSSymbol *                gIOBSDNameKey;
 169 const OSSymbol *                gIOBSDMajorKey;
 170 const OSSymbol *                gIOBSDMinorKey;
 171 const OSSymbol *                gIOBSDUnitKey;
 172
 173 const  OSSymbol *               gAKSGetKey;
 174 #if defined(__i386__) || defined(__x86_64__)
 175 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 176 #endif
 177
 178 static OSDictionary *           gNotifications;
 179 static IORecursiveLock *        gNotificationLock;
 180
 181 static IOService *              gIOResources;
 182 static IOService *              gIOServiceRoot;
 183
 184 static OSOrderedSet *           gJobs;
 185 static semaphore_port_t         gJobsSemaphore;
 186 static IOLock *                 gJobsLock;
 187 static int                      gOutstandingJobs;
 188 static int                      gNumConfigThreads;
 189 static int                      gNumWaitingThreads;
 190 static IOLock *                 gIOServiceBusyLock;
 191 bool                            gCPUsRunning;
 192
 193 static thread_t                 gIOTerminateThread;
 194 static thread_t                 gIOTerminateWorkerThread;
 195 static UInt32                   gIOTerminateWork;
 196 static OSArray *                gIOTerminatePhase2List;
 197 static OSArray *                gIOStopList;
 198 static OSArray *                gIOStopProviderList;
 199 static OSArray *                gIOFinalizeList;
 200
 201 static SInt32                   gIOConsoleUsersSeed;
 202 static OSData *                 gIOConsoleUsersSeedValue;
 203
 204 extern const OSSymbol *         gIODTPHandleKey;
 205
 206 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 207
 208 static IOLock *                 gIOConsoleUsersLock;
 209 static thread_call_t            gIOConsoleLockCallout;
 210 static IONotifier *             gIOServiceNullNotifier;
 211
 212 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 213
 214 #define LOCKREADNOTIFY()        \
 215     IORecursiveLockLock( gNotificationLock )
 216 #define LOCKWRITENOTIFY()       \
 217     IORecursiveLockLock( gNotificationLock )
 218 #define LOCKWRITE2READNOTIFY()
 219 #define UNLOCKNOTIFY()          \
 220     IORecursiveLockUnlock( gNotificationLock )
 221 #define SLEEPNOTIFY(event) \
 222     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 223 #define SLEEPNOTIFYTO(event, deadline) \
 224     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 225 #define WAKEUPNOTIFY(event) \
 226         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 227
 228 #define randomDelay()   \
 229         int del = read_processor_clock();                               \
 230         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 231         IOSleep( del );
 232
 233 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 234
 235 #define queue_element(entry, element, type, field) do { \
 236         vm_address_t __ele = (vm_address_t) (entry);    \
 237         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 238         (element) = (type) __ele;                       \
 239     } while(0)
 240
 241 #define iterqueue(que, elt)                             \
 242         for (queue_entry_t elt = queue_first(que);      \
 243              !queue_end(que, elt);                      \
 244              elt = queue_next(elt))
 245
 246 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 247
 248 struct IOInterruptAccountingReporter {
 249     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 250     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 251 };
 252
 253 struct ArbitrationLockQueueElement {
 254     queue_chain_t link;
 255     IOThread      thread;
 256     IOService *   service;
 257     unsigned      count;
 258     bool          required;
 259     bool          aborted;
 260 };
 261
 262 static queue_head_t gArbitrationLockQueueActive;
 263 static queue_head_t gArbitrationLockQueueWaiting;
 264 static queue_head_t gArbitrationLockQueueFree;
 265 static IOLock *     gArbitrationLockQueueLock;
 266
 267 bool IOService::isInactive( void ) const
 268     { return( 0 != (kIOServiceInactiveState & getState())); }
 269
 270 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 271
 272 #if defined(__i386__) || defined(__x86_64__)
 273
 274 // Only used by the intel implementation of
 275 //     IOService::requireMaxBusStall(UInt32 ns)
 276 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 277 struct CpuDelayEntry
 278 {
 279     IOService * fService;
 280     UInt32      fMaxDelay;
 281     UInt32      fDelayType;
 282 };
 283
 284 enum {
 285     kCpuDelayBusStall, kCpuDelayInterrupt,
 286     kCpuNumDelayTypes
 287 };
 288
 289 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 290 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 291 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 292 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 293 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 294 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 295 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 296
 297 static void
 298 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 299 static IOReturn
 300 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 301
 302 #endif /* defined(__i386__) || defined(__x86_64__) */
 303
 304 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 305
 306 void IOService::initialize( void )
 307 {
 308     kern_return_t       err;
 309
 310     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 311     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 312
 313     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 314     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 315     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 316     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 317     gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 318     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 319     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 320     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 321
 322     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 323     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 324                                         kIODefaultMatchCategoryKey );
 325     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 326                                         kIOMatchedServiceCountKey );
 327 #if !CONFIG_EMBEDDED
 328     gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
 329                                         kIOServiceLegacyMatchingRegistryIDKey );
 330 #endif
 331
 332     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 333
 334     gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 335     gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 336     gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 337
 338     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 339     gIOInterruptControllersKey
 340         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 341     gIOInterruptSpecifiersKey
 342         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 343
 344     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 345
 346     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 347
 348     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 349
 350     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 351     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 352     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 353     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 354     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 355
 356     gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 357     gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 358     gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 359     gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 360     gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 361
 362     gNotifications              = OSDictionary::withCapacity( 1 );
 363     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 364                                                  kIOPublishNotification );
 365     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 366                                                  kIOFirstPublishNotification );
 367     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 368                                                  kIOMatchedNotification );
 369     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 370                                                  kIOFirstMatchNotification );
 371     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 372                                                  kIOTerminatedNotification );
 373     gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
 374                                                  kIOWillTerminateNotification );
 375     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 376
 377     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 378     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 379     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 380     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 381
 382     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 383     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 384     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 385     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 386     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 387     gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 388
 389     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 390
 391     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 392 #if defined(__i386__) || defined(__x86_64__)
 393     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 394     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 395     uint32_t  idx;
 396     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 397     {
 398         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 399         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 400         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 401     }
 402     gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 403 #endif
 404     gNotificationLock           = IORecursiveLockAlloc();
 405
 406     gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 407
 408     assert( gIOServicePlane && gIODeviceMemoryKey
 409         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 410         && gIOResourcesKey && gNotifications && gNotificationLock
 411         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 412         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 413         && gIOPublishNotification && gIOMatchedNotification
 414         && gIOTerminatedNotification && gIOServiceKey
 415         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 416     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 417         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 418
 419     gJobsLock   = IOLockAlloc();
 420     gJobs       = OSOrderedSet::withCapacity( 10 );
 421
 422     gIOServiceBusyLock = IOLockAlloc();
 423
 424     gIOConsoleUsersLock = IOLockAlloc();
 425
 426     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 427
 428     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 429
 430     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 431
 432     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 433                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 434
 435     gIOResources = IOResources::resources();
 436     assert( gIOResources );
 437
 438     gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 439     assert(gIOServiceNullNotifier);
 440
 441     gArbitrationLockQueueLock = IOLockAlloc();
 442     queue_init(&gArbitrationLockQueueActive);
 443     queue_init(&gArbitrationLockQueueWaiting);
 444     queue_init(&gArbitrationLockQueueFree);
 445
 446     assert( gArbitrationLockQueueLock );
 447
 448     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 449     gIOStopList            = OSArray::withCapacity( 16 );
 450     gIOStopProviderList    = OSArray::withCapacity( 16 );
 451     gIOFinalizeList        = OSArray::withCapacity( 16 );
 452     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 453
 454     // worker thread that is responsible for terminating / cleaning up threads
 455     kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
 456     assert(gIOTerminateWorkerThread);
 457     thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
 458 }
 459
 460 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 461
 462 #if defined(__i386__) || defined(__x86_64__)
 463 extern "C" {
 464
 465 const char *getCpuDelayBusStallHolderName(void);
 466 const char *getCpuDelayBusStallHolderName(void) {
 467     return sCPULatencyHolderName[kCpuDelayBusStall];
 468 }
 469
 470 const char *getCpuInterruptDelayHolderName(void);
 471 const char *getCpuInterruptDelayHolderName(void) {
 472     return sCPULatencyHolderName[kCpuDelayInterrupt];
 473 }
 474
 475 }
 476 #endif
 477
 478 #if IOMATCHDEBUG
 479 static UInt64 getDebugFlags( OSDictionary * props )
 480 {
 481     OSNumber *  debugProp;
 482     UInt64      debugFlags;
 483
 484     debugProp = OSDynamicCast( OSNumber,
 485                 props->getObject( gIOKitDebugKey ));
 486     if( debugProp)
 487         debugFlags = debugProp->unsigned64BitValue();
 488     else
 489         debugFlags = gIOKitDebug;
 490
 491     return( debugFlags );
 492 }
 493
 494 static UInt64 getDebugFlags( IOService * inst )
 495 {
 496     OSObject *  prop;
 497     OSNumber *  debugProp;
 498     UInt64      debugFlags;
 499
 500     prop = inst->copyProperty(gIOKitDebugKey);
 501     debugProp = OSDynamicCast(OSNumber, prop);
 502     if( debugProp)
 503         debugFlags = debugProp->unsigned64BitValue();
 504     else
 505         debugFlags = gIOKitDebug;
 506
 507     OSSafeReleaseNULL(prop);
 508
 509     return( debugFlags );
 510 }
 511 #endif
 512
 513 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 514
 515 // Probe a matched service and return an instance to be started.
 516 // The default score is from the property table, & may be altered
 517 // during probe to change the start order.
 518
 519 IOService * IOService::probe(   IOService * provider,
 520                                 SInt32    * score )
 521 {
 522     return( this );
 523 }
 524
 525 bool IOService::start( IOService * provider )
 526 {
 527     return( true );
 528 }
 529
 530 void IOService::stop( IOService * provider )
 531 {
 532 }
 533
 534 bool IOService::init( OSDictionary * dictionary )
 535 {
 536     bool ret;
 537
 538     ret = super::init(dictionary);
 539     if (!ret)     return (false);
 540     if (reserved) return (true);
 541
 542     reserved = IONew(ExpansionData, 1);
 543     if (!reserved) return (false);
 544     bzero(reserved, sizeof(*reserved));
 545
 546     /*
 547      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 548      * lock based on the presence of specifiers ran into issues as some
 549      * platforms set up the specifiers after IOService initialization.
 550      *
 551      * We may be able to get away with a global lock, as this should only be
 552      * contended by IOReporting clients and driver start/stop (unless a
 553      * driver wants to remove/add handlers in the course of normal operation,
 554      * which should be unlikely).
 555      */
 556     reserved->interruptStatisticsLock = IOLockAlloc();
 557     if (!reserved->interruptStatisticsLock) return (false);
 558
 559     return (true);
 560 }
 561
 562 bool IOService::init( IORegistryEntry * from,
 563                       const IORegistryPlane * inPlane )
 564 {
 565     bool ret;
 566
 567     ret = super::init(from, inPlane);
 568     if (!ret)     return (false);
 569     if (reserved) return (true);
 570
 571     reserved = IONew(ExpansionData, 1);
 572     if (!reserved) return (false);
 573     bzero(reserved, sizeof(*reserved));
 574
 575     /*
 576      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 577      * lock based on the presence of specifiers ran into issues as some
 578      * platforms set up the specifiers after IOService initialization.
 579      *
 580      * We may be able to get away with a global lock, as this should only be
 581      * contended by IOReporting clients and driver start/stop (unless a
 582      * driver wants to remove/add handlers in the course of normal operation,
 583      * which should be unlikely).
 584      */
 585     reserved->interruptStatisticsLock = IOLockAlloc();
 586     if (!reserved->interruptStatisticsLock) return (false);
 587
 588     return (true);
 589 }
 590
 591 void IOService::free( void )
 592 {
 593     int i = 0;
 594     requireMaxBusStall(0);
 595     requireMaxInterruptDelay(0);
 596     if( getPropertyTable())
 597         unregisterAllInterest();
 598     PMfree();
 599
 600     if (reserved) {
 601         if (reserved->interruptStatisticsArray) {
 602             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 603                 if (reserved->interruptStatisticsArray[i].reporter)
 604                     reserved->interruptStatisticsArray[i].reporter->release();
 605             }
 606
 607             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 608         }
 609
 610         if (reserved->interruptStatisticsLock)
 611             IOLockFree(reserved->interruptStatisticsLock);
 612         IODelete(reserved, ExpansionData, 1);
 613     }
 614
 615     if (_numInterruptSources && _interruptSources)
 616     {
 617         for (i = 0; i < _numInterruptSources; i++) {
 618             void * block = _interruptSourcesPrivate(this)[i].vectorBlock;
 619             if (block) Block_release(block);
 620         }
 621         IOFree(_interruptSources,
 622                 _numInterruptSources * sizeofAllIOInterruptSource);
 623         _interruptSources = 0;
 624     }
 625
 626     super::free();
 627 }
 628
 629 /*
 630  * Attach in service plane
 631  */
 632 bool IOService::attach( IOService * provider )
 633 {
 634     bool         ok;
 635     uint32_t     count;
 636     AbsoluteTime deadline;
 637     int          waitResult = THREAD_AWAKENED;
 638     bool         wait, computeDeadline = true;
 639
 640     if( provider) {
 641
 642         if( gIOKitDebug & kIOLogAttach)
 643             LOG( "%s::attach(%s)\n", getName(),
 644                     provider->getName());
 645
 646         ok   = false;
 647         do
 648         {
 649             wait = false;
 650             provider->lockForArbitration();
 651             if (provider->__state[0] & kIOServiceInactiveState) ok = false;
 652             else
 653             {
 654                 count = provider->getChildCount(gIOServicePlane);
 655                 wait = (count > (kIOServiceBusyMax - 4));
 656                 if (!wait) ok = attachToParent(provider, gIOServicePlane);
 657                 else
 658                 {
 659                     IOLog("stalling for detach from %s\n", provider->getName());
 660                     IOLockLock( gIOServiceBusyLock );
 661                     provider->__state[1] |= kIOServiceWaitDetachState;
 662                 }
 663             }
 664             provider->unlockForArbitration();
 665             if (wait)
 666             {
 667                 if (computeDeadline)
 668                 {
 669                     clock_interval_to_deadline(15, kSecondScale, &deadline);
 670                     computeDeadline = false;
 671                 }
 672                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 673                 IOLockUnlock( gIOServiceBusyLock );
 674                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 675                 wait = (waitResult != THREAD_TIMED_OUT);
 676             }
 677         }
 678         while (wait);
 679
 680     } else {
 681         gIOServiceRoot = this;
 682         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 683     }
 684
 685     if (ok && !__provider) (void) getProvider();
 686
 687     return( ok );
 688 }
 689
 690 IOService * IOService::getServiceRoot( void )
 691 {
 692     return( gIOServiceRoot );
 693 }
 694
 695 void IOService::detach( IOService * provider )
 696 {
 697     IOService * newProvider = 0;
 698     SInt32      busy;
 699     bool        adjParent;
 700
 701     if( gIOKitDebug & kIOLogAttach)
 702         LOG("%s::detach(%s)\n", getName(), provider->getName());
 703
 704     lockForArbitration();
 705
 706     uint64_t regID1 = provider->getRegistryEntryID();
 707     uint64_t regID2 = getRegistryEntryID();
 708     IOServiceTrace(
 709         IOSERVICE_DETACH,
 710         (uintptr_t) regID1,
 711         (uintptr_t) (regID1 >> 32),
 712         (uintptr_t) regID2,
 713         (uintptr_t) (regID2 >> 32));
 714
 715     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 716                && (provider == getProvider()));
 717
 718     detachFromParent( provider, gIOServicePlane );
 719
 720     if( busy) {
 721         newProvider = getProvider();
 722         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 723             _adjustBusy( -busy );
 724     }
 725
 726     if (kIOServiceInactiveState & __state[0]) {
 727         getMetaClass()->removeInstance(this);
 728         IORemoveServicePlatformActions(this);
 729     }
 730
 731     unlockForArbitration();
 732
 733     if( newProvider && adjParent) {
 734         newProvider->lockForArbitration();
 735         newProvider->_adjustBusy(1);
 736         newProvider->unlockForArbitration();
 737     }
 738
 739     // check for last client detach from a terminated service
 740     if( provider->lockForArbitration( true ))
 741     {
 742         if (kIOServiceStartState & __state[1])
 743         {
 744             provider->scheduleTerminatePhase2();
 745         }
 746         if( adjParent) provider->_adjustBusy( -1 );
 747         if( (provider->__state[1] & kIOServiceTermPhase3State)
 748          && (0 == provider->getClient())) {
 749             provider->scheduleFinalize(false);
 750         }
 751
 752         IOLockLock( gIOServiceBusyLock );
 753         if (kIOServiceWaitDetachState & provider->__state[1])
 754         {
 755             provider->__state[1] &= ~kIOServiceWaitDetachState;
 756             thread_wakeup(&provider->__provider);
 757         }
 758         IOLockUnlock( gIOServiceBusyLock );
 759
 760         provider->unlockForArbitration();
 761     }
 762 }
 763
 764 /*
 765  * Register instance - publish it for matching
 766  */
 767
 768 void IOService::registerService( IOOptionBits options )
 769 {
 770     char *              pathBuf;
 771     const char *        path;
 772     char *              skip;
 773     int                 len;
 774     enum { kMaxPathLen  = 256 };
 775     enum { kMaxChars    = 63 };
 776
 777     IORegistryEntry * parent = this;
 778     IORegistryEntry * root = getRegistryRoot();
 779     while( parent && (parent != root))
 780         parent = parent->getParentEntry( gIOServicePlane);
 781
 782     if( parent != root) {
 783         IOLog("%s: not registry member at registerService()\n", getName());
 784         return;
 785     }
 786
 787     // Allow the Platform Expert to adjust this node.
 788     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 789         return;
 790
 791     IOInstallServicePlatformActions(this);
 792
 793     if( (this != gIOResources)
 794      && (kIOLogRegister & gIOKitDebug)) {
 795
 796         pathBuf = (char *) IOMalloc( kMaxPathLen );
 797
 798         IOLog( "Registering: " );
 799
 800         len = kMaxPathLen;
 801         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 802
 803             path = pathBuf;
 804             if( len > kMaxChars) {
 805                 IOLog("..");
 806                 len -= kMaxChars;
 807                 path += len;
 808                 if( (skip = strchr( path, '/')))
 809                     path = skip;
 810             }
 811         } else
 812             path = getName();
 813
 814         IOLog( "%s\n", path );
 815
 816         if( pathBuf)
 817             IOFree( pathBuf, kMaxPathLen );
 818     }
 819
 820     startMatching( options );
 821 }
 822
 823 void IOService::startMatching( IOOptionBits options )
 824 {
 825     IOService * provider;
 826     UInt32      prevBusy = 0;
 827     bool        needConfig;
 828     bool        needWake = false;
 829     bool        ok;
 830     bool        sync;
 831     bool        waitAgain;
 832
 833     lockForArbitration();
 834
 835     sync = (options & kIOServiceSynchronous)
 836         || ((provider = getProvider())
 837                 && (provider->__state[1] & kIOServiceSynchronousState));
 838
 839         if ( options & kIOServiceAsynchronous )
 840                 sync = false;
 841
 842     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
 843                && (0 == (__state[0] & kIOServiceInactiveState));
 844
 845     __state[1] |= kIOServiceNeedConfigState;
 846
 847 //    __state[0] &= ~kIOServiceInactiveState;
 848
 849 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 850 //                      OSKernelStackRemaining(), getName());
 851
 852     if( needConfig) {
 853         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 854     }
 855
 856     if( sync)
 857         __state[1] |= kIOServiceSynchronousState;
 858     else
 859         __state[1] &= ~kIOServiceSynchronousState;
 860
 861     if( needConfig) prevBusy = _adjustBusy( 1 );
 862
 863     unlockForArbitration();
 864
 865     if( needConfig) {
 866
 867         if( needWake) {
 868             IOLockLock( gIOServiceBusyLock );
 869             thread_wakeup( (event_t) this/*&__state[1]*/ );
 870             IOLockUnlock( gIOServiceBusyLock );
 871
 872         } else if( !sync || (kIOServiceAsynchronous & options)) {
 873
 874             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 875
 876         } else do {
 877
 878             if( (__state[1] & kIOServiceNeedConfigState))
 879                 doServiceMatch( options );
 880
 881             lockForArbitration();
 882             IOLockLock( gIOServiceBusyLock );
 883
 884             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 885                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 886
 887             if( waitAgain)
 888                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 889             else
 890                 __state[1] &= ~kIOServiceSyncPubState;
 891
 892             unlockForArbitration();
 893
 894             if( waitAgain)
 895                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 896
 897             IOLockUnlock( gIOServiceBusyLock );
 898             if( waitAgain)
 899                 thread_block(THREAD_CONTINUE_NULL);
 900
 901         } while( waitAgain );
 902     }
 903 }
 904
 905 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 906 {
 907     OSDictionary *      table;
 908     OSSet *             set;
 909     OSSet *             allSet = 0;
 910     IOService *         service;
 911 #if IOMATCHDEBUG
 912     SInt32              count = 0;
 913 #endif
 914
 915     newTables->retain();
 916
 917     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 918
 919         LOCKWRITENOTIFY();
 920         set = (OSSet *) copyExistingServices( table,
 921                                                 kIOServiceRegisteredState,
 922                                                 kIOServiceExistingSet);
 923         UNLOCKNOTIFY();
 924         if( set) {
 925
 926 #if IOMATCHDEBUG
 927             count += set->getCount();
 928 #endif
 929             if (allSet) {
 930                 allSet->merge((const OSSet *) set);
 931                 set->release();
 932             }
 933             else
 934                 allSet = set;
 935         }
 936
 937 #if IOMATCHDEBUG
 938         if( getDebugFlags( table ) & kIOLogMatch)
 939             LOG("Matching service count = %ld\n", (long)count);
 940 #endif
 941         newTables->removeObject(table);
 942     }
 943
 944     if (allSet) {
 945         while( (service = (IOService *) allSet->getAnyObject())) {
 946             service->startMatching(kIOServiceAsynchronous);
 947             allSet->removeObject(service);
 948         }
 949         allSet->release();
 950     }
 951
 952     newTables->release();
 953
 954     return( kIOReturnSuccess );
 955 }
 956
 957  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 958                                                 IOOptionBits options )
 959 {
 960     _IOServiceJob *     job;
 961
 962     job = new _IOServiceJob;
 963     if( job && !job->init()) {
 964         job->release();
 965         job = 0;
 966     }
 967
 968     if( job) {
 969         job->type       = type;
 970         job->nub        = nub;
 971         job->options    = options;
 972         nub->retain();                  // thread will release()
 973         pingConfig( job );
 974     }
 975
 976     return( job );
 977 }
 978
 979 /*
 980  * Called on a registered service to see if it matches
 981  * a property table.
 982  */
 983
 984 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 985 {
 986     return( matchPropertyTable(table) );
 987 }
 988
 989 bool IOService::matchPropertyTable( OSDictionary * table )
 990 {
 991     return( true );
 992 }
 993
 994 /*
 995  * Called on a matched service to allocate resources
 996  * before first driver is attached.
 997  */
 998
 999 IOReturn IOService::getResources( void )
1000 {
1001     return( kIOReturnSuccess);
1002 }
1003
1004 /*
1005  * Client/provider accessors
1006  */
1007
1008 IOService * IOService::getProvider( void ) const
1009 {
1010     IOService * self = (IOService *) this;
1011     IOService * parent;
1012     SInt32      generation;
1013
1014     generation = getRegistryEntryGenerationCount();
1015     if( __providerGeneration == generation)
1016         return( __provider );
1017
1018     parent = (IOService *) getParentEntry( gIOServicePlane);
1019     if( parent == IORegistryEntry::getRegistryRoot())
1020         /* root is not an IOService */
1021         parent = 0;
1022
1023     self->__provider = parent;
1024     OSMemoryBarrier();
1025     // save the count from before call to getParentEntry()
1026     self->__providerGeneration = generation;
1027
1028     return( parent );
1029 }
1030
1031 IOWorkLoop * IOService::getWorkLoop() const
1032 {
1033     IOService *provider = getProvider();
1034
1035     if (provider)
1036         return provider->getWorkLoop();
1037     else
1038         return 0;
1039 }
1040
1041 OSIterator * IOService::getProviderIterator( void ) const
1042 {
1043     return( getParentIterator( gIOServicePlane));
1044 }
1045
1046 IOService * IOService::getClient( void ) const
1047 {
1048     return( (IOService *) getChildEntry( gIOServicePlane));
1049 }
1050
1051 OSIterator * IOService::getClientIterator( void ) const
1052 {
1053     return( getChildIterator( gIOServicePlane));
1054 }
1055
1056 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
1057                                                 const IOService * client,
1058                                                 const IOService * provider )
1059 {
1060     _IOOpenServiceIterator * inst;
1061
1062     if( !_iter)
1063         return( 0 );
1064
1065     inst = new _IOOpenServiceIterator;
1066
1067     if( inst && !inst->init()) {
1068         inst->release();
1069         inst = 0;
1070     }
1071     if( inst) {
1072         inst->iter = _iter;
1073         inst->client = client;
1074         inst->provider = provider;
1075     }
1076
1077     return( inst );
1078 }
1079
1080 void _IOOpenServiceIterator::free()
1081 {
1082     iter->release();
1083     if( last)
1084         last->unlockForArbitration();
1085     OSIterator::free();
1086 }
1087
1088 OSObject * _IOOpenServiceIterator::getNextObject()
1089 {
1090     IOService * next;
1091
1092     if( last)
1093         last->unlockForArbitration();
1094
1095     while( (next = (IOService *) iter->getNextObject())) {
1096
1097         next->lockForArbitration();
1098         if( (client && (next->isOpen( client )))
1099          || (provider && (provider->isOpen( next ))) )
1100             break;
1101         next->unlockForArbitration();
1102     }
1103
1104     last = next;
1105
1106     return( next );
1107 }
1108
1109 bool _IOOpenServiceIterator::isValid()
1110 {
1111     return( iter->isValid() );
1112 }
1113
1114 void _IOOpenServiceIterator::reset()
1115 {
1116     if( last) {
1117         last->unlockForArbitration();
1118         last = 0;
1119     }
1120     iter->reset();
1121 }
1122
1123 OSIterator * IOService::getOpenProviderIterator( void ) const
1124 {
1125     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1126 }
1127
1128 OSIterator * IOService::getOpenClientIterator( void ) const
1129 {
1130     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1131 }
1132
1133
1134 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1135                                           bool waitForFunction,
1136                                           void *param1, void *param2,
1137                                           void *param3, void *param4 )
1138 {
1139   IOReturn  result = kIOReturnUnsupported;
1140   IOService *provider;
1141
1142   if (gIOPlatformFunctionHandlerSet == functionName)
1143   {
1144 #if defined(__i386__) || defined(__x86_64__)
1145     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1146     IOService *      target              = (IOService *) param2;
1147     bool             enable              = (param3 != 0);
1148
1149     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1150         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1151     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1152         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1153 #endif /* defined(__i386__) || defined(__x86_64__) */
1154   }
1155
1156   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1157     result = provider->callPlatformFunction(functionName, waitForFunction,
1158                                             param1, param2, param3, param4);
1159   }
1160
1161   return result;
1162 }
1163
1164 IOReturn IOService::callPlatformFunction( const char * functionName,
1165                                           bool waitForFunction,
1166                                           void *param1, void *param2,
1167                                           void *param3, void *param4 )
1168 {
1169   IOReturn result = kIOReturnNoMemory;
1170   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1171
1172   if (functionSymbol != 0) {
1173     result = callPlatformFunction(functionSymbol, waitForFunction,
1174                                   param1, param2, param3, param4);
1175     functionSymbol->release();
1176   }
1177
1178   return result;
1179 }
1180
1181
1182 /*
1183  * Accessors for global services
1184  */
1185
1186 IOPlatformExpert * IOService::getPlatform( void )
1187 {
1188     return( gIOPlatform);
1189 }
1190
1191 class IOPMrootDomain * IOService::getPMRootDomain( void )
1192 {
1193     return( gIOPMRootDomain);
1194 }
1195
1196 IOService * IOService::getResourceService( void )
1197 {
1198     return( gIOResources );
1199 }
1200
1201 void IOService::setPlatform( IOPlatformExpert * platform)
1202 {
1203     gIOPlatform = platform;
1204     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1205
1206 #if defined(__i386__) || defined(__x86_64__)
1207
1208     static const char * keys[kCpuNumDelayTypes] = {
1209         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1210     const OSObject * objs[2];
1211     OSArray * array;
1212     uint32_t  idx;
1213
1214     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1215     {
1216         objs[0] = sCPULatencySet[idx];
1217         objs[1] = sCPULatencyHolder[idx];
1218         array   = OSArray::withObjects(objs, 2);
1219         if (!array) break;
1220         platform->setProperty(keys[idx], array);
1221         array->release();
1222     }
1223 #endif /* defined(__i386__) || defined(__x86_64__) */
1224 }
1225
1226 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1227 {
1228     gIOPMRootDomain = rootDomain;
1229     publishResource("IOKit");
1230 }
1231
1232 /*
1233  * Stacking change
1234  */
1235
1236 bool IOService::lockForArbitration( bool isSuccessRequired )
1237 {
1238     bool                          found;
1239     bool                          success;
1240     ArbitrationLockQueueElement * element;
1241     ArbitrationLockQueueElement * active;
1242     ArbitrationLockQueueElement * waiting;
1243
1244     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1245
1246     // lock global access
1247     IOTakeLock( gArbitrationLockQueueLock );
1248
1249     // obtain an unused queue element
1250     if( !queue_empty( &gArbitrationLockQueueFree )) {
1251         queue_remove_first( &gArbitrationLockQueueFree,
1252                             element,
1253                             ArbitrationLockQueueElement *,
1254                             link );
1255     } else {
1256         element = IONew( ArbitrationLockQueueElement, 1 );
1257         assert( element );
1258     }
1259
1260     // prepare the queue element
1261     element->thread   = IOThreadSelf();
1262     element->service  = this;
1263     element->count    = 1;
1264     element->required = isSuccessRequired;
1265     element->aborted  = false;
1266
1267     // determine whether this object is already locked (ie. on active queue)
1268     found = false;
1269     queue_iterate( &gArbitrationLockQueueActive,
1270                     active,
1271                     ArbitrationLockQueueElement *,
1272                     link )
1273     {
1274         if( active->service == element->service ) {
1275             found = true;
1276             break;
1277         }
1278     }
1279
1280     if( found ) { // this object is already locked
1281
1282         // determine whether it is the same or a different thread trying to lock
1283         if( active->thread != element->thread ) { // it is a different thread
1284
1285             ArbitrationLockQueueElement * victim = 0;
1286
1287             // before placing this new thread on the waiting queue, we look for
1288             // a deadlock cycle...
1289
1290             while( 1 ) {
1291                 // determine whether the active thread holding the object we
1292                 // want is waiting for another object to be unlocked
1293                 found = false;
1294                 queue_iterate( &gArbitrationLockQueueWaiting,
1295                                waiting,
1296                                ArbitrationLockQueueElement *,
1297                                link )
1298                 {
1299                     if( waiting->thread == active->thread ) {
1300                         assert( false == waiting->aborted );
1301                         found = true;
1302                         break;
1303                     }
1304                 }
1305
1306                 if( found ) { // yes, active thread waiting for another object
1307
1308                     // this may be a candidate for rejection if the required
1309                     // flag is not set, should we detect a deadlock later on
1310                     if( false == waiting->required )
1311                         victim = waiting;
1312
1313                     // find the thread that is holding this other object, that
1314                     // is blocking the active thread from proceeding (fun :-)
1315                     found = false;
1316                     queue_iterate( &gArbitrationLockQueueActive,
1317                                    active,      // (reuse active queue element)
1318                                    ArbitrationLockQueueElement *,
1319                                    link )
1320                     {
1321                         if( active->service == waiting->service ) {
1322                             found = true;
1323                             break;
1324                         }
1325                     }
1326
1327                     // someone must be holding it or it wouldn't be waiting
1328                     assert( found );
1329
1330                     if( active->thread == element->thread ) {
1331
1332                         // doh, it's waiting for the thread that originated
1333                         // this whole lock (ie. current thread) -> deadlock
1334                         if( false == element->required ) { // willing to fail?
1335
1336                             // the originating thread doesn't have the required
1337                             // flag, so it can fail
1338                             success = false; // (fail originating lock request)
1339                             break; // (out of while)
1340
1341                         } else { // originating thread is not willing to fail
1342
1343                             // see if we came across a waiting thread that did
1344                             // not have the 'required' flag set: we'll fail it
1345                             if( victim ) {
1346
1347                                 // we do have a willing victim, fail it's lock
1348                                 victim->aborted = true;
1349
1350                                 // take the victim off the waiting queue
1351                                 queue_remove( &gArbitrationLockQueueWaiting,
1352                                               victim,
1353                                               ArbitrationLockQueueElement *,
1354                                               link );
1355
1356                                 // wake the victim
1357                                 IOLockWakeup( gArbitrationLockQueueLock,
1358                                               victim,
1359                                               /* one thread */ true );
1360
1361                                 // allow this thread to proceed (ie. wait)
1362                                 success = true; // (put request on wait queue)
1363                                 break; // (out of while)
1364                             } else {
1365
1366                                 // all the waiting threads we came across in
1367                                 // finding this loop had the 'required' flag
1368                                 // set, so we've got a deadlock we can't avoid
1369                                 panic("I/O Kit: Unrecoverable deadlock.");
1370                             }
1371                         }
1372                     } else {
1373                         // repeat while loop, redefining active thread to be the
1374                         // thread holding "this other object" (see above), and
1375                         // looking for threads waiting on it; note the active
1376                         // variable points to "this other object" already... so
1377                         // there nothing to do in this else clause.
1378                     }
1379                 } else { // no, active thread is not waiting for another object
1380
1381                     success = true; // (put request on wait queue)
1382                     break; // (out of while)
1383                 }
1384             } // while forever
1385
1386             if( success ) { // put the request on the waiting queue?
1387                 kern_return_t wait_result;
1388
1389                 // place this thread on the waiting queue and put it to sleep;
1390                 // we place it at the tail of the queue...
1391                 queue_enter( &gArbitrationLockQueueWaiting,
1392                              element,
1393                              ArbitrationLockQueueElement *,
1394                              link );
1395
1396                 // declare that this thread will wait for a given event
1397 restart_sleep:  wait_result = assert_wait( element,
1398                                            element->required ? THREAD_UNINT
1399                                            : THREAD_INTERRUPTIBLE );
1400
1401                 // unlock global access
1402                 IOUnlock( gArbitrationLockQueueLock );
1403
1404                 // put thread to sleep, waiting for our event to fire...
1405                 if (wait_result == THREAD_WAITING)
1406                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1407
1408
1409                 // ...and we've been woken up; we might be in one of two states:
1410                 // (a) we've been aborted and our queue element is not on
1411                 //     any of the three queues, but is floating around
1412                 // (b) we're allowed to proceed with the lock and we have
1413                 //     already been moved from the waiting queue to the
1414                 //     active queue.
1415                 // ...plus a 3rd state, should the thread have been interrupted:
1416                 // (c) we're still on the waiting queue
1417
1418                 // determine whether we were interrupted out of our sleep
1419                 if( THREAD_INTERRUPTED == wait_result ) {
1420
1421                     // re-lock global access
1422                     IOTakeLock( gArbitrationLockQueueLock );
1423
1424                     // determine whether we're still on the waiting queue
1425                     found = false;
1426                     queue_iterate( &gArbitrationLockQueueWaiting,
1427                                    waiting,     // (reuse waiting queue element)
1428                                    ArbitrationLockQueueElement *,
1429                                    link )
1430                     {
1431                         if( waiting == element ) {
1432                             found = true;
1433                             break;
1434                         }
1435                     }
1436
1437                     if( found ) { // yes, we're still on the waiting queue
1438
1439                         // determine whether we're willing to fail
1440                         if( false == element->required ) {
1441
1442                             // mark us as aborted
1443                             element->aborted = true;
1444
1445                             // take us off the waiting queue
1446                             queue_remove( &gArbitrationLockQueueWaiting,
1447                                           element,
1448                                           ArbitrationLockQueueElement *,
1449                                           link );
1450                         } else { // we are not willing to fail
1451
1452                             // ignore interruption, go back to sleep
1453                             goto restart_sleep;
1454                         }
1455                     }
1456
1457                     // unlock global access
1458                     IOUnlock( gArbitrationLockQueueLock );
1459
1460                     // proceed as though this were a normal wake up
1461                     wait_result = THREAD_AWAKENED;
1462                 }
1463
1464                 assert( THREAD_AWAKENED == wait_result );
1465
1466                 // determine whether we've been aborted while we were asleep
1467                 if( element->aborted ) {
1468                     assert( false == element->required );
1469
1470                     // re-lock global access
1471                     IOTakeLock( gArbitrationLockQueueLock );
1472
1473                     action = kPutOnFreeQueue;
1474                     success = false;
1475                 } else { // we weren't aborted, so we must be ready to go :-)
1476
1477                     // we've already been moved from waiting to active queue
1478                     return true;
1479                 }
1480
1481             } else { // the lock request is to be failed
1482
1483                 // return unused queue element to queue
1484                 action = kPutOnFreeQueue;
1485             }
1486         } else { // it is the same thread, recursive access is allowed
1487
1488             // add one level of recursion
1489             active->count++;
1490
1491             // return unused queue element to queue
1492             action = kPutOnFreeQueue;
1493             success = true;
1494         }
1495     } else { // this object is not already locked, so let this thread through
1496         action = kPutOnActiveQueue;
1497         success = true;
1498     }
1499
1500     // put the new element on a queue
1501     if( kPutOnActiveQueue == action ) {
1502         queue_enter( &gArbitrationLockQueueActive,
1503                      element,
1504                      ArbitrationLockQueueElement *,
1505                      link );
1506     } else if( kPutOnFreeQueue == action ) {
1507         queue_enter( &gArbitrationLockQueueFree,
1508                      element,
1509                      ArbitrationLockQueueElement *,
1510                      link );
1511     } else {
1512         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1513     }
1514
1515     // unlock global access
1516     IOUnlock( gArbitrationLockQueueLock );
1517
1518     return( success );
1519 }
1520
1521 void IOService::unlockForArbitration( void )
1522 {
1523     bool                          found;
1524     ArbitrationLockQueueElement * element;
1525
1526     // lock global access
1527     IOTakeLock( gArbitrationLockQueueLock );
1528
1529     // find the lock element for this object (ie. on active queue)
1530     found = false;
1531     queue_iterate( &gArbitrationLockQueueActive,
1532                     element,
1533                     ArbitrationLockQueueElement *,
1534                     link )
1535     {
1536         if( element->service == this ) {
1537             found = true;
1538             break;
1539         }
1540     }
1541
1542     assert( found );
1543
1544     // determine whether the lock has been taken recursively
1545     if( element->count > 1 ) {
1546         // undo one level of recursion
1547         element->count--;
1548
1549     } else {
1550
1551         // remove it from the active queue
1552         queue_remove( &gArbitrationLockQueueActive,
1553                       element,
1554                       ArbitrationLockQueueElement *,
1555                       link );
1556
1557         // put it on the free queue
1558         queue_enter( &gArbitrationLockQueueFree,
1559                      element,
1560                      ArbitrationLockQueueElement *,
1561                      link );
1562
1563         // determine whether a thread is waiting for object (head to tail scan)
1564         found = false;
1565         queue_iterate( &gArbitrationLockQueueWaiting,
1566                        element,
1567                        ArbitrationLockQueueElement *,
1568                        link )
1569         {
1570             if( element->service == this ) {
1571                 found = true;
1572                 break;
1573             }
1574         }
1575
1576         if ( found ) { // we found an interested thread on waiting queue
1577
1578             // remove it from the waiting queue
1579             queue_remove( &gArbitrationLockQueueWaiting,
1580                           element,
1581                           ArbitrationLockQueueElement *,
1582                           link );
1583
1584             // put it on the active queue
1585             queue_enter( &gArbitrationLockQueueActive,
1586                          element,
1587                          ArbitrationLockQueueElement *,
1588                          link );
1589
1590             // wake the waiting thread
1591             IOLockWakeup( gArbitrationLockQueueLock,
1592                           element,
1593                           /* one thread */ true );
1594         }
1595     }
1596
1597     // unlock global access
1598     IOUnlock( gArbitrationLockQueueLock );
1599 }
1600
1601 uint32_t IOService::isLockedForArbitration(IOService * service)
1602 {
1603 #if DEBUG_NOTIFIER_LOCKED
1604     uint32_t                      count;
1605     ArbitrationLockQueueElement * active;
1606
1607     // lock global access
1608     IOLockLock(gArbitrationLockQueueLock);
1609
1610     // determine whether this object is already locked (ie. on active queue)
1611     count = 0;
1612     queue_iterate(&gArbitrationLockQueueActive,
1613                   active,
1614                   ArbitrationLockQueueElement *,
1615                   link)
1616     {
1617         if ((active->thread == IOThreadSelf())
1618             && (!service || (active->service == service)))
1619         {
1620             count += 0x10000;
1621             count += active->count;
1622         }
1623     }
1624
1625     IOLockUnlock(gArbitrationLockQueueLock);
1626
1627     return (count);
1628
1629 #else /* DEBUG_NOTIFIER_LOCKED */
1630
1631     return (0);
1632
1633 #endif /* DEBUG_NOTIFIER_LOCKED */
1634 }
1635
1636 void IOService::applyToProviders( IOServiceApplierFunction applier,
1637                                   void * context )
1638 {
1639     applyToParents( (IORegistryEntryApplierFunction) applier,
1640                     context, gIOServicePlane );
1641 }
1642
1643 void IOService::applyToClients( IOServiceApplierFunction applier,
1644                                 void * context )
1645 {
1646     applyToChildren( (IORegistryEntryApplierFunction) applier,
1647                      context, gIOServicePlane );
1648 }
1649
1650
1651 /*
1652  * Client messages
1653  */
1654
1655
1656 // send a message to a client or interested party of this service
1657 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1658                                    void * argument, vm_size_t argSize )
1659 {
1660     IOReturn                            ret;
1661     IOService *                         service;
1662     _IOServiceInterestNotifier *        notify;
1663
1664     if( (service = OSDynamicCast( IOService, client)))
1665         ret = service->message( type, this, argument );
1666
1667     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1668
1669         _IOServiceNotifierInvocation invocation;
1670         bool                     willNotify;
1671
1672         invocation.thread = current_thread();
1673
1674         LOCKWRITENOTIFY();
1675         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1676
1677         if( willNotify) {
1678             queue_enter( &notify->handlerInvocations, &invocation,
1679                             _IOServiceNotifierInvocation *, link );
1680         }
1681         UNLOCKNOTIFY();
1682
1683         if( willNotify) {
1684
1685             ret = (*notify->handler)( notify->target, notify->ref,
1686                                           type, this, argument, argSize );
1687
1688             LOCKWRITENOTIFY();
1689             queue_remove( &notify->handlerInvocations, &invocation,
1690                             _IOServiceNotifierInvocation *, link );
1691             if( kIOServiceNotifyWaiter & notify->state) {
1692                 notify->state &= ~kIOServiceNotifyWaiter;
1693                 WAKEUPNOTIFY( notify );
1694             }
1695             UNLOCKNOTIFY();
1696
1697         } else
1698             ret = kIOReturnSuccess;
1699
1700     } else
1701         ret = kIOReturnBadArgument;
1702
1703     return( ret );
1704 }
1705
1706 static void
1707 applyToInterestNotifiers(const IORegistryEntry *target,
1708                          const OSSymbol * typeOfInterest,
1709                          OSObjectApplierFunction applier,
1710                          void * context )
1711 {
1712     OSArray *  copyArray = 0;
1713     OSObject * prop;
1714
1715     LOCKREADNOTIFY();
1716
1717     prop = target->copyProperty(typeOfInterest);
1718     IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
1719
1720     if( notifyList) {
1721         copyArray = OSArray::withCapacity(1);
1722
1723         // iterate over queue, entry is set to each element in the list
1724         iterqueue(&notifyList->fCommandChain, entry) {
1725             _IOServiceInterestNotifier * notify;
1726
1727             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1728             copyArray->setObject(notify);
1729         }
1730     }
1731     UNLOCKNOTIFY();
1732
1733     if( copyArray) {
1734         unsigned int    index;
1735         OSObject *      next;
1736
1737         for( index = 0; (next = copyArray->getObject( index )); index++)
1738             (*applier)(next, context);
1739         copyArray->release();
1740     }
1741
1742     OSSafeReleaseNULL(prop);
1743 }
1744
1745 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1746                                    OSObjectApplierFunction applier,
1747                                    void * context )
1748 {
1749     if (gIOGeneralInterest == typeOfInterest)
1750         applyToClients( (IOServiceApplierFunction) applier, context );
1751     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1752 }
1753
1754 struct MessageClientsContext {
1755     IOService * service;
1756     UInt32      type;
1757     void *      argument;
1758     vm_size_t   argSize;
1759     IOReturn    ret;
1760 };
1761
1762 static void messageClientsApplier( OSObject * object, void * ctx )
1763 {
1764     IOReturn                ret;
1765     MessageClientsContext * context = (MessageClientsContext *) ctx;
1766
1767     ret = context->service->messageClient( context->type,
1768                                            object, context->argument, context->argSize );
1769     if( kIOReturnSuccess != ret)
1770         context->ret = ret;
1771 }
1772
1773 // send a message to all clients
1774 IOReturn IOService::messageClients( UInt32 type,
1775                                     void * argument, vm_size_t argSize )
1776 {
1777     MessageClientsContext       context;
1778
1779     context.service     = this;
1780     context.type        = type;
1781     context.argument    = argument;
1782     context.argSize     = argSize;
1783     context.ret         = kIOReturnSuccess;
1784
1785     applyToInterested( gIOGeneralInterest,
1786                        &messageClientsApplier, &context );
1787
1788     return( context.ret );
1789 }
1790
1791 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1792                                               IOOptionBits response )
1793 {
1794     return( kIOReturnUnsupported );
1795 }
1796
1797 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1798                   IOServiceInterestHandler handler, void * target, void * ref )
1799 {
1800     _IOServiceInterestNotifier * notify = 0;
1801     IOReturn rc = kIOReturnError;
1802
1803     notify = new _IOServiceInterestNotifier;
1804     if (!notify) return NULL;
1805
1806     if(notify->init()) {
1807         rc = registerInterestForNotifier(notify, typeOfInterest,
1808                               handler, target, ref);
1809     }
1810
1811     if (rc != kIOReturnSuccess) {
1812         notify->release();
1813         notify = 0;
1814     }
1815
1816     return( notify );
1817 }
1818
1819
1820
1821 static IOReturn
1822 IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
1823                                               UInt32 messageType, IOService * provider,
1824                                               void * messageArgument, vm_size_t argSize )
1825 {
1826     return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
1827 }
1828
1829 IONotifier * IOService::registerInterest(const OSSymbol * typeOfInterest,
1830                   IOServiceInterestHandlerBlock handler)
1831 {
1832     IONotifier * notify;
1833     void       * block;
1834
1835     block = Block_copy(handler);
1836     if (!block) return (NULL);
1837
1838     notify = registerInterest(typeOfInterest, &IOServiceInterestHandlerToBlock, NULL, block);
1839
1840     if (!notify) Block_release(block);
1841
1842     return (notify);
1843 }
1844
1845 IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1846                   IOServiceInterestHandler handler, void * target, void * ref )
1847 {
1848     IOReturn rc = kIOReturnSuccess;
1849     _IOServiceInterestNotifier  *notify = 0;
1850
1851     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1852         return( kIOReturnBadArgument );
1853
1854     notify->handler = handler;
1855     notify->target = target;
1856     notify->ref = ref;
1857
1858     if( (typeOfInterest != gIOGeneralInterest)
1859      && (typeOfInterest != gIOBusyInterest)
1860      && (typeOfInterest != gIOAppPowerStateInterest)
1861      && (typeOfInterest != gIOConsoleSecurityInterest)
1862      && (typeOfInterest != gIOPriorityPowerStateInterest))
1863         return( kIOReturnBadArgument );
1864
1865     lockForArbitration();
1866     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1867
1868         notify->state = kIOServiceNotifyEnable;
1869
1870         ////// queue
1871
1872         LOCKWRITENOTIFY();
1873
1874         // Get the head of the notifier linked list
1875         IOCommand * notifyList;
1876         OSObject  * obj = copyProperty( typeOfInterest );
1877         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1878             notifyList = OSTypeAlloc(IOCommand);
1879             if (notifyList) {
1880                 notifyList->init();
1881                 bool ok = setProperty( typeOfInterest, notifyList);
1882                 notifyList->release();
1883                 if (!ok) notifyList = 0;
1884             }
1885         }
1886         if (obj) obj->release();
1887
1888         if (notifyList) {
1889             enqueue(&notifyList->fCommandChain, &notify->chain);
1890             notify->retain();   // ref'ed while in list
1891         }
1892
1893         UNLOCKNOTIFY();
1894     }
1895     else {
1896         rc = kIOReturnNotReady;
1897     }
1898     unlockForArbitration();
1899
1900     return rc;
1901 }
1902
1903 static void cleanInterestList( OSObject * head )
1904 {
1905     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1906     if (!notifyHead)
1907         return;
1908
1909     LOCKWRITENOTIFY();
1910     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1911         queue_next(entry) = queue_prev(entry) = 0;
1912
1913         _IOServiceInterestNotifier * notify;
1914
1915         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1916         notify->release();
1917     }
1918     UNLOCKNOTIFY();
1919 }
1920
1921 void IOService::unregisterAllInterest( void )
1922 {
1923     OSObject * prop;
1924
1925     prop = copyProperty(gIOGeneralInterest);
1926     cleanInterestList(prop);
1927     OSSafeReleaseNULL(prop);
1928
1929     prop = copyProperty(gIOBusyInterest);
1930     cleanInterestList(prop);
1931     OSSafeReleaseNULL(prop);
1932
1933     prop = copyProperty(gIOAppPowerStateInterest);
1934     cleanInterestList(prop);
1935     OSSafeReleaseNULL(prop);
1936
1937     prop = copyProperty(gIOPriorityPowerStateInterest);
1938     cleanInterestList(prop);
1939     OSSafeReleaseNULL(prop);
1940
1941     prop = copyProperty(gIOConsoleSecurityInterest);
1942     cleanInterestList(prop);
1943     OSSafeReleaseNULL(prop);
1944 }
1945
1946 /*
1947  * _IOServiceInterestNotifier
1948  */
1949
1950 // wait for all threads, other than the current one,
1951 //  to exit the handler
1952
1953 void _IOServiceInterestNotifier::wait()
1954 {
1955     _IOServiceNotifierInvocation * next;
1956     bool doWait;
1957
1958     do {
1959         doWait = false;
1960         queue_iterate( &handlerInvocations, next,
1961                         _IOServiceNotifierInvocation *, link) {
1962             if( next->thread != current_thread() ) {
1963                 doWait = true;
1964                 break;
1965             }
1966         }
1967         if( doWait) {
1968             state |= kIOServiceNotifyWaiter;
1969                SLEEPNOTIFY(this);
1970         }
1971
1972     } while( doWait );
1973 }
1974
1975 void _IOServiceInterestNotifier::free()
1976 {
1977     assert( queue_empty( &handlerInvocations ));
1978
1979     if (handler == &IOServiceInterestHandlerToBlock) Block_release(ref);
1980
1981     OSObject::free();
1982 }
1983
1984 void _IOServiceInterestNotifier::remove()
1985 {
1986     LOCKWRITENOTIFY();
1987
1988     if( queue_next( &chain )) {
1989         remqueue(&chain);
1990         queue_next( &chain) = queue_prev( &chain) = 0;
1991         release();
1992     }
1993
1994     state &= ~kIOServiceNotifyEnable;
1995
1996     wait();
1997
1998     UNLOCKNOTIFY();
1999
2000     release();
2001 }
2002
2003 bool _IOServiceInterestNotifier::disable()
2004 {
2005     bool        ret;
2006
2007     LOCKWRITENOTIFY();
2008
2009     ret = (0 != (kIOServiceNotifyEnable & state));
2010     state &= ~kIOServiceNotifyEnable;
2011     if( ret)
2012         wait();
2013
2014     UNLOCKNOTIFY();
2015
2016     return( ret );
2017 }
2018
2019 void _IOServiceInterestNotifier::enable( bool was )
2020 {
2021     LOCKWRITENOTIFY();
2022     if( was)
2023         state |= kIOServiceNotifyEnable;
2024     else
2025         state &= ~kIOServiceNotifyEnable;
2026     UNLOCKNOTIFY();
2027 }
2028
2029 bool _IOServiceInterestNotifier::init()
2030 {
2031     queue_init( &handlerInvocations );
2032     return (OSObject::init());
2033 }
2034 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2035
2036 /*
2037  * Termination
2038  */
2039
2040 #define tailQ(o)                setObject(o)
2041 #define headQ(o)                setObject(0, o)
2042 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2043
2044 static void _workLoopAction( IOWorkLoop::Action action,
2045                              IOService * service,
2046                              void * p0 = 0, void * p1 = 0,
2047                              void * p2 = 0, void * p3 = 0 )
2048 {
2049     IOWorkLoop * wl;
2050
2051     if( (wl = service->getWorkLoop())) {
2052         wl->retain();
2053         wl->runAction( action, service, p0, p1, p2, p3 );
2054         wl->release();
2055     } else
2056         (*action)( service, p0, p1, p2, p3 );
2057 }
2058
2059 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
2060 {
2061     bool ok;
2062
2063     // if its our only provider
2064     ok = isParent( provider, gIOServicePlane, true);
2065
2066     // -- compat
2067     if( ok) {
2068         provider->terminateClient( this, options | kIOServiceRecursing );
2069         ok = (0 != (kIOServiceInactiveState & __state[0]));
2070     }
2071     // --
2072
2073     return( ok );
2074 }
2075
2076 bool IOService::terminatePhase1( IOOptionBits options )
2077 {
2078     IOService *  victim;
2079     IOService *  client;
2080     OSIterator * iter;
2081     OSArray *    makeInactive;
2082     OSArray *    waitingInactive;
2083     int          waitResult = THREAD_AWAKENED;
2084     bool         wait;
2085     bool                 ok;
2086     bool                 didInactive;
2087     bool                 startPhase2 = false;
2088
2089     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2090
2091     uint64_t regID = getRegistryEntryID();
2092     IOServiceTrace(
2093         IOSERVICE_TERMINATE_PHASE1,
2094         (uintptr_t) regID,
2095         (uintptr_t) (regID >> 32),
2096         (uintptr_t) this,
2097         (uintptr_t) options);
2098
2099     // -- compat
2100     if( options & kIOServiceRecursing) {
2101         lockForArbitration();
2102         if (0 == (kIOServiceInactiveState & __state[0]))
2103         {
2104             __state[0] |= kIOServiceInactiveState;
2105             __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2106         }
2107         unlockForArbitration();
2108
2109         return( true );
2110     }
2111     // --
2112
2113     makeInactive    = OSArray::withCapacity( 16 );
2114     waitingInactive = OSArray::withCapacity( 16 );
2115     if(!makeInactive || !waitingInactive) return( false );
2116
2117     victim = this;
2118     victim->retain();
2119
2120     while( victim )
2121     {
2122         didInactive = victim->lockForArbitration( true );
2123         if( didInactive)
2124         {
2125             uint64_t regID1 = victim->getRegistryEntryID();
2126             IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2127                 (uintptr_t) regID1,
2128                 (uintptr_t) (regID1 >> 32),
2129                 (uintptr_t) victim->__state[1],
2130                 (uintptr_t) 0);
2131
2132             enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2133             didInactive = (kRP1 == (victim->__state[1] & kRP1))
2134                         || (0 == (victim->__state[0] & kIOServiceInactiveState));
2135
2136             if (!didInactive)
2137             {
2138                 // a multiply attached IOService can be visited twice
2139                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do
2140                 {
2141                     IOLockLock(gIOServiceBusyLock);
2142                     wait = (victim->__state[1] & kIOServiceTermPhase1State);
2143                     if( wait) {
2144                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2145                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2146                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2147                         victim->unlockForArbitration();
2148                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2149                     }
2150                     IOLockUnlock(gIOServiceBusyLock);
2151                     if( wait) {
2152                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2153                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2154                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2155                         victim->lockForArbitration();
2156                     }
2157                 }
2158                 while (wait && (waitResult != THREAD_TIMED_OUT));
2159             }
2160             else
2161             {
2162                 victim->__state[0] |= kIOServiceInactiveState;
2163                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2164                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2165                 victim->__state[1] &= ~kIOServiceRecursing;
2166                 victim->__state[1] |= kIOServiceTermPhase1State;
2167                 waitingInactive->headQ(victim);
2168                 if (victim == this)
2169                 {
2170                     if (kIOServiceTerminateNeedWillTerminate & options)
2171                     {
2172                         victim->__state[1] |= kIOServiceNeedWillTerminate;
2173                     }
2174                 }
2175                 victim->_adjustBusy( 1 );
2176             }
2177             victim->unlockForArbitration();
2178         }
2179         if( victim == this) startPhase2 = didInactive;
2180         if (didInactive)
2181         {
2182             OSArray * notifiers;
2183             notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2184             victim->invokeNotifiers(&notifiers);
2185
2186             IOUserClient::destroyUserReferences( victim );
2187
2188             iter = victim->getClientIterator();
2189             if( iter) {
2190                 while( (client = (IOService *) iter->getNextObject())) {
2191                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2192                             client->getName(), client->getRegistryEntryID(),
2193                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2194                     ok = client->requestTerminate( victim, options );
2195                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2196                             client->getName(), client->getRegistryEntryID(),
2197                             victim->getName(), victim->getRegistryEntryID(), ok);
2198
2199                     uint64_t regID1 = client->getRegistryEntryID();
2200                     uint64_t regID2 = victim->getRegistryEntryID();
2201                     IOServiceTrace(
2202                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2203                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
2204                         (uintptr_t) regID1,
2205                         (uintptr_t) (regID1 >> 32),
2206                         (uintptr_t) regID2,
2207                         (uintptr_t) (regID2 >> 32));
2208
2209                     if( ok)
2210                         makeInactive->setObject( client );
2211                 }
2212                 iter->release();
2213             }
2214         }
2215         victim->release();
2216         victim = (IOService *) makeInactive->getObject(0);
2217         if( victim) {
2218             victim->retain();
2219             makeInactive->removeObject(0);
2220         }
2221     }
2222     makeInactive->release();
2223
2224     while ((victim = (IOService *) waitingInactive->getObject(0)))
2225     {
2226         victim->retain();
2227         waitingInactive->removeObject(0);
2228
2229         victim->lockForArbitration();
2230         victim->__state[1] &= ~kIOServiceTermPhase1State;
2231         if (kIOServiceTerm1WaiterState & victim->__state[1])
2232         {
2233             victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2234             TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2235             IOLockLock( gIOServiceBusyLock );
2236             thread_wakeup( (event_t) &victim->__state[1]);
2237             IOLockUnlock( gIOServiceBusyLock );
2238         }
2239         victim->unlockForArbitration();
2240         victim->release();
2241     }
2242     waitingInactive->release();
2243
2244     if( startPhase2)
2245     {
2246         retain();
2247         lockForArbitration();
2248         scheduleTerminatePhase2(options);
2249         unlockForArbitration();
2250         release();
2251     }
2252
2253     return( true );
2254 }
2255
2256 void IOService::setTerminateDefer(IOService * provider, bool defer)
2257 {
2258     lockForArbitration();
2259     if (defer)  __state[1] |= kIOServiceStartState;
2260     else        __state[1] &= ~kIOServiceStartState;
2261     unlockForArbitration();
2262
2263     if (provider && !defer)
2264     {
2265         provider->lockForArbitration();
2266         provider->scheduleTerminatePhase2();
2267         provider->unlockForArbitration();
2268     }
2269 }
2270
2271 // Must call this while holding gJobsLock
2272 void IOService::waitToBecomeTerminateThread(void)
2273 {
2274     IOLockAssert(gJobsLock, kIOLockAssertOwned);
2275     bool wait;
2276     do {
2277         wait = (gIOTerminateThread != THREAD_NULL);
2278         if (wait) {
2279             IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2280         }
2281     } while (wait);
2282     gIOTerminateThread = current_thread();
2283 }
2284
2285 // call with lockForArbitration
2286 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2287 {
2288     AbsoluteTime        deadline;
2289     uint64_t            regID1;
2290     int                 waitResult = THREAD_AWAKENED;
2291     bool                wait = false, haveDeadline = false;
2292
2293     if (!(__state[0] & kIOServiceInactiveState)) return;
2294
2295     regID1 = getRegistryEntryID();
2296     IOServiceTrace(
2297         IOSERVICE_TERM_SCHED_PHASE2,
2298         (uintptr_t) regID1,
2299         (uintptr_t) (regID1 >> 32),
2300         (uintptr_t) __state[1],
2301         (uintptr_t) options);
2302
2303     if (__state[1] & kIOServiceTermPhase1State)         return;
2304
2305     retain();
2306     unlockForArbitration();
2307     options |= kIOServiceRequired;
2308     IOLockLock( gJobsLock );
2309
2310     if( (options & kIOServiceSynchronous)
2311         && (current_thread() != gIOTerminateThread)) {
2312
2313         waitToBecomeTerminateThread();
2314         gIOTerminatePhase2List->setObject( this );
2315         gIOTerminateWork++;
2316
2317         do {
2318             while( gIOTerminateWork )
2319                 terminateWorker( options );
2320             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2321             if( wait) {
2322                 /* wait for the victim to go non-busy */
2323                 if( !haveDeadline) {
2324                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2325                     haveDeadline = true;
2326                 }
2327                 /* let others do work while we wait */
2328                 gIOTerminateThread = 0;
2329                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2330                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2331                                                   deadline, THREAD_UNINT );
2332                 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2333                     panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2334                 }
2335                 waitToBecomeTerminateThread();
2336             }
2337         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2338
2339         gIOTerminateThread = 0;
2340         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2341
2342     } else {
2343         // ! kIOServiceSynchronous
2344
2345         gIOTerminatePhase2List->setObject( this );
2346         if( 0 == gIOTerminateWork++) {
2347             assert(gIOTerminateWorkerThread);
2348             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2349         }
2350     }
2351
2352     IOLockUnlock( gJobsLock );
2353     lockForArbitration();
2354     release();
2355 }
2356
2357 __attribute__((__noreturn__))
2358 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2359 {
2360     // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2361     IOLockLock(gJobsLock);
2362     while (true) {
2363         if (gIOTerminateThread != gIOTerminateWorkerThread) {
2364             waitToBecomeTerminateThread();
2365         }
2366
2367         while (gIOTerminateWork)
2368             terminateWorker( (uintptr_t)arg );
2369
2370         gIOTerminateThread = 0;
2371         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2372         IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2373     }
2374 }
2375
2376 void IOService::scheduleStop( IOService * provider )
2377 {
2378     uint64_t regID1 = getRegistryEntryID();
2379     uint64_t regID2 = provider->getRegistryEntryID();
2380
2381     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2382     IOServiceTrace(
2383         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2384         (uintptr_t) regID1,
2385         (uintptr_t) (regID1 >> 32),
2386         (uintptr_t) regID2,
2387         (uintptr_t) (regID2 >> 32));
2388
2389     IOLockLock( gJobsLock );
2390     gIOStopList->tailQ( this );
2391     gIOStopProviderList->tailQ( provider );
2392
2393     if( 0 == gIOTerminateWork++) {
2394         assert(gIOTerminateWorkerThread);
2395         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2396     }
2397
2398     IOLockUnlock( gJobsLock );
2399 }
2400
2401 void IOService::scheduleFinalize(bool now)
2402 {
2403     uint64_t regID1 = getRegistryEntryID();
2404
2405     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2406     IOServiceTrace(
2407         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2408         (uintptr_t) regID1,
2409         (uintptr_t) (regID1 >> 32),
2410         0, 0);
2411
2412     if (now || IOUserClient::finalizeUserReferences(this))
2413     {
2414         IOLockLock( gJobsLock );
2415         gIOFinalizeList->tailQ(this);
2416         if( 0 == gIOTerminateWork++) {
2417             assert(gIOTerminateWorkerThread);
2418             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2419         }
2420         IOLockUnlock( gJobsLock );
2421     }
2422 }
2423
2424 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2425 {
2426     return( true );
2427 }
2428
2429 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2430 {
2431     if( false == *defer) {
2432
2433         if( lockForArbitration( true )) {
2434             if( false == provider->handleIsOpen( this ))
2435                 scheduleStop( provider );
2436             // -- compat
2437             else {
2438                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2439                 if( false == provider->handleIsOpen( this ))
2440                     scheduleStop( provider );
2441             }
2442             // --
2443             unlockForArbitration();
2444         }
2445     }
2446
2447     return( true );
2448 }
2449
2450 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2451                                      OSArray * doPhase2List,
2452                                      void *unused2 __unused,
2453                                      void *unused3 __unused  )
2454 {
2455     OSIterator * iter;
2456     IOService *  client;
2457     bool         ok;
2458     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2459
2460     iter = victim->getClientIterator();
2461     if( iter) {
2462         while( (client = (IOService *) iter->getNextObject())) {
2463
2464             regID1 = client->getRegistryEntryID();
2465             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2466                     client->getName(), regID1,
2467                     victim->getName(), regID2, (long long)options);
2468             IOServiceTrace(
2469                 IOSERVICE_TERMINATE_WILL,
2470                 (uintptr_t) regID1,
2471                 (uintptr_t) (regID1 >> 32),
2472                 (uintptr_t) regID2,
2473                 (uintptr_t) (regID2 >> 32));
2474
2475             ok = client->willTerminate( victim, options );
2476             doPhase2List->tailQ( client );
2477         }
2478         iter->release();
2479     }
2480 }
2481
2482 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2483                             void *unused1 __unused, void *unused2 __unused,
2484                             void *unused3 __unused )
2485 {
2486     OSIterator * iter;
2487     IOService *  client;
2488     bool         defer;
2489     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2490
2491     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2492
2493     iter = victim->getClientIterator();
2494     if( iter) {
2495         while( (client = (IOService *) iter->getNextObject())) {
2496
2497             regID1 = client->getRegistryEntryID();
2498             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2499                     client->getName(), regID1,
2500                     victim->getName(), regID2, (long long)options);
2501             defer = false;
2502             client->didTerminate( victim, options, &defer );
2503
2504             IOServiceTrace(
2505                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2506                        : IOSERVICE_TERMINATE_DID),
2507                 (uintptr_t) regID1,
2508                 (uintptr_t) (regID1 >> 32),
2509                 (uintptr_t) regID2,
2510                 (uintptr_t) (regID2 >> 32));
2511
2512             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2513                     client->getName(), regID1,
2514                     victim->getName(), regID2, defer);
2515         }
2516         iter->release();
2517     }
2518 }
2519
2520
2521 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2522                             void *unused1 __unused, void *unused2 __unused,
2523                             void *unused3 __unused )
2524 {
2525     OSIterator * iter;
2526     IOService *  provider;
2527     bool         ok;
2528     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2529
2530     iter = victim->getProviderIterator();
2531     if( iter) {
2532         while( (provider = (IOService *) iter->getNextObject())) {
2533
2534             regID1 = provider->getRegistryEntryID();
2535             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2536                     victim->getName(), regID2,
2537                     provider->getName(), regID1, (long long)options);
2538             IOServiceTrace(
2539                 IOSERVICE_TERMINATE_WILL,
2540                 (uintptr_t) regID2,
2541                 (uintptr_t) (regID2 >> 32),
2542                 (uintptr_t) regID1,
2543                 (uintptr_t) (regID1 >> 32));
2544
2545             ok = victim->willTerminate( provider, options );
2546         }
2547         iter->release();
2548     }
2549 }
2550
2551 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2552                             void *unused1 __unused, void *unused2 __unused,
2553                             void *unused3 __unused )
2554 {
2555     OSIterator * iter;
2556     IOService *  provider;
2557     bool defer = false;
2558     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2559
2560     iter = victim->getProviderIterator();
2561     if( iter) {
2562         while( (provider = (IOService *) iter->getNextObject())) {
2563
2564             regID1 = provider->getRegistryEntryID();
2565             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2566                     victim->getName(), regID2,
2567                     provider->getName(), regID1, (long long)options);
2568             victim->didTerminate( provider, options, &defer );
2569
2570             IOServiceTrace(
2571                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2572                        : IOSERVICE_TERMINATE_DID),
2573                 (uintptr_t) regID2,
2574                 (uintptr_t) (regID2 >> 32),
2575                 (uintptr_t) regID1,
2576                 (uintptr_t) (regID1 >> 32));
2577
2578             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2579                     victim->getName(), regID2,
2580                     provider->getName(), regID1, defer);
2581         }
2582         iter->release();
2583     }
2584 }
2585
2586
2587 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2588                             void *unused1 __unused, void *unused2 __unused,
2589                             void *unused3 __unused )
2590 {
2591     uint64_t regID1 = victim->getRegistryEntryID();
2592     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2593     IOServiceTrace(
2594         IOSERVICE_TERMINATE_FINALIZE,
2595         (uintptr_t) regID1,
2596         (uintptr_t) (regID1 >> 32),
2597         0, 0);
2598
2599     victim->finalize( options );
2600 }
2601
2602 void IOService::actionStop( IOService * provider, IOService * client,
2603                             void *unused1 __unused, void *unused2 __unused,
2604                             void *unused3 __unused )
2605 {
2606     uint64_t regID1 = provider->getRegistryEntryID();
2607     uint64_t regID2 = client->getRegistryEntryID();
2608
2609     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2610     IOServiceTrace(
2611         IOSERVICE_TERMINATE_STOP,
2612         (uintptr_t) regID1,
2613         (uintptr_t) (regID1 >> 32),
2614         (uintptr_t) regID2,
2615         (uintptr_t) (regID2 >> 32));
2616
2617     client->stop( provider );
2618     if( provider->isOpen( client ))
2619         provider->close( client );
2620
2621     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2622     client->detach( provider );
2623 }
2624
2625 void IOService::terminateWorker( IOOptionBits options )
2626 {
2627     OSArray *           doPhase2List;
2628     OSArray *           didPhase2List;
2629     OSSet *             freeList;
2630     OSIterator *        iter;
2631     UInt32              workDone;
2632     IOService *         victim;
2633     IOService *         client;
2634     IOService *         provider;
2635     unsigned int        idx;
2636     bool                moreToDo;
2637     bool                doPhase2;
2638     bool                doPhase3;
2639
2640     options |= kIOServiceRequired;
2641
2642     doPhase2List  = OSArray::withCapacity( 16 );
2643     didPhase2List = OSArray::withCapacity( 16 );
2644     freeList      = OSSet::withCapacity( 16 );
2645     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2646         return;
2647
2648     do {
2649         workDone = gIOTerminateWork;
2650
2651         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2652
2653             victim->retain();
2654             gIOTerminatePhase2List->removeObject(0);
2655             IOLockUnlock( gJobsLock );
2656
2657             uint64_t regID1 = victim->getRegistryEntryID();
2658             IOServiceTrace(
2659                 IOSERVICE_TERM_START_PHASE2,
2660                 (uintptr_t) regID1,
2661                 (uintptr_t) (regID1 >> 32),
2662                 (uintptr_t) 0,
2663                 (uintptr_t) 0);
2664
2665             while( victim ) {
2666
2667                 doPhase2 = victim->lockForArbitration( true );
2668                 if( doPhase2) {
2669                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2670                     if( doPhase2) {
2671
2672                         uint64_t regID1 = victim->getRegistryEntryID();
2673                         IOServiceTrace(
2674                             IOSERVICE_TERM_TRY_PHASE2,
2675                             (uintptr_t) regID1,
2676                             (uintptr_t) (regID1 >> 32),
2677                             (uintptr_t) victim->__state[1],
2678                             (uintptr_t) 0);
2679
2680                         doPhase2 = (0 == (victim->__state[1] &
2681                                     (kIOServiceTermPhase1State
2682                                     | kIOServiceTermPhase2State
2683                                     | kIOServiceConfigState)));
2684
2685                         if (doPhase2 && (iter = victim->getClientIterator())) {
2686                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2687                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2688                                 if (!doPhase2)
2689                                 {
2690                                     uint64_t regID1 = client->getRegistryEntryID();
2691                                     IOServiceTrace(
2692                                         IOSERVICE_TERM_UC_DEFER,
2693                                         (uintptr_t) regID1,
2694                                         (uintptr_t) (regID1 >> 32),
2695                                         (uintptr_t) client->__state[1],
2696                                         (uintptr_t) 0);
2697                                     TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2698                                            victim->getName(), victim->getRegistryEntryID(),
2699                                            client->getName(), client->getRegistryEntryID());
2700                                 }
2701                             }
2702                             iter->release();
2703                         }
2704                         if( doPhase2)
2705                             victim->__state[1] |= kIOServiceTermPhase2State;
2706                     }
2707                     victim->unlockForArbitration();
2708                 }
2709                 if( doPhase2) {
2710
2711                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2712                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2713                                             victim, (void *)(uintptr_t) options, NULL );
2714                     }
2715
2716                     OSArray * notifiers;
2717                     notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
2718                     victim->invokeNotifiers(&notifiers);
2719
2720                     _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2721                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2722
2723                     didPhase2List->headQ( victim );
2724                 }
2725                 victim->release();
2726                 victim = (IOService *) doPhase2List->getObject(0);
2727                 if( victim) {
2728                     victim->retain();
2729                     doPhase2List->removeObject(0);
2730                 }
2731             }
2732
2733             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2734                 bool scheduleFinalize = false;
2735                 if( victim->lockForArbitration( true )) {
2736                     victim->__state[1] |= kIOServiceTermPhase3State;
2737                     scheduleFinalize = (0 == victim->getClient());
2738                     victim->unlockForArbitration();
2739                 }
2740                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2741                                     victim, (void *)(uintptr_t) options );
2742                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2743                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2744                                         victim, (void *)(uintptr_t) options, NULL );
2745                 }
2746                 // no clients - will go to finalize
2747                 if (scheduleFinalize) victim->scheduleFinalize(false);
2748                 didPhase2List->removeObject(0);
2749             }
2750             IOLockLock( gJobsLock );
2751         }
2752
2753         // phase 3
2754         do {
2755             doPhase3 = false;
2756             // finalize leaves
2757             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2758                 bool sendFinal = false;
2759                 IOLockUnlock( gJobsLock );
2760                 if (victim->lockForArbitration(true)) {
2761                     sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
2762                     if (sendFinal) victim->__state[1] |= kIOServiceFinalized;
2763                     victim->unlockForArbitration();
2764                 }
2765                 if (sendFinal) {
2766                     _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2767                                     victim, (void *)(uintptr_t) options );
2768                 }
2769                 IOLockLock( gJobsLock );
2770                 // hold off free
2771                 freeList->setObject( victim );
2772                 // safe if finalize list is append only
2773                 gIOFinalizeList->removeObject(0);
2774             }
2775
2776             for( idx = 0;
2777                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2778
2779                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2780                 assert( provider );
2781
2782                 uint64_t regID1 = provider->getRegistryEntryID();
2783                 uint64_t regID2 = client->getRegistryEntryID();
2784
2785                 if( !provider->isChild( client, gIOServicePlane )) {
2786                     // may be multiply queued - nop it
2787                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2788                     IOServiceTrace(
2789                         IOSERVICE_TERMINATE_STOP_NOP,
2790                         (uintptr_t) regID1,
2791                         (uintptr_t) (regID1 >> 32),
2792                         (uintptr_t) regID2,
2793                         (uintptr_t) (regID2 >> 32));
2794
2795                 } else {
2796                     // a terminated client is not ready for stop if it has clients, skip it
2797                     bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
2798                     IOLockUnlock( gJobsLock );
2799                     if (deferStop && client->lockForArbitration(true)) {
2800                         deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
2801                         //deferStop = (!deferStop && (0 != client->getClient()));
2802                         //deferStop = (0 != client->getClient());
2803                         client->unlockForArbitration();
2804                         if (deferStop) {
2805                             TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
2806                             IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
2807                                (uintptr_t) regID1,
2808                                (uintptr_t) (regID1 >> 32),
2809                                (uintptr_t) regID2,
2810                                (uintptr_t) (regID2 >> 32));
2811
2812                             idx++;
2813                             IOLockLock( gJobsLock );
2814                             continue;
2815                         }
2816                     }
2817                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2818                                      provider, (void *) client );
2819                     IOLockLock( gJobsLock );
2820                     // check the finalize list now
2821                     doPhase3 = true;
2822                 }
2823                 // hold off free
2824                 freeList->setObject( client );
2825                 freeList->setObject( provider );
2826
2827                 // safe if stop list is append only
2828                 gIOStopList->removeObject( idx );
2829                 gIOStopProviderList->removeObject( idx );
2830                 idx = 0;
2831             }
2832
2833         } while( doPhase3 );
2834
2835         gIOTerminateWork -= workDone;
2836         moreToDo = (gIOTerminateWork != 0);
2837
2838         if( !moreToDo) {
2839             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2840             IOServiceTrace(
2841                 IOSERVICE_TERMINATE_DONE,
2842                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2843         }
2844
2845     } while( moreToDo );
2846
2847     IOLockUnlock( gJobsLock );
2848
2849     freeList->release();
2850     doPhase2List->release();
2851     didPhase2List->release();
2852
2853     IOLockLock( gJobsLock );
2854 }
2855
2856 bool IOService::finalize( IOOptionBits options )
2857 {
2858     OSIterator *  iter;
2859     IOService *   provider;
2860     uint64_t      regID1, regID2 = getRegistryEntryID();
2861
2862     iter = getProviderIterator();
2863     assert( iter );
2864
2865     if( iter) {
2866         while( (provider = (IOService *) iter->getNextObject())) {
2867
2868             // -- compat
2869             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2870                 /* we come down here on programmatic terminate */
2871
2872                 regID1 = provider->getRegistryEntryID();
2873                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2874                 IOServiceTrace(
2875                     IOSERVICE_TERMINATE_STOP,
2876                     (uintptr_t) regID1,
2877                     (uintptr_t) (regID1 >> 32),
2878                     (uintptr_t) regID2,
2879                     (uintptr_t) (regID2 >> 32));
2880
2881                 stop( provider );
2882                 if( provider->isOpen( this ))
2883                     provider->close( this );
2884                 detach( provider );
2885             } else {
2886             //--
2887                 if( provider->lockForArbitration( true )) {
2888                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2889                         scheduleStop( provider );
2890                     provider->unlockForArbitration();
2891                 }
2892             }
2893         }
2894         iter->release();
2895     }
2896
2897     return( true );
2898 }
2899
2900 #undef tailQ
2901 #undef headQ
2902
2903 /*
2904  * Terminate
2905  */
2906
2907 void IOService::doServiceTerminate( IOOptionBits options )
2908 {
2909 }
2910
2911 // a method in case someone needs to override it
2912 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2913 {
2914     bool ok;
2915
2916     if( client->isParent( this, gIOServicePlane, true))
2917         // we are the clients only provider
2918         ok = client->terminate( options );
2919     else
2920         ok = true;
2921
2922     return( ok );
2923 }
2924
2925 bool IOService::terminate( IOOptionBits options )
2926 {
2927     options |= kIOServiceTerminate;
2928
2929     return( terminatePhase1( options ));
2930 }
2931
2932 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2933
2934 /*
2935  * Open & close
2936  */
2937
2938 struct ServiceOpenMessageContext
2939 {
2940     IOService *  service;
2941     UInt32       type;
2942     IOService *  excludeClient;
2943     IOOptionBits options;
2944 };
2945
2946 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2947 {
2948     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2949
2950     if( object != context->excludeClient)
2951         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2952 }
2953
2954 bool IOService::open(   IOService *     forClient,
2955                         IOOptionBits    options,
2956                         void *          arg )
2957 {
2958     bool                        ok;
2959     ServiceOpenMessageContext   context;
2960
2961     context.service             = this;
2962     context.type                = kIOMessageServiceIsAttemptingOpen;
2963     context.excludeClient       = forClient;
2964     context.options             = options;
2965
2966     applyToInterested( gIOGeneralInterest,
2967                         &serviceOpenMessageApplier, &context );
2968
2969     if( false == lockForArbitration(false) )
2970         return false;
2971
2972     ok = (0 == (__state[0] & kIOServiceInactiveState));
2973     if( ok)
2974         ok = handleOpen( forClient, options, arg );
2975
2976     unlockForArbitration();
2977
2978     return( ok );
2979 }
2980
2981 void IOService::close(  IOService *     forClient,
2982                         IOOptionBits    options )
2983 {
2984     bool                wasClosed;
2985     bool                last = false;
2986
2987     lockForArbitration();
2988
2989     wasClosed = handleIsOpen( forClient );
2990     if( wasClosed) {
2991         handleClose( forClient, options );
2992         last = (__state[1] & kIOServiceTermPhase3State);
2993     }
2994
2995     unlockForArbitration();
2996
2997     if( last)
2998         forClient->scheduleStop( this );
2999
3000     else if( wasClosed) {
3001
3002         ServiceOpenMessageContext context;
3003
3004         context.service         = this;
3005         context.type            = kIOMessageServiceWasClosed;
3006         context.excludeClient   = forClient;
3007         context.options         = options;
3008
3009         applyToInterested( gIOGeneralInterest,
3010                             &serviceOpenMessageApplier, &context );
3011     }
3012 }
3013
3014 bool IOService::isOpen( const IOService * forClient ) const
3015 {
3016     IOService * self = (IOService *) this;
3017     bool ok;
3018
3019     self->lockForArbitration();
3020
3021     ok = handleIsOpen( forClient );
3022
3023     self->unlockForArbitration();
3024
3025     return( ok );
3026 }
3027
3028 bool IOService::handleOpen(     IOService *     forClient,
3029                                 IOOptionBits    options,
3030                                 void *          arg )
3031 {
3032     bool        ok;
3033
3034     ok = (0 == __owner);
3035     if( ok )
3036         __owner = forClient;
3037
3038     else if( options & kIOServiceSeize ) {
3039         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3040                                 __owner, (void *)(uintptr_t) options ));
3041         if( ok && (0 == __owner ))
3042             __owner = forClient;
3043         else
3044             ok = false;
3045     }
3046     return( ok );
3047 }
3048
3049 void IOService::handleClose(    IOService *     forClient,
3050                                 IOOptionBits    options )
3051 {
3052     if( __owner == forClient)
3053         __owner = 0;
3054 }
3055
3056 bool IOService::handleIsOpen(   const IOService * forClient ) const
3057 {
3058     if( forClient)
3059         return( __owner == forClient );
3060     else
3061         return( __owner != forClient );
3062 }
3063
3064 /*
3065  * Probing & starting
3066  */
3067 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3068 {
3069     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3070     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3071     SInt32             val1;
3072     SInt32             val2;
3073
3074     val1 = 0;
3075     val2 = 0;
3076
3077     if ( obj1 )
3078         val1 = obj1->priority;
3079
3080     if ( obj2 )
3081         val2 = obj2->priority;
3082
3083     return ( val1 - val2 );
3084 }
3085
3086 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
3087 {
3088     OSDictionary *      dict;
3089     IOService *         service;
3090     _IOServiceNotifier * notify;
3091     OSSymbol *          key = (OSSymbol *) ref;
3092     OSNumber *          offset;
3093     OSObject *          prop;
3094     SInt32              result;
3095
3096     prop = 0;
3097     result = kIODefaultProbeScore;
3098     if( (dict = OSDynamicCast( OSDictionary, entry)))
3099         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3100     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
3101         return( notify->priority );
3102     else if( (service = OSDynamicCast( IOService, entry)))
3103     {
3104         prop = service->copyProperty(key);
3105         offset = OSDynamicCast(OSNumber, prop);
3106     }
3107     else {
3108         assert( false );
3109         offset = 0;
3110     }
3111
3112     if (offset) result = offset->unsigned32BitValue();
3113
3114     OSSafeReleaseNULL(prop);
3115
3116     return (result);
3117 }
3118
3119 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3120 {
3121     const OSObject *    obj1 = (const OSObject *) inObj1;
3122     const OSObject *    obj2 = (const OSObject *) inObj2;
3123     SInt32               val1;
3124     SInt32               val2;
3125
3126     val1 = 0;
3127     val2 = 0;
3128
3129     if ( obj1 )
3130         val1 = IOServiceObjectOrder( obj1, ref );
3131
3132     if ( obj2 )
3133         val2 = IOServiceObjectOrder( obj2, ref );
3134
3135     return ( val1 - val2 );
3136 }
3137
3138 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
3139 {
3140     IOService *         service = 0;
3141     OSIterator *        iter;
3142     const OSSymbol *    nextCat;
3143
3144     iter = getClientIterator();
3145     if( iter) {
3146         while( (service = (IOService *) iter->getNextObject())) {
3147             if( kIOServiceInactiveState & service->__state[0])
3148                 continue;
3149             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3150                         service->getProperty( gIOMatchCategoryKey ));
3151             if( category == nextCat)
3152             {
3153                 service->retain();
3154                 break;
3155             }
3156         }
3157         iter->release();
3158     }
3159     return( service );
3160 }
3161
3162 IOService * IOService::getClientWithCategory( const OSSymbol * category )
3163 {
3164     IOService *
3165     service = copyClientWithCategory(category);
3166     if (service)
3167         service->release();
3168     return (service);
3169 }
3170
3171 bool IOService::invokeNotifier( _IOServiceNotifier * notify )
3172 {
3173     _IOServiceNotifierInvocation invocation;
3174     bool                         willNotify;
3175     bool                         ret = true;
3176     invocation.thread = current_thread();
3177
3178 #if DEBUG_NOTIFIER_LOCKED
3179     uint32_t count;
3180     if ((count = isLockedForArbitration(0)))
3181     {
3182         IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3183         panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3184     }
3185 #endif /* DEBUG_NOTIFIER_LOCKED */
3186
3187     LOCKWRITENOTIFY();
3188     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3189
3190     if( willNotify) {
3191         queue_enter( &notify->handlerInvocations, &invocation,
3192                         _IOServiceNotifierInvocation *, link );
3193     }
3194     UNLOCKNOTIFY();
3195
3196     if( willNotify) {
3197
3198         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3199
3200         LOCKWRITENOTIFY();
3201         queue_remove( &notify->handlerInvocations, &invocation,
3202                         _IOServiceNotifierInvocation *, link );
3203         if( kIOServiceNotifyWaiter & notify->state) {
3204             notify->state &= ~kIOServiceNotifyWaiter;
3205             WAKEUPNOTIFY( notify );
3206         }
3207         UNLOCKNOTIFY();
3208     }
3209
3210     return( ret );
3211 }
3212
3213 bool IOService::invokeNotifiers(OSArray ** willSend)
3214 {
3215     OSArray *            array;
3216     _IOServiceNotifier * notify;
3217     bool                 ret = true;
3218
3219     array = *willSend;
3220     if (!array) return (true);
3221     *willSend = 0;
3222
3223     for( unsigned int idx = 0;
3224          (notify = (_IOServiceNotifier *) array->getObject(idx));
3225          idx++) {
3226         ret &= invokeNotifier(notify);
3227     }
3228     array->release();
3229
3230     return (ret);
3231 }
3232
3233
3234 /*
3235  * Alloc and probe matching classes,
3236  * called on the provider instance
3237  */
3238
3239 void IOService::probeCandidates( OSOrderedSet * matches )
3240 {
3241     OSDictionary        *       match = 0;
3242     OSSymbol            *       symbol;
3243     IOService           *       inst;
3244     IOService           *       newInst;
3245     OSDictionary        *       props;
3246     SInt32                      score;
3247     OSNumber            *       newPri;
3248     OSOrderedSet        *       familyMatches = 0;
3249     OSOrderedSet        *       startList;
3250     OSDictionary        *       startDict = 0;
3251     const OSSymbol      *       category;
3252     OSIterator          *       iter;
3253     _IOServiceNotifier  *               notify;
3254     OSObject            *       nextMatch = 0;
3255     bool                        started;
3256     bool                        needReloc = false;
3257 #if IOMATCHDEBUG
3258     SInt64                      debugFlags;
3259 #endif
3260     IOService * client = NULL;
3261
3262
3263     assert( matches );
3264     while( !needReloc && (nextMatch = matches->getFirstObject())) {
3265
3266         nextMatch->retain();
3267         matches->removeObject(nextMatch);
3268
3269         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3270
3271             if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify );
3272             nextMatch->release();
3273             nextMatch = 0;
3274             continue;
3275
3276         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3277             nextMatch->release();
3278             nextMatch = 0;
3279             continue;
3280         }
3281
3282         props = 0;
3283 #if IOMATCHDEBUG
3284         debugFlags = getDebugFlags( match );
3285 #endif
3286
3287         do {
3288             category = OSDynamicCast( OSSymbol,
3289                         match->getObject( gIOMatchCategoryKey ));
3290             if( 0 == category)
3291                 category = gIODefaultMatchCategoryKey;
3292
3293             if( (client = copyClientWithCategory(category)) ) {
3294 #if IOMATCHDEBUG
3295                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
3296                     LOG("%s: match category %s exists\n", getName(),
3297                                 category->getCStringNoCopy());
3298 #endif
3299                 nextMatch->release();
3300                 nextMatch = 0;
3301
3302                 client->release();
3303                 client = NULL;
3304
3305                 continue;
3306             }
3307
3308             // create a copy now in case its modified during matching
3309             props = OSDictionary::withDictionary( match, match->getCount());
3310             if( 0 == props)
3311                 continue;
3312             props->setCapacityIncrement(1);
3313
3314             // check the nub matches
3315             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
3316                 continue;
3317
3318             // Check to see if driver reloc has been loaded.
3319             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
3320             if( needReloc) {
3321 #if IOMATCHDEBUG
3322                 if( debugFlags & kIOLogCatalogue)
3323                     LOG("%s: stalling for module\n", getName());
3324 #endif
3325                 // If reloc hasn't been loaded, exit;
3326                 // reprobing will occur after reloc has been loaded.
3327                 continue;
3328             }
3329
3330             // reorder on family matchPropertyTable score.
3331             if( 0 == familyMatches)
3332                 familyMatches = OSOrderedSet::withCapacity( 1,
3333                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3334             if( familyMatches)
3335                 familyMatches->setObject( props );
3336
3337         } while( false );
3338
3339         if (nextMatch) {
3340             nextMatch->release();
3341             nextMatch = 0;
3342         }
3343         if( props)
3344             props->release();
3345     }
3346     matches->release();
3347     matches = 0;
3348
3349     if( familyMatches) {
3350
3351         while( !needReloc
3352              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3353
3354             props->retain();
3355             familyMatches->removeObject( props );
3356
3357             inst = 0;
3358             newInst = 0;
3359 #if IOMATCHDEBUG
3360             debugFlags = getDebugFlags( props );
3361 #endif
3362             do {
3363                 symbol = OSDynamicCast( OSSymbol,
3364                                 props->getObject( gIOClassKey));
3365                 if( !symbol)
3366                     continue;
3367
3368                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
3369
3370                 // alloc the driver instance
3371                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3372
3373                 if( !inst || !OSDynamicCast(IOService, inst)) {
3374                     IOLog("Couldn't alloc class \"%s\"\n",
3375                         symbol->getCStringNoCopy());
3376                     continue;
3377                 }
3378
3379                 // init driver instance
3380                 if( !(inst->init( props ))) {
3381 #if IOMATCHDEBUG
3382                     if( debugFlags & kIOLogStart)
3383                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3384 #endif
3385                     continue;
3386                 }
3387                 if( __state[1] & kIOServiceSynchronousState)
3388                     inst->__state[1] |= kIOServiceSynchronousState;
3389
3390                 // give the driver the default match category if not specified
3391                 category = OSDynamicCast( OSSymbol,
3392                             props->getObject( gIOMatchCategoryKey ));
3393                 if( 0 == category)
3394                     category = gIODefaultMatchCategoryKey;
3395                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3396                 // attach driver instance
3397                 if( !(inst->attach( this )))
3398                         continue;
3399
3400                 // pass in score from property table
3401                 score = familyMatches->orderObject( props );
3402
3403                 // & probe the new driver instance
3404 #if IOMATCHDEBUG
3405                 if( debugFlags & kIOLogProbe)
3406                     LOG("%s::probe(%s)\n",
3407                         inst->getMetaClass()->getClassName(), getName());
3408 #endif
3409
3410                 newInst = inst->probe( this, &score );
3411                 inst->detach( this );
3412                 if( 0 == newInst) {
3413 #if IOMATCHDEBUG
3414                     if( debugFlags & kIOLogProbe)
3415                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3416 #endif
3417                     continue;
3418                 }
3419
3420                 // save the score
3421                 newPri = OSNumber::withNumber( score, 32 );
3422                 if( newPri) {
3423                     newInst->setProperty( gIOProbeScoreKey, newPri );
3424                     newPri->release();
3425                 }
3426
3427                 // add to start list for the match category
3428                 if( 0 == startDict)
3429                     startDict = OSDictionary::withCapacity( 1 );
3430                 assert( startDict );
3431                 startList = (OSOrderedSet *)
3432                                 startDict->getObject( category );
3433                 if( 0 == startList) {
3434                     startList = OSOrderedSet::withCapacity( 1,
3435                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3436                     if( startDict && startList) {
3437                         startDict->setObject( category, startList );
3438                         startList->release();
3439                     }
3440                 }
3441                 assert( startList );
3442                 if( startList)
3443                     startList->setObject( newInst );
3444
3445             } while( false );
3446
3447             props->release();
3448             if( inst)
3449                 inst->release();
3450         }
3451         familyMatches->release();
3452         familyMatches = 0;
3453     }
3454
3455     // start the best (until success) of each category
3456
3457     iter = OSCollectionIterator::withCollection( startDict );
3458     if( iter) {
3459         while( (category = (const OSSymbol *) iter->getNextObject())) {
3460
3461             startList = (OSOrderedSet *) startDict->getObject( category );
3462             assert( startList );
3463             if( !startList)
3464                 continue;
3465
3466             started = false;
3467             while( true // (!started)
3468                    && (inst = (IOService *)startList->getFirstObject())) {
3469
3470                 inst->retain();
3471                 startList->removeObject(inst);
3472
3473 #if IOMATCHDEBUG
3474                 debugFlags = getDebugFlags( inst );
3475
3476                 if( debugFlags & kIOLogStart) {
3477                     if( started)
3478                         LOG( "match category exists, skipping " );
3479                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3480                          getName(), inst->getRetainCount());
3481                 }
3482 #endif
3483                 if( false == started)
3484                     started = startCandidate( inst );
3485 #if IOMATCHDEBUG
3486                 if( (debugFlags & kIOLogStart) && (false == started))
3487                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3488                          inst->getRetainCount());
3489 #endif
3490                 inst->release();
3491             }
3492         }
3493         iter->release();
3494     }
3495
3496
3497     // adjust the busy count by +1 if matching is stalled for a module,
3498     // or -1 if a previously stalled matching is complete.
3499     lockForArbitration();
3500     SInt32 adjBusy = 0;
3501     uint64_t regID = getRegistryEntryID();
3502
3503     if( needReloc) {
3504         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3505         if( adjBusy) {
3506
3507             IOServiceTrace(
3508                 IOSERVICE_MODULESTALL,
3509                 (uintptr_t) regID,
3510                 (uintptr_t) (regID >> 32),
3511                 (uintptr_t) this,
3512                 0);
3513
3514             __state[1] |= kIOServiceModuleStallState;
3515         }
3516
3517     } else if( __state[1] & kIOServiceModuleStallState) {
3518
3519             IOServiceTrace(
3520             IOSERVICE_MODULEUNSTALL,
3521             (uintptr_t) regID,
3522             (uintptr_t) (regID >> 32),
3523             (uintptr_t) this,
3524             0);
3525
3526         __state[1] &= ~kIOServiceModuleStallState;
3527         adjBusy = -1;
3528     }
3529     if( adjBusy)
3530         _adjustBusy( adjBusy );
3531     unlockForArbitration();
3532
3533     if( startDict)
3534         startDict->release();
3535 }
3536
3537 /*
3538  * Start a previously attached & probed instance,
3539  * called on exporting object instance
3540  */
3541
3542 bool IOService::startCandidate( IOService * service )
3543 {
3544     bool                ok;
3545
3546     ok = service->attach( this );
3547
3548     if( ok)
3549     {
3550         if (this != gIOResources)
3551         {
3552             // stall for any nub resources
3553             checkResources();
3554             // stall for any driver resources
3555             service->checkResources();
3556         }
3557
3558         AbsoluteTime startTime;
3559         AbsoluteTime endTime;
3560         UInt64       nano;
3561
3562         if (kIOLogStart & gIOKitDebug)
3563             clock_get_uptime(&startTime);
3564
3565         ok = service->start(this);
3566
3567         if (kIOLogStart & gIOKitDebug)
3568         {
3569             clock_get_uptime(&endTime);
3570
3571             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3572             {
3573                 SUB_ABSOLUTETIME(&endTime, &startTime);
3574                 absolutetime_to_nanoseconds(endTime, &nano);
3575                 if (nano > 500000000ULL)
3576                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3577             }
3578         }
3579         if( !ok)
3580             service->detach( this );
3581     }
3582     return( ok );
3583 }
3584
3585 void IOService::publishResource( const char * key, OSObject * value )
3586 {
3587     const OSSymbol *    sym;
3588
3589     if( (sym = OSSymbol::withCString( key))) {
3590         publishResource( sym, value);
3591         sym->release();
3592     }
3593 }
3594
3595 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3596 {
3597     if( 0 == value)
3598         value = (OSObject *) gIOServiceKey;
3599
3600     gIOResources->setProperty( key, value);
3601
3602     if( IORecursiveLockHaveLock( gNotificationLock))
3603         return;
3604
3605     gIOResourceGenerationCount++;
3606     gIOResources->registerService();
3607 }
3608
3609 bool IOService::addNeededResource( const char * key )
3610 {
3611     OSObject *  resourcesProp;
3612     OSSet *     set;
3613     OSString *  newKey;
3614     bool ret;
3615
3616     resourcesProp = copyProperty( gIOResourceMatchKey );
3617     if (!resourcesProp) return(false);
3618
3619     newKey = OSString::withCString( key );
3620     if (!newKey)
3621     {
3622         resourcesProp->release();
3623         return( false);
3624     }
3625
3626     set = OSDynamicCast( OSSet, resourcesProp );
3627     if( !set) {
3628         set = OSSet::withCapacity( 1 );
3629         if( set)
3630             set->setObject( resourcesProp );
3631     }
3632     else
3633         set->retain();
3634
3635     set->setObject( newKey );
3636     newKey->release();
3637     ret = setProperty( gIOResourceMatchKey, set );
3638     set->release();
3639     resourcesProp->release();
3640
3641     return( ret );
3642 }
3643
3644 bool IOService::checkResource( OSObject * matching )
3645 {
3646     OSString *          str;
3647     OSDictionary *      table;
3648
3649     if( (str = OSDynamicCast( OSString, matching ))) {
3650         if( gIOResources->getProperty( str ))
3651             return( true );
3652     }
3653
3654     if( str)
3655         table = resourceMatching( str );
3656     else if( (table = OSDynamicCast( OSDictionary, matching )))
3657         table->retain();
3658     else {
3659         IOLog("%s: Can't match using: %s\n", getName(),
3660                 matching->getMetaClass()->getClassName());
3661         /* false would stall forever */
3662         return( true );
3663     }
3664
3665     if( gIOKitDebug & kIOLogConfig)
3666         LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
3667
3668     waitForService( table );
3669
3670     if( gIOKitDebug & kIOLogConfig)
3671         LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
3672
3673     return( true );
3674 }
3675
3676 bool IOService::checkResources( void )
3677 {
3678     OSObject *          resourcesProp;
3679     OSSet *             set;
3680     OSIterator *        iter;
3681     bool                ok;
3682
3683     resourcesProp = copyProperty( gIOResourceMatchKey );
3684     if( 0 == resourcesProp)
3685         return( true );
3686
3687     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3688
3689         iter = OSCollectionIterator::withCollection( set );
3690         ok = (0 != iter);
3691         while( ok && (resourcesProp = iter->getNextObject()) )
3692             ok = checkResource( resourcesProp );
3693         if( iter)
3694             iter->release();
3695
3696     } else
3697         ok = checkResource( resourcesProp );
3698
3699     OSSafeReleaseNULL(resourcesProp);
3700
3701     return( ok );
3702 }
3703
3704
3705 void _IOConfigThread::configThread( void )
3706 {
3707     _IOConfigThread *   inst;
3708
3709     do {
3710         if( !(inst = new _IOConfigThread))
3711             continue;
3712         if( !inst->init())
3713             continue;
3714         thread_t unused;
3715         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3716             continue;
3717
3718         return;
3719
3720     } while( false);
3721
3722     if( inst)
3723         inst->release();
3724
3725     return;
3726 }
3727
3728 void _IOConfigThread::free( void )
3729 {
3730     thread_deallocate(current_thread());
3731     OSObject::free();
3732 }
3733
3734 void IOService::doServiceMatch( IOOptionBits options )
3735 {
3736     _IOServiceNotifier * notify;
3737     OSIterator *        iter;
3738     OSOrderedSet *      matches;
3739     OSArray *           resourceKeys = 0;
3740     SInt32              catalogGeneration;
3741     bool                keepGuessing = true;
3742     bool                reRegistered = true;
3743     bool                didRegister;
3744     OSArray *           notifiers[2] = {0};
3745
3746 //    job->nub->deliverNotification( gIOPublishNotification,
3747 //                              kIOServiceRegisteredState, 0xffffffff );
3748
3749     while( keepGuessing ) {
3750
3751         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3752         // the matches list should always be created by findDrivers()
3753         if( matches) {
3754
3755             lockForArbitration();
3756             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3757                 getMetaClass()->addInstance(this);
3758                 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
3759                                      kIOServiceFirstPublishState, 0xffffffff );
3760             }
3761             LOCKREADNOTIFY();
3762             __state[1] &= ~kIOServiceNeedConfigState;
3763             __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
3764             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3765             __state[0] |= kIOServiceRegisteredState;
3766
3767             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3768             if (reRegistered && keepGuessing) {
3769                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3770                         gNotifications->getObject( gIOPublishNotification ) );
3771                 if( iter) {
3772                     while((notify = (_IOServiceNotifier *)
3773                            iter->getNextObject())) {
3774
3775                         if( matchPassive(notify->matching, 0)
3776                          && (kIOServiceNotifyEnable & notify->state))
3777                             matches->setObject( notify );
3778                     }
3779                     iter->release();
3780                 }
3781             }
3782
3783             UNLOCKNOTIFY();
3784             unlockForArbitration();
3785             invokeNotifiers(&notifiers[0]);
3786
3787             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3788             {
3789                 if (this == gIOResources)
3790                 {
3791                    if (resourceKeys) resourceKeys->release();
3792                    resourceKeys = copyPropertyKeys();
3793                 }
3794                 probeCandidates( matches );
3795             }
3796             else
3797                 matches->release();
3798         }
3799
3800         lockForArbitration();
3801         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3802         keepGuessing =
3803                    (reRegistered || (catalogGeneration !=
3804                                         gIOCatalogue->getGenerationCount()))
3805                 && (0 == (__state[0] & kIOServiceInactiveState));
3806
3807         if( keepGuessing)
3808             unlockForArbitration();
3809     }
3810
3811     if( (0 == (__state[0] & kIOServiceInactiveState))
3812      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3813
3814         if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys);
3815
3816         notifiers[0] = copyNotifiers(gIOMatchedNotification,
3817                 kIOServiceMatchedState, 0xffffffff);
3818         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3819             notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
3820                 kIOServiceFirstMatchState, 0xffffffff);
3821     }
3822
3823     __state[1] &= ~kIOServiceConfigRunning;
3824     unlockForArbitration();
3825
3826     if (resourceKeys) resourceKeys->release();
3827
3828     invokeNotifiers(&notifiers[0]);
3829     invokeNotifiers(&notifiers[1]);
3830
3831     lockForArbitration();
3832     __state[1] &= ~kIOServiceConfigState;
3833     scheduleTerminatePhase2();
3834
3835     _adjustBusy( -1 );
3836     unlockForArbitration();
3837 }
3838
3839 UInt32 IOService::_adjustBusy( SInt32 delta )
3840 {
3841     IOService * next;
3842     UInt32      count;
3843     UInt32      result;
3844     bool        wasQuiet, nowQuiet, needWake;
3845
3846     next = this;
3847     result = __state[1] & kIOServiceBusyStateMask;
3848
3849     if( delta) do {
3850         if( next != this)
3851             next->lockForArbitration();
3852         count = next->__state[1] & kIOServiceBusyStateMask;
3853         wasQuiet = (0 == count);
3854         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3855             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3856         else
3857             count += delta;
3858         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3859         nowQuiet = (0 == count);
3860         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3861
3862         if( needWake) {
3863             next->__state[1] &= ~kIOServiceBusyWaiterState;
3864             IOLockLock( gIOServiceBusyLock );
3865             thread_wakeup( (event_t) next);
3866             IOLockUnlock( gIOServiceBusyLock );
3867         }
3868         if( next != this)
3869             next->unlockForArbitration();
3870
3871         if( (wasQuiet || nowQuiet) ) {
3872
3873             uint64_t regID = next->getRegistryEntryID();
3874             IOServiceTrace(
3875                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3876                 (uintptr_t) regID,
3877                 (uintptr_t) (regID >> 32),
3878                 (uintptr_t) next,
3879                 0);
3880
3881             if (wasQuiet)
3882             {
3883                 next->__timeBusy = mach_absolute_time();
3884             }
3885             else
3886             {
3887                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3888                 next->__timeBusy = 0;
3889             }
3890
3891             MessageClientsContext context;
3892
3893             context.service  = next;
3894             context.type     = kIOMessageServiceBusyStateChange;
3895             context.argument = (void *) wasQuiet;       /*nowBusy*/
3896             context.argSize  = 0;
3897
3898             applyToInterestNotifiers( next, gIOBusyInterest,
3899                                      &messageClientsApplier, &context );
3900
3901 #if !NO_KEXTD
3902             if( nowQuiet && (next == gIOServiceRoot)) {
3903                 OSKext::considerUnloads();
3904                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3905             }
3906 #endif
3907         }
3908
3909         delta = nowQuiet ? -1 : +1;
3910
3911     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3912
3913     return( result );
3914 }
3915
3916 void IOService::adjustBusy( SInt32 delta )
3917 {
3918     lockForArbitration();
3919     _adjustBusy( delta );
3920     unlockForArbitration();
3921 }
3922
3923 uint64_t IOService::getAccumulatedBusyTime( void )
3924 {
3925     uint64_t accumBusy = __accumBusy;
3926     uint64_t timeBusy = __timeBusy;
3927     uint64_t nano;
3928
3929     do
3930     {
3931         accumBusy = __accumBusy;
3932         timeBusy  = __timeBusy;
3933         if (timeBusy)
3934             accumBusy += mach_absolute_time() - timeBusy;
3935     }
3936     while (timeBusy != __timeBusy);
3937
3938     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3939
3940     return (nano);
3941 }
3942
3943 UInt32 IOService::getBusyState( void )
3944 {
3945     return( __state[1] & kIOServiceBusyStateMask );
3946 }
3947
3948 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3949                                   mach_timespec_t * timeout )
3950 {
3951     panic("waitForState");
3952     return (kIOReturnUnsupported);
3953 }
3954
3955 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3956                                    uint64_t timeout )
3957 {
3958     bool            wait;
3959     int             waitResult = THREAD_AWAKENED;
3960     bool            computeDeadline = true;
3961     AbsoluteTime    abstime;
3962
3963     do {
3964         lockForArbitration();
3965         IOLockLock( gIOServiceBusyLock );
3966         wait = (value != (__state[1] & mask));
3967         if( wait) {
3968             __state[1] |= kIOServiceBusyWaiterState;
3969             unlockForArbitration();
3970             if( timeout != UINT64_MAX ) {
3971                 if( computeDeadline ) {
3972                     AbsoluteTime  nsinterval;
3973                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3974                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3975                     computeDeadline = false;
3976                 }
3977                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3978             }
3979             else
3980                 assert_wait((event_t)this, THREAD_UNINT );
3981         } else
3982             unlockForArbitration();
3983         IOLockUnlock( gIOServiceBusyLock );
3984         if( wait)
3985             waitResult = thread_block(THREAD_CONTINUE_NULL);
3986
3987     } while( wait && (waitResult != THREAD_TIMED_OUT));
3988
3989     if( waitResult == THREAD_TIMED_OUT)
3990         return( kIOReturnTimeout );
3991     else
3992         return( kIOReturnSuccess );
3993 }
3994
3995 IOReturn IOService::waitQuiet( uint64_t timeout )
3996 {
3997     IOReturn ret;
3998     uint32_t loops;
3999     char *   string = NULL;
4000     char *   panicString = NULL;
4001     size_t   len;
4002     size_t   panicStringLen;
4003     uint64_t time;
4004     uint64_t nano;
4005     bool     kextdWait;
4006     bool     dopanic;
4007
4008     enum { kTimeoutExtensions = 4 };
4009
4010     time = mach_absolute_time();
4011     kextdWait = false;
4012     for (loops = 0; loops < kTimeoutExtensions; loops++)
4013     {
4014         ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
4015
4016         if (loops && (kIOReturnSuccess == ret))
4017         {
4018             time = mach_absolute_time() - time;
4019             absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
4020             IOLog("busy extended ok[%d], (%llds, %llds)\n",
4021                   loops, timeout / 1000000000ULL, nano / 1000000000ULL);
4022             break;
4023         }
4024         else if (kIOReturnTimeout != ret) break;
4025         else if (timeout < 41000000000)   break;
4026
4027         {
4028             IORegistryIterator * iter;
4029             OSOrderedSet       * set;
4030             OSOrderedSet       * leaves;
4031             IOService          * next;
4032             IOService          * nextParent;
4033             char               * s;
4034             size_t               l;
4035
4036             len = 256;
4037             panicStringLen = 256;
4038             if (!string)      string      = IONew(char, len);
4039             if (!panicString) panicString = IONew(char, panicStringLen);
4040             set = NULL;
4041             kextdWait = OSKext::isWaitingKextd();
4042             iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
4043             leaves = OSOrderedSet::withCapacity(4);
4044             if (iter) set = iter->iterateAll();
4045             if (string && panicString && leaves && set)
4046             {
4047                 string[0] = panicString[0] = 0;
4048                 set->setObject(this);
4049                 while ((next = (IOService *) set->getLastObject()))
4050                 {
4051                     if (next->getBusyState())
4052                     {
4053                         if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true;
4054                         leaves->setObject(next);
4055                         nextParent = next;
4056                         while ((nextParent = nextParent->getProvider()))
4057                         {
4058                             set->removeObject(nextParent);
4059                             leaves->removeObject(nextParent);
4060                         }
4061                     }
4062                     set->removeObject(next);
4063                 }
4064                 s = string;
4065                 while ((next = (IOService *) leaves->getLastObject()))
4066                 {
4067                     l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
4068                     if (l >= len) break;
4069                     s += l;
4070                     len -= l;
4071                     leaves->removeObject(next);
4072                 }
4073             }
4074             OSSafeReleaseNULL(leaves);
4075             OSSafeReleaseNULL(set);
4076             OSSafeReleaseNULL(iter);
4077         }
4078
4079         dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug));
4080         snprintf(panicString, panicStringLen,
4081                  "%s[%d], (%llds): %s",
4082                  kextdWait ? "kextd stall" : "busy timeout",
4083                  loops, timeout / 1000000000ULL,
4084                  string ? string : "");
4085         IOLog("%s\n", panicString);
4086         if (dopanic)     panic("%s", panicString);
4087         else if (!loops) getPMRootDomain()->startSpinDump(1);
4088     }
4089
4090     if (string)      IODelete(string, char, 256);
4091     if (panicString) IODelete(panicString, char, panicStringLen);
4092
4093     return (ret);
4094 }
4095
4096 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
4097 {
4098     uint64_t    timeoutNS;
4099
4100     if (timeout)
4101     {
4102         timeoutNS = timeout->tv_sec;
4103         timeoutNS *= kSecondScale;
4104         timeoutNS += timeout->tv_nsec;
4105     }
4106     else
4107         timeoutNS = UINT64_MAX;
4108
4109     return (waitQuiet(timeoutNS));
4110 }
4111
4112 bool IOService::serializeProperties( OSSerialize * s ) const
4113 {
4114 #if 0
4115     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
4116                 sizeof( __state), "__state");
4117 #endif
4118     return( super::serializeProperties(s) );
4119 }
4120
4121
4122 void _IOConfigThread::main(void * arg, wait_result_t result)
4123 {
4124     _IOConfigThread * self = (_IOConfigThread *) arg;
4125     _IOServiceJob * job;
4126     IOService   *   nub;
4127     bool            alive = true;
4128     kern_return_t   kr;
4129     thread_precedence_policy_data_t precedence = { -1 };
4130
4131     kr = thread_policy_set(current_thread(),
4132                             THREAD_PRECEDENCE_POLICY,
4133                             (thread_policy_t) &precedence,
4134                             THREAD_PRECEDENCE_POLICY_COUNT);
4135     if (KERN_SUCCESS != kr)
4136         IOLog("thread_policy_set(%d)\n", kr);
4137
4138     do {
4139
4140 //      randomDelay();
4141
4142         semaphore_wait( gJobsSemaphore );
4143
4144         IOTakeLock( gJobsLock );
4145         job = (_IOServiceJob *) gJobs->getFirstObject();
4146         job->retain();
4147         gJobs->removeObject(job);
4148         if( job) {
4149             gOutstandingJobs--;
4150 //          gNumConfigThreads--;        // we're out of service
4151             gNumWaitingThreads--;       // we're out of service
4152         }
4153         IOUnlock( gJobsLock );
4154
4155         if( job) {
4156
4157           nub = job->nub;
4158
4159           if( gIOKitDebug & kIOLogConfig)
4160             LOG("config(%p): starting on %s, %d\n",
4161                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
4162
4163           switch( job->type) {
4164
4165             case kMatchNubJob:
4166                 nub->doServiceMatch( job->options );
4167                 break;
4168
4169             default:
4170                 LOG("config(%p): strange type (%d)\n",
4171                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
4172                 break;
4173             }
4174
4175             nub->release();
4176             job->release();
4177
4178             IOTakeLock( gJobsLock );
4179             alive = (gOutstandingJobs > gNumWaitingThreads);
4180             if( alive)
4181                 gNumWaitingThreads++;   // back in service
4182 //              gNumConfigThreads++;
4183             else {
4184                 if( 0 == --gNumConfigThreads) {
4185 //                    IOLog("MATCH IDLE\n");
4186                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
4187                 }
4188             }
4189             IOUnlock( gJobsLock );
4190         }
4191
4192     } while( alive );
4193
4194     if( gIOKitDebug & kIOLogConfig)
4195         LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
4196
4197     self->release();
4198 }
4199
4200 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
4201 {
4202     bool            wait;
4203     int             waitResult = THREAD_AWAKENED;
4204     bool            computeDeadline = true;
4205     AbsoluteTime    deadline;
4206
4207     IOLockLock( gJobsLock );
4208     do {
4209         wait = (0 != gNumConfigThreads);
4210         if( wait) {
4211             if( msToWait) {
4212                 if( computeDeadline ) {
4213                     clock_interval_to_deadline(
4214                           msToWait, kMillisecondScale, &deadline );
4215                     computeDeadline = false;
4216                 }
4217                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
4218                                                                 deadline, THREAD_UNINT );
4219                    } else {
4220                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
4221                                                                 THREAD_UNINT );
4222                 }
4223         }
4224     } while( wait && (waitResult != THREAD_TIMED_OUT));
4225         IOLockUnlock( gJobsLock );
4226
4227     if( waitResult == THREAD_TIMED_OUT)
4228         return( kIOReturnTimeout );
4229     else
4230         return( kIOReturnSuccess );
4231 }
4232
4233 void IOService::cpusRunning(void)
4234 {
4235     gCPUsRunning = true;
4236 }
4237
4238 void _IOServiceJob::pingConfig( _IOServiceJob * job )
4239 {
4240     int         count;
4241     bool        create;
4242
4243     assert( job );
4244
4245     IOTakeLock( gJobsLock );
4246
4247     gOutstandingJobs++;
4248     gJobs->setLastObject( job );
4249
4250     count = gNumWaitingThreads;
4251 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
4252
4253     create = (  (gOutstandingJobs > count)
4254                 && ((gNumConfigThreads < kMaxConfigThreads)
4255             || (job->nub == gIOResources)
4256             || !gCPUsRunning));
4257     if( create) {
4258         gNumConfigThreads++;
4259         gNumWaitingThreads++;
4260     }
4261
4262     IOUnlock( gJobsLock );
4263
4264     job->release();
4265
4266     if( create) {
4267         if( gIOKitDebug & kIOLogConfig)
4268             LOG("config(%d): creating\n", gNumConfigThreads - 1);
4269         _IOConfigThread::configThread();
4270     }
4271
4272     semaphore_signal( gJobsSemaphore );
4273 }
4274
4275 struct IOServiceMatchContext
4276 {
4277     OSDictionary * table;
4278     OSObject *     result;
4279     uint32_t       options;
4280     uint32_t       state;
4281     uint32_t       count;
4282     uint32_t       done;
4283 };
4284
4285 bool IOService::instanceMatch(const OSObject * entry, void * context)
4286 {
4287     IOServiceMatchContext * ctx = (typeof(ctx)) context;
4288     IOService *    service = (typeof(service)) entry;
4289     OSDictionary * table   = ctx->table;
4290     uint32_t       options = ctx->options;
4291     uint32_t       state   = ctx->state;
4292     uint32_t       done;
4293     bool           match;
4294
4295     done = 0;
4296     do
4297     {
4298         match = ((state == (state & service->__state[0]))
4299                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
4300         if (!match) break;
4301         ctx->count += table->getCount();
4302         match = service->matchInternal(table, options, &done);
4303         ctx->done += done;
4304     }
4305     while (false);
4306     if (!match)
4307         return (false);
4308
4309     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
4310     {
4311         service->retain();
4312         ctx->result = service;
4313         return (true);
4314     }
4315     else if (!ctx->result)
4316     {
4317         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
4318     }
4319     else
4320     {
4321         ((OSSet *)ctx->result)->setObject(service);
4322     }
4323     return (false);
4324 }
4325
4326 // internal - call with gNotificationLock
4327 OSObject * IOService::copyExistingServices( OSDictionary * matching,
4328                  IOOptionBits inState, IOOptionBits options )
4329 {
4330     OSObject *   current = 0;
4331     OSIterator * iter;
4332     IOService *  service;
4333     OSObject *   obj;
4334     OSString *   str;
4335
4336     if( !matching)
4337         return( 0 );
4338
4339 #if MATCH_DEBUG
4340     OSSerialize * s = OSSerialize::withCapacity(128);
4341     matching->serialize(s);
4342 #endif
4343
4344     if((obj = matching->getObject(gIOProviderClassKey))
4345       && gIOResourcesKey
4346       && gIOResourcesKey->isEqualTo(obj)
4347       && (service = gIOResources))
4348     {
4349         if( (inState == (service->__state[0] & inState))
4350           && (0 == (service->__state[0] & kIOServiceInactiveState))
4351           &&  service->matchPassive(matching, options))
4352         {
4353             if( options & kIONotifyOnce)
4354             {
4355                 service->retain();
4356                 current = service;
4357             }
4358             else
4359                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
4360         }
4361     }
4362     else
4363     {
4364         IOServiceMatchContext ctx;
4365         ctx.table   = matching;
4366         ctx.state   = inState;
4367         ctx.count   = 0;
4368         ctx.done    = 0;
4369         ctx.options = options;
4370         ctx.result  = 0;
4371
4372         if ((str = OSDynamicCast(OSString, obj)))
4373         {
4374             const OSSymbol * sym = OSSymbol::withString(str);
4375             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
4376             sym->release();
4377         }
4378         else
4379         {
4380             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
4381         }
4382
4383
4384         current = ctx.result;
4385
4386         options |= kIOServiceInternalDone | kIOServiceClassDone;
4387         if (current && (ctx.done != ctx.count))
4388         {
4389             OSSet *
4390             source = OSDynamicCast(OSSet, current);
4391             current = 0;
4392             while ((service = (IOService *) source->getAnyObject()))
4393             {
4394                 if (service->matchPassive(matching, options))
4395                 {
4396                     if( options & kIONotifyOnce)
4397                     {
4398                         service->retain();
4399                         current = service;
4400                         break;
4401                     }
4402                     if( current)
4403                     {
4404                         ((OSSet *)current)->setObject( service );
4405                     }
4406                     else
4407                     {
4408                         current = OSSet::withObjects(
4409                                         (const OSObject **) &service, 1, 1 );
4410                     }
4411                 }
4412                 source->removeObject(service);
4413             }
4414             source->release();
4415         }
4416     }
4417
4418 #if MATCH_DEBUG
4419     {
4420         OSObject * _current = 0;
4421
4422         iter = IORegistryIterator::iterateOver( gIOServicePlane,
4423                                             kIORegistryIterateRecursively );
4424         if( iter) {
4425             do {
4426                 iter->reset();
4427                 while( (service = (IOService *) iter->getNextObject())) {
4428                     if( (inState == (service->__state[0] & inState))
4429                     && (0 == (service->__state[0] & kIOServiceInactiveState))
4430                     &&  service->matchPassive(matching, 0)) {
4431
4432                         if( options & kIONotifyOnce) {
4433                             service->retain();
4434                             _current = service;
4435                             break;
4436                         }
4437                         if( _current)
4438                             ((OSSet *)_current)->setObject( service );
4439                         else
4440                             _current = OSSet::withObjects(
4441                                             (const OSObject **) &service, 1, 1 );
4442                     }
4443                 }
4444             } while( !service && !iter->isValid());
4445             iter->release();
4446         }
4447
4448
4449         if ( ((current != 0) != (_current != 0))
4450         || (current && _current && !current->isEqualTo(_current)))
4451         {
4452             OSSerialize * s1 = OSSerialize::withCapacity(128);
4453             OSSerialize * s2 = OSSerialize::withCapacity(128);
4454             current->serialize(s1);
4455             _current->serialize(s2);
4456             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
4457                 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4458             s1->release();
4459             s2->release();
4460         }
4461
4462         if (_current) _current->release();
4463     }
4464
4465     s->release();
4466 #endif
4467
4468     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4469         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4470         current->release();
4471         current = iter;
4472     }
4473
4474     return( current );
4475 }
4476
4477 // public version
4478 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4479 {
4480     OSIterator *        iter;
4481
4482     // is a lock even needed?
4483     LOCKWRITENOTIFY();
4484
4485     iter = (OSIterator *) copyExistingServices( matching,
4486                                                 kIOServiceMatchedState );
4487
4488     UNLOCKNOTIFY();
4489
4490     return( iter );
4491 }
4492
4493 IOService * IOService::copyMatchingService( OSDictionary * matching )
4494 {
4495     IOService * service;
4496
4497     // is a lock even needed?
4498     LOCKWRITENOTIFY();
4499
4500     service = (IOService *) copyExistingServices( matching,
4501                                                 kIOServiceMatchedState, kIONotifyOnce );
4502
4503     UNLOCKNOTIFY();
4504
4505     return( service );
4506 }
4507
4508 struct _IOServiceMatchingNotificationHandlerRef
4509 {
4510     IOServiceNotificationHandler handler;
4511     void * ref;
4512 };
4513
4514 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4515                                                    IOService * newService,
4516                                                    IONotifier * notifier )
4517 {
4518     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4519 }
4520
4521 // internal - call with gNotificationLock
4522 IONotifier * IOService::setNotification(
4523             const OSSymbol * type, OSDictionary * matching,
4524             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4525             SInt32 priority )
4526 {
4527     _IOServiceNotifier * notify = 0;
4528     OSOrderedSet *      set;
4529
4530     if( !matching)
4531         return( 0 );
4532
4533     notify = new _IOServiceNotifier;
4534     if( notify && !notify->init()) {
4535         notify->release();
4536         notify = 0;
4537     }
4538
4539     if( notify) {
4540         notify->handler = handler;
4541         notify->target = target;
4542         notify->type = type;
4543         notify->matching = matching;
4544         matching->retain();
4545         if (handler == &_IOServiceMatchingNotificationHandler)
4546         {
4547             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4548             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4549         }
4550         else
4551             notify->ref = ref;
4552         notify->priority = priority;
4553         notify->state = kIOServiceNotifyEnable;
4554         queue_init( &notify->handlerInvocations );
4555
4556         ////// queue
4557
4558         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4559             set = OSOrderedSet::withCapacity( 1,
4560                         IONotifyOrdering, 0 );
4561             if( set) {
4562                 gNotifications->setObject( type, set );
4563                 set->release();
4564             }
4565         }
4566         notify->whence = set;
4567         if( set)
4568             set->setObject( notify );
4569     }
4570
4571     return( notify );
4572 }
4573
4574 // internal - call with gNotificationLock
4575 IONotifier * IOService::doInstallNotification(
4576                         const OSSymbol * type, OSDictionary * matching,
4577                         IOServiceMatchingNotificationHandler handler,
4578                         void * target, void * ref,
4579                         SInt32 priority, OSIterator ** existing )
4580 {
4581     OSIterator *        exist;
4582     IONotifier *        notify;
4583     IOOptionBits        inState;
4584
4585     if( !matching)
4586         return( 0 );
4587
4588     if( type == gIOPublishNotification)
4589         inState = kIOServiceRegisteredState;
4590
4591     else if( type == gIOFirstPublishNotification)
4592         inState = kIOServiceFirstPublishState;
4593
4594     else if (type == gIOMatchedNotification)
4595         inState = kIOServiceMatchedState;
4596
4597     else if (type == gIOFirstMatchNotification)
4598         inState = kIOServiceFirstMatchState;
4599
4600     else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification))
4601         inState = 0;
4602     else
4603         return( 0 );
4604
4605     notify = setNotification( type, matching, handler, target, ref, priority );
4606
4607     if( inState)
4608         // get the current set
4609         exist = (OSIterator *) copyExistingServices( matching, inState );
4610     else
4611         exist = 0;
4612
4613     *existing = exist;
4614
4615     return( notify );
4616 }
4617
4618 #if !defined(__LP64__)
4619 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4620                                         IOServiceNotificationHandler handler,
4621                                         void * target, void * refCon,
4622                                         SInt32 priority, OSIterator ** existing )
4623 {
4624     IONotifier * result;
4625     _IOServiceMatchingNotificationHandlerRef ref;
4626     ref.handler = handler;
4627     ref.ref     = refCon;
4628
4629     result = (_IOServiceNotifier *) installNotification( type, matching,
4630                          &_IOServiceMatchingNotificationHandler,
4631                         target, &ref, priority, existing );
4632     if (result)
4633         matching->release();
4634
4635     return (result);
4636 }
4637 #endif /* !defined(__LP64__) */
4638
4639
4640 IONotifier * IOService::installNotification(
4641                         const OSSymbol * type, OSDictionary * matching,
4642                         IOServiceMatchingNotificationHandler handler,
4643                         void * target, void * ref,
4644                         SInt32 priority, OSIterator ** existing )
4645 {
4646     IONotifier * notify;
4647
4648     LOCKWRITENOTIFY();
4649
4650     notify = doInstallNotification( type, matching, handler, target, ref,
4651                 priority, existing );
4652
4653     // in case handler remove()s
4654     if (notify) notify->retain();
4655
4656     UNLOCKNOTIFY();
4657
4658     return( notify );
4659 }
4660
4661 IONotifier * IOService::addNotification(
4662                         const OSSymbol * type, OSDictionary * matching,
4663                         IOServiceNotificationHandler handler,
4664                         void * target, void * refCon,
4665                         SInt32 priority )
4666 {
4667     IONotifier * result;
4668     _IOServiceMatchingNotificationHandlerRef ref;
4669
4670     ref.handler = handler;
4671     ref.ref     = refCon;
4672
4673     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4674                             target, &ref, priority);
4675
4676     if (result)
4677         matching->release();
4678
4679     return (result);
4680 }
4681
4682 IONotifier * IOService::addMatchingNotification(
4683                         const OSSymbol * type, OSDictionary * matching,
4684                         IOServiceMatchingNotificationHandler handler,
4685                         void * target, void * ref,
4686                         SInt32 priority )
4687 {
4688     OSIterator *                existing = NULL;
4689     IONotifier *                ret;
4690     _IOServiceNotifier *        notify;
4691     IOService *                 next;
4692
4693     ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
4694                 handler, target, ref, priority, &existing );
4695     if (!ret) return (0);
4696
4697     // send notifications for existing set
4698     if (existing)
4699     {
4700         while( (next = (IOService *) existing->getNextObject()))
4701         {
4702             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4703             {
4704                 next->invokeNotifier( notify );
4705             }
4706         }
4707         existing->release();
4708     }
4709
4710     LOCKWRITENOTIFY();
4711     bool removed = (0 == notify->whence);
4712     notify->release();
4713     if (removed) ret = gIOServiceNullNotifier;
4714     UNLOCKNOTIFY();
4715
4716     return( ret );
4717 }
4718
4719 static bool
4720 IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
4721                                   IOService * newService,
4722                                   IONotifier * notifier )
4723 {
4724     return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
4725 }
4726
4727 IONotifier * IOService::addMatchingNotification(
4728                             const OSSymbol * type, OSDictionary * matching,
4729                             SInt32 priority,
4730                             IOServiceMatchingNotificationHandlerBlock handler)
4731 {
4732     IONotifier * notify;
4733     void       * block;
4734
4735     block = Block_copy(handler);
4736     if (!block) return (NULL);
4737
4738     notify = addMatchingNotification(type, matching,
4739                 &IOServiceMatchingNotificationHandlerToBlock, NULL, block, priority);
4740
4741     if (!notify) Block_release(block);
4742
4743     return (notify);
4744 }
4745
4746
4747 bool IOService::syncNotificationHandler(
4748                         void * /* target */, void * ref,
4749                         IOService * newService,
4750                         IONotifier * notifier )
4751 {
4752
4753     LOCKWRITENOTIFY();
4754     if (!*((IOService **) ref))
4755     {
4756         newService->retain();
4757         (*(IOService **) ref) = newService;
4758         WAKEUPNOTIFY(ref);
4759     }
4760     UNLOCKNOTIFY();
4761
4762     return( false );
4763 }
4764
4765 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4766                                                 uint64_t timeout)
4767 {
4768     IONotifier *        notify = 0;
4769     // priority doesn't help us much since we need a thread wakeup
4770     SInt32              priority = 0;
4771     IOService *         result;
4772
4773     if (!matching)
4774         return( 0 );
4775
4776     result = NULL;
4777
4778     LOCKWRITENOTIFY();
4779     do
4780     {
4781         result = (IOService *) copyExistingServices( matching,
4782                             kIOServiceMatchedState, kIONotifyOnce );
4783         if (result)
4784             break;
4785         notify = IOService::setNotification( gIOMatchedNotification, matching,
4786                     &IOService::syncNotificationHandler, (void *) 0,
4787                     &result, priority );
4788          if (!notify)
4789             break;
4790         if (UINT64_MAX != timeout)
4791         {
4792             AbsoluteTime deadline;
4793             nanoseconds_to_absolutetime(timeout, &deadline);
4794             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4795             SLEEPNOTIFYTO(&result, deadline);
4796         }
4797         else
4798         {
4799             SLEEPNOTIFY(&result);
4800         }
4801     }
4802     while( false );
4803
4804     UNLOCKNOTIFY();
4805
4806     if (notify)
4807         notify->remove();       // dequeues
4808
4809     return( result );
4810 }
4811
4812 IOService * IOService::waitForService( OSDictionary * matching,
4813                                         mach_timespec_t * timeout )
4814 {
4815     IOService * result;
4816     uint64_t    timeoutNS;
4817
4818     if (timeout)
4819     {
4820         timeoutNS = timeout->tv_sec;
4821         timeoutNS *= kSecondScale;
4822         timeoutNS += timeout->tv_nsec;
4823     }
4824     else
4825         timeoutNS = UINT64_MAX;
4826
4827     result = waitForMatchingService(matching, timeoutNS);
4828
4829     matching->release();
4830     if (result)
4831         result->release();
4832
4833     return (result);
4834 }
4835
4836 void IOService::deliverNotification( const OSSymbol * type,
4837                             IOOptionBits orNewState, IOOptionBits andNewState )
4838 {
4839     panic("deliverNotification");
4840 }
4841
4842 OSArray * IOService::copyNotifiers(const OSSymbol * type,
4843                                    IOOptionBits orNewState, IOOptionBits andNewState )
4844 {
4845     _IOServiceNotifier * notify;
4846     OSIterator *         iter;
4847     OSArray *            willSend = 0;
4848
4849     lockForArbitration();
4850
4851     if( (0 == (__state[0] & kIOServiceInactiveState))
4852      || (type == gIOTerminatedNotification)
4853      || (type == gIOWillTerminateNotification)) {
4854
4855         LOCKREADNOTIFY();
4856
4857         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4858                     gNotifications->getObject( type ) );
4859
4860         if( iter) {
4861             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4862
4863                 if( matchPassive(notify->matching, 0)
4864                   && (kIOServiceNotifyEnable & notify->state)) {
4865                     if( 0 == willSend)
4866                         willSend = OSArray::withCapacity(8);
4867                     if( willSend)
4868                         willSend->setObject( notify );
4869                 }
4870             }
4871             iter->release();
4872         }
4873         __state[0] = (__state[0] | orNewState) & andNewState;
4874         UNLOCKNOTIFY();
4875     }
4876
4877     unlockForArbitration();
4878
4879     return (willSend);
4880
4881 }
4882
4883 IOOptionBits IOService::getState( void ) const
4884 {
4885     return( __state[0] );
4886 }
4887
4888 /*
4889  * Helpers to make matching objects for simple cases
4890  */
4891
4892 OSDictionary * IOService::serviceMatching( const OSString * name,
4893                         OSDictionary * table )
4894 {
4895
4896     const OSString *    str;
4897
4898     str = OSSymbol::withString(name);
4899     if( !str)
4900         return( 0 );
4901
4902     if( !table)
4903         table = OSDictionary::withCapacity( 2 );
4904     if( table)
4905         table->setObject(gIOProviderClassKey, (OSObject *)str );
4906     str->release();
4907
4908     return( table );
4909 }
4910
4911 OSDictionary * IOService::serviceMatching( const char * name,
4912                         OSDictionary * table )
4913 {
4914     const OSString *    str;
4915
4916     str = OSSymbol::withCString( name );
4917     if( !str)
4918         return( 0 );
4919
4920     table = serviceMatching( str, table );
4921     str->release();
4922     return( table );
4923 }
4924
4925 OSDictionary * IOService::nameMatching( const OSString * name,
4926                         OSDictionary * table )
4927 {
4928     if( !table)
4929         table = OSDictionary::withCapacity( 2 );
4930     if( table)
4931         table->setObject( gIONameMatchKey, (OSObject *)name );
4932
4933     return( table );
4934 }
4935
4936 OSDictionary * IOService::nameMatching( const char * name,
4937                         OSDictionary * table )
4938 {
4939     const OSString *    str;
4940
4941     str = OSSymbol::withCString( name );
4942     if( !str)
4943         return( 0 );
4944
4945     table = nameMatching( str, table );
4946     str->release();
4947     return( table );
4948 }
4949
4950 OSDictionary * IOService::resourceMatching( const OSString * str,
4951                         OSDictionary * table )
4952 {
4953     table = serviceMatching( gIOResourcesKey, table );
4954     if( table)
4955         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4956
4957     return( table );
4958 }
4959
4960 OSDictionary * IOService::resourceMatching( const char * name,
4961                         OSDictionary * table )
4962 {
4963     const OSSymbol *    str;
4964
4965     str = OSSymbol::withCString( name );
4966     if( !str)
4967         return( 0 );
4968
4969     table = resourceMatching( str, table );
4970     str->release();
4971
4972     return( table );
4973 }
4974
4975 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4976                         OSDictionary * table )
4977 {
4978     OSDictionary * properties;
4979
4980     properties = OSDictionary::withCapacity( 2 );
4981     if( !properties)
4982         return( 0 );
4983     properties->setObject( key, value );
4984
4985     if( !table)
4986         table = OSDictionary::withCapacity( 2 );
4987     if( table)
4988         table->setObject( gIOPropertyMatchKey, properties );
4989
4990     properties->release();
4991
4992     return( table );
4993 }
4994
4995 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4996                         OSDictionary * table )
4997 {
4998     OSNumber *     num;
4999
5000     num = OSNumber::withNumber( entryID, 64 );
5001     if( !num)
5002         return( 0 );
5003
5004     if( !table)
5005         table = OSDictionary::withCapacity( 2 );
5006     if( table)
5007         table->setObject( gIORegistryEntryIDKey, num );
5008
5009     if (num)
5010         num->release();
5011
5012     return( table );
5013 }
5014
5015
5016 /*
5017  * _IOServiceNotifier
5018  */
5019
5020 // wait for all threads, other than the current one,
5021 //  to exit the handler
5022
5023 void _IOServiceNotifier::wait()
5024 {
5025     _IOServiceNotifierInvocation * next;
5026     bool doWait;
5027
5028     do {
5029         doWait = false;
5030         queue_iterate( &handlerInvocations, next,
5031                         _IOServiceNotifierInvocation *, link) {
5032             if( next->thread != current_thread() ) {
5033                 doWait = true;
5034                 break;
5035             }
5036         }
5037         if( doWait) {
5038             state |= kIOServiceNotifyWaiter;
5039             SLEEPNOTIFY(this);
5040         }
5041
5042     } while( doWait );
5043 }
5044
5045 void _IOServiceNotifier::free()
5046 {
5047     assert( queue_empty( &handlerInvocations ));
5048
5049     if (handler == &IOServiceMatchingNotificationHandlerToBlock) Block_release(ref);
5050
5051     OSObject::free();
5052 }
5053
5054 void _IOServiceNotifier::remove()
5055 {
5056     LOCKWRITENOTIFY();
5057
5058     if( whence) {
5059         whence->removeObject( (OSObject *) this );
5060         whence = 0;
5061     }
5062     if( matching) {
5063         matching->release();
5064         matching = 0;
5065     }
5066
5067     state &= ~kIOServiceNotifyEnable;
5068
5069     wait();
5070
5071     UNLOCKNOTIFY();
5072
5073     release();
5074 }
5075
5076 bool _IOServiceNotifier::disable()
5077 {
5078     bool        ret;
5079
5080     LOCKWRITENOTIFY();
5081
5082     ret = (0 != (kIOServiceNotifyEnable & state));
5083     state &= ~kIOServiceNotifyEnable;
5084     if( ret)
5085         wait();
5086
5087     UNLOCKNOTIFY();
5088
5089     return( ret );
5090 }
5091
5092 void _IOServiceNotifier::enable( bool was )
5093 {
5094     LOCKWRITENOTIFY();
5095     if( was)
5096         state |= kIOServiceNotifyEnable;
5097     else
5098         state &= ~kIOServiceNotifyEnable;
5099     UNLOCKNOTIFY();
5100 }
5101
5102
5103 /*
5104  * _IOServiceNullNotifier
5105  */
5106
5107 void _IOServiceNullNotifier::taggedRetain(const void *tag) const                      {}
5108 void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const     {}
5109 void _IOServiceNullNotifier::free()                                                   {}
5110 void _IOServiceNullNotifier::wait()                                                   {}
5111 void _IOServiceNullNotifier::remove()                                                 {}
5112 void _IOServiceNullNotifier::enable(bool was)                                         {}
5113 bool _IOServiceNullNotifier::disable()                                { return(false); }
5114
5115 /*
5116  * IOResources
5117  */
5118
5119 IOService * IOResources::resources( void )
5120 {
5121     IOResources *       inst;
5122
5123     inst = new IOResources;
5124     if( inst && !inst->init()) {
5125         inst->release();
5126         inst = 0;
5127     }
5128
5129     return( inst );
5130 }
5131
5132 bool IOResources::init( OSDictionary * dictionary )
5133 {
5134     // Do super init first
5135     if ( !IOService::init() )
5136         return false;
5137
5138     // Allow PAL layer to publish a value
5139     const char *property_name;
5140     int property_value;
5141
5142     pal_get_resource_property( &property_name, &property_value );
5143
5144     if( property_name ) {
5145         OSNumber *num;
5146         const OSSymbol *        sym;
5147
5148         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
5149             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
5150                 this->setProperty( sym, num );
5151                 sym->release();
5152             }
5153             num->release();
5154         }
5155     }
5156
5157     return true;
5158 }
5159
5160 IOReturn IOResources::newUserClient(task_t owningTask, void * securityID,
5161                                     UInt32 type,  OSDictionary * properties,
5162                                     IOUserClient ** handler)
5163 {
5164     return( kIOReturnUnsupported );
5165 }
5166
5167 IOWorkLoop * IOResources::getWorkLoop() const
5168 {
5169     // If we are the resource root
5170     // then use the platform's workloop
5171     if (this == (IOResources *) gIOResources)
5172         return getPlatform()->getWorkLoop();
5173     else
5174         return IOService::getWorkLoop();
5175 }
5176
5177 bool IOResources::matchPropertyTable( OSDictionary * table )
5178 {
5179     OSObject *          prop;
5180     OSString *          str;
5181     OSSet *             set;
5182     OSIterator *        iter;
5183     OSObject *          obj;
5184     OSArray *           keys;
5185     bool                ok = true;
5186
5187     prop = table->getObject( gIOResourceMatchKey );
5188     str = OSDynamicCast( OSString, prop );
5189     if( str)
5190         ok = (0 != getProperty( str ));
5191
5192     else if( (set = OSDynamicCast( OSSet, prop))) {
5193
5194         iter = OSCollectionIterator::withCollection( set );
5195         ok = (iter != 0);
5196         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
5197             ok = (0 != getProperty( str ));
5198
5199         if( iter)
5200             iter->release();
5201     }
5202     else if ((prop = table->getObject(gIOResourceMatchedKey)))
5203     {
5204         obj = copyProperty(gIOResourceMatchedKey);
5205         keys = OSDynamicCast(OSArray, obj);
5206         ok = false;
5207         if (keys)
5208         {
5209             // assuming OSSymbol
5210             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
5211         }
5212         OSSafeReleaseNULL(obj);
5213     }
5214
5215     return( ok );
5216 }
5217
5218 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
5219 {
5220     IOService::updateConsoleUsers(NULL, 0);
5221 }
5222
5223 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
5224 {
5225     IORegistryEntry * regEntry;
5226     OSObject *        locked = kOSBooleanFalse;
5227     uint32_t          idx;
5228     bool              publish;
5229     OSDictionary *    user;
5230     static IOMessage  sSystemPower;
5231     clock_sec_t       now = 0;
5232     clock_usec_t      microsecs;
5233
5234     regEntry = IORegistryEntry::getRegistryRoot();
5235
5236     if (!gIOChosenEntry)
5237         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
5238
5239     IOLockLock(gIOConsoleUsersLock);
5240
5241     if (systemMessage)
5242     {
5243         sSystemPower = systemMessage;
5244 #if HIBERNATION
5245         if (kIOMessageSystemHasPoweredOn == systemMessage)
5246         {
5247             uint32_t lockState = IOHibernateWasScreenLocked();
5248             switch (lockState)
5249             {
5250                 case 0:
5251                     break;
5252                 case kIOScreenLockLocked:
5253                 case kIOScreenLockFileVaultDialog:
5254                     gIOConsoleBooterLockState = kOSBooleanTrue;
5255                     break;
5256                 case kIOScreenLockNoLock:
5257                     gIOConsoleBooterLockState = 0;
5258                     break;
5259                 case kIOScreenLockUnlocked:
5260                 default:
5261                     gIOConsoleBooterLockState = kOSBooleanFalse;
5262                     break;
5263             }
5264         }
5265 #endif /* HIBERNATION */
5266     }
5267
5268     if (consoleUsers)
5269     {
5270         OSNumber * num = 0;
5271         bool       loginLocked = true;
5272
5273         gIOConsoleLoggedIn = false;
5274         for (idx = 0;
5275               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
5276               idx++)
5277         {
5278             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
5279                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
5280
5281             loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
5282             if (!num)
5283             {
5284                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
5285             }
5286         }
5287 #if HIBERNATION
5288         if (!loginLocked) gIOConsoleBooterLockState = 0;
5289         IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
5290                 (num != 0), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
5291                 gIOConsoleLoggedIn, loginLocked);
5292 #endif /* HIBERNATION */
5293         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
5294     }
5295
5296     if (!gIOConsoleLoggedIn
5297      || (kIOMessageSystemWillSleep == sSystemPower)
5298      || (kIOMessageSystemPagingOff == sSystemPower))
5299     {
5300         locked = kOSBooleanTrue;
5301     }
5302 #if HIBERNATION
5303     else if (gIOConsoleBooterLockState)
5304     {
5305         locked = gIOConsoleBooterLockState;
5306     }
5307 #endif /* HIBERNATION */
5308     else if (gIOConsoleLockTime)
5309     {
5310         clock_get_calendar_microtime(&now, &microsecs);
5311         if (gIOConsoleLockTime > now)
5312         {
5313             AbsoluteTime deadline;
5314             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
5315             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
5316         }
5317         else
5318         {
5319             locked = kOSBooleanTrue;
5320         }
5321     }
5322
5323     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
5324     if (publish)
5325     {
5326         regEntry->setProperty(gIOConsoleLockedKey, locked);
5327         if (consoleUsers)
5328         {
5329             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
5330         }
5331         OSIncrementAtomic( &gIOConsoleUsersSeed );
5332     }
5333
5334 #if HIBERNATION
5335     if (gIOChosenEntry)
5336     {
5337         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
5338         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
5339         else                          gIOScreenLockState = kIOScreenLockNoLock;
5340         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
5341
5342         IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
5343                 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != 0), now, systemMessage);
5344     }
5345 #endif /* HIBERNATION */
5346
5347     IOLockUnlock(gIOConsoleUsersLock);
5348
5349     if (publish)
5350     {
5351         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
5352
5353         MessageClientsContext context;
5354
5355         context.service  = getServiceRoot();
5356         context.type     = kIOMessageConsoleSecurityChange;
5357         context.argument = (void *) regEntry;
5358         context.argSize  = 0;
5359
5360         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
5361                                  &messageClientsApplier, &context );
5362     }
5363 }
5364
5365 IOReturn IOResources::setProperties( OSObject * properties )
5366 {
5367     IOReturn                    err;
5368     const OSSymbol *            key;
5369     OSDictionary *              dict;
5370     OSCollectionIterator *      iter;
5371
5372     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
5373     if ( kIOReturnSuccess != err)
5374         return( err );
5375
5376     dict = OSDynamicCast(OSDictionary, properties);
5377     if( 0 == dict)
5378         return( kIOReturnBadArgument);
5379
5380     iter = OSCollectionIterator::withCollection( dict);
5381     if( 0 == iter)
5382         return( kIOReturnBadArgument);
5383
5384     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
5385     {
5386         if (gIOConsoleUsersKey == key) do
5387         {
5388             OSArray * consoleUsers;
5389             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
5390             if (!consoleUsers)
5391                 continue;
5392             IOService::updateConsoleUsers(consoleUsers, 0);
5393         }
5394         while (false);
5395
5396         publishResource( key, dict->getObject(key) );
5397     }
5398
5399     iter->release();
5400
5401     return( kIOReturnSuccess );
5402 }
5403
5404 /*
5405  * Helpers for matching dictionaries.
5406  * Keys existing in matching are checked in properties.
5407  * Keys may be a string or OSCollection of IOStrings
5408  */
5409
5410 bool IOService::compareProperty( OSDictionary * matching,
5411                                  const char *   key )
5412 {
5413     OSObject *  value;
5414     OSObject *  prop;
5415     bool        ok;
5416
5417     value = matching->getObject( key );
5418     if( value)
5419     {
5420         prop = copyProperty(key);
5421         ok = value->isEqualTo(prop);
5422         if (prop) prop->release();
5423     }
5424     else
5425         ok = true;
5426
5427     return( ok );
5428 }
5429
5430
5431 bool IOService::compareProperty( OSDictionary *   matching,
5432                                  const OSString * key )
5433 {
5434     OSObject *  value;
5435     OSObject *  prop;
5436     bool        ok;
5437
5438     value = matching->getObject( key );
5439     if( value)
5440     {
5441         prop = copyProperty(key);
5442         ok = value->isEqualTo(prop);
5443         if (prop) prop->release();
5444     }
5445     else
5446         ok = true;
5447
5448     return( ok );
5449 }
5450
5451 bool IOService::compareProperties( OSDictionary * matching,
5452                                    OSCollection * keys )
5453 {
5454     OSCollectionIterator *      iter;
5455     const OSString *            key;
5456     bool                        ok = true;
5457
5458     if( !matching || !keys)
5459         return( false );
5460
5461     iter = OSCollectionIterator::withCollection( keys );
5462
5463     if( iter) {
5464         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
5465             ok = compareProperty( matching, key );
5466
5467         iter->release();
5468     }
5469     keys->release();    // !! consume a ref !!
5470
5471     return( ok );
5472 }
5473
5474 /* Helper to add a location matching dict to the table */
5475
5476 OSDictionary * IOService::addLocation( OSDictionary * table )
5477 {
5478     OSDictionary *      dict;
5479
5480     if( !table)
5481         return( 0 );
5482
5483     dict = OSDictionary::withCapacity( 1 );
5484     if( dict) {
5485         table->setObject( gIOLocationMatchKey, dict );
5486         dict->release();
5487     }
5488
5489     return( dict );
5490 }
5491
5492 /*
5493  * Go looking for a provider to match a location dict.
5494  */
5495
5496 IOService * IOService::matchLocation( IOService * /* client */ )
5497 {
5498     IOService * parent;
5499
5500     parent = getProvider();
5501
5502     if( parent)
5503         parent = parent->matchLocation( this );
5504
5505     return( parent );
5506 }
5507
5508 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
5509 {
5510     OSString *          matched;
5511     OSObject *          obj;
5512     OSString *          str;
5513     IORegistryEntry *   entry;
5514     OSNumber *          num;
5515     bool                match = true;
5516     bool                changesOK = (0 != (kIOServiceChangesOK & options));
5517     uint32_t            count;
5518     uint32_t            done;
5519
5520     do
5521     {
5522         count = table->getCount();
5523         done = 0;
5524
5525         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
5526         if (str) {
5527             done++;
5528             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
5529 #if MATCH_DEBUG
5530             match = (0 != metaCast( str ));
5531             if ((kIOServiceClassDone & options) && !match) panic("classDone");
5532 #endif
5533             if ((!match) || (done == count)) break;
5534         }
5535
5536         obj = table->getObject( gIONameMatchKey );
5537         if( obj) {
5538             done++;
5539             match = compareNames( obj, changesOK ? &matched : 0 );
5540             if (!match) break;
5541             if( changesOK && matched) {
5542                 // leave a hint as to which name matched
5543                 table->setObject( gIONameMatchedKey, matched );
5544                 matched->release();
5545             }
5546             if (done == count) break;
5547         }
5548
5549         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
5550         if (str)
5551         {
5552             const OSSymbol * sym;
5553             done++;
5554             match = false;
5555             sym = copyLocation();
5556             if (sym) {
5557                 match = sym->isEqualTo( str );
5558                 sym->release();
5559             }
5560             if ((!match) || (done == count)) break;
5561         }
5562
5563         obj = table->getObject( gIOPropertyMatchKey );
5564         if( obj)
5565         {
5566             OSDictionary * dict;
5567             OSDictionary * nextDict;
5568             OSIterator *   iter;
5569             done++;
5570             match = false;
5571             dict = dictionaryWithProperties();
5572             if( dict) {
5573                 nextDict = OSDynamicCast( OSDictionary, obj);
5574                 if( nextDict)
5575                     iter = 0;
5576                 else
5577                     iter = OSCollectionIterator::withCollection(
5578                                 OSDynamicCast(OSCollection, obj));
5579
5580                 while( nextDict
5581                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5582                                             iter->getNextObject()))))) {
5583                     match = dict->isEqualTo( nextDict, nextDict);
5584                     if( match)
5585                         break;
5586                     nextDict = 0;
5587                 }
5588                 dict->release();
5589                 if( iter)
5590                     iter->release();
5591             }
5592             if ((!match) || (done == count)) break;
5593         }
5594
5595         obj = table->getObject( gIOPropertyExistsMatchKey );
5596         if( obj)
5597         {
5598             OSDictionary * dict;
5599             OSString     * nextKey;
5600             OSIterator *   iter;
5601             done++;
5602             match = false;
5603             dict = dictionaryWithProperties();
5604             if( dict) {
5605                 nextKey = OSDynamicCast( OSString, obj);
5606                 if( nextKey)
5607                     iter = 0;
5608                 else
5609                     iter = OSCollectionIterator::withCollection(
5610                                 OSDynamicCast(OSCollection, obj));
5611
5612                 while( nextKey
5613                     || (iter && (0 != (nextKey = OSDynamicCast(OSString,
5614                                             iter->getNextObject()))))) {
5615                     match = (0 != dict->getObject(nextKey));
5616                     if( match)
5617                         break;
5618                     nextKey = 0;
5619                 }
5620                 dict->release();
5621                 if( iter)
5622                     iter->release();
5623             }
5624             if ((!match) || (done == count)) break;
5625         }
5626
5627         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5628         if( str) {
5629             done++;
5630             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5631             match = (this == entry);
5632             if( entry)
5633                 entry->release();
5634             if ((!match) || (done == count)) break;
5635         }
5636
5637         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5638         if (num) {
5639             done++;
5640             match = (getRegistryEntryID() == num->unsigned64BitValue());
5641             if ((!match) || (done == count)) break;
5642         }
5643
5644         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5645         if( num)
5646         {
5647             OSIterator *        iter;
5648             IOService *         service = 0;
5649             UInt32              serviceCount = 0;
5650
5651             done++;
5652             iter = getClientIterator();
5653             if( iter) {
5654                 while( (service = (IOService *) iter->getNextObject())) {
5655                     if( kIOServiceInactiveState & service->__state[0])
5656                         continue;
5657                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5658                         continue;
5659                     ++serviceCount;
5660                 }
5661                 iter->release();
5662             }
5663             match = (serviceCount == num->unsigned32BitValue());
5664             if ((!match) || (done == count)) break;
5665         }
5666
5667 #define propMatch(key)                                  \
5668         obj = table->getObject(key);                    \
5669         if (obj)                                        \
5670         {                                               \
5671             OSObject * prop;                            \
5672             done++;                                     \
5673             prop = copyProperty(key);                   \
5674             match = obj->isEqualTo(prop);               \
5675             if (prop) prop->release();                  \
5676             if ((!match) || (done == count)) break;     \
5677         }
5678         propMatch(gIOBSDNameKey)
5679         propMatch(gIOBSDMajorKey)
5680         propMatch(gIOBSDMinorKey)
5681         propMatch(gIOBSDUnitKey)
5682 #undef propMatch
5683     }
5684     while (false);
5685
5686     if (did) *did = done;
5687     return (match);
5688 }
5689
5690 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5691 {
5692     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5693 }
5694
5695 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5696 {
5697     IOService *         where;
5698     OSDictionary *      nextTable;
5699     SInt32              score;
5700     OSNumber *          newPri;
5701     bool                match = true;
5702     bool                matchParent = false;
5703     uint32_t            count;
5704     uint32_t            done;
5705
5706     assert( table );
5707
5708 #if !CONFIG_EMBEDDED
5709     OSArray* aliasServiceRegIds = NULL;
5710     IOService* foundAlternateService = NULL;
5711 #endif
5712
5713 #if MATCH_DEBUG
5714     OSDictionary * root = table;
5715 #endif
5716
5717     where = this;
5718     do
5719     {
5720         do
5721         {
5722             count = table->getCount();
5723             if (!(kIOServiceInternalDone & options))
5724             {
5725                 match = where->matchInternal(table, options, &done);
5726                 // don't call family if we've done all the entries in the table
5727                 if ((!match) || (done == count)) break;
5728             }
5729
5730             // pass in score from property table
5731             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5732
5733             // do family specific matching
5734             match = where->matchPropertyTable( table, &score );
5735
5736             if( !match) {
5737 #if IOMATCHDEBUG
5738                 if( kIOLogMatch & getDebugFlags( table ))
5739                     LOG("%s: family specific matching fails\n", where->getName());
5740 #endif
5741                 break;
5742             }
5743
5744             if (kIOServiceChangesOK & options) {
5745                 // save the score
5746                 newPri = OSNumber::withNumber( score, 32 );
5747                 if( newPri) {
5748                     table->setObject( gIOProbeScoreKey, newPri );
5749                     newPri->release();
5750                 }
5751             }
5752
5753             options = 0;
5754             matchParent = false;
5755
5756             nextTable = OSDynamicCast(OSDictionary,
5757                   table->getObject( gIOParentMatchKey ));
5758             if( nextTable) {
5759                 // look for a matching entry anywhere up to root
5760                 match = false;
5761                 matchParent = true;
5762                 table = nextTable;
5763                 break;
5764             }
5765
5766             table = OSDynamicCast(OSDictionary,
5767                     table->getObject( gIOLocationMatchKey ));
5768             if (table) {
5769                 // look for a matching entry at matchLocation()
5770                 match = false;
5771                 where = where->getProvider();
5772                 if (where && (where = where->matchLocation(where))) continue;
5773             }
5774             break;
5775         }
5776         while (true);
5777
5778         if(match == true) {
5779             break;
5780         }
5781
5782         if(matchParent == true) {
5783 #if !CONFIG_EMBEDDED
5784             // check if service has an alias to search its other "parents" if a parent match isn't found
5785             OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
5786             OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
5787             if(alternateRegistryID != NULL) {
5788                 if(aliasServiceRegIds == NULL)
5789                 {
5790                     aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
5791                 }
5792                 aliasServiceRegIds->setObject(alternateRegistryID);
5793             }
5794             OSSafeReleaseNULL(prop);
5795 #endif
5796         }
5797         else {
5798             break;
5799         }
5800
5801         where = where->getProvider();
5802 #if !CONFIG_EMBEDDED
5803         if(where == NULL) {
5804             // there were no matching parent services, check to see if there are aliased services that have a matching parent
5805             if(aliasServiceRegIds != NULL) {
5806                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
5807                 if(numAliasedServices != 0) {
5808                     OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
5809                     if(alternateRegistryID != NULL) {
5810                         OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
5811                         aliasServiceRegIds->removeObject(numAliasedServices - 1);
5812                         if(alternateMatchingDict != NULL) {
5813                             OSSafeReleaseNULL(foundAlternateService);
5814                             foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
5815                             alternateMatchingDict->release();
5816                             if(foundAlternateService != NULL) {
5817                                 where = foundAlternateService;
5818                             }
5819                         }
5820                     }
5821                 }
5822             }
5823         }
5824 #endif
5825     }
5826     while( where != NULL );
5827
5828 #if !CONFIG_EMBEDDED
5829     OSSafeReleaseNULL(foundAlternateService);
5830     OSSafeReleaseNULL(aliasServiceRegIds);
5831 #endif
5832
5833 #if MATCH_DEBUG
5834     if (where != this)
5835     {
5836         OSSerialize * s = OSSerialize::withCapacity(128);
5837         root->serialize(s);
5838         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5839         s->release();
5840     }
5841 #endif
5842
5843     return( match );
5844 }
5845
5846
5847 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5848                                     UInt32 type,  OSDictionary * properties,
5849                                     IOUserClient ** handler )
5850 {
5851     const OSSymbol *userClientClass = 0;
5852     IOUserClient *client;
5853     OSObject *prop;
5854     OSObject *temp;
5855
5856     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5857         return kIOReturnSuccess;
5858
5859     // First try my own properties for a user client class name
5860     prop = copyProperty(gIOUserClientClassKey);
5861     if (prop) {
5862         if (OSDynamicCast(OSSymbol, prop))
5863             userClientClass = (const OSSymbol *) prop;
5864         else if (OSDynamicCast(OSString, prop)) {
5865             userClientClass = OSSymbol::withString((OSString *) prop);
5866             if (userClientClass)
5867                 setProperty(gIOUserClientClassKey,
5868                             (OSObject *) userClientClass);
5869         }
5870     }
5871
5872     // Didn't find one so lets just bomb out now without further ado.
5873     if (!userClientClass)
5874     {
5875         OSSafeReleaseNULL(prop);
5876         return kIOReturnUnsupported;
5877     }
5878
5879     // This reference is consumed by the IOServiceOpen call
5880     temp = OSMetaClass::allocClassWithName(userClientClass);
5881     OSSafeReleaseNULL(prop);
5882     if (!temp)
5883         return kIOReturnNoMemory;
5884
5885     if (OSDynamicCast(IOUserClient, temp))
5886         client = (IOUserClient *) temp;
5887     else {
5888         temp->release();
5889         return kIOReturnUnsupported;
5890     }
5891
5892     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5893         client->release();
5894         return kIOReturnBadArgument;
5895     }
5896
5897     if ( !client->attach(this) ) {
5898         client->release();
5899         return kIOReturnUnsupported;
5900     }
5901
5902     if ( !client->start(this) ) {
5903         client->detach(this);
5904         client->release();
5905         return kIOReturnUnsupported;
5906     }
5907
5908     *handler = client;
5909     return kIOReturnSuccess;
5910 }
5911
5912 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5913                                     UInt32 type, IOUserClient ** handler )
5914 {
5915     return( kIOReturnUnsupported );
5916 }
5917
5918 IOReturn IOService::requestProbe( IOOptionBits options )
5919 {
5920     return( kIOReturnUnsupported);
5921 }
5922
5923 /*
5924  * Convert an IOReturn to text. Subclasses which add additional
5925  * IOReturn's should override this method and call
5926  * super::stringFromReturn if the desired value is not found.
5927  */
5928
5929 const char * IOService::stringFromReturn( IOReturn rtn )
5930 {
5931     static const IONamedValue IOReturn_values[] = {
5932         {kIOReturnSuccess,          "success"                           },
5933         {kIOReturnError,            "general error"                     },
5934         {kIOReturnNoMemory,         "memory allocation error"           },
5935         {kIOReturnNoResources,      "resource shortage"                 },
5936         {kIOReturnIPCError,         "Mach IPC failure"                  },
5937         {kIOReturnNoDevice,         "no such device"                    },
5938         {kIOReturnNotPrivileged,    "privilege violation"               },
5939         {kIOReturnBadArgument,      "invalid argument"                  },
5940         {kIOReturnLockedRead,       "device is read locked"             },
5941         {kIOReturnLockedWrite,      "device is write locked"            },
5942         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5943         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5944         {kIOReturnUnsupported,      "unsupported function"              },
5945         {kIOReturnVMError,          "virtual memory error"              },
5946         {kIOReturnInternalError,    "internal driver error"             },
5947         {kIOReturnIOError,          "I/O error"                         },
5948         {kIOReturnCannotLock,       "cannot acquire lock"               },
5949         {kIOReturnNotOpen,          "device is not open"                },
5950         {kIOReturnNotReadable,      "device is not readable"            },
5951         {kIOReturnNotWritable,      "device is not writeable"           },
5952         {kIOReturnNotAligned,       "alignment error"                   },
5953         {kIOReturnBadMedia,         "media error"                       },
5954         {kIOReturnStillOpen,        "device is still open"              },
5955         {kIOReturnRLDError,         "rld failure"                       },
5956         {kIOReturnDMAError,         "DMA failure"                       },
5957         {kIOReturnBusy,             "device is busy"                    },
5958         {kIOReturnTimeout,          "I/O timeout"                       },
5959         {kIOReturnOffline,          "device is offline"                 },
5960         {kIOReturnNotReady,         "device is not ready"               },
5961         {kIOReturnNotAttached,      "device/channel is not attached"    },
5962         {kIOReturnNoChannels,       "no DMA channels available"         },
5963         {kIOReturnNoSpace,          "no space for data"                 },
5964         {kIOReturnPortExists,       "device port already exists"        },
5965         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5966         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5967         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5968         {kIOReturnMessageTooLarge,  "message is too large"              },
5969         {kIOReturnNotPermitted,     "operation is not permitted"        },
5970         {kIOReturnNoPower,          "device is without power"           },
5971         {kIOReturnNoMedia,          "media is not present"              },
5972         {kIOReturnUnformattedMedia, "media is not formatted"            },
5973         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5974         {kIOReturnUnderrun,         "data underrun"                     },
5975         {kIOReturnOverrun,          "data overrun"                      },
5976         {kIOReturnDeviceError,      "device error"                      },
5977         {kIOReturnNoCompletion,     "no completion routine"             },
5978         {kIOReturnAborted,          "operation was aborted"             },
5979         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5980         {kIOReturnNotResponding,    "device is not responding"          },
5981         {kIOReturnInvalid,          "unanticipated driver error"        },
5982         {0,                         NULL                                }
5983     };
5984
5985     return IOFindNameForValue(rtn, IOReturn_values);
5986 }
5987
5988 /*
5989  * Convert an IOReturn to an errno.
5990  */
5991 int IOService::errnoFromReturn( IOReturn rtn )
5992 {
5993     if (unix_err(err_get_code(rtn)) == rtn)
5994         return err_get_code(rtn);
5995
5996     switch(rtn) {
5997         // (obvious match)
5998         case kIOReturnSuccess:
5999             return(0);
6000         case kIOReturnNoMemory:
6001             return(ENOMEM);
6002         case kIOReturnNoDevice:
6003             return(ENXIO);
6004         case kIOReturnVMError:
6005             return(EFAULT);
6006         case kIOReturnNotPermitted:
6007             return(EPERM);
6008         case kIOReturnNotPrivileged:
6009             return(EACCES);
6010         case kIOReturnIOError:
6011             return(EIO);
6012         case kIOReturnNotWritable:
6013             return(EROFS);
6014         case kIOReturnBadArgument:
6015             return(EINVAL);
6016         case kIOReturnUnsupported:
6017             return(ENOTSUP);
6018         case kIOReturnBusy:
6019             return(EBUSY);
6020         case kIOReturnNoPower:
6021             return(EPWROFF);
6022         case kIOReturnDeviceError:
6023             return(EDEVERR);
6024         case kIOReturnTimeout:
6025             return(ETIMEDOUT);
6026         case kIOReturnMessageTooLarge:
6027             return(EMSGSIZE);
6028         case kIOReturnNoSpace:
6029             return(ENOSPC);
6030         case kIOReturnCannotLock:
6031             return(ENOLCK);
6032
6033         // (best match)
6034         case kIOReturnBadMessageID:
6035         case kIOReturnNoCompletion:
6036         case kIOReturnNotAligned:
6037             return(EINVAL);
6038         case kIOReturnNotReady:
6039             return(EBUSY);
6040         case kIOReturnRLDError:
6041             return(EBADMACHO);
6042         case kIOReturnPortExists:
6043         case kIOReturnStillOpen:
6044             return(EEXIST);
6045         case kIOReturnExclusiveAccess:
6046         case kIOReturnLockedRead:
6047         case kIOReturnLockedWrite:
6048         case kIOReturnNotOpen:
6049         case kIOReturnNotReadable:
6050             return(EACCES);
6051         case kIOReturnCannotWire:
6052         case kIOReturnNoResources:
6053             return(ENOMEM);
6054         case kIOReturnAborted:
6055         case kIOReturnOffline:
6056         case kIOReturnNotResponding:
6057             return(EBUSY);
6058         case kIOReturnBadMedia:
6059         case kIOReturnNoMedia:
6060         case kIOReturnNotAttached:
6061         case kIOReturnUnformattedMedia:
6062             return(ENXIO); // (media error)
6063         case kIOReturnDMAError:
6064         case kIOReturnOverrun:
6065         case kIOReturnUnderrun:
6066             return(EIO); // (transfer error)
6067         case kIOReturnNoBandwidth:
6068         case kIOReturnNoChannels:
6069         case kIOReturnNoFrames:
6070         case kIOReturnNoInterrupt:
6071             return(EIO); // (hardware error)
6072         case kIOReturnError:
6073         case kIOReturnInternalError:
6074         case kIOReturnInvalid:
6075             return(EIO); // (generic error)
6076         case kIOReturnIPCError:
6077             return(EIO); // (ipc error)
6078         default:
6079             return(EIO); // (all other errors)
6080     }
6081 }
6082
6083 IOReturn IOService::message( UInt32 type, IOService * provider,
6084                                 void * argument )
6085 {
6086     /*
6087      * Generic entry point for calls from the provider.  A return value of
6088      * kIOReturnSuccess indicates that the message was received, and where
6089      * applicable, that it was successful.
6090      */
6091
6092     return kIOReturnUnsupported;
6093 }
6094
6095 /*
6096  * Device memory
6097  */
6098
6099 IOItemCount IOService::getDeviceMemoryCount( void )
6100 {
6101     OSArray *           array;
6102     IOItemCount         count;
6103
6104     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6105     if( array)
6106         count = array->getCount();
6107     else
6108         count = 0;
6109
6110     return( count);
6111 }
6112
6113 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
6114 {
6115     OSArray *           array;
6116     IODeviceMemory *    range;
6117
6118     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6119     if( array)
6120         range = (IODeviceMemory *) array->getObject( index );
6121     else
6122         range = 0;
6123
6124     return( range);
6125 }
6126
6127 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
6128                                                 IOOptionBits options )
6129 {
6130     IODeviceMemory *    range;
6131     IOMemoryMap *       map;
6132
6133     range = getDeviceMemoryWithIndex( index );
6134     if( range)
6135         map = range->map( options );
6136     else
6137         map = 0;
6138
6139     return( map );
6140 }
6141
6142 OSArray * IOService::getDeviceMemory( void )
6143 {
6144     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
6145 }
6146
6147
6148 void IOService::setDeviceMemory( OSArray * array )
6149 {
6150     setProperty( gIODeviceMemoryKey, array);
6151 }
6152
6153 /*
6154  * For machines where the transfers on an I/O bus can stall because
6155  * the CPU is in an idle mode, These APIs allow a driver to specify
6156  * the maximum bus stall that they can handle.  0 indicates no limit.
6157  */
6158 void IOService::
6159 setCPUSnoopDelay(UInt32 __unused ns)
6160 {
6161 #if defined(__i386__) || defined(__x86_64__)
6162     ml_set_maxsnoop(ns);
6163 #endif /* defined(__i386__) || defined(__x86_64__) */
6164 }
6165
6166 UInt32 IOService::
6167 getCPUSnoopDelay()
6168 {
6169 #if defined(__i386__) || defined(__x86_64__)
6170     return ml_get_maxsnoop();
6171 #else
6172     return 0;
6173 #endif /* defined(__i386__) || defined(__x86_64__) */
6174 }
6175
6176 #if defined(__i386__) || defined(__x86_64__)
6177 static void
6178 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
6179 {
6180     static const UInt kNoReplace = -1U; // Must be an illegal index
6181     UInt replace = kNoReplace;
6182     bool setCpuDelay = false;
6183
6184     IORecursiveLockLock(sCpuDelayLock);
6185
6186     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6187     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6188     IOService * holder = NULL;
6189
6190     if (ns) {
6191         const CpuDelayEntry ne = {service, ns, delayType};
6192         holder = service;
6193         // Set maximum delay.
6194         for (UInt i = 0; i < count; i++) {
6195             IOService *thisService = entries[i].fService;
6196             bool sameType = (delayType == entries[i].fDelayType);
6197             if ((service == thisService) && sameType)
6198                 replace = i;
6199             else if (!thisService) {
6200                 if (kNoReplace == replace)
6201                     replace = i;
6202             }
6203             else if (sameType) {
6204                 const UInt32 thisMax = entries[i].fMaxDelay;
6205                 if (thisMax < ns)
6206                 {
6207                     ns = thisMax;
6208                     holder = thisService;
6209                 }
6210             }
6211         }
6212
6213         setCpuDelay = true;
6214         if (kNoReplace == replace)
6215             sCpuDelayData->appendBytes(&ne, sizeof(ne));
6216         else
6217             entries[replace] = ne;
6218     }
6219     else {
6220         ns = -1U;       // Set to max unsigned, i.e. no restriction
6221
6222         for (UInt i = 0; i < count; i++) {
6223             // Clear a maximum delay.
6224             IOService *thisService = entries[i].fService;
6225             if (thisService && (delayType == entries[i].fDelayType)) {
6226                 UInt32 thisMax = entries[i].fMaxDelay;
6227                 if (service == thisService)
6228                     replace = i;
6229                 else if (thisMax < ns) {
6230                     ns = thisMax;
6231                     holder = thisService;
6232                 }
6233             }
6234         }
6235
6236         // Check if entry found
6237         if (kNoReplace != replace) {
6238             entries[replace].fService = 0;      // Null the entry
6239             setCpuDelay = true;
6240         }
6241     }
6242
6243     if (setCpuDelay)
6244     {
6245         if (holder && debug_boot_arg) {
6246             strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
6247         }
6248
6249         // Must be safe to call from locked context
6250         if (delayType == kCpuDelayBusStall)
6251         {
6252             ml_set_maxbusdelay(ns);
6253         }
6254         else if (delayType == kCpuDelayInterrupt)
6255         {
6256             ml_set_maxintdelay(ns);
6257         }
6258         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
6259         sCPULatencySet   [delayType]->setValue(ns);
6260
6261         OSArray * handlers = sCpuLatencyHandlers[delayType];
6262         IOService * target;
6263         if (handlers) for (unsigned int idx = 0;
6264                             (target = (IOService *) handlers->getObject(idx));
6265                             idx++)
6266         {
6267             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6268                                             (void *) (uintptr_t) ns, holder,
6269                                             NULL, NULL);
6270         }
6271     }
6272
6273     IORecursiveLockUnlock(sCpuDelayLock);
6274 }
6275
6276 static IOReturn
6277 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
6278 {
6279     IOReturn result = kIOReturnNotFound;
6280     OSArray * array;
6281     unsigned int idx;
6282
6283     IORecursiveLockLock(sCpuDelayLock);
6284
6285     do
6286     {
6287         if (enable && !sCpuLatencyHandlers[delayType])
6288             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
6289         array = sCpuLatencyHandlers[delayType];
6290         if (!array)
6291             break;
6292         idx = array->getNextIndexOfObject(target, 0);
6293         if (!enable)
6294         {
6295             if (-1U != idx)
6296             {
6297                 array->removeObject(idx);
6298                 result = kIOReturnSuccess;
6299             }
6300         }
6301         else
6302         {
6303             if (-1U != idx) {
6304                 result = kIOReturnExclusiveAccess;
6305                 break;
6306             }
6307             array->setObject(target);
6308
6309             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6310             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6311             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
6312             IOService * holder = NULL;
6313
6314             for (UInt i = 0; i < count; i++) {
6315                 if (entries[i].fService
6316                   && (delayType == entries[i].fDelayType)
6317                   && (entries[i].fMaxDelay < ns)) {
6318                     ns = entries[i].fMaxDelay;
6319                     holder = entries[i].fService;
6320                 }
6321             }
6322             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6323                                             (void *) (uintptr_t) ns, holder,
6324                                             NULL, NULL);
6325             result = kIOReturnSuccess;
6326         }
6327     }
6328     while (false);
6329
6330     IORecursiveLockUnlock(sCpuDelayLock);
6331
6332     return (result);
6333 }
6334
6335 #endif /* defined(__i386__) || defined(__x86_64__) */
6336
6337 void IOService::
6338 requireMaxBusStall(UInt32 __unused ns)
6339 {
6340 #if defined(__i386__) || defined(__x86_64__)
6341     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
6342 #endif
6343 }
6344
6345 void IOService::
6346 requireMaxInterruptDelay(uint32_t __unused ns)
6347 {
6348 #if defined(__i386__) || defined(__x86_64__)
6349     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
6350 #endif
6351 }
6352
6353 /*
6354  * Device interrupts
6355  */
6356
6357 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
6358 {
6359   IOInterruptController *interruptController;
6360   OSArray               *array;
6361   OSData                *data;
6362   OSSymbol              *interruptControllerName;
6363   long                  numSources;
6364   IOInterruptSource     *interruptSources;
6365
6366   // Get the parents list from the nub.
6367   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
6368   if (array == 0) return kIOReturnNoResources;
6369
6370   // Allocate space for the IOInterruptSources if needed... then return early.
6371   if (nub->_interruptSources == 0) {
6372     numSources = array->getCount();
6373     interruptSources = (IOInterruptSource *)IOMalloc(
6374         numSources * sizeofAllIOInterruptSource);
6375     if (interruptSources == 0) return kIOReturnNoMemory;
6376
6377     bzero(interruptSources, numSources * sizeofAllIOInterruptSource);
6378
6379     nub->_numInterruptSources = numSources;
6380     nub->_interruptSources = interruptSources;
6381     return kIOReturnSuccess;
6382   }
6383
6384   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
6385   if (interruptControllerName == 0) return kIOReturnNoResources;
6386
6387   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
6388   if (interruptController == 0) return kIOReturnNoResources;
6389
6390   // Get the interrupt numbers from the nub.
6391   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
6392   if (array == 0) return kIOReturnNoResources;
6393   data = OSDynamicCast(OSData, array->getObject(source));
6394   if (data == 0) return kIOReturnNoResources;
6395
6396   // Set the interruptController and interruptSource in the nub's table.
6397   interruptSources = nub->_interruptSources;
6398   interruptSources[source].interruptController = interruptController;
6399   interruptSources[source].vectorData = data;
6400
6401   return kIOReturnSuccess;
6402 }
6403
6404 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
6405 {
6406   IOReturn ret;
6407
6408   /* Make sure the _interruptSources are set */
6409   if (_interruptSources == 0) {
6410     ret = resolveInterrupt(this, source);
6411     if (ret != kIOReturnSuccess) return ret;
6412   }
6413
6414   /* Make sure the local source number is valid */
6415   if ((source < 0) || (source >= _numInterruptSources))
6416     return kIOReturnNoInterrupt;
6417
6418   /* Look up the contoller for the local source */
6419   *interruptController = _interruptSources[source].interruptController;
6420
6421   if (*interruptController == NULL) {
6422     if (!resolve) return kIOReturnNoInterrupt;
6423
6424     /* Try to resolve the interrupt */
6425     ret = resolveInterrupt(this, source);
6426     if (ret != kIOReturnSuccess) return ret;
6427
6428     *interruptController = _interruptSources[source].interruptController;
6429   }
6430
6431   return kIOReturnSuccess;
6432 }
6433
6434 IOReturn IOService::registerInterrupt(int source, OSObject *target,
6435                                       IOInterruptAction handler,
6436                                       void *refCon)
6437 {
6438   IOInterruptController *interruptController;
6439   IOReturn              ret;
6440
6441   ret = lookupInterrupt(source, true, &interruptController);
6442   if (ret != kIOReturnSuccess) return ret;
6443
6444   /* Register the source */
6445   return interruptController->registerInterrupt(this, source, target,
6446                                                 (IOInterruptHandler)handler,
6447                                                 refCon);
6448 }
6449
6450 static void IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
6451                    IOService * nub, int source )
6452 {
6453   ((IOInterruptActionBlock)(refCon))(nub, source);
6454 }
6455
6456 IOReturn IOService::registerInterruptBlock(int source, OSObject *target,
6457                                       IOInterruptActionBlock handler)
6458 {
6459     IOReturn ret;
6460     void   * block;
6461
6462     block = Block_copy(handler);
6463     if (!block) return (kIOReturnNoMemory);
6464
6465     ret = registerInterrupt(source, target, &IOServiceInterruptActionToBlock, block);
6466     if (kIOReturnSuccess != ret) {
6467       Block_release(block);
6468       return (ret);
6469     }
6470     _interruptSourcesPrivate(this)[source].vectorBlock = block;
6471
6472     return (ret);
6473 }
6474
6475 IOReturn IOService::unregisterInterrupt(int source)
6476 {
6477   IOReturn              ret;
6478   IOInterruptController *interruptController;
6479   void                  *block;
6480
6481   ret = lookupInterrupt(source, false, &interruptController);
6482   if (ret != kIOReturnSuccess) return ret;
6483
6484   /* Unregister the source */
6485   block = _interruptSourcesPrivate(this)[source].vectorBlock;
6486   ret = interruptController->unregisterInterrupt(this, source);
6487   if ((kIOReturnSuccess == ret) && (block = _interruptSourcesPrivate(this)[source].vectorBlock)) {
6488     _interruptSourcesPrivate(this)[source].vectorBlock = NULL;
6489     Block_release(block);
6490   }
6491
6492   return ret;
6493 }
6494
6495 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
6496 {
6497   IOReportLegend * legend = NULL;
6498   IOInterruptAccountingData * oldValue = NULL;
6499   IOInterruptAccountingReporter * newArray = NULL;
6500   char subgroupName[64];
6501   int newArraySize = 0;
6502   int i = 0;
6503
6504   if (source < 0) {
6505     return kIOReturnBadArgument;
6506   }
6507
6508   /*
6509    * We support statistics on a maximum of 256 interrupts per nub; if a nub
6510    * has more than 256 interrupt specifiers associated with it, and tries
6511    * to register a high interrupt index with interrupt accounting, panic.
6512    * Having more than 256 interrupts associated with a single nub is
6513    * probably a sign that something fishy is going on.
6514    */
6515   if (source > IA_INDEX_MAX) {
6516     panic("addInterruptStatistics called for an excessively large index (%d)", source);
6517   }
6518
6519   /*
6520    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
6521    * leaving it as is because the likelihood of contention where we are
6522    * actually growing the array is minimal (we would realistically need
6523    * to be starting a driver for the first time, with an IOReporting
6524    * client already in place).  Nonetheless, cleanup that can be done
6525    * to adhere to best practices; it'll make the code more complicated,
6526    * unfortunately.
6527    */
6528   IOLockLock(reserved->interruptStatisticsLock);
6529
6530   /*
6531    * Lazily allocate the statistics array.
6532    */
6533   if (!reserved->interruptStatisticsArray) {
6534     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
6535     assert(reserved->interruptStatisticsArray);
6536     reserved->interruptStatisticsArrayCount = 1;
6537     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
6538   }
6539
6540   if (source >= reserved->interruptStatisticsArrayCount) {
6541     /*
6542      * We're still within the range of supported indices, but we are out
6543      * of space in the current array.  Do a nasty realloc (because
6544      * IORealloc isn't a thing) here.  We'll double the size with each
6545      * reallocation.
6546      *
6547      * Yes, the "next power of 2" could be more efficient; but this will
6548      * be invoked incredibly rarely.  Who cares.
6549      */
6550     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
6551
6552     while (newArraySize <= source)
6553       newArraySize = (newArraySize << 1);
6554     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
6555
6556     assert(newArray);
6557
6558     /*
6559      * TODO: This even zeroes the memory it is about to overwrite.
6560      * Shameful; fix it.  Not particularly high impact, however.
6561      */
6562     bzero(newArray, newArraySize * sizeof(*newArray));
6563     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
6564     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
6565     reserved->interruptStatisticsArray = newArray;
6566     reserved->interruptStatisticsArrayCount = newArraySize;
6567   }
6568
6569   if (!reserved->interruptStatisticsArray[source].reporter) {
6570     /*
6571      * We don't have a reporter associated with this index yet, so we
6572      * need to create one.
6573      */
6574     /*
6575      * TODO: Some statistics do in fact have common units (time); should this be
6576      * split into separate reporters to communicate this?
6577      */
6578      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
6579
6580     /*
6581      * Each statistic is given an identifier based on the interrupt index (which
6582      * should be unique relative to any single nub) and the statistic involved.
6583      * We should now have a sane (small and positive) index, so start
6584      * constructing the channels for statistics.
6585      */
6586     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
6587       /*
6588        * TODO: Currently, this does not add channels for disabled statistics.
6589        * Will this be confusing for clients?  If so, we should just add the
6590        * channels; we can avoid updating the channels even if they exist.
6591        */
6592       if (IA_GET_STATISTIC_ENABLED(i))
6593         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
6594     }
6595
6596     /*
6597      * We now need to add the legend for this reporter to the registry.
6598      */
6599     OSObject * prop = copyProperty(kIOReportLegendKey);
6600     legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
6601     OSSafeReleaseNULL(prop);
6602
6603     /*
6604      * Note that while we compose the subgroup name, we do not need to
6605      * manage its lifecycle (the reporter will handle this).
6606      */
6607     snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
6608     subgroupName[sizeof(subgroupName) - 1] = 0;
6609     legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
6610     setProperty(kIOReportLegendKey, legend->getLegend());
6611     legend->release();
6612
6613     /*
6614      * TODO: Is this a good idea?  Probably not; my assumption is it opts
6615      * all entities who register interrupts into public disclosure of all
6616      * IOReporting channels.  Unfortunately, this appears to be as fine
6617      * grain as it gets.
6618      */
6619     setProperty(kIOReportLegendPublicKey, true);
6620   }
6621
6622   /*
6623    * Don't stomp existing entries.  If we are about to, panic; this
6624    * probably means we failed to tear down our old interrupt source
6625    * correctly.
6626    */
6627   oldValue = reserved->interruptStatisticsArray[source].statistics;
6628
6629   if (oldValue) {
6630     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
6631   }
6632
6633   reserved->interruptStatisticsArray[source].statistics = statistics;
6634
6635   /*
6636    * Inherit the reporter values for each statistic.  The target may
6637    * be torn down as part of the runtime of the service (especially
6638    * for sleep/wake), so we inherit in order to avoid having values
6639    * reset for no apparent reason.  Our statistics are ultimately
6640    * tied to the index and the sevice, not to an individual target,
6641    * so we should maintain them accordingly.
6642    */
6643   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6644
6645   IOLockUnlock(reserved->interruptStatisticsLock);
6646
6647   return kIOReturnSuccess;
6648 }
6649
6650 IOReturn IOService::removeInterruptStatistics(int source)
6651 {
6652   IOInterruptAccountingData * value = NULL;
6653
6654   if (source < 0) {
6655     return kIOReturnBadArgument;
6656   }
6657
6658   IOLockLock(reserved->interruptStatisticsLock);
6659
6660   /*
6661    * We dynamically grow the statistics array, so an excessively
6662    * large index value has NEVER been registered.  This either
6663    * means our cap on the array size is too small (unlikely), or
6664    * that we have been passed a corrupt index (this must be passed
6665    * the plain index into the interrupt specifier list).
6666    */
6667   if (source >= reserved->interruptStatisticsArrayCount) {
6668     panic("removeInterruptStatistics called for index %d, which was never registered", source);
6669   }
6670
6671   assert(reserved->interruptStatisticsArray);
6672
6673   /*
6674    * If there is no existing entry, we are most likely trying to
6675    * free an interrupt owner twice, or we have corrupted the
6676    * index value.
6677    */
6678   value = reserved->interruptStatisticsArray[source].statistics;
6679
6680   if (!value) {
6681     panic("removeInterruptStatistics called for empty index %d", source);
6682   }
6683
6684   /*
6685    * We update the statistics, so that any delta with the reporter
6686    * state is not lost.
6687    */
6688   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6689   reserved->interruptStatisticsArray[source].statistics = NULL;
6690   IOLockUnlock(reserved->interruptStatisticsLock);
6691
6692   return kIOReturnSuccess;
6693 }
6694
6695 IOReturn IOService::getInterruptType(int source, int *interruptType)
6696 {
6697   IOInterruptController *interruptController;
6698   IOReturn              ret;
6699
6700   ret = lookupInterrupt(source, true, &interruptController);
6701   if (ret != kIOReturnSuccess) return ret;
6702
6703   /* Return the type */
6704   return interruptController->getInterruptType(this, source, interruptType);
6705 }
6706
6707 IOReturn IOService::enableInterrupt(int source)
6708 {
6709   IOInterruptController *interruptController;
6710   IOReturn              ret;
6711
6712   ret = lookupInterrupt(source, false, &interruptController);
6713   if (ret != kIOReturnSuccess) return ret;
6714
6715   /* Enable the source */
6716   return interruptController->enableInterrupt(this, source);
6717 }
6718
6719 IOReturn IOService::disableInterrupt(int source)
6720 {
6721   IOInterruptController *interruptController;
6722   IOReturn              ret;
6723
6724   ret = lookupInterrupt(source, false, &interruptController);
6725   if (ret != kIOReturnSuccess) return ret;
6726
6727   /* Disable the source */
6728   return interruptController->disableInterrupt(this, source);
6729 }
6730
6731 IOReturn IOService::causeInterrupt(int source)
6732 {
6733   IOInterruptController *interruptController;
6734   IOReturn              ret;
6735
6736   ret = lookupInterrupt(source, false, &interruptController);
6737   if (ret != kIOReturnSuccess) return ret;
6738
6739   /* Cause an interrupt for the source */
6740   return interruptController->causeInterrupt(this, source);
6741 }
6742
6743 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6744                                     IOReportConfigureAction action,
6745                                     void                   *result,
6746                                     void                   *destination)
6747 {
6748     unsigned cnt;
6749
6750     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6751         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6752             if (pwrMgt) configurePowerStatesReport(action, result);
6753             else  return kIOReturnUnsupported;
6754         }
6755         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6756             if (pwrMgt) configureSimplePowerReport(action, result);
6757             else  return kIOReturnUnsupported;
6758         }
6759     }
6760
6761     IOLockLock(reserved->interruptStatisticsLock);
6762
6763     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6764     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6765         if (reserved->interruptStatisticsArray[cnt].reporter) {
6766             /*
6767              * If the reporter is currently associated with the statistics
6768              * for an event source, we may need to update the reporter.
6769              */
6770             if (reserved->interruptStatisticsArray[cnt].statistics)
6771                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6772
6773             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6774         }
6775     }
6776
6777     IOLockUnlock(reserved->interruptStatisticsLock);
6778
6779     return kIOReturnSuccess;
6780 }
6781
6782 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6783                                  IOReportUpdateAction      action,
6784                                  void                     *result,
6785                                  void                     *destination)
6786 {
6787     unsigned cnt;
6788
6789     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6790         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6791             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6792             else return kIOReturnUnsupported;
6793         }
6794         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6795             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6796             else return kIOReturnUnsupported;
6797         }
6798     }
6799
6800     IOLockLock(reserved->interruptStatisticsLock);
6801
6802     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6803     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6804         if (reserved->interruptStatisticsArray[cnt].reporter) {
6805             /*
6806              * If the reporter is currently associated with the statistics
6807              * for an event source, we need to update the reporter.
6808              */
6809             if (reserved->interruptStatisticsArray[cnt].statistics)
6810                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6811
6812             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6813         }
6814     }
6815
6816     IOLockUnlock(reserved->interruptStatisticsLock);
6817
6818     return kIOReturnSuccess;
6819 }
6820
6821 uint64_t IOService::getAuthorizationID( void )
6822 {
6823     return reserved->authorizationID;
6824 }
6825
6826 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6827 {
6828     OSObject * entitlement;
6829     IOReturn status;
6830
6831     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6832
6833     if ( entitlement )
6834     {
6835         if ( entitlement == kOSBooleanTrue )
6836         {
6837             reserved->authorizationID = authorizationID;
6838
6839             status = kIOReturnSuccess;
6840         }
6841         else
6842         {
6843             status = kIOReturnNotPrivileged;
6844         }
6845
6846         entitlement->release( );
6847     }
6848     else
6849     {
6850         status = kIOReturnNotPrivileged;
6851     }
6852
6853     return status;
6854 }
6855
6856 #if __LP64__
6857 OSMetaClassDefineReservedUsed(IOService, 0);
6858 OSMetaClassDefineReservedUsed(IOService, 1);
6859 OSMetaClassDefineReservedUnused(IOService, 2);
6860 OSMetaClassDefineReservedUnused(IOService, 3);
6861 OSMetaClassDefineReservedUnused(IOService, 4);
6862 OSMetaClassDefineReservedUnused(IOService, 5);
6863 OSMetaClassDefineReservedUnused(IOService, 6);
6864 OSMetaClassDefineReservedUnused(IOService, 7);
6865 #else
6866 OSMetaClassDefineReservedUsed(IOService, 0);
6867 OSMetaClassDefineReservedUsed(IOService, 1);
6868 OSMetaClassDefineReservedUsed(IOService, 2);
6869 OSMetaClassDefineReservedUsed(IOService, 3);
6870 OSMetaClassDefineReservedUsed(IOService, 4);
6871 OSMetaClassDefineReservedUsed(IOService, 5);
6872 OSMetaClassDefineReservedUsed(IOService, 6);
6873 OSMetaClassDefineReservedUsed(IOService, 7);
6874 #endif
6875 OSMetaClassDefineReservedUnused(IOService, 8);
6876 OSMetaClassDefineReservedUnused(IOService, 9);
6877 OSMetaClassDefineReservedUnused(IOService, 10);
6878 OSMetaClassDefineReservedUnused(IOService, 11);
6879 OSMetaClassDefineReservedUnused(IOService, 12);
6880 OSMetaClassDefineReservedUnused(IOService, 13);
6881 OSMetaClassDefineReservedUnused(IOService, 14);
6882 OSMetaClassDefineReservedUnused(IOService, 15);
6883 OSMetaClassDefineReservedUnused(IOService, 16);
6884 OSMetaClassDefineReservedUnused(IOService, 17);
6885 OSMetaClassDefineReservedUnused(IOService, 18);
6886 OSMetaClassDefineReservedUnused(IOService, 19);
6887 OSMetaClassDefineReservedUnused(IOService, 20);
6888 OSMetaClassDefineReservedUnused(IOService, 21);
6889 OSMetaClassDefineReservedUnused(IOService, 22);
6890 OSMetaClassDefineReservedUnused(IOService, 23);
6891 OSMetaClassDefineReservedUnused(IOService, 24);
6892 OSMetaClassDefineReservedUnused(IOService, 25);
6893 OSMetaClassDefineReservedUnused(IOService, 26);
6894 OSMetaClassDefineReservedUnused(IOService, 27);
6895 OSMetaClassDefineReservedUnused(IOService, 28);
6896 OSMetaClassDefineReservedUnused(IOService, 29);
6897 OSMetaClassDefineReservedUnused(IOService, 30);
6898 OSMetaClassDefineReservedUnused(IOService, 31);
6899 OSMetaClassDefineReservedUnused(IOService, 32);
6900 OSMetaClassDefineReservedUnused(IOService, 33);
6901 OSMetaClassDefineReservedUnused(IOService, 34);
6902 OSMetaClassDefineReservedUnused(IOService, 35);
6903 OSMetaClassDefineReservedUnused(IOService, 36);
6904 OSMetaClassDefineReservedUnused(IOService, 37);
6905 OSMetaClassDefineReservedUnused(IOService, 38);
6906 OSMetaClassDefineReservedUnused(IOService, 39);
6907 OSMetaClassDefineReservedUnused(IOService, 40);
6908 OSMetaClassDefineReservedUnused(IOService, 41);
6909 OSMetaClassDefineReservedUnused(IOService, 42);
6910 OSMetaClassDefineReservedUnused(IOService, 43);
6911 OSMetaClassDefineReservedUnused(IOService, 44);
6912 OSMetaClassDefineReservedUnused(IOService, 45);
6913 OSMetaClassDefineReservedUnused(IOService, 46);
6914 OSMetaClassDefineReservedUnused(IOService, 47);