iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <IOKit/AppleKeyStoreInterface.h>
  54 #include <IOKit/pwr_mgt/RootDomain.h>
  55 #include <IOKit/IOCPU.h>
  56 #include <mach/sync_policy.h>
  57 #include <IOKit/assert.h>
  58 #include <sys/errno.h>
  59 #include <sys/kdebug.h>
  60 #include <string.h>
  61
  62 #include <machine/pal_routines.h>
  63
  64 #define LOG kprintf
  65 //#define LOG IOLog
  66 #define MATCH_DEBUG     0
  67 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  68
  69 // disabled since lockForArbitration() can be held externally
  70 #define DEBUG_NOTIFIER_LOCKED   0
  71
  72 #include "IOServicePrivate.h"
  73 #include "IOKitKernelInternal.h"
  74
  75 // take lockForArbitration before LOCKNOTIFY
  76
  77 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  78
  79 #define super IORegistryEntry
  80
  81 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  82
  83 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  84 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  85
  86 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  87
  88 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  89
  90 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  91
  92 OSDefineMetaClassAndStructors(IOResources, IOService)
  93
  94 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  95
  96 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  97
  98 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  99
 100 static IOPlatformExpert *       gIOPlatform;
 101 static class IOPMrootDomain *   gIOPMRootDomain;
 102 const IORegistryPlane *         gIOServicePlane;
 103 const IORegistryPlane *         gIOPowerPlane;
 104 const OSSymbol *                gIODeviceMemoryKey;
 105 const OSSymbol *                gIOInterruptControllersKey;
 106 const OSSymbol *                gIOInterruptSpecifiersKey;
 107
 108 const OSSymbol *                gIOResourcesKey;
 109 const OSSymbol *                gIOResourceMatchKey;
 110 const OSSymbol *                gIOResourceMatchedKey;
 111 const OSSymbol *                gIOProviderClassKey;
 112 const OSSymbol *                gIONameMatchKey;
 113 const OSSymbol *                gIONameMatchedKey;
 114 const OSSymbol *                gIOPropertyMatchKey;
 115 const OSSymbol *                gIOPropertyExistsMatchKey;
 116 const OSSymbol *                gIOLocationMatchKey;
 117 const OSSymbol *                gIOParentMatchKey;
 118 const OSSymbol *                gIOPathMatchKey;
 119 const OSSymbol *                gIOMatchCategoryKey;
 120 const OSSymbol *                gIODefaultMatchCategoryKey;
 121 const OSSymbol *                gIOMatchedServiceCountKey;
 122 #if !CONFIG_EMBEDDED
 123 const OSSymbol *                gIOServiceLegacyMatchingRegistryIDKey;
 124 #endif
 125
 126 const OSSymbol *                gIOMapperIDKey;
 127 const OSSymbol *                gIOUserClientClassKey;
 128 const OSSymbol *                gIOKitDebugKey;
 129
 130 const OSSymbol *                gIOCommandPoolSizeKey;
 131
 132 const OSSymbol *                gIOConsoleLockedKey;
 133 const OSSymbol *                gIOConsoleUsersKey;
 134 const OSSymbol *                gIOConsoleSessionUIDKey;
 135 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 136 const OSSymbol *                gIOConsoleUsersSeedKey;
 137 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 138 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 139 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 140 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 141 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 142 clock_sec_t                     gIOConsoleLockTime;
 143 static bool                     gIOConsoleLoggedIn;
 144 #if HIBERNATION
 145 static OSBoolean *              gIOConsoleBooterLockState;
 146 static uint32_t                 gIOScreenLockState;
 147 #endif
 148 static IORegistryEntry *        gIOChosenEntry;
 149
 150 static int                      gIOResourceGenerationCount;
 151
 152 const OSSymbol *                gIOServiceKey;
 153 const OSSymbol *                gIOPublishNotification;
 154 const OSSymbol *                gIOFirstPublishNotification;
 155 const OSSymbol *                gIOMatchedNotification;
 156 const OSSymbol *                gIOFirstMatchNotification;
 157 const OSSymbol *                gIOTerminatedNotification;
 158 const OSSymbol *                gIOWillTerminateNotification;
 159
 160 const OSSymbol *                gIOGeneralInterest;
 161 const OSSymbol *                gIOBusyInterest;
 162 const OSSymbol *                gIOAppPowerStateInterest;
 163 const OSSymbol *                gIOPriorityPowerStateInterest;
 164 const OSSymbol *                gIOConsoleSecurityInterest;
 165
 166 const OSSymbol *                gIOBSDKey;
 167 const OSSymbol *                gIOBSDNameKey;
 168 const OSSymbol *                gIOBSDMajorKey;
 169 const OSSymbol *                gIOBSDMinorKey;
 170 const OSSymbol *                gIOBSDUnitKey;
 171
 172 const  OSSymbol *               gAKSGetKey;
 173 #if defined(__i386__) || defined(__x86_64__)
 174 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 175 #endif
 176
 177 static OSDictionary *           gNotifications;
 178 static IORecursiveLock *        gNotificationLock;
 179
 180 static IOService *              gIOResources;
 181 static IOService *              gIOServiceRoot;
 182
 183 static OSOrderedSet *           gJobs;
 184 static semaphore_port_t         gJobsSemaphore;
 185 static IOLock *                 gJobsLock;
 186 static int                      gOutstandingJobs;
 187 static int                      gNumConfigThreads;
 188 static int                      gNumWaitingThreads;
 189 static IOLock *                 gIOServiceBusyLock;
 190 bool                            gCPUsRunning;
 191
 192 static thread_t                 gIOTerminateThread;
 193 static thread_t                 gIOTerminateWorkerThread;
 194 static UInt32                   gIOTerminateWork;
 195 static OSArray *                gIOTerminatePhase2List;
 196 static OSArray *                gIOStopList;
 197 static OSArray *                gIOStopProviderList;
 198 static OSArray *                gIOFinalizeList;
 199
 200 static SInt32                   gIOConsoleUsersSeed;
 201 static OSData *                 gIOConsoleUsersSeedValue;
 202
 203 extern const OSSymbol *         gIODTPHandleKey;
 204
 205 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 206
 207 static IOLock *                 gIOConsoleUsersLock;
 208 static thread_call_t            gIOConsoleLockCallout;
 209 static IONotifier *             gIOServiceNullNotifier;
 210
 211 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 212
 213 #define LOCKREADNOTIFY()        \
 214     IORecursiveLockLock( gNotificationLock )
 215 #define LOCKWRITENOTIFY()       \
 216     IORecursiveLockLock( gNotificationLock )
 217 #define LOCKWRITE2READNOTIFY()
 218 #define UNLOCKNOTIFY()          \
 219     IORecursiveLockUnlock( gNotificationLock )
 220 #define SLEEPNOTIFY(event) \
 221     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 222 #define SLEEPNOTIFYTO(event, deadline) \
 223     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 224 #define WAKEUPNOTIFY(event) \
 225         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 226
 227 #define randomDelay()   \
 228         int del = read_processor_clock();                               \
 229         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 230         IOSleep( del );
 231
 232 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 233
 234 #define queue_element(entry, element, type, field) do { \
 235         vm_address_t __ele = (vm_address_t) (entry);    \
 236         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 237         (element) = (type) __ele;                       \
 238     } while(0)
 239
 240 #define iterqueue(que, elt)                             \
 241         for (queue_entry_t elt = queue_first(que);      \
 242              !queue_end(que, elt);                      \
 243              elt = queue_next(elt))
 244
 245 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 246
 247 struct IOInterruptAccountingReporter {
 248     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 249     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 250 };
 251
 252 struct ArbitrationLockQueueElement {
 253     queue_chain_t link;
 254     IOThread      thread;
 255     IOService *   service;
 256     unsigned      count;
 257     bool          required;
 258     bool          aborted;
 259 };
 260
 261 static queue_head_t gArbitrationLockQueueActive;
 262 static queue_head_t gArbitrationLockQueueWaiting;
 263 static queue_head_t gArbitrationLockQueueFree;
 264 static IOLock *     gArbitrationLockQueueLock;
 265
 266 bool IOService::isInactive( void ) const
 267     { return( 0 != (kIOServiceInactiveState & getState())); }
 268
 269 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 270
 271 #if defined(__i386__) || defined(__x86_64__)
 272
 273 // Only used by the intel implementation of
 274 //     IOService::requireMaxBusStall(UInt32 ns)
 275 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 276 struct CpuDelayEntry
 277 {
 278     IOService * fService;
 279     UInt32      fMaxDelay;
 280     UInt32      fDelayType;
 281 };
 282
 283 enum {
 284     kCpuDelayBusStall, kCpuDelayInterrupt,
 285     kCpuNumDelayTypes
 286 };
 287
 288 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 289 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 290 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 291 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 292 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 293 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 294 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 295
 296 static void
 297 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 298 static IOReturn
 299 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 300
 301 #endif /* defined(__i386__) || defined(__x86_64__) */
 302
 303 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 304
 305 void IOService::initialize( void )
 306 {
 307     kern_return_t       err;
 308
 309     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 310     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 311
 312     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 313     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 314     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 315     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 316     gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 317     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 318     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 319     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 320
 321     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 322     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 323                                         kIODefaultMatchCategoryKey );
 324     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 325                                         kIOMatchedServiceCountKey );
 326 #if !CONFIG_EMBEDDED
 327     gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
 328                                         kIOServiceLegacyMatchingRegistryIDKey );
 329 #endif
 330
 331     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 332
 333     gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 334     gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 335     gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 336
 337     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 338     gIOInterruptControllersKey
 339         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 340     gIOInterruptSpecifiersKey
 341         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 342
 343     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 344
 345     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 346
 347     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 348
 349     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 350     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 351     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 352     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 353     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 354
 355     gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 356     gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 357     gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 358     gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 359     gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 360
 361     gNotifications              = OSDictionary::withCapacity( 1 );
 362     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 363                                                  kIOPublishNotification );
 364     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 365                                                  kIOFirstPublishNotification );
 366     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 367                                                  kIOMatchedNotification );
 368     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 369                                                  kIOFirstMatchNotification );
 370     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 371                                                  kIOTerminatedNotification );
 372     gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
 373                                                  kIOWillTerminateNotification );
 374     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 375
 376     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 377     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 378     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 379     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 380
 381     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 382     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 383     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 384     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 385     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 386     gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 387
 388     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 389
 390     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 391 #if defined(__i386__) || defined(__x86_64__)
 392     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 393     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 394     uint32_t  idx;
 395     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 396     {
 397         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 398         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 399         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 400     }
 401     gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 402 #endif
 403     gNotificationLock           = IORecursiveLockAlloc();
 404
 405     gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 406
 407     assert( gIOServicePlane && gIODeviceMemoryKey
 408         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 409         && gIOResourcesKey && gNotifications && gNotificationLock
 410         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 411         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 412         && gIOPublishNotification && gIOMatchedNotification
 413         && gIOTerminatedNotification && gIOServiceKey
 414         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 415     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 416         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 417
 418     gJobsLock   = IOLockAlloc();
 419     gJobs       = OSOrderedSet::withCapacity( 10 );
 420
 421     gIOServiceBusyLock = IOLockAlloc();
 422
 423     gIOConsoleUsersLock = IOLockAlloc();
 424
 425     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 426
 427     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 428
 429     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 430
 431     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 432                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 433
 434     gIOResources = IOResources::resources();
 435     assert( gIOResources );
 436
 437     gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 438     assert(gIOServiceNullNotifier);
 439
 440     gArbitrationLockQueueLock = IOLockAlloc();
 441     queue_init(&gArbitrationLockQueueActive);
 442     queue_init(&gArbitrationLockQueueWaiting);
 443     queue_init(&gArbitrationLockQueueFree);
 444
 445     assert( gArbitrationLockQueueLock );
 446
 447     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 448     gIOStopList            = OSArray::withCapacity( 16 );
 449     gIOStopProviderList    = OSArray::withCapacity( 16 );
 450     gIOFinalizeList        = OSArray::withCapacity( 16 );
 451     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 452
 453     // worker thread that is responsible for terminating / cleaning up threads
 454     kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
 455     assert(gIOTerminateWorkerThread);
 456     thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
 457 }
 458
 459 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 460
 461 #if defined(__i386__) || defined(__x86_64__)
 462 extern "C" {
 463
 464 const char *getCpuDelayBusStallHolderName(void);
 465 const char *getCpuDelayBusStallHolderName(void) {
 466     return sCPULatencyHolderName[kCpuDelayBusStall];
 467 }
 468
 469 const char *getCpuInterruptDelayHolderName(void);
 470 const char *getCpuInterruptDelayHolderName(void) {
 471     return sCPULatencyHolderName[kCpuDelayInterrupt];
 472 }
 473
 474 }
 475 #endif
 476
 477 #if IOMATCHDEBUG
 478 static UInt64 getDebugFlags( OSDictionary * props )
 479 {
 480     OSNumber *  debugProp;
 481     UInt64      debugFlags;
 482
 483     debugProp = OSDynamicCast( OSNumber,
 484                 props->getObject( gIOKitDebugKey ));
 485     if( debugProp)
 486         debugFlags = debugProp->unsigned64BitValue();
 487     else
 488         debugFlags = gIOKitDebug;
 489
 490     return( debugFlags );
 491 }
 492
 493 static UInt64 getDebugFlags( IOService * inst )
 494 {
 495     OSObject *  prop;
 496     OSNumber *  debugProp;
 497     UInt64      debugFlags;
 498
 499     prop = inst->copyProperty(gIOKitDebugKey);
 500     debugProp = OSDynamicCast(OSNumber, prop);
 501     if( debugProp)
 502         debugFlags = debugProp->unsigned64BitValue();
 503     else
 504         debugFlags = gIOKitDebug;
 505
 506     OSSafeReleaseNULL(prop);
 507
 508     return( debugFlags );
 509 }
 510 #endif
 511
 512 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 513
 514 // Probe a matched service and return an instance to be started.
 515 // The default score is from the property table, & may be altered
 516 // during probe to change the start order.
 517
 518 IOService * IOService::probe(   IOService * provider,
 519                                 SInt32    * score )
 520 {
 521     return( this );
 522 }
 523
 524 bool IOService::start( IOService * provider )
 525 {
 526     return( true );
 527 }
 528
 529 void IOService::stop( IOService * provider )
 530 {
 531 }
 532
 533 bool IOService::init( OSDictionary * dictionary )
 534 {
 535     bool ret;
 536
 537     ret = super::init(dictionary);
 538     if (!ret)     return (false);
 539     if (reserved) return (true);
 540
 541     reserved = IONew(ExpansionData, 1);
 542     if (!reserved) return (false);
 543     bzero(reserved, sizeof(*reserved));
 544
 545     /*
 546      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 547      * lock based on the presence of specifiers ran into issues as some
 548      * platforms set up the specifiers after IOService initialization.
 549      *
 550      * We may be able to get away with a global lock, as this should only be
 551      * contended by IOReporting clients and driver start/stop (unless a
 552      * driver wants to remove/add handlers in the course of normal operation,
 553      * which should be unlikely).
 554      */
 555     reserved->interruptStatisticsLock = IOLockAlloc();
 556     if (!reserved->interruptStatisticsLock) return (false);
 557
 558     return (true);
 559 }
 560
 561 bool IOService::init( IORegistryEntry * from,
 562                       const IORegistryPlane * inPlane )
 563 {
 564     bool ret;
 565
 566     ret = super::init(from, inPlane);
 567     if (!ret)     return (false);
 568     if (reserved) return (true);
 569
 570     reserved = IONew(ExpansionData, 1);
 571     if (!reserved) return (false);
 572     bzero(reserved, sizeof(*reserved));
 573
 574     /*
 575      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 576      * lock based on the presence of specifiers ran into issues as some
 577      * platforms set up the specifiers after IOService initialization.
 578      *
 579      * We may be able to get away with a global lock, as this should only be
 580      * contended by IOReporting clients and driver start/stop (unless a
 581      * driver wants to remove/add handlers in the course of normal operation,
 582      * which should be unlikely).
 583      */
 584     reserved->interruptStatisticsLock = IOLockAlloc();
 585     if (!reserved->interruptStatisticsLock) return (false);
 586
 587     return (true);
 588 }
 589
 590 void IOService::free( void )
 591 {
 592     int i = 0;
 593     requireMaxBusStall(0);
 594     requireMaxInterruptDelay(0);
 595     if( getPropertyTable())
 596         unregisterAllInterest();
 597     PMfree();
 598
 599     if (reserved) {
 600         if (reserved->interruptStatisticsArray) {
 601             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 602                 if (reserved->interruptStatisticsArray[i].reporter)
 603                     reserved->interruptStatisticsArray[i].reporter->release();
 604             }
 605
 606             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 607         }
 608
 609         if (reserved->interruptStatisticsLock)
 610             IOLockFree(reserved->interruptStatisticsLock);
 611         IODelete(reserved, ExpansionData, 1);
 612     }
 613
 614     if (_numInterruptSources && _interruptSources)
 615     {
 616         IOFree(_interruptSources, _numInterruptSources * sizeof(IOInterruptSource));
 617         _interruptSources = 0;
 618     }
 619
 620     super::free();
 621 }
 622
 623 /*
 624  * Attach in service plane
 625  */
 626 bool IOService::attach( IOService * provider )
 627 {
 628     bool         ok;
 629     uint32_t     count;
 630     AbsoluteTime deadline;
 631     int          waitResult = THREAD_AWAKENED;
 632     bool         wait, computeDeadline = true;
 633
 634     if( provider) {
 635
 636         if( gIOKitDebug & kIOLogAttach)
 637             LOG( "%s::attach(%s)\n", getName(),
 638                     provider->getName());
 639
 640         ok   = false;
 641         do
 642         {
 643             wait = false;
 644             provider->lockForArbitration();
 645             if (provider->__state[0] & kIOServiceInactiveState) ok = false;
 646             else
 647             {
 648                 count = provider->getChildCount(gIOServicePlane);
 649                 wait = (count > (kIOServiceBusyMax - 4));
 650                 if (!wait) ok = attachToParent(provider, gIOServicePlane);
 651                 else
 652                 {
 653                     IOLog("stalling for detach from %s\n", provider->getName());
 654                     IOLockLock( gIOServiceBusyLock );
 655                     provider->__state[1] |= kIOServiceWaitDetachState;
 656                 }
 657             }
 658             provider->unlockForArbitration();
 659             if (wait)
 660             {
 661                 if (computeDeadline)
 662                 {
 663                     clock_interval_to_deadline(15, kSecondScale, &deadline);
 664                     computeDeadline = false;
 665                 }
 666                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 667                 IOLockUnlock( gIOServiceBusyLock );
 668                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 669                 wait = (waitResult != THREAD_TIMED_OUT);
 670             }
 671         }
 672         while (wait);
 673
 674     } else {
 675         gIOServiceRoot = this;
 676         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 677     }
 678
 679     if (ok && !__provider) (void) getProvider();
 680
 681     return( ok );
 682 }
 683
 684 IOService * IOService::getServiceRoot( void )
 685 {
 686     return( gIOServiceRoot );
 687 }
 688
 689 void IOService::detach( IOService * provider )
 690 {
 691     IOService * newProvider = 0;
 692     SInt32      busy;
 693     bool        adjParent;
 694
 695     if( gIOKitDebug & kIOLogAttach)
 696         LOG("%s::detach(%s)\n", getName(), provider->getName());
 697
 698     lockForArbitration();
 699
 700     uint64_t regID1 = provider->getRegistryEntryID();
 701     uint64_t regID2 = getRegistryEntryID();
 702     IOServiceTrace(
 703         IOSERVICE_DETACH,
 704         (uintptr_t) regID1,
 705         (uintptr_t) (regID1 >> 32),
 706         (uintptr_t) regID2,
 707         (uintptr_t) (regID2 >> 32));
 708
 709     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 710                && (provider == getProvider()));
 711
 712     detachFromParent( provider, gIOServicePlane );
 713
 714     if( busy) {
 715         newProvider = getProvider();
 716         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 717             _adjustBusy( -busy );
 718     }
 719
 720     if (kIOServiceInactiveState & __state[0]) {
 721         getMetaClass()->removeInstance(this);
 722         IORemoveServicePlatformActions(this);
 723     }
 724
 725     unlockForArbitration();
 726
 727     if( newProvider) {
 728         newProvider->lockForArbitration();
 729         newProvider->_adjustBusy(1);
 730         newProvider->unlockForArbitration();
 731     }
 732
 733     // check for last client detach from a terminated service
 734     if( provider->lockForArbitration( true ))
 735     {
 736         if (kIOServiceStartState & __state[1])
 737         {
 738             provider->scheduleTerminatePhase2();
 739         }
 740         if( adjParent) provider->_adjustBusy( -1 );
 741         if( (provider->__state[1] & kIOServiceTermPhase3State)
 742          && (0 == provider->getClient())) {
 743             provider->scheduleFinalize(false);
 744         }
 745
 746         IOLockLock( gIOServiceBusyLock );
 747         if (kIOServiceWaitDetachState & provider->__state[1])
 748         {
 749             provider->__state[1] &= ~kIOServiceWaitDetachState;
 750             thread_wakeup(&provider->__provider);
 751         }
 752         IOLockUnlock( gIOServiceBusyLock );
 753
 754         provider->unlockForArbitration();
 755     }
 756 }
 757
 758 /*
 759  * Register instance - publish it for matching
 760  */
 761
 762 void IOService::registerService( IOOptionBits options )
 763 {
 764     char *              pathBuf;
 765     const char *        path;
 766     char *              skip;
 767     int                 len;
 768     enum { kMaxPathLen  = 256 };
 769     enum { kMaxChars    = 63 };
 770
 771     IORegistryEntry * parent = this;
 772     IORegistryEntry * root = getRegistryRoot();
 773     while( parent && (parent != root))
 774         parent = parent->getParentEntry( gIOServicePlane);
 775
 776     if( parent != root) {
 777         IOLog("%s: not registry member at registerService()\n", getName());
 778         return;
 779     }
 780
 781     // Allow the Platform Expert to adjust this node.
 782     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 783         return;
 784
 785     IOInstallServicePlatformActions(this);
 786
 787     if( (this != gIOResources)
 788      && (kIOLogRegister & gIOKitDebug)) {
 789
 790         pathBuf = (char *) IOMalloc( kMaxPathLen );
 791
 792         IOLog( "Registering: " );
 793
 794         len = kMaxPathLen;
 795         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 796
 797             path = pathBuf;
 798             if( len > kMaxChars) {
 799                 IOLog("..");
 800                 len -= kMaxChars;
 801                 path += len;
 802                 if( (skip = strchr( path, '/')))
 803                     path = skip;
 804             }
 805         } else
 806             path = getName();
 807
 808         IOLog( "%s\n", path );
 809
 810         if( pathBuf)
 811             IOFree( pathBuf, kMaxPathLen );
 812     }
 813
 814     startMatching( options );
 815 }
 816
 817 void IOService::startMatching( IOOptionBits options )
 818 {
 819     IOService * provider;
 820     UInt32      prevBusy = 0;
 821     bool        needConfig;
 822     bool        needWake = false;
 823     bool        ok;
 824     bool        sync;
 825     bool        waitAgain;
 826
 827     lockForArbitration();
 828
 829     sync = (options & kIOServiceSynchronous)
 830         || ((provider = getProvider())
 831                 && (provider->__state[1] & kIOServiceSynchronousState));
 832
 833         if ( options & kIOServiceAsynchronous )
 834                 sync = false;
 835
 836     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
 837                && (0 == (__state[0] & kIOServiceInactiveState));
 838
 839     __state[1] |= kIOServiceNeedConfigState;
 840
 841 //    __state[0] &= ~kIOServiceInactiveState;
 842
 843 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 844 //                      OSKernelStackRemaining(), getName());
 845
 846     if( needConfig) {
 847         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 848     }
 849
 850     if( sync)
 851         __state[1] |= kIOServiceSynchronousState;
 852     else
 853         __state[1] &= ~kIOServiceSynchronousState;
 854
 855     if( needConfig) prevBusy = _adjustBusy( 1 );
 856
 857     unlockForArbitration();
 858
 859     if( needConfig) {
 860
 861         if( needWake) {
 862             IOLockLock( gIOServiceBusyLock );
 863             thread_wakeup( (event_t) this/*&__state[1]*/ );
 864             IOLockUnlock( gIOServiceBusyLock );
 865
 866         } else if( !sync || (kIOServiceAsynchronous & options)) {
 867
 868             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 869
 870         } else do {
 871
 872             if( (__state[1] & kIOServiceNeedConfigState))
 873                 doServiceMatch( options );
 874
 875             lockForArbitration();
 876             IOLockLock( gIOServiceBusyLock );
 877
 878             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 879                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 880
 881             if( waitAgain)
 882                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 883             else
 884                 __state[1] &= ~kIOServiceSyncPubState;
 885
 886             unlockForArbitration();
 887
 888             if( waitAgain)
 889                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 890
 891             IOLockUnlock( gIOServiceBusyLock );
 892             if( waitAgain)
 893                 thread_block(THREAD_CONTINUE_NULL);
 894
 895         } while( waitAgain );
 896     }
 897 }
 898
 899 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 900 {
 901     OSDictionary *      table;
 902     OSSet *             set;
 903     OSSet *             allSet = 0;
 904     IOService *         service;
 905 #if IOMATCHDEBUG
 906     SInt32              count = 0;
 907 #endif
 908
 909     newTables->retain();
 910
 911     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 912
 913         LOCKWRITENOTIFY();
 914         set = (OSSet *) copyExistingServices( table,
 915                                                 kIOServiceRegisteredState,
 916                                                 kIOServiceExistingSet);
 917         UNLOCKNOTIFY();
 918         if( set) {
 919
 920 #if IOMATCHDEBUG
 921             count += set->getCount();
 922 #endif
 923             if (allSet) {
 924                 allSet->merge((const OSSet *) set);
 925                 set->release();
 926             }
 927             else
 928                 allSet = set;
 929         }
 930
 931 #if IOMATCHDEBUG
 932         if( getDebugFlags( table ) & kIOLogMatch)
 933             LOG("Matching service count = %ld\n", (long)count);
 934 #endif
 935         newTables->removeObject(table);
 936     }
 937
 938     if (allSet) {
 939         while( (service = (IOService *) allSet->getAnyObject())) {
 940             service->startMatching(kIOServiceAsynchronous);
 941             allSet->removeObject(service);
 942         }
 943         allSet->release();
 944     }
 945
 946     newTables->release();
 947
 948     return( kIOReturnSuccess );
 949 }
 950
 951  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 952                                                 IOOptionBits options )
 953 {
 954     _IOServiceJob *     job;
 955
 956     job = new _IOServiceJob;
 957     if( job && !job->init()) {
 958         job->release();
 959         job = 0;
 960     }
 961
 962     if( job) {
 963         job->type       = type;
 964         job->nub        = nub;
 965         job->options    = options;
 966         nub->retain();                  // thread will release()
 967         pingConfig( job );
 968     }
 969
 970     return( job );
 971 }
 972
 973 /*
 974  * Called on a registered service to see if it matches
 975  * a property table.
 976  */
 977
 978 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 979 {
 980     return( matchPropertyTable(table) );
 981 }
 982
 983 bool IOService::matchPropertyTable( OSDictionary * table )
 984 {
 985     return( true );
 986 }
 987
 988 /*
 989  * Called on a matched service to allocate resources
 990  * before first driver is attached.
 991  */
 992
 993 IOReturn IOService::getResources( void )
 994 {
 995     return( kIOReturnSuccess);
 996 }
 997
 998 /*
 999  * Client/provider accessors
1000  */
1001
1002 IOService * IOService::getProvider( void ) const
1003 {
1004     IOService * self = (IOService *) this;
1005     IOService * parent;
1006     SInt32      generation;
1007
1008     generation = getRegistryEntryGenerationCount();
1009     if( __providerGeneration == generation)
1010         return( __provider );
1011
1012     parent = (IOService *) getParentEntry( gIOServicePlane);
1013     if( parent == IORegistryEntry::getRegistryRoot())
1014         /* root is not an IOService */
1015         parent = 0;
1016
1017     self->__provider = parent;
1018     OSMemoryBarrier();
1019     // save the count from before call to getParentEntry()
1020     self->__providerGeneration = generation;
1021
1022     return( parent );
1023 }
1024
1025 IOWorkLoop * IOService::getWorkLoop() const
1026 {
1027     IOService *provider = getProvider();
1028
1029     if (provider)
1030         return provider->getWorkLoop();
1031     else
1032         return 0;
1033 }
1034
1035 OSIterator * IOService::getProviderIterator( void ) const
1036 {
1037     return( getParentIterator( gIOServicePlane));
1038 }
1039
1040 IOService * IOService::getClient( void ) const
1041 {
1042     return( (IOService *) getChildEntry( gIOServicePlane));
1043 }
1044
1045 OSIterator * IOService::getClientIterator( void ) const
1046 {
1047     return( getChildIterator( gIOServicePlane));
1048 }
1049
1050 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
1051                                                 const IOService * client,
1052                                                 const IOService * provider )
1053 {
1054     _IOOpenServiceIterator * inst;
1055
1056     if( !_iter)
1057         return( 0 );
1058
1059     inst = new _IOOpenServiceIterator;
1060
1061     if( inst && !inst->init()) {
1062         inst->release();
1063         inst = 0;
1064     }
1065     if( inst) {
1066         inst->iter = _iter;
1067         inst->client = client;
1068         inst->provider = provider;
1069     }
1070
1071     return( inst );
1072 }
1073
1074 void _IOOpenServiceIterator::free()
1075 {
1076     iter->release();
1077     if( last)
1078         last->unlockForArbitration();
1079     OSIterator::free();
1080 }
1081
1082 OSObject * _IOOpenServiceIterator::getNextObject()
1083 {
1084     IOService * next;
1085
1086     if( last)
1087         last->unlockForArbitration();
1088
1089     while( (next = (IOService *) iter->getNextObject())) {
1090
1091         next->lockForArbitration();
1092         if( (client && (next->isOpen( client )))
1093          || (provider && (provider->isOpen( next ))) )
1094             break;
1095         next->unlockForArbitration();
1096     }
1097
1098     last = next;
1099
1100     return( next );
1101 }
1102
1103 bool _IOOpenServiceIterator::isValid()
1104 {
1105     return( iter->isValid() );
1106 }
1107
1108 void _IOOpenServiceIterator::reset()
1109 {
1110     if( last) {
1111         last->unlockForArbitration();
1112         last = 0;
1113     }
1114     iter->reset();
1115 }
1116
1117 OSIterator * IOService::getOpenProviderIterator( void ) const
1118 {
1119     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1120 }
1121
1122 OSIterator * IOService::getOpenClientIterator( void ) const
1123 {
1124     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1125 }
1126
1127
1128 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1129                                           bool waitForFunction,
1130                                           void *param1, void *param2,
1131                                           void *param3, void *param4 )
1132 {
1133   IOReturn  result = kIOReturnUnsupported;
1134   IOService *provider;
1135
1136   if (gIOPlatformFunctionHandlerSet == functionName)
1137   {
1138 #if defined(__i386__) || defined(__x86_64__)
1139     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1140     IOService *      target              = (IOService *) param2;
1141     bool             enable              = (param3 != 0);
1142
1143     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1144         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1145     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1146         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1147 #endif /* defined(__i386__) || defined(__x86_64__) */
1148   }
1149
1150   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1151     result = provider->callPlatformFunction(functionName, waitForFunction,
1152                                             param1, param2, param3, param4);
1153   }
1154
1155   return result;
1156 }
1157
1158 IOReturn IOService::callPlatformFunction( const char * functionName,
1159                                           bool waitForFunction,
1160                                           void *param1, void *param2,
1161                                           void *param3, void *param4 )
1162 {
1163   IOReturn result = kIOReturnNoMemory;
1164   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1165
1166   if (functionSymbol != 0) {
1167     result = callPlatformFunction(functionSymbol, waitForFunction,
1168                                   param1, param2, param3, param4);
1169     functionSymbol->release();
1170   }
1171
1172   return result;
1173 }
1174
1175
1176 /*
1177  * Accessors for global services
1178  */
1179
1180 IOPlatformExpert * IOService::getPlatform( void )
1181 {
1182     return( gIOPlatform);
1183 }
1184
1185 class IOPMrootDomain * IOService::getPMRootDomain( void )
1186 {
1187     return( gIOPMRootDomain);
1188 }
1189
1190 IOService * IOService::getResourceService( void )
1191 {
1192     return( gIOResources );
1193 }
1194
1195 void IOService::setPlatform( IOPlatformExpert * platform)
1196 {
1197     gIOPlatform = platform;
1198     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1199
1200 #if defined(__i386__) || defined(__x86_64__)
1201
1202     static const char * keys[kCpuNumDelayTypes] = {
1203         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1204     const OSObject * objs[2];
1205     OSArray * array;
1206     uint32_t  idx;
1207
1208     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1209     {
1210         objs[0] = sCPULatencySet[idx];
1211         objs[1] = sCPULatencyHolder[idx];
1212         array   = OSArray::withObjects(objs, 2);
1213         if (!array) break;
1214         platform->setProperty(keys[idx], array);
1215         array->release();
1216     }
1217 #endif /* defined(__i386__) || defined(__x86_64__) */
1218 }
1219
1220 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1221 {
1222     gIOPMRootDomain = rootDomain;
1223     publishResource("IOKit");
1224 }
1225
1226 /*
1227  * Stacking change
1228  */
1229
1230 bool IOService::lockForArbitration( bool isSuccessRequired )
1231 {
1232     bool                          found;
1233     bool                          success;
1234     ArbitrationLockQueueElement * element;
1235     ArbitrationLockQueueElement * active;
1236     ArbitrationLockQueueElement * waiting;
1237
1238     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1239
1240     // lock global access
1241     IOTakeLock( gArbitrationLockQueueLock );
1242
1243     // obtain an unused queue element
1244     if( !queue_empty( &gArbitrationLockQueueFree )) {
1245         queue_remove_first( &gArbitrationLockQueueFree,
1246                             element,
1247                             ArbitrationLockQueueElement *,
1248                             link );
1249     } else {
1250         element = IONew( ArbitrationLockQueueElement, 1 );
1251         assert( element );
1252     }
1253
1254     // prepare the queue element
1255     element->thread   = IOThreadSelf();
1256     element->service  = this;
1257     element->count    = 1;
1258     element->required = isSuccessRequired;
1259     element->aborted  = false;
1260
1261     // determine whether this object is already locked (ie. on active queue)
1262     found = false;
1263     queue_iterate( &gArbitrationLockQueueActive,
1264                     active,
1265                     ArbitrationLockQueueElement *,
1266                     link )
1267     {
1268         if( active->service == element->service ) {
1269             found = true;
1270             break;
1271         }
1272     }
1273
1274     if( found ) { // this object is already locked
1275
1276         // determine whether it is the same or a different thread trying to lock
1277         if( active->thread != element->thread ) { // it is a different thread
1278
1279             ArbitrationLockQueueElement * victim = 0;
1280
1281             // before placing this new thread on the waiting queue, we look for
1282             // a deadlock cycle...
1283
1284             while( 1 ) {
1285                 // determine whether the active thread holding the object we
1286                 // want is waiting for another object to be unlocked
1287                 found = false;
1288                 queue_iterate( &gArbitrationLockQueueWaiting,
1289                                waiting,
1290                                ArbitrationLockQueueElement *,
1291                                link )
1292                 {
1293                     if( waiting->thread == active->thread ) {
1294                         assert( false == waiting->aborted );
1295                         found = true;
1296                         break;
1297                     }
1298                 }
1299
1300                 if( found ) { // yes, active thread waiting for another object
1301
1302                     // this may be a candidate for rejection if the required
1303                     // flag is not set, should we detect a deadlock later on
1304                     if( false == waiting->required )
1305                         victim = waiting;
1306
1307                     // find the thread that is holding this other object, that
1308                     // is blocking the active thread from proceeding (fun :-)
1309                     found = false;
1310                     queue_iterate( &gArbitrationLockQueueActive,
1311                                    active,      // (reuse active queue element)
1312                                    ArbitrationLockQueueElement *,
1313                                    link )
1314                     {
1315                         if( active->service == waiting->service ) {
1316                             found = true;
1317                             break;
1318                         }
1319                     }
1320
1321                     // someone must be holding it or it wouldn't be waiting
1322                     assert( found );
1323
1324                     if( active->thread == element->thread ) {
1325
1326                         // doh, it's waiting for the thread that originated
1327                         // this whole lock (ie. current thread) -> deadlock
1328                         if( false == element->required ) { // willing to fail?
1329
1330                             // the originating thread doesn't have the required
1331                             // flag, so it can fail
1332                             success = false; // (fail originating lock request)
1333                             break; // (out of while)
1334
1335                         } else { // originating thread is not willing to fail
1336
1337                             // see if we came across a waiting thread that did
1338                             // not have the 'required' flag set: we'll fail it
1339                             if( victim ) {
1340
1341                                 // we do have a willing victim, fail it's lock
1342                                 victim->aborted = true;
1343
1344                                 // take the victim off the waiting queue
1345                                 queue_remove( &gArbitrationLockQueueWaiting,
1346                                               victim,
1347                                               ArbitrationLockQueueElement *,
1348                                               link );
1349
1350                                 // wake the victim
1351                                 IOLockWakeup( gArbitrationLockQueueLock,
1352                                               victim,
1353                                               /* one thread */ true );
1354
1355                                 // allow this thread to proceed (ie. wait)
1356                                 success = true; // (put request on wait queue)
1357                                 break; // (out of while)
1358                             } else {
1359
1360                                 // all the waiting threads we came across in
1361                                 // finding this loop had the 'required' flag
1362                                 // set, so we've got a deadlock we can't avoid
1363                                 panic("I/O Kit: Unrecoverable deadlock.");
1364                             }
1365                         }
1366                     } else {
1367                         // repeat while loop, redefining active thread to be the
1368                         // thread holding "this other object" (see above), and
1369                         // looking for threads waiting on it; note the active
1370                         // variable points to "this other object" already... so
1371                         // there nothing to do in this else clause.
1372                     }
1373                 } else { // no, active thread is not waiting for another object
1374
1375                     success = true; // (put request on wait queue)
1376                     break; // (out of while)
1377                 }
1378             } // while forever
1379
1380             if( success ) { // put the request on the waiting queue?
1381                 kern_return_t wait_result;
1382
1383                 // place this thread on the waiting queue and put it to sleep;
1384                 // we place it at the tail of the queue...
1385                 queue_enter( &gArbitrationLockQueueWaiting,
1386                              element,
1387                              ArbitrationLockQueueElement *,
1388                              link );
1389
1390                 // declare that this thread will wait for a given event
1391 restart_sleep:  wait_result = assert_wait( element,
1392                                            element->required ? THREAD_UNINT
1393                                            : THREAD_INTERRUPTIBLE );
1394
1395                 // unlock global access
1396                 IOUnlock( gArbitrationLockQueueLock );
1397
1398                 // put thread to sleep, waiting for our event to fire...
1399                 if (wait_result == THREAD_WAITING)
1400                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1401
1402
1403                 // ...and we've been woken up; we might be in one of two states:
1404                 // (a) we've been aborted and our queue element is not on
1405                 //     any of the three queues, but is floating around
1406                 // (b) we're allowed to proceed with the lock and we have
1407                 //     already been moved from the waiting queue to the
1408                 //     active queue.
1409                 // ...plus a 3rd state, should the thread have been interrupted:
1410                 // (c) we're still on the waiting queue
1411
1412                 // determine whether we were interrupted out of our sleep
1413                 if( THREAD_INTERRUPTED == wait_result ) {
1414
1415                     // re-lock global access
1416                     IOTakeLock( gArbitrationLockQueueLock );
1417
1418                     // determine whether we're still on the waiting queue
1419                     found = false;
1420                     queue_iterate( &gArbitrationLockQueueWaiting,
1421                                    waiting,     // (reuse waiting queue element)
1422                                    ArbitrationLockQueueElement *,
1423                                    link )
1424                     {
1425                         if( waiting == element ) {
1426                             found = true;
1427                             break;
1428                         }
1429                     }
1430
1431                     if( found ) { // yes, we're still on the waiting queue
1432
1433                         // determine whether we're willing to fail
1434                         if( false == element->required ) {
1435
1436                             // mark us as aborted
1437                             element->aborted = true;
1438
1439                             // take us off the waiting queue
1440                             queue_remove( &gArbitrationLockQueueWaiting,
1441                                           element,
1442                                           ArbitrationLockQueueElement *,
1443                                           link );
1444                         } else { // we are not willing to fail
1445
1446                             // ignore interruption, go back to sleep
1447                             goto restart_sleep;
1448                         }
1449                     }
1450
1451                     // unlock global access
1452                     IOUnlock( gArbitrationLockQueueLock );
1453
1454                     // proceed as though this were a normal wake up
1455                     wait_result = THREAD_AWAKENED;
1456                 }
1457
1458                 assert( THREAD_AWAKENED == wait_result );
1459
1460                 // determine whether we've been aborted while we were asleep
1461                 if( element->aborted ) {
1462                     assert( false == element->required );
1463
1464                     // re-lock global access
1465                     IOTakeLock( gArbitrationLockQueueLock );
1466
1467                     action = kPutOnFreeQueue;
1468                     success = false;
1469                 } else { // we weren't aborted, so we must be ready to go :-)
1470
1471                     // we've already been moved from waiting to active queue
1472                     return true;
1473                 }
1474
1475             } else { // the lock request is to be failed
1476
1477                 // return unused queue element to queue
1478                 action = kPutOnFreeQueue;
1479             }
1480         } else { // it is the same thread, recursive access is allowed
1481
1482             // add one level of recursion
1483             active->count++;
1484
1485             // return unused queue element to queue
1486             action = kPutOnFreeQueue;
1487             success = true;
1488         }
1489     } else { // this object is not already locked, so let this thread through
1490         action = kPutOnActiveQueue;
1491         success = true;
1492     }
1493
1494     // put the new element on a queue
1495     if( kPutOnActiveQueue == action ) {
1496         queue_enter( &gArbitrationLockQueueActive,
1497                      element,
1498                      ArbitrationLockQueueElement *,
1499                      link );
1500     } else if( kPutOnFreeQueue == action ) {
1501         queue_enter( &gArbitrationLockQueueFree,
1502                      element,
1503                      ArbitrationLockQueueElement *,
1504                      link );
1505     } else {
1506         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1507     }
1508
1509     // unlock global access
1510     IOUnlock( gArbitrationLockQueueLock );
1511
1512     return( success );
1513 }
1514
1515 void IOService::unlockForArbitration( void )
1516 {
1517     bool                          found;
1518     ArbitrationLockQueueElement * element;
1519
1520     // lock global access
1521     IOTakeLock( gArbitrationLockQueueLock );
1522
1523     // find the lock element for this object (ie. on active queue)
1524     found = false;
1525     queue_iterate( &gArbitrationLockQueueActive,
1526                     element,
1527                     ArbitrationLockQueueElement *,
1528                     link )
1529     {
1530         if( element->service == this ) {
1531             found = true;
1532             break;
1533         }
1534     }
1535
1536     assert( found );
1537
1538     // determine whether the lock has been taken recursively
1539     if( element->count > 1 ) {
1540         // undo one level of recursion
1541         element->count--;
1542
1543     } else {
1544
1545         // remove it from the active queue
1546         queue_remove( &gArbitrationLockQueueActive,
1547                       element,
1548                       ArbitrationLockQueueElement *,
1549                       link );
1550
1551         // put it on the free queue
1552         queue_enter( &gArbitrationLockQueueFree,
1553                      element,
1554                      ArbitrationLockQueueElement *,
1555                      link );
1556
1557         // determine whether a thread is waiting for object (head to tail scan)
1558         found = false;
1559         queue_iterate( &gArbitrationLockQueueWaiting,
1560                        element,
1561                        ArbitrationLockQueueElement *,
1562                        link )
1563         {
1564             if( element->service == this ) {
1565                 found = true;
1566                 break;
1567             }
1568         }
1569
1570         if ( found ) { // we found an interested thread on waiting queue
1571
1572             // remove it from the waiting queue
1573             queue_remove( &gArbitrationLockQueueWaiting,
1574                           element,
1575                           ArbitrationLockQueueElement *,
1576                           link );
1577
1578             // put it on the active queue
1579             queue_enter( &gArbitrationLockQueueActive,
1580                          element,
1581                          ArbitrationLockQueueElement *,
1582                          link );
1583
1584             // wake the waiting thread
1585             IOLockWakeup( gArbitrationLockQueueLock,
1586                           element,
1587                           /* one thread */ true );
1588         }
1589     }
1590
1591     // unlock global access
1592     IOUnlock( gArbitrationLockQueueLock );
1593 }
1594
1595 uint32_t IOService::isLockedForArbitration(IOService * service)
1596 {
1597 #if DEBUG_NOTIFIER_LOCKED
1598     uint32_t                      count;
1599     ArbitrationLockQueueElement * active;
1600
1601     // lock global access
1602     IOLockLock(gArbitrationLockQueueLock);
1603
1604     // determine whether this object is already locked (ie. on active queue)
1605     count = 0;
1606     queue_iterate(&gArbitrationLockQueueActive,
1607                   active,
1608                   ArbitrationLockQueueElement *,
1609                   link)
1610     {
1611         if ((active->thread == IOThreadSelf())
1612             && (!service || (active->service == service)))
1613         {
1614             count += 0x10000;
1615             count += active->count;
1616         }
1617     }
1618
1619     IOLockUnlock(gArbitrationLockQueueLock);
1620
1621     return (count);
1622
1623 #else /* DEBUG_NOTIFIER_LOCKED */
1624
1625     return (0);
1626
1627 #endif /* DEBUG_NOTIFIER_LOCKED */
1628 }
1629
1630 void IOService::applyToProviders( IOServiceApplierFunction applier,
1631                                   void * context )
1632 {
1633     applyToParents( (IORegistryEntryApplierFunction) applier,
1634                     context, gIOServicePlane );
1635 }
1636
1637 void IOService::applyToClients( IOServiceApplierFunction applier,
1638                                 void * context )
1639 {
1640     applyToChildren( (IORegistryEntryApplierFunction) applier,
1641                      context, gIOServicePlane );
1642 }
1643
1644
1645 /*
1646  * Client messages
1647  */
1648
1649
1650 // send a message to a client or interested party of this service
1651 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1652                                    void * argument, vm_size_t argSize )
1653 {
1654     IOReturn                            ret;
1655     IOService *                         service;
1656     _IOServiceInterestNotifier *        notify;
1657
1658     if( (service = OSDynamicCast( IOService, client)))
1659         ret = service->message( type, this, argument );
1660
1661     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1662
1663         _IOServiceNotifierInvocation invocation;
1664         bool                     willNotify;
1665
1666         invocation.thread = current_thread();
1667
1668         LOCKWRITENOTIFY();
1669         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1670
1671         if( willNotify) {
1672             queue_enter( &notify->handlerInvocations, &invocation,
1673                             _IOServiceNotifierInvocation *, link );
1674         }
1675         UNLOCKNOTIFY();
1676
1677         if( willNotify) {
1678
1679             ret = (*notify->handler)( notify->target, notify->ref,
1680                                           type, this, argument, argSize );
1681
1682             LOCKWRITENOTIFY();
1683             queue_remove( &notify->handlerInvocations, &invocation,
1684                             _IOServiceNotifierInvocation *, link );
1685             if( kIOServiceNotifyWaiter & notify->state) {
1686                 notify->state &= ~kIOServiceNotifyWaiter;
1687                 WAKEUPNOTIFY( notify );
1688             }
1689             UNLOCKNOTIFY();
1690
1691         } else
1692             ret = kIOReturnSuccess;
1693
1694     } else
1695         ret = kIOReturnBadArgument;
1696
1697     return( ret );
1698 }
1699
1700 static void
1701 applyToInterestNotifiers(const IORegistryEntry *target,
1702                          const OSSymbol * typeOfInterest,
1703                          OSObjectApplierFunction applier,
1704                          void * context )
1705 {
1706     OSArray *  copyArray = 0;
1707     OSObject * prop;
1708
1709     LOCKREADNOTIFY();
1710
1711     prop = target->copyProperty(typeOfInterest);
1712     IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
1713
1714     if( notifyList) {
1715         copyArray = OSArray::withCapacity(1);
1716
1717         // iterate over queue, entry is set to each element in the list
1718         iterqueue(&notifyList->fCommandChain, entry) {
1719             _IOServiceInterestNotifier * notify;
1720
1721             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1722             copyArray->setObject(notify);
1723         }
1724     }
1725     UNLOCKNOTIFY();
1726
1727     if( copyArray) {
1728         unsigned int    index;
1729         OSObject *      next;
1730
1731         for( index = 0; (next = copyArray->getObject( index )); index++)
1732             (*applier)(next, context);
1733         copyArray->release();
1734     }
1735
1736     OSSafeReleaseNULL(prop);
1737 }
1738
1739 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1740                                    OSObjectApplierFunction applier,
1741                                    void * context )
1742 {
1743     if (gIOGeneralInterest == typeOfInterest)
1744         applyToClients( (IOServiceApplierFunction) applier, context );
1745     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1746 }
1747
1748 struct MessageClientsContext {
1749     IOService * service;
1750     UInt32      type;
1751     void *      argument;
1752     vm_size_t   argSize;
1753     IOReturn    ret;
1754 };
1755
1756 static void messageClientsApplier( OSObject * object, void * ctx )
1757 {
1758     IOReturn                ret;
1759     MessageClientsContext * context = (MessageClientsContext *) ctx;
1760
1761     ret = context->service->messageClient( context->type,
1762                                            object, context->argument, context->argSize );
1763     if( kIOReturnSuccess != ret)
1764         context->ret = ret;
1765 }
1766
1767 // send a message to all clients
1768 IOReturn IOService::messageClients( UInt32 type,
1769                                     void * argument, vm_size_t argSize )
1770 {
1771     MessageClientsContext       context;
1772
1773     context.service     = this;
1774     context.type        = type;
1775     context.argument    = argument;
1776     context.argSize     = argSize;
1777     context.ret         = kIOReturnSuccess;
1778
1779     applyToInterested( gIOGeneralInterest,
1780                        &messageClientsApplier, &context );
1781
1782     return( context.ret );
1783 }
1784
1785 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1786                                               IOOptionBits response )
1787 {
1788     return( kIOReturnUnsupported );
1789 }
1790
1791 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1792                   IOServiceInterestHandler handler, void * target, void * ref )
1793 {
1794     _IOServiceInterestNotifier * notify = 0;
1795     IOReturn rc = kIOReturnError;
1796
1797     notify = new _IOServiceInterestNotifier;
1798     if (!notify) return NULL;
1799
1800     if(notify->init()) {
1801         rc = registerInterestForNotifier(notify, typeOfInterest,
1802                               handler, target, ref);
1803     }
1804
1805     if (rc != kIOReturnSuccess) {
1806         notify->release();
1807         notify = 0;
1808     }
1809
1810     return( notify );
1811 }
1812
1813 IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1814                   IOServiceInterestHandler handler, void * target, void * ref )
1815 {
1816     IOReturn rc = kIOReturnSuccess;
1817     _IOServiceInterestNotifier  *notify = 0;
1818
1819     if( (typeOfInterest != gIOGeneralInterest)
1820      && (typeOfInterest != gIOBusyInterest)
1821      && (typeOfInterest != gIOAppPowerStateInterest)
1822      && (typeOfInterest != gIOConsoleSecurityInterest)
1823      && (typeOfInterest != gIOPriorityPowerStateInterest))
1824         return( kIOReturnBadArgument );
1825
1826     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1827         return( kIOReturnBadArgument );
1828
1829     lockForArbitration();
1830     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1831
1832         notify->handler = handler;
1833         notify->target = target;
1834         notify->ref = ref;
1835         notify->state = kIOServiceNotifyEnable;
1836
1837         ////// queue
1838
1839         LOCKWRITENOTIFY();
1840
1841         // Get the head of the notifier linked list
1842         IOCommand * notifyList;
1843         OSObject  * obj = copyProperty( typeOfInterest );
1844         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1845             notifyList = OSTypeAlloc(IOCommand);
1846             if (notifyList) {
1847                 notifyList->init();
1848                 bool ok = setProperty( typeOfInterest, notifyList);
1849                 notifyList->release();
1850                 if (!ok) notifyList = 0;
1851             }
1852         }
1853         if (obj) obj->release();
1854
1855         if (notifyList) {
1856             enqueue(&notifyList->fCommandChain, &notify->chain);
1857             notify->retain();   // ref'ed while in list
1858         }
1859
1860         UNLOCKNOTIFY();
1861     }
1862     else {
1863         rc = kIOReturnNotReady;
1864     }
1865     unlockForArbitration();
1866
1867     return rc;
1868 }
1869
1870 static void cleanInterestList( OSObject * head )
1871 {
1872     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1873     if (!notifyHead)
1874         return;
1875
1876     LOCKWRITENOTIFY();
1877     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1878         queue_next(entry) = queue_prev(entry) = 0;
1879
1880         _IOServiceInterestNotifier * notify;
1881
1882         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1883         notify->release();
1884     }
1885     UNLOCKNOTIFY();
1886 }
1887
1888 void IOService::unregisterAllInterest( void )
1889 {
1890     OSObject * prop;
1891
1892     prop = copyProperty(gIOGeneralInterest);
1893     cleanInterestList(prop);
1894     OSSafeReleaseNULL(prop);
1895
1896     prop = copyProperty(gIOBusyInterest);
1897     cleanInterestList(prop);
1898     OSSafeReleaseNULL(prop);
1899
1900     prop = copyProperty(gIOAppPowerStateInterest);
1901     cleanInterestList(prop);
1902     OSSafeReleaseNULL(prop);
1903
1904     prop = copyProperty(gIOPriorityPowerStateInterest);
1905     cleanInterestList(prop);
1906     OSSafeReleaseNULL(prop);
1907
1908     prop = copyProperty(gIOConsoleSecurityInterest);
1909     cleanInterestList(prop);
1910     OSSafeReleaseNULL(prop);
1911 }
1912
1913 /*
1914  * _IOServiceInterestNotifier
1915  */
1916
1917 // wait for all threads, other than the current one,
1918 //  to exit the handler
1919
1920 void _IOServiceInterestNotifier::wait()
1921 {
1922     _IOServiceNotifierInvocation * next;
1923     bool doWait;
1924
1925     do {
1926         doWait = false;
1927         queue_iterate( &handlerInvocations, next,
1928                         _IOServiceNotifierInvocation *, link) {
1929             if( next->thread != current_thread() ) {
1930                 doWait = true;
1931                 break;
1932             }
1933         }
1934         if( doWait) {
1935             state |= kIOServiceNotifyWaiter;
1936                SLEEPNOTIFY(this);
1937         }
1938
1939     } while( doWait );
1940 }
1941
1942 void _IOServiceInterestNotifier::free()
1943 {
1944     assert( queue_empty( &handlerInvocations ));
1945     OSObject::free();
1946 }
1947
1948 void _IOServiceInterestNotifier::remove()
1949 {
1950     LOCKWRITENOTIFY();
1951
1952     if( queue_next( &chain )) {
1953         remqueue(&chain);
1954         queue_next( &chain) = queue_prev( &chain) = 0;
1955         release();
1956     }
1957
1958     state &= ~kIOServiceNotifyEnable;
1959
1960     wait();
1961
1962     UNLOCKNOTIFY();
1963
1964     release();
1965 }
1966
1967 bool _IOServiceInterestNotifier::disable()
1968 {
1969     bool        ret;
1970
1971     LOCKWRITENOTIFY();
1972
1973     ret = (0 != (kIOServiceNotifyEnable & state));
1974     state &= ~kIOServiceNotifyEnable;
1975     if( ret)
1976         wait();
1977
1978     UNLOCKNOTIFY();
1979
1980     return( ret );
1981 }
1982
1983 void _IOServiceInterestNotifier::enable( bool was )
1984 {
1985     LOCKWRITENOTIFY();
1986     if( was)
1987         state |= kIOServiceNotifyEnable;
1988     else
1989         state &= ~kIOServiceNotifyEnable;
1990     UNLOCKNOTIFY();
1991 }
1992
1993 bool _IOServiceInterestNotifier::init()
1994 {
1995     queue_init( &handlerInvocations );
1996     return (OSObject::init());
1997 }
1998 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1999
2000 /*
2001  * Termination
2002  */
2003
2004 #define tailQ(o)                setObject(o)
2005 #define headQ(o)                setObject(0, o)
2006 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2007
2008 static void _workLoopAction( IOWorkLoop::Action action,
2009                              IOService * service,
2010                              void * p0 = 0, void * p1 = 0,
2011                              void * p2 = 0, void * p3 = 0 )
2012 {
2013     IOWorkLoop * wl;
2014
2015     if( (wl = service->getWorkLoop())) {
2016         wl->retain();
2017         wl->runAction( action, service, p0, p1, p2, p3 );
2018         wl->release();
2019     } else
2020         (*action)( service, p0, p1, p2, p3 );
2021 }
2022
2023 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
2024 {
2025     bool ok;
2026
2027     // if its our only provider
2028     ok = isParent( provider, gIOServicePlane, true);
2029
2030     // -- compat
2031     if( ok) {
2032         provider->terminateClient( this, options | kIOServiceRecursing );
2033         ok = (0 != (kIOServiceInactiveState & __state[0]));
2034     }
2035     // --
2036
2037     return( ok );
2038 }
2039
2040 bool IOService::terminatePhase1( IOOptionBits options )
2041 {
2042     IOService *  victim;
2043     IOService *  client;
2044     OSIterator * iter;
2045     OSArray *    makeInactive;
2046     OSArray *    waitingInactive;
2047     int          waitResult = THREAD_AWAKENED;
2048     bool         wait;
2049     bool                 ok;
2050     bool                 didInactive;
2051     bool                 startPhase2 = false;
2052
2053     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2054
2055     uint64_t regID = getRegistryEntryID();
2056     IOServiceTrace(
2057         IOSERVICE_TERMINATE_PHASE1,
2058         (uintptr_t) regID,
2059         (uintptr_t) (regID >> 32),
2060         (uintptr_t) this,
2061         (uintptr_t) options);
2062
2063     // -- compat
2064     if( options & kIOServiceRecursing) {
2065         lockForArbitration();
2066         if (0 == (kIOServiceInactiveState & __state[0]))
2067         {
2068             __state[0] |= kIOServiceInactiveState;
2069             __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2070         }
2071         unlockForArbitration();
2072
2073         return( true );
2074     }
2075     // --
2076
2077     makeInactive    = OSArray::withCapacity( 16 );
2078     waitingInactive = OSArray::withCapacity( 16 );
2079     if(!makeInactive || !waitingInactive) return( false );
2080
2081     victim = this;
2082     victim->retain();
2083
2084     while( victim )
2085     {
2086         didInactive = victim->lockForArbitration( true );
2087         if( didInactive)
2088         {
2089             uint64_t regID1 = victim->getRegistryEntryID();
2090             IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2091                 (uintptr_t) regID1,
2092                 (uintptr_t) (regID1 >> 32),
2093                 (uintptr_t) victim->__state[1],
2094                 (uintptr_t) 0);
2095
2096             enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2097             didInactive = (kRP1 == (victim->__state[1] & kRP1))
2098                         || (0 == (victim->__state[0] & kIOServiceInactiveState));
2099
2100             if (!didInactive)
2101             {
2102                 // a multiply attached IOService can be visited twice
2103                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do
2104                 {
2105                     IOLockLock(gIOServiceBusyLock);
2106                     wait = (victim->__state[1] & kIOServiceTermPhase1State);
2107                     if( wait) {
2108                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2109                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2110                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2111                         victim->unlockForArbitration();
2112                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2113                     }
2114                     IOLockUnlock(gIOServiceBusyLock);
2115                     if( wait) {
2116                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2117                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2118                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2119                         victim->lockForArbitration();
2120                     }
2121                 }
2122                 while (wait && (waitResult != THREAD_TIMED_OUT));
2123             }
2124             else
2125             {
2126                 victim->__state[0] |= kIOServiceInactiveState;
2127                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2128                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2129                 victim->__state[1] &= ~kIOServiceRecursing;
2130                 victim->__state[1] |= kIOServiceTermPhase1State;
2131                 waitingInactive->headQ(victim);
2132                 if (victim == this)
2133                 {
2134                     if (kIOServiceTerminateNeedWillTerminate & options)
2135                     {
2136                         victim->__state[1] |= kIOServiceNeedWillTerminate;
2137                     }
2138                 }
2139                 victim->_adjustBusy( 1 );
2140             }
2141             victim->unlockForArbitration();
2142         }
2143         if( victim == this) startPhase2 = didInactive;
2144         if (didInactive)
2145         {
2146             OSArray * notifiers;
2147             notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2148             victim->invokeNotifiers(&notifiers);
2149
2150             IOUserClient::destroyUserReferences( victim );
2151
2152             iter = victim->getClientIterator();
2153             if( iter) {
2154                 while( (client = (IOService *) iter->getNextObject())) {
2155                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2156                             client->getName(), client->getRegistryEntryID(),
2157                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2158                     ok = client->requestTerminate( victim, options );
2159                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2160                             client->getName(), client->getRegistryEntryID(),
2161                             victim->getName(), victim->getRegistryEntryID(), ok);
2162
2163                     uint64_t regID1 = client->getRegistryEntryID();
2164                     uint64_t regID2 = victim->getRegistryEntryID();
2165                     IOServiceTrace(
2166                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2167                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
2168                         (uintptr_t) regID1,
2169                         (uintptr_t) (regID1 >> 32),
2170                         (uintptr_t) regID2,
2171                         (uintptr_t) (regID2 >> 32));
2172
2173                     if( ok)
2174                         makeInactive->setObject( client );
2175                 }
2176                 iter->release();
2177             }
2178         }
2179         victim->release();
2180         victim = (IOService *) makeInactive->getObject(0);
2181         if( victim) {
2182             victim->retain();
2183             makeInactive->removeObject(0);
2184         }
2185     }
2186     makeInactive->release();
2187
2188     while ((victim = (IOService *) waitingInactive->getObject(0)))
2189     {
2190         victim->retain();
2191         waitingInactive->removeObject(0);
2192
2193         victim->lockForArbitration();
2194         victim->__state[1] &= ~kIOServiceTermPhase1State;
2195         if (kIOServiceTerm1WaiterState & victim->__state[1])
2196         {
2197             victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2198             TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2199             IOLockLock( gIOServiceBusyLock );
2200             thread_wakeup( (event_t) &victim->__state[1]);
2201             IOLockUnlock( gIOServiceBusyLock );
2202         }
2203         victim->unlockForArbitration();
2204         victim->release();
2205     }
2206     waitingInactive->release();
2207
2208     if( startPhase2)
2209     {
2210         retain();
2211         lockForArbitration();
2212         scheduleTerminatePhase2(options);
2213         unlockForArbitration();
2214         release();
2215     }
2216
2217     return( true );
2218 }
2219
2220 void IOService::setTerminateDefer(IOService * provider, bool defer)
2221 {
2222     lockForArbitration();
2223     if (defer)  __state[1] |= kIOServiceStartState;
2224     else        __state[1] &= ~kIOServiceStartState;
2225     unlockForArbitration();
2226
2227     if (provider && !defer)
2228     {
2229         provider->lockForArbitration();
2230         provider->scheduleTerminatePhase2();
2231         provider->unlockForArbitration();
2232     }
2233 }
2234
2235 // Must call this while holding gJobsLock
2236 void IOService::waitToBecomeTerminateThread(void)
2237 {
2238     IOLockAssert(gJobsLock, kIOLockAssertOwned);
2239     bool wait;
2240     do {
2241         wait = (gIOTerminateThread != THREAD_NULL);
2242         if (wait) {
2243             IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2244         }
2245     } while (wait);
2246     gIOTerminateThread = current_thread();
2247 }
2248
2249 // call with lockForArbitration
2250 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2251 {
2252     AbsoluteTime        deadline;
2253     uint64_t            regID1;
2254     int                 waitResult = THREAD_AWAKENED;
2255     bool                wait = false, haveDeadline = false;
2256
2257     if (!(__state[0] & kIOServiceInactiveState)) return;
2258
2259     regID1 = getRegistryEntryID();
2260     IOServiceTrace(
2261         IOSERVICE_TERM_SCHED_PHASE2,
2262         (uintptr_t) regID1,
2263         (uintptr_t) (regID1 >> 32),
2264         (uintptr_t) __state[1],
2265         (uintptr_t) options);
2266
2267     if (__state[1] & kIOServiceTermPhase1State)         return;
2268
2269     retain();
2270     unlockForArbitration();
2271     options |= kIOServiceRequired;
2272     IOLockLock( gJobsLock );
2273
2274     if( (options & kIOServiceSynchronous)
2275         && (current_thread() != gIOTerminateThread)) {
2276
2277         waitToBecomeTerminateThread();
2278         gIOTerminatePhase2List->setObject( this );
2279         gIOTerminateWork++;
2280
2281         do {
2282             while( gIOTerminateWork )
2283                 terminateWorker( options );
2284             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2285             if( wait) {
2286                 /* wait for the victim to go non-busy */
2287                 if( !haveDeadline) {
2288                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2289                     haveDeadline = true;
2290                 }
2291                 /* let others do work while we wait */
2292                 gIOTerminateThread = 0;
2293                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2294                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2295                                                   deadline, THREAD_UNINT );
2296                 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2297                     panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2298                 }
2299                 waitToBecomeTerminateThread();
2300             }
2301         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2302
2303         gIOTerminateThread = 0;
2304         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2305
2306     } else {
2307         // ! kIOServiceSynchronous
2308
2309         gIOTerminatePhase2List->setObject( this );
2310         if( 0 == gIOTerminateWork++) {
2311             assert(gIOTerminateWorkerThread);
2312             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2313         }
2314     }
2315
2316     IOLockUnlock( gJobsLock );
2317     lockForArbitration();
2318     release();
2319 }
2320
2321 __attribute__((__noreturn__))
2322 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2323 {
2324     // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2325     IOLockLock(gJobsLock);
2326     while (true) {
2327         if (gIOTerminateThread != gIOTerminateWorkerThread) {
2328             waitToBecomeTerminateThread();
2329         }
2330
2331         while (gIOTerminateWork)
2332             terminateWorker( (uintptr_t)arg );
2333
2334         gIOTerminateThread = 0;
2335         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2336         IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2337     }
2338 }
2339
2340 void IOService::scheduleStop( IOService * provider )
2341 {
2342     uint64_t regID1 = getRegistryEntryID();
2343     uint64_t regID2 = provider->getRegistryEntryID();
2344
2345     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2346     IOServiceTrace(
2347         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2348         (uintptr_t) regID1,
2349         (uintptr_t) (regID1 >> 32),
2350         (uintptr_t) regID2,
2351         (uintptr_t) (regID2 >> 32));
2352
2353     IOLockLock( gJobsLock );
2354     gIOStopList->tailQ( this );
2355     gIOStopProviderList->tailQ( provider );
2356
2357     if( 0 == gIOTerminateWork++) {
2358         assert(gIOTerminateWorkerThread);
2359         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2360     }
2361
2362     IOLockUnlock( gJobsLock );
2363 }
2364
2365 void IOService::scheduleFinalize(bool now)
2366 {
2367     uint64_t regID1 = getRegistryEntryID();
2368
2369     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2370     IOServiceTrace(
2371         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2372         (uintptr_t) regID1,
2373         (uintptr_t) (regID1 >> 32),
2374         0, 0);
2375
2376     if (now || IOUserClient::finalizeUserReferences(this))
2377     {
2378         IOLockLock( gJobsLock );
2379         gIOFinalizeList->tailQ(this);
2380         if( 0 == gIOTerminateWork++) {
2381             assert(gIOTerminateWorkerThread);
2382             IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2383         }
2384         IOLockUnlock( gJobsLock );
2385     }
2386 }
2387
2388 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2389 {
2390     return( true );
2391 }
2392
2393 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2394 {
2395     if( false == *defer) {
2396
2397         if( lockForArbitration( true )) {
2398             if( false == provider->handleIsOpen( this ))
2399                 scheduleStop( provider );
2400             // -- compat
2401             else {
2402                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2403                 if( false == provider->handleIsOpen( this ))
2404                     scheduleStop( provider );
2405             }
2406             // --
2407             unlockForArbitration();
2408         }
2409     }
2410
2411     return( true );
2412 }
2413
2414 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2415                                      OSArray * doPhase2List,
2416                                      void *unused2 __unused,
2417                                      void *unused3 __unused  )
2418 {
2419     OSIterator * iter;
2420     IOService *  client;
2421     bool         ok;
2422     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2423
2424     iter = victim->getClientIterator();
2425     if( iter) {
2426         while( (client = (IOService *) iter->getNextObject())) {
2427
2428             regID1 = client->getRegistryEntryID();
2429             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2430                     client->getName(), regID1,
2431                     victim->getName(), regID2, (long long)options);
2432             IOServiceTrace(
2433                 IOSERVICE_TERMINATE_WILL,
2434                 (uintptr_t) regID1,
2435                 (uintptr_t) (regID1 >> 32),
2436                 (uintptr_t) regID2,
2437                 (uintptr_t) (regID2 >> 32));
2438
2439             ok = client->willTerminate( victim, options );
2440             doPhase2List->tailQ( client );
2441         }
2442         iter->release();
2443     }
2444 }
2445
2446 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2447                             void *unused1 __unused, void *unused2 __unused,
2448                             void *unused3 __unused )
2449 {
2450     OSIterator * iter;
2451     IOService *  client;
2452     bool         defer;
2453     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2454
2455     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2456
2457     iter = victim->getClientIterator();
2458     if( iter) {
2459         while( (client = (IOService *) iter->getNextObject())) {
2460
2461             regID1 = client->getRegistryEntryID();
2462             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2463                     client->getName(), regID1,
2464                     victim->getName(), regID2, (long long)options);
2465             defer = false;
2466             client->didTerminate( victim, options, &defer );
2467
2468             IOServiceTrace(
2469                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2470                        : IOSERVICE_TERMINATE_DID),
2471                 (uintptr_t) regID1,
2472                 (uintptr_t) (regID1 >> 32),
2473                 (uintptr_t) regID2,
2474                 (uintptr_t) (regID2 >> 32));
2475
2476             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2477                     client->getName(), regID1,
2478                     victim->getName(), regID2, defer);
2479         }
2480         iter->release();
2481     }
2482 }
2483
2484
2485 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2486                             void *unused1 __unused, void *unused2 __unused,
2487                             void *unused3 __unused )
2488 {
2489     OSIterator * iter;
2490     IOService *  provider;
2491     bool         ok;
2492     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2493
2494     iter = victim->getProviderIterator();
2495     if( iter) {
2496         while( (provider = (IOService *) iter->getNextObject())) {
2497
2498             regID1 = provider->getRegistryEntryID();
2499             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2500                     victim->getName(), regID2,
2501                     provider->getName(), regID1, (long long)options);
2502             IOServiceTrace(
2503                 IOSERVICE_TERMINATE_WILL,
2504                 (uintptr_t) regID2,
2505                 (uintptr_t) (regID2 >> 32),
2506                 (uintptr_t) regID1,
2507                 (uintptr_t) (regID1 >> 32));
2508
2509             ok = victim->willTerminate( provider, options );
2510         }
2511         iter->release();
2512     }
2513 }
2514
2515 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2516                             void *unused1 __unused, void *unused2 __unused,
2517                             void *unused3 __unused )
2518 {
2519     OSIterator * iter;
2520     IOService *  provider;
2521     bool defer = false;
2522     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2523
2524     iter = victim->getProviderIterator();
2525     if( iter) {
2526         while( (provider = (IOService *) iter->getNextObject())) {
2527
2528             regID1 = provider->getRegistryEntryID();
2529             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2530                     victim->getName(), regID2,
2531                     provider->getName(), regID1, (long long)options);
2532             victim->didTerminate( provider, options, &defer );
2533
2534             IOServiceTrace(
2535                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2536                        : IOSERVICE_TERMINATE_DID),
2537                 (uintptr_t) regID2,
2538                 (uintptr_t) (regID2 >> 32),
2539                 (uintptr_t) regID1,
2540                 (uintptr_t) (regID1 >> 32));
2541
2542             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2543                     victim->getName(), regID2,
2544                     provider->getName(), regID1, defer);
2545         }
2546         iter->release();
2547     }
2548 }
2549
2550
2551 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2552                             void *unused1 __unused, void *unused2 __unused,
2553                             void *unused3 __unused )
2554 {
2555     uint64_t regID1 = victim->getRegistryEntryID();
2556     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2557     IOServiceTrace(
2558         IOSERVICE_TERMINATE_FINALIZE,
2559         (uintptr_t) regID1,
2560         (uintptr_t) (regID1 >> 32),
2561         0, 0);
2562
2563     victim->finalize( options );
2564 }
2565
2566 void IOService::actionStop( IOService * provider, IOService * client,
2567                             void *unused1 __unused, void *unused2 __unused,
2568                             void *unused3 __unused )
2569 {
2570     uint64_t regID1 = provider->getRegistryEntryID();
2571     uint64_t regID2 = client->getRegistryEntryID();
2572
2573     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2574     IOServiceTrace(
2575         IOSERVICE_TERMINATE_STOP,
2576         (uintptr_t) regID1,
2577         (uintptr_t) (regID1 >> 32),
2578         (uintptr_t) regID2,
2579         (uintptr_t) (regID2 >> 32));
2580
2581     client->stop( provider );
2582     if( provider->isOpen( client ))
2583         provider->close( client );
2584
2585     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2586     client->detach( provider );
2587 }
2588
2589 void IOService::terminateWorker( IOOptionBits options )
2590 {
2591     OSArray *           doPhase2List;
2592     OSArray *           didPhase2List;
2593     OSSet *             freeList;
2594     OSIterator *        iter;
2595     UInt32              workDone;
2596     IOService *         victim;
2597     IOService *         client;
2598     IOService *         provider;
2599     unsigned int        idx;
2600     bool                moreToDo;
2601     bool                doPhase2;
2602     bool                doPhase3;
2603
2604     options |= kIOServiceRequired;
2605
2606     doPhase2List  = OSArray::withCapacity( 16 );
2607     didPhase2List = OSArray::withCapacity( 16 );
2608     freeList      = OSSet::withCapacity( 16 );
2609     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2610         return;
2611
2612     do {
2613         workDone = gIOTerminateWork;
2614
2615         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2616
2617             victim->retain();
2618             gIOTerminatePhase2List->removeObject(0);
2619             IOLockUnlock( gJobsLock );
2620
2621             uint64_t regID1 = victim->getRegistryEntryID();
2622             IOServiceTrace(
2623                 IOSERVICE_TERM_START_PHASE2,
2624                 (uintptr_t) regID1,
2625                 (uintptr_t) (regID1 >> 32),
2626                 (uintptr_t) 0,
2627                 (uintptr_t) 0);
2628
2629             while( victim ) {
2630
2631                 doPhase2 = victim->lockForArbitration( true );
2632                 if( doPhase2) {
2633                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2634                     if( doPhase2) {
2635
2636                         uint64_t regID1 = victim->getRegistryEntryID();
2637                         IOServiceTrace(
2638                             IOSERVICE_TERM_TRY_PHASE2,
2639                             (uintptr_t) regID1,
2640                             (uintptr_t) (regID1 >> 32),
2641                             (uintptr_t) victim->__state[1],
2642                             (uintptr_t) 0);
2643
2644                         doPhase2 = (0 == (victim->__state[1] &
2645                                     (kIOServiceTermPhase1State
2646                                     | kIOServiceTermPhase2State
2647                                     | kIOServiceConfigState)));
2648
2649                         if (doPhase2 && (iter = victim->getClientIterator())) {
2650                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2651                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2652                                 if (!doPhase2)
2653                                 {
2654                                     uint64_t regID1 = client->getRegistryEntryID();
2655                                     IOServiceTrace(
2656                                         IOSERVICE_TERM_UC_DEFER,
2657                                         (uintptr_t) regID1,
2658                                         (uintptr_t) (regID1 >> 32),
2659                                         (uintptr_t) client->__state[1],
2660                                         (uintptr_t) 0);
2661                                     TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2662                                            victim->getName(), victim->getRegistryEntryID(),
2663                                            client->getName(), client->getRegistryEntryID());
2664                                 }
2665                             }
2666                             iter->release();
2667                         }
2668                         if( doPhase2)
2669                             victim->__state[1] |= kIOServiceTermPhase2State;
2670                     }
2671                     victim->unlockForArbitration();
2672                 }
2673                 if( doPhase2) {
2674
2675                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2676                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2677                                             victim, (void *)(uintptr_t) options, NULL );
2678                     }
2679
2680                     OSArray * notifiers;
2681                     notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
2682                     victim->invokeNotifiers(&notifiers);
2683
2684                     _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2685                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2686
2687                     didPhase2List->headQ( victim );
2688                 }
2689                 victim->release();
2690                 victim = (IOService *) doPhase2List->getObject(0);
2691                 if( victim) {
2692                     victim->retain();
2693                     doPhase2List->removeObject(0);
2694                 }
2695             }
2696
2697             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2698                 bool scheduleFinalize = false;
2699                 if( victim->lockForArbitration( true )) {
2700                     victim->__state[1] |= kIOServiceTermPhase3State;
2701                     scheduleFinalize = (0 == victim->getClient());
2702                     victim->unlockForArbitration();
2703                 }
2704                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2705                                     victim, (void *)(uintptr_t) options );
2706                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2707                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2708                                         victim, (void *)(uintptr_t) options, NULL );
2709                 }
2710                 // no clients - will go to finalize
2711                 if (scheduleFinalize) victim->scheduleFinalize(false);
2712                 didPhase2List->removeObject(0);
2713             }
2714             IOLockLock( gJobsLock );
2715         }
2716
2717         // phase 3
2718         do {
2719             doPhase3 = false;
2720             // finalize leaves
2721             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2722                 bool sendFinal = false;
2723                 IOLockUnlock( gJobsLock );
2724                 if (victim->lockForArbitration(true)) {
2725                     sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
2726                     if (sendFinal) victim->__state[1] |= kIOServiceFinalized;
2727                     victim->unlockForArbitration();
2728                 }
2729                 if (sendFinal) {
2730                     _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2731                                     victim, (void *)(uintptr_t) options );
2732                 }
2733                 IOLockLock( gJobsLock );
2734                 // hold off free
2735                 freeList->setObject( victim );
2736                 // safe if finalize list is append only
2737                 gIOFinalizeList->removeObject(0);
2738             }
2739
2740             for( idx = 0;
2741                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2742
2743                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2744                 assert( provider );
2745
2746                 uint64_t regID1 = provider->getRegistryEntryID();
2747                 uint64_t regID2 = client->getRegistryEntryID();
2748
2749                 if( !provider->isChild( client, gIOServicePlane )) {
2750                     // may be multiply queued - nop it
2751                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2752                     IOServiceTrace(
2753                         IOSERVICE_TERMINATE_STOP_NOP,
2754                         (uintptr_t) regID1,
2755                         (uintptr_t) (regID1 >> 32),
2756                         (uintptr_t) regID2,
2757                         (uintptr_t) (regID2 >> 32));
2758
2759                 } else {
2760                     // a terminated client is not ready for stop if it has clients, skip it
2761                     bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
2762                     IOLockUnlock( gJobsLock );
2763                     if (deferStop && client->lockForArbitration(true)) {
2764                         deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
2765                         //deferStop = (!deferStop && (0 != client->getClient()));
2766                         //deferStop = (0 != client->getClient());
2767                         client->unlockForArbitration();
2768                         if (deferStop) {
2769                             TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
2770                             IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
2771                                (uintptr_t) regID1,
2772                                (uintptr_t) (regID1 >> 32),
2773                                (uintptr_t) regID2,
2774                                (uintptr_t) (regID2 >> 32));
2775
2776                             idx++;
2777                             IOLockLock( gJobsLock );
2778                             continue;
2779                         }
2780                     }
2781                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2782                                      provider, (void *) client );
2783                     IOLockLock( gJobsLock );
2784                     // check the finalize list now
2785                     doPhase3 = true;
2786                 }
2787                 // hold off free
2788                 freeList->setObject( client );
2789                 freeList->setObject( provider );
2790
2791                 // safe if stop list is append only
2792                 gIOStopList->removeObject( idx );
2793                 gIOStopProviderList->removeObject( idx );
2794                 idx = 0;
2795             }
2796
2797         } while( doPhase3 );
2798
2799         gIOTerminateWork -= workDone;
2800         moreToDo = (gIOTerminateWork != 0);
2801
2802         if( !moreToDo) {
2803             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2804             IOServiceTrace(
2805                 IOSERVICE_TERMINATE_DONE,
2806                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2807         }
2808
2809     } while( moreToDo );
2810
2811     IOLockUnlock( gJobsLock );
2812
2813     freeList->release();
2814     doPhase2List->release();
2815     didPhase2List->release();
2816
2817     IOLockLock( gJobsLock );
2818 }
2819
2820 bool IOService::finalize( IOOptionBits options )
2821 {
2822     OSIterator *  iter;
2823     IOService *   provider;
2824     uint64_t      regID1, regID2 = getRegistryEntryID();
2825
2826     iter = getProviderIterator();
2827     assert( iter );
2828
2829     if( iter) {
2830         while( (provider = (IOService *) iter->getNextObject())) {
2831
2832             // -- compat
2833             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2834                 /* we come down here on programmatic terminate */
2835
2836                 regID1 = provider->getRegistryEntryID();
2837                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2838                 IOServiceTrace(
2839                     IOSERVICE_TERMINATE_STOP,
2840                     (uintptr_t) regID1,
2841                     (uintptr_t) (regID1 >> 32),
2842                     (uintptr_t) regID2,
2843                     (uintptr_t) (regID2 >> 32));
2844
2845                 stop( provider );
2846                 if( provider->isOpen( this ))
2847                     provider->close( this );
2848                 detach( provider );
2849             } else {
2850             //--
2851                 if( provider->lockForArbitration( true )) {
2852                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2853                         scheduleStop( provider );
2854                     provider->unlockForArbitration();
2855                 }
2856             }
2857         }
2858         iter->release();
2859     }
2860
2861     return( true );
2862 }
2863
2864 #undef tailQ
2865 #undef headQ
2866
2867 /*
2868  * Terminate
2869  */
2870
2871 void IOService::doServiceTerminate( IOOptionBits options )
2872 {
2873 }
2874
2875 // a method in case someone needs to override it
2876 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2877 {
2878     bool ok;
2879
2880     if( client->isParent( this, gIOServicePlane, true))
2881         // we are the clients only provider
2882         ok = client->terminate( options );
2883     else
2884         ok = true;
2885
2886     return( ok );
2887 }
2888
2889 bool IOService::terminate( IOOptionBits options )
2890 {
2891     options |= kIOServiceTerminate;
2892
2893     return( terminatePhase1( options ));
2894 }
2895
2896 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2897
2898 /*
2899  * Open & close
2900  */
2901
2902 struct ServiceOpenMessageContext
2903 {
2904     IOService *  service;
2905     UInt32       type;
2906     IOService *  excludeClient;
2907     IOOptionBits options;
2908 };
2909
2910 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2911 {
2912     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2913
2914     if( object != context->excludeClient)
2915         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2916 }
2917
2918 bool IOService::open(   IOService *     forClient,
2919                         IOOptionBits    options,
2920                         void *          arg )
2921 {
2922     bool                        ok;
2923     ServiceOpenMessageContext   context;
2924
2925     context.service             = this;
2926     context.type                = kIOMessageServiceIsAttemptingOpen;
2927     context.excludeClient       = forClient;
2928     context.options             = options;
2929
2930     applyToInterested( gIOGeneralInterest,
2931                         &serviceOpenMessageApplier, &context );
2932
2933     if( false == lockForArbitration(false) )
2934         return false;
2935
2936     ok = (0 == (__state[0] & kIOServiceInactiveState));
2937     if( ok)
2938         ok = handleOpen( forClient, options, arg );
2939
2940     unlockForArbitration();
2941
2942     return( ok );
2943 }
2944
2945 void IOService::close(  IOService *     forClient,
2946                         IOOptionBits    options )
2947 {
2948     bool                wasClosed;
2949     bool                last = false;
2950
2951     lockForArbitration();
2952
2953     wasClosed = handleIsOpen( forClient );
2954     if( wasClosed) {
2955         handleClose( forClient, options );
2956         last = (__state[1] & kIOServiceTermPhase3State);
2957     }
2958
2959     unlockForArbitration();
2960
2961     if( last)
2962         forClient->scheduleStop( this );
2963
2964     else if( wasClosed) {
2965
2966         ServiceOpenMessageContext context;
2967
2968         context.service         = this;
2969         context.type            = kIOMessageServiceWasClosed;
2970         context.excludeClient   = forClient;
2971         context.options         = options;
2972
2973         applyToInterested( gIOGeneralInterest,
2974                             &serviceOpenMessageApplier, &context );
2975     }
2976 }
2977
2978 bool IOService::isOpen( const IOService * forClient ) const
2979 {
2980     IOService * self = (IOService *) this;
2981     bool ok;
2982
2983     self->lockForArbitration();
2984
2985     ok = handleIsOpen( forClient );
2986
2987     self->unlockForArbitration();
2988
2989     return( ok );
2990 }
2991
2992 bool IOService::handleOpen(     IOService *     forClient,
2993                                 IOOptionBits    options,
2994                                 void *          arg )
2995 {
2996     bool        ok;
2997
2998     ok = (0 == __owner);
2999     if( ok )
3000         __owner = forClient;
3001
3002     else if( options & kIOServiceSeize ) {
3003         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3004                                 __owner, (void *)(uintptr_t) options ));
3005         if( ok && (0 == __owner ))
3006             __owner = forClient;
3007         else
3008             ok = false;
3009     }
3010     return( ok );
3011 }
3012
3013 void IOService::handleClose(    IOService *     forClient,
3014                                 IOOptionBits    options )
3015 {
3016     if( __owner == forClient)
3017         __owner = 0;
3018 }
3019
3020 bool IOService::handleIsOpen(   const IOService * forClient ) const
3021 {
3022     if( forClient)
3023         return( __owner == forClient );
3024     else
3025         return( __owner != forClient );
3026 }
3027
3028 /*
3029  * Probing & starting
3030  */
3031 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3032 {
3033     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3034     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3035     SInt32             val1;
3036     SInt32             val2;
3037
3038     val1 = 0;
3039     val2 = 0;
3040
3041     if ( obj1 )
3042         val1 = obj1->priority;
3043
3044     if ( obj2 )
3045         val2 = obj2->priority;
3046
3047     return ( val1 - val2 );
3048 }
3049
3050 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
3051 {
3052     OSDictionary *      dict;
3053     IOService *         service;
3054     _IOServiceNotifier * notify;
3055     OSSymbol *          key = (OSSymbol *) ref;
3056     OSNumber *          offset;
3057     OSObject *          prop;
3058     SInt32              result;
3059
3060     prop = 0;
3061     result = kIODefaultProbeScore;
3062     if( (dict = OSDynamicCast( OSDictionary, entry)))
3063         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3064     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
3065         return( notify->priority );
3066     else if( (service = OSDynamicCast( IOService, entry)))
3067     {
3068         prop = service->copyProperty(key);
3069         offset = OSDynamicCast(OSNumber, prop);
3070     }
3071     else {
3072         assert( false );
3073         offset = 0;
3074     }
3075
3076     if (offset) result = offset->unsigned32BitValue();
3077
3078     OSSafeReleaseNULL(prop);
3079
3080     return (result);
3081 }
3082
3083 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3084 {
3085     const OSObject *    obj1 = (const OSObject *) inObj1;
3086     const OSObject *    obj2 = (const OSObject *) inObj2;
3087     SInt32               val1;
3088     SInt32               val2;
3089
3090     val1 = 0;
3091     val2 = 0;
3092
3093     if ( obj1 )
3094         val1 = IOServiceObjectOrder( obj1, ref );
3095
3096     if ( obj2 )
3097         val2 = IOServiceObjectOrder( obj2, ref );
3098
3099     return ( val1 - val2 );
3100 }
3101
3102 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
3103 {
3104     IOService *         service = 0;
3105     OSIterator *        iter;
3106     const OSSymbol *    nextCat;
3107
3108     iter = getClientIterator();
3109     if( iter) {
3110         while( (service = (IOService *) iter->getNextObject())) {
3111             if( kIOServiceInactiveState & service->__state[0])
3112                 continue;
3113             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3114                         service->getProperty( gIOMatchCategoryKey ));
3115             if( category == nextCat)
3116             {
3117                 service->retain();
3118                 break;
3119             }
3120         }
3121         iter->release();
3122     }
3123     return( service );
3124 }
3125
3126 IOService * IOService::getClientWithCategory( const OSSymbol * category )
3127 {
3128     IOService *
3129     service = copyClientWithCategory(category);
3130     if (service)
3131         service->release();
3132     return (service);
3133 }
3134
3135 bool IOService::invokeNotifier( _IOServiceNotifier * notify )
3136 {
3137     _IOServiceNotifierInvocation invocation;
3138     bool                         willNotify;
3139     bool                         ret = true;
3140     invocation.thread = current_thread();
3141
3142 #if DEBUG_NOTIFIER_LOCKED
3143     uint32_t count;
3144     if ((count = isLockedForArbitration(0)))
3145     {
3146         IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3147         panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3148     }
3149 #endif /* DEBUG_NOTIFIER_LOCKED */
3150
3151     LOCKWRITENOTIFY();
3152     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3153
3154     if( willNotify) {
3155         queue_enter( &notify->handlerInvocations, &invocation,
3156                         _IOServiceNotifierInvocation *, link );
3157     }
3158     UNLOCKNOTIFY();
3159
3160     if( willNotify) {
3161
3162         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3163
3164         LOCKWRITENOTIFY();
3165         queue_remove( &notify->handlerInvocations, &invocation,
3166                         _IOServiceNotifierInvocation *, link );
3167         if( kIOServiceNotifyWaiter & notify->state) {
3168             notify->state &= ~kIOServiceNotifyWaiter;
3169             WAKEUPNOTIFY( notify );
3170         }
3171         UNLOCKNOTIFY();
3172     }
3173
3174     return( ret );
3175 }
3176
3177 bool IOService::invokeNotifiers(OSArray ** willSend)
3178 {
3179     OSArray *            array;
3180     _IOServiceNotifier * notify;
3181     bool                 ret = true;
3182
3183     array = *willSend;
3184     if (!array) return (true);
3185     *willSend = 0;
3186
3187     for( unsigned int idx = 0;
3188          (notify = (_IOServiceNotifier *) array->getObject(idx));
3189          idx++) {
3190         ret &= invokeNotifier(notify);
3191     }
3192     array->release();
3193
3194     return (ret);
3195 }
3196
3197
3198 /*
3199  * Alloc and probe matching classes,
3200  * called on the provider instance
3201  */
3202
3203 void IOService::probeCandidates( OSOrderedSet * matches )
3204 {
3205     OSDictionary        *       match = 0;
3206     OSSymbol            *       symbol;
3207     IOService           *       inst;
3208     IOService           *       newInst;
3209     OSDictionary        *       props;
3210     SInt32                      score;
3211     OSNumber            *       newPri;
3212     OSOrderedSet        *       familyMatches = 0;
3213     OSOrderedSet        *       startList;
3214     OSDictionary        *       startDict = 0;
3215     const OSSymbol      *       category;
3216     OSIterator          *       iter;
3217     _IOServiceNotifier  *               notify;
3218     OSObject            *       nextMatch = 0;
3219     bool                        started;
3220     bool                        needReloc = false;
3221 #if IOMATCHDEBUG
3222     SInt64                      debugFlags;
3223 #endif
3224     IOService * client = NULL;
3225
3226
3227     assert( matches );
3228     while( !needReloc && (nextMatch = matches->getFirstObject())) {
3229
3230         nextMatch->retain();
3231         matches->removeObject(nextMatch);
3232
3233         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3234
3235             if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify );
3236             nextMatch->release();
3237             nextMatch = 0;
3238             continue;
3239
3240         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3241             nextMatch->release();
3242             nextMatch = 0;
3243             continue;
3244         }
3245
3246         props = 0;
3247 #if IOMATCHDEBUG
3248         debugFlags = getDebugFlags( match );
3249 #endif
3250
3251         do {
3252             category = OSDynamicCast( OSSymbol,
3253                         match->getObject( gIOMatchCategoryKey ));
3254             if( 0 == category)
3255                 category = gIODefaultMatchCategoryKey;
3256
3257             if( (client = copyClientWithCategory(category)) ) {
3258 #if IOMATCHDEBUG
3259                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
3260                     LOG("%s: match category %s exists\n", getName(),
3261                                 category->getCStringNoCopy());
3262 #endif
3263                 nextMatch->release();
3264                 nextMatch = 0;
3265
3266                 client->release();
3267                 client = NULL;
3268
3269                 continue;
3270             }
3271
3272             // create a copy now in case its modified during matching
3273             props = OSDictionary::withDictionary( match, match->getCount());
3274             if( 0 == props)
3275                 continue;
3276             props->setCapacityIncrement(1);
3277
3278             // check the nub matches
3279             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
3280                 continue;
3281
3282             // Check to see if driver reloc has been loaded.
3283             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
3284             if( needReloc) {
3285 #if IOMATCHDEBUG
3286                 if( debugFlags & kIOLogCatalogue)
3287                     LOG("%s: stalling for module\n", getName());
3288 #endif
3289                 // If reloc hasn't been loaded, exit;
3290                 // reprobing will occur after reloc has been loaded.
3291                 continue;
3292             }
3293
3294             // reorder on family matchPropertyTable score.
3295             if( 0 == familyMatches)
3296                 familyMatches = OSOrderedSet::withCapacity( 1,
3297                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3298             if( familyMatches)
3299                 familyMatches->setObject( props );
3300
3301         } while( false );
3302
3303         if (nextMatch) {
3304             nextMatch->release();
3305             nextMatch = 0;
3306         }
3307         if( props)
3308             props->release();
3309     }
3310     matches->release();
3311     matches = 0;
3312
3313     if( familyMatches) {
3314
3315         while( !needReloc
3316              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3317
3318             props->retain();
3319             familyMatches->removeObject( props );
3320
3321             inst = 0;
3322             newInst = 0;
3323 #if IOMATCHDEBUG
3324             debugFlags = getDebugFlags( props );
3325 #endif
3326             do {
3327                 symbol = OSDynamicCast( OSSymbol,
3328                                 props->getObject( gIOClassKey));
3329                 if( !symbol)
3330                     continue;
3331
3332                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
3333
3334                 // alloc the driver instance
3335                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3336
3337                 if( !inst || !OSDynamicCast(IOService, inst)) {
3338                     IOLog("Couldn't alloc class \"%s\"\n",
3339                         symbol->getCStringNoCopy());
3340                     continue;
3341                 }
3342
3343                 // init driver instance
3344                 if( !(inst->init( props ))) {
3345 #if IOMATCHDEBUG
3346                     if( debugFlags & kIOLogStart)
3347                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3348 #endif
3349                     continue;
3350                 }
3351                 if( __state[1] & kIOServiceSynchronousState)
3352                     inst->__state[1] |= kIOServiceSynchronousState;
3353
3354                 // give the driver the default match category if not specified
3355                 category = OSDynamicCast( OSSymbol,
3356                             props->getObject( gIOMatchCategoryKey ));
3357                 if( 0 == category)
3358                     category = gIODefaultMatchCategoryKey;
3359                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3360                 // attach driver instance
3361                 if( !(inst->attach( this )))
3362                         continue;
3363
3364                 // pass in score from property table
3365                 score = familyMatches->orderObject( props );
3366
3367                 // & probe the new driver instance
3368 #if IOMATCHDEBUG
3369                 if( debugFlags & kIOLogProbe)
3370                     LOG("%s::probe(%s)\n",
3371                         inst->getMetaClass()->getClassName(), getName());
3372 #endif
3373
3374                 newInst = inst->probe( this, &score );
3375                 inst->detach( this );
3376                 if( 0 == newInst) {
3377 #if IOMATCHDEBUG
3378                     if( debugFlags & kIOLogProbe)
3379                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3380 #endif
3381                     continue;
3382                 }
3383
3384                 // save the score
3385                 newPri = OSNumber::withNumber( score, 32 );
3386                 if( newPri) {
3387                     newInst->setProperty( gIOProbeScoreKey, newPri );
3388                     newPri->release();
3389                 }
3390
3391                 // add to start list for the match category
3392                 if( 0 == startDict)
3393                     startDict = OSDictionary::withCapacity( 1 );
3394                 assert( startDict );
3395                 startList = (OSOrderedSet *)
3396                                 startDict->getObject( category );
3397                 if( 0 == startList) {
3398                     startList = OSOrderedSet::withCapacity( 1,
3399                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3400                     if( startDict && startList) {
3401                         startDict->setObject( category, startList );
3402                         startList->release();
3403                     }
3404                 }
3405                 assert( startList );
3406                 if( startList)
3407                     startList->setObject( newInst );
3408
3409             } while( false );
3410
3411             props->release();
3412             if( inst)
3413                 inst->release();
3414         }
3415         familyMatches->release();
3416         familyMatches = 0;
3417     }
3418
3419     // start the best (until success) of each category
3420
3421     iter = OSCollectionIterator::withCollection( startDict );
3422     if( iter) {
3423         while( (category = (const OSSymbol *) iter->getNextObject())) {
3424
3425             startList = (OSOrderedSet *) startDict->getObject( category );
3426             assert( startList );
3427             if( !startList)
3428                 continue;
3429
3430             started = false;
3431             while( true // (!started)
3432                    && (inst = (IOService *)startList->getFirstObject())) {
3433
3434                 inst->retain();
3435                 startList->removeObject(inst);
3436
3437 #if IOMATCHDEBUG
3438                 debugFlags = getDebugFlags( inst );
3439
3440                 if( debugFlags & kIOLogStart) {
3441                     if( started)
3442                         LOG( "match category exists, skipping " );
3443                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3444                          getName(), inst->getRetainCount());
3445                 }
3446 #endif
3447                 if( false == started)
3448                     started = startCandidate( inst );
3449 #if IOMATCHDEBUG
3450                 if( (debugFlags & kIOLogStart) && (false == started))
3451                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3452                          inst->getRetainCount());
3453 #endif
3454                 inst->release();
3455             }
3456         }
3457         iter->release();
3458     }
3459
3460
3461     // adjust the busy count by +1 if matching is stalled for a module,
3462     // or -1 if a previously stalled matching is complete.
3463     lockForArbitration();
3464     SInt32 adjBusy = 0;
3465     uint64_t regID = getRegistryEntryID();
3466
3467     if( needReloc) {
3468         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3469         if( adjBusy) {
3470
3471             IOServiceTrace(
3472                 IOSERVICE_MODULESTALL,
3473                 (uintptr_t) regID,
3474                 (uintptr_t) (regID >> 32),
3475                 (uintptr_t) this,
3476                 0);
3477
3478             __state[1] |= kIOServiceModuleStallState;
3479         }
3480
3481     } else if( __state[1] & kIOServiceModuleStallState) {
3482
3483             IOServiceTrace(
3484             IOSERVICE_MODULEUNSTALL,
3485             (uintptr_t) regID,
3486             (uintptr_t) (regID >> 32),
3487             (uintptr_t) this,
3488             0);
3489
3490         __state[1] &= ~kIOServiceModuleStallState;
3491         adjBusy = -1;
3492     }
3493     if( adjBusy)
3494         _adjustBusy( adjBusy );
3495     unlockForArbitration();
3496
3497     if( startDict)
3498         startDict->release();
3499 }
3500
3501 /*
3502  * Start a previously attached & probed instance,
3503  * called on exporting object instance
3504  */
3505
3506 bool IOService::startCandidate( IOService * service )
3507 {
3508     bool                ok;
3509
3510     ok = service->attach( this );
3511
3512     if( ok)
3513     {
3514         if (this != gIOResources)
3515         {
3516             // stall for any nub resources
3517             checkResources();
3518             // stall for any driver resources
3519             service->checkResources();
3520         }
3521
3522         AbsoluteTime startTime;
3523         AbsoluteTime endTime;
3524         UInt64       nano;
3525
3526         if (kIOLogStart & gIOKitDebug)
3527             clock_get_uptime(&startTime);
3528
3529         ok = service->start(this);
3530
3531         if (kIOLogStart & gIOKitDebug)
3532         {
3533             clock_get_uptime(&endTime);
3534
3535             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3536             {
3537                 SUB_ABSOLUTETIME(&endTime, &startTime);
3538                 absolutetime_to_nanoseconds(endTime, &nano);
3539                 if (nano > 500000000ULL)
3540                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3541             }
3542         }
3543         if( !ok)
3544             service->detach( this );
3545     }
3546     return( ok );
3547 }
3548
3549 void IOService::publishResource( const char * key, OSObject * value )
3550 {
3551     const OSSymbol *    sym;
3552
3553     if( (sym = OSSymbol::withCString( key))) {
3554         publishResource( sym, value);
3555         sym->release();
3556     }
3557 }
3558
3559 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3560 {
3561     if( 0 == value)
3562         value = (OSObject *) gIOServiceKey;
3563
3564     gIOResources->setProperty( key, value);
3565
3566     if( IORecursiveLockHaveLock( gNotificationLock))
3567         return;
3568
3569     gIOResourceGenerationCount++;
3570     gIOResources->registerService();
3571 }
3572
3573 bool IOService::addNeededResource( const char * key )
3574 {
3575     OSObject *  resourcesProp;
3576     OSSet *     set;
3577     OSString *  newKey;
3578     bool ret;
3579
3580     resourcesProp = copyProperty( gIOResourceMatchKey );
3581     if (!resourcesProp) return(false);
3582
3583     newKey = OSString::withCString( key );
3584     if (!newKey)
3585     {
3586         resourcesProp->release();
3587         return( false);
3588     }
3589
3590     set = OSDynamicCast( OSSet, resourcesProp );
3591     if( !set) {
3592         set = OSSet::withCapacity( 1 );
3593         if( set)
3594             set->setObject( resourcesProp );
3595     }
3596     else
3597         set->retain();
3598
3599     set->setObject( newKey );
3600     newKey->release();
3601     ret = setProperty( gIOResourceMatchKey, set );
3602     set->release();
3603     resourcesProp->release();
3604
3605     return( ret );
3606 }
3607
3608 bool IOService::checkResource( OSObject * matching )
3609 {
3610     OSString *          str;
3611     OSDictionary *      table;
3612
3613     if( (str = OSDynamicCast( OSString, matching ))) {
3614         if( gIOResources->getProperty( str ))
3615             return( true );
3616     }
3617
3618     if( str)
3619         table = resourceMatching( str );
3620     else if( (table = OSDynamicCast( OSDictionary, matching )))
3621         table->retain();
3622     else {
3623         IOLog("%s: Can't match using: %s\n", getName(),
3624                 matching->getMetaClass()->getClassName());
3625         /* false would stall forever */
3626         return( true );
3627     }
3628
3629     if( gIOKitDebug & kIOLogConfig)
3630         LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
3631
3632     waitForService( table );
3633
3634     if( gIOKitDebug & kIOLogConfig)
3635         LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
3636
3637     return( true );
3638 }
3639
3640 bool IOService::checkResources( void )
3641 {
3642     OSObject *          resourcesProp;
3643     OSSet *             set;
3644     OSIterator *        iter;
3645     bool                ok;
3646
3647     resourcesProp = copyProperty( gIOResourceMatchKey );
3648     if( 0 == resourcesProp)
3649         return( true );
3650
3651     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3652
3653         iter = OSCollectionIterator::withCollection( set );
3654         ok = (0 != iter);
3655         while( ok && (resourcesProp = iter->getNextObject()) )
3656             ok = checkResource( resourcesProp );
3657         if( iter)
3658             iter->release();
3659
3660     } else
3661         ok = checkResource( resourcesProp );
3662
3663     OSSafeReleaseNULL(resourcesProp);
3664
3665     return( ok );
3666 }
3667
3668
3669 void _IOConfigThread::configThread( void )
3670 {
3671     _IOConfigThread *   inst;
3672
3673     do {
3674         if( !(inst = new _IOConfigThread))
3675             continue;
3676         if( !inst->init())
3677             continue;
3678         thread_t unused;
3679         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3680             continue;
3681
3682         return;
3683
3684     } while( false);
3685
3686     if( inst)
3687         inst->release();
3688
3689     return;
3690 }
3691
3692 void _IOConfigThread::free( void )
3693 {
3694     thread_deallocate(current_thread());
3695     OSObject::free();
3696 }
3697
3698 void IOService::doServiceMatch( IOOptionBits options )
3699 {
3700     _IOServiceNotifier * notify;
3701     OSIterator *        iter;
3702     OSOrderedSet *      matches;
3703     OSArray *           resourceKeys = 0;
3704     SInt32              catalogGeneration;
3705     bool                keepGuessing = true;
3706     bool                reRegistered = true;
3707     bool                didRegister;
3708     OSArray *           notifiers[2] = {0};
3709
3710 //    job->nub->deliverNotification( gIOPublishNotification,
3711 //                              kIOServiceRegisteredState, 0xffffffff );
3712
3713     while( keepGuessing ) {
3714
3715         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3716         // the matches list should always be created by findDrivers()
3717         if( matches) {
3718
3719             lockForArbitration();
3720             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3721                 getMetaClass()->addInstance(this);
3722                 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
3723                                      kIOServiceFirstPublishState, 0xffffffff );
3724             }
3725             LOCKREADNOTIFY();
3726             __state[1] &= ~kIOServiceNeedConfigState;
3727             __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
3728             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3729             __state[0] |= kIOServiceRegisteredState;
3730
3731             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3732             if (reRegistered && keepGuessing) {
3733                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3734                         gNotifications->getObject( gIOPublishNotification ) );
3735                 if( iter) {
3736                     while((notify = (_IOServiceNotifier *)
3737                            iter->getNextObject())) {
3738
3739                         if( matchPassive(notify->matching, 0)
3740                          && (kIOServiceNotifyEnable & notify->state))
3741                             matches->setObject( notify );
3742                     }
3743                     iter->release();
3744                 }
3745             }
3746
3747             UNLOCKNOTIFY();
3748             unlockForArbitration();
3749             invokeNotifiers(&notifiers[0]);
3750
3751             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3752             {
3753                 if (this == gIOResources)
3754                 {
3755                    if (resourceKeys) resourceKeys->release();
3756                    resourceKeys = copyPropertyKeys();
3757                 }
3758                 probeCandidates( matches );
3759             }
3760             else
3761                 matches->release();
3762         }
3763
3764         lockForArbitration();
3765         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3766         keepGuessing =
3767                    (reRegistered || (catalogGeneration !=
3768                                         gIOCatalogue->getGenerationCount()))
3769                 && (0 == (__state[0] & kIOServiceInactiveState));
3770
3771         if( keepGuessing)
3772             unlockForArbitration();
3773     }
3774
3775     if( (0 == (__state[0] & kIOServiceInactiveState))
3776      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3777
3778         if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys);
3779
3780         notifiers[0] = copyNotifiers(gIOMatchedNotification,
3781                 kIOServiceMatchedState, 0xffffffff);
3782         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3783             notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
3784                 kIOServiceFirstMatchState, 0xffffffff);
3785     }
3786
3787     __state[1] &= ~kIOServiceConfigRunning;
3788     unlockForArbitration();
3789
3790     if (resourceKeys) resourceKeys->release();
3791
3792     invokeNotifiers(&notifiers[0]);
3793     invokeNotifiers(&notifiers[1]);
3794
3795     lockForArbitration();
3796     __state[1] &= ~kIOServiceConfigState;
3797     scheduleTerminatePhase2();
3798
3799     _adjustBusy( -1 );
3800     unlockForArbitration();
3801 }
3802
3803 UInt32 IOService::_adjustBusy( SInt32 delta )
3804 {
3805     IOService * next;
3806     UInt32      count;
3807     UInt32      result;
3808     bool        wasQuiet, nowQuiet, needWake;
3809
3810     next = this;
3811     result = __state[1] & kIOServiceBusyStateMask;
3812
3813     if( delta) do {
3814         if( next != this)
3815             next->lockForArbitration();
3816         count = next->__state[1] & kIOServiceBusyStateMask;
3817         wasQuiet = (0 == count);
3818         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3819             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3820         else
3821             count += delta;
3822         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3823         nowQuiet = (0 == count);
3824         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3825
3826         if( needWake) {
3827             next->__state[1] &= ~kIOServiceBusyWaiterState;
3828             IOLockLock( gIOServiceBusyLock );
3829             thread_wakeup( (event_t) next);
3830             IOLockUnlock( gIOServiceBusyLock );
3831         }
3832         if( next != this)
3833             next->unlockForArbitration();
3834
3835         if( (wasQuiet || nowQuiet) ) {
3836
3837             uint64_t regID = next->getRegistryEntryID();
3838             IOServiceTrace(
3839                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3840                 (uintptr_t) regID,
3841                 (uintptr_t) (regID >> 32),
3842                 (uintptr_t) next,
3843                 0);
3844
3845             if (wasQuiet)
3846             {
3847                 next->__timeBusy = mach_absolute_time();
3848             }
3849             else
3850             {
3851                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3852                 next->__timeBusy = 0;
3853             }
3854
3855             MessageClientsContext context;
3856
3857             context.service  = next;
3858             context.type     = kIOMessageServiceBusyStateChange;
3859             context.argument = (void *) wasQuiet;       /*nowBusy*/
3860             context.argSize  = 0;
3861
3862             applyToInterestNotifiers( next, gIOBusyInterest,
3863                                      &messageClientsApplier, &context );
3864
3865 #if !NO_KEXTD
3866             if( nowQuiet && (next == gIOServiceRoot)) {
3867                 OSKext::considerUnloads();
3868                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3869             }
3870 #endif
3871         }
3872
3873         delta = nowQuiet ? -1 : +1;
3874
3875     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3876
3877     return( result );
3878 }
3879
3880 void IOService::adjustBusy( SInt32 delta )
3881 {
3882     lockForArbitration();
3883     _adjustBusy( delta );
3884     unlockForArbitration();
3885 }
3886
3887 uint64_t IOService::getAccumulatedBusyTime( void )
3888 {
3889     uint64_t accumBusy = __accumBusy;
3890     uint64_t timeBusy = __timeBusy;
3891     uint64_t nano;
3892
3893     do
3894     {
3895         accumBusy = __accumBusy;
3896         timeBusy  = __timeBusy;
3897         if (timeBusy)
3898             accumBusy += mach_absolute_time() - timeBusy;
3899     }
3900     while (timeBusy != __timeBusy);
3901
3902     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3903
3904     return (nano);
3905 }
3906
3907 UInt32 IOService::getBusyState( void )
3908 {
3909     return( __state[1] & kIOServiceBusyStateMask );
3910 }
3911
3912 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3913                                   mach_timespec_t * timeout )
3914 {
3915     panic("waitForState");
3916     return (kIOReturnUnsupported);
3917 }
3918
3919 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3920                                    uint64_t timeout )
3921 {
3922     bool            wait;
3923     int             waitResult = THREAD_AWAKENED;
3924     bool            computeDeadline = true;
3925     AbsoluteTime    abstime;
3926
3927     do {
3928         lockForArbitration();
3929         IOLockLock( gIOServiceBusyLock );
3930         wait = (value != (__state[1] & mask));
3931         if( wait) {
3932             __state[1] |= kIOServiceBusyWaiterState;
3933             unlockForArbitration();
3934             if( timeout != UINT64_MAX ) {
3935                 if( computeDeadline ) {
3936                     AbsoluteTime  nsinterval;
3937                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3938                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3939                     computeDeadline = false;
3940                 }
3941                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3942             }
3943             else
3944                 assert_wait((event_t)this, THREAD_UNINT );
3945         } else
3946             unlockForArbitration();
3947         IOLockUnlock( gIOServiceBusyLock );
3948         if( wait)
3949             waitResult = thread_block(THREAD_CONTINUE_NULL);
3950
3951     } while( wait && (waitResult != THREAD_TIMED_OUT));
3952
3953     if( waitResult == THREAD_TIMED_OUT)
3954         return( kIOReturnTimeout );
3955     else
3956         return( kIOReturnSuccess );
3957 }
3958
3959 IOReturn IOService::waitQuiet( uint64_t timeout )
3960 {
3961     IOReturn ret;
3962     uint32_t loops;
3963     char *   string = NULL;
3964     char *   panicString = NULL;
3965     size_t   len;
3966     size_t   panicStringLen;
3967     uint64_t time;
3968     uint64_t nano;
3969     bool     kextdWait;
3970     bool     dopanic;
3971
3972     enum { kTimeoutExtensions = 4 };
3973
3974     time = mach_absolute_time();
3975     kextdWait = false;
3976     for (loops = 0; loops < kTimeoutExtensions; loops++)
3977     {
3978         ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3979
3980         if (loops && (kIOReturnSuccess == ret))
3981         {
3982             time = mach_absolute_time() - time;
3983             absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
3984             IOLog("busy extended ok[%d], (%llds, %llds)\n",
3985                   loops, timeout / 1000000000ULL, nano / 1000000000ULL);
3986             break;
3987         }
3988         else if (kIOReturnTimeout != ret) break;
3989         else if (timeout < 41000000000)   break;
3990
3991         {
3992             IORegistryIterator * iter;
3993             OSOrderedSet       * set;
3994             OSOrderedSet       * leaves;
3995             IOService          * next;
3996             IOService          * nextParent;
3997             char               * s;
3998             size_t               l;
3999
4000             len = 256;
4001             panicStringLen = 256;
4002             if (!string)      string      = IONew(char, len);
4003             if (!panicString) panicString = IONew(char, panicStringLen);
4004             set = NULL;
4005             kextdWait = OSKext::isWaitingKextd();
4006             iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
4007             leaves = OSOrderedSet::withCapacity(4);
4008             if (iter) set = iter->iterateAll();
4009             if (string && panicString && leaves && set)
4010             {
4011                 string[0] = panicString[0] = 0;
4012                 set->setObject(this);
4013                 while ((next = (IOService *) set->getLastObject()))
4014                 {
4015                     if (next->getBusyState())
4016                     {
4017                         if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true;
4018                         leaves->setObject(next);
4019                         nextParent = next;
4020                         while ((nextParent = nextParent->getProvider()))
4021                         {
4022                             set->removeObject(nextParent);
4023                             leaves->removeObject(nextParent);
4024                         }
4025                     }
4026                     set->removeObject(next);
4027                 }
4028                 s = string;
4029                 while ((next = (IOService *) leaves->getLastObject()))
4030                 {
4031                     l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
4032                     if (l >= len) break;
4033                     s += l;
4034                     len -= l;
4035                     leaves->removeObject(next);
4036                 }
4037             }
4038             OSSafeReleaseNULL(leaves);
4039             OSSafeReleaseNULL(set);
4040             OSSafeReleaseNULL(iter);
4041         }
4042
4043         dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug));
4044         snprintf(panicString, panicStringLen,
4045                  "%s[%d], (%llds): %s",
4046                  kextdWait ? "kextd stall" : "busy timeout",
4047                  loops, timeout / 1000000000ULL,
4048                  string ? string : "");
4049         IOLog("%s\n", panicString);
4050         if (dopanic)     panic("%s", panicString);
4051         else if (!loops) getPMRootDomain()->startSpinDump(1);
4052     }
4053
4054     if (string)      IODelete(string, char, 256);
4055     if (panicString) IODelete(panicString, char, panicStringLen);
4056
4057     return (ret);
4058 }
4059
4060 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
4061 {
4062     uint64_t    timeoutNS;
4063
4064     if (timeout)
4065     {
4066         timeoutNS = timeout->tv_sec;
4067         timeoutNS *= kSecondScale;
4068         timeoutNS += timeout->tv_nsec;
4069     }
4070     else
4071         timeoutNS = UINT64_MAX;
4072
4073     return (waitQuiet(timeoutNS));
4074 }
4075
4076 bool IOService::serializeProperties( OSSerialize * s ) const
4077 {
4078 #if 0
4079     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
4080                 sizeof( __state), "__state");
4081 #endif
4082     return( super::serializeProperties(s) );
4083 }
4084
4085
4086 void _IOConfigThread::main(void * arg, wait_result_t result)
4087 {
4088     _IOConfigThread * self = (_IOConfigThread *) arg;
4089     _IOServiceJob * job;
4090     IOService   *   nub;
4091     bool            alive = true;
4092     kern_return_t   kr;
4093     thread_precedence_policy_data_t precedence = { -1 };
4094
4095     kr = thread_policy_set(current_thread(),
4096                             THREAD_PRECEDENCE_POLICY,
4097                             (thread_policy_t) &precedence,
4098                             THREAD_PRECEDENCE_POLICY_COUNT);
4099     if (KERN_SUCCESS != kr)
4100         IOLog("thread_policy_set(%d)\n", kr);
4101
4102     do {
4103
4104 //      randomDelay();
4105
4106         semaphore_wait( gJobsSemaphore );
4107
4108         IOTakeLock( gJobsLock );
4109         job = (_IOServiceJob *) gJobs->getFirstObject();
4110         job->retain();
4111         gJobs->removeObject(job);
4112         if( job) {
4113             gOutstandingJobs--;
4114 //          gNumConfigThreads--;        // we're out of service
4115             gNumWaitingThreads--;       // we're out of service
4116         }
4117         IOUnlock( gJobsLock );
4118
4119         if( job) {
4120
4121           nub = job->nub;
4122
4123           if( gIOKitDebug & kIOLogConfig)
4124             LOG("config(%p): starting on %s, %d\n",
4125                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
4126
4127           switch( job->type) {
4128
4129             case kMatchNubJob:
4130                 nub->doServiceMatch( job->options );
4131                 break;
4132
4133             default:
4134                 LOG("config(%p): strange type (%d)\n",
4135                         IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
4136                 break;
4137             }
4138
4139             nub->release();
4140             job->release();
4141
4142             IOTakeLock( gJobsLock );
4143             alive = (gOutstandingJobs > gNumWaitingThreads);
4144             if( alive)
4145                 gNumWaitingThreads++;   // back in service
4146 //              gNumConfigThreads++;
4147             else {
4148                 if( 0 == --gNumConfigThreads) {
4149 //                    IOLog("MATCH IDLE\n");
4150                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
4151                 }
4152             }
4153             IOUnlock( gJobsLock );
4154         }
4155
4156     } while( alive );
4157
4158     if( gIOKitDebug & kIOLogConfig)
4159         LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) );
4160
4161     self->release();
4162 }
4163
4164 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
4165 {
4166     bool            wait;
4167     int             waitResult = THREAD_AWAKENED;
4168     bool            computeDeadline = true;
4169     AbsoluteTime    deadline;
4170
4171     IOLockLock( gJobsLock );
4172     do {
4173         wait = (0 != gNumConfigThreads);
4174         if( wait) {
4175             if( msToWait) {
4176                 if( computeDeadline ) {
4177                     clock_interval_to_deadline(
4178                           msToWait, kMillisecondScale, &deadline );
4179                     computeDeadline = false;
4180                 }
4181                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
4182                                                                 deadline, THREAD_UNINT );
4183                    } else {
4184                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
4185                                                                 THREAD_UNINT );
4186                 }
4187         }
4188     } while( wait && (waitResult != THREAD_TIMED_OUT));
4189         IOLockUnlock( gJobsLock );
4190
4191     if( waitResult == THREAD_TIMED_OUT)
4192         return( kIOReturnTimeout );
4193     else
4194         return( kIOReturnSuccess );
4195 }
4196
4197 void IOService::cpusRunning(void)
4198 {
4199     gCPUsRunning = true;
4200 }
4201
4202 void _IOServiceJob::pingConfig( _IOServiceJob * job )
4203 {
4204     int         count;
4205     bool        create;
4206
4207     assert( job );
4208
4209     IOTakeLock( gJobsLock );
4210
4211     gOutstandingJobs++;
4212     gJobs->setLastObject( job );
4213
4214     count = gNumWaitingThreads;
4215 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
4216
4217     create = (  (gOutstandingJobs > count)
4218                 && ((gNumConfigThreads < kMaxConfigThreads)
4219             || (job->nub == gIOResources)
4220             || !gCPUsRunning));
4221     if( create) {
4222         gNumConfigThreads++;
4223         gNumWaitingThreads++;
4224     }
4225
4226     IOUnlock( gJobsLock );
4227
4228     job->release();
4229
4230     if( create) {
4231         if( gIOKitDebug & kIOLogConfig)
4232             LOG("config(%d): creating\n", gNumConfigThreads - 1);
4233         _IOConfigThread::configThread();
4234     }
4235
4236     semaphore_signal( gJobsSemaphore );
4237 }
4238
4239 struct IOServiceMatchContext
4240 {
4241     OSDictionary * table;
4242     OSObject *     result;
4243     uint32_t       options;
4244     uint32_t       state;
4245     uint32_t       count;
4246     uint32_t       done;
4247 };
4248
4249 bool IOService::instanceMatch(const OSObject * entry, void * context)
4250 {
4251     IOServiceMatchContext * ctx = (typeof(ctx)) context;
4252     IOService *    service = (typeof(service)) entry;
4253     OSDictionary * table   = ctx->table;
4254     uint32_t       options = ctx->options;
4255     uint32_t       state   = ctx->state;
4256     uint32_t       done;
4257     bool           match;
4258
4259     done = 0;
4260     do
4261     {
4262         match = ((state == (state & service->__state[0]))
4263                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
4264         if (!match) break;
4265         ctx->count += table->getCount();
4266         match = service->matchInternal(table, options, &done);
4267         ctx->done += done;
4268     }
4269     while (false);
4270     if (!match)
4271         return (false);
4272
4273     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
4274     {
4275         service->retain();
4276         ctx->result = service;
4277         return (true);
4278     }
4279     else if (!ctx->result)
4280     {
4281         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
4282     }
4283     else
4284     {
4285         ((OSSet *)ctx->result)->setObject(service);
4286     }
4287     return (false);
4288 }
4289
4290 // internal - call with gNotificationLock
4291 OSObject * IOService::copyExistingServices( OSDictionary * matching,
4292                  IOOptionBits inState, IOOptionBits options )
4293 {
4294     OSObject *   current = 0;
4295     OSIterator * iter;
4296     IOService *  service;
4297     OSObject *   obj;
4298     OSString *   str;
4299
4300     if( !matching)
4301         return( 0 );
4302
4303 #if MATCH_DEBUG
4304     OSSerialize * s = OSSerialize::withCapacity(128);
4305     matching->serialize(s);
4306 #endif
4307
4308     if((obj = matching->getObject(gIOProviderClassKey))
4309       && gIOResourcesKey
4310       && gIOResourcesKey->isEqualTo(obj)
4311       && (service = gIOResources))
4312     {
4313         if( (inState == (service->__state[0] & inState))
4314           && (0 == (service->__state[0] & kIOServiceInactiveState))
4315           &&  service->matchPassive(matching, options))
4316         {
4317             if( options & kIONotifyOnce)
4318             {
4319                 service->retain();
4320                 current = service;
4321             }
4322             else
4323                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
4324         }
4325     }
4326     else
4327     {
4328         IOServiceMatchContext ctx;
4329         ctx.table   = matching;
4330         ctx.state   = inState;
4331         ctx.count   = 0;
4332         ctx.done    = 0;
4333         ctx.options = options;
4334         ctx.result  = 0;
4335
4336         if ((str = OSDynamicCast(OSString, obj)))
4337         {
4338             const OSSymbol * sym = OSSymbol::withString(str);
4339             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
4340             sym->release();
4341         }
4342         else
4343         {
4344             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
4345         }
4346
4347
4348         current = ctx.result;
4349
4350         options |= kIOServiceInternalDone | kIOServiceClassDone;
4351         if (current && (ctx.done != ctx.count))
4352         {
4353             OSSet *
4354             source = OSDynamicCast(OSSet, current);
4355             current = 0;
4356             while ((service = (IOService *) source->getAnyObject()))
4357             {
4358                 if (service->matchPassive(matching, options))
4359                 {
4360                     if( options & kIONotifyOnce)
4361                     {
4362                         service->retain();
4363                         current = service;
4364                         break;
4365                     }
4366                     if( current)
4367                     {
4368                         ((OSSet *)current)->setObject( service );
4369                     }
4370                     else
4371                     {
4372                         current = OSSet::withObjects(
4373                                         (const OSObject **) &service, 1, 1 );
4374                     }
4375                 }
4376                 source->removeObject(service);
4377             }
4378             source->release();
4379         }
4380     }
4381
4382 #if MATCH_DEBUG
4383     {
4384         OSObject * _current = 0;
4385
4386         iter = IORegistryIterator::iterateOver( gIOServicePlane,
4387                                             kIORegistryIterateRecursively );
4388         if( iter) {
4389             do {
4390                 iter->reset();
4391                 while( (service = (IOService *) iter->getNextObject())) {
4392                     if( (inState == (service->__state[0] & inState))
4393                     && (0 == (service->__state[0] & kIOServiceInactiveState))
4394                     &&  service->matchPassive(matching, 0)) {
4395
4396                         if( options & kIONotifyOnce) {
4397                             service->retain();
4398                             _current = service;
4399                             break;
4400                         }
4401                         if( _current)
4402                             ((OSSet *)_current)->setObject( service );
4403                         else
4404                             _current = OSSet::withObjects(
4405                                             (const OSObject **) &service, 1, 1 );
4406                     }
4407                 }
4408             } while( !service && !iter->isValid());
4409             iter->release();
4410         }
4411
4412
4413         if ( ((current != 0) != (_current != 0))
4414         || (current && _current && !current->isEqualTo(_current)))
4415         {
4416             OSSerialize * s1 = OSSerialize::withCapacity(128);
4417             OSSerialize * s2 = OSSerialize::withCapacity(128);
4418             current->serialize(s1);
4419             _current->serialize(s2);
4420             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
4421                 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4422             s1->release();
4423             s2->release();
4424         }
4425
4426         if (_current) _current->release();
4427     }
4428
4429     s->release();
4430 #endif
4431
4432     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4433         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4434         current->release();
4435         current = iter;
4436     }
4437
4438     return( current );
4439 }
4440
4441 // public version
4442 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4443 {
4444     OSIterator *        iter;
4445
4446     // is a lock even needed?
4447     LOCKWRITENOTIFY();
4448
4449     iter = (OSIterator *) copyExistingServices( matching,
4450                                                 kIOServiceMatchedState );
4451
4452     UNLOCKNOTIFY();
4453
4454     return( iter );
4455 }
4456
4457 IOService * IOService::copyMatchingService( OSDictionary * matching )
4458 {
4459     IOService * service;
4460
4461     // is a lock even needed?
4462     LOCKWRITENOTIFY();
4463
4464     service = (IOService *) copyExistingServices( matching,
4465                                                 kIOServiceMatchedState, kIONotifyOnce );
4466
4467     UNLOCKNOTIFY();
4468
4469     return( service );
4470 }
4471
4472 struct _IOServiceMatchingNotificationHandlerRef
4473 {
4474     IOServiceNotificationHandler handler;
4475     void * ref;
4476 };
4477
4478 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4479                                                    IOService * newService,
4480                                                    IONotifier * notifier )
4481 {
4482     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4483 }
4484
4485 // internal - call with gNotificationLock
4486 IONotifier * IOService::setNotification(
4487             const OSSymbol * type, OSDictionary * matching,
4488             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4489             SInt32 priority )
4490 {
4491     _IOServiceNotifier * notify = 0;
4492     OSOrderedSet *      set;
4493
4494     if( !matching)
4495         return( 0 );
4496
4497     notify = new _IOServiceNotifier;
4498     if( notify && !notify->init()) {
4499         notify->release();
4500         notify = 0;
4501     }
4502
4503     if( notify) {
4504         notify->handler = handler;
4505         notify->target = target;
4506         notify->type = type;
4507         notify->matching = matching;
4508         matching->retain();
4509         if (handler == &_IOServiceMatchingNotificationHandler)
4510         {
4511             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4512             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4513         }
4514         else
4515             notify->ref = ref;
4516         notify->priority = priority;
4517         notify->state = kIOServiceNotifyEnable;
4518         queue_init( &notify->handlerInvocations );
4519
4520         ////// queue
4521
4522         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4523             set = OSOrderedSet::withCapacity( 1,
4524                         IONotifyOrdering, 0 );
4525             if( set) {
4526                 gNotifications->setObject( type, set );
4527                 set->release();
4528             }
4529         }
4530         notify->whence = set;
4531         if( set)
4532             set->setObject( notify );
4533     }
4534
4535     return( notify );
4536 }
4537
4538 // internal - call with gNotificationLock
4539 IONotifier * IOService::doInstallNotification(
4540                         const OSSymbol * type, OSDictionary * matching,
4541                         IOServiceMatchingNotificationHandler handler,
4542                         void * target, void * ref,
4543                         SInt32 priority, OSIterator ** existing )
4544 {
4545     OSIterator *        exist;
4546     IONotifier *        notify;
4547     IOOptionBits        inState;
4548
4549     if( !matching)
4550         return( 0 );
4551
4552     if( type == gIOPublishNotification)
4553         inState = kIOServiceRegisteredState;
4554
4555     else if( type == gIOFirstPublishNotification)
4556         inState = kIOServiceFirstPublishState;
4557
4558     else if (type == gIOMatchedNotification)
4559         inState = kIOServiceMatchedState;
4560
4561     else if (type == gIOFirstMatchNotification)
4562         inState = kIOServiceFirstMatchState;
4563
4564     else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification))
4565         inState = 0;
4566     else
4567         return( 0 );
4568
4569     notify = setNotification( type, matching, handler, target, ref, priority );
4570
4571     if( inState)
4572         // get the current set
4573         exist = (OSIterator *) copyExistingServices( matching, inState );
4574     else
4575         exist = 0;
4576
4577     *existing = exist;
4578
4579     return( notify );
4580 }
4581
4582 #if !defined(__LP64__)
4583 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4584                                         IOServiceNotificationHandler handler,
4585                                         void * target, void * refCon,
4586                                         SInt32 priority, OSIterator ** existing )
4587 {
4588     IONotifier * result;
4589     _IOServiceMatchingNotificationHandlerRef ref;
4590     ref.handler = handler;
4591     ref.ref     = refCon;
4592
4593     result = (_IOServiceNotifier *) installNotification( type, matching,
4594                          &_IOServiceMatchingNotificationHandler,
4595                         target, &ref, priority, existing );
4596     if (result)
4597         matching->release();
4598
4599     return (result);
4600 }
4601 #endif /* !defined(__LP64__) */
4602
4603
4604 IONotifier * IOService::installNotification(
4605                         const OSSymbol * type, OSDictionary * matching,
4606                         IOServiceMatchingNotificationHandler handler,
4607                         void * target, void * ref,
4608                         SInt32 priority, OSIterator ** existing )
4609 {
4610     IONotifier * notify;
4611
4612     LOCKWRITENOTIFY();
4613
4614     notify = doInstallNotification( type, matching, handler, target, ref,
4615                 priority, existing );
4616
4617     // in case handler remove()s
4618     if (notify) notify->retain();
4619
4620     UNLOCKNOTIFY();
4621
4622     return( notify );
4623 }
4624
4625 IONotifier * IOService::addNotification(
4626                         const OSSymbol * type, OSDictionary * matching,
4627                         IOServiceNotificationHandler handler,
4628                         void * target, void * refCon,
4629                         SInt32 priority )
4630 {
4631     IONotifier * result;
4632     _IOServiceMatchingNotificationHandlerRef ref;
4633
4634     ref.handler = handler;
4635     ref.ref     = refCon;
4636
4637     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4638                             target, &ref, priority);
4639
4640     if (result)
4641         matching->release();
4642
4643     return (result);
4644 }
4645
4646 IONotifier * IOService::addMatchingNotification(
4647                         const OSSymbol * type, OSDictionary * matching,
4648                         IOServiceMatchingNotificationHandler handler,
4649                         void * target, void * ref,
4650                         SInt32 priority )
4651 {
4652     OSIterator *                existing = NULL;
4653     IONotifier *                ret;
4654     _IOServiceNotifier *        notify;
4655     IOService *                 next;
4656
4657     ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
4658                 handler, target, ref, priority, &existing );
4659     if (!ret) return (0);
4660
4661     // send notifications for existing set
4662     if (existing)
4663     {
4664         while( (next = (IOService *) existing->getNextObject()))
4665         {
4666             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4667             {
4668                 next->invokeNotifier( notify );
4669             }
4670         }
4671         existing->release();
4672     }
4673
4674     LOCKWRITENOTIFY();
4675     bool removed = (0 == notify->whence);
4676     notify->release();
4677     if (removed) ret = gIOServiceNullNotifier;
4678     UNLOCKNOTIFY();
4679
4680     return( ret );
4681 }
4682
4683 bool IOService::syncNotificationHandler(
4684                         void * /* target */, void * ref,
4685                         IOService * newService,
4686                         IONotifier * notifier )
4687 {
4688
4689     LOCKWRITENOTIFY();
4690     if (!*((IOService **) ref))
4691     {
4692         newService->retain();
4693         (*(IOService **) ref) = newService;
4694         WAKEUPNOTIFY(ref);
4695     }
4696     UNLOCKNOTIFY();
4697
4698     return( false );
4699 }
4700
4701 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4702                                                 uint64_t timeout)
4703 {
4704     IONotifier *        notify = 0;
4705     // priority doesn't help us much since we need a thread wakeup
4706     SInt32              priority = 0;
4707     IOService *         result;
4708
4709     if (!matching)
4710         return( 0 );
4711
4712     result = NULL;
4713
4714     LOCKWRITENOTIFY();
4715     do
4716     {
4717         result = (IOService *) copyExistingServices( matching,
4718                             kIOServiceMatchedState, kIONotifyOnce );
4719         if (result)
4720             break;
4721         notify = IOService::setNotification( gIOMatchedNotification, matching,
4722                     &IOService::syncNotificationHandler, (void *) 0,
4723                     &result, priority );
4724          if (!notify)
4725             break;
4726         if (UINT64_MAX != timeout)
4727         {
4728             AbsoluteTime deadline;
4729             nanoseconds_to_absolutetime(timeout, &deadline);
4730             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4731             SLEEPNOTIFYTO(&result, deadline);
4732         }
4733         else
4734         {
4735             SLEEPNOTIFY(&result);
4736         }
4737     }
4738     while( false );
4739
4740     UNLOCKNOTIFY();
4741
4742     if (notify)
4743         notify->remove();       // dequeues
4744
4745     return( result );
4746 }
4747
4748 IOService * IOService::waitForService( OSDictionary * matching,
4749                                         mach_timespec_t * timeout )
4750 {
4751     IOService * result;
4752     uint64_t    timeoutNS;
4753
4754     if (timeout)
4755     {
4756         timeoutNS = timeout->tv_sec;
4757         timeoutNS *= kSecondScale;
4758         timeoutNS += timeout->tv_nsec;
4759     }
4760     else
4761         timeoutNS = UINT64_MAX;
4762
4763     result = waitForMatchingService(matching, timeoutNS);
4764
4765     matching->release();
4766     if (result)
4767         result->release();
4768
4769     return (result);
4770 }
4771
4772 void IOService::deliverNotification( const OSSymbol * type,
4773                             IOOptionBits orNewState, IOOptionBits andNewState )
4774 {
4775     panic("deliverNotification");
4776 }
4777
4778 OSArray * IOService::copyNotifiers(const OSSymbol * type,
4779                                    IOOptionBits orNewState, IOOptionBits andNewState )
4780 {
4781     _IOServiceNotifier * notify;
4782     OSIterator *         iter;
4783     OSArray *            willSend = 0;
4784
4785     lockForArbitration();
4786
4787     if( (0 == (__state[0] & kIOServiceInactiveState))
4788      || (type == gIOTerminatedNotification)
4789      || (type == gIOWillTerminateNotification)) {
4790
4791         LOCKREADNOTIFY();
4792
4793         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4794                     gNotifications->getObject( type ) );
4795
4796         if( iter) {
4797             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4798
4799                 if( matchPassive(notify->matching, 0)
4800                   && (kIOServiceNotifyEnable & notify->state)) {
4801                     if( 0 == willSend)
4802                         willSend = OSArray::withCapacity(8);
4803                     if( willSend)
4804                         willSend->setObject( notify );
4805                 }
4806             }
4807             iter->release();
4808         }
4809         __state[0] = (__state[0] | orNewState) & andNewState;
4810         UNLOCKNOTIFY();
4811     }
4812
4813     unlockForArbitration();
4814
4815     return (willSend);
4816
4817 }
4818
4819 IOOptionBits IOService::getState( void ) const
4820 {
4821     return( __state[0] );
4822 }
4823
4824 /*
4825  * Helpers to make matching objects for simple cases
4826  */
4827
4828 OSDictionary * IOService::serviceMatching( const OSString * name,
4829                         OSDictionary * table )
4830 {
4831
4832     const OSString *    str;
4833
4834     str = OSSymbol::withString(name);
4835     if( !str)
4836         return( 0 );
4837
4838     if( !table)
4839         table = OSDictionary::withCapacity( 2 );
4840     if( table)
4841         table->setObject(gIOProviderClassKey, (OSObject *)str );
4842     str->release();
4843
4844     return( table );
4845 }
4846
4847 OSDictionary * IOService::serviceMatching( const char * name,
4848                         OSDictionary * table )
4849 {
4850     const OSString *    str;
4851
4852     str = OSSymbol::withCString( name );
4853     if( !str)
4854         return( 0 );
4855
4856     table = serviceMatching( str, table );
4857     str->release();
4858     return( table );
4859 }
4860
4861 OSDictionary * IOService::nameMatching( const OSString * name,
4862                         OSDictionary * table )
4863 {
4864     if( !table)
4865         table = OSDictionary::withCapacity( 2 );
4866     if( table)
4867         table->setObject( gIONameMatchKey, (OSObject *)name );
4868
4869     return( table );
4870 }
4871
4872 OSDictionary * IOService::nameMatching( const char * name,
4873                         OSDictionary * table )
4874 {
4875     const OSString *    str;
4876
4877     str = OSSymbol::withCString( name );
4878     if( !str)
4879         return( 0 );
4880
4881     table = nameMatching( str, table );
4882     str->release();
4883     return( table );
4884 }
4885
4886 OSDictionary * IOService::resourceMatching( const OSString * str,
4887                         OSDictionary * table )
4888 {
4889     table = serviceMatching( gIOResourcesKey, table );
4890     if( table)
4891         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4892
4893     return( table );
4894 }
4895
4896 OSDictionary * IOService::resourceMatching( const char * name,
4897                         OSDictionary * table )
4898 {
4899     const OSSymbol *    str;
4900
4901     str = OSSymbol::withCString( name );
4902     if( !str)
4903         return( 0 );
4904
4905     table = resourceMatching( str, table );
4906     str->release();
4907
4908     return( table );
4909 }
4910
4911 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4912                         OSDictionary * table )
4913 {
4914     OSDictionary * properties;
4915
4916     properties = OSDictionary::withCapacity( 2 );
4917     if( !properties)
4918         return( 0 );
4919     properties->setObject( key, value );
4920
4921     if( !table)
4922         table = OSDictionary::withCapacity( 2 );
4923     if( table)
4924         table->setObject( gIOPropertyMatchKey, properties );
4925
4926     properties->release();
4927
4928     return( table );
4929 }
4930
4931 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4932                         OSDictionary * table )
4933 {
4934     OSNumber *     num;
4935
4936     num = OSNumber::withNumber( entryID, 64 );
4937     if( !num)
4938         return( 0 );
4939
4940     if( !table)
4941         table = OSDictionary::withCapacity( 2 );
4942     if( table)
4943         table->setObject( gIORegistryEntryIDKey, num );
4944
4945     if (num)
4946         num->release();
4947
4948     return( table );
4949 }
4950
4951
4952 /*
4953  * _IOServiceNotifier
4954  */
4955
4956 // wait for all threads, other than the current one,
4957 //  to exit the handler
4958
4959 void _IOServiceNotifier::wait()
4960 {
4961     _IOServiceNotifierInvocation * next;
4962     bool doWait;
4963
4964     do {
4965         doWait = false;
4966         queue_iterate( &handlerInvocations, next,
4967                         _IOServiceNotifierInvocation *, link) {
4968             if( next->thread != current_thread() ) {
4969                 doWait = true;
4970                 break;
4971             }
4972         }
4973         if( doWait) {
4974             state |= kIOServiceNotifyWaiter;
4975             SLEEPNOTIFY(this);
4976         }
4977
4978     } while( doWait );
4979 }
4980
4981 void _IOServiceNotifier::free()
4982 {
4983     assert( queue_empty( &handlerInvocations ));
4984     OSObject::free();
4985 }
4986
4987 void _IOServiceNotifier::remove()
4988 {
4989     LOCKWRITENOTIFY();
4990
4991     if( whence) {
4992         whence->removeObject( (OSObject *) this );
4993         whence = 0;
4994     }
4995     if( matching) {
4996         matching->release();
4997         matching = 0;
4998     }
4999
5000     state &= ~kIOServiceNotifyEnable;
5001
5002     wait();
5003
5004     UNLOCKNOTIFY();
5005
5006     release();
5007 }
5008
5009 bool _IOServiceNotifier::disable()
5010 {
5011     bool        ret;
5012
5013     LOCKWRITENOTIFY();
5014
5015     ret = (0 != (kIOServiceNotifyEnable & state));
5016     state &= ~kIOServiceNotifyEnable;
5017     if( ret)
5018         wait();
5019
5020     UNLOCKNOTIFY();
5021
5022     return( ret );
5023 }
5024
5025 void _IOServiceNotifier::enable( bool was )
5026 {
5027     LOCKWRITENOTIFY();
5028     if( was)
5029         state |= kIOServiceNotifyEnable;
5030     else
5031         state &= ~kIOServiceNotifyEnable;
5032     UNLOCKNOTIFY();
5033 }
5034
5035
5036 /*
5037  * _IOServiceNullNotifier
5038  */
5039
5040 void _IOServiceNullNotifier::taggedRetain(const void *tag) const                      {}
5041 void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const     {}
5042 void _IOServiceNullNotifier::free()                                                   {}
5043 void _IOServiceNullNotifier::wait()                                                   {}
5044 void _IOServiceNullNotifier::remove()                                                 {}
5045 void _IOServiceNullNotifier::enable(bool was)                                         {}
5046 bool _IOServiceNullNotifier::disable()                                { return(false); }
5047
5048 /*
5049  * IOResources
5050  */
5051
5052 IOService * IOResources::resources( void )
5053 {
5054     IOResources *       inst;
5055
5056     inst = new IOResources;
5057     if( inst && !inst->init()) {
5058         inst->release();
5059         inst = 0;
5060     }
5061
5062     return( inst );
5063 }
5064
5065 bool IOResources::init( OSDictionary * dictionary )
5066 {
5067     // Do super init first
5068     if ( !IOService::init() )
5069         return false;
5070
5071     // Allow PAL layer to publish a value
5072     const char *property_name;
5073     int property_value;
5074
5075     pal_get_resource_property( &property_name, &property_value );
5076
5077     if( property_name ) {
5078         OSNumber *num;
5079         const OSSymbol *        sym;
5080
5081         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
5082             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
5083                 this->setProperty( sym, num );
5084                 sym->release();
5085             }
5086             num->release();
5087         }
5088     }
5089
5090     return true;
5091 }
5092
5093 IOReturn IOResources::newUserClient(task_t owningTask, void * securityID,
5094                                     UInt32 type,  OSDictionary * properties,
5095                                     IOUserClient ** handler)
5096 {
5097     return( kIOReturnUnsupported );
5098 }
5099
5100 IOWorkLoop * IOResources::getWorkLoop() const
5101 {
5102     // If we are the resource root
5103     // then use the platform's workloop
5104     if (this == (IOResources *) gIOResources)
5105         return getPlatform()->getWorkLoop();
5106     else
5107         return IOService::getWorkLoop();
5108 }
5109
5110 bool IOResources::matchPropertyTable( OSDictionary * table )
5111 {
5112     OSObject *          prop;
5113     OSString *          str;
5114     OSSet *             set;
5115     OSIterator *        iter;
5116     OSObject *          obj;
5117     OSArray *           keys;
5118     bool                ok = true;
5119
5120     prop = table->getObject( gIOResourceMatchKey );
5121     str = OSDynamicCast( OSString, prop );
5122     if( str)
5123         ok = (0 != getProperty( str ));
5124
5125     else if( (set = OSDynamicCast( OSSet, prop))) {
5126
5127         iter = OSCollectionIterator::withCollection( set );
5128         ok = (iter != 0);
5129         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
5130             ok = (0 != getProperty( str ));
5131
5132         if( iter)
5133             iter->release();
5134     }
5135     else if ((prop = table->getObject(gIOResourceMatchedKey)))
5136     {
5137         obj = copyProperty(gIOResourceMatchedKey);
5138         keys = OSDynamicCast(OSArray, obj);
5139         ok = false;
5140         if (keys)
5141         {
5142             // assuming OSSymbol
5143             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
5144         }
5145         OSSafeReleaseNULL(obj);
5146     }
5147
5148     return( ok );
5149 }
5150
5151 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
5152 {
5153     IOService::updateConsoleUsers(NULL, 0);
5154 }
5155
5156 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
5157 {
5158     IORegistryEntry * regEntry;
5159     OSObject *        locked = kOSBooleanFalse;
5160     uint32_t          idx;
5161     bool              publish;
5162     OSDictionary *    user;
5163     static IOMessage  sSystemPower;
5164     clock_sec_t       now = 0;
5165     clock_usec_t      microsecs;
5166
5167     regEntry = IORegistryEntry::getRegistryRoot();
5168
5169     if (!gIOChosenEntry)
5170         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
5171
5172     IOLockLock(gIOConsoleUsersLock);
5173
5174     if (systemMessage)
5175     {
5176         sSystemPower = systemMessage;
5177 #if HIBERNATION
5178         if (kIOMessageSystemHasPoweredOn == systemMessage)
5179         {
5180             uint32_t lockState = IOHibernateWasScreenLocked();
5181             switch (lockState)
5182             {
5183                 case 0:
5184                     break;
5185                 case kIOScreenLockLocked:
5186                 case kIOScreenLockFileVaultDialog:
5187                     gIOConsoleBooterLockState = kOSBooleanTrue;
5188                     break;
5189                 case kIOScreenLockNoLock:
5190                     gIOConsoleBooterLockState = 0;
5191                     break;
5192                 case kIOScreenLockUnlocked:
5193                 default:
5194                     gIOConsoleBooterLockState = kOSBooleanFalse;
5195                     break;
5196             }
5197         }
5198 #endif /* HIBERNATION */
5199     }
5200
5201     if (consoleUsers)
5202     {
5203         OSNumber * num = 0;
5204         bool       loginLocked = true;
5205
5206         gIOConsoleLoggedIn = false;
5207         for (idx = 0;
5208               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
5209               idx++)
5210         {
5211             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
5212                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
5213
5214             loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
5215             if (!num)
5216             {
5217                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
5218             }
5219         }
5220 #if HIBERNATION
5221         if (!loginLocked) gIOConsoleBooterLockState = 0;
5222         IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
5223                 (num != 0), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
5224                 gIOConsoleLoggedIn, loginLocked);
5225 #endif /* HIBERNATION */
5226         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
5227     }
5228
5229     if (!gIOConsoleLoggedIn
5230      || (kIOMessageSystemWillSleep == sSystemPower)
5231      || (kIOMessageSystemPagingOff == sSystemPower))
5232     {
5233         locked = kOSBooleanTrue;
5234     }
5235 #if HIBERNATION
5236     else if (gIOConsoleBooterLockState)
5237     {
5238         locked = gIOConsoleBooterLockState;
5239     }
5240 #endif /* HIBERNATION */
5241     else if (gIOConsoleLockTime)
5242     {
5243         clock_get_calendar_microtime(&now, &microsecs);
5244         if (gIOConsoleLockTime > now)
5245         {
5246             AbsoluteTime deadline;
5247             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
5248             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
5249         }
5250         else
5251         {
5252             locked = kOSBooleanTrue;
5253         }
5254     }
5255
5256     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
5257     if (publish)
5258     {
5259         regEntry->setProperty(gIOConsoleLockedKey, locked);
5260         if (consoleUsers)
5261         {
5262             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
5263         }
5264         OSIncrementAtomic( &gIOConsoleUsersSeed );
5265     }
5266
5267 #if HIBERNATION
5268     if (gIOChosenEntry)
5269     {
5270         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
5271         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
5272         else                          gIOScreenLockState = kIOScreenLockNoLock;
5273         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
5274
5275         IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
5276                 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != 0), now, systemMessage);
5277     }
5278 #endif /* HIBERNATION */
5279
5280     IOLockUnlock(gIOConsoleUsersLock);
5281
5282     if (publish)
5283     {
5284         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
5285
5286         MessageClientsContext context;
5287
5288         context.service  = getServiceRoot();
5289         context.type     = kIOMessageConsoleSecurityChange;
5290         context.argument = (void *) regEntry;
5291         context.argSize  = 0;
5292
5293         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
5294                                  &messageClientsApplier, &context );
5295     }
5296 }
5297
5298 IOReturn IOResources::setProperties( OSObject * properties )
5299 {
5300     IOReturn                    err;
5301     const OSSymbol *            key;
5302     OSDictionary *              dict;
5303     OSCollectionIterator *      iter;
5304
5305     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
5306     if ( kIOReturnSuccess != err)
5307         return( err );
5308
5309     dict = OSDynamicCast(OSDictionary, properties);
5310     if( 0 == dict)
5311         return( kIOReturnBadArgument);
5312
5313     iter = OSCollectionIterator::withCollection( dict);
5314     if( 0 == iter)
5315         return( kIOReturnBadArgument);
5316
5317     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
5318     {
5319         if (gIOConsoleUsersKey == key) do
5320         {
5321             OSArray * consoleUsers;
5322             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
5323             if (!consoleUsers)
5324                 continue;
5325             IOService::updateConsoleUsers(consoleUsers, 0);
5326         }
5327         while (false);
5328
5329         publishResource( key, dict->getObject(key) );
5330     }
5331
5332     iter->release();
5333
5334     return( kIOReturnSuccess );
5335 }
5336
5337 /*
5338  * Helpers for matching dictionaries.
5339  * Keys existing in matching are checked in properties.
5340  * Keys may be a string or OSCollection of IOStrings
5341  */
5342
5343 bool IOService::compareProperty( OSDictionary * matching,
5344                                  const char *   key )
5345 {
5346     OSObject *  value;
5347     OSObject *  prop;
5348     bool        ok;
5349
5350     value = matching->getObject( key );
5351     if( value)
5352     {
5353         prop = copyProperty(key);
5354         ok = value->isEqualTo(prop);
5355         if (prop) prop->release();
5356     }
5357     else
5358         ok = true;
5359
5360     return( ok );
5361 }
5362
5363
5364 bool IOService::compareProperty( OSDictionary *   matching,
5365                                  const OSString * key )
5366 {
5367     OSObject *  value;
5368     OSObject *  prop;
5369     bool        ok;
5370
5371     value = matching->getObject( key );
5372     if( value)
5373     {
5374         prop = copyProperty(key);
5375         ok = value->isEqualTo(prop);
5376         if (prop) prop->release();
5377     }
5378     else
5379         ok = true;
5380
5381     return( ok );
5382 }
5383
5384 bool IOService::compareProperties( OSDictionary * matching,
5385                                    OSCollection * keys )
5386 {
5387     OSCollectionIterator *      iter;
5388     const OSString *            key;
5389     bool                        ok = true;
5390
5391     if( !matching || !keys)
5392         return( false );
5393
5394     iter = OSCollectionIterator::withCollection( keys );
5395
5396     if( iter) {
5397         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
5398             ok = compareProperty( matching, key );
5399
5400         iter->release();
5401     }
5402     keys->release();    // !! consume a ref !!
5403
5404     return( ok );
5405 }
5406
5407 /* Helper to add a location matching dict to the table */
5408
5409 OSDictionary * IOService::addLocation( OSDictionary * table )
5410 {
5411     OSDictionary *      dict;
5412
5413     if( !table)
5414         return( 0 );
5415
5416     dict = OSDictionary::withCapacity( 1 );
5417     if( dict) {
5418         table->setObject( gIOLocationMatchKey, dict );
5419         dict->release();
5420     }
5421
5422     return( dict );
5423 }
5424
5425 /*
5426  * Go looking for a provider to match a location dict.
5427  */
5428
5429 IOService * IOService::matchLocation( IOService * /* client */ )
5430 {
5431     IOService * parent;
5432
5433     parent = getProvider();
5434
5435     if( parent)
5436         parent = parent->matchLocation( this );
5437
5438     return( parent );
5439 }
5440
5441 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
5442 {
5443     OSString *          matched;
5444     OSObject *          obj;
5445     OSString *          str;
5446     IORegistryEntry *   entry;
5447     OSNumber *          num;
5448     bool                match = true;
5449     bool                changesOK = (0 != (kIOServiceChangesOK & options));
5450     uint32_t            count;
5451     uint32_t            done;
5452
5453     do
5454     {
5455         count = table->getCount();
5456         done = 0;
5457
5458         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
5459         if (str) {
5460             done++;
5461             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
5462 #if MATCH_DEBUG
5463             match = (0 != metaCast( str ));
5464             if ((kIOServiceClassDone & options) && !match) panic("classDone");
5465 #endif
5466             if ((!match) || (done == count)) break;
5467         }
5468
5469         obj = table->getObject( gIONameMatchKey );
5470         if( obj) {
5471             done++;
5472             match = compareNames( obj, changesOK ? &matched : 0 );
5473             if (!match) break;
5474             if( changesOK && matched) {
5475                 // leave a hint as to which name matched
5476                 table->setObject( gIONameMatchedKey, matched );
5477                 matched->release();
5478             }
5479             if (done == count) break;
5480         }
5481
5482         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
5483         if (str)
5484         {
5485             const OSSymbol * sym;
5486             done++;
5487             match = false;
5488             sym = copyLocation();
5489             if (sym) {
5490                 match = sym->isEqualTo( str );
5491                 sym->release();
5492             }
5493             if ((!match) || (done == count)) break;
5494         }
5495
5496         obj = table->getObject( gIOPropertyMatchKey );
5497         if( obj)
5498         {
5499             OSDictionary * dict;
5500             OSDictionary * nextDict;
5501             OSIterator *   iter;
5502             done++;
5503             match = false;
5504             dict = dictionaryWithProperties();
5505             if( dict) {
5506                 nextDict = OSDynamicCast( OSDictionary, obj);
5507                 if( nextDict)
5508                     iter = 0;
5509                 else
5510                     iter = OSCollectionIterator::withCollection(
5511                                 OSDynamicCast(OSCollection, obj));
5512
5513                 while( nextDict
5514                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5515                                             iter->getNextObject()))))) {
5516                     match = dict->isEqualTo( nextDict, nextDict);
5517                     if( match)
5518                         break;
5519                     nextDict = 0;
5520                 }
5521                 dict->release();
5522                 if( iter)
5523                     iter->release();
5524             }
5525             if ((!match) || (done == count)) break;
5526         }
5527
5528         obj = table->getObject( gIOPropertyExistsMatchKey );
5529         if( obj)
5530         {
5531             OSDictionary * dict;
5532             OSString     * nextKey;
5533             OSIterator *   iter;
5534             done++;
5535             match = false;
5536             dict = dictionaryWithProperties();
5537             if( dict) {
5538                 nextKey = OSDynamicCast( OSString, obj);
5539                 if( nextKey)
5540                     iter = 0;
5541                 else
5542                     iter = OSCollectionIterator::withCollection(
5543                                 OSDynamicCast(OSCollection, obj));
5544
5545                 while( nextKey
5546                     || (iter && (0 != (nextKey = OSDynamicCast(OSString,
5547                                             iter->getNextObject()))))) {
5548                     match = (0 != dict->getObject(nextKey));
5549                     if( match)
5550                         break;
5551                     nextKey = 0;
5552                 }
5553                 dict->release();
5554                 if( iter)
5555                     iter->release();
5556             }
5557             if ((!match) || (done == count)) break;
5558         }
5559
5560         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5561         if( str) {
5562             done++;
5563             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5564             match = (this == entry);
5565             if( entry)
5566                 entry->release();
5567             if ((!match) || (done == count)) break;
5568         }
5569
5570         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5571         if (num) {
5572             done++;
5573             match = (getRegistryEntryID() == num->unsigned64BitValue());
5574             if ((!match) || (done == count)) break;
5575         }
5576
5577         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5578         if( num)
5579         {
5580             OSIterator *        iter;
5581             IOService *         service = 0;
5582             UInt32              serviceCount = 0;
5583
5584             done++;
5585             iter = getClientIterator();
5586             if( iter) {
5587                 while( (service = (IOService *) iter->getNextObject())) {
5588                     if( kIOServiceInactiveState & service->__state[0])
5589                         continue;
5590                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5591                         continue;
5592                     ++serviceCount;
5593                 }
5594                 iter->release();
5595             }
5596             match = (serviceCount == num->unsigned32BitValue());
5597             if ((!match) || (done == count)) break;
5598         }
5599
5600 #define propMatch(key)                                  \
5601         obj = table->getObject(key);                    \
5602         if (obj)                                        \
5603         {                                               \
5604             OSObject * prop;                            \
5605             done++;                                     \
5606             prop = copyProperty(key);                   \
5607             match = obj->isEqualTo(prop);               \
5608             if (prop) prop->release();                  \
5609             if ((!match) || (done == count)) break;     \
5610         }
5611         propMatch(gIOBSDNameKey)
5612         propMatch(gIOBSDMajorKey)
5613         propMatch(gIOBSDMinorKey)
5614         propMatch(gIOBSDUnitKey)
5615 #undef propMatch
5616     }
5617     while (false);
5618
5619     if (did) *did = done;
5620     return (match);
5621 }
5622
5623 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5624 {
5625     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5626 }
5627
5628 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5629 {
5630     IOService *         where;
5631     OSDictionary *      nextTable;
5632     SInt32              score;
5633     OSNumber *          newPri;
5634     bool                match = true;
5635     bool                matchParent = false;
5636     uint32_t            count;
5637     uint32_t            done;
5638
5639     assert( table );
5640
5641 #if !CONFIG_EMBEDDED
5642     OSArray* aliasServiceRegIds = NULL;
5643     IOService* foundAlternateService = NULL;
5644 #endif
5645
5646 #if MATCH_DEBUG
5647     OSDictionary * root = table;
5648 #endif
5649
5650     where = this;
5651     do
5652     {
5653         do
5654         {
5655             count = table->getCount();
5656             if (!(kIOServiceInternalDone & options))
5657             {
5658                 match = where->matchInternal(table, options, &done);
5659                 // don't call family if we've done all the entries in the table
5660                 if ((!match) || (done == count)) break;
5661             }
5662
5663             // pass in score from property table
5664             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5665
5666             // do family specific matching
5667             match = where->matchPropertyTable( table, &score );
5668
5669             if( !match) {
5670 #if IOMATCHDEBUG
5671                 if( kIOLogMatch & getDebugFlags( table ))
5672                     LOG("%s: family specific matching fails\n", where->getName());
5673 #endif
5674                 break;
5675             }
5676
5677             if (kIOServiceChangesOK & options) {
5678                 // save the score
5679                 newPri = OSNumber::withNumber( score, 32 );
5680                 if( newPri) {
5681                     table->setObject( gIOProbeScoreKey, newPri );
5682                     newPri->release();
5683                 }
5684             }
5685
5686             options = 0;
5687             matchParent = false;
5688
5689             nextTable = OSDynamicCast(OSDictionary,
5690                   table->getObject( gIOParentMatchKey ));
5691             if( nextTable) {
5692                 // look for a matching entry anywhere up to root
5693                 match = false;
5694                 matchParent = true;
5695                 table = nextTable;
5696                 break;
5697             }
5698
5699             table = OSDynamicCast(OSDictionary,
5700                     table->getObject( gIOLocationMatchKey ));
5701             if (table) {
5702                 // look for a matching entry at matchLocation()
5703                 match = false;
5704                 where = where->getProvider();
5705                 if (where && (where = where->matchLocation(where))) continue;
5706             }
5707             break;
5708         }
5709         while (true);
5710
5711         if(match == true) {
5712             break;
5713         }
5714
5715         if(matchParent == true) {
5716 #if !CONFIG_EMBEDDED
5717             // check if service has an alias to search its other "parents" if a parent match isn't found
5718             OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
5719             OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
5720             if(alternateRegistryID != NULL) {
5721                 if(aliasServiceRegIds == NULL)
5722                 {
5723                     aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
5724                 }
5725                 aliasServiceRegIds->setObject(alternateRegistryID);
5726             }
5727             OSSafeReleaseNULL(prop);
5728 #endif
5729         }
5730         else {
5731             break;
5732         }
5733
5734         where = where->getProvider();
5735 #if !CONFIG_EMBEDDED
5736         if(where == NULL) {
5737             // there were no matching parent services, check to see if there are aliased services that have a matching parent
5738             if(aliasServiceRegIds != NULL) {
5739                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
5740                 if(numAliasedServices != 0) {
5741                     OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
5742                     if(alternateRegistryID != NULL) {
5743                         OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
5744                         aliasServiceRegIds->removeObject(numAliasedServices - 1);
5745                         if(alternateMatchingDict != NULL) {
5746                             OSSafeReleaseNULL(foundAlternateService);
5747                             foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
5748                             alternateMatchingDict->release();
5749                             if(foundAlternateService != NULL) {
5750                                 where = foundAlternateService;
5751                             }
5752                         }
5753                     }
5754                 }
5755             }
5756         }
5757 #endif
5758     }
5759     while( where != NULL );
5760
5761 #if !CONFIG_EMBEDDED
5762     OSSafeReleaseNULL(foundAlternateService);
5763     OSSafeReleaseNULL(aliasServiceRegIds);
5764 #endif
5765
5766 #if MATCH_DEBUG
5767     if (where != this)
5768     {
5769         OSSerialize * s = OSSerialize::withCapacity(128);
5770         root->serialize(s);
5771         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5772         s->release();
5773     }
5774 #endif
5775
5776     return( match );
5777 }
5778
5779
5780 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5781                                     UInt32 type,  OSDictionary * properties,
5782                                     IOUserClient ** handler )
5783 {
5784     const OSSymbol *userClientClass = 0;
5785     IOUserClient *client;
5786     OSObject *prop;
5787     OSObject *temp;
5788
5789     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5790         return kIOReturnSuccess;
5791
5792     // First try my own properties for a user client class name
5793     prop = copyProperty(gIOUserClientClassKey);
5794     if (prop) {
5795         if (OSDynamicCast(OSSymbol, prop))
5796             userClientClass = (const OSSymbol *) prop;
5797         else if (OSDynamicCast(OSString, prop)) {
5798             userClientClass = OSSymbol::withString((OSString *) prop);
5799             if (userClientClass)
5800                 setProperty(gIOUserClientClassKey,
5801                             (OSObject *) userClientClass);
5802         }
5803     }
5804
5805     // Didn't find one so lets just bomb out now without further ado.
5806     if (!userClientClass)
5807     {
5808         OSSafeReleaseNULL(prop);
5809         return kIOReturnUnsupported;
5810     }
5811
5812     // This reference is consumed by the IOServiceOpen call
5813     temp = OSMetaClass::allocClassWithName(userClientClass);
5814     OSSafeReleaseNULL(prop);
5815     if (!temp)
5816         return kIOReturnNoMemory;
5817
5818     if (OSDynamicCast(IOUserClient, temp))
5819         client = (IOUserClient *) temp;
5820     else {
5821         temp->release();
5822         return kIOReturnUnsupported;
5823     }
5824
5825     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5826         client->release();
5827         return kIOReturnBadArgument;
5828     }
5829
5830     if ( !client->attach(this) ) {
5831         client->release();
5832         return kIOReturnUnsupported;
5833     }
5834
5835     if ( !client->start(this) ) {
5836         client->detach(this);
5837         client->release();
5838         return kIOReturnUnsupported;
5839     }
5840
5841     *handler = client;
5842     return kIOReturnSuccess;
5843 }
5844
5845 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5846                                     UInt32 type, IOUserClient ** handler )
5847 {
5848     return( kIOReturnUnsupported );
5849 }
5850
5851 IOReturn IOService::requestProbe( IOOptionBits options )
5852 {
5853     return( kIOReturnUnsupported);
5854 }
5855
5856 /*
5857  * Convert an IOReturn to text. Subclasses which add additional
5858  * IOReturn's should override this method and call
5859  * super::stringFromReturn if the desired value is not found.
5860  */
5861
5862 const char * IOService::stringFromReturn( IOReturn rtn )
5863 {
5864     static const IONamedValue IOReturn_values[] = {
5865         {kIOReturnSuccess,          "success"                           },
5866         {kIOReturnError,            "general error"                     },
5867         {kIOReturnNoMemory,         "memory allocation error"           },
5868         {kIOReturnNoResources,      "resource shortage"                 },
5869         {kIOReturnIPCError,         "Mach IPC failure"                  },
5870         {kIOReturnNoDevice,         "no such device"                    },
5871         {kIOReturnNotPrivileged,    "privilege violation"               },
5872         {kIOReturnBadArgument,      "invalid argument"                  },
5873         {kIOReturnLockedRead,       "device is read locked"             },
5874         {kIOReturnLockedWrite,      "device is write locked"            },
5875         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5876         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5877         {kIOReturnUnsupported,      "unsupported function"              },
5878         {kIOReturnVMError,          "virtual memory error"              },
5879         {kIOReturnInternalError,    "internal driver error"             },
5880         {kIOReturnIOError,          "I/O error"                         },
5881         {kIOReturnCannotLock,       "cannot acquire lock"               },
5882         {kIOReturnNotOpen,          "device is not open"                },
5883         {kIOReturnNotReadable,      "device is not readable"            },
5884         {kIOReturnNotWritable,      "device is not writeable"           },
5885         {kIOReturnNotAligned,       "alignment error"                   },
5886         {kIOReturnBadMedia,         "media error"                       },
5887         {kIOReturnStillOpen,        "device is still open"              },
5888         {kIOReturnRLDError,         "rld failure"                       },
5889         {kIOReturnDMAError,         "DMA failure"                       },
5890         {kIOReturnBusy,             "device is busy"                    },
5891         {kIOReturnTimeout,          "I/O timeout"                       },
5892         {kIOReturnOffline,          "device is offline"                 },
5893         {kIOReturnNotReady,         "device is not ready"               },
5894         {kIOReturnNotAttached,      "device/channel is not attached"    },
5895         {kIOReturnNoChannels,       "no DMA channels available"         },
5896         {kIOReturnNoSpace,          "no space for data"                 },
5897         {kIOReturnPortExists,       "device port already exists"        },
5898         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5899         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5900         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5901         {kIOReturnMessageTooLarge,  "message is too large"              },
5902         {kIOReturnNotPermitted,     "operation is not permitted"        },
5903         {kIOReturnNoPower,          "device is without power"           },
5904         {kIOReturnNoMedia,          "media is not present"              },
5905         {kIOReturnUnformattedMedia, "media is not formatted"            },
5906         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5907         {kIOReturnUnderrun,         "data underrun"                     },
5908         {kIOReturnOverrun,          "data overrun"                      },
5909         {kIOReturnDeviceError,      "device error"                      },
5910         {kIOReturnNoCompletion,     "no completion routine"             },
5911         {kIOReturnAborted,          "operation was aborted"             },
5912         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5913         {kIOReturnNotResponding,    "device is not responding"          },
5914         {kIOReturnInvalid,          "unanticipated driver error"        },
5915         {0,                         NULL                                }
5916     };
5917
5918     return IOFindNameForValue(rtn, IOReturn_values);
5919 }
5920
5921 /*
5922  * Convert an IOReturn to an errno.
5923  */
5924 int IOService::errnoFromReturn( IOReturn rtn )
5925 {
5926     if (unix_err(err_get_code(rtn)) == rtn)
5927         return err_get_code(rtn);
5928
5929     switch(rtn) {
5930         // (obvious match)
5931         case kIOReturnSuccess:
5932             return(0);
5933         case kIOReturnNoMemory:
5934             return(ENOMEM);
5935         case kIOReturnNoDevice:
5936             return(ENXIO);
5937         case kIOReturnVMError:
5938             return(EFAULT);
5939         case kIOReturnNotPermitted:
5940             return(EPERM);
5941         case kIOReturnNotPrivileged:
5942             return(EACCES);
5943         case kIOReturnIOError:
5944             return(EIO);
5945         case kIOReturnNotWritable:
5946             return(EROFS);
5947         case kIOReturnBadArgument:
5948             return(EINVAL);
5949         case kIOReturnUnsupported:
5950             return(ENOTSUP);
5951         case kIOReturnBusy:
5952             return(EBUSY);
5953         case kIOReturnNoPower:
5954             return(EPWROFF);
5955         case kIOReturnDeviceError:
5956             return(EDEVERR);
5957         case kIOReturnTimeout:
5958             return(ETIMEDOUT);
5959         case kIOReturnMessageTooLarge:
5960             return(EMSGSIZE);
5961         case kIOReturnNoSpace:
5962             return(ENOSPC);
5963         case kIOReturnCannotLock:
5964             return(ENOLCK);
5965
5966         // (best match)
5967         case kIOReturnBadMessageID:
5968         case kIOReturnNoCompletion:
5969         case kIOReturnNotAligned:
5970             return(EINVAL);
5971         case kIOReturnNotReady:
5972             return(EBUSY);
5973         case kIOReturnRLDError:
5974             return(EBADMACHO);
5975         case kIOReturnPortExists:
5976         case kIOReturnStillOpen:
5977             return(EEXIST);
5978         case kIOReturnExclusiveAccess:
5979         case kIOReturnLockedRead:
5980         case kIOReturnLockedWrite:
5981         case kIOReturnNotOpen:
5982         case kIOReturnNotReadable:
5983             return(EACCES);
5984         case kIOReturnCannotWire:
5985         case kIOReturnNoResources:
5986             return(ENOMEM);
5987         case kIOReturnAborted:
5988         case kIOReturnOffline:
5989         case kIOReturnNotResponding:
5990             return(EBUSY);
5991         case kIOReturnBadMedia:
5992         case kIOReturnNoMedia:
5993         case kIOReturnNotAttached:
5994         case kIOReturnUnformattedMedia:
5995             return(ENXIO); // (media error)
5996         case kIOReturnDMAError:
5997         case kIOReturnOverrun:
5998         case kIOReturnUnderrun:
5999             return(EIO); // (transfer error)
6000         case kIOReturnNoBandwidth:
6001         case kIOReturnNoChannels:
6002         case kIOReturnNoFrames:
6003         case kIOReturnNoInterrupt:
6004             return(EIO); // (hardware error)
6005         case kIOReturnError:
6006         case kIOReturnInternalError:
6007         case kIOReturnInvalid:
6008             return(EIO); // (generic error)
6009         case kIOReturnIPCError:
6010             return(EIO); // (ipc error)
6011         default:
6012             return(EIO); // (all other errors)
6013     }
6014 }
6015
6016 IOReturn IOService::message( UInt32 type, IOService * provider,
6017                                 void * argument )
6018 {
6019     /*
6020      * Generic entry point for calls from the provider.  A return value of
6021      * kIOReturnSuccess indicates that the message was received, and where
6022      * applicable, that it was successful.
6023      */
6024
6025     return kIOReturnUnsupported;
6026 }
6027
6028 /*
6029  * Device memory
6030  */
6031
6032 IOItemCount IOService::getDeviceMemoryCount( void )
6033 {
6034     OSArray *           array;
6035     IOItemCount         count;
6036
6037     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6038     if( array)
6039         count = array->getCount();
6040     else
6041         count = 0;
6042
6043     return( count);
6044 }
6045
6046 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
6047 {
6048     OSArray *           array;
6049     IODeviceMemory *    range;
6050
6051     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
6052     if( array)
6053         range = (IODeviceMemory *) array->getObject( index );
6054     else
6055         range = 0;
6056
6057     return( range);
6058 }
6059
6060 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
6061                                                 IOOptionBits options )
6062 {
6063     IODeviceMemory *    range;
6064     IOMemoryMap *       map;
6065
6066     range = getDeviceMemoryWithIndex( index );
6067     if( range)
6068         map = range->map( options );
6069     else
6070         map = 0;
6071
6072     return( map );
6073 }
6074
6075 OSArray * IOService::getDeviceMemory( void )
6076 {
6077     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
6078 }
6079
6080
6081 void IOService::setDeviceMemory( OSArray * array )
6082 {
6083     setProperty( gIODeviceMemoryKey, array);
6084 }
6085
6086 /*
6087  * For machines where the transfers on an I/O bus can stall because
6088  * the CPU is in an idle mode, These APIs allow a driver to specify
6089  * the maximum bus stall that they can handle.  0 indicates no limit.
6090  */
6091 void IOService::
6092 setCPUSnoopDelay(UInt32 __unused ns)
6093 {
6094 #if defined(__i386__) || defined(__x86_64__)
6095     ml_set_maxsnoop(ns);
6096 #endif /* defined(__i386__) || defined(__x86_64__) */
6097 }
6098
6099 UInt32 IOService::
6100 getCPUSnoopDelay()
6101 {
6102 #if defined(__i386__) || defined(__x86_64__)
6103     return ml_get_maxsnoop();
6104 #else
6105     return 0;
6106 #endif /* defined(__i386__) || defined(__x86_64__) */
6107 }
6108
6109 #if defined(__i386__) || defined(__x86_64__)
6110 static void
6111 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
6112 {
6113     static const UInt kNoReplace = -1U; // Must be an illegal index
6114     UInt replace = kNoReplace;
6115     bool setCpuDelay = false;
6116
6117     IORecursiveLockLock(sCpuDelayLock);
6118
6119     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6120     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6121     IOService * holder = NULL;
6122
6123     if (ns) {
6124         const CpuDelayEntry ne = {service, ns, delayType};
6125         holder = service;
6126         // Set maximum delay.
6127         for (UInt i = 0; i < count; i++) {
6128             IOService *thisService = entries[i].fService;
6129             bool sameType = (delayType == entries[i].fDelayType);
6130             if ((service == thisService) && sameType)
6131                 replace = i;
6132             else if (!thisService) {
6133                 if (kNoReplace == replace)
6134                     replace = i;
6135             }
6136             else if (sameType) {
6137                 const UInt32 thisMax = entries[i].fMaxDelay;
6138                 if (thisMax < ns)
6139                 {
6140                     ns = thisMax;
6141                     holder = thisService;
6142                 }
6143             }
6144         }
6145
6146         setCpuDelay = true;
6147         if (kNoReplace == replace)
6148             sCpuDelayData->appendBytes(&ne, sizeof(ne));
6149         else
6150             entries[replace] = ne;
6151     }
6152     else {
6153         ns = -1U;       // Set to max unsigned, i.e. no restriction
6154
6155         for (UInt i = 0; i < count; i++) {
6156             // Clear a maximum delay.
6157             IOService *thisService = entries[i].fService;
6158             if (thisService && (delayType == entries[i].fDelayType)) {
6159                 UInt32 thisMax = entries[i].fMaxDelay;
6160                 if (service == thisService)
6161                     replace = i;
6162                 else if (thisMax < ns) {
6163                     ns = thisMax;
6164                     holder = thisService;
6165                 }
6166             }
6167         }
6168
6169         // Check if entry found
6170         if (kNoReplace != replace) {
6171             entries[replace].fService = 0;      // Null the entry
6172             setCpuDelay = true;
6173         }
6174     }
6175
6176     if (setCpuDelay)
6177     {
6178         if (holder && debug_boot_arg) {
6179             strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
6180         }
6181
6182         // Must be safe to call from locked context
6183         if (delayType == kCpuDelayBusStall)
6184         {
6185             ml_set_maxbusdelay(ns);
6186         }
6187         else if (delayType == kCpuDelayInterrupt)
6188         {
6189             ml_set_maxintdelay(ns);
6190         }
6191         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
6192         sCPULatencySet   [delayType]->setValue(ns);
6193
6194         OSArray * handlers = sCpuLatencyHandlers[delayType];
6195         IOService * target;
6196         if (handlers) for (unsigned int idx = 0;
6197                             (target = (IOService *) handlers->getObject(idx));
6198                             idx++)
6199         {
6200             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6201                                             (void *) (uintptr_t) ns, holder,
6202                                             NULL, NULL);
6203         }
6204     }
6205
6206     IORecursiveLockUnlock(sCpuDelayLock);
6207 }
6208
6209 static IOReturn
6210 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
6211 {
6212     IOReturn result = kIOReturnNotFound;
6213     OSArray * array;
6214     unsigned int idx;
6215
6216     IORecursiveLockLock(sCpuDelayLock);
6217
6218     do
6219     {
6220         if (enable && !sCpuLatencyHandlers[delayType])
6221             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
6222         array = sCpuLatencyHandlers[delayType];
6223         if (!array)
6224             break;
6225         idx = array->getNextIndexOfObject(target, 0);
6226         if (!enable)
6227         {
6228             if (-1U != idx)
6229             {
6230                 array->removeObject(idx);
6231                 result = kIOReturnSuccess;
6232             }
6233         }
6234         else
6235         {
6236             if (-1U != idx) {
6237                 result = kIOReturnExclusiveAccess;
6238                 break;
6239             }
6240             array->setObject(target);
6241
6242             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6243             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6244             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
6245             IOService * holder = NULL;
6246
6247             for (UInt i = 0; i < count; i++) {
6248                 if (entries[i].fService
6249                   && (delayType == entries[i].fDelayType)
6250                   && (entries[i].fMaxDelay < ns)) {
6251                     ns = entries[i].fMaxDelay;
6252                     holder = entries[i].fService;
6253                 }
6254             }
6255             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6256                                             (void *) (uintptr_t) ns, holder,
6257                                             NULL, NULL);
6258             result = kIOReturnSuccess;
6259         }
6260     }
6261     while (false);
6262
6263     IORecursiveLockUnlock(sCpuDelayLock);
6264
6265     return (result);
6266 }
6267
6268 #endif /* defined(__i386__) || defined(__x86_64__) */
6269
6270 void IOService::
6271 requireMaxBusStall(UInt32 __unused ns)
6272 {
6273 #if defined(__i386__) || defined(__x86_64__)
6274     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
6275 #endif
6276 }
6277
6278 void IOService::
6279 requireMaxInterruptDelay(uint32_t __unused ns)
6280 {
6281 #if defined(__i386__) || defined(__x86_64__)
6282     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
6283 #endif
6284 }
6285
6286 /*
6287  * Device interrupts
6288  */
6289
6290 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
6291 {
6292   IOInterruptController *interruptController;
6293   OSArray               *array;
6294   OSData                *data;
6295   OSSymbol              *interruptControllerName;
6296   long                  numSources;
6297   IOInterruptSource     *interruptSources;
6298
6299   // Get the parents list from the nub.
6300   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
6301   if (array == 0) return kIOReturnNoResources;
6302
6303   // Allocate space for the IOInterruptSources if needed... then return early.
6304   if (nub->_interruptSources == 0) {
6305     numSources = array->getCount();
6306     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
6307     if (interruptSources == 0) return kIOReturnNoMemory;
6308
6309     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
6310
6311     nub->_numInterruptSources = numSources;
6312     nub->_interruptSources = interruptSources;
6313     return kIOReturnSuccess;
6314   }
6315
6316   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
6317   if (interruptControllerName == 0) return kIOReturnNoResources;
6318
6319   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
6320   if (interruptController == 0) return kIOReturnNoResources;
6321
6322   // Get the interrupt numbers from the nub.
6323   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
6324   if (array == 0) return kIOReturnNoResources;
6325   data = OSDynamicCast(OSData, array->getObject(source));
6326   if (data == 0) return kIOReturnNoResources;
6327
6328   // Set the interruptController and interruptSource in the nub's table.
6329   interruptSources = nub->_interruptSources;
6330   interruptSources[source].interruptController = interruptController;
6331   interruptSources[source].vectorData = data;
6332
6333   return kIOReturnSuccess;
6334 }
6335
6336 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
6337 {
6338   IOReturn ret;
6339
6340   /* Make sure the _interruptSources are set */
6341   if (_interruptSources == 0) {
6342     ret = resolveInterrupt(this, source);
6343     if (ret != kIOReturnSuccess) return ret;
6344   }
6345
6346   /* Make sure the local source number is valid */
6347   if ((source < 0) || (source >= _numInterruptSources))
6348     return kIOReturnNoInterrupt;
6349
6350   /* Look up the contoller for the local source */
6351   *interruptController = _interruptSources[source].interruptController;
6352
6353   if (*interruptController == NULL) {
6354     if (!resolve) return kIOReturnNoInterrupt;
6355
6356     /* Try to reslove the interrupt */
6357     ret = resolveInterrupt(this, source);
6358     if (ret != kIOReturnSuccess) return ret;
6359
6360     *interruptController = _interruptSources[source].interruptController;
6361   }
6362
6363   return kIOReturnSuccess;
6364 }
6365
6366 IOReturn IOService::registerInterrupt(int source, OSObject *target,
6367                                       IOInterruptAction handler,
6368                                       void *refCon)
6369 {
6370   IOInterruptController *interruptController;
6371   IOReturn              ret;
6372
6373   ret = lookupInterrupt(source, true, &interruptController);
6374   if (ret != kIOReturnSuccess) return ret;
6375
6376   /* Register the source */
6377   return interruptController->registerInterrupt(this, source, target,
6378                                                 (IOInterruptHandler)handler,
6379                                                 refCon);
6380 }
6381
6382 IOReturn IOService::unregisterInterrupt(int source)
6383 {
6384   IOInterruptController *interruptController;
6385   IOReturn              ret;
6386
6387   ret = lookupInterrupt(source, false, &interruptController);
6388   if (ret != kIOReturnSuccess) return ret;
6389
6390   /* Unregister the source */
6391   return interruptController->unregisterInterrupt(this, source);
6392 }
6393
6394 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
6395 {
6396   IOReportLegend * legend = NULL;
6397   IOInterruptAccountingData * oldValue = NULL;
6398   IOInterruptAccountingReporter * newArray = NULL;
6399   char subgroupName[64];
6400   int newArraySize = 0;
6401   int i = 0;
6402
6403   if (source < 0) {
6404     return kIOReturnBadArgument;
6405   }
6406
6407   /*
6408    * We support statistics on a maximum of 256 interrupts per nub; if a nub
6409    * has more than 256 interrupt specifiers associated with it, and tries
6410    * to register a high interrupt index with interrupt accounting, panic.
6411    * Having more than 256 interrupts associated with a single nub is
6412    * probably a sign that something fishy is going on.
6413    */
6414   if (source > IA_INDEX_MAX) {
6415     panic("addInterruptStatistics called for an excessively large index (%d)", source);
6416   }
6417
6418   /*
6419    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
6420    * leaving it as is because the likelihood of contention where we are
6421    * actually growing the array is minimal (we would realistically need
6422    * to be starting a driver for the first time, with an IOReporting
6423    * client already in place).  Nonetheless, cleanup that can be done
6424    * to adhere to best practices; it'll make the code more complicated,
6425    * unfortunately.
6426    */
6427   IOLockLock(reserved->interruptStatisticsLock);
6428
6429   /*
6430    * Lazily allocate the statistics array.
6431    */
6432   if (!reserved->interruptStatisticsArray) {
6433     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
6434     assert(reserved->interruptStatisticsArray);
6435     reserved->interruptStatisticsArrayCount = 1;
6436     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
6437   }
6438
6439   if (source >= reserved->interruptStatisticsArrayCount) {
6440     /*
6441      * We're still within the range of supported indices, but we are out
6442      * of space in the current array.  Do a nasty realloc (because
6443      * IORealloc isn't a thing) here.  We'll double the size with each
6444      * reallocation.
6445      *
6446      * Yes, the "next power of 2" could be more efficient; but this will
6447      * be invoked incredibly rarely.  Who cares.
6448      */
6449     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
6450
6451     while (newArraySize <= source)
6452       newArraySize = (newArraySize << 1);
6453     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
6454
6455     assert(newArray);
6456
6457     /*
6458      * TODO: This even zeroes the memory it is about to overwrite.
6459      * Shameful; fix it.  Not particularly high impact, however.
6460      */
6461     bzero(newArray, newArraySize * sizeof(*newArray));
6462     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
6463     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
6464     reserved->interruptStatisticsArray = newArray;
6465     reserved->interruptStatisticsArrayCount = newArraySize;
6466   }
6467
6468   if (!reserved->interruptStatisticsArray[source].reporter) {
6469     /*
6470      * We don't have a reporter associated with this index yet, so we
6471      * need to create one.
6472      */
6473     /*
6474      * TODO: Some statistics do in fact have common units (time); should this be
6475      * split into separate reporters to communicate this?
6476      */
6477      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
6478
6479     /*
6480      * Each statistic is given an identifier based on the interrupt index (which
6481      * should be unique relative to any single nub) and the statistic involved.
6482      * We should now have a sane (small and positive) index, so start
6483      * constructing the channels for statistics.
6484      */
6485     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
6486       /*
6487        * TODO: Currently, this does not add channels for disabled statistics.
6488        * Will this be confusing for clients?  If so, we should just add the
6489        * channels; we can avoid updating the channels even if they exist.
6490        */
6491       if (IA_GET_STATISTIC_ENABLED(i))
6492         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
6493     }
6494
6495     /*
6496      * We now need to add the legend for this reporter to the registry.
6497      */
6498     OSObject * prop = copyProperty(kIOReportLegendKey);
6499     legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
6500     OSSafeReleaseNULL(prop);
6501
6502     /*
6503      * Note that while we compose the subgroup name, we do not need to
6504      * manage its lifecycle (the reporter will handle this).
6505      */
6506     snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
6507     subgroupName[sizeof(subgroupName) - 1] = 0;
6508     legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
6509     setProperty(kIOReportLegendKey, legend->getLegend());
6510     legend->release();
6511
6512     /*
6513      * TODO: Is this a good idea?  Probably not; my assumption is it opts
6514      * all entities who register interrupts into public disclosure of all
6515      * IOReporting channels.  Unfortunately, this appears to be as fine
6516      * grain as it gets.
6517      */
6518     setProperty(kIOReportLegendPublicKey, true);
6519   }
6520
6521   /*
6522    * Don't stomp existing entries.  If we are about to, panic; this
6523    * probably means we failed to tear down our old interrupt source
6524    * correctly.
6525    */
6526   oldValue = reserved->interruptStatisticsArray[source].statistics;
6527
6528   if (oldValue) {
6529     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
6530   }
6531
6532   reserved->interruptStatisticsArray[source].statistics = statistics;
6533
6534   /*
6535    * Inherit the reporter values for each statistic.  The target may
6536    * be torn down as part of the runtime of the service (especially
6537    * for sleep/wake), so we inherit in order to avoid having values
6538    * reset for no apparent reason.  Our statistics are ultimately
6539    * tied to the index and the sevice, not to an individual target,
6540    * so we should maintain them accordingly.
6541    */
6542   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6543
6544   IOLockUnlock(reserved->interruptStatisticsLock);
6545
6546   return kIOReturnSuccess;
6547 }
6548
6549 IOReturn IOService::removeInterruptStatistics(int source)
6550 {
6551   IOInterruptAccountingData * value = NULL;
6552
6553   if (source < 0) {
6554     return kIOReturnBadArgument;
6555   }
6556
6557   IOLockLock(reserved->interruptStatisticsLock);
6558
6559   /*
6560    * We dynamically grow the statistics array, so an excessively
6561    * large index value has NEVER been registered.  This either
6562    * means our cap on the array size is too small (unlikely), or
6563    * that we have been passed a corrupt index (this must be passed
6564    * the plain index into the interrupt specifier list).
6565    */
6566   if (source >= reserved->interruptStatisticsArrayCount) {
6567     panic("removeInterruptStatistics called for index %d, which was never registered", source);
6568   }
6569
6570   assert(reserved->interruptStatisticsArray);
6571
6572   /*
6573    * If there is no existing entry, we are most likely trying to
6574    * free an interrupt owner twice, or we have corrupted the
6575    * index value.
6576    */
6577   value = reserved->interruptStatisticsArray[source].statistics;
6578
6579   if (!value) {
6580     panic("removeInterruptStatistics called for empty index %d", source);
6581   }
6582
6583   /*
6584    * We update the statistics, so that any delta with the reporter
6585    * state is not lost.
6586    */
6587   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6588   reserved->interruptStatisticsArray[source].statistics = NULL;
6589   IOLockUnlock(reserved->interruptStatisticsLock);
6590
6591   return kIOReturnSuccess;
6592 }
6593
6594 IOReturn IOService::getInterruptType(int source, int *interruptType)
6595 {
6596   IOInterruptController *interruptController;
6597   IOReturn              ret;
6598
6599   ret = lookupInterrupt(source, true, &interruptController);
6600   if (ret != kIOReturnSuccess) return ret;
6601
6602   /* Return the type */
6603   return interruptController->getInterruptType(this, source, interruptType);
6604 }
6605
6606 IOReturn IOService::enableInterrupt(int source)
6607 {
6608   IOInterruptController *interruptController;
6609   IOReturn              ret;
6610
6611   ret = lookupInterrupt(source, false, &interruptController);
6612   if (ret != kIOReturnSuccess) return ret;
6613
6614   /* Enable the source */
6615   return interruptController->enableInterrupt(this, source);
6616 }
6617
6618 IOReturn IOService::disableInterrupt(int source)
6619 {
6620   IOInterruptController *interruptController;
6621   IOReturn              ret;
6622
6623   ret = lookupInterrupt(source, false, &interruptController);
6624   if (ret != kIOReturnSuccess) return ret;
6625
6626   /* Disable the source */
6627   return interruptController->disableInterrupt(this, source);
6628 }
6629
6630 IOReturn IOService::causeInterrupt(int source)
6631 {
6632   IOInterruptController *interruptController;
6633   IOReturn              ret;
6634
6635   ret = lookupInterrupt(source, false, &interruptController);
6636   if (ret != kIOReturnSuccess) return ret;
6637
6638   /* Cause an interrupt for the source */
6639   return interruptController->causeInterrupt(this, source);
6640 }
6641
6642 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6643                                     IOReportConfigureAction action,
6644                                     void                   *result,
6645                                     void                   *destination)
6646 {
6647     unsigned cnt;
6648
6649     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6650         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6651             if (pwrMgt) configurePowerStatesReport(action, result);
6652             else  return kIOReturnUnsupported;
6653         }
6654         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6655             if (pwrMgt) configureSimplePowerReport(action, result);
6656             else  return kIOReturnUnsupported;
6657         }
6658     }
6659
6660     IOLockLock(reserved->interruptStatisticsLock);
6661
6662     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6663     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6664         if (reserved->interruptStatisticsArray[cnt].reporter) {
6665             /*
6666              * If the reporter is currently associated with the statistics
6667              * for an event source, we may need to update the reporter.
6668              */
6669             if (reserved->interruptStatisticsArray[cnt].statistics)
6670                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6671
6672             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6673         }
6674     }
6675
6676     IOLockUnlock(reserved->interruptStatisticsLock);
6677
6678     return kIOReturnSuccess;
6679 }
6680
6681 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6682                                  IOReportUpdateAction      action,
6683                                  void                     *result,
6684                                  void                     *destination)
6685 {
6686     unsigned cnt;
6687
6688     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6689         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6690             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6691             else return kIOReturnUnsupported;
6692         }
6693         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6694             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6695             else return kIOReturnUnsupported;
6696         }
6697     }
6698
6699     IOLockLock(reserved->interruptStatisticsLock);
6700
6701     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6702     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6703         if (reserved->interruptStatisticsArray[cnt].reporter) {
6704             /*
6705              * If the reporter is currently associated with the statistics
6706              * for an event source, we need to update the reporter.
6707              */
6708             if (reserved->interruptStatisticsArray[cnt].statistics)
6709                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6710
6711             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6712         }
6713     }
6714
6715     IOLockUnlock(reserved->interruptStatisticsLock);
6716
6717     return kIOReturnSuccess;
6718 }
6719
6720 uint64_t IOService::getAuthorizationID( void )
6721 {
6722     return reserved->authorizationID;
6723 }
6724
6725 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6726 {
6727     OSObject * entitlement;
6728     IOReturn status;
6729
6730     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6731
6732     if ( entitlement )
6733     {
6734         if ( entitlement == kOSBooleanTrue )
6735         {
6736             reserved->authorizationID = authorizationID;
6737
6738             status = kIOReturnSuccess;
6739         }
6740         else
6741         {
6742             status = kIOReturnNotPrivileged;
6743         }
6744
6745         entitlement->release( );
6746     }
6747     else
6748     {
6749         status = kIOReturnNotPrivileged;
6750     }
6751
6752     return status;
6753 }
6754
6755 #if __LP64__
6756 OSMetaClassDefineReservedUsed(IOService, 0);
6757 OSMetaClassDefineReservedUsed(IOService, 1);
6758 OSMetaClassDefineReservedUnused(IOService, 2);
6759 OSMetaClassDefineReservedUnused(IOService, 3);
6760 OSMetaClassDefineReservedUnused(IOService, 4);
6761 OSMetaClassDefineReservedUnused(IOService, 5);
6762 OSMetaClassDefineReservedUnused(IOService, 6);
6763 OSMetaClassDefineReservedUnused(IOService, 7);
6764 #else
6765 OSMetaClassDefineReservedUsed(IOService, 0);
6766 OSMetaClassDefineReservedUsed(IOService, 1);
6767 OSMetaClassDefineReservedUsed(IOService, 2);
6768 OSMetaClassDefineReservedUsed(IOService, 3);
6769 OSMetaClassDefineReservedUsed(IOService, 4);
6770 OSMetaClassDefineReservedUsed(IOService, 5);
6771 OSMetaClassDefineReservedUsed(IOService, 6);
6772 OSMetaClassDefineReservedUsed(IOService, 7);
6773 #endif
6774 OSMetaClassDefineReservedUnused(IOService, 8);
6775 OSMetaClassDefineReservedUnused(IOService, 9);
6776 OSMetaClassDefineReservedUnused(IOService, 10);
6777 OSMetaClassDefineReservedUnused(IOService, 11);
6778 OSMetaClassDefineReservedUnused(IOService, 12);
6779 OSMetaClassDefineReservedUnused(IOService, 13);
6780 OSMetaClassDefineReservedUnused(IOService, 14);
6781 OSMetaClassDefineReservedUnused(IOService, 15);
6782 OSMetaClassDefineReservedUnused(IOService, 16);
6783 OSMetaClassDefineReservedUnused(IOService, 17);
6784 OSMetaClassDefineReservedUnused(IOService, 18);
6785 OSMetaClassDefineReservedUnused(IOService, 19);
6786 OSMetaClassDefineReservedUnused(IOService, 20);
6787 OSMetaClassDefineReservedUnused(IOService, 21);
6788 OSMetaClassDefineReservedUnused(IOService, 22);
6789 OSMetaClassDefineReservedUnused(IOService, 23);
6790 OSMetaClassDefineReservedUnused(IOService, 24);
6791 OSMetaClassDefineReservedUnused(IOService, 25);
6792 OSMetaClassDefineReservedUnused(IOService, 26);
6793 OSMetaClassDefineReservedUnused(IOService, 27);
6794 OSMetaClassDefineReservedUnused(IOService, 28);
6795 OSMetaClassDefineReservedUnused(IOService, 29);
6796 OSMetaClassDefineReservedUnused(IOService, 30);
6797 OSMetaClassDefineReservedUnused(IOService, 31);
6798 OSMetaClassDefineReservedUnused(IOService, 32);
6799 OSMetaClassDefineReservedUnused(IOService, 33);
6800 OSMetaClassDefineReservedUnused(IOService, 34);
6801 OSMetaClassDefineReservedUnused(IOService, 35);
6802 OSMetaClassDefineReservedUnused(IOService, 36);
6803 OSMetaClassDefineReservedUnused(IOService, 37);
6804 OSMetaClassDefineReservedUnused(IOService, 38);
6805 OSMetaClassDefineReservedUnused(IOService, 39);
6806 OSMetaClassDefineReservedUnused(IOService, 40);
6807 OSMetaClassDefineReservedUnused(IOService, 41);
6808 OSMetaClassDefineReservedUnused(IOService, 42);
6809 OSMetaClassDefineReservedUnused(IOService, 43);
6810 OSMetaClassDefineReservedUnused(IOService, 44);
6811 OSMetaClassDefineReservedUnused(IOService, 45);
6812 OSMetaClassDefineReservedUnused(IOService, 46);
6813 OSMetaClassDefineReservedUnused(IOService, 47);