iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <IOKit/AppleKeyStoreInterface.h>
  54 #include <IOKit/IOCPU.h>
  55 #include <mach/sync_policy.h>
  56 #include <IOKit/assert.h>
  57 #include <sys/errno.h>
  58 #include <sys/kdebug.h>
  59 #include <string.h>
  60
  61 #include <machine/pal_routines.h>
  62
  63 #define LOG kprintf
  64 //#define LOG IOLog
  65 #define MATCH_DEBUG     0
  66 #define OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  67
  68 #include "IOServicePrivate.h"
  69 #include "IOKitKernelInternal.h"
  70
  71 // take lockForArbitration before LOCKNOTIFY
  72
  73 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  74
  75 #define super IORegistryEntry
  76
  77 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  78
  79 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  80 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  81
  82 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  83
  84 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  85
  86 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  87
  88 OSDefineMetaClassAndStructors(IOResources, IOService)
  89
  90 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  91
  92 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  93
  94 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  95
  96 static IOPlatformExpert *       gIOPlatform;
  97 static class IOPMrootDomain *   gIOPMRootDomain;
  98 const IORegistryPlane *         gIOServicePlane;
  99 const IORegistryPlane *         gIOPowerPlane;
 100 const OSSymbol *                gIODeviceMemoryKey;
 101 const OSSymbol *                gIOInterruptControllersKey;
 102 const OSSymbol *                gIOInterruptSpecifiersKey;
 103
 104 const OSSymbol *                gIOResourcesKey;
 105 const OSSymbol *                gIOResourceMatchKey;
 106 const OSSymbol *                gIOResourceMatchedKey;
 107 const OSSymbol *                gIOProviderClassKey;
 108 const OSSymbol *                gIONameMatchKey;
 109 const OSSymbol *                gIONameMatchedKey;
 110 const OSSymbol *                gIOPropertyMatchKey;
 111 const OSSymbol *                gIOPropertyExistsMatchKey;
 112 const OSSymbol *                gIOLocationMatchKey;
 113 const OSSymbol *                gIOParentMatchKey;
 114 const OSSymbol *                gIOPathMatchKey;
 115 const OSSymbol *                gIOMatchCategoryKey;
 116 const OSSymbol *                gIODefaultMatchCategoryKey;
 117 const OSSymbol *                gIOMatchedServiceCountKey;
 118
 119 const OSSymbol *                gIOMapperIDKey;
 120 const OSSymbol *                gIOUserClientClassKey;
 121 const OSSymbol *                gIOKitDebugKey;
 122
 123 const OSSymbol *                gIOCommandPoolSizeKey;
 124
 125 const OSSymbol *                gIOConsoleLockedKey;
 126 const OSSymbol *                gIOConsoleUsersKey;
 127 const OSSymbol *                gIOConsoleSessionUIDKey;
 128 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 129 const OSSymbol *                gIOConsoleUsersSeedKey;
 130 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 131 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 132 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 133 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 134 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 135 clock_sec_t                     gIOConsoleLockTime;
 136 static bool                     gIOConsoleLoggedIn;
 137 #if HIBERNATION
 138 static OSBoolean *              gIOConsoleBooterLockState;
 139 static uint32_t                 gIOScreenLockState;
 140 #endif
 141 static IORegistryEntry *        gIOChosenEntry;
 142
 143 static int                      gIOResourceGenerationCount;
 144
 145 const OSSymbol *                gIOServiceKey;
 146 const OSSymbol *                gIOPublishNotification;
 147 const OSSymbol *                gIOFirstPublishNotification;
 148 const OSSymbol *                gIOMatchedNotification;
 149 const OSSymbol *                gIOFirstMatchNotification;
 150 const OSSymbol *                gIOTerminatedNotification;
 151
 152 const OSSymbol *                gIOGeneralInterest;
 153 const OSSymbol *                gIOBusyInterest;
 154 const OSSymbol *                gIOAppPowerStateInterest;
 155 const OSSymbol *                gIOPriorityPowerStateInterest;
 156 const OSSymbol *                gIOConsoleSecurityInterest;
 157
 158 const OSSymbol *                gIOBSDKey;
 159 const OSSymbol *                gIOBSDNameKey;
 160 const OSSymbol *                gIOBSDMajorKey;
 161 const OSSymbol *                gIOBSDMinorKey;
 162 const OSSymbol *                gIOBSDUnitKey;
 163
 164 const  OSSymbol *               gAKSGetKey;
 165 #if defined(__i386__) || defined(__x86_64__)
 166 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 167 #endif
 168
 169 static OSDictionary *           gNotifications;
 170 static IORecursiveLock *        gNotificationLock;
 171
 172 static IOService *              gIOResources;
 173 static IOService *              gIOServiceRoot;
 174
 175 static OSOrderedSet *           gJobs;
 176 static semaphore_port_t         gJobsSemaphore;
 177 static IOLock *                 gJobsLock;
 178 static int                      gOutstandingJobs;
 179 static int                      gNumConfigThreads;
 180 static int                      gNumWaitingThreads;
 181 static IOLock *                 gIOServiceBusyLock;
 182 static bool             gCPUsRunning;
 183
 184 static thread_t                 gIOTerminateThread;
 185 static UInt32                   gIOTerminateWork;
 186 static OSArray *                gIOTerminatePhase2List;
 187 static OSArray *                gIOStopList;
 188 static OSArray *                gIOStopProviderList;
 189 static OSArray *                gIOFinalizeList;
 190
 191 static SInt32                   gIOConsoleUsersSeed;
 192 static OSData *                 gIOConsoleUsersSeedValue;
 193
 194 extern const OSSymbol *         gIODTPHandleKey;
 195
 196 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 197
 198 static IOLock *                 gIOConsoleUsersLock;
 199 static thread_call_t            gIOConsoleLockCallout;
 200 static IONotifier *             gIOServiceNullNotifier;
 201
 202 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 203
 204 #define LOCKREADNOTIFY()        \
 205     IORecursiveLockLock( gNotificationLock )
 206 #define LOCKWRITENOTIFY()       \
 207     IORecursiveLockLock( gNotificationLock )
 208 #define LOCKWRITE2READNOTIFY()
 209 #define UNLOCKNOTIFY()          \
 210     IORecursiveLockUnlock( gNotificationLock )
 211 #define SLEEPNOTIFY(event) \
 212     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 213 #define SLEEPNOTIFYTO(event, deadline) \
 214     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 215 #define WAKEUPNOTIFY(event) \
 216         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 217
 218 #define randomDelay()   \
 219         int del = read_processor_clock();                               \
 220         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 221         IOSleep( del );
 222
 223 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 224
 225 #define queue_element(entry, element, type, field) do { \
 226         vm_address_t __ele = (vm_address_t) (entry);    \
 227         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 228         (element) = (type) __ele;                       \
 229     } while(0)
 230
 231 #define iterqueue(que, elt)                             \
 232         for (queue_entry_t elt = queue_first(que);      \
 233              !queue_end(que, elt);                      \
 234              elt = queue_next(elt))
 235
 236 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 237
 238 struct IOInterruptAccountingReporter {
 239     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 240     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 241 };
 242
 243 struct ArbitrationLockQueueElement {
 244     queue_chain_t link;
 245     IOThread      thread;
 246     IOService *   service;
 247     unsigned      count;
 248     bool          required;
 249     bool          aborted;
 250 };
 251
 252 static queue_head_t gArbitrationLockQueueActive;
 253 static queue_head_t gArbitrationLockQueueWaiting;
 254 static queue_head_t gArbitrationLockQueueFree;
 255 static IOLock *     gArbitrationLockQueueLock;
 256
 257 bool IOService::isInactive( void ) const
 258     { return( 0 != (kIOServiceInactiveState & getState())); }
 259
 260 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 261
 262 #if defined(__i386__) || defined(__x86_64__)
 263
 264 // Only used by the intel implementation of
 265 //     IOService::requireMaxBusStall(UInt32 ns)
 266 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 267 struct CpuDelayEntry
 268 {
 269     IOService * fService;
 270     UInt32      fMaxDelay;
 271     UInt32      fDelayType;
 272 };
 273
 274 enum {
 275     kCpuDelayBusStall, kCpuDelayInterrupt,
 276     kCpuNumDelayTypes
 277 };
 278
 279 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 280 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 281 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 282 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 283 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 284 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 285 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 286
 287 static void
 288 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 289 static IOReturn
 290 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 291
 292 #endif /* defined(__i386__) || defined(__x86_64__) */
 293
 294 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 295
 296 void IOService::initialize( void )
 297 {
 298     kern_return_t       err;
 299
 300     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 301     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 302
 303     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 304     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 305     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 306     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 307     gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 308     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 309     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 310     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 311
 312     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 313     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 314                                         kIODefaultMatchCategoryKey );
 315     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 316                                         kIOMatchedServiceCountKey );
 317
 318     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 319
 320     gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 321     gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 322     gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 323
 324     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 325     gIOInterruptControllersKey
 326         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 327     gIOInterruptSpecifiersKey
 328         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 329
 330     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 331
 332     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 333
 334     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 335
 336     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 337     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 338     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 339     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 340     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 341
 342     gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 343     gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 344     gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 345     gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 346     gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 347
 348     gNotifications              = OSDictionary::withCapacity( 1 );
 349     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 350                                                  kIOPublishNotification );
 351     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 352                                                  kIOFirstPublishNotification );
 353     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 354                                                  kIOMatchedNotification );
 355     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 356                                                  kIOFirstMatchNotification );
 357     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 358                                                  kIOTerminatedNotification );
 359     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 360
 361     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 362     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 363     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 364     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 365
 366     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 367     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 368     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 369     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 370     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 371     gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 372
 373     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 374
 375     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 376 #if defined(__i386__) || defined(__x86_64__)
 377     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 378     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 379     uint32_t  idx;
 380     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 381     {
 382         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 383         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 384         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 385     }
 386     gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 387 #endif
 388     gNotificationLock           = IORecursiveLockAlloc();
 389
 390     gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 391
 392     assert( gIOServicePlane && gIODeviceMemoryKey
 393         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 394         && gIOResourcesKey && gNotifications && gNotificationLock
 395         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 396         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 397         && gIOPublishNotification && gIOMatchedNotification
 398         && gIOTerminatedNotification && gIOServiceKey
 399         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 400     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 401         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 402
 403     gJobsLock   = IOLockAlloc();
 404     gJobs       = OSOrderedSet::withCapacity( 10 );
 405
 406     gIOServiceBusyLock = IOLockAlloc();
 407
 408     gIOConsoleUsersLock = IOLockAlloc();
 409
 410     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 411
 412     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 413
 414     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 415
 416     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 417                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 418
 419     gIOResources = IOResources::resources();
 420     assert( gIOResources );
 421
 422     gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 423     assert(gIOServiceNullNotifier);
 424
 425     gArbitrationLockQueueLock = IOLockAlloc();
 426     queue_init(&gArbitrationLockQueueActive);
 427     queue_init(&gArbitrationLockQueueWaiting);
 428     queue_init(&gArbitrationLockQueueFree);
 429
 430     assert( gArbitrationLockQueueLock );
 431
 432     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 433     gIOStopList            = OSArray::withCapacity( 16 );
 434     gIOStopProviderList    = OSArray::withCapacity( 16 );
 435     gIOFinalizeList        = OSArray::withCapacity( 16 );
 436     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 437 }
 438
 439 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 440
 441 #if defined(__i386__) || defined(__x86_64__)
 442 extern "C" {
 443
 444 const char *getCpuDelayBusStallHolderName(void);
 445 const char *getCpuDelayBusStallHolderName(void) {
 446     return sCPULatencyHolderName[kCpuDelayBusStall];
 447 }
 448
 449 }
 450 #endif
 451
 452 #if IOMATCHDEBUG
 453 static UInt64 getDebugFlags( OSDictionary * props )
 454 {
 455     OSNumber *  debugProp;
 456     UInt64      debugFlags;
 457
 458     debugProp = OSDynamicCast( OSNumber,
 459                 props->getObject( gIOKitDebugKey ));
 460     if( debugProp)
 461         debugFlags = debugProp->unsigned64BitValue();
 462     else
 463         debugFlags = gIOKitDebug;
 464
 465     return( debugFlags );
 466 }
 467 #endif
 468
 469 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 470
 471 // Probe a matched service and return an instance to be started.
 472 // The default score is from the property table, & may be altered
 473 // during probe to change the start order.
 474
 475 IOService * IOService::probe(   IOService * provider,
 476                                 SInt32    * score )
 477 {
 478     return( this );
 479 }
 480
 481 bool IOService::start( IOService * provider )
 482 {
 483     return( true );
 484 }
 485
 486 void IOService::stop( IOService * provider )
 487 {
 488 }
 489
 490 bool IOService::init( OSDictionary * dictionary )
 491 {
 492     bool ret;
 493
 494     ret = super::init(dictionary);
 495     if (!ret)     return (false);
 496     if (reserved) return (true);
 497
 498     reserved = IONew(ExpansionData, 1);
 499     if (!reserved) return (false);
 500     bzero(reserved, sizeof(*reserved));
 501
 502     /*
 503      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 504      * lock based on the presence of specifiers ran into issues as some
 505      * platforms set up the specifiers after IOService initialization.
 506      *
 507      * We may be able to get away with a global lock, as this should only be
 508      * contended by IOReporting clients and driver start/stop (unless a
 509      * driver wants to remove/add handlers in the course of normal operation,
 510      * which should be unlikely).
 511      */
 512     reserved->interruptStatisticsLock = IOLockAlloc();
 513     if (!reserved->interruptStatisticsLock) return (false);
 514
 515     return (true);
 516 }
 517
 518 bool IOService::init( IORegistryEntry * from,
 519                       const IORegistryPlane * inPlane )
 520 {
 521     bool ret;
 522
 523     ret = super::init(from, inPlane);
 524     if (!ret)     return (false);
 525     if (reserved) return (true);
 526
 527     reserved = IONew(ExpansionData, 1);
 528     if (!reserved) return (false);
 529     bzero(reserved, sizeof(*reserved));
 530
 531     /*
 532      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 533      * lock based on the presence of specifiers ran into issues as some
 534      * platforms set up the specifiers after IOService initialization.
 535      *
 536      * We may be able to get away with a global lock, as this should only be
 537      * contended by IOReporting clients and driver start/stop (unless a
 538      * driver wants to remove/add handlers in the course of normal operation,
 539      * which should be unlikely).
 540      */
 541     reserved->interruptStatisticsLock = IOLockAlloc();
 542     if (!reserved->interruptStatisticsLock) return (false);
 543
 544     return (true);
 545 }
 546
 547 void IOService::free( void )
 548 {
 549     int i = 0;
 550     requireMaxBusStall(0);
 551     requireMaxInterruptDelay(0);
 552     if( getPropertyTable())
 553         unregisterAllInterest();
 554     PMfree();
 555
 556     if (reserved) {
 557         if (reserved->interruptStatisticsArray) {
 558             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 559                 if (reserved->interruptStatisticsArray[i].reporter)
 560                     reserved->interruptStatisticsArray[i].reporter->release();
 561             }
 562
 563             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 564         }
 565
 566         if (reserved->interruptStatisticsLock)
 567             IOLockFree(reserved->interruptStatisticsLock);
 568         IODelete(reserved, ExpansionData, 1);
 569     }
 570
 571     if (_numInterruptSources && _interruptSources)
 572     {
 573         IOFree(_interruptSources, _numInterruptSources * sizeof(IOInterruptSource));
 574         _interruptSources = 0;
 575     }
 576
 577     super::free();
 578 }
 579
 580 /*
 581  * Attach in service plane
 582  */
 583 bool IOService::attach( IOService * provider )
 584 {
 585     bool         ok;
 586     uint32_t     count;
 587     AbsoluteTime deadline;
 588     int          waitResult = THREAD_AWAKENED;
 589     bool         wait, computeDeadline = true;
 590
 591     if( provider) {
 592
 593         if( gIOKitDebug & kIOLogAttach)
 594             LOG( "%s::attach(%s)\n", getName(),
 595                     provider->getName());
 596
 597         ok   = false;
 598         do
 599         {
 600             wait = false;
 601             provider->lockForArbitration();
 602             if (provider->__state[0] & kIOServiceInactiveState) ok = false;
 603             else
 604             {
 605                 count = provider->getChildCount(gIOServicePlane);
 606                 wait = (count > (kIOServiceBusyMax - 4));
 607                 if (!wait) ok = attachToParent(provider, gIOServicePlane);
 608                 else
 609                 {
 610                     IOLog("stalling for detach from %s\n", provider->getName());
 611                     IOLockLock( gIOServiceBusyLock );
 612                     provider->__state[1] |= kIOServiceWaitDetachState;
 613                 }
 614             }
 615             provider->unlockForArbitration();
 616             if (wait)
 617             {
 618                 if (computeDeadline)
 619                 {
 620                     clock_interval_to_deadline(15, kSecondScale, &deadline);
 621                     computeDeadline = false;
 622                 }
 623                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 624                 IOLockUnlock( gIOServiceBusyLock );
 625                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 626                 wait = (waitResult != THREAD_TIMED_OUT);
 627             }
 628         }
 629         while (wait);
 630
 631     } else {
 632         gIOServiceRoot = this;
 633         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 634     }
 635
 636     if (ok && !__provider) (void) getProvider();
 637
 638     return( ok );
 639 }
 640
 641 IOService * IOService::getServiceRoot( void )
 642 {
 643     return( gIOServiceRoot );
 644 }
 645
 646 void IOService::detach( IOService * provider )
 647 {
 648     IOService * newProvider = 0;
 649     SInt32      busy;
 650     bool        adjParent;
 651
 652     if( gIOKitDebug & kIOLogAttach)
 653         LOG("%s::detach(%s)\n", getName(), provider->getName());
 654
 655     lockForArbitration();
 656
 657     uint64_t regID1 = provider->getRegistryEntryID();
 658     uint64_t regID2 = getRegistryEntryID();
 659     IOServiceTrace(
 660         IOSERVICE_DETACH,
 661         (uintptr_t) regID1,
 662         (uintptr_t) (regID1 >> 32),
 663         (uintptr_t) regID2,
 664         (uintptr_t) (regID2 >> 32));
 665
 666     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 667                && (provider == getProvider()));
 668
 669     detachFromParent( provider, gIOServicePlane );
 670
 671     if( busy) {
 672         newProvider = getProvider();
 673         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 674             _adjustBusy( -busy );
 675     }
 676
 677     if (kIOServiceInactiveState & __state[0]) {
 678         getMetaClass()->removeInstance(this);
 679         IORemoveServicePlatformActions(this);
 680     }
 681
 682     unlockForArbitration();
 683
 684     if( newProvider) {
 685         newProvider->lockForArbitration();
 686         newProvider->_adjustBusy(1);
 687         newProvider->unlockForArbitration();
 688     }
 689
 690     // check for last client detach from a terminated service
 691     if( provider->lockForArbitration( true ))
 692     {
 693         if (kIOServiceStartState & __state[1])
 694         {
 695             provider->scheduleTerminatePhase2();
 696         }
 697         if( adjParent) provider->_adjustBusy( -1 );
 698         if( (provider->__state[1] & kIOServiceTermPhase3State)
 699          && (0 == provider->getClient())) {
 700             provider->scheduleFinalize(false);
 701         }
 702
 703         IOLockLock( gIOServiceBusyLock );
 704         if (kIOServiceWaitDetachState & provider->__state[1])
 705         {
 706             provider->__state[1] &= ~kIOServiceWaitDetachState;
 707             thread_wakeup(&provider->__provider);
 708         }
 709         IOLockUnlock( gIOServiceBusyLock );
 710
 711         provider->unlockForArbitration();
 712     }
 713 }
 714
 715 /*
 716  * Register instance - publish it for matching
 717  */
 718
 719 void IOService::registerService( IOOptionBits options )
 720 {
 721     char *              pathBuf;
 722     const char *        path;
 723     char *              skip;
 724     int                 len;
 725     enum { kMaxPathLen  = 256 };
 726     enum { kMaxChars    = 63 };
 727
 728     IORegistryEntry * parent = this;
 729     IORegistryEntry * root = getRegistryRoot();
 730     while( parent && (parent != root))
 731         parent = parent->getParentEntry( gIOServicePlane);
 732
 733     if( parent != root) {
 734         IOLog("%s: not registry member at registerService()\n", getName());
 735         return;
 736     }
 737
 738     // Allow the Platform Expert to adjust this node.
 739     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 740         return;
 741
 742     IOInstallServicePlatformActions(this);
 743
 744     if( (this != gIOResources)
 745      && (kIOLogRegister & gIOKitDebug)) {
 746
 747         pathBuf = (char *) IOMalloc( kMaxPathLen );
 748
 749         IOLog( "Registering: " );
 750
 751         len = kMaxPathLen;
 752         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 753
 754             path = pathBuf;
 755             if( len > kMaxChars) {
 756                 IOLog("..");
 757                 len -= kMaxChars;
 758                 path += len;
 759                 if( (skip = strchr( path, '/')))
 760                     path = skip;
 761             }
 762         } else
 763             path = getName();
 764
 765         IOLog( "%s\n", path );
 766
 767         if( pathBuf)
 768             IOFree( pathBuf, kMaxPathLen );
 769     }
 770
 771     startMatching( options );
 772 }
 773
 774 void IOService::startMatching( IOOptionBits options )
 775 {
 776     IOService * provider;
 777     UInt32      prevBusy = 0;
 778     bool        needConfig;
 779     bool        needWake = false;
 780     bool        ok;
 781     bool        sync;
 782     bool        waitAgain;
 783
 784     lockForArbitration();
 785
 786     sync = (options & kIOServiceSynchronous)
 787         || ((provider = getProvider())
 788                 && (provider->__state[1] & kIOServiceSynchronousState));
 789
 790         if ( options & kIOServiceAsynchronous )
 791                 sync = false;
 792
 793     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState)))
 794                && (0 == (__state[0] & kIOServiceInactiveState));
 795
 796     __state[1] |= kIOServiceNeedConfigState;
 797
 798 //    __state[0] &= ~kIOServiceInactiveState;
 799
 800 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 801 //                      OSKernelStackRemaining(), getName());
 802
 803     if( needConfig) {
 804         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 805     }
 806
 807     if( sync)
 808         __state[1] |= kIOServiceSynchronousState;
 809     else
 810         __state[1] &= ~kIOServiceSynchronousState;
 811
 812     if( needConfig) prevBusy = _adjustBusy( 1 );
 813
 814     unlockForArbitration();
 815
 816     if( needConfig) {
 817
 818         if( needWake) {
 819             IOLockLock( gIOServiceBusyLock );
 820             thread_wakeup( (event_t) this/*&__state[1]*/ );
 821             IOLockUnlock( gIOServiceBusyLock );
 822
 823         } else if( !sync || (kIOServiceAsynchronous & options)) {
 824
 825             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 826
 827         } else do {
 828
 829             if( (__state[1] & kIOServiceNeedConfigState))
 830                 doServiceMatch( options );
 831
 832             lockForArbitration();
 833             IOLockLock( gIOServiceBusyLock );
 834
 835             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 836                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 837
 838             if( waitAgain)
 839                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 840             else
 841                 __state[1] &= ~kIOServiceSyncPubState;
 842
 843             unlockForArbitration();
 844
 845             if( waitAgain)
 846                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 847
 848             IOLockUnlock( gIOServiceBusyLock );
 849             if( waitAgain)
 850                 thread_block(THREAD_CONTINUE_NULL);
 851
 852         } while( waitAgain );
 853     }
 854 }
 855
 856 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 857 {
 858     OSDictionary *      table;
 859     OSSet *             set;
 860     OSSet *             allSet = 0;
 861     IOService *         service;
 862 #if IOMATCHDEBUG
 863     SInt32              count = 0;
 864 #endif
 865
 866     newTables->retain();
 867
 868     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 869
 870         LOCKWRITENOTIFY();
 871         set = (OSSet *) copyExistingServices( table,
 872                                                 kIOServiceRegisteredState,
 873                                                 kIOServiceExistingSet);
 874         UNLOCKNOTIFY();
 875         if( set) {
 876
 877 #if IOMATCHDEBUG
 878             count += set->getCount();
 879 #endif
 880             if (allSet) {
 881                 allSet->merge((const OSSet *) set);
 882                 set->release();
 883             }
 884             else
 885                 allSet = set;
 886         }
 887
 888 #if IOMATCHDEBUG
 889         if( getDebugFlags( table ) & kIOLogMatch)
 890             LOG("Matching service count = %ld\n", (long)count);
 891 #endif
 892         newTables->removeObject(table);
 893     }
 894
 895     if (allSet) {
 896         while( (service = (IOService *) allSet->getAnyObject())) {
 897             service->startMatching(kIOServiceAsynchronous);
 898             allSet->removeObject(service);
 899         }
 900         allSet->release();
 901     }
 902
 903     newTables->release();
 904
 905     return( kIOReturnSuccess );
 906 }
 907
 908  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 909                                                 IOOptionBits options )
 910 {
 911     _IOServiceJob *     job;
 912
 913     job = new _IOServiceJob;
 914     if( job && !job->init()) {
 915         job->release();
 916         job = 0;
 917     }
 918
 919     if( job) {
 920         job->type       = type;
 921         job->nub        = nub;
 922         job->options    = options;
 923         nub->retain();                  // thread will release()
 924         pingConfig( job );
 925     }
 926
 927     return( job );
 928 }
 929
 930 /*
 931  * Called on a registered service to see if it matches
 932  * a property table.
 933  */
 934
 935 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 936 {
 937     return( matchPropertyTable(table) );
 938 }
 939
 940 bool IOService::matchPropertyTable( OSDictionary * table )
 941 {
 942     return( true );
 943 }
 944
 945 /*
 946  * Called on a matched service to allocate resources
 947  * before first driver is attached.
 948  */
 949
 950 IOReturn IOService::getResources( void )
 951 {
 952     return( kIOReturnSuccess);
 953 }
 954
 955 /*
 956  * Client/provider accessors
 957  */
 958
 959 IOService * IOService::getProvider( void ) const
 960 {
 961     IOService * self = (IOService *) this;
 962     IOService * parent;
 963     SInt32      generation;
 964
 965     generation = getRegistryEntryGenerationCount();
 966     if( __providerGeneration == generation)
 967         return( __provider );
 968
 969     parent = (IOService *) getParentEntry( gIOServicePlane);
 970     if( parent == IORegistryEntry::getRegistryRoot())
 971         /* root is not an IOService */
 972         parent = 0;
 973
 974     self->__provider = parent;
 975     OSMemoryBarrier();
 976     // save the count from before call to getParentEntry()
 977     self->__providerGeneration = generation;
 978
 979     return( parent );
 980 }
 981
 982 IOWorkLoop * IOService::getWorkLoop() const
 983 {
 984     IOService *provider = getProvider();
 985
 986     if (provider)
 987         return provider->getWorkLoop();
 988     else
 989         return 0;
 990 }
 991
 992 OSIterator * IOService::getProviderIterator( void ) const
 993 {
 994     return( getParentIterator( gIOServicePlane));
 995 }
 996
 997 IOService * IOService::getClient( void ) const
 998 {
 999     return( (IOService *) getChildEntry( gIOServicePlane));
1000 }
1001
1002 OSIterator * IOService::getClientIterator( void ) const
1003 {
1004     return( getChildIterator( gIOServicePlane));
1005 }
1006
1007 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
1008                                                 const IOService * client,
1009                                                 const IOService * provider )
1010 {
1011     _IOOpenServiceIterator * inst;
1012
1013     if( !_iter)
1014         return( 0 );
1015
1016     inst = new _IOOpenServiceIterator;
1017
1018     if( inst && !inst->init()) {
1019         inst->release();
1020         inst = 0;
1021     }
1022     if( inst) {
1023         inst->iter = _iter;
1024         inst->client = client;
1025         inst->provider = provider;
1026     }
1027
1028     return( inst );
1029 }
1030
1031 void _IOOpenServiceIterator::free()
1032 {
1033     iter->release();
1034     if( last)
1035         last->unlockForArbitration();
1036     OSIterator::free();
1037 }
1038
1039 OSObject * _IOOpenServiceIterator::getNextObject()
1040 {
1041     IOService * next;
1042
1043     if( last)
1044         last->unlockForArbitration();
1045
1046     while( (next = (IOService *) iter->getNextObject())) {
1047
1048         next->lockForArbitration();
1049         if( (client && (next->isOpen( client )))
1050          || (provider && (provider->isOpen( next ))) )
1051             break;
1052         next->unlockForArbitration();
1053     }
1054
1055     last = next;
1056
1057     return( next );
1058 }
1059
1060 bool _IOOpenServiceIterator::isValid()
1061 {
1062     return( iter->isValid() );
1063 }
1064
1065 void _IOOpenServiceIterator::reset()
1066 {
1067     if( last) {
1068         last->unlockForArbitration();
1069         last = 0;
1070     }
1071     iter->reset();
1072 }
1073
1074 OSIterator * IOService::getOpenProviderIterator( void ) const
1075 {
1076     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1077 }
1078
1079 OSIterator * IOService::getOpenClientIterator( void ) const
1080 {
1081     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1082 }
1083
1084
1085 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1086                                           bool waitForFunction,
1087                                           void *param1, void *param2,
1088                                           void *param3, void *param4 )
1089 {
1090   IOReturn  result = kIOReturnUnsupported;
1091   IOService *provider;
1092
1093   if (gIOPlatformFunctionHandlerSet == functionName)
1094   {
1095 #if defined(__i386__) || defined(__x86_64__)
1096     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1097     IOService *      target              = (IOService *) param2;
1098     bool             enable              = (param3 != 0);
1099
1100     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1101         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1102     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1103         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1104 #endif /* defined(__i386__) || defined(__x86_64__) */
1105   }
1106
1107   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1108     result = provider->callPlatformFunction(functionName, waitForFunction,
1109                                             param1, param2, param3, param4);
1110   }
1111
1112   return result;
1113 }
1114
1115 IOReturn IOService::callPlatformFunction( const char * functionName,
1116                                           bool waitForFunction,
1117                                           void *param1, void *param2,
1118                                           void *param3, void *param4 )
1119 {
1120   IOReturn result = kIOReturnNoMemory;
1121   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1122
1123   if (functionSymbol != 0) {
1124     result = callPlatformFunction(functionSymbol, waitForFunction,
1125                                   param1, param2, param3, param4);
1126     functionSymbol->release();
1127   }
1128
1129   return result;
1130 }
1131
1132
1133 /*
1134  * Accessors for global services
1135  */
1136
1137 IOPlatformExpert * IOService::getPlatform( void )
1138 {
1139     return( gIOPlatform);
1140 }
1141
1142 class IOPMrootDomain * IOService::getPMRootDomain( void )
1143 {
1144     return( gIOPMRootDomain);
1145 }
1146
1147 IOService * IOService::getResourceService( void )
1148 {
1149     return( gIOResources );
1150 }
1151
1152 void IOService::setPlatform( IOPlatformExpert * platform)
1153 {
1154     gIOPlatform = platform;
1155     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1156
1157 #if defined(__i386__) || defined(__x86_64__)
1158
1159     static const char * keys[kCpuNumDelayTypes] = {
1160         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1161     const OSObject * objs[2];
1162     OSArray * array;
1163     uint32_t  idx;
1164
1165     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1166     {
1167         objs[0] = sCPULatencySet[idx];
1168         objs[1] = sCPULatencyHolder[idx];
1169         array   = OSArray::withObjects(objs, 2);
1170         if (!array) break;
1171         platform->setProperty(keys[idx], array);
1172         array->release();
1173     }
1174 #endif /* defined(__i386__) || defined(__x86_64__) */
1175 }
1176
1177 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1178 {
1179     gIOPMRootDomain = rootDomain;
1180     publishResource("IOKit");
1181 }
1182
1183 /*
1184  * Stacking change
1185  */
1186
1187 bool IOService::lockForArbitration( bool isSuccessRequired )
1188 {
1189     bool                          found;
1190     bool                          success;
1191     ArbitrationLockQueueElement * element;
1192     ArbitrationLockQueueElement * active;
1193     ArbitrationLockQueueElement * waiting;
1194
1195     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1196
1197     // lock global access
1198     IOTakeLock( gArbitrationLockQueueLock );
1199
1200     // obtain an unused queue element
1201     if( !queue_empty( &gArbitrationLockQueueFree )) {
1202         queue_remove_first( &gArbitrationLockQueueFree,
1203                             element,
1204                             ArbitrationLockQueueElement *,
1205                             link );
1206     } else {
1207         element = IONew( ArbitrationLockQueueElement, 1 );
1208         assert( element );
1209     }
1210
1211     // prepare the queue element
1212     element->thread   = IOThreadSelf();
1213     element->service  = this;
1214     element->count    = 1;
1215     element->required = isSuccessRequired;
1216     element->aborted  = false;
1217
1218     // determine whether this object is already locked (ie. on active queue)
1219     found = false;
1220     queue_iterate( &gArbitrationLockQueueActive,
1221                     active,
1222                     ArbitrationLockQueueElement *,
1223                     link )
1224     {
1225         if( active->service == element->service ) {
1226             found = true;
1227             break;
1228         }
1229     }
1230
1231     if( found ) { // this object is already locked
1232
1233         // determine whether it is the same or a different thread trying to lock
1234         if( active->thread != element->thread ) { // it is a different thread
1235
1236             ArbitrationLockQueueElement * victim = 0;
1237
1238             // before placing this new thread on the waiting queue, we look for
1239             // a deadlock cycle...
1240
1241             while( 1 ) {
1242                 // determine whether the active thread holding the object we
1243                 // want is waiting for another object to be unlocked
1244                 found = false;
1245                 queue_iterate( &gArbitrationLockQueueWaiting,
1246                                waiting,
1247                                ArbitrationLockQueueElement *,
1248                                link )
1249                 {
1250                     if( waiting->thread == active->thread ) {
1251                         assert( false == waiting->aborted );
1252                         found = true;
1253                         break;
1254                     }
1255                 }
1256
1257                 if( found ) { // yes, active thread waiting for another object
1258
1259                     // this may be a candidate for rejection if the required
1260                     // flag is not set, should we detect a deadlock later on
1261                     if( false == waiting->required )
1262                         victim = waiting;
1263
1264                     // find the thread that is holding this other object, that
1265                     // is blocking the active thread from proceeding (fun :-)
1266                     found = false;
1267                     queue_iterate( &gArbitrationLockQueueActive,
1268                                    active,      // (reuse active queue element)
1269                                    ArbitrationLockQueueElement *,
1270                                    link )
1271                     {
1272                         if( active->service == waiting->service ) {
1273                             found = true;
1274                             break;
1275                         }
1276                     }
1277
1278                     // someone must be holding it or it wouldn't be waiting
1279                     assert( found );
1280
1281                     if( active->thread == element->thread ) {
1282
1283                         // doh, it's waiting for the thread that originated
1284                         // this whole lock (ie. current thread) -> deadlock
1285                         if( false == element->required ) { // willing to fail?
1286
1287                             // the originating thread doesn't have the required
1288                             // flag, so it can fail
1289                             success = false; // (fail originating lock request)
1290                             break; // (out of while)
1291
1292                         } else { // originating thread is not willing to fail
1293
1294                             // see if we came across a waiting thread that did
1295                             // not have the 'required' flag set: we'll fail it
1296                             if( victim ) {
1297
1298                                 // we do have a willing victim, fail it's lock
1299                                 victim->aborted = true;
1300
1301                                 // take the victim off the waiting queue
1302                                 queue_remove( &gArbitrationLockQueueWaiting,
1303                                               victim,
1304                                               ArbitrationLockQueueElement *,
1305                                               link );
1306
1307                                 // wake the victim
1308                                 IOLockWakeup( gArbitrationLockQueueLock,
1309                                               victim,
1310                                               /* one thread */ true );
1311
1312                                 // allow this thread to proceed (ie. wait)
1313                                 success = true; // (put request on wait queue)
1314                                 break; // (out of while)
1315                             } else {
1316
1317                                 // all the waiting threads we came across in
1318                                 // finding this loop had the 'required' flag
1319                                 // set, so we've got a deadlock we can't avoid
1320                                 panic("I/O Kit: Unrecoverable deadlock.");
1321                             }
1322                         }
1323                     } else {
1324                         // repeat while loop, redefining active thread to be the
1325                         // thread holding "this other object" (see above), and
1326                         // looking for threads waiting on it; note the active
1327                         // variable points to "this other object" already... so
1328                         // there nothing to do in this else clause.
1329                     }
1330                 } else { // no, active thread is not waiting for another object
1331
1332                     success = true; // (put request on wait queue)
1333                     break; // (out of while)
1334                 }
1335             } // while forever
1336
1337             if( success ) { // put the request on the waiting queue?
1338                 kern_return_t wait_result;
1339
1340                 // place this thread on the waiting queue and put it to sleep;
1341                 // we place it at the tail of the queue...
1342                 queue_enter( &gArbitrationLockQueueWaiting,
1343                              element,
1344                              ArbitrationLockQueueElement *,
1345                              link );
1346
1347                 // declare that this thread will wait for a given event
1348 restart_sleep:  wait_result = assert_wait( element,
1349                                            element->required ? THREAD_UNINT
1350                                            : THREAD_INTERRUPTIBLE );
1351
1352                 // unlock global access
1353                 IOUnlock( gArbitrationLockQueueLock );
1354
1355                 // put thread to sleep, waiting for our event to fire...
1356                 if (wait_result == THREAD_WAITING)
1357                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1358
1359
1360                 // ...and we've been woken up; we might be in one of two states:
1361                 // (a) we've been aborted and our queue element is not on
1362                 //     any of the three queues, but is floating around
1363                 // (b) we're allowed to proceed with the lock and we have
1364                 //     already been moved from the waiting queue to the
1365                 //     active queue.
1366                 // ...plus a 3rd state, should the thread have been interrupted:
1367                 // (c) we're still on the waiting queue
1368
1369                 // determine whether we were interrupted out of our sleep
1370                 if( THREAD_INTERRUPTED == wait_result ) {
1371
1372                     // re-lock global access
1373                     IOTakeLock( gArbitrationLockQueueLock );
1374
1375                     // determine whether we're still on the waiting queue
1376                     found = false;
1377                     queue_iterate( &gArbitrationLockQueueWaiting,
1378                                    waiting,     // (reuse waiting queue element)
1379                                    ArbitrationLockQueueElement *,
1380                                    link )
1381                     {
1382                         if( waiting == element ) {
1383                             found = true;
1384                             break;
1385                         }
1386                     }
1387
1388                     if( found ) { // yes, we're still on the waiting queue
1389
1390                         // determine whether we're willing to fail
1391                         if( false == element->required ) {
1392
1393                             // mark us as aborted
1394                             element->aborted = true;
1395
1396                             // take us off the waiting queue
1397                             queue_remove( &gArbitrationLockQueueWaiting,
1398                                           element,
1399                                           ArbitrationLockQueueElement *,
1400                                           link );
1401                         } else { // we are not willing to fail
1402
1403                             // ignore interruption, go back to sleep
1404                             goto restart_sleep;
1405                         }
1406                     }
1407
1408                     // unlock global access
1409                     IOUnlock( gArbitrationLockQueueLock );
1410
1411                     // proceed as though this were a normal wake up
1412                     wait_result = THREAD_AWAKENED;
1413                 }
1414
1415                 assert( THREAD_AWAKENED == wait_result );
1416
1417                 // determine whether we've been aborted while we were asleep
1418                 if( element->aborted ) {
1419                     assert( false == element->required );
1420
1421                     // re-lock global access
1422                     IOTakeLock( gArbitrationLockQueueLock );
1423
1424                     action = kPutOnFreeQueue;
1425                     success = false;
1426                 } else { // we weren't aborted, so we must be ready to go :-)
1427
1428                     // we've already been moved from waiting to active queue
1429                     return true;
1430                 }
1431
1432             } else { // the lock request is to be failed
1433
1434                 // return unused queue element to queue
1435                 action = kPutOnFreeQueue;
1436             }
1437         } else { // it is the same thread, recursive access is allowed
1438
1439             // add one level of recursion
1440             active->count++;
1441
1442             // return unused queue element to queue
1443             action = kPutOnFreeQueue;
1444             success = true;
1445         }
1446     } else { // this object is not already locked, so let this thread through
1447         action = kPutOnActiveQueue;
1448         success = true;
1449     }
1450
1451     // put the new element on a queue
1452     if( kPutOnActiveQueue == action ) {
1453         queue_enter( &gArbitrationLockQueueActive,
1454                      element,
1455                      ArbitrationLockQueueElement *,
1456                      link );
1457     } else if( kPutOnFreeQueue == action ) {
1458         queue_enter( &gArbitrationLockQueueFree,
1459                      element,
1460                      ArbitrationLockQueueElement *,
1461                      link );
1462     } else {
1463         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1464     }
1465
1466     // unlock global access
1467     IOUnlock( gArbitrationLockQueueLock );
1468
1469     return( success );
1470 }
1471
1472 void IOService::unlockForArbitration( void )
1473 {
1474     bool                          found;
1475     ArbitrationLockQueueElement * element;
1476
1477     // lock global access
1478     IOTakeLock( gArbitrationLockQueueLock );
1479
1480     // find the lock element for this object (ie. on active queue)
1481     found = false;
1482     queue_iterate( &gArbitrationLockQueueActive,
1483                     element,
1484                     ArbitrationLockQueueElement *,
1485                     link )
1486     {
1487         if( element->service == this ) {
1488             found = true;
1489             break;
1490         }
1491     }
1492
1493     assert( found );
1494
1495     // determine whether the lock has been taken recursively
1496     if( element->count > 1 ) {
1497         // undo one level of recursion
1498         element->count--;
1499
1500     } else {
1501
1502         // remove it from the active queue
1503         queue_remove( &gArbitrationLockQueueActive,
1504                       element,
1505                       ArbitrationLockQueueElement *,
1506                       link );
1507
1508         // put it on the free queue
1509         queue_enter( &gArbitrationLockQueueFree,
1510                      element,
1511                      ArbitrationLockQueueElement *,
1512                      link );
1513
1514         // determine whether a thread is waiting for object (head to tail scan)
1515         found = false;
1516         queue_iterate( &gArbitrationLockQueueWaiting,
1517                        element,
1518                        ArbitrationLockQueueElement *,
1519                        link )
1520         {
1521             if( element->service == this ) {
1522                 found = true;
1523                 break;
1524             }
1525         }
1526
1527         if ( found ) { // we found an interested thread on waiting queue
1528
1529             // remove it from the waiting queue
1530             queue_remove( &gArbitrationLockQueueWaiting,
1531                           element,
1532                           ArbitrationLockQueueElement *,
1533                           link );
1534
1535             // put it on the active queue
1536             queue_enter( &gArbitrationLockQueueActive,
1537                          element,
1538                          ArbitrationLockQueueElement *,
1539                          link );
1540
1541             // wake the waiting thread
1542             IOLockWakeup( gArbitrationLockQueueLock,
1543                           element,
1544                           /* one thread */ true );
1545         }
1546     }
1547
1548     // unlock global access
1549     IOUnlock( gArbitrationLockQueueLock );
1550 }
1551
1552 void IOService::applyToProviders( IOServiceApplierFunction applier,
1553                                   void * context )
1554 {
1555     applyToParents( (IORegistryEntryApplierFunction) applier,
1556                     context, gIOServicePlane );
1557 }
1558
1559 void IOService::applyToClients( IOServiceApplierFunction applier,
1560                                 void * context )
1561 {
1562     applyToChildren( (IORegistryEntryApplierFunction) applier,
1563                      context, gIOServicePlane );
1564 }
1565
1566
1567 /*
1568  * Client messages
1569  */
1570
1571
1572 // send a message to a client or interested party of this service
1573 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1574                                    void * argument, vm_size_t argSize )
1575 {
1576     IOReturn                            ret;
1577     IOService *                         service;
1578     _IOServiceInterestNotifier *        notify;
1579
1580     if( (service = OSDynamicCast( IOService, client)))
1581         ret = service->message( type, this, argument );
1582
1583     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1584
1585         _IOServiceNotifierInvocation invocation;
1586         bool                     willNotify;
1587
1588         invocation.thread = current_thread();
1589
1590         LOCKWRITENOTIFY();
1591         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1592
1593         if( willNotify) {
1594             queue_enter( &notify->handlerInvocations, &invocation,
1595                             _IOServiceNotifierInvocation *, link );
1596         }
1597         UNLOCKNOTIFY();
1598
1599         if( willNotify) {
1600
1601             ret = (*notify->handler)( notify->target, notify->ref,
1602                                           type, this, argument, argSize );
1603
1604             LOCKWRITENOTIFY();
1605             queue_remove( &notify->handlerInvocations, &invocation,
1606                             _IOServiceNotifierInvocation *, link );
1607             if( kIOServiceNotifyWaiter & notify->state) {
1608                 notify->state &= ~kIOServiceNotifyWaiter;
1609                 WAKEUPNOTIFY( notify );
1610             }
1611             UNLOCKNOTIFY();
1612
1613         } else
1614             ret = kIOReturnSuccess;
1615
1616     } else
1617         ret = kIOReturnBadArgument;
1618
1619     return( ret );
1620 }
1621
1622 static void
1623 applyToInterestNotifiers(const IORegistryEntry *target,
1624                          const OSSymbol * typeOfInterest,
1625                          OSObjectApplierFunction applier,
1626                          void * context )
1627 {
1628     OSArray *           copyArray = 0;
1629
1630     LOCKREADNOTIFY();
1631
1632     IOCommand *notifyList =
1633         OSDynamicCast( IOCommand, target->getProperty( typeOfInterest ));
1634
1635     if( notifyList) {
1636         copyArray = OSArray::withCapacity(1);
1637
1638         // iterate over queue, entry is set to each element in the list
1639         iterqueue(&notifyList->fCommandChain, entry) {
1640             _IOServiceInterestNotifier * notify;
1641
1642             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1643             copyArray->setObject(notify);
1644         }
1645     }
1646     UNLOCKNOTIFY();
1647
1648     if( copyArray) {
1649         unsigned int    index;
1650         OSObject *      next;
1651
1652         for( index = 0; (next = copyArray->getObject( index )); index++)
1653             (*applier)(next, context);
1654         copyArray->release();
1655     }
1656 }
1657
1658 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1659                                    OSObjectApplierFunction applier,
1660                                    void * context )
1661 {
1662     if (gIOGeneralInterest == typeOfInterest)
1663         applyToClients( (IOServiceApplierFunction) applier, context );
1664     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1665 }
1666
1667 struct MessageClientsContext {
1668     IOService * service;
1669     UInt32      type;
1670     void *      argument;
1671     vm_size_t   argSize;
1672     IOReturn    ret;
1673 };
1674
1675 static void messageClientsApplier( OSObject * object, void * ctx )
1676 {
1677     IOReturn                ret;
1678     MessageClientsContext * context = (MessageClientsContext *) ctx;
1679
1680     ret = context->service->messageClient( context->type,
1681                                            object, context->argument, context->argSize );
1682     if( kIOReturnSuccess != ret)
1683         context->ret = ret;
1684 }
1685
1686 // send a message to all clients
1687 IOReturn IOService::messageClients( UInt32 type,
1688                                     void * argument, vm_size_t argSize )
1689 {
1690     MessageClientsContext       context;
1691
1692     context.service     = this;
1693     context.type        = type;
1694     context.argument    = argument;
1695     context.argSize     = argSize;
1696     context.ret         = kIOReturnSuccess;
1697
1698     applyToInterested( gIOGeneralInterest,
1699                        &messageClientsApplier, &context );
1700
1701     return( context.ret );
1702 }
1703
1704 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1705                                               IOOptionBits response )
1706 {
1707     return( kIOReturnUnsupported );
1708 }
1709
1710 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1711                   IOServiceInterestHandler handler, void * target, void * ref )
1712 {
1713     _IOServiceInterestNotifier * notify = 0;
1714     IOReturn rc = kIOReturnError;
1715
1716     notify = new _IOServiceInterestNotifier;
1717     if (!notify) return NULL;
1718
1719     if(notify->init()) {
1720         rc = registerInterestForNotifer(notify, typeOfInterest,
1721                               handler, target, ref);
1722     }
1723
1724     if (rc != kIOReturnSuccess) {
1725         notify->release();
1726         notify = 0;
1727     }
1728
1729     return( notify );
1730 }
1731
1732 IOReturn IOService::registerInterestForNotifer( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1733                   IOServiceInterestHandler handler, void * target, void * ref )
1734 {
1735     IOReturn rc = kIOReturnSuccess;
1736     _IOServiceInterestNotifier  *notify = 0;
1737
1738     if( (typeOfInterest != gIOGeneralInterest)
1739      && (typeOfInterest != gIOBusyInterest)
1740      && (typeOfInterest != gIOAppPowerStateInterest)
1741      && (typeOfInterest != gIOConsoleSecurityInterest)
1742      && (typeOfInterest != gIOPriorityPowerStateInterest))
1743         return( kIOReturnBadArgument );
1744
1745     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1746         return( kIOReturnBadArgument );
1747
1748     lockForArbitration();
1749     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1750
1751         notify->handler = handler;
1752         notify->target = target;
1753         notify->ref = ref;
1754         notify->state = kIOServiceNotifyEnable;
1755
1756         ////// queue
1757
1758         LOCKWRITENOTIFY();
1759
1760         // Get the head of the notifier linked list
1761         IOCommand * notifyList;
1762         OSObject  * obj = copyProperty( typeOfInterest );
1763         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1764             notifyList = OSTypeAlloc(IOCommand);
1765             if (notifyList) {
1766                 notifyList->init();
1767                 bool ok = setProperty( typeOfInterest, notifyList);
1768                 notifyList->release();
1769                 if (!ok) notifyList = 0;
1770             }
1771         }
1772         if (obj) obj->release();
1773
1774         if (notifyList) {
1775             enqueue(&notifyList->fCommandChain, &notify->chain);
1776             notify->retain();   // ref'ed while in list
1777         }
1778
1779         UNLOCKNOTIFY();
1780     }
1781     else {
1782         rc = kIOReturnNotReady;
1783     }
1784     unlockForArbitration();
1785
1786     return rc;
1787 }
1788
1789 static void cleanInterestList( OSObject * head )
1790 {
1791     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1792     if (!notifyHead)
1793         return;
1794
1795     LOCKWRITENOTIFY();
1796     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1797         queue_next(entry) = queue_prev(entry) = 0;
1798
1799         _IOServiceInterestNotifier * notify;
1800
1801         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1802         notify->release();
1803     }
1804     UNLOCKNOTIFY();
1805 }
1806
1807 void IOService::unregisterAllInterest( void )
1808 {
1809     cleanInterestList( getProperty( gIOGeneralInterest ));
1810     cleanInterestList( getProperty( gIOBusyInterest ));
1811     cleanInterestList( getProperty( gIOAppPowerStateInterest ));
1812     cleanInterestList( getProperty( gIOPriorityPowerStateInterest ));
1813     cleanInterestList( getProperty( gIOConsoleSecurityInterest ));
1814 }
1815
1816 /*
1817  * _IOServiceInterestNotifier
1818  */
1819
1820 // wait for all threads, other than the current one,
1821 //  to exit the handler
1822
1823 void _IOServiceInterestNotifier::wait()
1824 {
1825     _IOServiceNotifierInvocation * next;
1826     bool doWait;
1827
1828     do {
1829         doWait = false;
1830         queue_iterate( &handlerInvocations, next,
1831                         _IOServiceNotifierInvocation *, link) {
1832             if( next->thread != current_thread() ) {
1833                 doWait = true;
1834                 break;
1835             }
1836         }
1837         if( doWait) {
1838             state |= kIOServiceNotifyWaiter;
1839                SLEEPNOTIFY(this);
1840         }
1841
1842     } while( doWait );
1843 }
1844
1845 void _IOServiceInterestNotifier::free()
1846 {
1847     assert( queue_empty( &handlerInvocations ));
1848     OSObject::free();
1849 }
1850
1851 void _IOServiceInterestNotifier::remove()
1852 {
1853     LOCKWRITENOTIFY();
1854
1855     if( queue_next( &chain )) {
1856         remqueue(&chain);
1857         queue_next( &chain) = queue_prev( &chain) = 0;
1858         release();
1859     }
1860
1861     state &= ~kIOServiceNotifyEnable;
1862
1863     wait();
1864
1865     UNLOCKNOTIFY();
1866
1867     release();
1868 }
1869
1870 bool _IOServiceInterestNotifier::disable()
1871 {
1872     bool        ret;
1873
1874     LOCKWRITENOTIFY();
1875
1876     ret = (0 != (kIOServiceNotifyEnable & state));
1877     state &= ~kIOServiceNotifyEnable;
1878     if( ret)
1879         wait();
1880
1881     UNLOCKNOTIFY();
1882
1883     return( ret );
1884 }
1885
1886 void _IOServiceInterestNotifier::enable( bool was )
1887 {
1888     LOCKWRITENOTIFY();
1889     if( was)
1890         state |= kIOServiceNotifyEnable;
1891     else
1892         state &= ~kIOServiceNotifyEnable;
1893     UNLOCKNOTIFY();
1894 }
1895
1896 bool _IOServiceInterestNotifier::init()
1897 {
1898     queue_init( &handlerInvocations );
1899     return (OSObject::init());
1900 }
1901 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1902
1903 /*
1904  * Termination
1905  */
1906
1907 #define tailQ(o)                setObject(o)
1908 #define headQ(o)                setObject(0, o)
1909 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
1910
1911 static void _workLoopAction( IOWorkLoop::Action action,
1912                              IOService * service,
1913                              void * p0 = 0, void * p1 = 0,
1914                              void * p2 = 0, void * p3 = 0 )
1915 {
1916     IOWorkLoop * wl;
1917
1918     if( (wl = service->getWorkLoop())) {
1919         wl->retain();
1920         wl->runAction( action, service, p0, p1, p2, p3 );
1921         wl->release();
1922     } else
1923         (*action)( service, p0, p1, p2, p3 );
1924 }
1925
1926 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
1927 {
1928     bool ok;
1929
1930     // if its our only provider
1931     ok = isParent( provider, gIOServicePlane, true);
1932
1933     // -- compat
1934     if( ok) {
1935         provider->terminateClient( this, options | kIOServiceRecursing );
1936         ok = (0 != (kIOServiceInactiveState & __state[0]));
1937     }
1938     // --
1939
1940     return( ok );
1941 }
1942
1943 bool IOService::terminatePhase1( IOOptionBits options )
1944 {
1945     IOService *  victim;
1946     IOService *  client;
1947     OSIterator * iter;
1948     OSArray *    makeInactive;
1949     OSArray *    waitingInactive;
1950     int          waitResult = THREAD_AWAKENED;
1951     bool         wait;
1952     bool                 ok;
1953     bool                 didInactive;
1954     bool                 startPhase2 = false;
1955
1956     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
1957
1958     uint64_t regID = getRegistryEntryID();
1959     IOServiceTrace(
1960         IOSERVICE_TERMINATE_PHASE1,
1961         (uintptr_t) regID,
1962         (uintptr_t) (regID >> 32),
1963         (uintptr_t) this,
1964         (uintptr_t) options);
1965
1966     // -- compat
1967     if( options & kIOServiceRecursing) {
1968         lockForArbitration();
1969         if (0 == (kIOServiceInactiveState & __state[0]))
1970         {
1971             __state[0] |= kIOServiceInactiveState;
1972             __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
1973         }
1974         unlockForArbitration();
1975
1976         return( true );
1977     }
1978     // --
1979
1980     makeInactive    = OSArray::withCapacity( 16 );
1981     waitingInactive = OSArray::withCapacity( 16 );
1982     if(!makeInactive || !waitingInactive) return( false );
1983
1984     victim = this;
1985     victim->retain();
1986
1987     while( victim )
1988     {
1989         didInactive = victim->lockForArbitration( true );
1990         if( didInactive)
1991         {
1992             uint64_t regID1 = victim->getRegistryEntryID();
1993             IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
1994                 (uintptr_t) regID1,
1995                 (uintptr_t) (regID1 >> 32),
1996                 (uintptr_t) victim->__state[1],
1997                 (uintptr_t) 0);
1998
1999             enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2000             didInactive = (kRP1 == (victim->__state[1] & kRP1))
2001                         || (0 == (victim->__state[0] & kIOServiceInactiveState));
2002
2003             if (!didInactive)
2004             {
2005                 // a multiply attached IOService can be visited twice
2006                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do
2007                 {
2008                     IOLockLock(gIOServiceBusyLock);
2009                     wait = (victim->__state[1] & kIOServiceTermPhase1State);
2010                     if( wait) {
2011                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2012                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2013                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2014                         victim->unlockForArbitration();
2015                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2016                     }
2017                     IOLockUnlock(gIOServiceBusyLock);
2018                     if( wait) {
2019                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2020                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2021                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2022                         victim->lockForArbitration();
2023                     }
2024                 }
2025                 while (wait && (waitResult != THREAD_TIMED_OUT));
2026             }
2027             else
2028             {
2029                 victim->__state[0] |= kIOServiceInactiveState;
2030                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2031                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2032                 victim->__state[1] &= ~kIOServiceRecursing;
2033                 victim->__state[1] |= kIOServiceTermPhase1State;
2034                 waitingInactive->headQ(victim);
2035                 if (victim == this)
2036                 {
2037                     if (kIOServiceTerminateNeedWillTerminate & options)
2038                     {
2039                         victim->__state[1] |= kIOServiceNeedWillTerminate;
2040                     }
2041                 }
2042                 victim->_adjustBusy( 1 );
2043             }
2044             victim->unlockForArbitration();
2045         }
2046         if( victim == this) startPhase2 = didInactive;
2047         if (didInactive)
2048         {
2049             victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff );
2050             IOUserClient::destroyUserReferences( victim );
2051
2052             iter = victim->getClientIterator();
2053             if( iter) {
2054                 while( (client = (IOService *) iter->getNextObject())) {
2055                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2056                             client->getName(), client->getRegistryEntryID(),
2057                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2058                     ok = client->requestTerminate( victim, options );
2059                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2060                             client->getName(), client->getRegistryEntryID(),
2061                             victim->getName(), victim->getRegistryEntryID(), ok);
2062
2063                     uint64_t regID1 = client->getRegistryEntryID();
2064                     uint64_t regID2 = victim->getRegistryEntryID();
2065                     IOServiceTrace(
2066                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2067                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
2068                         (uintptr_t) regID1,
2069                         (uintptr_t) (regID1 >> 32),
2070                         (uintptr_t) regID2,
2071                         (uintptr_t) (regID2 >> 32));
2072
2073                     if( ok)
2074                         makeInactive->setObject( client );
2075                 }
2076                 iter->release();
2077             }
2078         }
2079         victim->release();
2080         victim = (IOService *) makeInactive->getObject(0);
2081         if( victim) {
2082             victim->retain();
2083             makeInactive->removeObject(0);
2084         }
2085     }
2086     makeInactive->release();
2087
2088     while ((victim = (IOService *) waitingInactive->getObject(0)))
2089     {
2090         victim->retain();
2091         waitingInactive->removeObject(0);
2092
2093         victim->lockForArbitration();
2094         victim->__state[1] &= ~kIOServiceTermPhase1State;
2095         if (kIOServiceTerm1WaiterState & victim->__state[1])
2096         {
2097             victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2098             TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2099             IOLockLock( gIOServiceBusyLock );
2100             thread_wakeup( (event_t) &victim->__state[1]);
2101             IOLockUnlock( gIOServiceBusyLock );
2102         }
2103         victim->unlockForArbitration();
2104         victim->release();
2105     }
2106     waitingInactive->release();
2107
2108     if( startPhase2)
2109     {
2110         retain();
2111         lockForArbitration();
2112         scheduleTerminatePhase2(options);
2113         unlockForArbitration();
2114         release();
2115     }
2116
2117     return( true );
2118 }
2119
2120 void IOService::setTerminateDefer(IOService * provider, bool defer)
2121 {
2122     lockForArbitration();
2123     if (defer)  __state[1] |= kIOServiceStartState;
2124     else        __state[1] &= ~kIOServiceStartState;
2125     unlockForArbitration();
2126
2127     if (provider && !defer)
2128     {
2129         provider->lockForArbitration();
2130         provider->scheduleTerminatePhase2();
2131         provider->unlockForArbitration();
2132     }
2133 }
2134
2135 // call with lockForArbitration
2136 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2137 {
2138     AbsoluteTime        deadline;
2139     uint64_t            regID1;
2140     int                 waitResult = THREAD_AWAKENED;
2141     bool                wait, haveDeadline = false;
2142
2143     if (!(__state[0] & kIOServiceInactiveState)) return;
2144
2145     regID1 = getRegistryEntryID();
2146     IOServiceTrace(
2147         IOSERVICE_TERM_SCHED_PHASE2,
2148         (uintptr_t) regID1,
2149         (uintptr_t) (regID1 >> 32),
2150         (uintptr_t) __state[1],
2151         (uintptr_t) options);
2152
2153     if (__state[1] & kIOServiceTermPhase1State)         return;
2154
2155     retain();
2156     unlockForArbitration();
2157     options |= kIOServiceRequired;
2158     IOLockLock( gJobsLock );
2159
2160     if( (options & kIOServiceSynchronous)
2161         && (current_thread() != gIOTerminateThread)) {
2162
2163         do {
2164             wait = (gIOTerminateThread != 0);
2165             if( wait) {
2166                 // wait to become the terminate thread
2167                 IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2168             }
2169         } while( wait );
2170
2171         gIOTerminateThread = current_thread();
2172         gIOTerminatePhase2List->setObject( this );
2173         gIOTerminateWork++;
2174
2175         do {
2176             while( gIOTerminateWork )
2177                 terminateWorker( options );
2178             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2179             if( wait) {
2180                 // wait for the victim to go non-busy
2181                 if( !haveDeadline) {
2182                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2183                     haveDeadline = true;
2184                 }
2185                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2186                                                   deadline, THREAD_UNINT );
2187                 if( waitResult == THREAD_TIMED_OUT) {
2188                     IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2189                 }
2190             }
2191         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2192
2193         gIOTerminateThread = 0;
2194         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2195
2196     } else {
2197         // ! kIOServiceSynchronous
2198
2199         gIOTerminatePhase2List->setObject( this );
2200         if( 0 == gIOTerminateWork++) {
2201             if( !gIOTerminateThread)
2202                 kernel_thread_start(&terminateThread, (void *)(uintptr_t) options, &gIOTerminateThread);
2203             else
2204                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2205         }
2206     }
2207
2208     IOLockUnlock( gJobsLock );
2209     lockForArbitration();
2210     release();
2211 }
2212
2213 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2214 {
2215     IOLockLock( gJobsLock );
2216
2217     while (gIOTerminateWork)
2218         terminateWorker( (uintptr_t) arg );
2219
2220     thread_deallocate(gIOTerminateThread);
2221     gIOTerminateThread = 0;
2222     IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2223
2224     IOLockUnlock( gJobsLock );
2225 }
2226
2227 void IOService::scheduleStop( IOService * provider )
2228 {
2229     uint64_t regID1 = getRegistryEntryID();
2230     uint64_t regID2 = provider->getRegistryEntryID();
2231
2232     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2233     IOServiceTrace(
2234         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2235         (uintptr_t) regID1,
2236         (uintptr_t) (regID1 >> 32),
2237         (uintptr_t) regID2,
2238         (uintptr_t) (regID2 >> 32));
2239
2240     IOLockLock( gJobsLock );
2241     gIOStopList->tailQ( this );
2242     gIOStopProviderList->tailQ( provider );
2243
2244     if( 0 == gIOTerminateWork++) {
2245         if( !gIOTerminateThread)
2246             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2247         else
2248             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2249     }
2250
2251     IOLockUnlock( gJobsLock );
2252 }
2253
2254 void IOService::scheduleFinalize(bool now)
2255 {
2256     uint64_t regID1 = getRegistryEntryID();
2257
2258     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2259     IOServiceTrace(
2260         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2261         (uintptr_t) regID1,
2262         (uintptr_t) (regID1 >> 32),
2263         0, 0);
2264
2265     if (now || IOUserClient::finalizeUserReferences(this))
2266     {
2267         IOLockLock( gJobsLock );
2268         gIOFinalizeList->tailQ(this);
2269         if( 0 == gIOTerminateWork++)
2270         {
2271             if( !gIOTerminateThread)
2272                 kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2273             else
2274                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2275         }
2276         IOLockUnlock( gJobsLock );
2277     }
2278 }
2279
2280 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2281 {
2282     return( true );
2283 }
2284
2285 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2286 {
2287     if( false == *defer) {
2288
2289         if( lockForArbitration( true )) {
2290             if( false == provider->handleIsOpen( this ))
2291                 scheduleStop( provider );
2292             // -- compat
2293             else {
2294                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2295                 if( false == provider->handleIsOpen( this ))
2296                     scheduleStop( provider );
2297             }
2298             // --
2299             unlockForArbitration();
2300         }
2301     }
2302
2303     return( true );
2304 }
2305
2306 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2307                                      OSArray * doPhase2List,
2308                                      void *unused2 __unused,
2309                                      void *unused3 __unused  )
2310 {
2311     OSIterator * iter;
2312     IOService *  client;
2313     bool         ok;
2314     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2315
2316     iter = victim->getClientIterator();
2317     if( iter) {
2318         while( (client = (IOService *) iter->getNextObject())) {
2319
2320             regID1 = client->getRegistryEntryID();
2321             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2322                     client->getName(), regID1,
2323                     victim->getName(), regID2, (long long)options);
2324             IOServiceTrace(
2325                 IOSERVICE_TERMINATE_WILL,
2326                 (uintptr_t) regID1,
2327                 (uintptr_t) (regID1 >> 32),
2328                 (uintptr_t) regID2,
2329                 (uintptr_t) (regID2 >> 32));
2330
2331             ok = client->willTerminate( victim, options );
2332             doPhase2List->tailQ( client );
2333         }
2334         iter->release();
2335     }
2336 }
2337
2338 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2339                             void *unused1 __unused, void *unused2 __unused,
2340                             void *unused3 __unused )
2341 {
2342     OSIterator * iter;
2343     IOService *  client;
2344     bool         defer;
2345     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2346
2347     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2348
2349     iter = victim->getClientIterator();
2350     if( iter) {
2351         while( (client = (IOService *) iter->getNextObject())) {
2352
2353             regID1 = client->getRegistryEntryID();
2354             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2355                     client->getName(), regID1,
2356                     victim->getName(), regID2, (long long)options);
2357             defer = false;
2358             client->didTerminate( victim, options, &defer );
2359
2360             IOServiceTrace(
2361                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2362                        : IOSERVICE_TERMINATE_DID),
2363                 (uintptr_t) regID1,
2364                 (uintptr_t) (regID1 >> 32),
2365                 (uintptr_t) regID2,
2366                 (uintptr_t) (regID2 >> 32));
2367
2368             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2369                     client->getName(), regID1,
2370                     victim->getName(), regID2, defer);
2371         }
2372         iter->release();
2373     }
2374 }
2375
2376
2377 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2378                             void *unused1 __unused, void *unused2 __unused,
2379                             void *unused3 __unused )
2380 {
2381     OSIterator * iter;
2382     IOService *  provider;
2383     bool         ok;
2384     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2385
2386     iter = victim->getProviderIterator();
2387     if( iter) {
2388         while( (provider = (IOService *) iter->getNextObject())) {
2389
2390             regID1 = provider->getRegistryEntryID();
2391             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2392                     victim->getName(), regID2,
2393                     provider->getName(), regID1, (long long)options);
2394             IOServiceTrace(
2395                 IOSERVICE_TERMINATE_WILL,
2396                 (uintptr_t) regID2,
2397                 (uintptr_t) (regID2 >> 32),
2398                 (uintptr_t) regID1,
2399                 (uintptr_t) (regID1 >> 32));
2400
2401             ok = victim->willTerminate( provider, options );
2402         }
2403         iter->release();
2404     }
2405 }
2406
2407 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2408                             void *unused1 __unused, void *unused2 __unused,
2409                             void *unused3 __unused )
2410 {
2411     OSIterator * iter;
2412     IOService *  provider;
2413     bool defer = false;
2414     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2415
2416     iter = victim->getProviderIterator();
2417     if( iter) {
2418         while( (provider = (IOService *) iter->getNextObject())) {
2419
2420             regID1 = provider->getRegistryEntryID();
2421             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2422                     victim->getName(), regID2,
2423                     provider->getName(), regID1, (long long)options);
2424             victim->didTerminate( provider, options, &defer );
2425
2426             IOServiceTrace(
2427                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2428                        : IOSERVICE_TERMINATE_DID),
2429                 (uintptr_t) regID2,
2430                 (uintptr_t) (regID2 >> 32),
2431                 (uintptr_t) regID1,
2432                 (uintptr_t) (regID1 >> 32));
2433
2434             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2435                     victim->getName(), regID2,
2436                     provider->getName(), regID1, defer);
2437         }
2438         iter->release();
2439     }
2440 }
2441
2442
2443 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2444                             void *unused1 __unused, void *unused2 __unused,
2445                             void *unused3 __unused )
2446 {
2447     uint64_t regID1 = victim->getRegistryEntryID();
2448     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2449     IOServiceTrace(
2450         IOSERVICE_TERMINATE_FINALIZE,
2451         (uintptr_t) regID1,
2452         (uintptr_t) (regID1 >> 32),
2453         0, 0);
2454
2455     victim->finalize( options );
2456 }
2457
2458 void IOService::actionStop( IOService * provider, IOService * client,
2459                             void *unused1 __unused, void *unused2 __unused,
2460                             void *unused3 __unused )
2461 {
2462     uint64_t regID1 = provider->getRegistryEntryID();
2463     uint64_t regID2 = client->getRegistryEntryID();
2464
2465     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2466     IOServiceTrace(
2467         IOSERVICE_TERMINATE_STOP,
2468         (uintptr_t) regID1,
2469         (uintptr_t) (regID1 >> 32),
2470         (uintptr_t) regID2,
2471         (uintptr_t) (regID2 >> 32));
2472
2473     client->stop( provider );
2474     if( provider->isOpen( client ))
2475         provider->close( client );
2476
2477     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2478     client->detach( provider );
2479 }
2480
2481 void IOService::terminateWorker( IOOptionBits options )
2482 {
2483     OSArray *           doPhase2List;
2484     OSArray *           didPhase2List;
2485     OSSet *             freeList;
2486     OSIterator *        iter;
2487     UInt32              workDone;
2488     IOService *         victim;
2489     IOService *         client;
2490     IOService *         provider;
2491     unsigned int        idx;
2492     bool                moreToDo;
2493     bool                doPhase2;
2494     bool                doPhase3;
2495
2496     options |= kIOServiceRequired;
2497
2498     doPhase2List  = OSArray::withCapacity( 16 );
2499     didPhase2List = OSArray::withCapacity( 16 );
2500     freeList      = OSSet::withCapacity( 16 );
2501     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2502         return;
2503
2504     do {
2505         workDone = gIOTerminateWork;
2506
2507         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2508
2509             victim->retain();
2510             gIOTerminatePhase2List->removeObject(0);
2511             IOLockUnlock( gJobsLock );
2512
2513             uint64_t regID1 = victim->getRegistryEntryID();
2514             IOServiceTrace(
2515                 IOSERVICE_TERM_START_PHASE2,
2516                 (uintptr_t) regID1,
2517                 (uintptr_t) (regID1 >> 32),
2518                 (uintptr_t) 0,
2519                 (uintptr_t) 0);
2520
2521             while( victim ) {
2522
2523                 doPhase2 = victim->lockForArbitration( true );
2524                 if( doPhase2) {
2525                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2526                     if( doPhase2) {
2527
2528                         uint64_t regID1 = victim->getRegistryEntryID();
2529                         IOServiceTrace(
2530                             IOSERVICE_TERM_TRY_PHASE2,
2531                             (uintptr_t) regID1,
2532                             (uintptr_t) (regID1 >> 32),
2533                             (uintptr_t) victim->__state[1],
2534                             (uintptr_t) 0);
2535
2536                         doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State))
2537                                 && (0 == (victim->__state[1] & kIOServiceConfigState));
2538
2539                         if (doPhase2 && (iter = victim->getClientIterator())) {
2540                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2541                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2542                                 if (!doPhase2)
2543                                 {
2544                                     uint64_t regID1 = client->getRegistryEntryID();
2545                                     IOServiceTrace(
2546                                         IOSERVICE_TERM_UC_DEFER,
2547                                         (uintptr_t) regID1,
2548                                         (uintptr_t) (regID1 >> 32),
2549                                         (uintptr_t) client->__state[1],
2550                                         (uintptr_t) 0);
2551                                     TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2552                                            victim->getName(), victim->getRegistryEntryID(),
2553                                            client->getName(), client->getRegistryEntryID());
2554                                 }
2555                             }
2556                             iter->release();
2557                         }
2558                         if( doPhase2)
2559                             victim->__state[1] |= kIOServiceTermPhase2State;
2560                     }
2561                     victim->unlockForArbitration();
2562                 }
2563                 if( doPhase2) {
2564
2565                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2566                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2567                                             victim, (void *)(uintptr_t) options, NULL );
2568                     }
2569
2570                     if( 0 == victim->getClient()) {
2571
2572                         // no clients - will go to finalize
2573                         victim->scheduleFinalize(false);
2574
2575                     } else {
2576                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2577                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2578                     }
2579                     didPhase2List->headQ( victim );
2580                 }
2581                 victim->release();
2582                 victim = (IOService *) doPhase2List->getObject(0);
2583                 if( victim) {
2584                     victim->retain();
2585                     doPhase2List->removeObject(0);
2586                 }
2587             }
2588
2589             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2590
2591                 if( victim->lockForArbitration( true )) {
2592                     victim->__state[1] |= kIOServiceTermPhase3State;
2593                     victim->unlockForArbitration();
2594                 }
2595                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2596                                     victim, (void *)(uintptr_t) options );
2597                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2598                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2599                                         victim, (void *)(uintptr_t) options, NULL );
2600                 }
2601                 didPhase2List->removeObject(0);
2602             }
2603             IOLockLock( gJobsLock );
2604         }
2605
2606         // phase 3
2607         do {
2608             doPhase3 = false;
2609             // finalize leaves
2610             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2611
2612                 IOLockUnlock( gJobsLock );
2613                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2614                                     victim, (void *)(uintptr_t) options );
2615                 IOLockLock( gJobsLock );
2616                 // hold off free
2617                 freeList->setObject( victim );
2618                 // safe if finalize list is append only
2619                 gIOFinalizeList->removeObject(0);
2620             }
2621
2622             for( idx = 0;
2623                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2624
2625                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2626                 assert( provider );
2627
2628                 uint64_t regID1 = provider->getRegistryEntryID();
2629                 uint64_t regID2 = client->getRegistryEntryID();
2630
2631                 if( !provider->isChild( client, gIOServicePlane )) {
2632                     // may be multiply queued - nop it
2633                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2634                     IOServiceTrace(
2635                         IOSERVICE_TERMINATE_STOP_NOP,
2636                         (uintptr_t) regID1,
2637                         (uintptr_t) (regID1 >> 32),
2638                         (uintptr_t) regID2,
2639                         (uintptr_t) (regID2 >> 32));
2640
2641                 } else {
2642                     // a terminated client is not ready for stop if it has clients, skip it
2643                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2644                         TLOG("%s[0x%qx]::defer stop(%s[0x%qx])\n",
2645                                 client->getName(), regID2,
2646                                 client->getClient()->getName(), client->getClient()->getRegistryEntryID());
2647                         IOServiceTrace(
2648                             IOSERVICE_TERMINATE_STOP_DEFER,
2649                             (uintptr_t) regID1,
2650                             (uintptr_t) (regID1 >> 32),
2651                             (uintptr_t) regID2,
2652                             (uintptr_t) (regID2 >> 32));
2653
2654                         idx++;
2655                         continue;
2656                     }
2657
2658                     IOLockUnlock( gJobsLock );
2659                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2660                                      provider, (void *) client );
2661                     IOLockLock( gJobsLock );
2662                     // check the finalize list now
2663                     doPhase3 = true;
2664                 }
2665                 // hold off free
2666                 freeList->setObject( client );
2667                 freeList->setObject( provider );
2668
2669                 // safe if stop list is append only
2670                 gIOStopList->removeObject( idx );
2671                 gIOStopProviderList->removeObject( idx );
2672                 idx = 0;
2673             }
2674
2675         } while( doPhase3 );
2676
2677         gIOTerminateWork -= workDone;
2678         moreToDo = (gIOTerminateWork != 0);
2679
2680         if( !moreToDo) {
2681             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2682             IOServiceTrace(
2683                 IOSERVICE_TERMINATE_DONE,
2684                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2685         }
2686
2687     } while( moreToDo );
2688
2689     IOLockUnlock( gJobsLock );
2690
2691     freeList->release();
2692     doPhase2List->release();
2693     didPhase2List->release();
2694
2695     IOLockLock( gJobsLock );
2696 }
2697
2698 bool IOService::finalize( IOOptionBits options )
2699 {
2700     OSIterator *  iter;
2701     IOService *   provider;
2702     uint64_t      regID1, regID2 = getRegistryEntryID();
2703
2704     iter = getProviderIterator();
2705     assert( iter );
2706
2707     if( iter) {
2708         while( (provider = (IOService *) iter->getNextObject())) {
2709
2710             // -- compat
2711             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2712                 /* we come down here on programmatic terminate */
2713
2714                 regID1 = provider->getRegistryEntryID();
2715                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2716                 IOServiceTrace(
2717                     IOSERVICE_TERMINATE_STOP,
2718                     (uintptr_t) regID1,
2719                     (uintptr_t) (regID1 >> 32),
2720                     (uintptr_t) regID2,
2721                     (uintptr_t) (regID2 >> 32));
2722
2723                 stop( provider );
2724                 if( provider->isOpen( this ))
2725                     provider->close( this );
2726                 detach( provider );
2727             } else {
2728             //--
2729                 if( provider->lockForArbitration( true )) {
2730                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2731                         scheduleStop( provider );
2732                     provider->unlockForArbitration();
2733                 }
2734             }
2735         }
2736         iter->release();
2737     }
2738
2739     return( true );
2740 }
2741
2742 #undef tailQ
2743 #undef headQ
2744
2745 /*
2746  * Terminate
2747  */
2748
2749 void IOService::doServiceTerminate( IOOptionBits options )
2750 {
2751 }
2752
2753 // a method in case someone needs to override it
2754 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2755 {
2756     bool ok;
2757
2758     if( client->isParent( this, gIOServicePlane, true))
2759         // we are the clients only provider
2760         ok = client->terminate( options );
2761     else
2762         ok = true;
2763
2764     return( ok );
2765 }
2766
2767 bool IOService::terminate( IOOptionBits options )
2768 {
2769     options |= kIOServiceTerminate;
2770
2771     return( terminatePhase1( options ));
2772 }
2773
2774 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2775
2776 /*
2777  * Open & close
2778  */
2779
2780 struct ServiceOpenMessageContext
2781 {
2782     IOService *  service;
2783     UInt32       type;
2784     IOService *  excludeClient;
2785     IOOptionBits options;
2786 };
2787
2788 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2789 {
2790     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2791
2792     if( object != context->excludeClient)
2793         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2794 }
2795
2796 bool IOService::open(   IOService *     forClient,
2797                         IOOptionBits    options,
2798                         void *          arg )
2799 {
2800     bool                        ok;
2801     ServiceOpenMessageContext   context;
2802
2803     context.service             = this;
2804     context.type                = kIOMessageServiceIsAttemptingOpen;
2805     context.excludeClient       = forClient;
2806     context.options             = options;
2807
2808     applyToInterested( gIOGeneralInterest,
2809                         &serviceOpenMessageApplier, &context );
2810
2811     if( false == lockForArbitration(false) )
2812         return false;
2813
2814     ok = (0 == (__state[0] & kIOServiceInactiveState));
2815     if( ok)
2816         ok = handleOpen( forClient, options, arg );
2817
2818     unlockForArbitration();
2819
2820     return( ok );
2821 }
2822
2823 void IOService::close(  IOService *     forClient,
2824                         IOOptionBits    options )
2825 {
2826     bool                wasClosed;
2827     bool                last = false;
2828
2829     lockForArbitration();
2830
2831     wasClosed = handleIsOpen( forClient );
2832     if( wasClosed) {
2833         handleClose( forClient, options );
2834         last = (__state[1] & kIOServiceTermPhase3State);
2835     }
2836
2837     unlockForArbitration();
2838
2839     if( last)
2840         forClient->scheduleStop( this );
2841
2842     else if( wasClosed) {
2843
2844         ServiceOpenMessageContext context;
2845
2846         context.service         = this;
2847         context.type            = kIOMessageServiceWasClosed;
2848         context.excludeClient   = forClient;
2849         context.options         = options;
2850
2851         applyToInterested( gIOGeneralInterest,
2852                             &serviceOpenMessageApplier, &context );
2853     }
2854 }
2855
2856 bool IOService::isOpen( const IOService * forClient ) const
2857 {
2858     IOService * self = (IOService *) this;
2859     bool ok;
2860
2861     self->lockForArbitration();
2862
2863     ok = handleIsOpen( forClient );
2864
2865     self->unlockForArbitration();
2866
2867     return( ok );
2868 }
2869
2870 bool IOService::handleOpen(     IOService *     forClient,
2871                                 IOOptionBits    options,
2872                                 void *          arg )
2873 {
2874     bool        ok;
2875
2876     ok = (0 == __owner);
2877     if( ok )
2878         __owner = forClient;
2879
2880     else if( options & kIOServiceSeize ) {
2881         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2882                                 __owner, (void *)(uintptr_t) options ));
2883         if( ok && (0 == __owner ))
2884             __owner = forClient;
2885         else
2886             ok = false;
2887     }
2888     return( ok );
2889 }
2890
2891 void IOService::handleClose(    IOService *     forClient,
2892                                 IOOptionBits    options )
2893 {
2894     if( __owner == forClient)
2895         __owner = 0;
2896 }
2897
2898 bool IOService::handleIsOpen(   const IOService * forClient ) const
2899 {
2900     if( forClient)
2901         return( __owner == forClient );
2902     else
2903         return( __owner != forClient );
2904 }
2905
2906 /*
2907  * Probing & starting
2908  */
2909 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2910 {
2911     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
2912     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
2913     SInt32             val1;
2914     SInt32             val2;
2915
2916     val1 = 0;
2917     val2 = 0;
2918
2919     if ( obj1 )
2920         val1 = obj1->priority;
2921
2922     if ( obj2 )
2923         val2 = obj2->priority;
2924
2925     return ( val1 - val2 );
2926 }
2927
2928 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
2929 {
2930     OSDictionary *      dict;
2931     IOService *         service;
2932     _IOServiceNotifier * notify;
2933     OSSymbol *          key = (OSSymbol *) ref;
2934     OSNumber *          offset;
2935
2936     if( (dict = OSDynamicCast( OSDictionary, entry)))
2937         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
2938     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
2939         return( notify->priority );
2940
2941     else if( (service = OSDynamicCast( IOService, entry)))
2942         offset = OSDynamicCast(OSNumber, service->getProperty( key ));
2943     else {
2944         assert( false );
2945         offset = 0;
2946     }
2947
2948     if( offset)
2949         return( (SInt32) offset->unsigned32BitValue());
2950     else
2951         return( kIODefaultProbeScore );
2952 }
2953
2954 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2955 {
2956     const OSObject *    obj1 = (const OSObject *) inObj1;
2957     const OSObject *    obj2 = (const OSObject *) inObj2;
2958     SInt32               val1;
2959     SInt32               val2;
2960
2961     val1 = 0;
2962     val2 = 0;
2963
2964     if ( obj1 )
2965         val1 = IOServiceObjectOrder( obj1, ref );
2966
2967     if ( obj2 )
2968         val2 = IOServiceObjectOrder( obj2, ref );
2969
2970     return ( val1 - val2 );
2971 }
2972
2973 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
2974 {
2975     IOService *         service = 0;
2976     OSIterator *        iter;
2977     const OSSymbol *    nextCat;
2978
2979     iter = getClientIterator();
2980     if( iter) {
2981         while( (service = (IOService *) iter->getNextObject())) {
2982             if( kIOServiceInactiveState & service->__state[0])
2983                 continue;
2984             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
2985                         service->getProperty( gIOMatchCategoryKey ));
2986             if( category == nextCat)
2987             {
2988                 service->retain();
2989                 break;
2990             }
2991         }
2992         iter->release();
2993     }
2994     return( service );
2995 }
2996
2997 IOService * IOService::getClientWithCategory( const OSSymbol * category )
2998 {
2999     IOService *
3000     service = copyClientWithCategory(category);
3001     if (service)
3002         service->release();
3003     return (service);
3004 }
3005
3006 bool IOService::invokeNotifer( _IOServiceNotifier * notify )
3007 {
3008     _IOServiceNotifierInvocation invocation;
3009     bool                         willNotify;
3010     bool                         ret = true;
3011
3012     invocation.thread = current_thread();
3013
3014     LOCKWRITENOTIFY();
3015     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3016
3017     if( willNotify) {
3018         queue_enter( &notify->handlerInvocations, &invocation,
3019                         _IOServiceNotifierInvocation *, link );
3020     }
3021     UNLOCKNOTIFY();
3022
3023     if( willNotify) {
3024
3025         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3026
3027         LOCKWRITENOTIFY();
3028         queue_remove( &notify->handlerInvocations, &invocation,
3029                         _IOServiceNotifierInvocation *, link );
3030         if( kIOServiceNotifyWaiter & notify->state) {
3031             notify->state &= ~kIOServiceNotifyWaiter;
3032             WAKEUPNOTIFY( notify );
3033         }
3034         UNLOCKNOTIFY();
3035     }
3036
3037     return( ret );
3038 }
3039
3040 /*
3041  * Alloc and probe matching classes,
3042  * called on the provider instance
3043  */
3044
3045 void IOService::probeCandidates( OSOrderedSet * matches )
3046 {
3047     OSDictionary        *       match = 0;
3048     OSSymbol            *       symbol;
3049     IOService           *       inst;
3050     IOService           *       newInst;
3051     OSDictionary        *       props;
3052     SInt32                      score;
3053     OSNumber            *       newPri;
3054     OSOrderedSet        *       familyMatches = 0;
3055     OSOrderedSet        *       startList;
3056     OSDictionary        *       startDict = 0;
3057     const OSSymbol      *       category;
3058     OSIterator          *       iter;
3059     _IOServiceNotifier  *               notify;
3060     OSObject            *       nextMatch = 0;
3061     bool                        started;
3062     bool                        needReloc = false;
3063 #if IOMATCHDEBUG
3064     SInt64                      debugFlags;
3065 #endif
3066     IOService * client = NULL;
3067
3068
3069     assert( matches );
3070     while( !needReloc && (nextMatch = matches->getFirstObject())) {
3071
3072         nextMatch->retain();
3073         matches->removeObject(nextMatch);
3074
3075         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3076
3077             lockForArbitration();
3078             if( 0 == (__state[0] & kIOServiceInactiveState))
3079                 invokeNotifer( notify );
3080             unlockForArbitration();
3081             nextMatch->release();
3082             nextMatch = 0;
3083             continue;
3084
3085         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3086             nextMatch->release();
3087             nextMatch = 0;
3088             continue;
3089         }
3090
3091         props = 0;
3092 #if IOMATCHDEBUG
3093         debugFlags = getDebugFlags( match );
3094 #endif
3095
3096         do {
3097             category = OSDynamicCast( OSSymbol,
3098                         match->getObject( gIOMatchCategoryKey ));
3099             if( 0 == category)
3100                 category = gIODefaultMatchCategoryKey;
3101
3102             if( (client = copyClientWithCategory(category)) ) {
3103 #if IOMATCHDEBUG
3104                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
3105                     LOG("%s: match category %s exists\n", getName(),
3106                                 category->getCStringNoCopy());
3107 #endif
3108                 nextMatch->release();
3109                 nextMatch = 0;
3110
3111                 client->release();
3112                 client = NULL;
3113
3114                 continue;
3115             }
3116
3117             // create a copy now in case its modified during matching
3118             props = OSDictionary::withDictionary( match, match->getCount());
3119             if( 0 == props)
3120                 continue;
3121             props->setCapacityIncrement(1);
3122
3123             // check the nub matches
3124             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
3125                 continue;
3126
3127             // Check to see if driver reloc has been loaded.
3128             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
3129             if( needReloc) {
3130 #if IOMATCHDEBUG
3131                 if( debugFlags & kIOLogCatalogue)
3132                     LOG("%s: stalling for module\n", getName());
3133 #endif
3134                 // If reloc hasn't been loaded, exit;
3135                 // reprobing will occur after reloc has been loaded.
3136                 continue;
3137             }
3138
3139             // reorder on family matchPropertyTable score.
3140             if( 0 == familyMatches)
3141                 familyMatches = OSOrderedSet::withCapacity( 1,
3142                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3143             if( familyMatches)
3144                 familyMatches->setObject( props );
3145
3146         } while( false );
3147
3148         if (nextMatch) {
3149             nextMatch->release();
3150             nextMatch = 0;
3151         }
3152         if( props)
3153             props->release();
3154     }
3155     matches->release();
3156     matches = 0;
3157
3158     if( familyMatches) {
3159
3160         while( !needReloc
3161              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3162
3163             props->retain();
3164             familyMatches->removeObject( props );
3165
3166             inst = 0;
3167             newInst = 0;
3168 #if IOMATCHDEBUG
3169             debugFlags = getDebugFlags( props );
3170 #endif
3171             do {
3172                 symbol = OSDynamicCast( OSSymbol,
3173                                 props->getObject( gIOClassKey));
3174                 if( !symbol)
3175                     continue;
3176
3177                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), OBFUSCATE(symbol), OBFUSCATE(props));
3178
3179                 // alloc the driver instance
3180                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3181
3182                 if( !inst || !OSDynamicCast(IOService, inst)) {
3183                     IOLog("Couldn't alloc class \"%s\"\n",
3184                         symbol->getCStringNoCopy());
3185                     continue;
3186                 }
3187
3188                 // init driver instance
3189                 if( !(inst->init( props ))) {
3190 #if IOMATCHDEBUG
3191                     if( debugFlags & kIOLogStart)
3192                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3193 #endif
3194                     continue;
3195                 }
3196                 if( __state[1] & kIOServiceSynchronousState)
3197                     inst->__state[1] |= kIOServiceSynchronousState;
3198
3199                 // give the driver the default match category if not specified
3200                 category = OSDynamicCast( OSSymbol,
3201                             props->getObject( gIOMatchCategoryKey ));
3202                 if( 0 == category)
3203                     category = gIODefaultMatchCategoryKey;
3204                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3205                 // attach driver instance
3206                 if( !(inst->attach( this )))
3207                         continue;
3208
3209                 // pass in score from property table
3210                 score = familyMatches->orderObject( props );
3211
3212                 // & probe the new driver instance
3213 #if IOMATCHDEBUG
3214                 if( debugFlags & kIOLogProbe)
3215                     LOG("%s::probe(%s)\n",
3216                         inst->getMetaClass()->getClassName(), getName());
3217 #endif
3218
3219                 newInst = inst->probe( this, &score );
3220                 inst->detach( this );
3221                 if( 0 == newInst) {
3222 #if IOMATCHDEBUG
3223                     if( debugFlags & kIOLogProbe)
3224                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3225 #endif
3226                     continue;
3227                 }
3228
3229                 // save the score
3230                 newPri = OSNumber::withNumber( score, 32 );
3231                 if( newPri) {
3232                     newInst->setProperty( gIOProbeScoreKey, newPri );
3233                     newPri->release();
3234                 }
3235
3236                 // add to start list for the match category
3237                 if( 0 == startDict)
3238                     startDict = OSDictionary::withCapacity( 1 );
3239                 assert( startDict );
3240                 startList = (OSOrderedSet *)
3241                                 startDict->getObject( category );
3242                 if( 0 == startList) {
3243                     startList = OSOrderedSet::withCapacity( 1,
3244                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3245                     if( startDict && startList) {
3246                         startDict->setObject( category, startList );
3247                         startList->release();
3248                     }
3249                 }
3250                 assert( startList );
3251                 if( startList)
3252                     startList->setObject( newInst );
3253
3254             } while( false );
3255
3256             props->release();
3257             if( inst)
3258                 inst->release();
3259         }
3260         familyMatches->release();
3261         familyMatches = 0;
3262     }
3263
3264     // start the best (until success) of each category
3265
3266     iter = OSCollectionIterator::withCollection( startDict );
3267     if( iter) {
3268         while( (category = (const OSSymbol *) iter->getNextObject())) {
3269
3270             startList = (OSOrderedSet *) startDict->getObject( category );
3271             assert( startList );
3272             if( !startList)
3273                 continue;
3274
3275             started = false;
3276             while( true // (!started)
3277                    && (inst = (IOService *)startList->getFirstObject())) {
3278
3279                 inst->retain();
3280                 startList->removeObject(inst);
3281
3282 #if IOMATCHDEBUG
3283                 debugFlags = getDebugFlags( inst->getPropertyTable() );
3284
3285                 if( debugFlags & kIOLogStart) {
3286                     if( started)
3287                         LOG( "match category exists, skipping " );
3288                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3289                          getName(), inst->getRetainCount());
3290                 }
3291 #endif
3292                 if( false == started)
3293                     started = startCandidate( inst );
3294 #if IOMATCHDEBUG
3295                 if( (debugFlags & kIOLogStart) && (false == started))
3296                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3297                          inst->getRetainCount());
3298 #endif
3299                 inst->release();
3300             }
3301         }
3302         iter->release();
3303     }
3304
3305
3306     // adjust the busy count by +1 if matching is stalled for a module,
3307     // or -1 if a previously stalled matching is complete.
3308     lockForArbitration();
3309     SInt32 adjBusy = 0;
3310     uint64_t regID = getRegistryEntryID();
3311
3312     if( needReloc) {
3313         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3314         if( adjBusy) {
3315
3316             IOServiceTrace(
3317                 IOSERVICE_MODULESTALL,
3318                 (uintptr_t) regID,
3319                 (uintptr_t) (regID >> 32),
3320                 (uintptr_t) this,
3321                 0);
3322
3323             __state[1] |= kIOServiceModuleStallState;
3324         }
3325
3326     } else if( __state[1] & kIOServiceModuleStallState) {
3327
3328             IOServiceTrace(
3329             IOSERVICE_MODULEUNSTALL,
3330             (uintptr_t) regID,
3331             (uintptr_t) (regID >> 32),
3332             (uintptr_t) this,
3333             0);
3334
3335         __state[1] &= ~kIOServiceModuleStallState;
3336         adjBusy = -1;
3337     }
3338     if( adjBusy)
3339         _adjustBusy( adjBusy );
3340     unlockForArbitration();
3341
3342     if( startDict)
3343         startDict->release();
3344 }
3345
3346 /*
3347  * Start a previously attached & probed instance,
3348  * called on exporting object instance
3349  */
3350
3351 bool IOService::startCandidate( IOService * service )
3352 {
3353     bool                ok;
3354
3355     ok = service->attach( this );
3356
3357     if( ok)
3358     {
3359         if (this != gIOResources)
3360         {
3361             // stall for any nub resources
3362             checkResources();
3363             // stall for any driver resources
3364             service->checkResources();
3365         }
3366
3367         AbsoluteTime startTime;
3368         AbsoluteTime endTime;
3369         UInt64       nano;
3370
3371         if (kIOLogStart & gIOKitDebug)
3372             clock_get_uptime(&startTime);
3373
3374         ok = service->start(this);
3375
3376         if (kIOLogStart & gIOKitDebug)
3377         {
3378             clock_get_uptime(&endTime);
3379
3380             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3381             {
3382                 SUB_ABSOLUTETIME(&endTime, &startTime);
3383                 absolutetime_to_nanoseconds(endTime, &nano);
3384                 if (nano > 500000000ULL)
3385                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3386             }
3387         }
3388         if( !ok)
3389             service->detach( this );
3390     }
3391     return( ok );
3392 }
3393
3394 void IOService::publishResource( const char * key, OSObject * value )
3395 {
3396     const OSSymbol *    sym;
3397
3398     if( (sym = OSSymbol::withCString( key))) {
3399         publishResource( sym, value);
3400         sym->release();
3401     }
3402 }
3403
3404 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3405 {
3406     if( 0 == value)
3407         value = (OSObject *) gIOServiceKey;
3408
3409     gIOResources->setProperty( key, value);
3410
3411     if( IORecursiveLockHaveLock( gNotificationLock))
3412         return;
3413
3414     gIOResourceGenerationCount++;
3415     gIOResources->registerService();
3416 }
3417
3418 bool IOService::addNeededResource( const char * key )
3419 {
3420     OSObject *  resourcesProp;
3421     OSSet *     set;
3422     OSString *  newKey;
3423     bool ret;
3424
3425     resourcesProp = getProperty( gIOResourceMatchKey );
3426
3427     newKey = OSString::withCString( key );
3428     if( (0 == resourcesProp) || (0 == newKey))
3429         return( false);
3430
3431     set = OSDynamicCast( OSSet, resourcesProp );
3432     if( !set) {
3433         set = OSSet::withCapacity( 1 );
3434         if( set)
3435             set->setObject( resourcesProp );
3436     }
3437     else
3438         set->retain();
3439
3440     set->setObject( newKey );
3441     newKey->release();
3442     ret = setProperty( gIOResourceMatchKey, set );
3443     set->release();
3444
3445     return( ret );
3446 }
3447
3448 bool IOService::checkResource( OSObject * matching )
3449 {
3450     OSString *          str;
3451     OSDictionary *      table;
3452
3453     if( (str = OSDynamicCast( OSString, matching ))) {
3454         if( gIOResources->getProperty( str ))
3455             return( true );
3456     }
3457
3458     if( str)
3459         table = resourceMatching( str );
3460     else if( (table = OSDynamicCast( OSDictionary, matching )))
3461         table->retain();
3462     else {
3463         IOLog("%s: Can't match using: %s\n", getName(),
3464                 matching->getMetaClass()->getClassName());
3465         /* false would stall forever */
3466         return( true );
3467     }
3468
3469     if( gIOKitDebug & kIOLogConfig)
3470         LOG("config(%p): stalling %s\n", OBFUSCATE(IOThreadSelf()), getName());
3471
3472     waitForService( table );
3473
3474     if( gIOKitDebug & kIOLogConfig)
3475         LOG("config(%p): waking\n", OBFUSCATE(IOThreadSelf()) );
3476
3477     return( true );
3478 }
3479
3480 bool IOService::checkResources( void )
3481 {
3482     OSObject *          resourcesProp;
3483     OSSet *             set;
3484     OSIterator *        iter;
3485     bool                ok;
3486
3487     resourcesProp = getProperty( gIOResourceMatchKey );
3488     if( 0 == resourcesProp)
3489         return( true );
3490
3491     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3492
3493         iter = OSCollectionIterator::withCollection( set );
3494         ok = (0 != iter);
3495         while( ok && (resourcesProp = iter->getNextObject()) )
3496             ok = checkResource( resourcesProp );
3497         if( iter)
3498             iter->release();
3499
3500     } else
3501         ok = checkResource( resourcesProp );
3502
3503     return( ok );
3504 }
3505
3506
3507 void _IOConfigThread::configThread( void )
3508 {
3509     _IOConfigThread *   inst;
3510
3511     do {
3512         if( !(inst = new _IOConfigThread))
3513             continue;
3514         if( !inst->init())
3515             continue;
3516         thread_t unused;
3517         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3518             continue;
3519
3520         return;
3521
3522     } while( false);
3523
3524     if( inst)
3525         inst->release();
3526
3527     return;
3528 }
3529
3530 void _IOConfigThread::free( void )
3531 {
3532     thread_deallocate(current_thread());
3533     OSObject::free();
3534 }
3535
3536 void IOService::doServiceMatch( IOOptionBits options )
3537 {
3538     _IOServiceNotifier * notify;
3539     OSIterator *        iter;
3540     OSOrderedSet *      matches;
3541     OSArray *           resourceKeys = 0;
3542     SInt32              catalogGeneration;
3543     bool                keepGuessing = true;
3544     bool                reRegistered = true;
3545     bool                didRegister;
3546
3547 //    job->nub->deliverNotification( gIOPublishNotification,
3548 //                              kIOServiceRegisteredState, 0xffffffff );
3549
3550     while( keepGuessing ) {
3551
3552         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3553         // the matches list should always be created by findDrivers()
3554         if( matches) {
3555
3556             lockForArbitration();
3557             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3558                 getMetaClass()->addInstance(this);
3559                 deliverNotification( gIOFirstPublishNotification,
3560                                      kIOServiceFirstPublishState, 0xffffffff );
3561             }
3562             LOCKREADNOTIFY();
3563             __state[1] &= ~kIOServiceNeedConfigState;
3564             __state[1] |= kIOServiceConfigState;
3565             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3566             __state[0] |= kIOServiceRegisteredState;
3567
3568             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3569             if (reRegistered && keepGuessing) {
3570                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3571                         gNotifications->getObject( gIOPublishNotification ) );
3572                 if( iter) {
3573                     while((notify = (_IOServiceNotifier *)
3574                            iter->getNextObject())) {
3575
3576                         if( matchPassive(notify->matching, 0)
3577                          && (kIOServiceNotifyEnable & notify->state))
3578                             matches->setObject( notify );
3579                     }
3580                     iter->release();
3581                 }
3582             }
3583
3584             UNLOCKNOTIFY();
3585             unlockForArbitration();
3586
3587             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3588             {
3589                 if (this == gIOResources)
3590                 {
3591                    if (resourceKeys) resourceKeys->release();
3592                    resourceKeys = copyPropertyKeys();
3593                 }
3594                 probeCandidates( matches );
3595             }
3596             else
3597                 matches->release();
3598         }
3599
3600         lockForArbitration();
3601         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3602         keepGuessing =
3603                    (reRegistered || (catalogGeneration !=
3604                                         gIOCatalogue->getGenerationCount()))
3605                 && (0 == (__state[0] & kIOServiceInactiveState));
3606
3607         if( keepGuessing)
3608             unlockForArbitration();
3609     }
3610
3611     if( (0 == (__state[0] & kIOServiceInactiveState))
3612      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3613
3614         if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys);
3615
3616         deliverNotification( gIOMatchedNotification,
3617                 kIOServiceMatchedState, 0xffffffff );
3618         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3619             deliverNotification( gIOFirstMatchNotification,
3620                 kIOServiceFirstMatchState, 0xffffffff );
3621     }
3622
3623     if (resourceKeys) resourceKeys->release();
3624
3625     __state[1] &= ~kIOServiceConfigState;
3626     scheduleTerminatePhase2();
3627
3628     _adjustBusy( -1 );
3629     unlockForArbitration();
3630 }
3631
3632 UInt32 IOService::_adjustBusy( SInt32 delta )
3633 {
3634     IOService * next;
3635     UInt32      count;
3636     UInt32      result;
3637     bool        wasQuiet, nowQuiet, needWake;
3638
3639     next = this;
3640     result = __state[1] & kIOServiceBusyStateMask;
3641
3642     if( delta) do {
3643         if( next != this)
3644             next->lockForArbitration();
3645         count = next->__state[1] & kIOServiceBusyStateMask;
3646         wasQuiet = (0 == count);
3647         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3648             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3649         else
3650             count += delta;
3651         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3652         nowQuiet = (0 == count);
3653         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3654
3655         if( needWake) {
3656             next->__state[1] &= ~kIOServiceBusyWaiterState;
3657             IOLockLock( gIOServiceBusyLock );
3658             thread_wakeup( (event_t) next);
3659             IOLockUnlock( gIOServiceBusyLock );
3660         }
3661         if( next != this)
3662             next->unlockForArbitration();
3663
3664         if( (wasQuiet || nowQuiet) ) {
3665
3666             uint64_t regID = next->getRegistryEntryID();
3667             IOServiceTrace(
3668                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3669                 (uintptr_t) regID,
3670                 (uintptr_t) (regID >> 32),
3671                 (uintptr_t) next,
3672                 0);
3673
3674             if (wasQuiet)
3675             {
3676                 next->__timeBusy = mach_absolute_time();
3677             }
3678             else
3679             {
3680                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3681                 next->__timeBusy = 0;
3682             }
3683
3684             MessageClientsContext context;
3685
3686             context.service  = next;
3687             context.type     = kIOMessageServiceBusyStateChange;
3688             context.argument = (void *) wasQuiet;       /*nowBusy*/
3689             context.argSize  = 0;
3690
3691             applyToInterestNotifiers( next, gIOBusyInterest,
3692                                      &messageClientsApplier, &context );
3693
3694 #if !NO_KEXTD
3695             if( nowQuiet && (next == gIOServiceRoot)) {
3696                 OSKext::considerUnloads();
3697                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3698             }
3699 #endif
3700         }
3701
3702         delta = nowQuiet ? -1 : +1;
3703
3704     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3705
3706     return( result );
3707 }
3708
3709 void IOService::adjustBusy( SInt32 delta )
3710 {
3711     lockForArbitration();
3712     _adjustBusy( delta );
3713     unlockForArbitration();
3714 }
3715
3716 uint64_t IOService::getAccumulatedBusyTime( void )
3717 {
3718     uint64_t accumBusy = __accumBusy;
3719     uint64_t timeBusy = __timeBusy;
3720     uint64_t nano;
3721
3722     do
3723     {
3724         accumBusy = __accumBusy;
3725         timeBusy  = __timeBusy;
3726         if (timeBusy)
3727             accumBusy += mach_absolute_time() - timeBusy;
3728     }
3729     while (timeBusy != __timeBusy);
3730
3731     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3732
3733     return (nano);
3734 }
3735
3736 UInt32 IOService::getBusyState( void )
3737 {
3738     return( __state[1] & kIOServiceBusyStateMask );
3739 }
3740
3741 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3742                                   mach_timespec_t * timeout )
3743 {
3744     panic("waitForState");
3745     return (kIOReturnUnsupported);
3746 }
3747
3748 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3749                                    uint64_t timeout )
3750 {
3751     bool            wait;
3752     int             waitResult = THREAD_AWAKENED;
3753     bool            computeDeadline = true;
3754     AbsoluteTime    abstime;
3755
3756     do {
3757         lockForArbitration();
3758         IOLockLock( gIOServiceBusyLock );
3759         wait = (value != (__state[1] & mask));
3760         if( wait) {
3761             __state[1] |= kIOServiceBusyWaiterState;
3762             unlockForArbitration();
3763             if( timeout != UINT64_MAX ) {
3764                 if( computeDeadline ) {
3765                     AbsoluteTime  nsinterval;
3766                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3767                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3768                     computeDeadline = false;
3769                 }
3770                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3771             }
3772             else
3773                 assert_wait((event_t)this, THREAD_UNINT );
3774         } else
3775             unlockForArbitration();
3776         IOLockUnlock( gIOServiceBusyLock );
3777         if( wait)
3778             waitResult = thread_block(THREAD_CONTINUE_NULL);
3779
3780     } while( wait && (waitResult != THREAD_TIMED_OUT));
3781
3782     if( waitResult == THREAD_TIMED_OUT)
3783         return( kIOReturnTimeout );
3784     else
3785         return( kIOReturnSuccess );
3786 }
3787
3788 #if NO_KEXTD
3789 #define WAITING_KEXTD     false
3790 #else
3791 extern bool gIOKextdClearedBusy;
3792 #define WAITING_KEXTD     (false == gIOKextdClearedBusy)
3793 #endif
3794
3795 IOReturn IOService::waitQuiet( uint64_t timeout )
3796 {
3797     IOReturn ret;
3798     uint32_t loops;
3799     char * string = NULL;
3800     size_t len;
3801     uint64_t time;
3802     uint64_t nano;
3803
3804     time = mach_absolute_time();
3805     for (loops = 0; loops < 2; loops++)
3806     {
3807         ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3808
3809         if (loops && (kIOReturnSuccess == ret))
3810         {
3811             time = mach_absolute_time() - time;
3812             absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
3813             IOLog("busy extended ok[%d], (%llds, %llds), kextd wait(%d): %s\n",
3814                   loops, timeout / 1000000000ULL, nano / 1000000000ULL, WAITING_KEXTD,
3815                   string ? string : "");
3816             break;
3817         }
3818         else if (kIOReturnTimeout != ret) break;
3819         else if (timeout < 41000000000)   break;
3820
3821         if (!loops)
3822         {
3823             IORegistryIterator * iter;
3824             OSOrderedSet       * set;
3825             OSOrderedSet       * leaves;
3826             IOService          * next;
3827             IOService          * nextParent;
3828             char               * s;
3829             size_t               l;
3830
3831             len = 256;
3832             string = IONew(char, len);
3833             set = NULL;
3834             iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
3835             leaves = OSOrderedSet::withCapacity(4);
3836             if (iter) set = iter->iterateAll();
3837             if (string && leaves && set)
3838             {
3839                 while ((next = (IOService *) set->getLastObject()))
3840                 {
3841                     if (next->getBusyState())
3842                     {
3843                         leaves->setObject(next);
3844                         nextParent = next;
3845                         while ((nextParent = nextParent->getProvider()))
3846                         {
3847                             set->removeObject(nextParent);
3848                             leaves->removeObject(nextParent);
3849                         }
3850                     }
3851                     set->removeObject(next);
3852                 }
3853                 s = string;
3854                 while ((next = (IOService *) leaves->getLastObject()))
3855                 {
3856                     l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
3857                     if (l >= len) break;
3858                     s += l;
3859                     len -= l;
3860                     leaves->removeObject(next);
3861                 }
3862             }
3863             OSSafeReleaseNULL(leaves);
3864             OSSafeReleaseNULL(set);
3865             OSSafeReleaseNULL(iter);
3866         }
3867         if (loops && (kIOWaitQuietPanics & gIOKitDebug))
3868         {
3869             panic("busy timeout[%d], (%llds), kextd wait(%d): %s",
3870                     loops, timeout / 1000000000ULL, WAITING_KEXTD,
3871                     string ? string : "");
3872         }
3873         else
3874         {
3875             IOLog("busy timeout[%d], (%llds), kextd wait(%d): %s\n",
3876                     loops, timeout / 1000000000ULL, WAITING_KEXTD,
3877                     string ? string : "");
3878         }
3879     }
3880
3881     if (string) IODelete(string, char, 256);
3882
3883     return (ret);
3884 }
3885
3886 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
3887 {
3888     uint64_t    timeoutNS;
3889
3890     if (timeout)
3891     {
3892         timeoutNS = timeout->tv_sec;
3893         timeoutNS *= kSecondScale;
3894         timeoutNS += timeout->tv_nsec;
3895     }
3896     else
3897         timeoutNS = UINT64_MAX;
3898
3899     return (waitQuiet(timeoutNS));
3900 }
3901
3902 bool IOService::serializeProperties( OSSerialize * s ) const
3903 {
3904 #if 0
3905     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
3906                 sizeof( __state), "__state");
3907 #endif
3908     return( super::serializeProperties(s) );
3909 }
3910
3911
3912 void _IOConfigThread::main(void * arg, wait_result_t result)
3913 {
3914     _IOConfigThread * self = (_IOConfigThread *) arg;
3915     _IOServiceJob * job;
3916     IOService   *   nub;
3917     bool            alive = true;
3918     kern_return_t   kr;
3919     thread_precedence_policy_data_t precedence = { -1 };
3920
3921     kr = thread_policy_set(current_thread(),
3922                             THREAD_PRECEDENCE_POLICY,
3923                             (thread_policy_t) &precedence,
3924                             THREAD_PRECEDENCE_POLICY_COUNT);
3925     if (KERN_SUCCESS != kr)
3926         IOLog("thread_policy_set(%d)\n", kr);
3927
3928     do {
3929
3930 //      randomDelay();
3931
3932         semaphore_wait( gJobsSemaphore );
3933
3934         IOTakeLock( gJobsLock );
3935         job = (_IOServiceJob *) gJobs->getFirstObject();
3936         job->retain();
3937         gJobs->removeObject(job);
3938         if( job) {
3939             gOutstandingJobs--;
3940 //          gNumConfigThreads--;        // we're out of service
3941             gNumWaitingThreads--;       // we're out of service
3942         }
3943         IOUnlock( gJobsLock );
3944
3945         if( job) {
3946
3947           nub = job->nub;
3948
3949           if( gIOKitDebug & kIOLogConfig)
3950             LOG("config(%p): starting on %s, %d\n",
3951                         OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
3952
3953           switch( job->type) {
3954
3955             case kMatchNubJob:
3956                 nub->doServiceMatch( job->options );
3957                 break;
3958
3959             default:
3960                 LOG("config(%p): strange type (%d)\n",
3961                         OBFUSCATE(IOThreadSelf()), job->type );
3962                 break;
3963             }
3964
3965             nub->release();
3966             job->release();
3967
3968             IOTakeLock( gJobsLock );
3969             alive = (gOutstandingJobs > gNumWaitingThreads);
3970             if( alive)
3971                 gNumWaitingThreads++;   // back in service
3972 //              gNumConfigThreads++;
3973             else {
3974                 if( 0 == --gNumConfigThreads) {
3975 //                    IOLog("MATCH IDLE\n");
3976                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
3977                 }
3978             }
3979             IOUnlock( gJobsLock );
3980         }
3981
3982     } while( alive );
3983
3984     if( gIOKitDebug & kIOLogConfig)
3985         LOG("config(%p): terminating\n", OBFUSCATE(IOThreadSelf()) );
3986
3987     self->release();
3988 }
3989
3990 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
3991 {
3992     bool            wait;
3993     int             waitResult = THREAD_AWAKENED;
3994     bool            computeDeadline = true;
3995     AbsoluteTime    deadline;
3996
3997     IOLockLock( gJobsLock );
3998     do {
3999         wait = (0 != gNumConfigThreads);
4000         if( wait) {
4001             if( msToWait) {
4002                 if( computeDeadline ) {
4003                     clock_interval_to_deadline(
4004                           msToWait, kMillisecondScale, &deadline );
4005                     computeDeadline = false;
4006                 }
4007                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
4008                                                                 deadline, THREAD_UNINT );
4009                    } else {
4010                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
4011                                                                 THREAD_UNINT );
4012                 }
4013         }
4014     } while( wait && (waitResult != THREAD_TIMED_OUT));
4015         IOLockUnlock( gJobsLock );
4016
4017     if( waitResult == THREAD_TIMED_OUT)
4018         return( kIOReturnTimeout );
4019     else
4020         return( kIOReturnSuccess );
4021 }
4022
4023 void IOService::cpusRunning(void)
4024 {
4025     gCPUsRunning = true;
4026 }
4027
4028 void _IOServiceJob::pingConfig( _IOServiceJob * job )
4029 {
4030     int         count;
4031     bool        create;
4032
4033     assert( job );
4034
4035     IOTakeLock( gJobsLock );
4036
4037     gOutstandingJobs++;
4038     gJobs->setLastObject( job );
4039
4040     count = gNumWaitingThreads;
4041 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
4042
4043     create = (  (gOutstandingJobs > count)
4044                 && ((gNumConfigThreads < kMaxConfigThreads)
4045             || (job->nub == gIOResources)
4046             || !gCPUsRunning));
4047     if( create) {
4048         gNumConfigThreads++;
4049         gNumWaitingThreads++;
4050     }
4051
4052     IOUnlock( gJobsLock );
4053
4054     job->release();
4055
4056     if( create) {
4057         if( gIOKitDebug & kIOLogConfig)
4058             LOG("config(%d): creating\n", gNumConfigThreads - 1);
4059         _IOConfigThread::configThread();
4060     }
4061
4062     semaphore_signal( gJobsSemaphore );
4063 }
4064
4065 struct IOServiceMatchContext
4066 {
4067     OSDictionary * table;
4068     OSObject *     result;
4069     uint32_t       options;
4070     uint32_t       state;
4071     uint32_t       count;
4072     uint32_t       done;
4073 };
4074
4075 bool IOService::instanceMatch(const OSObject * entry, void * context)
4076 {
4077     IOServiceMatchContext * ctx = (typeof(ctx)) context;
4078     IOService *    service = (typeof(service)) entry;
4079     OSDictionary * table   = ctx->table;
4080     uint32_t       options = ctx->options;
4081     uint32_t       state   = ctx->state;
4082     uint32_t       done;
4083     bool           match;
4084
4085     done = 0;
4086     do
4087     {
4088         match = ((state == (state & service->__state[0]))
4089                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
4090         if (!match) break;
4091         ctx->count += table->getCount();
4092         match = service->matchInternal(table, options, &done);
4093         ctx->done += done;
4094     }
4095     while (false);
4096     if (!match)
4097         return (false);
4098
4099     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
4100     {
4101         service->retain();
4102         ctx->result = service;
4103         return (true);
4104     }
4105     else if (!ctx->result)
4106     {
4107         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
4108     }
4109     else
4110     {
4111         ((OSSet *)ctx->result)->setObject(service);
4112     }
4113     return (false);
4114 }
4115
4116 // internal - call with gNotificationLock
4117 OSObject * IOService::copyExistingServices( OSDictionary * matching,
4118                  IOOptionBits inState, IOOptionBits options )
4119 {
4120     OSObject *   current = 0;
4121     OSIterator * iter;
4122     IOService *  service;
4123     OSObject *   obj;
4124     OSString *   str;
4125
4126     if( !matching)
4127         return( 0 );
4128
4129 #if MATCH_DEBUG
4130     OSSerialize * s = OSSerialize::withCapacity(128);
4131     matching->serialize(s);
4132 #endif
4133
4134     if((obj = matching->getObject(gIOProviderClassKey))
4135       && gIOResourcesKey
4136       && gIOResourcesKey->isEqualTo(obj)
4137       && (service = gIOResources))
4138     {
4139         if( (inState == (service->__state[0] & inState))
4140           && (0 == (service->__state[0] & kIOServiceInactiveState))
4141           &&  service->matchPassive(matching, options))
4142         {
4143             if( options & kIONotifyOnce)
4144             {
4145                 service->retain();
4146                 current = service;
4147             }
4148             else
4149                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
4150         }
4151     }
4152     else
4153     {
4154         IOServiceMatchContext ctx;
4155         ctx.table   = matching;
4156         ctx.state   = inState;
4157         ctx.count   = 0;
4158         ctx.done    = 0;
4159         ctx.options = options;
4160         ctx.result  = 0;
4161
4162         if ((str = OSDynamicCast(OSString, obj)))
4163         {
4164             const OSSymbol * sym = OSSymbol::withString(str);
4165             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
4166             sym->release();
4167         }
4168         else
4169         {
4170             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
4171         }
4172
4173
4174         current = ctx.result;
4175
4176         options |= kIOServiceInternalDone | kIOServiceClassDone;
4177         if (current && (ctx.done != ctx.count))
4178         {
4179             OSSet *
4180             source = OSDynamicCast(OSSet, current);
4181             current = 0;
4182             while ((service = (IOService *) source->getAnyObject()))
4183             {
4184                 if (service->matchPassive(matching, options))
4185                 {
4186                     if( options & kIONotifyOnce)
4187                     {
4188                         service->retain();
4189                         current = service;
4190                         break;
4191                     }
4192                     if( current)
4193                     {
4194                         ((OSSet *)current)->setObject( service );
4195                     }
4196                     else
4197                     {
4198                         current = OSSet::withObjects(
4199                                         (const OSObject **) &service, 1, 1 );
4200                     }
4201                 }
4202                 source->removeObject(service);
4203             }
4204             source->release();
4205         }
4206     }
4207
4208 #if MATCH_DEBUG
4209     {
4210         OSObject * _current = 0;
4211
4212         iter = IORegistryIterator::iterateOver( gIOServicePlane,
4213                                             kIORegistryIterateRecursively );
4214         if( iter) {
4215             do {
4216                 iter->reset();
4217                 while( (service = (IOService *) iter->getNextObject())) {
4218                     if( (inState == (service->__state[0] & inState))
4219                     && (0 == (service->__state[0] & kIOServiceInactiveState))
4220                     &&  service->matchPassive(matching, 0)) {
4221
4222                         if( options & kIONotifyOnce) {
4223                             service->retain();
4224                             _current = service;
4225                             break;
4226                         }
4227                         if( _current)
4228                             ((OSSet *)_current)->setObject( service );
4229                         else
4230                             _current = OSSet::withObjects(
4231                                             (const OSObject **) &service, 1, 1 );
4232                     }
4233                 }
4234             } while( !service && !iter->isValid());
4235             iter->release();
4236         }
4237
4238
4239         if ( ((current != 0) != (_current != 0))
4240         || (current && _current && !current->isEqualTo(_current)))
4241         {
4242             OSSerialize * s1 = OSSerialize::withCapacity(128);
4243             OSSerialize * s2 = OSSerialize::withCapacity(128);
4244             current->serialize(s1);
4245             _current->serialize(s2);
4246             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", OBFUSCATE(current),
4247                 OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4248             s1->release();
4249             s2->release();
4250         }
4251
4252         if (_current) _current->release();
4253     }
4254
4255     s->release();
4256 #endif
4257
4258     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4259         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4260         current->release();
4261         current = iter;
4262     }
4263
4264     return( current );
4265 }
4266
4267 // public version
4268 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4269 {
4270     OSIterator *        iter;
4271
4272     // is a lock even needed?
4273     LOCKWRITENOTIFY();
4274
4275     iter = (OSIterator *) copyExistingServices( matching,
4276                                                 kIOServiceMatchedState );
4277
4278     UNLOCKNOTIFY();
4279
4280     return( iter );
4281 }
4282
4283 IOService * IOService::copyMatchingService( OSDictionary * matching )
4284 {
4285     IOService * service;
4286
4287     // is a lock even needed?
4288     LOCKWRITENOTIFY();
4289
4290     service = (IOService *) copyExistingServices( matching,
4291                                                 kIOServiceMatchedState, kIONotifyOnce );
4292
4293     UNLOCKNOTIFY();
4294
4295     return( service );
4296 }
4297
4298 struct _IOServiceMatchingNotificationHandlerRef
4299 {
4300     IOServiceNotificationHandler handler;
4301     void * ref;
4302 };
4303
4304 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4305                                                    IOService * newService,
4306                                                    IONotifier * notifier )
4307 {
4308     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4309 }
4310
4311 // internal - call with gNotificationLock
4312 IONotifier * IOService::setNotification(
4313             const OSSymbol * type, OSDictionary * matching,
4314             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4315             SInt32 priority )
4316 {
4317     _IOServiceNotifier * notify = 0;
4318     OSOrderedSet *      set;
4319
4320     if( !matching)
4321         return( 0 );
4322
4323     notify = new _IOServiceNotifier;
4324     if( notify && !notify->init()) {
4325         notify->release();
4326         notify = 0;
4327     }
4328
4329     if( notify) {
4330         notify->handler = handler;
4331         notify->target = target;
4332         notify->matching = matching;
4333         matching->retain();
4334         if (handler == &_IOServiceMatchingNotificationHandler)
4335         {
4336             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4337             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4338         }
4339         else
4340             notify->ref = ref;
4341         notify->priority = priority;
4342         notify->state = kIOServiceNotifyEnable;
4343         queue_init( &notify->handlerInvocations );
4344
4345         ////// queue
4346
4347         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4348             set = OSOrderedSet::withCapacity( 1,
4349                         IONotifyOrdering, 0 );
4350             if( set) {
4351                 gNotifications->setObject( type, set );
4352                 set->release();
4353             }
4354         }
4355         notify->whence = set;
4356         if( set)
4357             set->setObject( notify );
4358     }
4359
4360     return( notify );
4361 }
4362
4363 // internal - call with gNotificationLock
4364 IONotifier * IOService::doInstallNotification(
4365                         const OSSymbol * type, OSDictionary * matching,
4366                         IOServiceMatchingNotificationHandler handler,
4367                         void * target, void * ref,
4368                         SInt32 priority, OSIterator ** existing )
4369 {
4370     OSIterator *        exist;
4371     IONotifier *        notify;
4372     IOOptionBits        inState;
4373
4374     if( !matching)
4375         return( 0 );
4376
4377     if( type == gIOPublishNotification)
4378         inState = kIOServiceRegisteredState;
4379
4380     else if( type == gIOFirstPublishNotification)
4381         inState = kIOServiceFirstPublishState;
4382
4383     else if (type == gIOMatchedNotification)
4384         inState = kIOServiceMatchedState;
4385
4386     else if (type == gIOFirstMatchNotification)
4387         inState = kIOServiceFirstMatchState;
4388
4389     else if( type == gIOTerminatedNotification)
4390         inState = 0;
4391     else
4392         return( 0 );
4393
4394     notify = setNotification( type, matching, handler, target, ref, priority );
4395
4396     if( inState)
4397         // get the current set
4398         exist = (OSIterator *) copyExistingServices( matching, inState );
4399     else
4400         exist = 0;
4401
4402     *existing = exist;
4403
4404     return( notify );
4405 }
4406
4407 #if !defined(__LP64__)
4408 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4409                                         IOServiceNotificationHandler handler,
4410                                         void * target, void * refCon,
4411                                         SInt32 priority, OSIterator ** existing )
4412 {
4413     IONotifier * result;
4414     _IOServiceMatchingNotificationHandlerRef ref;
4415     ref.handler = handler;
4416     ref.ref     = refCon;
4417
4418     result = (_IOServiceNotifier *) installNotification( type, matching,
4419                          &_IOServiceMatchingNotificationHandler,
4420                         target, &ref, priority, existing );
4421     if (result)
4422         matching->release();
4423
4424     return (result);
4425 }
4426 #endif /* !defined(__LP64__) */
4427
4428
4429 IONotifier * IOService::installNotification(
4430                         const OSSymbol * type, OSDictionary * matching,
4431                         IOServiceMatchingNotificationHandler handler,
4432                         void * target, void * ref,
4433                         SInt32 priority, OSIterator ** existing )
4434 {
4435     IONotifier * notify;
4436
4437     LOCKWRITENOTIFY();
4438
4439     notify = doInstallNotification( type, matching, handler, target, ref,
4440                 priority, existing );
4441
4442     // in case handler remove()s
4443     if (notify) notify->retain();
4444
4445     UNLOCKNOTIFY();
4446
4447     return( notify );
4448 }
4449
4450 IONotifier * IOService::addNotification(
4451                         const OSSymbol * type, OSDictionary * matching,
4452                         IOServiceNotificationHandler handler,
4453                         void * target, void * refCon,
4454                         SInt32 priority )
4455 {
4456     IONotifier * result;
4457     _IOServiceMatchingNotificationHandlerRef ref;
4458
4459     ref.handler = handler;
4460     ref.ref     = refCon;
4461
4462     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4463                             target, &ref, priority);
4464
4465     if (result)
4466         matching->release();
4467
4468     return (result);
4469 }
4470
4471 IONotifier * IOService::addMatchingNotification(
4472                         const OSSymbol * type, OSDictionary * matching,
4473                         IOServiceMatchingNotificationHandler handler,
4474                         void * target, void * ref,
4475                         SInt32 priority )
4476 {
4477     OSIterator *                existing = NULL;
4478     IONotifier *                ret;
4479     _IOServiceNotifier *        notify;
4480     IOService *                 next;
4481
4482     ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
4483                 handler, target, ref, priority, &existing );
4484     if (!ret) return (0);
4485
4486     // send notifications for existing set
4487     if (existing) {
4488
4489         while( (next = (IOService *) existing->getNextObject())) {
4490
4491             next->lockForArbitration();
4492             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4493                 next->invokeNotifer( notify );
4494             next->unlockForArbitration();
4495         }
4496         existing->release();
4497     }
4498
4499     LOCKWRITENOTIFY();
4500     bool removed = (0 == notify->whence);
4501     notify->release();
4502     if (removed) ret = gIOServiceNullNotifier;
4503     UNLOCKNOTIFY();
4504
4505     return( ret );
4506 }
4507
4508 bool IOService::syncNotificationHandler(
4509                         void * /* target */, void * ref,
4510                         IOService * newService,
4511                         IONotifier * notifier )
4512 {
4513
4514     LOCKWRITENOTIFY();
4515     if (!*((IOService **) ref))
4516     {
4517         newService->retain();
4518         (*(IOService **) ref) = newService;
4519         WAKEUPNOTIFY(ref);
4520     }
4521     UNLOCKNOTIFY();
4522
4523     return( false );
4524 }
4525
4526 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4527                                                 uint64_t timeout)
4528 {
4529     IONotifier *        notify = 0;
4530     // priority doesn't help us much since we need a thread wakeup
4531     SInt32              priority = 0;
4532     IOService *         result;
4533
4534     if (!matching)
4535         return( 0 );
4536
4537     result = NULL;
4538
4539     LOCKWRITENOTIFY();
4540     do
4541     {
4542         result = (IOService *) copyExistingServices( matching,
4543                             kIOServiceMatchedState, kIONotifyOnce );
4544         if (result)
4545             break;
4546         notify = IOService::setNotification( gIOMatchedNotification, matching,
4547                     &IOService::syncNotificationHandler, (void *) 0,
4548                     &result, priority );
4549          if (!notify)
4550             break;
4551         if (UINT64_MAX != timeout)
4552         {
4553             AbsoluteTime deadline;
4554             nanoseconds_to_absolutetime(timeout, &deadline);
4555             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4556             SLEEPNOTIFYTO(&result, deadline);
4557         }
4558         else
4559         {
4560             SLEEPNOTIFY(&result);
4561         }
4562     }
4563     while( false );
4564
4565     UNLOCKNOTIFY();
4566
4567     if (notify)
4568         notify->remove();       // dequeues
4569
4570     return( result );
4571 }
4572
4573 IOService * IOService::waitForService( OSDictionary * matching,
4574                                         mach_timespec_t * timeout )
4575 {
4576     IOService * result;
4577     uint64_t    timeoutNS;
4578
4579     if (timeout)
4580     {
4581         timeoutNS = timeout->tv_sec;
4582         timeoutNS *= kSecondScale;
4583         timeoutNS += timeout->tv_nsec;
4584     }
4585     else
4586         timeoutNS = UINT64_MAX;
4587
4588     result = waitForMatchingService(matching, timeoutNS);
4589
4590     matching->release();
4591     if (result)
4592         result->release();
4593
4594     return (result);
4595 }
4596
4597 void IOService::deliverNotification( const OSSymbol * type,
4598                             IOOptionBits orNewState, IOOptionBits andNewState )
4599 {
4600     _IOServiceNotifier * notify;
4601     OSIterator *         iter;
4602     OSArray *            willSend = 0;
4603
4604     lockForArbitration();
4605
4606     if( (0 == (__state[0] & kIOServiceInactiveState))
4607      || (type == gIOTerminatedNotification)) {
4608
4609         LOCKREADNOTIFY();
4610
4611         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4612                     gNotifications->getObject( type ) );
4613
4614         if( iter) {
4615             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4616
4617                 if( matchPassive(notify->matching, 0)
4618                   && (kIOServiceNotifyEnable & notify->state)) {
4619                     if( 0 == willSend)
4620                         willSend = OSArray::withCapacity(8);
4621                     if( willSend)
4622                         willSend->setObject( notify );
4623                 }
4624             }
4625             iter->release();
4626         }
4627
4628         __state[0] = (__state[0] | orNewState) & andNewState;
4629
4630         UNLOCKNOTIFY();
4631     }
4632
4633     if( willSend) {
4634         for( unsigned int idx = 0;
4635              (notify = (_IOServiceNotifier *) willSend->getObject(idx));
4636              idx++) {
4637             invokeNotifer( notify );
4638         }
4639         willSend->release();
4640     }
4641     unlockForArbitration();
4642 }
4643
4644 IOOptionBits IOService::getState( void ) const
4645 {
4646     return( __state[0] );
4647 }
4648
4649 /*
4650  * Helpers to make matching objects for simple cases
4651  */
4652
4653 OSDictionary * IOService::serviceMatching( const OSString * name,
4654                         OSDictionary * table )
4655 {
4656
4657     const OSString *    str;
4658
4659     str = OSSymbol::withString(name);
4660     if( !str)
4661         return( 0 );
4662
4663     if( !table)
4664         table = OSDictionary::withCapacity( 2 );
4665     if( table)
4666         table->setObject(gIOProviderClassKey, (OSObject *)str );
4667     str->release();
4668
4669     return( table );
4670 }
4671
4672 OSDictionary * IOService::serviceMatching( const char * name,
4673                         OSDictionary * table )
4674 {
4675     const OSString *    str;
4676
4677     str = OSSymbol::withCString( name );
4678     if( !str)
4679         return( 0 );
4680
4681     table = serviceMatching( str, table );
4682     str->release();
4683     return( table );
4684 }
4685
4686 OSDictionary * IOService::nameMatching( const OSString * name,
4687                         OSDictionary * table )
4688 {
4689     if( !table)
4690         table = OSDictionary::withCapacity( 2 );
4691     if( table)
4692         table->setObject( gIONameMatchKey, (OSObject *)name );
4693
4694     return( table );
4695 }
4696
4697 OSDictionary * IOService::nameMatching( const char * name,
4698                         OSDictionary * table )
4699 {
4700     const OSString *    str;
4701
4702     str = OSSymbol::withCString( name );
4703     if( !str)
4704         return( 0 );
4705
4706     table = nameMatching( str, table );
4707     str->release();
4708     return( table );
4709 }
4710
4711 OSDictionary * IOService::resourceMatching( const OSString * str,
4712                         OSDictionary * table )
4713 {
4714     table = serviceMatching( gIOResourcesKey, table );
4715     if( table)
4716         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4717
4718     return( table );
4719 }
4720
4721 OSDictionary * IOService::resourceMatching( const char * name,
4722                         OSDictionary * table )
4723 {
4724     const OSSymbol *    str;
4725
4726     str = OSSymbol::withCString( name );
4727     if( !str)
4728         return( 0 );
4729
4730     table = resourceMatching( str, table );
4731     str->release();
4732
4733     return( table );
4734 }
4735
4736 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4737                         OSDictionary * table )
4738 {
4739     OSDictionary * properties;
4740
4741     properties = OSDictionary::withCapacity( 2 );
4742     if( !properties)
4743         return( 0 );
4744     properties->setObject( key, value );
4745
4746     if( !table)
4747         table = OSDictionary::withCapacity( 2 );
4748     if( table)
4749         table->setObject( gIOPropertyMatchKey, properties );
4750
4751     properties->release();
4752
4753     return( table );
4754 }
4755
4756 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4757                         OSDictionary * table )
4758 {
4759     OSNumber *     num;
4760
4761     num = OSNumber::withNumber( entryID, 64 );
4762     if( !num)
4763         return( 0 );
4764
4765     if( !table)
4766         table = OSDictionary::withCapacity( 2 );
4767     if( table)
4768         table->setObject( gIORegistryEntryIDKey, num );
4769
4770     if (num)
4771         num->release();
4772
4773     return( table );
4774 }
4775
4776
4777 /*
4778  * _IOServiceNotifier
4779  */
4780
4781 // wait for all threads, other than the current one,
4782 //  to exit the handler
4783
4784 void _IOServiceNotifier::wait()
4785 {
4786     _IOServiceNotifierInvocation * next;
4787     bool doWait;
4788
4789     do {
4790         doWait = false;
4791         queue_iterate( &handlerInvocations, next,
4792                         _IOServiceNotifierInvocation *, link) {
4793             if( next->thread != current_thread() ) {
4794                 doWait = true;
4795                 break;
4796             }
4797         }
4798         if( doWait) {
4799             state |= kIOServiceNotifyWaiter;
4800             SLEEPNOTIFY(this);
4801         }
4802
4803     } while( doWait );
4804 }
4805
4806 void _IOServiceNotifier::free()
4807 {
4808     assert( queue_empty( &handlerInvocations ));
4809     OSObject::free();
4810 }
4811
4812 void _IOServiceNotifier::remove()
4813 {
4814     LOCKWRITENOTIFY();
4815
4816     if( whence) {
4817         whence->removeObject( (OSObject *) this );
4818         whence = 0;
4819     }
4820     if( matching) {
4821         matching->release();
4822         matching = 0;
4823     }
4824
4825     state &= ~kIOServiceNotifyEnable;
4826
4827     wait();
4828
4829     UNLOCKNOTIFY();
4830
4831     release();
4832 }
4833
4834 bool _IOServiceNotifier::disable()
4835 {
4836     bool        ret;
4837
4838     LOCKWRITENOTIFY();
4839
4840     ret = (0 != (kIOServiceNotifyEnable & state));
4841     state &= ~kIOServiceNotifyEnable;
4842     if( ret)
4843         wait();
4844
4845     UNLOCKNOTIFY();
4846
4847     return( ret );
4848 }
4849
4850 void _IOServiceNotifier::enable( bool was )
4851 {
4852     LOCKWRITENOTIFY();
4853     if( was)
4854         state |= kIOServiceNotifyEnable;
4855     else
4856         state &= ~kIOServiceNotifyEnable;
4857     UNLOCKNOTIFY();
4858 }
4859
4860
4861 /*
4862  * _IOServiceNullNotifier
4863  */
4864
4865 void _IOServiceNullNotifier::taggedRetain(const void *tag) const                      {}
4866 void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const     {}
4867 void _IOServiceNullNotifier::free()                                                   {}
4868 void _IOServiceNullNotifier::wait()                                                   {}
4869 void _IOServiceNullNotifier::remove()                                                 {}
4870 void _IOServiceNullNotifier::enable(bool was)                                         {}
4871 bool _IOServiceNullNotifier::disable()                                { return(false); }
4872
4873 /*
4874  * IOResources
4875  */
4876
4877 IOService * IOResources::resources( void )
4878 {
4879     IOResources *       inst;
4880
4881     inst = new IOResources;
4882     if( inst && !inst->init()) {
4883         inst->release();
4884         inst = 0;
4885     }
4886
4887     return( inst );
4888 }
4889
4890 bool IOResources::init( OSDictionary * dictionary )
4891 {
4892     // Do super init first
4893     if ( !IOService::init() )
4894         return false;
4895
4896     // Allow PAL layer to publish a value
4897     const char *property_name;
4898     int property_value;
4899
4900     pal_get_resource_property( &property_name, &property_value );
4901
4902     if( property_name ) {
4903         OSNumber *num;
4904         const OSSymbol *        sym;
4905
4906         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
4907             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
4908                 this->setProperty( sym, num );
4909                 sym->release();
4910             }
4911             num->release();
4912         }
4913     }
4914
4915     return true;
4916 }
4917
4918 IOReturn IOResources::newUserClient(task_t owningTask, void * securityID,
4919                                     UInt32 type,  OSDictionary * properties,
4920                                     IOUserClient ** handler)
4921 {
4922     return( kIOReturnUnsupported );
4923 }
4924
4925 IOWorkLoop * IOResources::getWorkLoop() const
4926 {
4927     // If we are the resource root
4928     // then use the platform's workloop
4929     if (this == (IOResources *) gIOResources)
4930         return getPlatform()->getWorkLoop();
4931     else
4932         return IOService::getWorkLoop();
4933 }
4934
4935 bool IOResources::matchPropertyTable( OSDictionary * table )
4936 {
4937     OSObject *          prop;
4938     OSString *          str;
4939     OSSet *             set;
4940     OSIterator *        iter;
4941     OSArray *           keys;
4942     bool                ok = true;
4943
4944     prop = table->getObject( gIOResourceMatchKey );
4945     str = OSDynamicCast( OSString, prop );
4946     if( str)
4947         ok = (0 != getProperty( str ));
4948
4949     else if( (set = OSDynamicCast( OSSet, prop))) {
4950
4951         iter = OSCollectionIterator::withCollection( set );
4952         ok = (iter != 0);
4953         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
4954             ok = (0 != getProperty( str ));
4955
4956         if( iter)
4957             iter->release();
4958     }
4959     else if ((prop = table->getObject(gIOResourceMatchedKey)))
4960     {
4961         keys = (OSArray *) copyProperty(gIOResourceMatchedKey);
4962         ok = false;
4963         if (keys)
4964         {
4965             // assuming OSSymbol
4966             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
4967             keys->release();
4968         }
4969     }
4970
4971     return( ok );
4972 }
4973
4974 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
4975 {
4976     IOService::updateConsoleUsers(NULL, 0);
4977 }
4978
4979 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
4980 {
4981     IORegistryEntry * regEntry;
4982     OSObject *        locked = kOSBooleanFalse;
4983     uint32_t          idx;
4984     bool              publish;
4985     OSDictionary *    user;
4986     static IOMessage  sSystemPower;
4987     clock_sec_t       now = 0;
4988     clock_usec_t      microsecs;
4989
4990     regEntry = IORegistryEntry::getRegistryRoot();
4991
4992     if (!gIOChosenEntry)
4993         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
4994
4995     IOLockLock(gIOConsoleUsersLock);
4996
4997     if (systemMessage)
4998     {
4999         sSystemPower = systemMessage;
5000 #if HIBERNATION
5001         if (kIOMessageSystemHasPoweredOn == systemMessage)
5002         {
5003             uint32_t lockState = IOHibernateWasScreenLocked();
5004             switch (lockState)
5005             {
5006                 case 0:
5007                     break;
5008                 case kIOScreenLockLocked:
5009                 case kIOScreenLockFileVaultDialog:
5010                     gIOConsoleBooterLockState = kOSBooleanTrue;
5011                     break;
5012                 case kIOScreenLockNoLock:
5013                     gIOConsoleBooterLockState = 0;
5014                     break;
5015                 case kIOScreenLockUnlocked:
5016                 default:
5017                     gIOConsoleBooterLockState = kOSBooleanFalse;
5018                     break;
5019             }
5020         }
5021 #endif /* HIBERNATION */
5022     }
5023
5024     if (consoleUsers)
5025     {
5026         OSNumber * num = 0;
5027         bool       loginLocked = true;
5028
5029         gIOConsoleLoggedIn = false;
5030         for (idx = 0;
5031               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
5032               idx++)
5033         {
5034             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
5035                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
5036
5037             loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
5038             if (!num)
5039             {
5040                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
5041             }
5042         }
5043 #if HIBERNATION
5044         if (!loginLocked) gIOConsoleBooterLockState = 0;
5045         IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
5046                 (num != 0), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
5047                 gIOConsoleLoggedIn, loginLocked);
5048 #endif /* HIBERNATION */
5049         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
5050     }
5051
5052     if (!gIOConsoleLoggedIn
5053      || (kIOMessageSystemWillSleep == sSystemPower)
5054      || (kIOMessageSystemPagingOff == sSystemPower))
5055     {
5056         locked = kOSBooleanTrue;
5057     }
5058 #if HIBERNATION
5059     else if (gIOConsoleBooterLockState)
5060     {
5061         locked = gIOConsoleBooterLockState;
5062     }
5063 #endif /* HIBERNATION */
5064     else if (gIOConsoleLockTime)
5065     {
5066         clock_get_calendar_microtime(&now, &microsecs);
5067         if (gIOConsoleLockTime > now)
5068         {
5069             AbsoluteTime deadline;
5070             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
5071             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
5072         }
5073         else
5074         {
5075             locked = kOSBooleanTrue;
5076         }
5077     }
5078
5079     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
5080     if (publish)
5081     {
5082         regEntry->setProperty(gIOConsoleLockedKey, locked);
5083         if (consoleUsers)
5084         {
5085             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
5086         }
5087         OSIncrementAtomic( &gIOConsoleUsersSeed );
5088     }
5089
5090 #if HIBERNATION
5091     if (gIOChosenEntry)
5092     {
5093         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
5094         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
5095         else                          gIOScreenLockState = kIOScreenLockNoLock;
5096         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
5097
5098         IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
5099                 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != 0), now, systemMessage);
5100     }
5101 #endif /* HIBERNATION */
5102
5103     IOLockUnlock(gIOConsoleUsersLock);
5104
5105     if (publish)
5106     {
5107         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
5108
5109         MessageClientsContext context;
5110
5111         context.service  = getServiceRoot();
5112         context.type     = kIOMessageConsoleSecurityChange;
5113         context.argument = (void *) regEntry;
5114         context.argSize  = 0;
5115
5116         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
5117                                  &messageClientsApplier, &context );
5118     }
5119 }
5120
5121 IOReturn IOResources::setProperties( OSObject * properties )
5122 {
5123     IOReturn                    err;
5124     const OSSymbol *            key;
5125     OSDictionary *              dict;
5126     OSCollectionIterator *      iter;
5127
5128     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
5129     if ( kIOReturnSuccess != err)
5130         return( err );
5131
5132     dict = OSDynamicCast(OSDictionary, properties);
5133     if( 0 == dict)
5134         return( kIOReturnBadArgument);
5135
5136     iter = OSCollectionIterator::withCollection( dict);
5137     if( 0 == iter)
5138         return( kIOReturnBadArgument);
5139
5140     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
5141     {
5142         if (gIOConsoleUsersKey == key) do
5143         {
5144             OSArray * consoleUsers;
5145             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
5146             if (!consoleUsers)
5147                 continue;
5148             IOService::updateConsoleUsers(consoleUsers, 0);
5149         }
5150         while (false);
5151
5152         publishResource( key, dict->getObject(key) );
5153     }
5154
5155     iter->release();
5156
5157     return( kIOReturnSuccess );
5158 }
5159
5160 /*
5161  * Helpers for matching dictionaries.
5162  * Keys existing in matching are checked in properties.
5163  * Keys may be a string or OSCollection of IOStrings
5164  */
5165
5166 bool IOService::compareProperty( OSDictionary * matching,
5167                                  const char *   key )
5168 {
5169     OSObject *  value;
5170     OSObject *  prop;
5171     bool        ok;
5172
5173     value = matching->getObject( key );
5174     if( value)
5175     {
5176         prop = copyProperty(key);
5177         ok = value->isEqualTo(prop);
5178         if (prop) prop->release();
5179     }
5180     else
5181         ok = true;
5182
5183     return( ok );
5184 }
5185
5186
5187 bool IOService::compareProperty( OSDictionary *   matching,
5188                                  const OSString * key )
5189 {
5190     OSObject *  value;
5191     OSObject *  prop;
5192     bool        ok;
5193
5194     value = matching->getObject( key );
5195     if( value)
5196     {
5197         prop = copyProperty(key);
5198         ok = value->isEqualTo(prop);
5199         if (prop) prop->release();
5200     }
5201     else
5202         ok = true;
5203
5204     return( ok );
5205 }
5206
5207 bool IOService::compareProperties( OSDictionary * matching,
5208                                    OSCollection * keys )
5209 {
5210     OSCollectionIterator *      iter;
5211     const OSString *            key;
5212     bool                        ok = true;
5213
5214     if( !matching || !keys)
5215         return( false );
5216
5217     iter = OSCollectionIterator::withCollection( keys );
5218
5219     if( iter) {
5220         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
5221             ok = compareProperty( matching, key );
5222
5223         iter->release();
5224     }
5225     keys->release();    // !! consume a ref !!
5226
5227     return( ok );
5228 }
5229
5230 /* Helper to add a location matching dict to the table */
5231
5232 OSDictionary * IOService::addLocation( OSDictionary * table )
5233 {
5234     OSDictionary *      dict;
5235
5236     if( !table)
5237         return( 0 );
5238
5239     dict = OSDictionary::withCapacity( 1 );
5240     if( dict) {
5241         table->setObject( gIOLocationMatchKey, dict );
5242         dict->release();
5243     }
5244
5245     return( dict );
5246 }
5247
5248 /*
5249  * Go looking for a provider to match a location dict.
5250  */
5251
5252 IOService * IOService::matchLocation( IOService * /* client */ )
5253 {
5254     IOService * parent;
5255
5256     parent = getProvider();
5257
5258     if( parent)
5259         parent = parent->matchLocation( this );
5260
5261     return( parent );
5262 }
5263
5264 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
5265 {
5266     OSString *          matched;
5267     OSObject *          obj;
5268     OSString *          str;
5269     IORegistryEntry *   entry;
5270     OSNumber *          num;
5271     bool                match = true;
5272     bool                changesOK = (0 != (kIOServiceChangesOK & options));
5273     uint32_t            count;
5274     uint32_t            done;
5275
5276     do
5277     {
5278         count = table->getCount();
5279         done = 0;
5280
5281         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
5282         if (str) {
5283             done++;
5284             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
5285 #if MATCH_DEBUG
5286             match = (0 != metaCast( str ));
5287             if ((kIOServiceClassDone & options) && !match) panic("classDone");
5288 #endif
5289             if ((!match) || (done == count)) break;
5290         }
5291
5292         obj = table->getObject( gIONameMatchKey );
5293         if( obj) {
5294             done++;
5295             match = compareNames( obj, changesOK ? &matched : 0 );
5296             if (!match) break;
5297             if( changesOK && matched) {
5298                 // leave a hint as to which name matched
5299                 table->setObject( gIONameMatchedKey, matched );
5300                 matched->release();
5301             }
5302             if (done == count) break;
5303         }
5304
5305         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
5306         if (str)
5307         {
5308             const OSSymbol * sym;
5309             done++;
5310             match = false;
5311             sym = copyLocation();
5312             if (sym) {
5313                 match = sym->isEqualTo( str );
5314                 sym->release();
5315             }
5316             if ((!match) || (done == count)) break;
5317         }
5318
5319         obj = table->getObject( gIOPropertyMatchKey );
5320         if( obj)
5321         {
5322             OSDictionary * dict;
5323             OSDictionary * nextDict;
5324             OSIterator *   iter;
5325             done++;
5326             match = false;
5327             dict = dictionaryWithProperties();
5328             if( dict) {
5329                 nextDict = OSDynamicCast( OSDictionary, obj);
5330                 if( nextDict)
5331                     iter = 0;
5332                 else
5333                     iter = OSCollectionIterator::withCollection(
5334                                 OSDynamicCast(OSCollection, obj));
5335
5336                 while( nextDict
5337                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5338                                             iter->getNextObject()))))) {
5339                     match = dict->isEqualTo( nextDict, nextDict);
5340                     if( match)
5341                         break;
5342                     nextDict = 0;
5343                 }
5344                 dict->release();
5345                 if( iter)
5346                     iter->release();
5347             }
5348             if ((!match) || (done == count)) break;
5349         }
5350
5351         obj = table->getObject( gIOPropertyExistsMatchKey );
5352         if( obj)
5353         {
5354             OSDictionary * dict;
5355             OSString     * nextKey;
5356             OSIterator *   iter;
5357             done++;
5358             match = false;
5359             dict = dictionaryWithProperties();
5360             if( dict) {
5361                 nextKey = OSDynamicCast( OSString, obj);
5362                 if( nextKey)
5363                     iter = 0;
5364                 else
5365                     iter = OSCollectionIterator::withCollection(
5366                                 OSDynamicCast(OSCollection, obj));
5367
5368                 while( nextKey
5369                     || (iter && (0 != (nextKey = OSDynamicCast(OSString,
5370                                             iter->getNextObject()))))) {
5371                     match = (0 != dict->getObject(nextKey));
5372                     if( match)
5373                         break;
5374                     nextKey = 0;
5375                 }
5376                 dict->release();
5377                 if( iter)
5378                     iter->release();
5379             }
5380             if ((!match) || (done == count)) break;
5381         }
5382
5383         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5384         if( str) {
5385             done++;
5386             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5387             match = (this == entry);
5388             if( entry)
5389                 entry->release();
5390             if ((!match) || (done == count)) break;
5391         }
5392
5393         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5394         if (num) {
5395             done++;
5396             match = (getRegistryEntryID() == num->unsigned64BitValue());
5397             if ((!match) || (done == count)) break;
5398         }
5399
5400         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5401         if( num)
5402         {
5403             OSIterator *        iter;
5404             IOService *         service = 0;
5405             UInt32              serviceCount = 0;
5406
5407             done++;
5408             iter = getClientIterator();
5409             if( iter) {
5410                 while( (service = (IOService *) iter->getNextObject())) {
5411                     if( kIOServiceInactiveState & service->__state[0])
5412                         continue;
5413                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5414                         continue;
5415                     ++serviceCount;
5416                 }
5417                 iter->release();
5418             }
5419             match = (serviceCount == num->unsigned32BitValue());
5420             if ((!match) || (done == count)) break;
5421         }
5422
5423 #define propMatch(key)                                  \
5424         obj = table->getObject(key);                    \
5425         if (obj)                                        \
5426         {                                               \
5427             OSObject * prop;                            \
5428             done++;                                     \
5429             prop = copyProperty(key);                   \
5430             match = obj->isEqualTo(prop);               \
5431             if (prop) prop->release();                  \
5432             if ((!match) || (done == count)) break;     \
5433         }
5434         propMatch(gIOBSDNameKey)
5435         propMatch(gIOBSDMajorKey)
5436         propMatch(gIOBSDMinorKey)
5437         propMatch(gIOBSDUnitKey)
5438 #undef propMatch
5439     }
5440     while (false);
5441
5442     if (did) *did = done;
5443     return (match);
5444 }
5445
5446 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5447 {
5448     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5449 }
5450
5451 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5452 {
5453     IOService *         where;
5454     OSDictionary *      nextTable;
5455     SInt32              score;
5456     OSNumber *          newPri;
5457     bool                match = true;
5458     bool                matchParent = false;
5459     uint32_t            count;
5460     uint32_t            done;
5461
5462     assert( table );
5463
5464     OSArray* aliasServiceRegIds = NULL;
5465     IOService* foundAlternateService = NULL;
5466
5467 #if MATCH_DEBUG
5468     OSDictionary * root = table;
5469 #endif
5470
5471     where = this;
5472     do
5473     {
5474         do
5475         {
5476             count = table->getCount();
5477             if (!(kIOServiceInternalDone & options))
5478             {
5479                 match = where->matchInternal(table, options, &done);
5480                 // don't call family if we've done all the entries in the table
5481                 if ((!match) || (done == count)) break;
5482             }
5483
5484             // pass in score from property table
5485             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5486
5487             // do family specific matching
5488             match = where->matchPropertyTable( table, &score );
5489
5490             if( !match) {
5491 #if IOMATCHDEBUG
5492                 if( kIOLogMatch & getDebugFlags( table ))
5493                     LOG("%s: family specific matching fails\n", where->getName());
5494 #endif
5495                 break;
5496             }
5497
5498             if (kIOServiceChangesOK & options) {
5499                 // save the score
5500                 newPri = OSNumber::withNumber( score, 32 );
5501                 if( newPri) {
5502                     table->setObject( gIOProbeScoreKey, newPri );
5503                     newPri->release();
5504                 }
5505             }
5506
5507             options = 0;
5508             matchParent = false;
5509
5510             nextTable = OSDynamicCast(OSDictionary,
5511                   table->getObject( gIOParentMatchKey ));
5512             if( nextTable) {
5513                 // look for a matching entry anywhere up to root
5514                 match = false;
5515                 matchParent = true;
5516                 table = nextTable;
5517                 break;
5518             }
5519
5520             table = OSDynamicCast(OSDictionary,
5521                     table->getObject( gIOLocationMatchKey ));
5522             if (table) {
5523                 // look for a matching entry at matchLocation()
5524                 match = false;
5525                 where = where->getProvider();
5526                 if (where && (where = where->matchLocation(where))) continue;
5527             }
5528             break;
5529         }
5530         while (true);
5531
5532         if(match == true) {
5533             break;
5534         }
5535
5536         if(matchParent == true) {
5537             // check if service has an alias to search its other "parents" if a parent match isn't found
5538             OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, where->getProperty(kIOServiceLegacyMatchingRegistryIDKey));
5539             if(alternateRegistryID != NULL) {
5540                 if(aliasServiceRegIds == NULL)
5541                 {
5542                     aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
5543                 }
5544                 aliasServiceRegIds->setObject(alternateRegistryID);
5545             }
5546         }
5547         else {
5548             break;
5549         }
5550
5551         where = where->getProvider();
5552         if(where == NULL) {
5553             // there were no matching parent services, check to see if there are aliased services that have a matching parent
5554             if(aliasServiceRegIds != NULL) {
5555                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
5556                 if(numAliasedServices != 0) {
5557                     OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
5558                     if(alternateRegistryID != NULL) {
5559                         OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
5560                         aliasServiceRegIds->removeObject(numAliasedServices - 1);
5561                         if(alternateMatchingDict != NULL) {
5562                             OSSafeReleaseNULL(foundAlternateService);
5563                             foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
5564                             alternateMatchingDict->release();
5565                             if(foundAlternateService != NULL) {
5566                                 where = foundAlternateService;
5567                             }
5568                         }
5569                     }
5570                 }
5571             }
5572         }
5573     }
5574     while( where != NULL );
5575
5576     OSSafeReleaseNULL(foundAlternateService);
5577     OSSafeReleaseNULL(aliasServiceRegIds);
5578
5579 #if MATCH_DEBUG
5580     if (where != this)
5581     {
5582         OSSerialize * s = OSSerialize::withCapacity(128);
5583         root->serialize(s);
5584         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5585         s->release();
5586     }
5587 #endif
5588
5589     return( match );
5590 }
5591
5592
5593 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5594                                     UInt32 type,  OSDictionary * properties,
5595                                     IOUserClient ** handler )
5596 {
5597     const OSSymbol *userClientClass = 0;
5598     IOUserClient *client;
5599     OSObject *temp;
5600
5601     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5602         return kIOReturnSuccess;
5603
5604     // First try my own properties for a user client class name
5605     temp = getProperty(gIOUserClientClassKey);
5606     if (temp) {
5607         if (OSDynamicCast(OSSymbol, temp))
5608             userClientClass = (const OSSymbol *) temp;
5609         else if (OSDynamicCast(OSString, temp)) {
5610             userClientClass = OSSymbol::withString((OSString *) temp);
5611             if (userClientClass)
5612                 setProperty(kIOUserClientClassKey,
5613                             (OSObject *) userClientClass);
5614         }
5615     }
5616
5617     // Didn't find one so lets just bomb out now without further ado.
5618     if (!userClientClass)
5619         return kIOReturnUnsupported;
5620
5621     // This reference is consumed by the IOServiceOpen call
5622     temp = OSMetaClass::allocClassWithName(userClientClass);
5623     if (!temp)
5624         return kIOReturnNoMemory;
5625
5626     if (OSDynamicCast(IOUserClient, temp))
5627         client = (IOUserClient *) temp;
5628     else {
5629         temp->release();
5630         return kIOReturnUnsupported;
5631     }
5632
5633     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5634         client->release();
5635         return kIOReturnBadArgument;
5636     }
5637
5638     if ( !client->attach(this) ) {
5639         client->release();
5640         return kIOReturnUnsupported;
5641     }
5642
5643     if ( !client->start(this) ) {
5644         client->detach(this);
5645         client->release();
5646         return kIOReturnUnsupported;
5647     }
5648
5649     *handler = client;
5650     return kIOReturnSuccess;
5651 }
5652
5653 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5654                                     UInt32 type, IOUserClient ** handler )
5655 {
5656     return( kIOReturnUnsupported );
5657 }
5658
5659 IOReturn IOService::requestProbe( IOOptionBits options )
5660 {
5661     return( kIOReturnUnsupported);
5662 }
5663
5664 /*
5665  * Convert an IOReturn to text. Subclasses which add additional
5666  * IOReturn's should override this method and call
5667  * super::stringFromReturn if the desired value is not found.
5668  */
5669
5670 const char * IOService::stringFromReturn( IOReturn rtn )
5671 {
5672     static const IONamedValue IOReturn_values[] = {
5673         {kIOReturnSuccess,          "success"                           },
5674         {kIOReturnError,            "general error"                     },
5675         {kIOReturnNoMemory,         "memory allocation error"           },
5676         {kIOReturnNoResources,      "resource shortage"                 },
5677         {kIOReturnIPCError,         "Mach IPC failure"                  },
5678         {kIOReturnNoDevice,         "no such device"                    },
5679         {kIOReturnNotPrivileged,    "privilege violation"               },
5680         {kIOReturnBadArgument,      "invalid argument"                  },
5681         {kIOReturnLockedRead,       "device is read locked"             },
5682         {kIOReturnLockedWrite,      "device is write locked"            },
5683         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5684         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5685         {kIOReturnUnsupported,      "unsupported function"              },
5686         {kIOReturnVMError,          "virtual memory error"              },
5687         {kIOReturnInternalError,    "internal driver error"             },
5688         {kIOReturnIOError,          "I/O error"                         },
5689         {kIOReturnCannotLock,       "cannot acquire lock"               },
5690         {kIOReturnNotOpen,          "device is not open"                },
5691         {kIOReturnNotReadable,      "device is not readable"            },
5692         {kIOReturnNotWritable,      "device is not writeable"           },
5693         {kIOReturnNotAligned,       "alignment error"                   },
5694         {kIOReturnBadMedia,         "media error"                       },
5695         {kIOReturnStillOpen,        "device is still open"              },
5696         {kIOReturnRLDError,         "rld failure"                       },
5697         {kIOReturnDMAError,         "DMA failure"                       },
5698         {kIOReturnBusy,             "device is busy"                    },
5699         {kIOReturnTimeout,          "I/O timeout"                       },
5700         {kIOReturnOffline,          "device is offline"                 },
5701         {kIOReturnNotReady,         "device is not ready"               },
5702         {kIOReturnNotAttached,      "device/channel is not attached"    },
5703         {kIOReturnNoChannels,       "no DMA channels available"         },
5704         {kIOReturnNoSpace,          "no space for data"                 },
5705         {kIOReturnPortExists,       "device port already exists"        },
5706         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5707         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5708         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5709         {kIOReturnMessageTooLarge,  "message is too large"              },
5710         {kIOReturnNotPermitted,     "operation is not permitted"        },
5711         {kIOReturnNoPower,          "device is without power"           },
5712         {kIOReturnNoMedia,          "media is not present"              },
5713         {kIOReturnUnformattedMedia, "media is not formatted"            },
5714         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5715         {kIOReturnUnderrun,         "data underrun"                     },
5716         {kIOReturnOverrun,          "data overrun"                      },
5717         {kIOReturnDeviceError,      "device error"                      },
5718         {kIOReturnNoCompletion,     "no completion routine"             },
5719         {kIOReturnAborted,          "operation was aborted"             },
5720         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5721         {kIOReturnNotResponding,    "device is not responding"          },
5722         {kIOReturnInvalid,          "unanticipated driver error"        },
5723         {0,                         NULL                                }
5724     };
5725
5726     return IOFindNameForValue(rtn, IOReturn_values);
5727 }
5728
5729 /*
5730  * Convert an IOReturn to an errno.
5731  */
5732 int IOService::errnoFromReturn( IOReturn rtn )
5733 {
5734     if (unix_err(err_get_code(rtn)) == rtn)
5735         return err_get_code(rtn);
5736
5737     switch(rtn) {
5738         // (obvious match)
5739         case kIOReturnSuccess:
5740             return(0);
5741         case kIOReturnNoMemory:
5742             return(ENOMEM);
5743         case kIOReturnNoDevice:
5744             return(ENXIO);
5745         case kIOReturnVMError:
5746             return(EFAULT);
5747         case kIOReturnNotPermitted:
5748             return(EPERM);
5749         case kIOReturnNotPrivileged:
5750             return(EACCES);
5751         case kIOReturnIOError:
5752             return(EIO);
5753         case kIOReturnNotWritable:
5754             return(EROFS);
5755         case kIOReturnBadArgument:
5756             return(EINVAL);
5757         case kIOReturnUnsupported:
5758             return(ENOTSUP);
5759         case kIOReturnBusy:
5760             return(EBUSY);
5761         case kIOReturnNoPower:
5762             return(EPWROFF);
5763         case kIOReturnDeviceError:
5764             return(EDEVERR);
5765         case kIOReturnTimeout:
5766             return(ETIMEDOUT);
5767         case kIOReturnMessageTooLarge:
5768             return(EMSGSIZE);
5769         case kIOReturnNoSpace:
5770             return(ENOSPC);
5771         case kIOReturnCannotLock:
5772             return(ENOLCK);
5773
5774         // (best match)
5775         case kIOReturnBadMessageID:
5776         case kIOReturnNoCompletion:
5777         case kIOReturnNotAligned:
5778             return(EINVAL);
5779         case kIOReturnNotReady:
5780             return(EBUSY);
5781         case kIOReturnRLDError:
5782             return(EBADMACHO);
5783         case kIOReturnPortExists:
5784         case kIOReturnStillOpen:
5785             return(EEXIST);
5786         case kIOReturnExclusiveAccess:
5787         case kIOReturnLockedRead:
5788         case kIOReturnLockedWrite:
5789         case kIOReturnNotOpen:
5790         case kIOReturnNotReadable:
5791             return(EACCES);
5792         case kIOReturnCannotWire:
5793         case kIOReturnNoResources:
5794             return(ENOMEM);
5795         case kIOReturnAborted:
5796         case kIOReturnOffline:
5797         case kIOReturnNotResponding:
5798             return(EBUSY);
5799         case kIOReturnBadMedia:
5800         case kIOReturnNoMedia:
5801         case kIOReturnNotAttached:
5802         case kIOReturnUnformattedMedia:
5803             return(ENXIO); // (media error)
5804         case kIOReturnDMAError:
5805         case kIOReturnOverrun:
5806         case kIOReturnUnderrun:
5807             return(EIO); // (transfer error)
5808         case kIOReturnNoBandwidth:
5809         case kIOReturnNoChannels:
5810         case kIOReturnNoFrames:
5811         case kIOReturnNoInterrupt:
5812             return(EIO); // (hardware error)
5813         case kIOReturnError:
5814         case kIOReturnInternalError:
5815         case kIOReturnInvalid:
5816             return(EIO); // (generic error)
5817         case kIOReturnIPCError:
5818             return(EIO); // (ipc error)
5819         default:
5820             return(EIO); // (all other errors)
5821     }
5822 }
5823
5824 IOReturn IOService::message( UInt32 type, IOService * provider,
5825                                 void * argument )
5826 {
5827     /*
5828      * Generic entry point for calls from the provider.  A return value of
5829      * kIOReturnSuccess indicates that the message was received, and where
5830      * applicable, that it was successful.
5831      */
5832
5833     return kIOReturnUnsupported;
5834 }
5835
5836 /*
5837  * Device memory
5838  */
5839
5840 IOItemCount IOService::getDeviceMemoryCount( void )
5841 {
5842     OSArray *           array;
5843     IOItemCount         count;
5844
5845     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5846     if( array)
5847         count = array->getCount();
5848     else
5849         count = 0;
5850
5851     return( count);
5852 }
5853
5854 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
5855 {
5856     OSArray *           array;
5857     IODeviceMemory *    range;
5858
5859     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5860     if( array)
5861         range = (IODeviceMemory *) array->getObject( index );
5862     else
5863         range = 0;
5864
5865     return( range);
5866 }
5867
5868 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
5869                                                 IOOptionBits options )
5870 {
5871     IODeviceMemory *    range;
5872     IOMemoryMap *       map;
5873
5874     range = getDeviceMemoryWithIndex( index );
5875     if( range)
5876         map = range->map( options );
5877     else
5878         map = 0;
5879
5880     return( map );
5881 }
5882
5883 OSArray * IOService::getDeviceMemory( void )
5884 {
5885     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
5886 }
5887
5888
5889 void IOService::setDeviceMemory( OSArray * array )
5890 {
5891     setProperty( gIODeviceMemoryKey, array);
5892 }
5893
5894 /*
5895  * For machines where the transfers on an I/O bus can stall because
5896  * the CPU is in an idle mode, These APIs allow a driver to specify
5897  * the maximum bus stall that they can handle.  0 indicates no limit.
5898  */
5899 void IOService::
5900 setCPUSnoopDelay(UInt32 __unused ns)
5901 {
5902 #if defined(__i386__) || defined(__x86_64__)
5903     ml_set_maxsnoop(ns);
5904 #endif /* defined(__i386__) || defined(__x86_64__) */
5905 }
5906
5907 UInt32 IOService::
5908 getCPUSnoopDelay()
5909 {
5910 #if defined(__i386__) || defined(__x86_64__)
5911     return ml_get_maxsnoop();
5912 #else
5913     return 0;
5914 #endif /* defined(__i386__) || defined(__x86_64__) */
5915 }
5916
5917 #if defined(__i386__) || defined(__x86_64__)
5918 static void
5919 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
5920 {
5921     static const UInt kNoReplace = -1U; // Must be an illegal index
5922     UInt replace = kNoReplace;
5923     bool setCpuDelay = false;
5924
5925     IORecursiveLockLock(sCpuDelayLock);
5926
5927     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5928     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5929     IOService * holder = NULL;
5930
5931     if (ns) {
5932         const CpuDelayEntry ne = {service, ns, delayType};
5933         holder = service;
5934         // Set maximum delay.
5935         for (UInt i = 0; i < count; i++) {
5936             IOService *thisService = entries[i].fService;
5937             bool sameType = (delayType == entries[i].fDelayType);
5938             if ((service == thisService) && sameType)
5939                 replace = i;
5940             else if (!thisService) {
5941                 if (kNoReplace == replace)
5942                     replace = i;
5943             }
5944             else if (sameType) {
5945                 const UInt32 thisMax = entries[i].fMaxDelay;
5946                 if (thisMax < ns)
5947                 {
5948                     ns = thisMax;
5949                     holder = thisService;
5950                 }
5951             }
5952         }
5953
5954         setCpuDelay = true;
5955         if (kNoReplace == replace)
5956             sCpuDelayData->appendBytes(&ne, sizeof(ne));
5957         else
5958             entries[replace] = ne;
5959     }
5960     else {
5961         ns = -1U;       // Set to max unsigned, i.e. no restriction
5962
5963         for (UInt i = 0; i < count; i++) {
5964             // Clear a maximum delay.
5965             IOService *thisService = entries[i].fService;
5966             if (thisService && (delayType == entries[i].fDelayType)) {
5967                 UInt32 thisMax = entries[i].fMaxDelay;
5968                 if (service == thisService)
5969                     replace = i;
5970                 else if (thisMax < ns) {
5971                     ns = thisMax;
5972                     holder = thisService;
5973                 }
5974             }
5975         }
5976
5977         // Check if entry found
5978         if (kNoReplace != replace) {
5979             entries[replace].fService = 0;      // Null the entry
5980             setCpuDelay = true;
5981         }
5982     }
5983
5984     if (setCpuDelay)
5985     {
5986         if (holder && debug_boot_arg) {
5987             strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
5988         }
5989
5990         // Must be safe to call from locked context
5991         if (delayType == kCpuDelayBusStall)
5992         {
5993             ml_set_maxbusdelay(ns);
5994         }
5995         else if (delayType == kCpuDelayInterrupt)
5996         {
5997             ml_set_maxintdelay(ns);
5998         }
5999         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
6000         sCPULatencySet   [delayType]->setValue(ns);
6001
6002         OSArray * handlers = sCpuLatencyHandlers[delayType];
6003         IOService * target;
6004         if (handlers) for (unsigned int idx = 0;
6005                             (target = (IOService *) handlers->getObject(idx));
6006                             idx++)
6007         {
6008             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6009                                             (void *) (uintptr_t) ns, holder,
6010                                             NULL, NULL);
6011         }
6012     }
6013
6014     IORecursiveLockUnlock(sCpuDelayLock);
6015 }
6016
6017 static IOReturn
6018 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
6019 {
6020     IOReturn result = kIOReturnNotFound;
6021     OSArray * array;
6022     unsigned int idx;
6023
6024     IORecursiveLockLock(sCpuDelayLock);
6025
6026     do
6027     {
6028         if (enable && !sCpuLatencyHandlers[delayType])
6029             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
6030         array = sCpuLatencyHandlers[delayType];
6031         if (!array)
6032             break;
6033         idx = array->getNextIndexOfObject(target, 0);
6034         if (!enable)
6035         {
6036             if (-1U != idx)
6037             {
6038                 array->removeObject(idx);
6039                 result = kIOReturnSuccess;
6040             }
6041         }
6042         else
6043         {
6044             if (-1U != idx) {
6045                 result = kIOReturnExclusiveAccess;
6046                 break;
6047             }
6048             array->setObject(target);
6049
6050             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6051             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6052             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
6053             IOService * holder = NULL;
6054
6055             for (UInt i = 0; i < count; i++) {
6056                 if (entries[i].fService
6057                   && (delayType == entries[i].fDelayType)
6058                   && (entries[i].fMaxDelay < ns)) {
6059                     ns = entries[i].fMaxDelay;
6060                     holder = entries[i].fService;
6061                 }
6062             }
6063             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6064                                             (void *) (uintptr_t) ns, holder,
6065                                             NULL, NULL);
6066             result = kIOReturnSuccess;
6067         }
6068     }
6069     while (false);
6070
6071     IORecursiveLockUnlock(sCpuDelayLock);
6072
6073     return (result);
6074 }
6075
6076 #endif /* defined(__i386__) || defined(__x86_64__) */
6077
6078 void IOService::
6079 requireMaxBusStall(UInt32 __unused ns)
6080 {
6081 #if defined(__i386__) || defined(__x86_64__)
6082     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
6083 #endif
6084 }
6085
6086 void IOService::
6087 requireMaxInterruptDelay(uint32_t __unused ns)
6088 {
6089 #if defined(__i386__) || defined(__x86_64__)
6090     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
6091 #endif
6092 }
6093
6094 /*
6095  * Device interrupts
6096  */
6097
6098 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
6099 {
6100   IOInterruptController *interruptController;
6101   OSArray               *array;
6102   OSData                *data;
6103   OSSymbol              *interruptControllerName;
6104   long                  numSources;
6105   IOInterruptSource     *interruptSources;
6106
6107   // Get the parents list from the nub.
6108   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
6109   if (array == 0) return kIOReturnNoResources;
6110
6111   // Allocate space for the IOInterruptSources if needed... then return early.
6112   if (nub->_interruptSources == 0) {
6113     numSources = array->getCount();
6114     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
6115     if (interruptSources == 0) return kIOReturnNoMemory;
6116
6117     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
6118
6119     nub->_numInterruptSources = numSources;
6120     nub->_interruptSources = interruptSources;
6121     return kIOReturnSuccess;
6122   }
6123
6124   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
6125   if (interruptControllerName == 0) return kIOReturnNoResources;
6126
6127   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
6128   if (interruptController == 0) return kIOReturnNoResources;
6129
6130   // Get the interrupt numbers from the nub.
6131   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
6132   if (array == 0) return kIOReturnNoResources;
6133   data = OSDynamicCast(OSData, array->getObject(source));
6134   if (data == 0) return kIOReturnNoResources;
6135
6136   // Set the interruptController and interruptSource in the nub's table.
6137   interruptSources = nub->_interruptSources;
6138   interruptSources[source].interruptController = interruptController;
6139   interruptSources[source].vectorData = data;
6140
6141   return kIOReturnSuccess;
6142 }
6143
6144 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
6145 {
6146   IOReturn ret;
6147
6148   /* Make sure the _interruptSources are set */
6149   if (_interruptSources == 0) {
6150     ret = resolveInterrupt(this, source);
6151     if (ret != kIOReturnSuccess) return ret;
6152   }
6153
6154   /* Make sure the local source number is valid */
6155   if ((source < 0) || (source >= _numInterruptSources))
6156     return kIOReturnNoInterrupt;
6157
6158   /* Look up the contoller for the local source */
6159   *interruptController = _interruptSources[source].interruptController;
6160
6161   if (*interruptController == NULL) {
6162     if (!resolve) return kIOReturnNoInterrupt;
6163
6164     /* Try to reslove the interrupt */
6165     ret = resolveInterrupt(this, source);
6166     if (ret != kIOReturnSuccess) return ret;
6167
6168     *interruptController = _interruptSources[source].interruptController;
6169   }
6170
6171   return kIOReturnSuccess;
6172 }
6173
6174 IOReturn IOService::registerInterrupt(int source, OSObject *target,
6175                                       IOInterruptAction handler,
6176                                       void *refCon)
6177 {
6178   IOInterruptController *interruptController;
6179   IOReturn              ret;
6180
6181   ret = lookupInterrupt(source, true, &interruptController);
6182   if (ret != kIOReturnSuccess) return ret;
6183
6184   /* Register the source */
6185   return interruptController->registerInterrupt(this, source, target,
6186                                                 (IOInterruptHandler)handler,
6187                                                 refCon);
6188 }
6189
6190 IOReturn IOService::unregisterInterrupt(int source)
6191 {
6192   IOInterruptController *interruptController;
6193   IOReturn              ret;
6194
6195   ret = lookupInterrupt(source, false, &interruptController);
6196   if (ret != kIOReturnSuccess) return ret;
6197
6198   /* Unregister the source */
6199   return interruptController->unregisterInterrupt(this, source);
6200 }
6201
6202 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
6203 {
6204   IOReportLegend * legend = NULL;
6205   IOInterruptAccountingData * oldValue = NULL;
6206   IOInterruptAccountingReporter * newArray = NULL;
6207   char subgroupName[64];
6208   int newArraySize = 0;
6209   int i = 0;
6210
6211   if (source < 0) {
6212     return kIOReturnBadArgument;
6213   }
6214
6215   /*
6216    * We support statistics on a maximum of 256 interrupts per nub; if a nub
6217    * has more than 256 interrupt specifiers associated with it, and tries
6218    * to register a high interrupt index with interrupt accounting, panic.
6219    * Having more than 256 interrupts associated with a single nub is
6220    * probably a sign that something fishy is going on.
6221    */
6222   if (source > IA_INDEX_MAX) {
6223     panic("addInterruptStatistics called for an excessively large index (%d)", source);
6224   }
6225
6226   /*
6227    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
6228    * leaving it as is because the likelihood of contention where we are
6229    * actually growing the array is minimal (we would realistically need
6230    * to be starting a driver for the first time, with an IOReporting
6231    * client already in place).  Nonetheless, cleanup that can be done
6232    * to adhere to best practices; it'll make the code more complicated,
6233    * unfortunately.
6234    */
6235   IOLockLock(reserved->interruptStatisticsLock);
6236
6237   /*
6238    * Lazily allocate the statistics array.
6239    */
6240   if (!reserved->interruptStatisticsArray) {
6241     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
6242     assert(reserved->interruptStatisticsArray);
6243     reserved->interruptStatisticsArrayCount = 1;
6244     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
6245   }
6246
6247   if (source >= reserved->interruptStatisticsArrayCount) {
6248     /*
6249      * We're still within the range of supported indices, but we are out
6250      * of space in the current array.  Do a nasty realloc (because
6251      * IORealloc isn't a thing) here.  We'll double the size with each
6252      * reallocation.
6253      *
6254      * Yes, the "next power of 2" could be more efficient; but this will
6255      * be invoked incredibly rarely.  Who cares.
6256      */
6257     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
6258
6259     while (newArraySize <= source)
6260       newArraySize = (newArraySize << 1);
6261     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
6262
6263     assert(newArray);
6264
6265     /*
6266      * TODO: This even zeroes the memory it is about to overwrite.
6267      * Shameful; fix it.  Not particularly high impact, however.
6268      */
6269     bzero(newArray, newArraySize * sizeof(*newArray));
6270     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
6271     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
6272     reserved->interruptStatisticsArray = newArray;
6273     reserved->interruptStatisticsArrayCount = newArraySize;
6274   }
6275
6276   if (!reserved->interruptStatisticsArray[source].reporter) {
6277     /*
6278      * We don't have a reporter associated with this index yet, so we
6279      * need to create one.
6280      */
6281     /*
6282      * TODO: Some statistics do in fact have common units (time); should this be
6283      * split into separate reporters to communicate this?
6284      */
6285      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
6286
6287     /*
6288      * Each statistic is given an identifier based on the interrupt index (which
6289      * should be unique relative to any single nub) and the statistic involved.
6290      * We should now have a sane (small and positive) index, so start
6291      * constructing the channels for statistics.
6292      */
6293     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
6294       /*
6295        * TODO: Currently, this does not add channels for disabled statistics.
6296        * Will this be confusing for clients?  If so, we should just add the
6297        * channels; we can avoid updating the channels even if they exist.
6298        */
6299       if (IA_GET_STATISTIC_ENABLED(i))
6300         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
6301     }
6302
6303     /*
6304      * We now need to add the legend for this reporter to the registry.
6305      */
6306     legend = IOReportLegend::with(OSDynamicCast(OSArray, getProperty(kIOReportLegendKey)));
6307
6308     /*
6309      * Note that while we compose the subgroup name, we do not need to
6310      * manage its lifecycle (the reporter will handle this).
6311      */
6312     snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
6313     subgroupName[sizeof(subgroupName) - 1] = 0;
6314     legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
6315     setProperty(kIOReportLegendKey, legend->getLegend());
6316     legend->release();
6317
6318     /*
6319      * TODO: Is this a good idea?  Probably not; my assumption is it opts
6320      * all entities who register interrupts into public disclosure of all
6321      * IOReporting channels.  Unfortunately, this appears to be as fine
6322      * grain as it gets.
6323      */
6324     setProperty(kIOReportLegendPublicKey, true);
6325   }
6326
6327   /*
6328    * Don't stomp existing entries.  If we are about to, panic; this
6329    * probably means we failed to tear down our old interrupt source
6330    * correctly.
6331    */
6332   oldValue = reserved->interruptStatisticsArray[source].statistics;
6333
6334   if (oldValue) {
6335     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
6336   }
6337
6338   reserved->interruptStatisticsArray[source].statistics = statistics;
6339
6340   /*
6341    * Inherit the reporter values for each statistic.  The target may
6342    * be torn down as part of the runtime of the service (especially
6343    * for sleep/wake), so we inherit in order to avoid having values
6344    * reset for no apparent reason.  Our statistics are ultimately
6345    * tied to the index and the sevice, not to an individual target,
6346    * so we should maintain them accordingly.
6347    */
6348   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6349
6350   IOLockUnlock(reserved->interruptStatisticsLock);
6351
6352   return kIOReturnSuccess;
6353 }
6354
6355 IOReturn IOService::removeInterruptStatistics(int source)
6356 {
6357   IOInterruptAccountingData * value = NULL;
6358
6359   if (source < 0) {
6360     return kIOReturnBadArgument;
6361   }
6362
6363   IOLockLock(reserved->interruptStatisticsLock);
6364
6365   /*
6366    * We dynamically grow the statistics array, so an excessively
6367    * large index value has NEVER been registered.  This either
6368    * means our cap on the array size is too small (unlikely), or
6369    * that we have been passed a corrupt index (this must be passed
6370    * the plain index into the interrupt specifier list).
6371    */
6372   if (source >= reserved->interruptStatisticsArrayCount) {
6373     panic("removeInterruptStatistics called for index %d, which was never registered", source);
6374   }
6375
6376   assert(reserved->interruptStatisticsArray);
6377
6378   /*
6379    * If there is no existing entry, we are most likely trying to
6380    * free an interrupt owner twice, or we have corrupted the
6381    * index value.
6382    */
6383   value = reserved->interruptStatisticsArray[source].statistics;
6384
6385   if (!value) {
6386     panic("removeInterruptStatistics called for empty index %d", source);
6387   }
6388
6389   /*
6390    * We update the statistics, so that any delta with the reporter
6391    * state is not lost.
6392    */
6393   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6394   reserved->interruptStatisticsArray[source].statistics = NULL;
6395   IOLockUnlock(reserved->interruptStatisticsLock);
6396
6397   return kIOReturnSuccess;
6398 }
6399
6400 IOReturn IOService::getInterruptType(int source, int *interruptType)
6401 {
6402   IOInterruptController *interruptController;
6403   IOReturn              ret;
6404
6405   ret = lookupInterrupt(source, true, &interruptController);
6406   if (ret != kIOReturnSuccess) return ret;
6407
6408   /* Return the type */
6409   return interruptController->getInterruptType(this, source, interruptType);
6410 }
6411
6412 IOReturn IOService::enableInterrupt(int source)
6413 {
6414   IOInterruptController *interruptController;
6415   IOReturn              ret;
6416
6417   ret = lookupInterrupt(source, false, &interruptController);
6418   if (ret != kIOReturnSuccess) return ret;
6419
6420   /* Enable the source */
6421   return interruptController->enableInterrupt(this, source);
6422 }
6423
6424 IOReturn IOService::disableInterrupt(int source)
6425 {
6426   IOInterruptController *interruptController;
6427   IOReturn              ret;
6428
6429   ret = lookupInterrupt(source, false, &interruptController);
6430   if (ret != kIOReturnSuccess) return ret;
6431
6432   /* Disable the source */
6433   return interruptController->disableInterrupt(this, source);
6434 }
6435
6436 IOReturn IOService::causeInterrupt(int source)
6437 {
6438   IOInterruptController *interruptController;
6439   IOReturn              ret;
6440
6441   ret = lookupInterrupt(source, false, &interruptController);
6442   if (ret != kIOReturnSuccess) return ret;
6443
6444   /* Cause an interrupt for the source */
6445   return interruptController->causeInterrupt(this, source);
6446 }
6447
6448 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6449                                     IOReportConfigureAction action,
6450                                     void                   *result,
6451                                     void                   *destination)
6452 {
6453     unsigned cnt;
6454
6455     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6456         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6457             if (pwrMgt) configurePowerStatesReport(action, result);
6458             else  return kIOReturnUnsupported;
6459         }
6460         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6461             if (pwrMgt) configureSimplePowerReport(action, result);
6462             else  return kIOReturnUnsupported;
6463         }
6464     }
6465
6466     IOLockLock(reserved->interruptStatisticsLock);
6467
6468     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6469     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6470         if (reserved->interruptStatisticsArray[cnt].reporter) {
6471             /*
6472              * If the reporter is currently associated with the statistics
6473              * for an event source, we may need to update the reporter.
6474              */
6475             if (reserved->interruptStatisticsArray[cnt].statistics)
6476                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6477
6478             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6479         }
6480     }
6481
6482     IOLockUnlock(reserved->interruptStatisticsLock);
6483
6484     return kIOReturnSuccess;
6485 }
6486
6487 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6488                                  IOReportUpdateAction      action,
6489                                  void                     *result,
6490                                  void                     *destination)
6491 {
6492     unsigned cnt;
6493
6494     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6495         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6496             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6497             else return kIOReturnUnsupported;
6498         }
6499         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6500             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6501             else return kIOReturnUnsupported;
6502         }
6503     }
6504
6505     IOLockLock(reserved->interruptStatisticsLock);
6506
6507     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6508     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6509         if (reserved->interruptStatisticsArray[cnt].reporter) {
6510             /*
6511              * If the reporter is currently associated with the statistics
6512              * for an event source, we need to update the reporter.
6513              */
6514             if (reserved->interruptStatisticsArray[cnt].statistics)
6515                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6516
6517             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6518         }
6519     }
6520
6521     IOLockUnlock(reserved->interruptStatisticsLock);
6522
6523     return kIOReturnSuccess;
6524 }
6525
6526 uint64_t IOService::getAuthorizationID( void )
6527 {
6528     return reserved->authorizationID;
6529 }
6530
6531 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6532 {
6533     OSObject * entitlement;
6534     IOReturn status;
6535
6536     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6537
6538     if ( entitlement )
6539     {
6540         if ( entitlement == kOSBooleanTrue )
6541         {
6542             reserved->authorizationID = authorizationID;
6543
6544             status = kIOReturnSuccess;
6545         }
6546         else
6547         {
6548             status = kIOReturnNotPrivileged;
6549         }
6550
6551         entitlement->release( );
6552     }
6553     else
6554     {
6555         status = kIOReturnNotPrivileged;
6556     }
6557
6558     return status;
6559 }
6560
6561 #if __LP64__
6562 OSMetaClassDefineReservedUsed(IOService, 0);
6563 OSMetaClassDefineReservedUsed(IOService, 1);
6564 OSMetaClassDefineReservedUnused(IOService, 2);
6565 OSMetaClassDefineReservedUnused(IOService, 3);
6566 OSMetaClassDefineReservedUnused(IOService, 4);
6567 OSMetaClassDefineReservedUnused(IOService, 5);
6568 OSMetaClassDefineReservedUnused(IOService, 6);
6569 OSMetaClassDefineReservedUnused(IOService, 7);
6570 #else
6571 OSMetaClassDefineReservedUsed(IOService, 0);
6572 OSMetaClassDefineReservedUsed(IOService, 1);
6573 OSMetaClassDefineReservedUsed(IOService, 2);
6574 OSMetaClassDefineReservedUsed(IOService, 3);
6575 OSMetaClassDefineReservedUsed(IOService, 4);
6576 OSMetaClassDefineReservedUsed(IOService, 5);
6577 OSMetaClassDefineReservedUsed(IOService, 6);
6578 OSMetaClassDefineReservedUsed(IOService, 7);
6579 #endif
6580 OSMetaClassDefineReservedUnused(IOService, 8);
6581 OSMetaClassDefineReservedUnused(IOService, 9);
6582 OSMetaClassDefineReservedUnused(IOService, 10);
6583 OSMetaClassDefineReservedUnused(IOService, 11);
6584 OSMetaClassDefineReservedUnused(IOService, 12);
6585 OSMetaClassDefineReservedUnused(IOService, 13);
6586 OSMetaClassDefineReservedUnused(IOService, 14);
6587 OSMetaClassDefineReservedUnused(IOService, 15);
6588 OSMetaClassDefineReservedUnused(IOService, 16);
6589 OSMetaClassDefineReservedUnused(IOService, 17);
6590 OSMetaClassDefineReservedUnused(IOService, 18);
6591 OSMetaClassDefineReservedUnused(IOService, 19);
6592 OSMetaClassDefineReservedUnused(IOService, 20);
6593 OSMetaClassDefineReservedUnused(IOService, 21);
6594 OSMetaClassDefineReservedUnused(IOService, 22);
6595 OSMetaClassDefineReservedUnused(IOService, 23);
6596 OSMetaClassDefineReservedUnused(IOService, 24);
6597 OSMetaClassDefineReservedUnused(IOService, 25);
6598 OSMetaClassDefineReservedUnused(IOService, 26);
6599 OSMetaClassDefineReservedUnused(IOService, 27);
6600 OSMetaClassDefineReservedUnused(IOService, 28);
6601 OSMetaClassDefineReservedUnused(IOService, 29);
6602 OSMetaClassDefineReservedUnused(IOService, 30);
6603 OSMetaClassDefineReservedUnused(IOService, 31);
6604 OSMetaClassDefineReservedUnused(IOService, 32);
6605 OSMetaClassDefineReservedUnused(IOService, 33);
6606 OSMetaClassDefineReservedUnused(IOService, 34);
6607 OSMetaClassDefineReservedUnused(IOService, 35);
6608 OSMetaClassDefineReservedUnused(IOService, 36);
6609 OSMetaClassDefineReservedUnused(IOService, 37);
6610 OSMetaClassDefineReservedUnused(IOService, 38);
6611 OSMetaClassDefineReservedUnused(IOService, 39);
6612 OSMetaClassDefineReservedUnused(IOService, 40);
6613 OSMetaClassDefineReservedUnused(IOService, 41);
6614 OSMetaClassDefineReservedUnused(IOService, 42);
6615 OSMetaClassDefineReservedUnused(IOService, 43);
6616 OSMetaClassDefineReservedUnused(IOService, 44);
6617 OSMetaClassDefineReservedUnused(IOService, 45);
6618 OSMetaClassDefineReservedUnused(IOService, 46);
6619 OSMetaClassDefineReservedUnused(IOService, 47);