iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30
  31 #include <IOKit/IOService.h>
  32 #include <libkern/OSDebug.h>
  33 #include <libkern/c++/OSContainers.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/c++/OSUnserialize.h>
  36 #include <IOKit/IOCatalogue.h>
  37 #include <IOKit/IOCommand.h>
  38 #include <IOKit/IODeviceTreeSupport.h>
  39 #include <IOKit/IODeviceMemory.h>
  40 #include <IOKit/IOInterrupts.h>
  41 #include <IOKit/IOInterruptController.h>
  42 #include <IOKit/IOPlatformExpert.h>
  43 #include <IOKit/IOMessage.h>
  44 #include <IOKit/IOLib.h>
  45 #include <IOKit/IOKitKeysPrivate.h>
  46 #include <IOKit/IOBSD.h>
  47 #include <IOKit/IOUserClient.h>
  48 #include <IOKit/IOWorkLoop.h>
  49 #include <IOKit/IOTimeStamp.h>
  50 #include <IOKit/IOHibernatePrivate.h>
  51 #include <IOKit/IOInterruptAccountingPrivate.h>
  52 #include <IOKit/IOKernelReporters.h>
  53 #include <IOKit/AppleKeyStoreInterface.h>
  54 #include <IOKit/IOCPU.h>
  55 #include <mach/sync_policy.h>
  56 #include <IOKit/assert.h>
  57 #include <sys/errno.h>
  58 #include <sys/kdebug.h>
  59 #include <string.h>
  60
  61 #include <machine/pal_routines.h>
  62
  63 #define LOG kprintf
  64 //#define LOG IOLog
  65 #define MATCH_DEBUG     0
  66 #define OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  67
  68 #include "IOServicePrivate.h"
  69 #include "IOKitKernelInternal.h"
  70
  71 // take lockForArbitration before LOCKNOTIFY
  72
  73 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  74
  75 #define super IORegistryEntry
  76
  77 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  78
  79 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  80 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  81
  82 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  83
  84 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  85
  86 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  87
  88 OSDefineMetaClassAndStructors(IOResources, IOService)
  89
  90 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
  91
  92 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
  93
  94 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  95
  96 static IOPlatformExpert *       gIOPlatform;
  97 static class IOPMrootDomain *   gIOPMRootDomain;
  98 const IORegistryPlane *         gIOServicePlane;
  99 const IORegistryPlane *         gIOPowerPlane;
 100 const OSSymbol *                gIODeviceMemoryKey;
 101 const OSSymbol *                gIOInterruptControllersKey;
 102 const OSSymbol *                gIOInterruptSpecifiersKey;
 103
 104 const OSSymbol *                gIOResourcesKey;
 105 const OSSymbol *                gIOResourceMatchKey;
 106 const OSSymbol *                gIOResourceMatchedKey;
 107 const OSSymbol *                gIOProviderClassKey;
 108 const OSSymbol *                gIONameMatchKey;
 109 const OSSymbol *                gIONameMatchedKey;
 110 const OSSymbol *                gIOPropertyMatchKey;
 111 const OSSymbol *                gIOPropertyExistsMatchKey;
 112 const OSSymbol *                gIOLocationMatchKey;
 113 const OSSymbol *                gIOParentMatchKey;
 114 const OSSymbol *                gIOPathMatchKey;
 115 const OSSymbol *                gIOMatchCategoryKey;
 116 const OSSymbol *                gIODefaultMatchCategoryKey;
 117 const OSSymbol *                gIOMatchedServiceCountKey;
 118
 119 const OSSymbol *                gIOMapperIDKey;
 120 const OSSymbol *                gIOUserClientClassKey;
 121 const OSSymbol *                gIOKitDebugKey;
 122
 123 const OSSymbol *                gIOCommandPoolSizeKey;
 124
 125 const OSSymbol *                gIOConsoleLockedKey;
 126 const OSSymbol *                gIOConsoleUsersKey;
 127 const OSSymbol *                gIOConsoleSessionUIDKey;
 128 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 129 const OSSymbol *                gIOConsoleUsersSeedKey;
 130 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 131 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 132 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 133 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 134
 135 clock_sec_t                     gIOConsoleLockTime;
 136 static bool                     gIOConsoleLoggedIn;
 137 #if HIBERNATION
 138 static uint32_t                 gIOScreenLockState;
 139 #endif
 140 static IORegistryEntry *        gIOChosenEntry;
 141
 142 static int                      gIOResourceGenerationCount;
 143
 144 const OSSymbol *                gIOServiceKey;
 145 const OSSymbol *                gIOPublishNotification;
 146 const OSSymbol *                gIOFirstPublishNotification;
 147 const OSSymbol *                gIOMatchedNotification;
 148 const OSSymbol *                gIOFirstMatchNotification;
 149 const OSSymbol *                gIOTerminatedNotification;
 150
 151 const OSSymbol *                gIOGeneralInterest;
 152 const OSSymbol *                gIOBusyInterest;
 153 const OSSymbol *                gIOAppPowerStateInterest;
 154 const OSSymbol *                gIOPriorityPowerStateInterest;
 155 const OSSymbol *                gIOConsoleSecurityInterest;
 156
 157 const OSSymbol *                gIOBSDKey;
 158 const OSSymbol *                gIOBSDNameKey;
 159 const OSSymbol *                gIOBSDMajorKey;
 160 const OSSymbol *                gIOBSDMinorKey;
 161 const OSSymbol *                gIOBSDUnitKey;
 162
 163 const  OSSymbol *               gAKSGetKey;
 164 #if defined(__i386__) || defined(__x86_64__)
 165 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 166 #endif
 167
 168 static OSDictionary *           gNotifications;
 169 static IORecursiveLock *        gNotificationLock;
 170
 171 static IOService *              gIOResources;
 172 static IOService *              gIOServiceRoot;
 173
 174 static OSOrderedSet *           gJobs;
 175 static semaphore_port_t         gJobsSemaphore;
 176 static IOLock *                 gJobsLock;
 177 static int                      gOutstandingJobs;
 178 static int                      gNumConfigThreads;
 179 static int                      gNumWaitingThreads;
 180 static IOLock *                 gIOServiceBusyLock;
 181 static bool             gCPUsRunning;
 182
 183 static thread_t                 gIOTerminateThread;
 184 static UInt32                   gIOTerminateWork;
 185 static OSArray *                gIOTerminatePhase2List;
 186 static OSArray *                gIOStopList;
 187 static OSArray *                gIOStopProviderList;
 188 static OSArray *                gIOFinalizeList;
 189
 190 static SInt32                   gIOConsoleUsersSeed;
 191 static OSData *                 gIOConsoleUsersSeedValue;
 192
 193 extern const OSSymbol *         gIODTPHandleKey;
 194
 195 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 196
 197 static IOLock *                 gIOConsoleUsersLock;
 198 static thread_call_t            gIOConsoleLockCallout;
 199 static IONotifier *             gIOServiceNullNotifier;
 200
 201 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 202
 203 #define LOCKREADNOTIFY()        \
 204     IORecursiveLockLock( gNotificationLock )
 205 #define LOCKWRITENOTIFY()       \
 206     IORecursiveLockLock( gNotificationLock )
 207 #define LOCKWRITE2READNOTIFY()
 208 #define UNLOCKNOTIFY()          \
 209     IORecursiveLockUnlock( gNotificationLock )
 210 #define SLEEPNOTIFY(event) \
 211     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 212 #define SLEEPNOTIFYTO(event, deadline) \
 213     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 214 #define WAKEUPNOTIFY(event) \
 215         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 216
 217 #define randomDelay()   \
 218         int del = read_processor_clock();                               \
 219         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 220         IOSleep( del );
 221
 222 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 223
 224 #define queue_element(entry, element, type, field) do { \
 225         vm_address_t __ele = (vm_address_t) (entry);    \
 226         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 227         (element) = (type) __ele;                       \
 228     } while(0)
 229
 230 #define iterqueue(que, elt)                             \
 231         for (queue_entry_t elt = queue_first(que);      \
 232              !queue_end(que, elt);                      \
 233              elt = queue_next(elt))
 234
 235 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 236
 237 struct IOInterruptAccountingReporter {
 238     IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 239     IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 240 };
 241
 242 struct ArbitrationLockQueueElement {
 243     queue_chain_t link;
 244     IOThread      thread;
 245     IOService *   service;
 246     unsigned      count;
 247     bool          required;
 248     bool          aborted;
 249 };
 250
 251 static queue_head_t gArbitrationLockQueueActive;
 252 static queue_head_t gArbitrationLockQueueWaiting;
 253 static queue_head_t gArbitrationLockQueueFree;
 254 static IOLock *     gArbitrationLockQueueLock;
 255
 256 bool IOService::isInactive( void ) const
 257     { return( 0 != (kIOServiceInactiveState & getState())); }
 258
 259 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 260
 261 #if defined(__i386__) || defined(__x86_64__)
 262
 263 // Only used by the intel implementation of
 264 //     IOService::requireMaxBusStall(UInt32 ns)
 265 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 266 struct CpuDelayEntry
 267 {
 268     IOService * fService;
 269     UInt32      fMaxDelay;
 270     UInt32      fDelayType;
 271 };
 272
 273 enum {
 274     kCpuDelayBusStall, kCpuDelayInterrupt,
 275     kCpuNumDelayTypes
 276 };
 277
 278 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 279 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 280 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 281 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 282 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 283 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 284 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 285
 286 static void
 287 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 288 static IOReturn
 289 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 290
 291 #endif /* defined(__i386__) || defined(__x86_64__) */
 292
 293 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 294
 295 void IOService::initialize( void )
 296 {
 297     kern_return_t       err;
 298
 299     gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 300     gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 301
 302     gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 303     gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 304     gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 305     gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 306     gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 307     gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 308     gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 309     gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 310
 311     gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 312     gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 313                                         kIODefaultMatchCategoryKey );
 314     gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 315                                         kIOMatchedServiceCountKey );
 316
 317     gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 318
 319     gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 320     gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 321     gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 322
 323     gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 324     gIOInterruptControllersKey
 325         = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 326     gIOInterruptSpecifiersKey
 327         = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 328
 329     gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 330
 331     gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 332
 333     gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 334
 335     gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 336     gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 337     gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 338     gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 339     gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 340
 341     gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 342     gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 343     gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 344     gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 345     gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 346
 347     gNotifications              = OSDictionary::withCapacity( 1 );
 348     gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 349                                                  kIOPublishNotification );
 350     gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 351                                                  kIOFirstPublishNotification );
 352     gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 353                                                  kIOMatchedNotification );
 354     gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 355                                                  kIOFirstMatchNotification );
 356     gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 357                                                  kIOTerminatedNotification );
 358     gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 359
 360     gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 361     gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 362     gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 363     gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 364
 365     gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 366     gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 367     gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 368     gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 369     gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 370
 371     gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 372
 373     gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 374 #if defined(__i386__) || defined(__x86_64__)
 375     sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 376     sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 377     uint32_t  idx;
 378     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
 379     {
 380         sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 381         sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 382         assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 383     }
 384     gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 385 #endif
 386     gNotificationLock           = IORecursiveLockAlloc();
 387
 388     gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 389
 390     assert( gIOServicePlane && gIODeviceMemoryKey
 391         && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 392         && gIOResourcesKey && gNotifications && gNotificationLock
 393         && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 394         && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 395         && gIOPublishNotification && gIOMatchedNotification
 396         && gIOTerminatedNotification && gIOServiceKey
 397         && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 398     && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 399         && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 400
 401     gJobsLock   = IOLockAlloc();
 402     gJobs       = OSOrderedSet::withCapacity( 10 );
 403
 404     gIOServiceBusyLock = IOLockAlloc();
 405
 406     gIOConsoleUsersLock = IOLockAlloc();
 407
 408     err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 409
 410     gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 411
 412     IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 413
 414     assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 415                 && gIOConsoleLockCallout && (err == KERN_SUCCESS) );
 416
 417     gIOResources = IOResources::resources();
 418     assert( gIOResources );
 419
 420     gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 421     assert(gIOServiceNullNotifier);
 422
 423     gArbitrationLockQueueLock = IOLockAlloc();
 424     queue_init(&gArbitrationLockQueueActive);
 425     queue_init(&gArbitrationLockQueueWaiting);
 426     queue_init(&gArbitrationLockQueueFree);
 427
 428     assert( gArbitrationLockQueueLock );
 429
 430     gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 431     gIOStopList            = OSArray::withCapacity( 16 );
 432     gIOStopProviderList    = OSArray::withCapacity( 16 );
 433     gIOFinalizeList        = OSArray::withCapacity( 16 );
 434     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 435 }
 436
 437 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 438
 439 #if defined(__i386__) || defined(__x86_64__)
 440 extern "C" {
 441
 442 const char *getCpuDelayBusStallHolderName(void);
 443 const char *getCpuDelayBusStallHolderName(void) {
 444     return sCPULatencyHolderName[kCpuDelayBusStall];
 445 }
 446
 447 }
 448 #endif
 449
 450 #if IOMATCHDEBUG
 451 static UInt64 getDebugFlags( OSDictionary * props )
 452 {
 453     OSNumber *  debugProp;
 454     UInt64      debugFlags;
 455
 456     debugProp = OSDynamicCast( OSNumber,
 457                 props->getObject( gIOKitDebugKey ));
 458     if( debugProp)
 459         debugFlags = debugProp->unsigned64BitValue();
 460     else
 461         debugFlags = gIOKitDebug;
 462
 463     return( debugFlags );
 464 }
 465 #endif
 466
 467 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 468
 469 // Probe a matched service and return an instance to be started.
 470 // The default score is from the property table, & may be altered
 471 // during probe to change the start order.
 472
 473 IOService * IOService::probe(   IOService * provider,
 474                                 SInt32    * score )
 475 {
 476     return( this );
 477 }
 478
 479 bool IOService::start( IOService * provider )
 480 {
 481     return( true );
 482 }
 483
 484 void IOService::stop( IOService * provider )
 485 {
 486 }
 487
 488 bool IOService::init( OSDictionary * dictionary )
 489 {
 490     bool ret;
 491
 492     ret = super::init(dictionary);
 493     if (!ret)     return (false);
 494     if (reserved) return (true);
 495
 496     reserved = IONew(ExpansionData, 1);
 497     if (!reserved) return (false);
 498     bzero(reserved, sizeof(*reserved));
 499
 500     /*
 501      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 502      * lock based on the presence of specifiers ran into issues as some
 503      * platforms set up the specifiers after IOService initialization.
 504      *
 505      * We may be able to get away with a global lock, as this should only be
 506      * contended by IOReporting clients and driver start/stop (unless a
 507      * driver wants to remove/add handlers in the course of normal operation,
 508      * which should be unlikely).
 509      */
 510     reserved->interruptStatisticsLock = IOLockAlloc();
 511     if (!reserved->interruptStatisticsLock) return (false);
 512
 513     return (true);
 514 }
 515
 516 bool IOService::init( IORegistryEntry * from,
 517                       const IORegistryPlane * inPlane )
 518 {
 519     bool ret;
 520
 521     ret = super::init(from, inPlane);
 522     if (!ret)     return (false);
 523     if (reserved) return (true);
 524
 525     reserved = IONew(ExpansionData, 1);
 526     if (!reserved) return (false);
 527     bzero(reserved, sizeof(*reserved));
 528
 529     /*
 530      * TODO: Improve on this.  Previous efforts to more lazily allocate this
 531      * lock based on the presence of specifiers ran into issues as some
 532      * platforms set up the specifiers after IOService initialization.
 533      *
 534      * We may be able to get away with a global lock, as this should only be
 535      * contended by IOReporting clients and driver start/stop (unless a
 536      * driver wants to remove/add handlers in the course of normal operation,
 537      * which should be unlikely).
 538      */
 539     reserved->interruptStatisticsLock = IOLockAlloc();
 540     if (!reserved->interruptStatisticsLock) return (false);
 541
 542     return (true);
 543 }
 544
 545 void IOService::free( void )
 546 {
 547     int i = 0;
 548     requireMaxBusStall(0);
 549     requireMaxInterruptDelay(0);
 550     if( getPropertyTable())
 551         unregisterAllInterest();
 552     PMfree();
 553
 554     if (reserved) {
 555         if (reserved->interruptStatisticsArray) {
 556             for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 557                 if (reserved->interruptStatisticsArray[i].reporter)
 558                     reserved->interruptStatisticsArray[i].reporter->release();
 559             }
 560
 561             IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 562         }
 563
 564         if (reserved->interruptStatisticsLock)
 565             IOLockFree(reserved->interruptStatisticsLock);
 566         IODelete(reserved, ExpansionData, 1);
 567     }
 568
 569     if (_numInterruptSources && _interruptSources)
 570     {
 571         IOFree(_interruptSources, _numInterruptSources * sizeof(IOInterruptSource));
 572         _interruptSources = 0;
 573     }
 574
 575     super::free();
 576 }
 577
 578 /*
 579  * Attach in service plane
 580  */
 581 bool IOService::attach( IOService * provider )
 582 {
 583     bool         ok;
 584     uint32_t     count;
 585     AbsoluteTime deadline;
 586     int          waitResult = THREAD_AWAKENED;
 587     bool         wait, computeDeadline = true;
 588
 589     if( provider) {
 590
 591         if( gIOKitDebug & kIOLogAttach)
 592             LOG( "%s::attach(%s)\n", getName(),
 593                     provider->getName());
 594
 595         ok   = false;
 596         do
 597         {
 598             wait = false;
 599             provider->lockForArbitration();
 600             if (provider->__state[0] & kIOServiceInactiveState) ok = false;
 601             else
 602             {
 603                 count = provider->getChildCount(gIOServicePlane);
 604                 wait = (count > (kIOServiceBusyMax - 4));
 605                 if (!wait) ok = attachToParent(provider, gIOServicePlane);
 606                 else
 607                 {
 608                     IOLog("stalling for detach from %s\n", provider->getName());
 609                     IOLockLock( gIOServiceBusyLock );
 610                     provider->__state[1] |= kIOServiceWaitDetachState;
 611                 }
 612             }
 613             provider->unlockForArbitration();
 614             if (wait)
 615             {
 616                 if (computeDeadline)
 617                 {
 618                     clock_interval_to_deadline(15, kSecondScale, &deadline);
 619                     computeDeadline = false;
 620                 }
 621                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 622                 IOLockUnlock( gIOServiceBusyLock );
 623                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 624                 wait = (waitResult != THREAD_TIMED_OUT);
 625             }
 626         }
 627         while (wait);
 628
 629     } else {
 630         gIOServiceRoot = this;
 631         ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 632     }
 633
 634     if (ok && !__provider) (void) getProvider();
 635
 636     return( ok );
 637 }
 638
 639 IOService * IOService::getServiceRoot( void )
 640 {
 641     return( gIOServiceRoot );
 642 }
 643
 644 void IOService::detach( IOService * provider )
 645 {
 646     IOService * newProvider = 0;
 647     SInt32      busy;
 648     bool        adjParent;
 649
 650     if( gIOKitDebug & kIOLogAttach)
 651         LOG("%s::detach(%s)\n", getName(), provider->getName());
 652
 653     lockForArbitration();
 654
 655     uint64_t regID1 = provider->getRegistryEntryID();
 656     uint64_t regID2 = getRegistryEntryID();
 657     IOServiceTrace(
 658         IOSERVICE_DETACH,
 659         (uintptr_t) regID1,
 660         (uintptr_t) (regID1 >> 32),
 661         (uintptr_t) regID2,
 662         (uintptr_t) (regID2 >> 32));
 663
 664     adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 665                && (provider == getProvider()));
 666
 667     detachFromParent( provider, gIOServicePlane );
 668
 669     if( busy) {
 670         newProvider = getProvider();
 671         if( busy && (__state[1] & kIOServiceTermPhase3State) && (0 == newProvider))
 672             _adjustBusy( -busy );
 673     }
 674
 675     if (kIOServiceInactiveState & __state[0]) {
 676         getMetaClass()->removeInstance(this);
 677         IORemoveServicePlatformActions(this);
 678     }
 679
 680     unlockForArbitration();
 681
 682     if( newProvider) {
 683         newProvider->lockForArbitration();
 684         newProvider->_adjustBusy(1);
 685         newProvider->unlockForArbitration();
 686     }
 687
 688     // check for last client detach from a terminated service
 689     if( provider->lockForArbitration( true ))
 690     {
 691         if (kIOServiceStartState & __state[1])
 692         {
 693             provider->scheduleTerminatePhase2();
 694         }
 695         if( adjParent) provider->_adjustBusy( -1 );
 696         if( (provider->__state[1] & kIOServiceTermPhase3State)
 697          && (0 == provider->getClient())) {
 698             provider->scheduleFinalize(false);
 699         }
 700
 701         IOLockLock( gIOServiceBusyLock );
 702         if (kIOServiceWaitDetachState & provider->__state[1])
 703         {
 704             provider->__state[1] &= ~kIOServiceWaitDetachState;
 705             thread_wakeup(&provider->__provider);
 706         }
 707         IOLockUnlock( gIOServiceBusyLock );
 708
 709         provider->unlockForArbitration();
 710     }
 711 }
 712
 713 /*
 714  * Register instance - publish it for matching
 715  */
 716
 717 void IOService::registerService( IOOptionBits options )
 718 {
 719     char *              pathBuf;
 720     const char *        path;
 721     char *              skip;
 722     int                 len;
 723     enum { kMaxPathLen  = 256 };
 724     enum { kMaxChars    = 63 };
 725
 726     IORegistryEntry * parent = this;
 727     IORegistryEntry * root = getRegistryRoot();
 728     while( parent && (parent != root))
 729         parent = parent->getParentEntry( gIOServicePlane);
 730
 731     if( parent != root) {
 732         IOLog("%s: not registry member at registerService()\n", getName());
 733         return;
 734     }
 735
 736     // Allow the Platform Expert to adjust this node.
 737     if( gIOPlatform && (!gIOPlatform->platformAdjustService(this)))
 738         return;
 739
 740     IOInstallServicePlatformActions(this);
 741
 742     if( (this != gIOResources)
 743      && (kIOLogRegister & gIOKitDebug)) {
 744
 745         pathBuf = (char *) IOMalloc( kMaxPathLen );
 746
 747         IOLog( "Registering: " );
 748
 749         len = kMaxPathLen;
 750         if( pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 751
 752             path = pathBuf;
 753             if( len > kMaxChars) {
 754                 IOLog("..");
 755                 len -= kMaxChars;
 756                 path += len;
 757                 if( (skip = strchr( path, '/')))
 758                     path = skip;
 759             }
 760         } else
 761             path = getName();
 762
 763         IOLog( "%s\n", path );
 764
 765         if( pathBuf)
 766             IOFree( pathBuf, kMaxPathLen );
 767     }
 768
 769     startMatching( options );
 770 }
 771
 772 void IOService::startMatching( IOOptionBits options )
 773 {
 774     IOService * provider;
 775     UInt32      prevBusy = 0;
 776     bool        needConfig;
 777     bool        needWake = false;
 778     bool        ok;
 779     bool        sync;
 780     bool        waitAgain;
 781
 782     lockForArbitration();
 783
 784     sync = (options & kIOServiceSynchronous)
 785         || ((provider = getProvider())
 786                 && (provider->__state[1] & kIOServiceSynchronousState));
 787
 788         if ( options & kIOServiceAsynchronous )
 789                 sync = false;
 790
 791     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState)))
 792                && (0 == (__state[0] & kIOServiceInactiveState));
 793
 794     __state[1] |= kIOServiceNeedConfigState;
 795
 796 //    __state[0] &= ~kIOServiceInactiveState;
 797
 798 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
 799 //                      OSKernelStackRemaining(), getName());
 800
 801     if( needConfig) {
 802         needWake = (0 != (kIOServiceSyncPubState & __state[1]));
 803     }
 804
 805     if( sync)
 806         __state[1] |= kIOServiceSynchronousState;
 807     else
 808         __state[1] &= ~kIOServiceSynchronousState;
 809
 810     if( needConfig) prevBusy = _adjustBusy( 1 );
 811
 812     unlockForArbitration();
 813
 814     if( needConfig) {
 815
 816         if( needWake) {
 817             IOLockLock( gIOServiceBusyLock );
 818             thread_wakeup( (event_t) this/*&__state[1]*/ );
 819             IOLockUnlock( gIOServiceBusyLock );
 820
 821         } else if( !sync || (kIOServiceAsynchronous & options)) {
 822
 823             ok = (0 != _IOServiceJob::startJob( this, kMatchNubJob, options ));
 824
 825         } else do {
 826
 827             if( (__state[1] & kIOServiceNeedConfigState))
 828                 doServiceMatch( options );
 829
 830             lockForArbitration();
 831             IOLockLock( gIOServiceBusyLock );
 832
 833             waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
 834                                        && (0 == (__state[0] & kIOServiceInactiveState)));
 835
 836             if( waitAgain)
 837                 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
 838             else
 839                 __state[1] &= ~kIOServiceSyncPubState;
 840
 841             unlockForArbitration();
 842
 843             if( waitAgain)
 844                 assert_wait( (event_t) this/*&__state[1]*/, THREAD_UNINT);
 845
 846             IOLockUnlock( gIOServiceBusyLock );
 847             if( waitAgain)
 848                 thread_block(THREAD_CONTINUE_NULL);
 849
 850         } while( waitAgain );
 851     }
 852 }
 853
 854 IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables )
 855 {
 856     OSDictionary *      table;
 857     OSSet *             set;
 858     OSSet *             allSet = 0;
 859     IOService *         service;
 860 #if IOMATCHDEBUG
 861     SInt32              count = 0;
 862 #endif
 863
 864     newTables->retain();
 865
 866     while( (table = (OSDictionary *) newTables->getFirstObject())) {
 867
 868         LOCKWRITENOTIFY();
 869         set = (OSSet *) copyExistingServices( table,
 870                                                 kIOServiceRegisteredState,
 871                                                 kIOServiceExistingSet);
 872         UNLOCKNOTIFY();
 873         if( set) {
 874
 875 #if IOMATCHDEBUG
 876             count += set->getCount();
 877 #endif
 878             if (allSet) {
 879                 allSet->merge((const OSSet *) set);
 880                 set->release();
 881             }
 882             else
 883                 allSet = set;
 884         }
 885
 886 #if IOMATCHDEBUG
 887         if( getDebugFlags( table ) & kIOLogMatch)
 888             LOG("Matching service count = %ld\n", (long)count);
 889 #endif
 890         newTables->removeObject(table);
 891     }
 892
 893     if (allSet) {
 894         while( (service = (IOService *) allSet->getAnyObject())) {
 895             service->startMatching(kIOServiceAsynchronous);
 896             allSet->removeObject(service);
 897         }
 898         allSet->release();
 899     }
 900
 901     newTables->release();
 902
 903     return( kIOReturnSuccess );
 904 }
 905
 906  _IOServiceJob * _IOServiceJob::startJob( IOService * nub, int type,
 907                                                 IOOptionBits options )
 908 {
 909     _IOServiceJob *     job;
 910
 911     job = new _IOServiceJob;
 912     if( job && !job->init()) {
 913         job->release();
 914         job = 0;
 915     }
 916
 917     if( job) {
 918         job->type       = type;
 919         job->nub        = nub;
 920         job->options    = options;
 921         nub->retain();                  // thread will release()
 922         pingConfig( job );
 923     }
 924
 925     return( job );
 926 }
 927
 928 /*
 929  * Called on a registered service to see if it matches
 930  * a property table.
 931  */
 932
 933 bool IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
 934 {
 935     return( matchPropertyTable(table) );
 936 }
 937
 938 bool IOService::matchPropertyTable( OSDictionary * table )
 939 {
 940     return( true );
 941 }
 942
 943 /*
 944  * Called on a matched service to allocate resources
 945  * before first driver is attached.
 946  */
 947
 948 IOReturn IOService::getResources( void )
 949 {
 950     return( kIOReturnSuccess);
 951 }
 952
 953 /*
 954  * Client/provider accessors
 955  */
 956
 957 IOService * IOService::getProvider( void ) const
 958 {
 959     IOService * self = (IOService *) this;
 960     IOService * parent;
 961     SInt32      generation;
 962
 963     generation = getRegistryEntryGenerationCount();
 964     if( __providerGeneration == generation)
 965         return( __provider );
 966
 967     parent = (IOService *) getParentEntry( gIOServicePlane);
 968     if( parent == IORegistryEntry::getRegistryRoot())
 969         /* root is not an IOService */
 970         parent = 0;
 971
 972     self->__provider = parent;
 973     OSMemoryBarrier();
 974     // save the count from before call to getParentEntry()
 975     self->__providerGeneration = generation;
 976
 977     return( parent );
 978 }
 979
 980 IOWorkLoop * IOService::getWorkLoop() const
 981 {
 982     IOService *provider = getProvider();
 983
 984     if (provider)
 985         return provider->getWorkLoop();
 986     else
 987         return 0;
 988 }
 989
 990 OSIterator * IOService::getProviderIterator( void ) const
 991 {
 992     return( getParentIterator( gIOServicePlane));
 993 }
 994
 995 IOService * IOService::getClient( void ) const
 996 {
 997     return( (IOService *) getChildEntry( gIOServicePlane));
 998 }
 999
1000 OSIterator * IOService::getClientIterator( void ) const
1001 {
1002     return( getChildIterator( gIOServicePlane));
1003 }
1004
1005 OSIterator * _IOOpenServiceIterator::iterator( OSIterator * _iter,
1006                                                 const IOService * client,
1007                                                 const IOService * provider )
1008 {
1009     _IOOpenServiceIterator * inst;
1010
1011     if( !_iter)
1012         return( 0 );
1013
1014     inst = new _IOOpenServiceIterator;
1015
1016     if( inst && !inst->init()) {
1017         inst->release();
1018         inst = 0;
1019     }
1020     if( inst) {
1021         inst->iter = _iter;
1022         inst->client = client;
1023         inst->provider = provider;
1024     }
1025
1026     return( inst );
1027 }
1028
1029 void _IOOpenServiceIterator::free()
1030 {
1031     iter->release();
1032     if( last)
1033         last->unlockForArbitration();
1034     OSIterator::free();
1035 }
1036
1037 OSObject * _IOOpenServiceIterator::getNextObject()
1038 {
1039     IOService * next;
1040
1041     if( last)
1042         last->unlockForArbitration();
1043
1044     while( (next = (IOService *) iter->getNextObject())) {
1045
1046         next->lockForArbitration();
1047         if( (client && (next->isOpen( client )))
1048          || (provider && (provider->isOpen( next ))) )
1049             break;
1050         next->unlockForArbitration();
1051     }
1052
1053     last = next;
1054
1055     return( next );
1056 }
1057
1058 bool _IOOpenServiceIterator::isValid()
1059 {
1060     return( iter->isValid() );
1061 }
1062
1063 void _IOOpenServiceIterator::reset()
1064 {
1065     if( last) {
1066         last->unlockForArbitration();
1067         last = 0;
1068     }
1069     iter->reset();
1070 }
1071
1072 OSIterator * IOService::getOpenProviderIterator( void ) const
1073 {
1074     return( _IOOpenServiceIterator::iterator( getProviderIterator(), this, 0 ));
1075 }
1076
1077 OSIterator * IOService::getOpenClientIterator( void ) const
1078 {
1079     return( _IOOpenServiceIterator::iterator( getClientIterator(), 0, this ));
1080 }
1081
1082
1083 IOReturn IOService::callPlatformFunction( const OSSymbol * functionName,
1084                                           bool waitForFunction,
1085                                           void *param1, void *param2,
1086                                           void *param3, void *param4 )
1087 {
1088   IOReturn  result = kIOReturnUnsupported;
1089   IOService *provider;
1090
1091   if (gIOPlatformFunctionHandlerSet == functionName)
1092   {
1093 #if defined(__i386__) || defined(__x86_64__)
1094     const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1095     IOService *      target              = (IOService *) param2;
1096     bool             enable              = (param3 != 0);
1097
1098     if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName)
1099         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1100     else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1)
1101         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1102 #endif /* defined(__i386__) || defined(__x86_64__) */
1103   }
1104
1105   if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1106     result = provider->callPlatformFunction(functionName, waitForFunction,
1107                                             param1, param2, param3, param4);
1108   }
1109
1110   return result;
1111 }
1112
1113 IOReturn IOService::callPlatformFunction( const char * functionName,
1114                                           bool waitForFunction,
1115                                           void *param1, void *param2,
1116                                           void *param3, void *param4 )
1117 {
1118   IOReturn result = kIOReturnNoMemory;
1119   const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1120
1121   if (functionSymbol != 0) {
1122     result = callPlatformFunction(functionSymbol, waitForFunction,
1123                                   param1, param2, param3, param4);
1124     functionSymbol->release();
1125   }
1126
1127   return result;
1128 }
1129
1130
1131 /*
1132  * Accessors for global services
1133  */
1134
1135 IOPlatformExpert * IOService::getPlatform( void )
1136 {
1137     return( gIOPlatform);
1138 }
1139
1140 class IOPMrootDomain * IOService::getPMRootDomain( void )
1141 {
1142     return( gIOPMRootDomain);
1143 }
1144
1145 IOService * IOService::getResourceService( void )
1146 {
1147     return( gIOResources );
1148 }
1149
1150 void IOService::setPlatform( IOPlatformExpert * platform)
1151 {
1152     gIOPlatform = platform;
1153     gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1154
1155 #if defined(__i386__) || defined(__x86_64__)
1156
1157     static const char * keys[kCpuNumDelayTypes] = {
1158         kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1159     const OSObject * objs[2];
1160     OSArray * array;
1161     uint32_t  idx;
1162
1163     for (idx = 0; idx < kCpuNumDelayTypes; idx++)
1164     {
1165         objs[0] = sCPULatencySet[idx];
1166         objs[1] = sCPULatencyHolder[idx];
1167         array   = OSArray::withObjects(objs, 2);
1168         if (!array) break;
1169         platform->setProperty(keys[idx], array);
1170         array->release();
1171     }
1172 #endif /* defined(__i386__) || defined(__x86_64__) */
1173 }
1174
1175 void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1176 {
1177     gIOPMRootDomain = rootDomain;
1178     publishResource("IOKit");
1179 }
1180
1181 /*
1182  * Stacking change
1183  */
1184
1185 bool IOService::lockForArbitration( bool isSuccessRequired )
1186 {
1187     bool                          found;
1188     bool                          success;
1189     ArbitrationLockQueueElement * element;
1190     ArbitrationLockQueueElement * active;
1191     ArbitrationLockQueueElement * waiting;
1192
1193     enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1194
1195     // lock global access
1196     IOTakeLock( gArbitrationLockQueueLock );
1197
1198     // obtain an unused queue element
1199     if( !queue_empty( &gArbitrationLockQueueFree )) {
1200         queue_remove_first( &gArbitrationLockQueueFree,
1201                             element,
1202                             ArbitrationLockQueueElement *,
1203                             link );
1204     } else {
1205         element = IONew( ArbitrationLockQueueElement, 1 );
1206         assert( element );
1207     }
1208
1209     // prepare the queue element
1210     element->thread   = IOThreadSelf();
1211     element->service  = this;
1212     element->count    = 1;
1213     element->required = isSuccessRequired;
1214     element->aborted  = false;
1215
1216     // determine whether this object is already locked (ie. on active queue)
1217     found = false;
1218     queue_iterate( &gArbitrationLockQueueActive,
1219                     active,
1220                     ArbitrationLockQueueElement *,
1221                     link )
1222     {
1223         if( active->service == element->service ) {
1224             found = true;
1225             break;
1226         }
1227     }
1228
1229     if( found ) { // this object is already locked
1230
1231         // determine whether it is the same or a different thread trying to lock
1232         if( active->thread != element->thread ) { // it is a different thread
1233
1234             ArbitrationLockQueueElement * victim = 0;
1235
1236             // before placing this new thread on the waiting queue, we look for
1237             // a deadlock cycle...
1238
1239             while( 1 ) {
1240                 // determine whether the active thread holding the object we
1241                 // want is waiting for another object to be unlocked
1242                 found = false;
1243                 queue_iterate( &gArbitrationLockQueueWaiting,
1244                                waiting,
1245                                ArbitrationLockQueueElement *,
1246                                link )
1247                 {
1248                     if( waiting->thread == active->thread ) {
1249                         assert( false == waiting->aborted );
1250                         found = true;
1251                         break;
1252                     }
1253                 }
1254
1255                 if( found ) { // yes, active thread waiting for another object
1256
1257                     // this may be a candidate for rejection if the required
1258                     // flag is not set, should we detect a deadlock later on
1259                     if( false == waiting->required )
1260                         victim = waiting;
1261
1262                     // find the thread that is holding this other object, that
1263                     // is blocking the active thread from proceeding (fun :-)
1264                     found = false;
1265                     queue_iterate( &gArbitrationLockQueueActive,
1266                                    active,      // (reuse active queue element)
1267                                    ArbitrationLockQueueElement *,
1268                                    link )
1269                     {
1270                         if( active->service == waiting->service ) {
1271                             found = true;
1272                             break;
1273                         }
1274                     }
1275
1276                     // someone must be holding it or it wouldn't be waiting
1277                     assert( found );
1278
1279                     if( active->thread == element->thread ) {
1280
1281                         // doh, it's waiting for the thread that originated
1282                         // this whole lock (ie. current thread) -> deadlock
1283                         if( false == element->required ) { // willing to fail?
1284
1285                             // the originating thread doesn't have the required
1286                             // flag, so it can fail
1287                             success = false; // (fail originating lock request)
1288                             break; // (out of while)
1289
1290                         } else { // originating thread is not willing to fail
1291
1292                             // see if we came across a waiting thread that did
1293                             // not have the 'required' flag set: we'll fail it
1294                             if( victim ) {
1295
1296                                 // we do have a willing victim, fail it's lock
1297                                 victim->aborted = true;
1298
1299                                 // take the victim off the waiting queue
1300                                 queue_remove( &gArbitrationLockQueueWaiting,
1301                                               victim,
1302                                               ArbitrationLockQueueElement *,
1303                                               link );
1304
1305                                 // wake the victim
1306                                 IOLockWakeup( gArbitrationLockQueueLock,
1307                                               victim,
1308                                               /* one thread */ true );
1309
1310                                 // allow this thread to proceed (ie. wait)
1311                                 success = true; // (put request on wait queue)
1312                                 break; // (out of while)
1313                             } else {
1314
1315                                 // all the waiting threads we came across in
1316                                 // finding this loop had the 'required' flag
1317                                 // set, so we've got a deadlock we can't avoid
1318                                 panic("I/O Kit: Unrecoverable deadlock.");
1319                             }
1320                         }
1321                     } else {
1322                         // repeat while loop, redefining active thread to be the
1323                         // thread holding "this other object" (see above), and
1324                         // looking for threads waiting on it; note the active
1325                         // variable points to "this other object" already... so
1326                         // there nothing to do in this else clause.
1327                     }
1328                 } else { // no, active thread is not waiting for another object
1329
1330                     success = true; // (put request on wait queue)
1331                     break; // (out of while)
1332                 }
1333             } // while forever
1334
1335             if( success ) { // put the request on the waiting queue?
1336                 kern_return_t wait_result;
1337
1338                 // place this thread on the waiting queue and put it to sleep;
1339                 // we place it at the tail of the queue...
1340                 queue_enter( &gArbitrationLockQueueWaiting,
1341                              element,
1342                              ArbitrationLockQueueElement *,
1343                              link );
1344
1345                 // declare that this thread will wait for a given event
1346 restart_sleep:  wait_result = assert_wait( element,
1347                                            element->required ? THREAD_UNINT
1348                                            : THREAD_INTERRUPTIBLE );
1349
1350                 // unlock global access
1351                 IOUnlock( gArbitrationLockQueueLock );
1352
1353                 // put thread to sleep, waiting for our event to fire...
1354                 if (wait_result == THREAD_WAITING)
1355                     wait_result = thread_block(THREAD_CONTINUE_NULL);
1356
1357
1358                 // ...and we've been woken up; we might be in one of two states:
1359                 // (a) we've been aborted and our queue element is not on
1360                 //     any of the three queues, but is floating around
1361                 // (b) we're allowed to proceed with the lock and we have
1362                 //     already been moved from the waiting queue to the
1363                 //     active queue.
1364                 // ...plus a 3rd state, should the thread have been interrupted:
1365                 // (c) we're still on the waiting queue
1366
1367                 // determine whether we were interrupted out of our sleep
1368                 if( THREAD_INTERRUPTED == wait_result ) {
1369
1370                     // re-lock global access
1371                     IOTakeLock( gArbitrationLockQueueLock );
1372
1373                     // determine whether we're still on the waiting queue
1374                     found = false;
1375                     queue_iterate( &gArbitrationLockQueueWaiting,
1376                                    waiting,     // (reuse waiting queue element)
1377                                    ArbitrationLockQueueElement *,
1378                                    link )
1379                     {
1380                         if( waiting == element ) {
1381                             found = true;
1382                             break;
1383                         }
1384                     }
1385
1386                     if( found ) { // yes, we're still on the waiting queue
1387
1388                         // determine whether we're willing to fail
1389                         if( false == element->required ) {
1390
1391                             // mark us as aborted
1392                             element->aborted = true;
1393
1394                             // take us off the waiting queue
1395                             queue_remove( &gArbitrationLockQueueWaiting,
1396                                           element,
1397                                           ArbitrationLockQueueElement *,
1398                                           link );
1399                         } else { // we are not willing to fail
1400
1401                             // ignore interruption, go back to sleep
1402                             goto restart_sleep;
1403                         }
1404                     }
1405
1406                     // unlock global access
1407                     IOUnlock( gArbitrationLockQueueLock );
1408
1409                     // proceed as though this were a normal wake up
1410                     wait_result = THREAD_AWAKENED;
1411                 }
1412
1413                 assert( THREAD_AWAKENED == wait_result );
1414
1415                 // determine whether we've been aborted while we were asleep
1416                 if( element->aborted ) {
1417                     assert( false == element->required );
1418
1419                     // re-lock global access
1420                     IOTakeLock( gArbitrationLockQueueLock );
1421
1422                     action = kPutOnFreeQueue;
1423                     success = false;
1424                 } else { // we weren't aborted, so we must be ready to go :-)
1425
1426                     // we've already been moved from waiting to active queue
1427                     return true;
1428                 }
1429
1430             } else { // the lock request is to be failed
1431
1432                 // return unused queue element to queue
1433                 action = kPutOnFreeQueue;
1434             }
1435         } else { // it is the same thread, recursive access is allowed
1436
1437             // add one level of recursion
1438             active->count++;
1439
1440             // return unused queue element to queue
1441             action = kPutOnFreeQueue;
1442             success = true;
1443         }
1444     } else { // this object is not already locked, so let this thread through
1445         action = kPutOnActiveQueue;
1446         success = true;
1447     }
1448
1449     // put the new element on a queue
1450     if( kPutOnActiveQueue == action ) {
1451         queue_enter( &gArbitrationLockQueueActive,
1452                      element,
1453                      ArbitrationLockQueueElement *,
1454                      link );
1455     } else if( kPutOnFreeQueue == action ) {
1456         queue_enter( &gArbitrationLockQueueFree,
1457                      element,
1458                      ArbitrationLockQueueElement *,
1459                      link );
1460     } else {
1461         assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1462     }
1463
1464     // unlock global access
1465     IOUnlock( gArbitrationLockQueueLock );
1466
1467     return( success );
1468 }
1469
1470 void IOService::unlockForArbitration( void )
1471 {
1472     bool                          found;
1473     ArbitrationLockQueueElement * element;
1474
1475     // lock global access
1476     IOTakeLock( gArbitrationLockQueueLock );
1477
1478     // find the lock element for this object (ie. on active queue)
1479     found = false;
1480     queue_iterate( &gArbitrationLockQueueActive,
1481                     element,
1482                     ArbitrationLockQueueElement *,
1483                     link )
1484     {
1485         if( element->service == this ) {
1486             found = true;
1487             break;
1488         }
1489     }
1490
1491     assert( found );
1492
1493     // determine whether the lock has been taken recursively
1494     if( element->count > 1 ) {
1495         // undo one level of recursion
1496         element->count--;
1497
1498     } else {
1499
1500         // remove it from the active queue
1501         queue_remove( &gArbitrationLockQueueActive,
1502                       element,
1503                       ArbitrationLockQueueElement *,
1504                       link );
1505
1506         // put it on the free queue
1507         queue_enter( &gArbitrationLockQueueFree,
1508                      element,
1509                      ArbitrationLockQueueElement *,
1510                      link );
1511
1512         // determine whether a thread is waiting for object (head to tail scan)
1513         found = false;
1514         queue_iterate( &gArbitrationLockQueueWaiting,
1515                        element,
1516                        ArbitrationLockQueueElement *,
1517                        link )
1518         {
1519             if( element->service == this ) {
1520                 found = true;
1521                 break;
1522             }
1523         }
1524
1525         if ( found ) { // we found an interested thread on waiting queue
1526
1527             // remove it from the waiting queue
1528             queue_remove( &gArbitrationLockQueueWaiting,
1529                           element,
1530                           ArbitrationLockQueueElement *,
1531                           link );
1532
1533             // put it on the active queue
1534             queue_enter( &gArbitrationLockQueueActive,
1535                          element,
1536                          ArbitrationLockQueueElement *,
1537                          link );
1538
1539             // wake the waiting thread
1540             IOLockWakeup( gArbitrationLockQueueLock,
1541                           element,
1542                           /* one thread */ true );
1543         }
1544     }
1545
1546     // unlock global access
1547     IOUnlock( gArbitrationLockQueueLock );
1548 }
1549
1550 void IOService::applyToProviders( IOServiceApplierFunction applier,
1551                                   void * context )
1552 {
1553     applyToParents( (IORegistryEntryApplierFunction) applier,
1554                     context, gIOServicePlane );
1555 }
1556
1557 void IOService::applyToClients( IOServiceApplierFunction applier,
1558                                 void * context )
1559 {
1560     applyToChildren( (IORegistryEntryApplierFunction) applier,
1561                      context, gIOServicePlane );
1562 }
1563
1564
1565 /*
1566  * Client messages
1567  */
1568
1569
1570 // send a message to a client or interested party of this service
1571 IOReturn IOService::messageClient( UInt32 type, OSObject * client,
1572                                    void * argument, vm_size_t argSize )
1573 {
1574     IOReturn                            ret;
1575     IOService *                         service;
1576     _IOServiceInterestNotifier *        notify;
1577
1578     if( (service = OSDynamicCast( IOService, client)))
1579         ret = service->message( type, this, argument );
1580
1581     else if( (notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1582
1583         _IOServiceNotifierInvocation invocation;
1584         bool                     willNotify;
1585
1586         invocation.thread = current_thread();
1587
1588         LOCKWRITENOTIFY();
1589         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1590
1591         if( willNotify) {
1592             queue_enter( &notify->handlerInvocations, &invocation,
1593                             _IOServiceNotifierInvocation *, link );
1594         }
1595         UNLOCKNOTIFY();
1596
1597         if( willNotify) {
1598
1599             ret = (*notify->handler)( notify->target, notify->ref,
1600                                           type, this, argument, argSize );
1601
1602             LOCKWRITENOTIFY();
1603             queue_remove( &notify->handlerInvocations, &invocation,
1604                             _IOServiceNotifierInvocation *, link );
1605             if( kIOServiceNotifyWaiter & notify->state) {
1606                 notify->state &= ~kIOServiceNotifyWaiter;
1607                 WAKEUPNOTIFY( notify );
1608             }
1609             UNLOCKNOTIFY();
1610
1611         } else
1612             ret = kIOReturnSuccess;
1613
1614     } else
1615         ret = kIOReturnBadArgument;
1616
1617     return( ret );
1618 }
1619
1620 static void
1621 applyToInterestNotifiers(const IORegistryEntry *target,
1622                          const OSSymbol * typeOfInterest,
1623                          OSObjectApplierFunction applier,
1624                          void * context )
1625 {
1626     OSArray *           copyArray = 0;
1627
1628     LOCKREADNOTIFY();
1629
1630     IOCommand *notifyList =
1631         OSDynamicCast( IOCommand, target->getProperty( typeOfInterest ));
1632
1633     if( notifyList) {
1634         copyArray = OSArray::withCapacity(1);
1635
1636         // iterate over queue, entry is set to each element in the list
1637         iterqueue(&notifyList->fCommandChain, entry) {
1638             _IOServiceInterestNotifier * notify;
1639
1640             queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1641             copyArray->setObject(notify);
1642         }
1643     }
1644     UNLOCKNOTIFY();
1645
1646     if( copyArray) {
1647         unsigned int    index;
1648         OSObject *      next;
1649
1650         for( index = 0; (next = copyArray->getObject( index )); index++)
1651             (*applier)(next, context);
1652         copyArray->release();
1653     }
1654 }
1655
1656 void IOService::applyToInterested( const OSSymbol * typeOfInterest,
1657                                    OSObjectApplierFunction applier,
1658                                    void * context )
1659 {
1660     if (gIOGeneralInterest == typeOfInterest)
1661         applyToClients( (IOServiceApplierFunction) applier, context );
1662     applyToInterestNotifiers(this, typeOfInterest, applier, context);
1663 }
1664
1665 struct MessageClientsContext {
1666     IOService * service;
1667     UInt32      type;
1668     void *      argument;
1669     vm_size_t   argSize;
1670     IOReturn    ret;
1671 };
1672
1673 static void messageClientsApplier( OSObject * object, void * ctx )
1674 {
1675     IOReturn                ret;
1676     MessageClientsContext * context = (MessageClientsContext *) ctx;
1677
1678     ret = context->service->messageClient( context->type,
1679                                            object, context->argument, context->argSize );
1680     if( kIOReturnSuccess != ret)
1681         context->ret = ret;
1682 }
1683
1684 // send a message to all clients
1685 IOReturn IOService::messageClients( UInt32 type,
1686                                     void * argument, vm_size_t argSize )
1687 {
1688     MessageClientsContext       context;
1689
1690     context.service     = this;
1691     context.type        = type;
1692     context.argument    = argument;
1693     context.argSize     = argSize;
1694     context.ret         = kIOReturnSuccess;
1695
1696     applyToInterested( gIOGeneralInterest,
1697                        &messageClientsApplier, &context );
1698
1699     return( context.ret );
1700 }
1701
1702 IOReturn IOService::acknowledgeNotification( IONotificationRef notification,
1703                                               IOOptionBits response )
1704 {
1705     return( kIOReturnUnsupported );
1706 }
1707
1708 IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest,
1709                   IOServiceInterestHandler handler, void * target, void * ref )
1710 {
1711     _IOServiceInterestNotifier * notify = 0;
1712     IOReturn rc = kIOReturnError;
1713
1714     notify = new _IOServiceInterestNotifier;
1715     if (!notify) return NULL;
1716
1717     if(notify->init()) {
1718         rc = registerInterestForNotifer(notify, typeOfInterest,
1719                               handler, target, ref);
1720     }
1721
1722     if (rc != kIOReturnSuccess) {
1723         notify->release();
1724         notify = 0;
1725     }
1726
1727     return( notify );
1728 }
1729
1730 IOReturn IOService::registerInterestForNotifer( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
1731                   IOServiceInterestHandler handler, void * target, void * ref )
1732 {
1733     IOReturn rc = kIOReturnSuccess;
1734     _IOServiceInterestNotifier  *notify = 0;
1735
1736     if( (typeOfInterest != gIOGeneralInterest)
1737      && (typeOfInterest != gIOBusyInterest)
1738      && (typeOfInterest != gIOAppPowerStateInterest)
1739      && (typeOfInterest != gIOConsoleSecurityInterest)
1740      && (typeOfInterest != gIOPriorityPowerStateInterest))
1741         return( kIOReturnBadArgument );
1742
1743     if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)))
1744         return( kIOReturnBadArgument );
1745
1746     lockForArbitration();
1747     if( 0 == (__state[0] & kIOServiceInactiveState)) {
1748
1749         notify->handler = handler;
1750         notify->target = target;
1751         notify->ref = ref;
1752         notify->state = kIOServiceNotifyEnable;
1753
1754         ////// queue
1755
1756         LOCKWRITENOTIFY();
1757
1758         // Get the head of the notifier linked list
1759         IOCommand * notifyList;
1760         OSObject  * obj = copyProperty( typeOfInterest );
1761         if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
1762             notifyList = OSTypeAlloc(IOCommand);
1763             if (notifyList) {
1764                 notifyList->init();
1765                 bool ok = setProperty( typeOfInterest, notifyList);
1766                 notifyList->release();
1767                 if (!ok) notifyList = 0;
1768             }
1769         }
1770         if (obj) obj->release();
1771
1772         if (notifyList) {
1773             enqueue(&notifyList->fCommandChain, &notify->chain);
1774             notify->retain();   // ref'ed while in list
1775         }
1776
1777         UNLOCKNOTIFY();
1778     }
1779     else {
1780         rc = kIOReturnNotReady;
1781     }
1782     unlockForArbitration();
1783
1784     return rc;
1785 }
1786
1787 static void cleanInterestList( OSObject * head )
1788 {
1789     IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
1790     if (!notifyHead)
1791         return;
1792
1793     LOCKWRITENOTIFY();
1794     while ( queue_entry_t entry = dequeue(&notifyHead->fCommandChain) ) {
1795         queue_next(entry) = queue_prev(entry) = 0;
1796
1797         _IOServiceInterestNotifier * notify;
1798
1799         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1800         notify->release();
1801     }
1802     UNLOCKNOTIFY();
1803 }
1804
1805 void IOService::unregisterAllInterest( void )
1806 {
1807     cleanInterestList( getProperty( gIOGeneralInterest ));
1808     cleanInterestList( getProperty( gIOBusyInterest ));
1809     cleanInterestList( getProperty( gIOAppPowerStateInterest ));
1810     cleanInterestList( getProperty( gIOPriorityPowerStateInterest ));
1811     cleanInterestList( getProperty( gIOConsoleSecurityInterest ));
1812 }
1813
1814 /*
1815  * _IOServiceInterestNotifier
1816  */
1817
1818 // wait for all threads, other than the current one,
1819 //  to exit the handler
1820
1821 void _IOServiceInterestNotifier::wait()
1822 {
1823     _IOServiceNotifierInvocation * next;
1824     bool doWait;
1825
1826     do {
1827         doWait = false;
1828         queue_iterate( &handlerInvocations, next,
1829                         _IOServiceNotifierInvocation *, link) {
1830             if( next->thread != current_thread() ) {
1831                 doWait = true;
1832                 break;
1833             }
1834         }
1835         if( doWait) {
1836             state |= kIOServiceNotifyWaiter;
1837                SLEEPNOTIFY(this);
1838         }
1839
1840     } while( doWait );
1841 }
1842
1843 void _IOServiceInterestNotifier::free()
1844 {
1845     assert( queue_empty( &handlerInvocations ));
1846     OSObject::free();
1847 }
1848
1849 void _IOServiceInterestNotifier::remove()
1850 {
1851     LOCKWRITENOTIFY();
1852
1853     if( queue_next( &chain )) {
1854         remqueue(&chain);
1855         queue_next( &chain) = queue_prev( &chain) = 0;
1856         release();
1857     }
1858
1859     state &= ~kIOServiceNotifyEnable;
1860
1861     wait();
1862
1863     UNLOCKNOTIFY();
1864
1865     release();
1866 }
1867
1868 bool _IOServiceInterestNotifier::disable()
1869 {
1870     bool        ret;
1871
1872     LOCKWRITENOTIFY();
1873
1874     ret = (0 != (kIOServiceNotifyEnable & state));
1875     state &= ~kIOServiceNotifyEnable;
1876     if( ret)
1877         wait();
1878
1879     UNLOCKNOTIFY();
1880
1881     return( ret );
1882 }
1883
1884 void _IOServiceInterestNotifier::enable( bool was )
1885 {
1886     LOCKWRITENOTIFY();
1887     if( was)
1888         state |= kIOServiceNotifyEnable;
1889     else
1890         state &= ~kIOServiceNotifyEnable;
1891     UNLOCKNOTIFY();
1892 }
1893
1894 bool _IOServiceInterestNotifier::init()
1895 {
1896     queue_init( &handlerInvocations );
1897     return (OSObject::init());
1898 }
1899 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1900
1901 /*
1902  * Termination
1903  */
1904
1905 #define tailQ(o)                setObject(o)
1906 #define headQ(o)                setObject(0, o)
1907 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
1908
1909 static void _workLoopAction( IOWorkLoop::Action action,
1910                              IOService * service,
1911                              void * p0 = 0, void * p1 = 0,
1912                              void * p2 = 0, void * p3 = 0 )
1913 {
1914     IOWorkLoop * wl;
1915
1916     if( (wl = service->getWorkLoop())) {
1917         wl->retain();
1918         wl->runAction( action, service, p0, p1, p2, p3 );
1919         wl->release();
1920     } else
1921         (*action)( service, p0, p1, p2, p3 );
1922 }
1923
1924 bool IOService::requestTerminate( IOService * provider, IOOptionBits options )
1925 {
1926     bool ok;
1927
1928     // if its our only provider
1929     ok = isParent( provider, gIOServicePlane, true);
1930
1931     // -- compat
1932     if( ok) {
1933         provider->terminateClient( this, options | kIOServiceRecursing );
1934         ok = (0 != (kIOServiceInactiveState & __state[0]));
1935     }
1936     // --
1937
1938     return( ok );
1939 }
1940
1941 bool IOService::terminatePhase1( IOOptionBits options )
1942 {
1943     IOService *  victim;
1944     IOService *  client;
1945     OSIterator * iter;
1946     OSArray *    makeInactive;
1947     OSArray *    waitingInactive;
1948     int          waitResult = THREAD_AWAKENED;
1949     bool         wait;
1950     bool                 ok;
1951     bool                 didInactive;
1952     bool                 startPhase2 = false;
1953
1954     TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
1955
1956     uint64_t regID = getRegistryEntryID();
1957     IOServiceTrace(
1958         IOSERVICE_TERMINATE_PHASE1,
1959         (uintptr_t) regID,
1960         (uintptr_t) (regID >> 32),
1961         (uintptr_t) this,
1962         (uintptr_t) options);
1963
1964     // -- compat
1965     if( options & kIOServiceRecursing) {
1966         lockForArbitration();
1967         if (0 == (kIOServiceInactiveState & __state[0]))
1968         {
1969             __state[0] |= kIOServiceInactiveState;
1970             __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
1971         }
1972         unlockForArbitration();
1973
1974         return( true );
1975     }
1976     // --
1977
1978     makeInactive    = OSArray::withCapacity( 16 );
1979     waitingInactive = OSArray::withCapacity( 16 );
1980     if(!makeInactive || !waitingInactive) return( false );
1981
1982     victim = this;
1983     victim->retain();
1984
1985     while( victim )
1986     {
1987         didInactive = victim->lockForArbitration( true );
1988         if( didInactive)
1989         {
1990             uint64_t regID1 = victim->getRegistryEntryID();
1991             IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
1992                 (uintptr_t) regID1,
1993                 (uintptr_t) (regID1 >> 32),
1994                 (uintptr_t) victim->__state[1],
1995                 (uintptr_t) 0);
1996
1997             enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
1998             didInactive = (kRP1 == (victim->__state[1] & kRP1))
1999                         || (0 == (victim->__state[0] & kIOServiceInactiveState));
2000
2001             if (!didInactive)
2002             {
2003                 // a multiply attached IOService can be visited twice
2004                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do
2005                 {
2006                     IOLockLock(gIOServiceBusyLock);
2007                     wait = (victim->__state[1] & kIOServiceTermPhase1State);
2008                     if( wait) {
2009                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2010                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2011                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2012                         victim->unlockForArbitration();
2013                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2014                     }
2015                     IOLockUnlock(gIOServiceBusyLock);
2016                     if( wait) {
2017                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2018                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2019                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2020                         victim->lockForArbitration();
2021                     }
2022                 }
2023                 while (wait && (waitResult != THREAD_TIMED_OUT));
2024             }
2025             else
2026             {
2027                 victim->__state[0] |= kIOServiceInactiveState;
2028                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2029                                         | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2030                 victim->__state[1] &= ~kIOServiceRecursing;
2031                 victim->__state[1] |= kIOServiceTermPhase1State;
2032                 waitingInactive->headQ(victim);
2033                 if (victim == this)
2034                 {
2035                     if (kIOServiceTerminateNeedWillTerminate & options)
2036                     {
2037                         victim->__state[1] |= kIOServiceNeedWillTerminate;
2038                     }
2039                 }
2040                 victim->_adjustBusy( 1 );
2041             }
2042             victim->unlockForArbitration();
2043         }
2044         if( victim == this) startPhase2 = didInactive;
2045         if (didInactive)
2046         {
2047             victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff );
2048             IOUserClient::destroyUserReferences( victim );
2049
2050             iter = victim->getClientIterator();
2051             if( iter) {
2052                 while( (client = (IOService *) iter->getNextObject())) {
2053                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2054                             client->getName(), client->getRegistryEntryID(),
2055                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2056                     ok = client->requestTerminate( victim, options );
2057                     TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2058                             client->getName(), client->getRegistryEntryID(),
2059                             victim->getName(), victim->getRegistryEntryID(), ok);
2060
2061                     uint64_t regID1 = client->getRegistryEntryID();
2062                     uint64_t regID2 = victim->getRegistryEntryID();
2063                     IOServiceTrace(
2064                         (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2065                            : IOSERVICE_TERMINATE_REQUEST_FAIL),
2066                         (uintptr_t) regID1,
2067                         (uintptr_t) (regID1 >> 32),
2068                         (uintptr_t) regID2,
2069                         (uintptr_t) (regID2 >> 32));
2070
2071                     if( ok)
2072                         makeInactive->setObject( client );
2073                 }
2074                 iter->release();
2075             }
2076         }
2077         victim->release();
2078         victim = (IOService *) makeInactive->getObject(0);
2079         if( victim) {
2080             victim->retain();
2081             makeInactive->removeObject(0);
2082         }
2083     }
2084     makeInactive->release();
2085
2086     while ((victim = (IOService *) waitingInactive->getObject(0)))
2087     {
2088         victim->retain();
2089         waitingInactive->removeObject(0);
2090
2091         victim->lockForArbitration();
2092         victim->__state[1] &= ~kIOServiceTermPhase1State;
2093         if (kIOServiceTerm1WaiterState & victim->__state[1])
2094         {
2095             victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2096             TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2097             IOLockLock( gIOServiceBusyLock );
2098             thread_wakeup( (event_t) &victim->__state[1]);
2099             IOLockUnlock( gIOServiceBusyLock );
2100         }
2101         victim->unlockForArbitration();
2102         victim->release();
2103     }
2104     waitingInactive->release();
2105
2106     if( startPhase2)
2107     {
2108         retain();
2109         lockForArbitration();
2110         scheduleTerminatePhase2(options);
2111         unlockForArbitration();
2112         release();
2113     }
2114
2115     return( true );
2116 }
2117
2118 void IOService::setTerminateDefer(IOService * provider, bool defer)
2119 {
2120     lockForArbitration();
2121     if (defer)  __state[1] |= kIOServiceStartState;
2122     else        __state[1] &= ~kIOServiceStartState;
2123     unlockForArbitration();
2124
2125     if (provider && !defer)
2126     {
2127         provider->lockForArbitration();
2128         provider->scheduleTerminatePhase2();
2129         provider->unlockForArbitration();
2130     }
2131 }
2132
2133 // call with lockForArbitration
2134 void IOService::scheduleTerminatePhase2( IOOptionBits options )
2135 {
2136     AbsoluteTime        deadline;
2137     uint64_t            regID1;
2138     int                 waitResult = THREAD_AWAKENED;
2139     bool                wait, haveDeadline = false;
2140
2141     if (!(__state[0] & kIOServiceInactiveState)) return;
2142
2143     regID1 = getRegistryEntryID();
2144     IOServiceTrace(
2145         IOSERVICE_TERM_SCHED_PHASE2,
2146         (uintptr_t) regID1,
2147         (uintptr_t) (regID1 >> 32),
2148         (uintptr_t) __state[1],
2149         (uintptr_t) options);
2150
2151     if (__state[1] & kIOServiceTermPhase1State)         return;
2152
2153     retain();
2154     unlockForArbitration();
2155     options |= kIOServiceRequired;
2156     IOLockLock( gJobsLock );
2157
2158     if( (options & kIOServiceSynchronous)
2159         && (current_thread() != gIOTerminateThread)) {
2160
2161         do {
2162             wait = (gIOTerminateThread != 0);
2163             if( wait) {
2164                 // wait to become the terminate thread
2165                 IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2166             }
2167         } while( wait );
2168
2169         gIOTerminateThread = current_thread();
2170         gIOTerminatePhase2List->setObject( this );
2171         gIOTerminateWork++;
2172
2173         do {
2174             while( gIOTerminateWork )
2175                 terminateWorker( options );
2176             wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2177             if( wait) {
2178                 // wait for the victim to go non-busy
2179                 if( !haveDeadline) {
2180                     clock_interval_to_deadline( 15, kSecondScale, &deadline );
2181                     haveDeadline = true;
2182                 }
2183                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2184                                                   deadline, THREAD_UNINT );
2185                 if( waitResult == THREAD_TIMED_OUT) {
2186                     IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
2187                 }
2188             }
2189         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2190
2191         gIOTerminateThread = 0;
2192         IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2193
2194     } else {
2195         // ! kIOServiceSynchronous
2196
2197         gIOTerminatePhase2List->setObject( this );
2198         if( 0 == gIOTerminateWork++) {
2199             if( !gIOTerminateThread)
2200                 kernel_thread_start(&terminateThread, (void *)(uintptr_t) options, &gIOTerminateThread);
2201             else
2202                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2203         }
2204     }
2205
2206     IOLockUnlock( gJobsLock );
2207     lockForArbitration();
2208     release();
2209 }
2210
2211 void IOService::terminateThread( void * arg, wait_result_t waitResult )
2212 {
2213     IOLockLock( gJobsLock );
2214
2215     while (gIOTerminateWork)
2216         terminateWorker( (uintptr_t) arg );
2217
2218     thread_deallocate(gIOTerminateThread);
2219     gIOTerminateThread = 0;
2220     IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2221
2222     IOLockUnlock( gJobsLock );
2223 }
2224
2225 void IOService::scheduleStop( IOService * provider )
2226 {
2227     uint64_t regID1 = getRegistryEntryID();
2228     uint64_t regID2 = provider->getRegistryEntryID();
2229
2230     TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2231     IOServiceTrace(
2232         IOSERVICE_TERMINATE_SCHEDULE_STOP,
2233         (uintptr_t) regID1,
2234         (uintptr_t) (regID1 >> 32),
2235         (uintptr_t) regID2,
2236         (uintptr_t) (regID2 >> 32));
2237
2238     IOLockLock( gJobsLock );
2239     gIOStopList->tailQ( this );
2240     gIOStopProviderList->tailQ( provider );
2241
2242     if( 0 == gIOTerminateWork++) {
2243         if( !gIOTerminateThread)
2244             kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2245         else
2246             IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2247     }
2248
2249     IOLockUnlock( gJobsLock );
2250 }
2251
2252 void IOService::scheduleFinalize(bool now)
2253 {
2254     uint64_t regID1 = getRegistryEntryID();
2255
2256     TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2257     IOServiceTrace(
2258         IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2259         (uintptr_t) regID1,
2260         (uintptr_t) (regID1 >> 32),
2261         0, 0);
2262
2263     if (now || IOUserClient::finalizeUserReferences(this))
2264     {
2265         IOLockLock( gJobsLock );
2266         gIOFinalizeList->tailQ(this);
2267         if( 0 == gIOTerminateWork++)
2268         {
2269             if( !gIOTerminateThread)
2270                 kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
2271             else
2272                 IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
2273         }
2274         IOLockUnlock( gJobsLock );
2275     }
2276 }
2277
2278 bool IOService::willTerminate( IOService * provider, IOOptionBits options )
2279 {
2280     return( true );
2281 }
2282
2283 bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2284 {
2285     if( false == *defer) {
2286
2287         if( lockForArbitration( true )) {
2288             if( false == provider->handleIsOpen( this ))
2289                 scheduleStop( provider );
2290             // -- compat
2291             else {
2292                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2293                 if( false == provider->handleIsOpen( this ))
2294                     scheduleStop( provider );
2295             }
2296             // --
2297             unlockForArbitration();
2298         }
2299     }
2300
2301     return( true );
2302 }
2303
2304 void IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2305                                      OSArray * doPhase2List,
2306                                      void *unused2 __unused,
2307                                      void *unused3 __unused  )
2308 {
2309     OSIterator * iter;
2310     IOService *  client;
2311     bool         ok;
2312     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2313
2314     iter = victim->getClientIterator();
2315     if( iter) {
2316         while( (client = (IOService *) iter->getNextObject())) {
2317
2318             regID1 = client->getRegistryEntryID();
2319             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2320                     client->getName(), regID1,
2321                     victim->getName(), regID2, (long long)options);
2322             IOServiceTrace(
2323                 IOSERVICE_TERMINATE_WILL,
2324                 (uintptr_t) regID1,
2325                 (uintptr_t) (regID1 >> 32),
2326                 (uintptr_t) regID2,
2327                 (uintptr_t) (regID2 >> 32));
2328
2329             ok = client->willTerminate( victim, options );
2330             doPhase2List->tailQ( client );
2331         }
2332         iter->release();
2333     }
2334 }
2335
2336 void IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2337                             void *unused1 __unused, void *unused2 __unused,
2338                             void *unused3 __unused )
2339 {
2340     OSIterator * iter;
2341     IOService *  client;
2342     bool         defer;
2343     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2344
2345     victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2346
2347     iter = victim->getClientIterator();
2348     if( iter) {
2349         while( (client = (IOService *) iter->getNextObject())) {
2350
2351             regID1 = client->getRegistryEntryID();
2352             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2353                     client->getName(), regID1,
2354                     victim->getName(), regID2, (long long)options);
2355             defer = false;
2356             client->didTerminate( victim, options, &defer );
2357
2358             IOServiceTrace(
2359                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2360                        : IOSERVICE_TERMINATE_DID),
2361                 (uintptr_t) regID1,
2362                 (uintptr_t) (regID1 >> 32),
2363                 (uintptr_t) regID2,
2364                 (uintptr_t) (regID2 >> 32));
2365
2366             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2367                     client->getName(), regID1,
2368                     victim->getName(), regID2, defer);
2369         }
2370         iter->release();
2371     }
2372 }
2373
2374
2375 void IOService::actionWillStop( IOService * victim, IOOptionBits options,
2376                             void *unused1 __unused, void *unused2 __unused,
2377                             void *unused3 __unused )
2378 {
2379     OSIterator * iter;
2380     IOService *  provider;
2381     bool         ok;
2382     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2383
2384     iter = victim->getProviderIterator();
2385     if( iter) {
2386         while( (provider = (IOService *) iter->getNextObject())) {
2387
2388             regID1 = provider->getRegistryEntryID();
2389             TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2390                     victim->getName(), regID2,
2391                     provider->getName(), regID1, (long long)options);
2392             IOServiceTrace(
2393                 IOSERVICE_TERMINATE_WILL,
2394                 (uintptr_t) regID2,
2395                 (uintptr_t) (regID2 >> 32),
2396                 (uintptr_t) regID1,
2397                 (uintptr_t) (regID1 >> 32));
2398
2399             ok = victim->willTerminate( provider, options );
2400         }
2401         iter->release();
2402     }
2403 }
2404
2405 void IOService::actionDidStop( IOService * victim, IOOptionBits options,
2406                             void *unused1 __unused, void *unused2 __unused,
2407                             void *unused3 __unused )
2408 {
2409     OSIterator * iter;
2410     IOService *  provider;
2411     bool defer = false;
2412     uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2413
2414     iter = victim->getProviderIterator();
2415     if( iter) {
2416         while( (provider = (IOService *) iter->getNextObject())) {
2417
2418             regID1 = provider->getRegistryEntryID();
2419             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2420                     victim->getName(), regID2,
2421                     provider->getName(), regID1, (long long)options);
2422             victim->didTerminate( provider, options, &defer );
2423
2424             IOServiceTrace(
2425                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2426                        : IOSERVICE_TERMINATE_DID),
2427                 (uintptr_t) regID2,
2428                 (uintptr_t) (regID2 >> 32),
2429                 (uintptr_t) regID1,
2430                 (uintptr_t) (regID1 >> 32));
2431
2432             TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2433                     victim->getName(), regID2,
2434                     provider->getName(), regID1, defer);
2435         }
2436         iter->release();
2437     }
2438 }
2439
2440
2441 void IOService::actionFinalize( IOService * victim, IOOptionBits options,
2442                             void *unused1 __unused, void *unused2 __unused,
2443                             void *unused3 __unused )
2444 {
2445     uint64_t regID1 = victim->getRegistryEntryID();
2446     TLOG("%s[0x%qx]::finalize(%08llx)\n",  victim->getName(), regID1, (long long)options);
2447     IOServiceTrace(
2448         IOSERVICE_TERMINATE_FINALIZE,
2449         (uintptr_t) regID1,
2450         (uintptr_t) (regID1 >> 32),
2451         0, 0);
2452
2453     victim->finalize( options );
2454 }
2455
2456 void IOService::actionStop( IOService * provider, IOService * client,
2457                             void *unused1 __unused, void *unused2 __unused,
2458                             void *unused3 __unused )
2459 {
2460     uint64_t regID1 = provider->getRegistryEntryID();
2461     uint64_t regID2 = client->getRegistryEntryID();
2462
2463     TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2464     IOServiceTrace(
2465         IOSERVICE_TERMINATE_STOP,
2466         (uintptr_t) regID1,
2467         (uintptr_t) (regID1 >> 32),
2468         (uintptr_t) regID2,
2469         (uintptr_t) (regID2 >> 32));
2470
2471     client->stop( provider );
2472     if( provider->isOpen( client ))
2473         provider->close( client );
2474
2475     TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2476     client->detach( provider );
2477 }
2478
2479 void IOService::terminateWorker( IOOptionBits options )
2480 {
2481     OSArray *           doPhase2List;
2482     OSArray *           didPhase2List;
2483     OSSet *             freeList;
2484     OSIterator *        iter;
2485     UInt32              workDone;
2486     IOService *         victim;
2487     IOService *         client;
2488     IOService *         provider;
2489     unsigned int        idx;
2490     bool                moreToDo;
2491     bool                doPhase2;
2492     bool                doPhase3;
2493
2494     options |= kIOServiceRequired;
2495
2496     doPhase2List  = OSArray::withCapacity( 16 );
2497     didPhase2List = OSArray::withCapacity( 16 );
2498     freeList      = OSSet::withCapacity( 16 );
2499     if( (0 == doPhase2List) || (0 == didPhase2List) || (0 == freeList))
2500         return;
2501
2502     do {
2503         workDone = gIOTerminateWork;
2504
2505         while( (victim = (IOService *) gIOTerminatePhase2List->getObject(0) )) {
2506
2507             victim->retain();
2508             gIOTerminatePhase2List->removeObject(0);
2509             IOLockUnlock( gJobsLock );
2510
2511             uint64_t regID1 = victim->getRegistryEntryID();
2512             IOServiceTrace(
2513                 IOSERVICE_TERM_START_PHASE2,
2514                 (uintptr_t) regID1,
2515                 (uintptr_t) (regID1 >> 32),
2516                 (uintptr_t) 0,
2517                 (uintptr_t) 0);
2518
2519             while( victim ) {
2520
2521                 doPhase2 = victim->lockForArbitration( true );
2522                 if( doPhase2) {
2523                     doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2524                     if( doPhase2) {
2525
2526                         uint64_t regID1 = victim->getRegistryEntryID();
2527                         IOServiceTrace(
2528                             IOSERVICE_TERM_TRY_PHASE2,
2529                             (uintptr_t) regID1,
2530                             (uintptr_t) (regID1 >> 32),
2531                             (uintptr_t) victim->__state[1],
2532                             (uintptr_t) 0);
2533
2534                         doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State))
2535                                 && (0 == (victim->__state[1] & kIOServiceConfigState));
2536
2537                         if (doPhase2 && (iter = victim->getClientIterator())) {
2538                             while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
2539                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
2540                                 if (!doPhase2)
2541                                 {
2542                                     uint64_t regID1 = client->getRegistryEntryID();
2543                                     IOServiceTrace(
2544                                         IOSERVICE_TERM_UC_DEFER,
2545                                         (uintptr_t) regID1,
2546                                         (uintptr_t) (regID1 >> 32),
2547                                         (uintptr_t) client->__state[1],
2548                                         (uintptr_t) 0);
2549                                     TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
2550                                            victim->getName(), victim->getRegistryEntryID(),
2551                                            client->getName(), client->getRegistryEntryID());
2552                                 }
2553                             }
2554                             iter->release();
2555                         }
2556                         if( doPhase2)
2557                             victim->__state[1] |= kIOServiceTermPhase2State;
2558                     }
2559                     victim->unlockForArbitration();
2560                 }
2561                 if( doPhase2) {
2562
2563                     if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2564                         _workLoopAction( (IOWorkLoop::Action) &actionWillStop,
2565                                             victim, (void *)(uintptr_t) options, NULL );
2566                     }
2567
2568                     if( 0 == victim->getClient()) {
2569
2570                         // no clients - will go to finalize
2571                         victim->scheduleFinalize(false);
2572
2573                     } else {
2574                         _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate,
2575                                             victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List );
2576                     }
2577                     didPhase2List->headQ( victim );
2578                 }
2579                 victim->release();
2580                 victim = (IOService *) doPhase2List->getObject(0);
2581                 if( victim) {
2582                     victim->retain();
2583                     doPhase2List->removeObject(0);
2584                 }
2585             }
2586
2587             while( (victim = (IOService *) didPhase2List->getObject(0)) ) {
2588
2589                 if( victim->lockForArbitration( true )) {
2590                     victim->__state[1] |= kIOServiceTermPhase3State;
2591                     victim->unlockForArbitration();
2592                 }
2593                 _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate,
2594                                     victim, (void *)(uintptr_t) options );
2595                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
2596                     _workLoopAction( (IOWorkLoop::Action) &actionDidStop,
2597                                         victim, (void *)(uintptr_t) options, NULL );
2598                 }
2599                 didPhase2List->removeObject(0);
2600             }
2601             IOLockLock( gJobsLock );
2602         }
2603
2604         // phase 3
2605         do {
2606             doPhase3 = false;
2607             // finalize leaves
2608             while( (victim = (IOService *) gIOFinalizeList->getObject(0))) {
2609
2610                 IOLockUnlock( gJobsLock );
2611                 _workLoopAction( (IOWorkLoop::Action) &actionFinalize,
2612                                     victim, (void *)(uintptr_t) options );
2613                 IOLockLock( gJobsLock );
2614                 // hold off free
2615                 freeList->setObject( victim );
2616                 // safe if finalize list is append only
2617                 gIOFinalizeList->removeObject(0);
2618             }
2619
2620             for( idx = 0;
2621                  (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx)); ) {
2622
2623                 provider = (IOService *) gIOStopProviderList->getObject(idx);
2624                 assert( provider );
2625
2626                 uint64_t regID1 = provider->getRegistryEntryID();
2627                 uint64_t regID2 = client->getRegistryEntryID();
2628
2629                 if( !provider->isChild( client, gIOServicePlane )) {
2630                     // may be multiply queued - nop it
2631                     TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2632                     IOServiceTrace(
2633                         IOSERVICE_TERMINATE_STOP_NOP,
2634                         (uintptr_t) regID1,
2635                         (uintptr_t) (regID1 >> 32),
2636                         (uintptr_t) regID2,
2637                         (uintptr_t) (regID2 >> 32));
2638
2639                 } else {
2640                     // a terminated client is not ready for stop if it has clients, skip it
2641                     if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) {
2642                         TLOG("%s[0x%qx]::defer stop(%s[0x%qx])\n",
2643                                 client->getName(), regID2,
2644                                 client->getClient()->getName(), client->getClient()->getRegistryEntryID());
2645                         IOServiceTrace(
2646                             IOSERVICE_TERMINATE_STOP_DEFER,
2647                             (uintptr_t) regID1,
2648                             (uintptr_t) (regID1 >> 32),
2649                             (uintptr_t) regID2,
2650                             (uintptr_t) (regID2 >> 32));
2651
2652                         idx++;
2653                         continue;
2654                     }
2655
2656                     IOLockUnlock( gJobsLock );
2657                     _workLoopAction( (IOWorkLoop::Action) &actionStop,
2658                                      provider, (void *) client );
2659                     IOLockLock( gJobsLock );
2660                     // check the finalize list now
2661                     doPhase3 = true;
2662                 }
2663                 // hold off free
2664                 freeList->setObject( client );
2665                 freeList->setObject( provider );
2666
2667                 // safe if stop list is append only
2668                 gIOStopList->removeObject( idx );
2669                 gIOStopProviderList->removeObject( idx );
2670                 idx = 0;
2671             }
2672
2673         } while( doPhase3 );
2674
2675         gIOTerminateWork -= workDone;
2676         moreToDo = (gIOTerminateWork != 0);
2677
2678         if( !moreToDo) {
2679             TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
2680             IOServiceTrace(
2681                 IOSERVICE_TERMINATE_DONE,
2682                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
2683         }
2684
2685     } while( moreToDo );
2686
2687     IOLockUnlock( gJobsLock );
2688
2689     freeList->release();
2690     doPhase2List->release();
2691     didPhase2List->release();
2692
2693     IOLockLock( gJobsLock );
2694 }
2695
2696 bool IOService::finalize( IOOptionBits options )
2697 {
2698     OSIterator *  iter;
2699     IOService *   provider;
2700     uint64_t      regID1, regID2 = getRegistryEntryID();
2701
2702     iter = getProviderIterator();
2703     assert( iter );
2704
2705     if( iter) {
2706         while( (provider = (IOService *) iter->getNextObject())) {
2707
2708             // -- compat
2709             if( 0 == (__state[1] & kIOServiceTermPhase3State)) {
2710                 /* we come down here on programmatic terminate */
2711
2712                 regID1 = provider->getRegistryEntryID();
2713                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
2714                 IOServiceTrace(
2715                     IOSERVICE_TERMINATE_STOP,
2716                     (uintptr_t) regID1,
2717                     (uintptr_t) (regID1 >> 32),
2718                     (uintptr_t) regID2,
2719                     (uintptr_t) (regID2 >> 32));
2720
2721                 stop( provider );
2722                 if( provider->isOpen( this ))
2723                     provider->close( this );
2724                 detach( provider );
2725             } else {
2726             //--
2727                 if( provider->lockForArbitration( true )) {
2728                     if( 0 == (provider->__state[1] & kIOServiceTermPhase3State))
2729                         scheduleStop( provider );
2730                     provider->unlockForArbitration();
2731                 }
2732             }
2733         }
2734         iter->release();
2735     }
2736
2737     return( true );
2738 }
2739
2740 #undef tailQ
2741 #undef headQ
2742
2743 /*
2744  * Terminate
2745  */
2746
2747 void IOService::doServiceTerminate( IOOptionBits options )
2748 {
2749 }
2750
2751 // a method in case someone needs to override it
2752 bool IOService::terminateClient( IOService * client, IOOptionBits options )
2753 {
2754     bool ok;
2755
2756     if( client->isParent( this, gIOServicePlane, true))
2757         // we are the clients only provider
2758         ok = client->terminate( options );
2759     else
2760         ok = true;
2761
2762     return( ok );
2763 }
2764
2765 bool IOService::terminate( IOOptionBits options )
2766 {
2767     options |= kIOServiceTerminate;
2768
2769     return( terminatePhase1( options ));
2770 }
2771
2772 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2773
2774 /*
2775  * Open & close
2776  */
2777
2778 struct ServiceOpenMessageContext
2779 {
2780     IOService *  service;
2781     UInt32       type;
2782     IOService *  excludeClient;
2783     IOOptionBits options;
2784 };
2785
2786 static void serviceOpenMessageApplier( OSObject * object, void * ctx )
2787 {
2788     ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
2789
2790     if( object != context->excludeClient)
2791         context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
2792 }
2793
2794 bool IOService::open(   IOService *     forClient,
2795                         IOOptionBits    options,
2796                         void *          arg )
2797 {
2798     bool                        ok;
2799     ServiceOpenMessageContext   context;
2800
2801     context.service             = this;
2802     context.type                = kIOMessageServiceIsAttemptingOpen;
2803     context.excludeClient       = forClient;
2804     context.options             = options;
2805
2806     applyToInterested( gIOGeneralInterest,
2807                         &serviceOpenMessageApplier, &context );
2808
2809     if( false == lockForArbitration(false) )
2810         return false;
2811
2812     ok = (0 == (__state[0] & kIOServiceInactiveState));
2813     if( ok)
2814         ok = handleOpen( forClient, options, arg );
2815
2816     unlockForArbitration();
2817
2818     return( ok );
2819 }
2820
2821 void IOService::close(  IOService *     forClient,
2822                         IOOptionBits    options )
2823 {
2824     bool                wasClosed;
2825     bool                last = false;
2826
2827     lockForArbitration();
2828
2829     wasClosed = handleIsOpen( forClient );
2830     if( wasClosed) {
2831         handleClose( forClient, options );
2832         last = (__state[1] & kIOServiceTermPhase3State);
2833     }
2834
2835     unlockForArbitration();
2836
2837     if( last)
2838         forClient->scheduleStop( this );
2839
2840     else if( wasClosed) {
2841
2842         ServiceOpenMessageContext context;
2843
2844         context.service         = this;
2845         context.type            = kIOMessageServiceWasClosed;
2846         context.excludeClient   = forClient;
2847         context.options         = options;
2848
2849         applyToInterested( gIOGeneralInterest,
2850                             &serviceOpenMessageApplier, &context );
2851     }
2852 }
2853
2854 bool IOService::isOpen( const IOService * forClient ) const
2855 {
2856     IOService * self = (IOService *) this;
2857     bool ok;
2858
2859     self->lockForArbitration();
2860
2861     ok = handleIsOpen( forClient );
2862
2863     self->unlockForArbitration();
2864
2865     return( ok );
2866 }
2867
2868 bool IOService::handleOpen(     IOService *     forClient,
2869                                 IOOptionBits    options,
2870                                 void *          arg )
2871 {
2872     bool        ok;
2873
2874     ok = (0 == __owner);
2875     if( ok )
2876         __owner = forClient;
2877
2878     else if( options & kIOServiceSeize ) {
2879         ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
2880                                 __owner, (void *)(uintptr_t) options ));
2881         if( ok && (0 == __owner ))
2882             __owner = forClient;
2883         else
2884             ok = false;
2885     }
2886     return( ok );
2887 }
2888
2889 void IOService::handleClose(    IOService *     forClient,
2890                                 IOOptionBits    options )
2891 {
2892     if( __owner == forClient)
2893         __owner = 0;
2894 }
2895
2896 bool IOService::handleIsOpen(   const IOService * forClient ) const
2897 {
2898     if( forClient)
2899         return( __owner == forClient );
2900     else
2901         return( __owner != forClient );
2902 }
2903
2904 /*
2905  * Probing & starting
2906  */
2907 static SInt32 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2908 {
2909     const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
2910     const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
2911     SInt32             val1;
2912     SInt32             val2;
2913
2914     val1 = 0;
2915     val2 = 0;
2916
2917     if ( obj1 )
2918         val1 = obj1->priority;
2919
2920     if ( obj2 )
2921         val2 = obj2->priority;
2922
2923     return ( val1 - val2 );
2924 }
2925
2926 static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref)
2927 {
2928     OSDictionary *      dict;
2929     IOService *         service;
2930     _IOServiceNotifier * notify;
2931     OSSymbol *          key = (OSSymbol *) ref;
2932     OSNumber *          offset;
2933
2934     if( (dict = OSDynamicCast( OSDictionary, entry)))
2935         offset = OSDynamicCast(OSNumber, dict->getObject( key ));
2936     else if( (notify = OSDynamicCast( _IOServiceNotifier, entry)))
2937         return( notify->priority );
2938
2939     else if( (service = OSDynamicCast( IOService, entry)))
2940         offset = OSDynamicCast(OSNumber, service->getProperty( key ));
2941     else {
2942         assert( false );
2943         offset = 0;
2944     }
2945
2946     if( offset)
2947         return( (SInt32) offset->unsigned32BitValue());
2948     else
2949         return( kIODefaultProbeScore );
2950 }
2951
2952 SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
2953 {
2954     const OSObject *    obj1 = (const OSObject *) inObj1;
2955     const OSObject *    obj2 = (const OSObject *) inObj2;
2956     SInt32               val1;
2957     SInt32               val2;
2958
2959     val1 = 0;
2960     val2 = 0;
2961
2962     if ( obj1 )
2963         val1 = IOServiceObjectOrder( obj1, ref );
2964
2965     if ( obj2 )
2966         val2 = IOServiceObjectOrder( obj2, ref );
2967
2968     return ( val1 - val2 );
2969 }
2970
2971 IOService * IOService::copyClientWithCategory( const OSSymbol * category )
2972 {
2973     IOService *         service = 0;
2974     OSIterator *        iter;
2975     const OSSymbol *    nextCat;
2976
2977     iter = getClientIterator();
2978     if( iter) {
2979         while( (service = (IOService *) iter->getNextObject())) {
2980             if( kIOServiceInactiveState & service->__state[0])
2981                 continue;
2982             nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
2983                         service->getProperty( gIOMatchCategoryKey ));
2984             if( category == nextCat)
2985             {
2986                 service->retain();
2987                 break;
2988             }
2989         }
2990         iter->release();
2991     }
2992     return( service );
2993 }
2994
2995 IOService * IOService::getClientWithCategory( const OSSymbol * category )
2996 {
2997     IOService *
2998     service = copyClientWithCategory(category);
2999     if (service)
3000         service->release();
3001     return (service);
3002 }
3003
3004 bool IOService::invokeNotifer( _IOServiceNotifier * notify )
3005 {
3006     _IOServiceNotifierInvocation invocation;
3007     bool                         willNotify;
3008     bool                         ret = true;
3009
3010     invocation.thread = current_thread();
3011
3012     LOCKWRITENOTIFY();
3013     willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3014
3015     if( willNotify) {
3016         queue_enter( &notify->handlerInvocations, &invocation,
3017                         _IOServiceNotifierInvocation *, link );
3018     }
3019     UNLOCKNOTIFY();
3020
3021     if( willNotify) {
3022
3023         ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3024
3025         LOCKWRITENOTIFY();
3026         queue_remove( &notify->handlerInvocations, &invocation,
3027                         _IOServiceNotifierInvocation *, link );
3028         if( kIOServiceNotifyWaiter & notify->state) {
3029             notify->state &= ~kIOServiceNotifyWaiter;
3030             WAKEUPNOTIFY( notify );
3031         }
3032         UNLOCKNOTIFY();
3033     }
3034
3035     return( ret );
3036 }
3037
3038 /*
3039  * Alloc and probe matching classes,
3040  * called on the provider instance
3041  */
3042
3043 void IOService::probeCandidates( OSOrderedSet * matches )
3044 {
3045     OSDictionary        *       match = 0;
3046     OSSymbol            *       symbol;
3047     IOService           *       inst;
3048     IOService           *       newInst;
3049     OSDictionary        *       props;
3050     SInt32                      score;
3051     OSNumber            *       newPri;
3052     OSOrderedSet        *       familyMatches = 0;
3053     OSOrderedSet        *       startList;
3054     OSDictionary        *       startDict = 0;
3055     const OSSymbol      *       category;
3056     OSIterator          *       iter;
3057     _IOServiceNotifier  *               notify;
3058     OSObject            *       nextMatch = 0;
3059     bool                        started;
3060     bool                        needReloc = false;
3061 #if IOMATCHDEBUG
3062     SInt64                      debugFlags;
3063 #endif
3064     IOService * client = NULL;
3065
3066
3067     assert( matches );
3068     while( !needReloc && (nextMatch = matches->getFirstObject())) {
3069
3070         nextMatch->retain();
3071         matches->removeObject(nextMatch);
3072
3073         if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3074
3075             lockForArbitration();
3076             if( 0 == (__state[0] & kIOServiceInactiveState))
3077                 invokeNotifer( notify );
3078             unlockForArbitration();
3079             nextMatch->release();
3080             nextMatch = 0;
3081             continue;
3082
3083         } else if( !(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3084             nextMatch->release();
3085             nextMatch = 0;
3086             continue;
3087         }
3088
3089         props = 0;
3090 #if IOMATCHDEBUG
3091         debugFlags = getDebugFlags( match );
3092 #endif
3093
3094         do {
3095             category = OSDynamicCast( OSSymbol,
3096                         match->getObject( gIOMatchCategoryKey ));
3097             if( 0 == category)
3098                 category = gIODefaultMatchCategoryKey;
3099
3100             if( (client = copyClientWithCategory(category)) ) {
3101 #if IOMATCHDEBUG
3102                 if( (debugFlags & kIOLogMatch) && (this != gIOResources))
3103                     LOG("%s: match category %s exists\n", getName(),
3104                                 category->getCStringNoCopy());
3105 #endif
3106                 nextMatch->release();
3107                 nextMatch = 0;
3108
3109                 client->release();
3110                 client = NULL;
3111
3112                 continue;
3113             }
3114
3115             // create a copy now in case its modified during matching
3116             props = OSDictionary::withDictionary( match, match->getCount());
3117             if( 0 == props)
3118                 continue;
3119             props->setCapacityIncrement(1);
3120
3121             // check the nub matches
3122             if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone))
3123                 continue;
3124
3125             // Check to see if driver reloc has been loaded.
3126             needReloc = (false == gIOCatalogue->isModuleLoaded( match ));
3127             if( needReloc) {
3128 #if IOMATCHDEBUG
3129                 if( debugFlags & kIOLogCatalogue)
3130                     LOG("%s: stalling for module\n", getName());
3131 #endif
3132                 // If reloc hasn't been loaded, exit;
3133                 // reprobing will occur after reloc has been loaded.
3134                 continue;
3135             }
3136
3137             // reorder on family matchPropertyTable score.
3138             if( 0 == familyMatches)
3139                 familyMatches = OSOrderedSet::withCapacity( 1,
3140                         IOServiceOrdering, (void *) gIOProbeScoreKey );
3141             if( familyMatches)
3142                 familyMatches->setObject( props );
3143
3144         } while( false );
3145
3146         if (nextMatch) {
3147             nextMatch->release();
3148             nextMatch = 0;
3149         }
3150         if( props)
3151             props->release();
3152     }
3153     matches->release();
3154     matches = 0;
3155
3156     if( familyMatches) {
3157
3158         while( !needReloc
3159              && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3160
3161             props->retain();
3162             familyMatches->removeObject( props );
3163
3164             inst = 0;
3165             newInst = 0;
3166 #if IOMATCHDEBUG
3167             debugFlags = getDebugFlags( props );
3168 #endif
3169             do {
3170                 symbol = OSDynamicCast( OSSymbol,
3171                                 props->getObject( gIOClassKey));
3172                 if( !symbol)
3173                     continue;
3174
3175                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), OBFUSCATE(symbol), OBFUSCATE(props));
3176
3177                 // alloc the driver instance
3178                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3179
3180                 if( !inst || !OSDynamicCast(IOService, inst)) {
3181                     IOLog("Couldn't alloc class \"%s\"\n",
3182                         symbol->getCStringNoCopy());
3183                     continue;
3184                 }
3185
3186                 // init driver instance
3187                 if( !(inst->init( props ))) {
3188 #if IOMATCHDEBUG
3189                     if( debugFlags & kIOLogStart)
3190                         IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3191 #endif
3192                     continue;
3193                 }
3194                 if( __state[1] & kIOServiceSynchronousState)
3195                     inst->__state[1] |= kIOServiceSynchronousState;
3196
3197                 // give the driver the default match category if not specified
3198                 category = OSDynamicCast( OSSymbol,
3199                             props->getObject( gIOMatchCategoryKey ));
3200                 if( 0 == category)
3201                     category = gIODefaultMatchCategoryKey;
3202                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3203                 // attach driver instance
3204                 if( !(inst->attach( this )))
3205                         continue;
3206
3207                 // pass in score from property table
3208                 score = familyMatches->orderObject( props );
3209
3210                 // & probe the new driver instance
3211 #if IOMATCHDEBUG
3212                 if( debugFlags & kIOLogProbe)
3213                     LOG("%s::probe(%s)\n",
3214                         inst->getMetaClass()->getClassName(), getName());
3215 #endif
3216
3217                 newInst = inst->probe( this, &score );
3218                 inst->detach( this );
3219                 if( 0 == newInst) {
3220 #if IOMATCHDEBUG
3221                     if( debugFlags & kIOLogProbe)
3222                         IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3223 #endif
3224                     continue;
3225                 }
3226
3227                 // save the score
3228                 newPri = OSNumber::withNumber( score, 32 );
3229                 if( newPri) {
3230                     newInst->setProperty( gIOProbeScoreKey, newPri );
3231                     newPri->release();
3232                 }
3233
3234                 // add to start list for the match category
3235                 if( 0 == startDict)
3236                     startDict = OSDictionary::withCapacity( 1 );
3237                 assert( startDict );
3238                 startList = (OSOrderedSet *)
3239                                 startDict->getObject( category );
3240                 if( 0 == startList) {
3241                     startList = OSOrderedSet::withCapacity( 1,
3242                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3243                     if( startDict && startList) {
3244                         startDict->setObject( category, startList );
3245                         startList->release();
3246                     }
3247                 }
3248                 assert( startList );
3249                 if( startList)
3250                     startList->setObject( newInst );
3251
3252             } while( false );
3253
3254             props->release();
3255             if( inst)
3256                 inst->release();
3257         }
3258         familyMatches->release();
3259         familyMatches = 0;
3260     }
3261
3262     // start the best (until success) of each category
3263
3264     iter = OSCollectionIterator::withCollection( startDict );
3265     if( iter) {
3266         while( (category = (const OSSymbol *) iter->getNextObject())) {
3267
3268             startList = (OSOrderedSet *) startDict->getObject( category );
3269             assert( startList );
3270             if( !startList)
3271                 continue;
3272
3273             started = false;
3274             while( true // (!started)
3275                    && (inst = (IOService *)startList->getFirstObject())) {
3276
3277                 inst->retain();
3278                 startList->removeObject(inst);
3279
3280 #if IOMATCHDEBUG
3281                 debugFlags = getDebugFlags( inst->getPropertyTable() );
3282
3283                 if( debugFlags & kIOLogStart) {
3284                     if( started)
3285                         LOG( "match category exists, skipping " );
3286                     LOG( "%s::start(%s) <%d>\n", inst->getName(),
3287                          getName(), inst->getRetainCount());
3288                 }
3289 #endif
3290                 if( false == started)
3291                     started = startCandidate( inst );
3292 #if IOMATCHDEBUG
3293                 if( (debugFlags & kIOLogStart) && (false == started))
3294                     LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3295                          inst->getRetainCount());
3296 #endif
3297                 inst->release();
3298             }
3299         }
3300         iter->release();
3301     }
3302
3303
3304     // adjust the busy count by +1 if matching is stalled for a module,
3305     // or -1 if a previously stalled matching is complete.
3306     lockForArbitration();
3307     SInt32 adjBusy = 0;
3308     uint64_t regID = getRegistryEntryID();
3309
3310     if( needReloc) {
3311         adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3312         if( adjBusy) {
3313
3314             IOServiceTrace(
3315                 IOSERVICE_MODULESTALL,
3316                 (uintptr_t) regID,
3317                 (uintptr_t) (regID >> 32),
3318                 (uintptr_t) this,
3319                 0);
3320
3321             __state[1] |= kIOServiceModuleStallState;
3322         }
3323
3324     } else if( __state[1] & kIOServiceModuleStallState) {
3325
3326             IOServiceTrace(
3327             IOSERVICE_MODULEUNSTALL,
3328             (uintptr_t) regID,
3329             (uintptr_t) (regID >> 32),
3330             (uintptr_t) this,
3331             0);
3332
3333         __state[1] &= ~kIOServiceModuleStallState;
3334         adjBusy = -1;
3335     }
3336     if( adjBusy)
3337         _adjustBusy( adjBusy );
3338     unlockForArbitration();
3339
3340     if( startDict)
3341         startDict->release();
3342 }
3343
3344 /*
3345  * Start a previously attached & probed instance,
3346  * called on exporting object instance
3347  */
3348
3349 bool IOService::startCandidate( IOService * service )
3350 {
3351     bool                ok;
3352
3353     ok = service->attach( this );
3354
3355     if( ok)
3356     {
3357         if (this != gIOResources)
3358         {
3359             // stall for any nub resources
3360             checkResources();
3361             // stall for any driver resources
3362             service->checkResources();
3363         }
3364
3365         AbsoluteTime startTime;
3366         AbsoluteTime endTime;
3367         UInt64       nano;
3368
3369         if (kIOLogStart & gIOKitDebug)
3370             clock_get_uptime(&startTime);
3371
3372         ok = service->start(this);
3373
3374         if (kIOLogStart & gIOKitDebug)
3375         {
3376             clock_get_uptime(&endTime);
3377
3378             if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
3379             {
3380                 SUB_ABSOLUTETIME(&endTime, &startTime);
3381                 absolutetime_to_nanoseconds(endTime, &nano);
3382                 if (nano > 500000000ULL)
3383                     IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
3384             }
3385         }
3386         if( !ok)
3387             service->detach( this );
3388     }
3389     return( ok );
3390 }
3391
3392 void IOService::publishResource( const char * key, OSObject * value )
3393 {
3394     const OSSymbol *    sym;
3395
3396     if( (sym = OSSymbol::withCString( key))) {
3397         publishResource( sym, value);
3398         sym->release();
3399     }
3400 }
3401
3402 void IOService::publishResource( const OSSymbol * key, OSObject * value )
3403 {
3404     if( 0 == value)
3405         value = (OSObject *) gIOServiceKey;
3406
3407     gIOResources->setProperty( key, value);
3408
3409     if( IORecursiveLockHaveLock( gNotificationLock))
3410         return;
3411
3412     gIOResourceGenerationCount++;
3413     gIOResources->registerService();
3414 }
3415
3416 bool IOService::addNeededResource( const char * key )
3417 {
3418     OSObject *  resourcesProp;
3419     OSSet *     set;
3420     OSString *  newKey;
3421     bool ret;
3422
3423     resourcesProp = getProperty( gIOResourceMatchKey );
3424
3425     newKey = OSString::withCString( key );
3426     if( (0 == resourcesProp) || (0 == newKey))
3427         return( false);
3428
3429     set = OSDynamicCast( OSSet, resourcesProp );
3430     if( !set) {
3431         set = OSSet::withCapacity( 1 );
3432         if( set)
3433             set->setObject( resourcesProp );
3434     }
3435     else
3436         set->retain();
3437
3438     set->setObject( newKey );
3439     newKey->release();
3440     ret = setProperty( gIOResourceMatchKey, set );
3441     set->release();
3442
3443     return( ret );
3444 }
3445
3446 bool IOService::checkResource( OSObject * matching )
3447 {
3448     OSString *          str;
3449     OSDictionary *      table;
3450
3451     if( (str = OSDynamicCast( OSString, matching ))) {
3452         if( gIOResources->getProperty( str ))
3453             return( true );
3454     }
3455
3456     if( str)
3457         table = resourceMatching( str );
3458     else if( (table = OSDynamicCast( OSDictionary, matching )))
3459         table->retain();
3460     else {
3461         IOLog("%s: Can't match using: %s\n", getName(),
3462                 matching->getMetaClass()->getClassName());
3463         /* false would stall forever */
3464         return( true );
3465     }
3466
3467     if( gIOKitDebug & kIOLogConfig)
3468         LOG("config(%p): stalling %s\n", OBFUSCATE(IOThreadSelf()), getName());
3469
3470     waitForService( table );
3471
3472     if( gIOKitDebug & kIOLogConfig)
3473         LOG("config(%p): waking\n", OBFUSCATE(IOThreadSelf()) );
3474
3475     return( true );
3476 }
3477
3478 bool IOService::checkResources( void )
3479 {
3480     OSObject *          resourcesProp;
3481     OSSet *             set;
3482     OSIterator *        iter;
3483     bool                ok;
3484
3485     resourcesProp = getProperty( gIOResourceMatchKey );
3486     if( 0 == resourcesProp)
3487         return( true );
3488
3489     if( (set = OSDynamicCast( OSSet, resourcesProp ))) {
3490
3491         iter = OSCollectionIterator::withCollection( set );
3492         ok = (0 != iter);
3493         while( ok && (resourcesProp = iter->getNextObject()) )
3494             ok = checkResource( resourcesProp );
3495         if( iter)
3496             iter->release();
3497
3498     } else
3499         ok = checkResource( resourcesProp );
3500
3501     return( ok );
3502 }
3503
3504
3505 void _IOConfigThread::configThread( void )
3506 {
3507     _IOConfigThread *   inst;
3508
3509     do {
3510         if( !(inst = new _IOConfigThread))
3511             continue;
3512         if( !inst->init())
3513             continue;
3514         thread_t unused;
3515         if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused))
3516             continue;
3517
3518         return;
3519
3520     } while( false);
3521
3522     if( inst)
3523         inst->release();
3524
3525     return;
3526 }
3527
3528 void _IOConfigThread::free( void )
3529 {
3530     thread_deallocate(current_thread());
3531     OSObject::free();
3532 }
3533
3534 void IOService::doServiceMatch( IOOptionBits options )
3535 {
3536     _IOServiceNotifier * notify;
3537     OSIterator *        iter;
3538     OSOrderedSet *      matches;
3539     OSArray *           resourceKeys = 0;
3540     SInt32              catalogGeneration;
3541     bool                keepGuessing = true;
3542     bool                reRegistered = true;
3543     bool                didRegister;
3544
3545 //    job->nub->deliverNotification( gIOPublishNotification,
3546 //                              kIOServiceRegisteredState, 0xffffffff );
3547
3548     while( keepGuessing ) {
3549
3550         matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
3551         // the matches list should always be created by findDrivers()
3552         if( matches) {
3553
3554             lockForArbitration();
3555             if( 0 == (__state[0] & kIOServiceFirstPublishState)) {
3556                 getMetaClass()->addInstance(this);
3557                 deliverNotification( gIOFirstPublishNotification,
3558                                      kIOServiceFirstPublishState, 0xffffffff );
3559             }
3560             LOCKREADNOTIFY();
3561             __state[1] &= ~kIOServiceNeedConfigState;
3562             __state[1] |= kIOServiceConfigState;
3563             didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
3564             __state[0] |= kIOServiceRegisteredState;
3565
3566             keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
3567             if (reRegistered && keepGuessing) {
3568                 iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
3569                         gNotifications->getObject( gIOPublishNotification ) );
3570                 if( iter) {
3571                     while((notify = (_IOServiceNotifier *)
3572                            iter->getNextObject())) {
3573
3574                         if( matchPassive(notify->matching, 0)
3575                          && (kIOServiceNotifyEnable & notify->state))
3576                             matches->setObject( notify );
3577                     }
3578                     iter->release();
3579                 }
3580             }
3581
3582             UNLOCKNOTIFY();
3583             unlockForArbitration();
3584
3585             if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources()))
3586             {
3587                 if (this == gIOResources)
3588                 {
3589                    if (resourceKeys) resourceKeys->release();
3590                    resourceKeys = copyPropertyKeys();
3591                 }
3592                 probeCandidates( matches );
3593             }
3594             else
3595                 matches->release();
3596         }
3597
3598         lockForArbitration();
3599         reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
3600         keepGuessing =
3601                    (reRegistered || (catalogGeneration !=
3602                                         gIOCatalogue->getGenerationCount()))
3603                 && (0 == (__state[0] & kIOServiceInactiveState));
3604
3605         if( keepGuessing)
3606             unlockForArbitration();
3607     }
3608
3609     if( (0 == (__state[0] & kIOServiceInactiveState))
3610      && (0 == (__state[1] & kIOServiceModuleStallState)) ) {
3611
3612         if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys);
3613
3614         deliverNotification( gIOMatchedNotification,
3615                 kIOServiceMatchedState, 0xffffffff );
3616         if( 0 == (__state[0] & kIOServiceFirstMatchState))
3617             deliverNotification( gIOFirstMatchNotification,
3618                 kIOServiceFirstMatchState, 0xffffffff );
3619     }
3620
3621     if (resourceKeys) resourceKeys->release();
3622
3623     __state[1] &= ~kIOServiceConfigState;
3624     scheduleTerminatePhase2();
3625
3626     _adjustBusy( -1 );
3627     unlockForArbitration();
3628 }
3629
3630 UInt32 IOService::_adjustBusy( SInt32 delta )
3631 {
3632     IOService * next;
3633     UInt32      count;
3634     UInt32      result;
3635     bool        wasQuiet, nowQuiet, needWake;
3636
3637     next = this;
3638     result = __state[1] & kIOServiceBusyStateMask;
3639
3640     if( delta) do {
3641         if( next != this)
3642             next->lockForArbitration();
3643         count = next->__state[1] & kIOServiceBusyStateMask;
3644         wasQuiet = (0 == count);
3645         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count)))
3646             OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
3647         else
3648             count += delta;
3649         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
3650         nowQuiet = (0 == count);
3651         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
3652
3653         if( needWake) {
3654             next->__state[1] &= ~kIOServiceBusyWaiterState;
3655             IOLockLock( gIOServiceBusyLock );
3656             thread_wakeup( (event_t) next);
3657             IOLockUnlock( gIOServiceBusyLock );
3658         }
3659         if( next != this)
3660             next->unlockForArbitration();
3661
3662         if( (wasQuiet || nowQuiet) ) {
3663
3664             uint64_t regID = next->getRegistryEntryID();
3665             IOServiceTrace(
3666                 ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
3667                 (uintptr_t) regID,
3668                 (uintptr_t) (regID >> 32),
3669                 (uintptr_t) next,
3670                 0);
3671
3672             if (wasQuiet)
3673             {
3674                 next->__timeBusy = mach_absolute_time();
3675             }
3676             else
3677             {
3678                 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
3679                 next->__timeBusy = 0;
3680             }
3681
3682             MessageClientsContext context;
3683
3684             context.service  = next;
3685             context.type     = kIOMessageServiceBusyStateChange;
3686             context.argument = (void *) wasQuiet;       /*nowBusy*/
3687             context.argSize  = 0;
3688
3689             applyToInterestNotifiers( next, gIOBusyInterest,
3690                                      &messageClientsApplier, &context );
3691
3692 #if !NO_KEXTD
3693             if( nowQuiet && (next == gIOServiceRoot)) {
3694                 OSKext::considerUnloads();
3695                 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
3696             }
3697 #endif
3698         }
3699
3700         delta = nowQuiet ? -1 : +1;
3701
3702     } while( (wasQuiet || nowQuiet) && (next = next->getProvider()));
3703
3704     return( result );
3705 }
3706
3707 void IOService::adjustBusy( SInt32 delta )
3708 {
3709     lockForArbitration();
3710     _adjustBusy( delta );
3711     unlockForArbitration();
3712 }
3713
3714 uint64_t IOService::getAccumulatedBusyTime( void )
3715 {
3716     uint64_t accumBusy = __accumBusy;
3717     uint64_t timeBusy = __timeBusy;
3718     uint64_t nano;
3719
3720     do
3721     {
3722         accumBusy = __accumBusy;
3723         timeBusy  = __timeBusy;
3724         if (timeBusy)
3725             accumBusy += mach_absolute_time() - timeBusy;
3726     }
3727     while (timeBusy != __timeBusy);
3728
3729     absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
3730
3731     return (nano);
3732 }
3733
3734 UInt32 IOService::getBusyState( void )
3735 {
3736     return( __state[1] & kIOServiceBusyStateMask );
3737 }
3738
3739 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3740                                   mach_timespec_t * timeout )
3741 {
3742     panic("waitForState");
3743     return (kIOReturnUnsupported);
3744 }
3745
3746 IOReturn IOService::waitForState( UInt32 mask, UInt32 value,
3747                                    uint64_t timeout )
3748 {
3749     bool            wait;
3750     int             waitResult = THREAD_AWAKENED;
3751     bool            computeDeadline = true;
3752     AbsoluteTime    abstime;
3753
3754     do {
3755         lockForArbitration();
3756         IOLockLock( gIOServiceBusyLock );
3757         wait = (value != (__state[1] & mask));
3758         if( wait) {
3759             __state[1] |= kIOServiceBusyWaiterState;
3760             unlockForArbitration();
3761             if( timeout != UINT64_MAX ) {
3762                 if( computeDeadline ) {
3763                     AbsoluteTime  nsinterval;
3764                     nanoseconds_to_absolutetime(timeout, &nsinterval );
3765                     clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
3766                     computeDeadline = false;
3767                 }
3768                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
3769             }
3770             else
3771                 assert_wait((event_t)this, THREAD_UNINT );
3772         } else
3773             unlockForArbitration();
3774         IOLockUnlock( gIOServiceBusyLock );
3775         if( wait)
3776             waitResult = thread_block(THREAD_CONTINUE_NULL);
3777
3778     } while( wait && (waitResult != THREAD_TIMED_OUT));
3779
3780     if( waitResult == THREAD_TIMED_OUT)
3781         return( kIOReturnTimeout );
3782     else
3783         return( kIOReturnSuccess );
3784 }
3785
3786 #if NO_KEXTD
3787 #define WAITING_KEXTD     false
3788 #else
3789 extern bool gIOKextdClearedBusy;
3790 #define WAITING_KEXTD     (false == gIOKextdClearedBusy)
3791 #endif
3792
3793 IOReturn IOService::waitQuiet( uint64_t timeout )
3794 {
3795     IOReturn ret;
3796     uint32_t loops;
3797     char * string = NULL;
3798     size_t len;
3799     uint64_t time;
3800     uint64_t nano;
3801
3802     time = mach_absolute_time();
3803     for (loops = 0; loops < 2; loops++)
3804     {
3805         ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
3806
3807         if (loops && (kIOReturnSuccess == ret))
3808         {
3809             time = mach_absolute_time() - time;
3810             absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
3811             IOLog("busy extended ok[%d], (%llds, %llds), kextd wait(%d): %s\n",
3812                   loops, timeout / 1000000000ULL, nano / 1000000000ULL, WAITING_KEXTD,
3813                   string ? string : "");
3814             break;
3815         }
3816         else if (kIOReturnTimeout != ret) break;
3817         else if (timeout < 41000000000)   break;
3818
3819         if (!loops)
3820         {
3821             IORegistryIterator * iter;
3822             OSOrderedSet       * set;
3823             OSOrderedSet       * leaves;
3824             IOService          * next;
3825             IOService          * nextParent;
3826             char               * s;
3827             size_t               l;
3828
3829             len = 256;
3830             string = IONew(char, len);
3831             set = NULL;
3832             iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
3833             leaves = OSOrderedSet::withCapacity(4);
3834             if (iter) set = iter->iterateAll();
3835             if (string && leaves && set)
3836             {
3837                 while ((next = (IOService *) set->getLastObject()))
3838                 {
3839                     if (next->getBusyState())
3840                     {
3841                         leaves->setObject(next);
3842                         nextParent = next;
3843                         while ((nextParent = nextParent->getProvider()))
3844                         {
3845                             set->removeObject(nextParent);
3846                             leaves->removeObject(nextParent);
3847                         }
3848                     }
3849                     set->removeObject(next);
3850                 }
3851                 s = string;
3852                 while ((next = (IOService *) leaves->getLastObject()))
3853                 {
3854                     l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
3855                     if (l >= len) break;
3856                     s += l;
3857                     len -= l;
3858                     leaves->removeObject(next);
3859                 }
3860             }
3861             OSSafeReleaseNULL(leaves);
3862             OSSafeReleaseNULL(set);
3863             OSSafeReleaseNULL(iter);
3864         }
3865         if (loops && (kIOWaitQuietPanics & gIOKitDebug))
3866         {
3867             panic("busy timeout[%d], (%llds), kextd wait(%d): %s",
3868                     loops, timeout / 1000000000ULL, WAITING_KEXTD,
3869                     string ? string : "");
3870         }
3871         else
3872         {
3873             IOLog("busy timeout[%d], (%llds), kextd wait(%d): %s\n",
3874                     loops, timeout / 1000000000ULL, WAITING_KEXTD,
3875                     string ? string : "");
3876         }
3877     }
3878
3879     if (string) IODelete(string, char, 256);
3880
3881     return (ret);
3882 }
3883
3884 IOReturn IOService::waitQuiet( mach_timespec_t * timeout )
3885 {
3886     uint64_t    timeoutNS;
3887
3888     if (timeout)
3889     {
3890         timeoutNS = timeout->tv_sec;
3891         timeoutNS *= kSecondScale;
3892         timeoutNS += timeout->tv_nsec;
3893     }
3894     else
3895         timeoutNS = UINT64_MAX;
3896
3897     return (waitQuiet(timeoutNS));
3898 }
3899
3900 bool IOService::serializeProperties( OSSerialize * s ) const
3901 {
3902 #if 0
3903     ((IOService *)this)->setProperty( ((IOService *)this)->__state,
3904                 sizeof( __state), "__state");
3905 #endif
3906     return( super::serializeProperties(s) );
3907 }
3908
3909
3910 void _IOConfigThread::main(void * arg, wait_result_t result)
3911 {
3912     _IOConfigThread * self = (_IOConfigThread *) arg;
3913     _IOServiceJob * job;
3914     IOService   *   nub;
3915     bool            alive = true;
3916     kern_return_t   kr;
3917     thread_precedence_policy_data_t precedence = { -1 };
3918
3919     kr = thread_policy_set(current_thread(),
3920                             THREAD_PRECEDENCE_POLICY,
3921                             (thread_policy_t) &precedence,
3922                             THREAD_PRECEDENCE_POLICY_COUNT);
3923     if (KERN_SUCCESS != kr)
3924         IOLog("thread_policy_set(%d)\n", kr);
3925
3926     do {
3927
3928 //      randomDelay();
3929
3930         semaphore_wait( gJobsSemaphore );
3931
3932         IOTakeLock( gJobsLock );
3933         job = (_IOServiceJob *) gJobs->getFirstObject();
3934         job->retain();
3935         gJobs->removeObject(job);
3936         if( job) {
3937             gOutstandingJobs--;
3938 //          gNumConfigThreads--;        // we're out of service
3939             gNumWaitingThreads--;       // we're out of service
3940         }
3941         IOUnlock( gJobsLock );
3942
3943         if( job) {
3944
3945           nub = job->nub;
3946
3947           if( gIOKitDebug & kIOLogConfig)
3948             LOG("config(%p): starting on %s, %d\n",
3949                         OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
3950
3951           switch( job->type) {
3952
3953             case kMatchNubJob:
3954                 nub->doServiceMatch( job->options );
3955                 break;
3956
3957             default:
3958                 LOG("config(%p): strange type (%d)\n",
3959                         OBFUSCATE(IOThreadSelf()), job->type );
3960                 break;
3961             }
3962
3963             nub->release();
3964             job->release();
3965
3966             IOTakeLock( gJobsLock );
3967             alive = (gOutstandingJobs > gNumWaitingThreads);
3968             if( alive)
3969                 gNumWaitingThreads++;   // back in service
3970 //              gNumConfigThreads++;
3971             else {
3972                 if( 0 == --gNumConfigThreads) {
3973 //                    IOLog("MATCH IDLE\n");
3974                     IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
3975                 }
3976             }
3977             IOUnlock( gJobsLock );
3978         }
3979
3980     } while( alive );
3981
3982     if( gIOKitDebug & kIOLogConfig)
3983         LOG("config(%p): terminating\n", OBFUSCATE(IOThreadSelf()) );
3984
3985     self->release();
3986 }
3987
3988 IOReturn IOService::waitMatchIdle( UInt32 msToWait )
3989 {
3990     bool            wait;
3991     int             waitResult = THREAD_AWAKENED;
3992     bool            computeDeadline = true;
3993     AbsoluteTime    deadline;
3994
3995     IOLockLock( gJobsLock );
3996     do {
3997         wait = (0 != gNumConfigThreads);
3998         if( wait) {
3999             if( msToWait) {
4000                 if( computeDeadline ) {
4001                     clock_interval_to_deadline(
4002                           msToWait, kMillisecondScale, &deadline );
4003                     computeDeadline = false;
4004                 }
4005                           waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
4006                                                                 deadline, THREAD_UNINT );
4007                    } else {
4008                           waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
4009                                                                 THREAD_UNINT );
4010                 }
4011         }
4012     } while( wait && (waitResult != THREAD_TIMED_OUT));
4013         IOLockUnlock( gJobsLock );
4014
4015     if( waitResult == THREAD_TIMED_OUT)
4016         return( kIOReturnTimeout );
4017     else
4018         return( kIOReturnSuccess );
4019 }
4020
4021 void IOService::cpusRunning(void)
4022 {
4023     gCPUsRunning = true;
4024 }
4025
4026 void _IOServiceJob::pingConfig( _IOServiceJob * job )
4027 {
4028     int         count;
4029     bool        create;
4030
4031     assert( job );
4032
4033     IOTakeLock( gJobsLock );
4034
4035     gOutstandingJobs++;
4036     gJobs->setLastObject( job );
4037
4038     count = gNumWaitingThreads;
4039 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
4040
4041     create = (  (gOutstandingJobs > count)
4042                 && ((gNumConfigThreads < kMaxConfigThreads)
4043             || (job->nub == gIOResources)
4044             || !gCPUsRunning));
4045     if( create) {
4046         gNumConfigThreads++;
4047         gNumWaitingThreads++;
4048     }
4049
4050     IOUnlock( gJobsLock );
4051
4052     job->release();
4053
4054     if( create) {
4055         if( gIOKitDebug & kIOLogConfig)
4056             LOG("config(%d): creating\n", gNumConfigThreads - 1);
4057         _IOConfigThread::configThread();
4058     }
4059
4060     semaphore_signal( gJobsSemaphore );
4061 }
4062
4063 struct IOServiceMatchContext
4064 {
4065     OSDictionary * table;
4066     OSObject *     result;
4067     uint32_t       options;
4068     uint32_t       state;
4069     uint32_t       count;
4070     uint32_t       done;
4071 };
4072
4073 bool IOService::instanceMatch(const OSObject * entry, void * context)
4074 {
4075     IOServiceMatchContext * ctx = (typeof(ctx)) context;
4076     IOService *    service = (typeof(service)) entry;
4077     OSDictionary * table   = ctx->table;
4078     uint32_t       options = ctx->options;
4079     uint32_t       state   = ctx->state;
4080     uint32_t       done;
4081     bool           match;
4082
4083     done = 0;
4084     do
4085     {
4086         match = ((state == (state & service->__state[0]))
4087                 && (0 == (service->__state[0] & kIOServiceInactiveState)));
4088         if (!match) break;
4089         ctx->count += table->getCount();
4090         match = service->matchInternal(table, options, &done);
4091         ctx->done += done;
4092     }
4093     while (false);
4094     if (!match)
4095         return (false);
4096
4097     if ((kIONotifyOnce & options) && (ctx->done == ctx->count))
4098     {
4099         service->retain();
4100         ctx->result = service;
4101         return (true);
4102     }
4103     else if (!ctx->result)
4104     {
4105         ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
4106     }
4107     else
4108     {
4109         ((OSSet *)ctx->result)->setObject(service);
4110     }
4111     return (false);
4112 }
4113
4114 // internal - call with gNotificationLock
4115 OSObject * IOService::copyExistingServices( OSDictionary * matching,
4116                  IOOptionBits inState, IOOptionBits options )
4117 {
4118     OSObject *   current = 0;
4119     OSIterator * iter;
4120     IOService *  service;
4121     OSObject *   obj;
4122     OSString *   str;
4123
4124     if( !matching)
4125         return( 0 );
4126
4127 #if MATCH_DEBUG
4128     OSSerialize * s = OSSerialize::withCapacity(128);
4129     matching->serialize(s);
4130 #endif
4131
4132     if((obj = matching->getObject(gIOProviderClassKey))
4133       && gIOResourcesKey
4134       && gIOResourcesKey->isEqualTo(obj)
4135       && (service = gIOResources))
4136     {
4137         if( (inState == (service->__state[0] & inState))
4138           && (0 == (service->__state[0] & kIOServiceInactiveState))
4139           &&  service->matchPassive(matching, options))
4140         {
4141             if( options & kIONotifyOnce)
4142             {
4143                 service->retain();
4144                 current = service;
4145             }
4146             else
4147                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
4148         }
4149     }
4150     else
4151     {
4152         IOServiceMatchContext ctx;
4153         ctx.table   = matching;
4154         ctx.state   = inState;
4155         ctx.count   = 0;
4156         ctx.done    = 0;
4157         ctx.options = options;
4158         ctx.result  = 0;
4159
4160         if ((str = OSDynamicCast(OSString, obj)))
4161         {
4162             const OSSymbol * sym = OSSymbol::withString(str);
4163             OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
4164             sym->release();
4165         }
4166         else
4167         {
4168             IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
4169         }
4170
4171
4172         current = ctx.result;
4173
4174         options |= kIOServiceInternalDone | kIOServiceClassDone;
4175         if (current && (ctx.done != ctx.count))
4176         {
4177             OSSet *
4178             source = OSDynamicCast(OSSet, current);
4179             current = 0;
4180             while ((service = (IOService *) source->getAnyObject()))
4181             {
4182                 if (service->matchPassive(matching, options))
4183                 {
4184                     if( options & kIONotifyOnce)
4185                     {
4186                         service->retain();
4187                         current = service;
4188                         break;
4189                     }
4190                     if( current)
4191                     {
4192                         ((OSSet *)current)->setObject( service );
4193                     }
4194                     else
4195                     {
4196                         current = OSSet::withObjects(
4197                                         (const OSObject **) &service, 1, 1 );
4198                     }
4199                 }
4200                 source->removeObject(service);
4201             }
4202             source->release();
4203         }
4204     }
4205
4206 #if MATCH_DEBUG
4207     {
4208         OSObject * _current = 0;
4209
4210         iter = IORegistryIterator::iterateOver( gIOServicePlane,
4211                                             kIORegistryIterateRecursively );
4212         if( iter) {
4213             do {
4214                 iter->reset();
4215                 while( (service = (IOService *) iter->getNextObject())) {
4216                     if( (inState == (service->__state[0] & inState))
4217                     && (0 == (service->__state[0] & kIOServiceInactiveState))
4218                     &&  service->matchPassive(matching, 0)) {
4219
4220                         if( options & kIONotifyOnce) {
4221                             service->retain();
4222                             _current = service;
4223                             break;
4224                         }
4225                         if( _current)
4226                             ((OSSet *)_current)->setObject( service );
4227                         else
4228                             _current = OSSet::withObjects(
4229                                             (const OSObject **) &service, 1, 1 );
4230                     }
4231                 }
4232             } while( !service && !iter->isValid());
4233             iter->release();
4234         }
4235
4236
4237         if ( ((current != 0) != (_current != 0))
4238         || (current && _current && !current->isEqualTo(_current)))
4239         {
4240             OSSerialize * s1 = OSSerialize::withCapacity(128);
4241             OSSerialize * s2 = OSSerialize::withCapacity(128);
4242             current->serialize(s1);
4243             _current->serialize(s2);
4244             kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", OBFUSCATE(current),
4245                 OBFUSCATE(_current), s->text(), s1->text(), s2->text());
4246             s1->release();
4247             s2->release();
4248         }
4249
4250         if (_current) _current->release();
4251     }
4252
4253     s->release();
4254 #endif
4255
4256     if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
4257         iter = OSCollectionIterator::withCollection( (OSSet *)current );
4258         current->release();
4259         current = iter;
4260     }
4261
4262     return( current );
4263 }
4264
4265 // public version
4266 OSIterator * IOService::getMatchingServices( OSDictionary * matching )
4267 {
4268     OSIterator *        iter;
4269
4270     // is a lock even needed?
4271     LOCKWRITENOTIFY();
4272
4273     iter = (OSIterator *) copyExistingServices( matching,
4274                                                 kIOServiceMatchedState );
4275
4276     UNLOCKNOTIFY();
4277
4278     return( iter );
4279 }
4280
4281 IOService * IOService::copyMatchingService( OSDictionary * matching )
4282 {
4283     IOService * service;
4284
4285     // is a lock even needed?
4286     LOCKWRITENOTIFY();
4287
4288     service = (IOService *) copyExistingServices( matching,
4289                                                 kIOServiceMatchedState, kIONotifyOnce );
4290
4291     UNLOCKNOTIFY();
4292
4293     return( service );
4294 }
4295
4296 struct _IOServiceMatchingNotificationHandlerRef
4297 {
4298     IOServiceNotificationHandler handler;
4299     void * ref;
4300 };
4301
4302 static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon,
4303                                                    IOService * newService,
4304                                                    IONotifier * notifier )
4305 {
4306     return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService));
4307 }
4308
4309 // internal - call with gNotificationLock
4310 IONotifier * IOService::setNotification(
4311             const OSSymbol * type, OSDictionary * matching,
4312             IOServiceMatchingNotificationHandler handler, void * target, void * ref,
4313             SInt32 priority )
4314 {
4315     _IOServiceNotifier * notify = 0;
4316     OSOrderedSet *      set;
4317
4318     if( !matching)
4319         return( 0 );
4320
4321     notify = new _IOServiceNotifier;
4322     if( notify && !notify->init()) {
4323         notify->release();
4324         notify = 0;
4325     }
4326
4327     if( notify) {
4328         notify->handler = handler;
4329         notify->target = target;
4330         notify->matching = matching;
4331         matching->retain();
4332         if (handler == &_IOServiceMatchingNotificationHandler)
4333         {
4334             notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
4335             notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
4336         }
4337         else
4338             notify->ref = ref;
4339         notify->priority = priority;
4340         notify->state = kIOServiceNotifyEnable;
4341         queue_init( &notify->handlerInvocations );
4342
4343         ////// queue
4344
4345         if( 0 == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
4346             set = OSOrderedSet::withCapacity( 1,
4347                         IONotifyOrdering, 0 );
4348             if( set) {
4349                 gNotifications->setObject( type, set );
4350                 set->release();
4351             }
4352         }
4353         notify->whence = set;
4354         if( set)
4355             set->setObject( notify );
4356     }
4357
4358     return( notify );
4359 }
4360
4361 // internal - call with gNotificationLock
4362 IONotifier * IOService::doInstallNotification(
4363                         const OSSymbol * type, OSDictionary * matching,
4364                         IOServiceMatchingNotificationHandler handler,
4365                         void * target, void * ref,
4366                         SInt32 priority, OSIterator ** existing )
4367 {
4368     OSIterator *        exist;
4369     IONotifier *        notify;
4370     IOOptionBits        inState;
4371
4372     if( !matching)
4373         return( 0 );
4374
4375     if( type == gIOPublishNotification)
4376         inState = kIOServiceRegisteredState;
4377
4378     else if( type == gIOFirstPublishNotification)
4379         inState = kIOServiceFirstPublishState;
4380
4381     else if (type == gIOMatchedNotification)
4382         inState = kIOServiceMatchedState;
4383
4384     else if (type == gIOFirstMatchNotification)
4385         inState = kIOServiceFirstMatchState;
4386
4387     else if( type == gIOTerminatedNotification)
4388         inState = 0;
4389     else
4390         return( 0 );
4391
4392     notify = setNotification( type, matching, handler, target, ref, priority );
4393
4394     if( inState)
4395         // get the current set
4396         exist = (OSIterator *) copyExistingServices( matching, inState );
4397     else
4398         exist = 0;
4399
4400     *existing = exist;
4401
4402     return( notify );
4403 }
4404
4405 #if !defined(__LP64__)
4406 IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
4407                                         IOServiceNotificationHandler handler,
4408                                         void * target, void * refCon,
4409                                         SInt32 priority, OSIterator ** existing )
4410 {
4411     IONotifier * result;
4412     _IOServiceMatchingNotificationHandlerRef ref;
4413     ref.handler = handler;
4414     ref.ref     = refCon;
4415
4416     result = (_IOServiceNotifier *) installNotification( type, matching,
4417                          &_IOServiceMatchingNotificationHandler,
4418                         target, &ref, priority, existing );
4419     if (result)
4420         matching->release();
4421
4422     return (result);
4423 }
4424 #endif /* !defined(__LP64__) */
4425
4426
4427 IONotifier * IOService::installNotification(
4428                         const OSSymbol * type, OSDictionary * matching,
4429                         IOServiceMatchingNotificationHandler handler,
4430                         void * target, void * ref,
4431                         SInt32 priority, OSIterator ** existing )
4432 {
4433     IONotifier * notify;
4434
4435     LOCKWRITENOTIFY();
4436
4437     notify = doInstallNotification( type, matching, handler, target, ref,
4438                 priority, existing );
4439
4440     // in case handler remove()s
4441     if (notify) notify->retain();
4442
4443     UNLOCKNOTIFY();
4444
4445     return( notify );
4446 }
4447
4448 IONotifier * IOService::addNotification(
4449                         const OSSymbol * type, OSDictionary * matching,
4450                         IOServiceNotificationHandler handler,
4451                         void * target, void * refCon,
4452                         SInt32 priority )
4453 {
4454     IONotifier * result;
4455     _IOServiceMatchingNotificationHandlerRef ref;
4456
4457     ref.handler = handler;
4458     ref.ref     = refCon;
4459
4460     result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
4461                             target, &ref, priority);
4462
4463     if (result)
4464         matching->release();
4465
4466     return (result);
4467 }
4468
4469 IONotifier * IOService::addMatchingNotification(
4470                         const OSSymbol * type, OSDictionary * matching,
4471                         IOServiceMatchingNotificationHandler handler,
4472                         void * target, void * ref,
4473                         SInt32 priority )
4474 {
4475     OSIterator *                existing = NULL;
4476     IONotifier *                ret;
4477     _IOServiceNotifier *        notify;
4478     IOService *                 next;
4479
4480     ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
4481                 handler, target, ref, priority, &existing );
4482     if (!ret) return (0);
4483
4484     // send notifications for existing set
4485     if (existing) {
4486
4487         while( (next = (IOService *) existing->getNextObject())) {
4488
4489             next->lockForArbitration();
4490             if( 0 == (next->__state[0] & kIOServiceInactiveState))
4491                 next->invokeNotifer( notify );
4492             next->unlockForArbitration();
4493         }
4494         existing->release();
4495     }
4496
4497     LOCKWRITENOTIFY();
4498     bool removed = (0 == notify->whence);
4499     notify->release();
4500     if (removed) ret = gIOServiceNullNotifier;
4501     UNLOCKNOTIFY();
4502
4503     return( ret );
4504 }
4505
4506 bool IOService::syncNotificationHandler(
4507                         void * /* target */, void * ref,
4508                         IOService * newService,
4509                         IONotifier * notifier )
4510 {
4511
4512     LOCKWRITENOTIFY();
4513     if (!*((IOService **) ref))
4514     {
4515         newService->retain();
4516         (*(IOService **) ref) = newService;
4517         WAKEUPNOTIFY(ref);
4518     }
4519     UNLOCKNOTIFY();
4520
4521     return( false );
4522 }
4523
4524 IOService * IOService::waitForMatchingService( OSDictionary * matching,
4525                                                 uint64_t timeout)
4526 {
4527     IONotifier *        notify = 0;
4528     // priority doesn't help us much since we need a thread wakeup
4529     SInt32              priority = 0;
4530     IOService *         result;
4531
4532     if (!matching)
4533         return( 0 );
4534
4535     result = NULL;
4536
4537     LOCKWRITENOTIFY();
4538     do
4539     {
4540         result = (IOService *) copyExistingServices( matching,
4541                             kIOServiceMatchedState, kIONotifyOnce );
4542         if (result)
4543             break;
4544         notify = IOService::setNotification( gIOMatchedNotification, matching,
4545                     &IOService::syncNotificationHandler, (void *) 0,
4546                     &result, priority );
4547          if (!notify)
4548             break;
4549         if (UINT64_MAX != timeout)
4550         {
4551             AbsoluteTime deadline;
4552             nanoseconds_to_absolutetime(timeout, &deadline);
4553             clock_absolutetime_interval_to_deadline(deadline, &deadline);
4554             SLEEPNOTIFYTO(&result, deadline);
4555         }
4556         else
4557         {
4558             SLEEPNOTIFY(&result);
4559         }
4560     }
4561     while( false );
4562
4563     UNLOCKNOTIFY();
4564
4565     if (notify)
4566         notify->remove();       // dequeues
4567
4568     return( result );
4569 }
4570
4571 IOService * IOService::waitForService( OSDictionary * matching,
4572                                         mach_timespec_t * timeout )
4573 {
4574     IOService * result;
4575     uint64_t    timeoutNS;
4576
4577     if (timeout)
4578     {
4579         timeoutNS = timeout->tv_sec;
4580         timeoutNS *= kSecondScale;
4581         timeoutNS += timeout->tv_nsec;
4582     }
4583     else
4584         timeoutNS = UINT64_MAX;
4585
4586     result = waitForMatchingService(matching, timeoutNS);
4587
4588     matching->release();
4589     if (result)
4590         result->release();
4591
4592     return (result);
4593 }
4594
4595 void IOService::deliverNotification( const OSSymbol * type,
4596                             IOOptionBits orNewState, IOOptionBits andNewState )
4597 {
4598     _IOServiceNotifier * notify;
4599     OSIterator *         iter;
4600     OSArray *            willSend = 0;
4601
4602     lockForArbitration();
4603
4604     if( (0 == (__state[0] & kIOServiceInactiveState))
4605      || (type == gIOTerminatedNotification)) {
4606
4607         LOCKREADNOTIFY();
4608
4609         iter = OSCollectionIterator::withCollection( (OSOrderedSet *)
4610                     gNotifications->getObject( type ) );
4611
4612         if( iter) {
4613             while( (notify = (_IOServiceNotifier *) iter->getNextObject())) {
4614
4615                 if( matchPassive(notify->matching, 0)
4616                   && (kIOServiceNotifyEnable & notify->state)) {
4617                     if( 0 == willSend)
4618                         willSend = OSArray::withCapacity(8);
4619                     if( willSend)
4620                         willSend->setObject( notify );
4621                 }
4622             }
4623             iter->release();
4624         }
4625
4626         __state[0] = (__state[0] | orNewState) & andNewState;
4627
4628         UNLOCKNOTIFY();
4629     }
4630
4631     if( willSend) {
4632         for( unsigned int idx = 0;
4633              (notify = (_IOServiceNotifier *) willSend->getObject(idx));
4634              idx++) {
4635             invokeNotifer( notify );
4636         }
4637         willSend->release();
4638     }
4639     unlockForArbitration();
4640 }
4641
4642 IOOptionBits IOService::getState( void ) const
4643 {
4644     return( __state[0] );
4645 }
4646
4647 /*
4648  * Helpers to make matching objects for simple cases
4649  */
4650
4651 OSDictionary * IOService::serviceMatching( const OSString * name,
4652                         OSDictionary * table )
4653 {
4654
4655     const OSString *    str;
4656
4657     str = OSSymbol::withString(name);
4658     if( !str)
4659         return( 0 );
4660
4661     if( !table)
4662         table = OSDictionary::withCapacity( 2 );
4663     if( table)
4664         table->setObject(gIOProviderClassKey, (OSObject *)str );
4665     str->release();
4666
4667     return( table );
4668 }
4669
4670 OSDictionary * IOService::serviceMatching( const char * name,
4671                         OSDictionary * table )
4672 {
4673     const OSString *    str;
4674
4675     str = OSSymbol::withCString( name );
4676     if( !str)
4677         return( 0 );
4678
4679     table = serviceMatching( str, table );
4680     str->release();
4681     return( table );
4682 }
4683
4684 OSDictionary * IOService::nameMatching( const OSString * name,
4685                         OSDictionary * table )
4686 {
4687     if( !table)
4688         table = OSDictionary::withCapacity( 2 );
4689     if( table)
4690         table->setObject( gIONameMatchKey, (OSObject *)name );
4691
4692     return( table );
4693 }
4694
4695 OSDictionary * IOService::nameMatching( const char * name,
4696                         OSDictionary * table )
4697 {
4698     const OSString *    str;
4699
4700     str = OSSymbol::withCString( name );
4701     if( !str)
4702         return( 0 );
4703
4704     table = nameMatching( str, table );
4705     str->release();
4706     return( table );
4707 }
4708
4709 OSDictionary * IOService::resourceMatching( const OSString * str,
4710                         OSDictionary * table )
4711 {
4712     table = serviceMatching( gIOResourcesKey, table );
4713     if( table)
4714         table->setObject( gIOResourceMatchKey, (OSObject *) str );
4715
4716     return( table );
4717 }
4718
4719 OSDictionary * IOService::resourceMatching( const char * name,
4720                         OSDictionary * table )
4721 {
4722     const OSSymbol *    str;
4723
4724     str = OSSymbol::withCString( name );
4725     if( !str)
4726         return( 0 );
4727
4728     table = resourceMatching( str, table );
4729     str->release();
4730
4731     return( table );
4732 }
4733
4734 OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
4735                         OSDictionary * table )
4736 {
4737     OSDictionary * properties;
4738
4739     properties = OSDictionary::withCapacity( 2 );
4740     if( !properties)
4741         return( 0 );
4742     properties->setObject( key, value );
4743
4744     if( !table)
4745         table = OSDictionary::withCapacity( 2 );
4746     if( table)
4747         table->setObject( gIOPropertyMatchKey, properties );
4748
4749     properties->release();
4750
4751     return( table );
4752 }
4753
4754 OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID,
4755                         OSDictionary * table )
4756 {
4757     OSNumber *     num;
4758
4759     num = OSNumber::withNumber( entryID, 64 );
4760     if( !num)
4761         return( 0 );
4762
4763     if( !table)
4764         table = OSDictionary::withCapacity( 2 );
4765     if( table)
4766         table->setObject( gIORegistryEntryIDKey, num );
4767
4768     if (num)
4769         num->release();
4770
4771     return( table );
4772 }
4773
4774
4775 /*
4776  * _IOServiceNotifier
4777  */
4778
4779 // wait for all threads, other than the current one,
4780 //  to exit the handler
4781
4782 void _IOServiceNotifier::wait()
4783 {
4784     _IOServiceNotifierInvocation * next;
4785     bool doWait;
4786
4787     do {
4788         doWait = false;
4789         queue_iterate( &handlerInvocations, next,
4790                         _IOServiceNotifierInvocation *, link) {
4791             if( next->thread != current_thread() ) {
4792                 doWait = true;
4793                 break;
4794             }
4795         }
4796         if( doWait) {
4797             state |= kIOServiceNotifyWaiter;
4798             SLEEPNOTIFY(this);
4799         }
4800
4801     } while( doWait );
4802 }
4803
4804 void _IOServiceNotifier::free()
4805 {
4806     assert( queue_empty( &handlerInvocations ));
4807     OSObject::free();
4808 }
4809
4810 void _IOServiceNotifier::remove()
4811 {
4812     LOCKWRITENOTIFY();
4813
4814     if( whence) {
4815         whence->removeObject( (OSObject *) this );
4816         whence = 0;
4817     }
4818     if( matching) {
4819         matching->release();
4820         matching = 0;
4821     }
4822
4823     state &= ~kIOServiceNotifyEnable;
4824
4825     wait();
4826
4827     UNLOCKNOTIFY();
4828
4829     release();
4830 }
4831
4832 bool _IOServiceNotifier::disable()
4833 {
4834     bool        ret;
4835
4836     LOCKWRITENOTIFY();
4837
4838     ret = (0 != (kIOServiceNotifyEnable & state));
4839     state &= ~kIOServiceNotifyEnable;
4840     if( ret)
4841         wait();
4842
4843     UNLOCKNOTIFY();
4844
4845     return( ret );
4846 }
4847
4848 void _IOServiceNotifier::enable( bool was )
4849 {
4850     LOCKWRITENOTIFY();
4851     if( was)
4852         state |= kIOServiceNotifyEnable;
4853     else
4854         state &= ~kIOServiceNotifyEnable;
4855     UNLOCKNOTIFY();
4856 }
4857
4858
4859 /*
4860  * _IOServiceNullNotifier
4861  */
4862
4863 void _IOServiceNullNotifier::taggedRetain(const void *tag) const                      {}
4864 void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const     {}
4865 void _IOServiceNullNotifier::free()                                                   {}
4866 void _IOServiceNullNotifier::wait()                                                   {}
4867 void _IOServiceNullNotifier::remove()                                                 {}
4868 void _IOServiceNullNotifier::enable(bool was)                                         {}
4869 bool _IOServiceNullNotifier::disable()                                { return(false); }
4870
4871 /*
4872  * IOResources
4873  */
4874
4875 IOService * IOResources::resources( void )
4876 {
4877     IOResources *       inst;
4878
4879     inst = new IOResources;
4880     if( inst && !inst->init()) {
4881         inst->release();
4882         inst = 0;
4883     }
4884
4885     return( inst );
4886 }
4887
4888 bool IOResources::init( OSDictionary * dictionary )
4889 {
4890     // Do super init first
4891     if ( !IOService::init() )
4892         return false;
4893
4894     // Allow PAL layer to publish a value
4895     const char *property_name;
4896     int property_value;
4897
4898     pal_get_resource_property( &property_name, &property_value );
4899
4900     if( property_name ) {
4901         OSNumber *num;
4902         const OSSymbol *        sym;
4903
4904         if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) {
4905             if( (sym = OSSymbol::withCString( property_name)) != 0 ) {
4906                 this->setProperty( sym, num );
4907                 sym->release();
4908             }
4909             num->release();
4910         }
4911     }
4912
4913     return true;
4914 }
4915
4916 IOReturn IOResources::newUserClient(task_t owningTask, void * securityID,
4917                                     UInt32 type,  OSDictionary * properties,
4918                                     IOUserClient ** handler)
4919 {
4920     return( kIOReturnUnsupported );
4921 }
4922
4923 IOWorkLoop * IOResources::getWorkLoop() const
4924 {
4925     // If we are the resource root
4926     // then use the platform's workloop
4927     if (this == (IOResources *) gIOResources)
4928         return getPlatform()->getWorkLoop();
4929     else
4930         return IOService::getWorkLoop();
4931 }
4932
4933 bool IOResources::matchPropertyTable( OSDictionary * table )
4934 {
4935     OSObject *          prop;
4936     OSString *          str;
4937     OSSet *             set;
4938     OSIterator *        iter;
4939     OSArray *           keys;
4940     bool                ok = true;
4941
4942     prop = table->getObject( gIOResourceMatchKey );
4943     str = OSDynamicCast( OSString, prop );
4944     if( str)
4945         ok = (0 != getProperty( str ));
4946
4947     else if( (set = OSDynamicCast( OSSet, prop))) {
4948
4949         iter = OSCollectionIterator::withCollection( set );
4950         ok = (iter != 0);
4951         while( ok && (str = OSDynamicCast( OSString, iter->getNextObject()) ))
4952             ok = (0 != getProperty( str ));
4953
4954         if( iter)
4955             iter->release();
4956     }
4957     else if ((prop = table->getObject(gIOResourceMatchedKey)))
4958     {
4959         keys = (OSArray *) copyProperty(gIOResourceMatchedKey);
4960         ok = false;
4961         if (keys)
4962         {
4963             // assuming OSSymbol
4964             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
4965             keys->release();
4966         }
4967     }
4968
4969     return( ok );
4970 }
4971
4972 void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
4973 {
4974     IOService::updateConsoleUsers(NULL, 0);
4975 }
4976
4977 void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
4978 {
4979     IORegistryEntry * regEntry;
4980     OSObject *        locked = kOSBooleanFalse;
4981     uint32_t          idx;
4982     bool              publish;
4983     OSDictionary *    user;
4984     static IOMessage  sSystemPower;
4985
4986     regEntry = IORegistryEntry::getRegistryRoot();
4987
4988     if (!gIOChosenEntry)
4989         gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
4990
4991     IOLockLock(gIOConsoleUsersLock);
4992
4993     if (systemMessage)
4994     {
4995         sSystemPower = systemMessage;
4996 #if HIBERNATION
4997         if ((kIOMessageSystemHasPoweredOn == systemMessage) && IOHibernateWasScreenLocked())
4998         {
4999             locked = kOSBooleanTrue;
5000         }
5001 #endif /* HIBERNATION */
5002     }
5003
5004     if (consoleUsers)
5005     {
5006         OSNumber * num = 0;
5007         gIOConsoleLoggedIn = false;
5008         for (idx = 0;
5009               (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
5010               idx++)
5011         {
5012             gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
5013                         && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
5014             if (!num)
5015             {
5016                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
5017             }
5018         }
5019         gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
5020     }
5021
5022     if (!gIOConsoleLoggedIn
5023      || (kIOMessageSystemWillSleep == sSystemPower)
5024      || (kIOMessageSystemPagingOff == sSystemPower))
5025     {
5026         locked = kOSBooleanTrue;
5027     }
5028     else if (gIOConsoleLockTime)
5029     {
5030         clock_sec_t  now;
5031         clock_usec_t microsecs;
5032
5033         clock_get_calendar_microtime(&now, &microsecs);
5034         if (gIOConsoleLockTime > now)
5035         {
5036             AbsoluteTime deadline;
5037             clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
5038             thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
5039         }
5040         else
5041         {
5042             locked = kOSBooleanTrue;
5043         }
5044     }
5045
5046     publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
5047     if (publish)
5048     {
5049         regEntry->setProperty(gIOConsoleLockedKey, locked);
5050         if (consoleUsers)
5051         {
5052             regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
5053         }
5054         OSIncrementAtomic( &gIOConsoleUsersSeed );
5055     }
5056
5057 #if HIBERNATION
5058     if (gIOChosenEntry)
5059     {
5060         if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked;
5061         else if (gIOConsoleLockTime)  gIOScreenLockState = kIOScreenLockUnlocked;
5062         else                          gIOScreenLockState = kIOScreenLockNoLock;
5063         gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
5064     }
5065 #endif /* HIBERNATION */
5066
5067     IOLockUnlock(gIOConsoleUsersLock);
5068
5069     if (publish)
5070     {
5071         publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
5072
5073         MessageClientsContext context;
5074
5075         context.service  = getServiceRoot();
5076         context.type     = kIOMessageConsoleSecurityChange;
5077         context.argument = (void *) regEntry;
5078         context.argSize  = 0;
5079
5080         applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
5081                                  &messageClientsApplier, &context );
5082     }
5083 }
5084
5085 IOReturn IOResources::setProperties( OSObject * properties )
5086 {
5087     IOReturn                    err;
5088     const OSSymbol *            key;
5089     OSDictionary *              dict;
5090     OSCollectionIterator *      iter;
5091
5092     err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
5093     if ( kIOReturnSuccess != err)
5094         return( err );
5095
5096     dict = OSDynamicCast(OSDictionary, properties);
5097     if( 0 == dict)
5098         return( kIOReturnBadArgument);
5099
5100     iter = OSCollectionIterator::withCollection( dict);
5101     if( 0 == iter)
5102         return( kIOReturnBadArgument);
5103
5104     while( (key = OSDynamicCast(OSSymbol, iter->getNextObject())))
5105     {
5106         if (gIOConsoleUsersKey == key) do
5107         {
5108             OSArray * consoleUsers;
5109             consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
5110             if (!consoleUsers)
5111                 continue;
5112             IOService::updateConsoleUsers(consoleUsers, 0);
5113         }
5114         while (false);
5115
5116         publishResource( key, dict->getObject(key) );
5117     }
5118
5119     iter->release();
5120
5121     return( kIOReturnSuccess );
5122 }
5123
5124 /*
5125  * Helpers for matching dictionaries.
5126  * Keys existing in matching are checked in properties.
5127  * Keys may be a string or OSCollection of IOStrings
5128  */
5129
5130 bool IOService::compareProperty( OSDictionary * matching,
5131                                  const char *   key )
5132 {
5133     OSObject *  value;
5134     OSObject *  prop;
5135     bool        ok;
5136
5137     value = matching->getObject( key );
5138     if( value)
5139     {
5140         prop = copyProperty(key);
5141         ok = value->isEqualTo(prop);
5142         if (prop) prop->release();
5143     }
5144     else
5145         ok = true;
5146
5147     return( ok );
5148 }
5149
5150
5151 bool IOService::compareProperty( OSDictionary *   matching,
5152                                  const OSString * key )
5153 {
5154     OSObject *  value;
5155     OSObject *  prop;
5156     bool        ok;
5157
5158     value = matching->getObject( key );
5159     if( value)
5160     {
5161         prop = copyProperty(key);
5162         ok = value->isEqualTo(prop);
5163         if (prop) prop->release();
5164     }
5165     else
5166         ok = true;
5167
5168     return( ok );
5169 }
5170
5171 bool IOService::compareProperties( OSDictionary * matching,
5172                                    OSCollection * keys )
5173 {
5174     OSCollectionIterator *      iter;
5175     const OSString *            key;
5176     bool                        ok = true;
5177
5178     if( !matching || !keys)
5179         return( false );
5180
5181     iter = OSCollectionIterator::withCollection( keys );
5182
5183     if( iter) {
5184         while( ok && (key = OSDynamicCast( OSString, iter->getNextObject())))
5185             ok = compareProperty( matching, key );
5186
5187         iter->release();
5188     }
5189     keys->release();    // !! consume a ref !!
5190
5191     return( ok );
5192 }
5193
5194 /* Helper to add a location matching dict to the table */
5195
5196 OSDictionary * IOService::addLocation( OSDictionary * table )
5197 {
5198     OSDictionary *      dict;
5199
5200     if( !table)
5201         return( 0 );
5202
5203     dict = OSDictionary::withCapacity( 1 );
5204     if( dict) {
5205         table->setObject( gIOLocationMatchKey, dict );
5206         dict->release();
5207     }
5208
5209     return( dict );
5210 }
5211
5212 /*
5213  * Go looking for a provider to match a location dict.
5214  */
5215
5216 IOService * IOService::matchLocation( IOService * /* client */ )
5217 {
5218     IOService * parent;
5219
5220     parent = getProvider();
5221
5222     if( parent)
5223         parent = parent->matchLocation( this );
5224
5225     return( parent );
5226 }
5227
5228 bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
5229 {
5230     OSString *          matched;
5231     OSObject *          obj;
5232     OSString *          str;
5233     IORegistryEntry *   entry;
5234     OSNumber *          num;
5235     bool                match = true;
5236     bool                changesOK = (0 != (kIOServiceChangesOK & options));
5237     uint32_t            count;
5238     uint32_t            done;
5239
5240     do
5241     {
5242         count = table->getCount();
5243         done = 0;
5244
5245         str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
5246         if (str) {
5247             done++;
5248             match = ((kIOServiceClassDone & options) || (0 != metaCast(str)));
5249 #if MATCH_DEBUG
5250             match = (0 != metaCast( str ));
5251             if ((kIOServiceClassDone & options) && !match) panic("classDone");
5252 #endif
5253             if ((!match) || (done == count)) break;
5254         }
5255
5256         obj = table->getObject( gIONameMatchKey );
5257         if( obj) {
5258             done++;
5259             match = compareNames( obj, changesOK ? &matched : 0 );
5260             if (!match) break;
5261             if( changesOK && matched) {
5262                 // leave a hint as to which name matched
5263                 table->setObject( gIONameMatchedKey, matched );
5264                 matched->release();
5265             }
5266             if (done == count) break;
5267         }
5268
5269         str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
5270         if (str)
5271         {
5272             const OSSymbol * sym;
5273             done++;
5274             match = false;
5275             sym = copyLocation();
5276             if (sym) {
5277                 match = sym->isEqualTo( str );
5278                 sym->release();
5279             }
5280             if ((!match) || (done == count)) break;
5281         }
5282
5283         obj = table->getObject( gIOPropertyMatchKey );
5284         if( obj)
5285         {
5286             OSDictionary * dict;
5287             OSDictionary * nextDict;
5288             OSIterator *   iter;
5289             done++;
5290             match = false;
5291             dict = dictionaryWithProperties();
5292             if( dict) {
5293                 nextDict = OSDynamicCast( OSDictionary, obj);
5294                 if( nextDict)
5295                     iter = 0;
5296                 else
5297                     iter = OSCollectionIterator::withCollection(
5298                                 OSDynamicCast(OSCollection, obj));
5299
5300                 while( nextDict
5301                     || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary,
5302                                             iter->getNextObject()))))) {
5303                     match = dict->isEqualTo( nextDict, nextDict);
5304                     if( match)
5305                         break;
5306                     nextDict = 0;
5307                 }
5308                 dict->release();
5309                 if( iter)
5310                     iter->release();
5311             }
5312             if ((!match) || (done == count)) break;
5313         }
5314
5315         obj = table->getObject( gIOPropertyExistsMatchKey );
5316         if( obj)
5317         {
5318             OSDictionary * dict;
5319             OSString     * nextKey;
5320             OSIterator *   iter;
5321             done++;
5322             match = false;
5323             dict = dictionaryWithProperties();
5324             if( dict) {
5325                 nextKey = OSDynamicCast( OSString, obj);
5326                 if( nextKey)
5327                     iter = 0;
5328                 else
5329                     iter = OSCollectionIterator::withCollection(
5330                                 OSDynamicCast(OSCollection, obj));
5331
5332                 while( nextKey
5333                     || (iter && (0 != (nextKey = OSDynamicCast(OSString,
5334                                             iter->getNextObject()))))) {
5335                     match = (0 != dict->getObject(nextKey));
5336                     if( match)
5337                         break;
5338                     nextKey = 0;
5339                 }
5340                 dict->release();
5341                 if( iter)
5342                     iter->release();
5343             }
5344             if ((!match) || (done == count)) break;
5345         }
5346
5347         str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
5348         if( str) {
5349             done++;
5350             entry = IORegistryEntry::fromPath( str->getCStringNoCopy() );
5351             match = (this == entry);
5352             if( entry)
5353                 entry->release();
5354             if ((!match) || (done == count)) break;
5355         }
5356
5357         num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
5358         if (num) {
5359             done++;
5360             match = (getRegistryEntryID() == num->unsigned64BitValue());
5361             if ((!match) || (done == count)) break;
5362         }
5363
5364         num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
5365         if( num)
5366         {
5367             OSIterator *        iter;
5368             IOService *         service = 0;
5369             UInt32              serviceCount = 0;
5370
5371             done++;
5372             iter = getClientIterator();
5373             if( iter) {
5374                 while( (service = (IOService *) iter->getNextObject())) {
5375                     if( kIOServiceInactiveState & service->__state[0])
5376                         continue;
5377                     if( 0 == service->getProperty( gIOMatchCategoryKey ))
5378                         continue;
5379                     ++serviceCount;
5380                 }
5381                 iter->release();
5382             }
5383             match = (serviceCount == num->unsigned32BitValue());
5384             if ((!match) || (done == count)) break;
5385         }
5386
5387 #define propMatch(key)                                  \
5388         obj = table->getObject(key);                    \
5389         if (obj)                                        \
5390         {                                               \
5391             OSObject * prop;                            \
5392             done++;                                     \
5393             prop = copyProperty(key);                   \
5394             match = obj->isEqualTo(prop);               \
5395             if (prop) prop->release();                  \
5396             if ((!match) || (done == count)) break;     \
5397         }
5398         propMatch(gIOBSDNameKey)
5399         propMatch(gIOBSDMajorKey)
5400         propMatch(gIOBSDMinorKey)
5401         propMatch(gIOBSDUnitKey)
5402 #undef propMatch
5403     }
5404     while (false);
5405
5406     if (did) *did = done;
5407     return (match);
5408 }
5409
5410 bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
5411 {
5412     return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0));
5413 }
5414
5415 bool IOService::matchPassive(OSDictionary * table, uint32_t options)
5416 {
5417     IOService *         where;
5418     OSDictionary *      nextTable;
5419     SInt32              score;
5420     OSNumber *          newPri;
5421     bool                match = true;
5422     bool                matchParent = false;
5423     uint32_t            count;
5424     uint32_t            done;
5425
5426     assert( table );
5427
5428     OSArray* aliasServiceRegIds = NULL;
5429     IOService* foundAlternateService = NULL;
5430
5431 #if MATCH_DEBUG
5432     OSDictionary * root = table;
5433 #endif
5434
5435     where = this;
5436     do
5437     {
5438         do
5439         {
5440             count = table->getCount();
5441             if (!(kIOServiceInternalDone & options))
5442             {
5443                 match = where->matchInternal(table, options, &done);
5444                 // don't call family if we've done all the entries in the table
5445                 if ((!match) || (done == count)) break;
5446             }
5447
5448             // pass in score from property table
5449             score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
5450
5451             // do family specific matching
5452             match = where->matchPropertyTable( table, &score );
5453
5454             if( !match) {
5455 #if IOMATCHDEBUG
5456                 if( kIOLogMatch & getDebugFlags( table ))
5457                     LOG("%s: family specific matching fails\n", where->getName());
5458 #endif
5459                 break;
5460             }
5461
5462             if (kIOServiceChangesOK & options) {
5463                 // save the score
5464                 newPri = OSNumber::withNumber( score, 32 );
5465                 if( newPri) {
5466                     table->setObject( gIOProbeScoreKey, newPri );
5467                     newPri->release();
5468                 }
5469             }
5470
5471             options = 0;
5472             matchParent = false;
5473
5474             nextTable = OSDynamicCast(OSDictionary,
5475                   table->getObject( gIOParentMatchKey ));
5476             if( nextTable) {
5477                 // look for a matching entry anywhere up to root
5478                 match = false;
5479                 matchParent = true;
5480                 table = nextTable;
5481                 break;
5482             }
5483
5484             table = OSDynamicCast(OSDictionary,
5485                     table->getObject( gIOLocationMatchKey ));
5486             if (table) {
5487                 // look for a matching entry at matchLocation()
5488                 match = false;
5489                 where = where->getProvider();
5490                 if (where && (where = where->matchLocation(where))) continue;
5491             }
5492             break;
5493         }
5494         while (true);
5495
5496         if(match == true) {
5497             break;
5498         }
5499
5500         if(matchParent == true) {
5501             // check if service has an alias to search its other "parents" if a parent match isn't found
5502             OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, where->getProperty(kIOServiceLegacyMatchingRegistryIDKey));
5503             if(alternateRegistryID != NULL) {
5504                 if(aliasServiceRegIds == NULL)
5505                 {
5506                     aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
5507                 }
5508                 aliasServiceRegIds->setObject(alternateRegistryID);
5509             }
5510         }
5511         else {
5512             break;
5513         }
5514
5515         where = where->getProvider();
5516         if(where == NULL) {
5517             // there were no matching parent services, check to see if there are aliased services that have a matching parent
5518             if(aliasServiceRegIds != NULL) {
5519                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
5520                 if(numAliasedServices != 0) {
5521                     OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
5522                     if(alternateRegistryID != NULL) {
5523                         OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
5524                         aliasServiceRegIds->removeObject(numAliasedServices - 1);
5525                         if(alternateMatchingDict != NULL) {
5526                             OSSafeReleaseNULL(foundAlternateService);
5527                             foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
5528                             alternateMatchingDict->release();
5529                             if(foundAlternateService != NULL) {
5530                                 where = foundAlternateService;
5531                             }
5532                         }
5533                     }
5534                 }
5535             }
5536         }
5537     }
5538     while( where != NULL );
5539
5540     OSSafeReleaseNULL(foundAlternateService);
5541     OSSafeReleaseNULL(aliasServiceRegIds);
5542
5543 #if MATCH_DEBUG
5544     if (where != this)
5545     {
5546         OSSerialize * s = OSSerialize::withCapacity(128);
5547         root->serialize(s);
5548         kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
5549         s->release();
5550     }
5551 #endif
5552
5553     return( match );
5554 }
5555
5556
5557 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5558                                     UInt32 type,  OSDictionary * properties,
5559                                     IOUserClient ** handler )
5560 {
5561     const OSSymbol *userClientClass = 0;
5562     IOUserClient *client;
5563     OSObject *temp;
5564
5565     if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
5566         return kIOReturnSuccess;
5567
5568     // First try my own properties for a user client class name
5569     temp = getProperty(gIOUserClientClassKey);
5570     if (temp) {
5571         if (OSDynamicCast(OSSymbol, temp))
5572             userClientClass = (const OSSymbol *) temp;
5573         else if (OSDynamicCast(OSString, temp)) {
5574             userClientClass = OSSymbol::withString((OSString *) temp);
5575             if (userClientClass)
5576                 setProperty(kIOUserClientClassKey,
5577                             (OSObject *) userClientClass);
5578         }
5579     }
5580
5581     // Didn't find one so lets just bomb out now without further ado.
5582     if (!userClientClass)
5583         return kIOReturnUnsupported;
5584
5585     // This reference is consumed by the IOServiceOpen call
5586     temp = OSMetaClass::allocClassWithName(userClientClass);
5587     if (!temp)
5588         return kIOReturnNoMemory;
5589
5590     if (OSDynamicCast(IOUserClient, temp))
5591         client = (IOUserClient *) temp;
5592     else {
5593         temp->release();
5594         return kIOReturnUnsupported;
5595     }
5596
5597     if ( !client->initWithTask(owningTask, securityID, type, properties) ) {
5598         client->release();
5599         return kIOReturnBadArgument;
5600     }
5601
5602     if ( !client->attach(this) ) {
5603         client->release();
5604         return kIOReturnUnsupported;
5605     }
5606
5607     if ( !client->start(this) ) {
5608         client->detach(this);
5609         client->release();
5610         return kIOReturnUnsupported;
5611     }
5612
5613     *handler = client;
5614     return kIOReturnSuccess;
5615 }
5616
5617 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
5618                                     UInt32 type, IOUserClient ** handler )
5619 {
5620     return( kIOReturnUnsupported );
5621 }
5622
5623 IOReturn IOService::requestProbe( IOOptionBits options )
5624 {
5625     return( kIOReturnUnsupported);
5626 }
5627
5628 /*
5629  * Convert an IOReturn to text. Subclasses which add additional
5630  * IOReturn's should override this method and call
5631  * super::stringFromReturn if the desired value is not found.
5632  */
5633
5634 const char * IOService::stringFromReturn( IOReturn rtn )
5635 {
5636     static const IONamedValue IOReturn_values[] = {
5637         {kIOReturnSuccess,          "success"                           },
5638         {kIOReturnError,            "general error"                     },
5639         {kIOReturnNoMemory,         "memory allocation error"           },
5640         {kIOReturnNoResources,      "resource shortage"                 },
5641         {kIOReturnIPCError,         "Mach IPC failure"                  },
5642         {kIOReturnNoDevice,         "no such device"                    },
5643         {kIOReturnNotPrivileged,    "privilege violation"               },
5644         {kIOReturnBadArgument,      "invalid argument"                  },
5645         {kIOReturnLockedRead,       "device is read locked"             },
5646         {kIOReturnLockedWrite,      "device is write locked"            },
5647         {kIOReturnExclusiveAccess,  "device is exclusive access"        },
5648         {kIOReturnBadMessageID,     "bad IPC message ID"                },
5649         {kIOReturnUnsupported,      "unsupported function"              },
5650         {kIOReturnVMError,          "virtual memory error"              },
5651         {kIOReturnInternalError,    "internal driver error"             },
5652         {kIOReturnIOError,          "I/O error"                         },
5653         {kIOReturnCannotLock,       "cannot acquire lock"               },
5654         {kIOReturnNotOpen,          "device is not open"                },
5655         {kIOReturnNotReadable,      "device is not readable"            },
5656         {kIOReturnNotWritable,      "device is not writeable"           },
5657         {kIOReturnNotAligned,       "alignment error"                   },
5658         {kIOReturnBadMedia,         "media error"                       },
5659         {kIOReturnStillOpen,        "device is still open"              },
5660         {kIOReturnRLDError,         "rld failure"                       },
5661         {kIOReturnDMAError,         "DMA failure"                       },
5662         {kIOReturnBusy,             "device is busy"                    },
5663         {kIOReturnTimeout,          "I/O timeout"                       },
5664         {kIOReturnOffline,          "device is offline"                 },
5665         {kIOReturnNotReady,         "device is not ready"               },
5666         {kIOReturnNotAttached,      "device/channel is not attached"    },
5667         {kIOReturnNoChannels,       "no DMA channels available"         },
5668         {kIOReturnNoSpace,          "no space for data"                 },
5669         {kIOReturnPortExists,       "device port already exists"        },
5670         {kIOReturnCannotWire,       "cannot wire physical memory"       },
5671         {kIOReturnNoInterrupt,      "no interrupt attached"             },
5672         {kIOReturnNoFrames,         "no DMA frames enqueued"            },
5673         {kIOReturnMessageTooLarge,  "message is too large"              },
5674         {kIOReturnNotPermitted,     "operation is not permitted"        },
5675         {kIOReturnNoPower,          "device is without power"           },
5676         {kIOReturnNoMedia,          "media is not present"              },
5677         {kIOReturnUnformattedMedia, "media is not formatted"            },
5678         {kIOReturnUnsupportedMode,  "unsupported mode"                  },
5679         {kIOReturnUnderrun,         "data underrun"                     },
5680         {kIOReturnOverrun,          "data overrun"                      },
5681         {kIOReturnDeviceError,      "device error"                      },
5682         {kIOReturnNoCompletion,     "no completion routine"             },
5683         {kIOReturnAborted,          "operation was aborted"             },
5684         {kIOReturnNoBandwidth,      "bus bandwidth would be exceeded"   },
5685         {kIOReturnNotResponding,    "device is not responding"          },
5686         {kIOReturnInvalid,          "unanticipated driver error"        },
5687         {0,                         NULL                                }
5688     };
5689
5690     return IOFindNameForValue(rtn, IOReturn_values);
5691 }
5692
5693 /*
5694  * Convert an IOReturn to an errno.
5695  */
5696 int IOService::errnoFromReturn( IOReturn rtn )
5697 {
5698     if (unix_err(err_get_code(rtn)) == rtn)
5699         return err_get_code(rtn);
5700
5701     switch(rtn) {
5702         // (obvious match)
5703         case kIOReturnSuccess:
5704             return(0);
5705         case kIOReturnNoMemory:
5706             return(ENOMEM);
5707         case kIOReturnNoDevice:
5708             return(ENXIO);
5709         case kIOReturnVMError:
5710             return(EFAULT);
5711         case kIOReturnNotPermitted:
5712             return(EPERM);
5713         case kIOReturnNotPrivileged:
5714             return(EACCES);
5715         case kIOReturnIOError:
5716             return(EIO);
5717         case kIOReturnNotWritable:
5718             return(EROFS);
5719         case kIOReturnBadArgument:
5720             return(EINVAL);
5721         case kIOReturnUnsupported:
5722             return(ENOTSUP);
5723         case kIOReturnBusy:
5724             return(EBUSY);
5725         case kIOReturnNoPower:
5726             return(EPWROFF);
5727         case kIOReturnDeviceError:
5728             return(EDEVERR);
5729         case kIOReturnTimeout:
5730             return(ETIMEDOUT);
5731         case kIOReturnMessageTooLarge:
5732             return(EMSGSIZE);
5733         case kIOReturnNoSpace:
5734             return(ENOSPC);
5735         case kIOReturnCannotLock:
5736             return(ENOLCK);
5737
5738         // (best match)
5739         case kIOReturnBadMessageID:
5740         case kIOReturnNoCompletion:
5741         case kIOReturnNotAligned:
5742             return(EINVAL);
5743         case kIOReturnNotReady:
5744             return(EBUSY);
5745         case kIOReturnRLDError:
5746             return(EBADMACHO);
5747         case kIOReturnPortExists:
5748         case kIOReturnStillOpen:
5749             return(EEXIST);
5750         case kIOReturnExclusiveAccess:
5751         case kIOReturnLockedRead:
5752         case kIOReturnLockedWrite:
5753         case kIOReturnNotOpen:
5754         case kIOReturnNotReadable:
5755             return(EACCES);
5756         case kIOReturnCannotWire:
5757         case kIOReturnNoResources:
5758             return(ENOMEM);
5759         case kIOReturnAborted:
5760         case kIOReturnOffline:
5761         case kIOReturnNotResponding:
5762             return(EBUSY);
5763         case kIOReturnBadMedia:
5764         case kIOReturnNoMedia:
5765         case kIOReturnNotAttached:
5766         case kIOReturnUnformattedMedia:
5767             return(ENXIO); // (media error)
5768         case kIOReturnDMAError:
5769         case kIOReturnOverrun:
5770         case kIOReturnUnderrun:
5771             return(EIO); // (transfer error)
5772         case kIOReturnNoBandwidth:
5773         case kIOReturnNoChannels:
5774         case kIOReturnNoFrames:
5775         case kIOReturnNoInterrupt:
5776             return(EIO); // (hardware error)
5777         case kIOReturnError:
5778         case kIOReturnInternalError:
5779         case kIOReturnInvalid:
5780             return(EIO); // (generic error)
5781         case kIOReturnIPCError:
5782             return(EIO); // (ipc error)
5783         default:
5784             return(EIO); // (all other errors)
5785     }
5786 }
5787
5788 IOReturn IOService::message( UInt32 type, IOService * provider,
5789                                 void * argument )
5790 {
5791     /*
5792      * Generic entry point for calls from the provider.  A return value of
5793      * kIOReturnSuccess indicates that the message was received, and where
5794      * applicable, that it was successful.
5795      */
5796
5797     return kIOReturnUnsupported;
5798 }
5799
5800 /*
5801  * Device memory
5802  */
5803
5804 IOItemCount IOService::getDeviceMemoryCount( void )
5805 {
5806     OSArray *           array;
5807     IOItemCount         count;
5808
5809     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5810     if( array)
5811         count = array->getCount();
5812     else
5813         count = 0;
5814
5815     return( count);
5816 }
5817
5818 IODeviceMemory * IOService::getDeviceMemoryWithIndex( unsigned int index )
5819 {
5820     OSArray *           array;
5821     IODeviceMemory *    range;
5822
5823     array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
5824     if( array)
5825         range = (IODeviceMemory *) array->getObject( index );
5826     else
5827         range = 0;
5828
5829     return( range);
5830 }
5831
5832 IOMemoryMap * IOService::mapDeviceMemoryWithIndex( unsigned int index,
5833                                                 IOOptionBits options )
5834 {
5835     IODeviceMemory *    range;
5836     IOMemoryMap *       map;
5837
5838     range = getDeviceMemoryWithIndex( index );
5839     if( range)
5840         map = range->map( options );
5841     else
5842         map = 0;
5843
5844     return( map );
5845 }
5846
5847 OSArray * IOService::getDeviceMemory( void )
5848 {
5849     return( OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey)));
5850 }
5851
5852
5853 void IOService::setDeviceMemory( OSArray * array )
5854 {
5855     setProperty( gIODeviceMemoryKey, array);
5856 }
5857
5858 /*
5859  * For machines where the transfers on an I/O bus can stall because
5860  * the CPU is in an idle mode, These APIs allow a driver to specify
5861  * the maximum bus stall that they can handle.  0 indicates no limit.
5862  */
5863 void IOService::
5864 setCPUSnoopDelay(UInt32 __unused ns)
5865 {
5866 #if defined(__i386__) || defined(__x86_64__)
5867     ml_set_maxsnoop(ns);
5868 #endif /* defined(__i386__) || defined(__x86_64__) */
5869 }
5870
5871 UInt32 IOService::
5872 getCPUSnoopDelay()
5873 {
5874 #if defined(__i386__) || defined(__x86_64__)
5875     return ml_get_maxsnoop();
5876 #else
5877     return 0;
5878 #endif /* defined(__i386__) || defined(__x86_64__) */
5879 }
5880
5881 #if defined(__i386__) || defined(__x86_64__)
5882 static void
5883 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
5884 {
5885     static const UInt kNoReplace = -1U; // Must be an illegal index
5886     UInt replace = kNoReplace;
5887     bool setCpuDelay = false;
5888
5889     IORecursiveLockLock(sCpuDelayLock);
5890
5891     UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
5892     CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
5893     IOService * holder = NULL;
5894
5895     if (ns) {
5896         const CpuDelayEntry ne = {service, ns, delayType};
5897         holder = service;
5898         // Set maximum delay.
5899         for (UInt i = 0; i < count; i++) {
5900             IOService *thisService = entries[i].fService;
5901             bool sameType = (delayType == entries[i].fDelayType);
5902             if ((service == thisService) && sameType)
5903                 replace = i;
5904             else if (!thisService) {
5905                 if (kNoReplace == replace)
5906                     replace = i;
5907             }
5908             else if (sameType) {
5909                 const UInt32 thisMax = entries[i].fMaxDelay;
5910                 if (thisMax < ns)
5911                 {
5912                     ns = thisMax;
5913                     holder = thisService;
5914                 }
5915             }
5916         }
5917
5918         setCpuDelay = true;
5919         if (kNoReplace == replace)
5920             sCpuDelayData->appendBytes(&ne, sizeof(ne));
5921         else
5922             entries[replace] = ne;
5923     }
5924     else {
5925         ns = -1U;       // Set to max unsigned, i.e. no restriction
5926
5927         for (UInt i = 0; i < count; i++) {
5928             // Clear a maximum delay.
5929             IOService *thisService = entries[i].fService;
5930             if (thisService && (delayType == entries[i].fDelayType)) {
5931                 UInt32 thisMax = entries[i].fMaxDelay;
5932                 if (service == thisService)
5933                     replace = i;
5934                 else if (thisMax < ns) {
5935                     ns = thisMax;
5936                     holder = thisService;
5937                 }
5938             }
5939         }
5940
5941         // Check if entry found
5942         if (kNoReplace != replace) {
5943             entries[replace].fService = 0;      // Null the entry
5944             setCpuDelay = true;
5945         }
5946     }
5947
5948     if (setCpuDelay)
5949     {
5950         if (holder && debug_boot_arg) {
5951             strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
5952         }
5953
5954         // Must be safe to call from locked context
5955         if (delayType == kCpuDelayBusStall)
5956         {
5957             ml_set_maxbusdelay(ns);
5958         }
5959         else if (delayType == kCpuDelayInterrupt)
5960         {
5961             ml_set_maxintdelay(ns);
5962         }
5963         sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
5964         sCPULatencySet   [delayType]->setValue(ns);
5965
5966         OSArray * handlers = sCpuLatencyHandlers[delayType];
5967         IOService * target;
5968         if (handlers) for (unsigned int idx = 0;
5969                             (target = (IOService *) handlers->getObject(idx));
5970                             idx++)
5971         {
5972             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
5973                                             (void *) (uintptr_t) ns, holder,
5974                                             NULL, NULL);
5975         }
5976     }
5977
5978     IORecursiveLockUnlock(sCpuDelayLock);
5979 }
5980
5981 static IOReturn
5982 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
5983 {
5984     IOReturn result = kIOReturnNotFound;
5985     OSArray * array;
5986     unsigned int idx;
5987
5988     IORecursiveLockLock(sCpuDelayLock);
5989
5990     do
5991     {
5992         if (enable && !sCpuLatencyHandlers[delayType])
5993             sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
5994         array = sCpuLatencyHandlers[delayType];
5995         if (!array)
5996             break;
5997         idx = array->getNextIndexOfObject(target, 0);
5998         if (!enable)
5999         {
6000             if (-1U != idx)
6001             {
6002                 array->removeObject(idx);
6003                 result = kIOReturnSuccess;
6004             }
6005         }
6006         else
6007         {
6008             if (-1U != idx) {
6009                 result = kIOReturnExclusiveAccess;
6010                 break;
6011             }
6012             array->setObject(target);
6013
6014             UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
6015             CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
6016             UInt32 ns = -1U;    // Set to max unsigned, i.e. no restriction
6017             IOService * holder = NULL;
6018
6019             for (UInt i = 0; i < count; i++) {
6020                 if (entries[i].fService
6021                   && (delayType == entries[i].fDelayType)
6022                   && (entries[i].fMaxDelay < ns)) {
6023                     ns = entries[i].fMaxDelay;
6024                     holder = entries[i].fService;
6025                 }
6026             }
6027             target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
6028                                             (void *) (uintptr_t) ns, holder,
6029                                             NULL, NULL);
6030             result = kIOReturnSuccess;
6031         }
6032     }
6033     while (false);
6034
6035     IORecursiveLockUnlock(sCpuDelayLock);
6036
6037     return (result);
6038 }
6039
6040 #endif /* defined(__i386__) || defined(__x86_64__) */
6041
6042 void IOService::
6043 requireMaxBusStall(UInt32 __unused ns)
6044 {
6045 #if defined(__i386__) || defined(__x86_64__)
6046     requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
6047 #endif
6048 }
6049
6050 void IOService::
6051 requireMaxInterruptDelay(uint32_t __unused ns)
6052 {
6053 #if defined(__i386__) || defined(__x86_64__)
6054     requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
6055 #endif
6056 }
6057
6058 /*
6059  * Device interrupts
6060  */
6061
6062 IOReturn IOService::resolveInterrupt(IOService *nub, int source)
6063 {
6064   IOInterruptController *interruptController;
6065   OSArray               *array;
6066   OSData                *data;
6067   OSSymbol              *interruptControllerName;
6068   long                  numSources;
6069   IOInterruptSource     *interruptSources;
6070
6071   // Get the parents list from the nub.
6072   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
6073   if (array == 0) return kIOReturnNoResources;
6074
6075   // Allocate space for the IOInterruptSources if needed... then return early.
6076   if (nub->_interruptSources == 0) {
6077     numSources = array->getCount();
6078     interruptSources = (IOInterruptSource *)IOMalloc(numSources * sizeof(IOInterruptSource));
6079     if (interruptSources == 0) return kIOReturnNoMemory;
6080
6081     bzero(interruptSources, numSources * sizeof(IOInterruptSource));
6082
6083     nub->_numInterruptSources = numSources;
6084     nub->_interruptSources = interruptSources;
6085     return kIOReturnSuccess;
6086   }
6087
6088   interruptControllerName = OSDynamicCast(OSSymbol,array->getObject(source));
6089   if (interruptControllerName == 0) return kIOReturnNoResources;
6090
6091   interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
6092   if (interruptController == 0) return kIOReturnNoResources;
6093
6094   // Get the interrupt numbers from the nub.
6095   array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
6096   if (array == 0) return kIOReturnNoResources;
6097   data = OSDynamicCast(OSData, array->getObject(source));
6098   if (data == 0) return kIOReturnNoResources;
6099
6100   // Set the interruptController and interruptSource in the nub's table.
6101   interruptSources = nub->_interruptSources;
6102   interruptSources[source].interruptController = interruptController;
6103   interruptSources[source].vectorData = data;
6104
6105   return kIOReturnSuccess;
6106 }
6107
6108 IOReturn IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
6109 {
6110   IOReturn ret;
6111
6112   /* Make sure the _interruptSources are set */
6113   if (_interruptSources == 0) {
6114     ret = resolveInterrupt(this, source);
6115     if (ret != kIOReturnSuccess) return ret;
6116   }
6117
6118   /* Make sure the local source number is valid */
6119   if ((source < 0) || (source >= _numInterruptSources))
6120     return kIOReturnNoInterrupt;
6121
6122   /* Look up the contoller for the local source */
6123   *interruptController = _interruptSources[source].interruptController;
6124
6125   if (*interruptController == NULL) {
6126     if (!resolve) return kIOReturnNoInterrupt;
6127
6128     /* Try to reslove the interrupt */
6129     ret = resolveInterrupt(this, source);
6130     if (ret != kIOReturnSuccess) return ret;
6131
6132     *interruptController = _interruptSources[source].interruptController;
6133   }
6134
6135   return kIOReturnSuccess;
6136 }
6137
6138 IOReturn IOService::registerInterrupt(int source, OSObject *target,
6139                                       IOInterruptAction handler,
6140                                       void *refCon)
6141 {
6142   IOInterruptController *interruptController;
6143   IOReturn              ret;
6144
6145   ret = lookupInterrupt(source, true, &interruptController);
6146   if (ret != kIOReturnSuccess) return ret;
6147
6148   /* Register the source */
6149   return interruptController->registerInterrupt(this, source, target,
6150                                                 (IOInterruptHandler)handler,
6151                                                 refCon);
6152 }
6153
6154 IOReturn IOService::unregisterInterrupt(int source)
6155 {
6156   IOInterruptController *interruptController;
6157   IOReturn              ret;
6158
6159   ret = lookupInterrupt(source, false, &interruptController);
6160   if (ret != kIOReturnSuccess) return ret;
6161
6162   /* Unregister the source */
6163   return interruptController->unregisterInterrupt(this, source);
6164 }
6165
6166 IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
6167 {
6168   IOReportLegend * legend = NULL;
6169   IOInterruptAccountingData * oldValue = NULL;
6170   IOInterruptAccountingReporter * newArray = NULL;
6171   char subgroupName[64];
6172   int newArraySize = 0;
6173   int i = 0;
6174
6175   if (source < 0) {
6176     return kIOReturnBadArgument;
6177   }
6178
6179   /*
6180    * We support statistics on a maximum of 256 interrupts per nub; if a nub
6181    * has more than 256 interrupt specifiers associated with it, and tries
6182    * to register a high interrupt index with interrupt accounting, panic.
6183    * Having more than 256 interrupts associated with a single nub is
6184    * probably a sign that something fishy is going on.
6185    */
6186   if (source > IA_INDEX_MAX) {
6187     panic("addInterruptStatistics called for an excessively large index (%d)", source);
6188   }
6189
6190   /*
6191    * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
6192    * leaving it as is because the likelihood of contention where we are
6193    * actually growing the array is minimal (we would realistically need
6194    * to be starting a driver for the first time, with an IOReporting
6195    * client already in place).  Nonetheless, cleanup that can be done
6196    * to adhere to best practices; it'll make the code more complicated,
6197    * unfortunately.
6198    */
6199   IOLockLock(reserved->interruptStatisticsLock);
6200
6201   /*
6202    * Lazily allocate the statistics array.
6203    */
6204   if (!reserved->interruptStatisticsArray) {
6205     reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
6206     assert(reserved->interruptStatisticsArray);
6207     reserved->interruptStatisticsArrayCount = 1;
6208     bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
6209   }
6210
6211   if (source >= reserved->interruptStatisticsArrayCount) {
6212     /*
6213      * We're still within the range of supported indices, but we are out
6214      * of space in the current array.  Do a nasty realloc (because
6215      * IORealloc isn't a thing) here.  We'll double the size with each
6216      * reallocation.
6217      *
6218      * Yes, the "next power of 2" could be more efficient; but this will
6219      * be invoked incredibly rarely.  Who cares.
6220      */
6221     newArraySize = (reserved->interruptStatisticsArrayCount << 1);
6222
6223     while (newArraySize <= source)
6224       newArraySize = (newArraySize << 1);
6225     newArray = IONew(IOInterruptAccountingReporter, newArraySize);
6226
6227     assert(newArray);
6228
6229     /*
6230      * TODO: This even zeroes the memory it is about to overwrite.
6231      * Shameful; fix it.  Not particularly high impact, however.
6232      */
6233     bzero(newArray, newArraySize * sizeof(*newArray));
6234     memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
6235     IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
6236     reserved->interruptStatisticsArray = newArray;
6237     reserved->interruptStatisticsArrayCount = newArraySize;
6238   }
6239
6240   if (!reserved->interruptStatisticsArray[source].reporter) {
6241     /*
6242      * We don't have a reporter associated with this index yet, so we
6243      * need to create one.
6244      */
6245     /*
6246      * TODO: Some statistics do in fact have common units (time); should this be
6247      * split into separate reporters to communicate this?
6248      */
6249      reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
6250
6251     /*
6252      * Each statistic is given an identifier based on the interrupt index (which
6253      * should be unique relative to any single nub) and the statistic involved.
6254      * We should now have a sane (small and positive) index, so start
6255      * constructing the channels for statistics.
6256      */
6257     for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
6258       /*
6259        * TODO: Currently, this does not add channels for disabled statistics.
6260        * Will this be confusing for clients?  If so, we should just add the
6261        * channels; we can avoid updating the channels even if they exist.
6262        */
6263       if (IA_GET_STATISTIC_ENABLED(i))
6264         reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
6265     }
6266
6267     /*
6268      * We now need to add the legend for this reporter to the registry.
6269      */
6270     legend = IOReportLegend::with(OSDynamicCast(OSArray, getProperty(kIOReportLegendKey)));
6271
6272     /*
6273      * Note that while we compose the subgroup name, we do not need to
6274      * manage its lifecycle (the reporter will handle this).
6275      */
6276     snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
6277     subgroupName[sizeof(subgroupName) - 1] = 0;
6278     legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
6279     setProperty(kIOReportLegendKey, legend->getLegend());
6280     legend->release();
6281
6282     /*
6283      * TODO: Is this a good idea?  Probably not; my assumption is it opts
6284      * all entities who register interrupts into public disclosure of all
6285      * IOReporting channels.  Unfortunately, this appears to be as fine
6286      * grain as it gets.
6287      */
6288     setProperty(kIOReportLegendPublicKey, true);
6289   }
6290
6291   /*
6292    * Don't stomp existing entries.  If we are about to, panic; this
6293    * probably means we failed to tear down our old interrupt source
6294    * correctly.
6295    */
6296   oldValue = reserved->interruptStatisticsArray[source].statistics;
6297
6298   if (oldValue) {
6299     panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
6300   }
6301
6302   reserved->interruptStatisticsArray[source].statistics = statistics;
6303
6304   /*
6305    * Inherit the reporter values for each statistic.  The target may
6306    * be torn down as part of the runtime of the service (especially
6307    * for sleep/wake), so we inherit in order to avoid having values
6308    * reset for no apparent reason.  Our statistics are ultimately
6309    * tied to the index and the sevice, not to an individual target,
6310    * so we should maintain them accordingly.
6311    */
6312   interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6313
6314   IOLockUnlock(reserved->interruptStatisticsLock);
6315
6316   return kIOReturnSuccess;
6317 }
6318
6319 IOReturn IOService::removeInterruptStatistics(int source)
6320 {
6321   IOInterruptAccountingData * value = NULL;
6322
6323   if (source < 0) {
6324     return kIOReturnBadArgument;
6325   }
6326
6327   IOLockLock(reserved->interruptStatisticsLock);
6328
6329   /*
6330    * We dynamically grow the statistics array, so an excessively
6331    * large index value has NEVER been registered.  This either
6332    * means our cap on the array size is too small (unlikely), or
6333    * that we have been passed a corrupt index (this must be passed
6334    * the plain index into the interrupt specifier list).
6335    */
6336   if (source >= reserved->interruptStatisticsArrayCount) {
6337     panic("removeInterruptStatistics called for index %d, which was never registered", source);
6338   }
6339
6340   assert(reserved->interruptStatisticsArray);
6341
6342   /*
6343    * If there is no existing entry, we are most likely trying to
6344    * free an interrupt owner twice, or we have corrupted the
6345    * index value.
6346    */
6347   value = reserved->interruptStatisticsArray[source].statistics;
6348
6349   if (!value) {
6350     panic("removeInterruptStatistics called for empty index %d", source);
6351   }
6352
6353   /*
6354    * We update the statistics, so that any delta with the reporter
6355    * state is not lost.
6356    */
6357   interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
6358   reserved->interruptStatisticsArray[source].statistics = NULL;
6359   IOLockUnlock(reserved->interruptStatisticsLock);
6360
6361   return kIOReturnSuccess;
6362 }
6363
6364 IOReturn IOService::getInterruptType(int source, int *interruptType)
6365 {
6366   IOInterruptController *interruptController;
6367   IOReturn              ret;
6368
6369   ret = lookupInterrupt(source, true, &interruptController);
6370   if (ret != kIOReturnSuccess) return ret;
6371
6372   /* Return the type */
6373   return interruptController->getInterruptType(this, source, interruptType);
6374 }
6375
6376 IOReturn IOService::enableInterrupt(int source)
6377 {
6378   IOInterruptController *interruptController;
6379   IOReturn              ret;
6380
6381   ret = lookupInterrupt(source, false, &interruptController);
6382   if (ret != kIOReturnSuccess) return ret;
6383
6384   /* Enable the source */
6385   return interruptController->enableInterrupt(this, source);
6386 }
6387
6388 IOReturn IOService::disableInterrupt(int source)
6389 {
6390   IOInterruptController *interruptController;
6391   IOReturn              ret;
6392
6393   ret = lookupInterrupt(source, false, &interruptController);
6394   if (ret != kIOReturnSuccess) return ret;
6395
6396   /* Disable the source */
6397   return interruptController->disableInterrupt(this, source);
6398 }
6399
6400 IOReturn IOService::causeInterrupt(int source)
6401 {
6402   IOInterruptController *interruptController;
6403   IOReturn              ret;
6404
6405   ret = lookupInterrupt(source, false, &interruptController);
6406   if (ret != kIOReturnSuccess) return ret;
6407
6408   /* Cause an interrupt for the source */
6409   return interruptController->causeInterrupt(this, source);
6410 }
6411
6412 IOReturn IOService::configureReport(IOReportChannelList    *channelList,
6413                                     IOReportConfigureAction action,
6414                                     void                   *result,
6415                                     void                   *destination)
6416 {
6417     unsigned cnt;
6418
6419     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6420         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6421             if (pwrMgt) configurePowerStatesReport(action, result);
6422             else  return kIOReturnUnsupported;
6423         }
6424         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6425             if (pwrMgt) configureSimplePowerReport(action, result);
6426             else  return kIOReturnUnsupported;
6427         }
6428     }
6429
6430     IOLockLock(reserved->interruptStatisticsLock);
6431
6432     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6433     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6434         if (reserved->interruptStatisticsArray[cnt].reporter) {
6435             /*
6436              * If the reporter is currently associated with the statistics
6437              * for an event source, we may need to update the reporter.
6438              */
6439             if (reserved->interruptStatisticsArray[cnt].statistics)
6440                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6441
6442             reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
6443         }
6444     }
6445
6446     IOLockUnlock(reserved->interruptStatisticsLock);
6447
6448     return kIOReturnSuccess;
6449 }
6450
6451 IOReturn IOService::updateReport(IOReportChannelList      *channelList,
6452                                  IOReportUpdateAction      action,
6453                                  void                     *result,
6454                                  void                     *destination)
6455 {
6456     unsigned cnt;
6457
6458     for (cnt = 0; cnt < channelList->nchannels; cnt++) {
6459         if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) {
6460             if (pwrMgt) updatePowerStatesReport(action, result, destination);
6461             else return kIOReturnUnsupported;
6462         }
6463         else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) {
6464             if (pwrMgt) updateSimplePowerReport(action, result, destination);
6465             else return kIOReturnUnsupported;
6466         }
6467     }
6468
6469     IOLockLock(reserved->interruptStatisticsLock);
6470
6471     /* The array count is signed (because the interrupt indices are signed), hence the cast */
6472     for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
6473         if (reserved->interruptStatisticsArray[cnt].reporter) {
6474             /*
6475              * If the reporter is currently associated with the statistics
6476              * for an event source, we need to update the reporter.
6477              */
6478             if (reserved->interruptStatisticsArray[cnt].statistics)
6479                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
6480
6481             reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
6482         }
6483     }
6484
6485     IOLockUnlock(reserved->interruptStatisticsLock);
6486
6487     return kIOReturnSuccess;
6488 }
6489
6490 uint64_t IOService::getAuthorizationID( void )
6491 {
6492     return reserved->authorizationID;
6493 }
6494
6495 IOReturn IOService::setAuthorizationID( uint64_t authorizationID )
6496 {
6497     OSObject * entitlement;
6498     IOReturn status;
6499
6500     entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
6501
6502     if ( entitlement )
6503     {
6504         if ( entitlement == kOSBooleanTrue )
6505         {
6506             reserved->authorizationID = authorizationID;
6507
6508             status = kIOReturnSuccess;
6509         }
6510         else
6511         {
6512             status = kIOReturnNotPrivileged;
6513         }
6514
6515         entitlement->release( );
6516     }
6517     else
6518     {
6519         status = kIOReturnNotPrivileged;
6520     }
6521
6522     return status;
6523 }
6524
6525 #if __LP64__
6526 OSMetaClassDefineReservedUsed(IOService, 0);
6527 OSMetaClassDefineReservedUsed(IOService, 1);
6528 OSMetaClassDefineReservedUnused(IOService, 2);
6529 OSMetaClassDefineReservedUnused(IOService, 3);
6530 OSMetaClassDefineReservedUnused(IOService, 4);
6531 OSMetaClassDefineReservedUnused(IOService, 5);
6532 OSMetaClassDefineReservedUnused(IOService, 6);
6533 OSMetaClassDefineReservedUnused(IOService, 7);
6534 #else
6535 OSMetaClassDefineReservedUsed(IOService, 0);
6536 OSMetaClassDefineReservedUsed(IOService, 1);
6537 OSMetaClassDefineReservedUsed(IOService, 2);
6538 OSMetaClassDefineReservedUsed(IOService, 3);
6539 OSMetaClassDefineReservedUsed(IOService, 4);
6540 OSMetaClassDefineReservedUsed(IOService, 5);
6541 OSMetaClassDefineReservedUsed(IOService, 6);
6542 OSMetaClassDefineReservedUsed(IOService, 7);
6543 #endif
6544 OSMetaClassDefineReservedUnused(IOService, 8);
6545 OSMetaClassDefineReservedUnused(IOService, 9);
6546 OSMetaClassDefineReservedUnused(IOService, 10);
6547 OSMetaClassDefineReservedUnused(IOService, 11);
6548 OSMetaClassDefineReservedUnused(IOService, 12);
6549 OSMetaClassDefineReservedUnused(IOService, 13);
6550 OSMetaClassDefineReservedUnused(IOService, 14);
6551 OSMetaClassDefineReservedUnused(IOService, 15);
6552 OSMetaClassDefineReservedUnused(IOService, 16);
6553 OSMetaClassDefineReservedUnused(IOService, 17);
6554 OSMetaClassDefineReservedUnused(IOService, 18);
6555 OSMetaClassDefineReservedUnused(IOService, 19);
6556 OSMetaClassDefineReservedUnused(IOService, 20);
6557 OSMetaClassDefineReservedUnused(IOService, 21);
6558 OSMetaClassDefineReservedUnused(IOService, 22);
6559 OSMetaClassDefineReservedUnused(IOService, 23);
6560 OSMetaClassDefineReservedUnused(IOService, 24);
6561 OSMetaClassDefineReservedUnused(IOService, 25);
6562 OSMetaClassDefineReservedUnused(IOService, 26);
6563 OSMetaClassDefineReservedUnused(IOService, 27);
6564 OSMetaClassDefineReservedUnused(IOService, 28);
6565 OSMetaClassDefineReservedUnused(IOService, 29);
6566 OSMetaClassDefineReservedUnused(IOService, 30);
6567 OSMetaClassDefineReservedUnused(IOService, 31);
6568 OSMetaClassDefineReservedUnused(IOService, 32);
6569 OSMetaClassDefineReservedUnused(IOService, 33);
6570 OSMetaClassDefineReservedUnused(IOService, 34);
6571 OSMetaClassDefineReservedUnused(IOService, 35);
6572 OSMetaClassDefineReservedUnused(IOService, 36);
6573 OSMetaClassDefineReservedUnused(IOService, 37);
6574 OSMetaClassDefineReservedUnused(IOService, 38);
6575 OSMetaClassDefineReservedUnused(IOService, 39);
6576 OSMetaClassDefineReservedUnused(IOService, 40);
6577 OSMetaClassDefineReservedUnused(IOService, 41);
6578 OSMetaClassDefineReservedUnused(IOService, 42);
6579 OSMetaClassDefineReservedUnused(IOService, 43);
6580 OSMetaClassDefineReservedUnused(IOService, 44);
6581 OSMetaClassDefineReservedUnused(IOService, 45);
6582 OSMetaClassDefineReservedUnused(IOService, 46);
6583 OSMetaClassDefineReservedUnused(IOService, 47);