iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30 #include <IOKit/IOService.h>
  31 #include <libkern/OSDebug.h>
  32 #include <libkern/c++/OSContainers.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/c++/OSUnserialize.h>
  35 #include <libkern/c++/OSKext.h>
  36 #include <libkern/Block.h>
  37 #include <IOKit/IOCatalogue.h>
  38 #include <IOKit/IOCommand.h>
  39 #include <IOKit/IODeviceTreeSupport.h>
  40 #include <IOKit/IODeviceMemory.h>
  41 #include <IOKit/IOInterrupts.h>
  42 #include <IOKit/IOInterruptController.h>
  43 #include <IOKit/IOPlatformExpert.h>
  44 #include <IOKit/IOMessage.h>
  45 #include <IOKit/IOLib.h>
  46 #include <IOKit/IOKitKeysPrivate.h>
  47 #include <IOKit/IOBSD.h>
  48 #include <IOKit/IOUserClient.h>
  49 #include <IOKit/IOUserServer.h>
  50 #include <IOKit/IOWorkLoop.h>
  51 #include <IOKit/IOTimeStamp.h>
  52 #include <IOKit/IOHibernatePrivate.h>
  53 #include <IOKit/IOInterruptAccountingPrivate.h>
  54 #include <IOKit/IOKernelReporters.h>
  55 #include <IOKit/AppleKeyStoreInterface.h>
  56 #include <IOKit/pwr_mgt/RootDomain.h>
  57 #include <IOKit/IOCPU.h>
  58 #include <mach/sync_policy.h>
  59 #include <mach/thread_info.h>
  60 #include <IOKit/assert.h>
  61 #include <sys/errno.h>
  62 #include <sys/kdebug.h>
  63 #include <string.h>
  64
  65 #include <machine/pal_routines.h>
  66
  67 #define LOG kprintf
  68 //#define LOG IOLog
  69 #define MATCH_DEBUG     0
  70 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  71
  72 // disabled since lockForArbitration() can be held externally
  73 #define DEBUG_NOTIFIER_LOCKED   0
  74
  75 enum{
  76         kIOUserServerCheckInTimeoutSecs = 120ULL
  77 };
  78
  79 #include "IOServicePrivate.h"
  80 #include "IOKitKernelInternal.h"
  81
  82 // take lockForArbitration before LOCKNOTIFY
  83
  84 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  85
  86 #define super IORegistryEntry
  87
  88 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  89
  90 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  91 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  92
  93 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  94
  95 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  96
  97 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  98
  99 OSDefineMetaClassAndStructors(IOResources, IOService)
 100 OSDefineMetaClassAndStructors(IOUserResources, IOService)
 101
 102 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
 103
 104 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
 105
 106 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 107
 108 static IOPlatformExpert *       gIOPlatform;
 109 static class IOPMrootDomain *   gIOPMRootDomain;
 110 const IORegistryPlane *         gIOServicePlane;
 111 const IORegistryPlane *         gIOPowerPlane;
 112 const OSSymbol *                gIODeviceMemoryKey;
 113 const OSSymbol *                gIOInterruptControllersKey;
 114 const OSSymbol *                gIOInterruptSpecifiersKey;
 115
 116 const OSSymbol *                gIOResourcesKey;
 117 const OSSymbol *                gIOUserResourcesKey;
 118 const OSSymbol *                gIOResourceMatchKey;
 119 const OSSymbol *                gIOResourceMatchedKey;
 120 const OSSymbol *                gIOResourceIOKitKey;
 121
 122 const OSSymbol *                gIOProviderClassKey;
 123 const OSSymbol *                gIONameMatchKey;
 124 const OSSymbol *                gIONameMatchedKey;
 125 const OSSymbol *                gIOPropertyMatchKey;
 126 const OSSymbol *                gIOPropertyExistsMatchKey;
 127 const OSSymbol *                gIOLocationMatchKey;
 128 const OSSymbol *                gIOParentMatchKey;
 129 const OSSymbol *                gIOPathMatchKey;
 130 const OSSymbol *                gIOMatchCategoryKey;
 131 const OSSymbol *                gIODefaultMatchCategoryKey;
 132 const OSSymbol *                gIOMatchedServiceCountKey;
 133 const OSSymbol *                gIOMatchedPersonalityKey;
 134 const OSSymbol *                gIORematchPersonalityKey;
 135 const OSSymbol *                gIORematchCountKey;
 136 const OSSymbol *                gIODEXTMatchCountKey;
 137 #if !CONFIG_EMBEDDED
 138 const OSSymbol *                gIOServiceLegacyMatchingRegistryIDKey;
 139 #endif
 140
 141 const OSSymbol *                gIOMapperIDKey;
 142 const OSSymbol *                gIOUserClientClassKey;
 143
 144 const OSSymbol *                gIOUserClassKey;
 145 const OSSymbol *                gIOUserServerClassKey;
 146 const OSSymbol *                gIOUserServerNameKey;
 147 const OSSymbol *                gIOUserServerTagKey;
 148 const OSSymbol *                gIOUserServerCDHashKey;
 149 const OSSymbol *                gIOUserUserClientKey;
 150
 151 const OSSymbol *                gIOKitDebugKey;
 152
 153 const OSSymbol *                gIOCommandPoolSizeKey;
 154
 155 const OSSymbol *                gIOConsoleLockedKey;
 156 const OSSymbol *                gIOConsoleUsersKey;
 157 const OSSymbol *                gIOConsoleSessionUIDKey;
 158 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 159 const OSSymbol *                gIOConsoleUsersSeedKey;
 160 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 161 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 162 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 163 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 164 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 165 clock_sec_t                     gIOConsoleLockTime;
 166 static bool                     gIOConsoleLoggedIn;
 167 #if HIBERNATION
 168 static OSBoolean *              gIOConsoleBooterLockState;
 169 static uint32_t                 gIOScreenLockState;
 170 #endif
 171 static IORegistryEntry *        gIOChosenEntry;
 172
 173 static int                      gIOResourceGenerationCount;
 174
 175 const OSSymbol *                gIOServiceKey;
 176 const OSSymbol *                gIOPublishNotification;
 177 const OSSymbol *                gIOFirstPublishNotification;
 178 const OSSymbol *                gIOMatchedNotification;
 179 const OSSymbol *                gIOFirstMatchNotification;
 180 const OSSymbol *                gIOTerminatedNotification;
 181 const OSSymbol *                gIOWillTerminateNotification;
 182
 183 const OSSymbol *                gIOServiceDEXTEntitlementsKey;
 184 const OSSymbol *                gIODriverKitEntitlementKey;
 185 const OSSymbol *                gIODriverKitUserClientEntitlementsKey;
 186 const OSSymbol *                gIOMatchDeferKey;
 187
 188 const OSSymbol *                gIOGeneralInterest;
 189 const OSSymbol *                gIOBusyInterest;
 190 const OSSymbol *                gIOAppPowerStateInterest;
 191 const OSSymbol *                gIOPriorityPowerStateInterest;
 192 const OSSymbol *                gIOConsoleSecurityInterest;
 193
 194 const OSSymbol *                gIOBSDKey;
 195 const OSSymbol *                gIOBSDNameKey;
 196 const OSSymbol *                gIOBSDMajorKey;
 197 const OSSymbol *                gIOBSDMinorKey;
 198 const OSSymbol *                gIOBSDUnitKey;
 199
 200 const  OSSymbol *               gAKSGetKey;
 201 #if defined(__i386__) || defined(__x86_64__)
 202 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 203 #endif
 204
 205 static OSDictionary *           gNotifications;
 206 static IORecursiveLock *        gNotificationLock;
 207
 208 static IOService *              gIOResources;
 209 static IOService *              gIOUserResources;
 210 static IOService *              gIOServiceRoot;
 211
 212 static OSOrderedSet *           gJobs;
 213 static semaphore_port_t         gJobsSemaphore;
 214 static IOLock *                 gJobsLock;
 215 static int                      gOutstandingJobs;
 216 static int                      gNumConfigThreads;
 217 static int                      gNumWaitingThreads;
 218 static IOLock *                 gIOServiceBusyLock;
 219 bool                            gCPUsRunning;
 220 bool                                                    gKextdWillTerminate;
 221
 222 static thread_t                 gIOTerminateThread;
 223 static thread_t                 gIOTerminateWorkerThread;
 224 static UInt32                   gIOTerminateWork;
 225 static OSArray *                gIOTerminatePhase2List;
 226 static OSArray *                gIOStopList;
 227 static OSArray *                gIOStopProviderList;
 228 static OSArray *                gIOFinalizeList;
 229
 230 #if !NO_KEXTD
 231 static OSArray *                gIOMatchDeferList;
 232 #endif
 233
 234 static SInt32                   gIOConsoleUsersSeed;
 235 static OSData *                 gIOConsoleUsersSeedValue;
 236
 237 extern const OSSymbol *         gIODTPHandleKey;
 238
 239 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 240
 241
 242 static IOLock *                 gIOConsoleUsersLock;
 243 static thread_call_t            gIOConsoleLockCallout;
 244 static IONotifier *             gIOServiceNullNotifier;
 245
 246 static uint32_t                 gIODextRelaunchMax = 1000;
 247
 248 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 249
 250 #define LOCKREADNOTIFY()        \
 251     IORecursiveLockLock( gNotificationLock )
 252 #define LOCKWRITENOTIFY()       \
 253     IORecursiveLockLock( gNotificationLock )
 254 #define LOCKWRITE2READNOTIFY()
 255 #define UNLOCKNOTIFY()          \
 256     IORecursiveLockUnlock( gNotificationLock )
 257 #define SLEEPNOTIFY(event) \
 258     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 259 #define SLEEPNOTIFYTO(event, deadline) \
 260     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 261 #define WAKEUPNOTIFY(event) \
 262         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 263
 264 #define randomDelay()   \
 265         int del = read_processor_clock();                               \
 266         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 267         IOSleep( del );
 268
 269 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 270
 271 #define queue_element(entry, element, type, field) do { \
 272         vm_address_t __ele = (vm_address_t) (entry);    \
 273         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 274         (element) = (type) __ele;                       \
 275     } while(0)
 276
 277 #define iterqueue(que, elt)                             \
 278         for (queue_entry_t elt = queue_first(que);      \
 279              !queue_end(que, elt);                      \
 280              elt = queue_next(elt))
 281
 282 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 283
 284 struct IOInterruptAccountingReporter {
 285         IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 286         IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 287 };
 288
 289 struct ArbitrationLockQueueElement {
 290         queue_chain_t link;
 291         IOThread      thread;
 292         IOService *   service;
 293         unsigned      count;
 294         bool          required;
 295         bool          aborted;
 296 };
 297
 298 static queue_head_t gArbitrationLockQueueActive;
 299 static queue_head_t gArbitrationLockQueueWaiting;
 300 static queue_head_t gArbitrationLockQueueFree;
 301 static IOLock *     gArbitrationLockQueueLock;
 302
 303 bool
 304 IOService::isInactive( void ) const
 305 {
 306         return 0 != (kIOServiceInactiveState & getState());
 307 }
 308
 309 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 310
 311 #if defined(__i386__) || defined(__x86_64__)
 312
 313 // Only used by the intel implementation of
 314 //     IOService::requireMaxBusStall(UInt32 ns)
 315 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 316 struct CpuDelayEntry {
 317         IOService * fService;
 318         UInt32      fMaxDelay;
 319         UInt32      fDelayType;
 320 };
 321
 322 enum {
 323         kCpuDelayBusStall, kCpuDelayInterrupt,
 324         kCpuNumDelayTypes
 325 };
 326
 327 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 328 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 329 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 330 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 331 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 332 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 333 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 334
 335 static void
 336 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 337 static IOReturn
 338 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 339
 340 #endif /* defined(__i386__) || defined(__x86_64__) */
 341
 342 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 343
 344 namespace IOServicePH
 345 {
 346 IONotifier          * fRootNotifier;
 347 OSArray             * fUserServers;
 348 OSArray             * fUserServersWait;
 349 OSArray             * fMatchingWork;
 350 OSArray             * fMatchingDelayed;
 351 IOService           * fSystemPowerAckTo;
 352 uint32_t              fSystemPowerAckRef;
 353 uint8_t               fSystemOff;
 354 uint8_t               fUserServerOff;
 355
 356 void lock();
 357 void unlock();
 358
 359 void init(IOPMrootDomain * root);
 360
 361 IOReturn systemPowerChange(
 362         void * target,
 363         void * refCon,
 364         UInt32 messageType, IOService * service,
 365         void * messageArgument, vm_size_t argSize);
 366
 367 bool matchingStart(IOService * service);
 368 void matchingEnd(IOService * service);
 369 };
 370
 371 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 372
 373 void
 374 IOService::initialize( void )
 375 {
 376         kern_return_t       err;
 377
 378         gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 379         gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 380
 381         gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 382         gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 383         gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 384         gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 385         gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 386         gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 387         gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 388         gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 389
 390         gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 391         gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 392                 kIODefaultMatchCategoryKey );
 393         gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 394                 kIOMatchedServiceCountKey );
 395         gIOMatchedPersonalityKey = OSSymbol::withCStringNoCopy(
 396                 kIOMatchedPersonalityKey );
 397         gIORematchPersonalityKey = OSSymbol::withCStringNoCopy(
 398                 kIORematchPersonalityKey );
 399         gIORematchCountKey = OSSymbol::withCStringNoCopy(
 400                 kIORematchCountKey );
 401         gIODEXTMatchCountKey = OSSymbol::withCStringNoCopy(
 402                 kIODEXTMatchCountKey );
 403
 404 #if !CONFIG_EMBEDDED
 405         gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
 406                 kIOServiceLegacyMatchingRegistryIDKey );
 407 #endif
 408
 409         PE_parse_boot_argn("dextrelaunch", &gIODextRelaunchMax, sizeof(gIODextRelaunchMax));
 410
 411         gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 412
 413         gIOUserClassKey       = OSSymbol::withCStringNoCopy(kIOUserClassKey);
 414
 415         gIOUserServerClassKey  = OSSymbol::withCStringNoCopy(kIOUserServerClassKey);
 416         gIOUserServerNameKey   = OSSymbol::withCStringNoCopy(kIOUserServerNameKey);
 417         gIOUserServerTagKey    = OSSymbol::withCStringNoCopy(kIOUserServerTagKey);
 418         gIOUserServerCDHashKey = OSSymbol::withCStringNoCopy(kIOUserServerCDHashKey);
 419         gIOUserUserClientKey   = OSSymbol::withCStringNoCopy(kIOUserUserClientKey);
 420
 421         gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 422         gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 423         gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 424         gIOResourceIOKitKey   = OSSymbol::withCStringNoCopy("IOKit");
 425
 426         gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 427         gIOInterruptControllersKey
 428                 = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 429         gIOInterruptSpecifiersKey
 430                 = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 431
 432         gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 433
 434         gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 435
 436         gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 437
 438         gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 439         gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 440         gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 441         gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 442         gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 443
 444         gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 445         gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 446         gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 447         gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 448         gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 449
 450         gNotifications              = OSDictionary::withCapacity( 1 );
 451         gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 452                 kIOPublishNotification );
 453         gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 454                 kIOFirstPublishNotification );
 455         gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 456                 kIOMatchedNotification );
 457         gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 458                 kIOFirstMatchNotification );
 459         gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 460                 kIOTerminatedNotification );
 461         gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
 462                 kIOWillTerminateNotification );
 463         gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 464
 465
 466         gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 467         gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 468         gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 469         gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 470
 471         gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 472         gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 473         gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 474         gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 475         gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 476         gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 477
 478         gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 479
 480         gIOServiceDEXTEntitlementsKey           = OSSymbol::withCStringNoCopy( kIOServiceDEXTEntitlementsKey );
 481         gIODriverKitEntitlementKey             = OSSymbol::withCStringNoCopy( kIODriverKitEntitlementKey );
 482         gIODriverKitUserClientEntitlementsKey   = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementsKey );
 483         gIOMatchDeferKey                        = OSSymbol::withCStringNoCopy( kIOMatchDeferKey );
 484
 485         gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 486 #if defined(__i386__) || defined(__x86_64__)
 487         sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 488         sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 489         uint32_t  idx;
 490         for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
 491                 sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 492                 sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 493                 assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 494         }
 495         gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 496 #endif
 497         gNotificationLock           = IORecursiveLockAlloc();
 498
 499         gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 500
 501         assert( gIOServicePlane && gIODeviceMemoryKey
 502             && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 503             && gIOResourcesKey && gNotifications && gNotificationLock
 504             && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 505             && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 506             && gIOPublishNotification && gIOMatchedNotification
 507             && gIOTerminatedNotification && gIOServiceKey
 508             && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 509             && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 510             && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 511
 512         gJobsLock   = IOLockAlloc();
 513         gJobs       = OSOrderedSet::withCapacity( 10 );
 514
 515         gIOServiceBusyLock = IOLockAlloc();
 516
 517         gIOConsoleUsersLock = IOLockAlloc();
 518
 519         err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 520
 521         gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 522
 523         IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 524
 525         assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 526             && gIOConsoleLockCallout && (err == KERN_SUCCESS));
 527
 528         gIOResources = IOResources::resources();
 529         gIOUserResources = IOUserResources::resources();
 530         assert( gIOResources && gIOUserResources );
 531
 532         gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 533         assert(gIOServiceNullNotifier);
 534
 535         gArbitrationLockQueueLock = IOLockAlloc();
 536         queue_init(&gArbitrationLockQueueActive);
 537         queue_init(&gArbitrationLockQueueWaiting);
 538         queue_init(&gArbitrationLockQueueFree);
 539
 540         assert( gArbitrationLockQueueLock );
 541
 542         gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 543         gIOStopList            = OSArray::withCapacity( 16 );
 544         gIOStopProviderList    = OSArray::withCapacity( 16 );
 545         gIOFinalizeList        = OSArray::withCapacity( 16 );
 546 #if !NO_KEXTD
 547         gIOMatchDeferList      = OSArray::withCapacity( 16 );
 548 #endif
 549         assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 550
 551         // worker thread that is responsible for terminating / cleaning up threads
 552         kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
 553         assert(gIOTerminateWorkerThread);
 554         thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
 555 }
 556
 557 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 558
 559 #if defined(__i386__) || defined(__x86_64__)
 560 extern "C" {
 561 const char *getCpuDelayBusStallHolderName(void);
 562 const char *
 563 getCpuDelayBusStallHolderName(void)
 564 {
 565         return sCPULatencyHolderName[kCpuDelayBusStall];
 566 }
 567
 568 const char *getCpuInterruptDelayHolderName(void);
 569 const char *
 570 getCpuInterruptDelayHolderName(void)
 571 {
 572         return sCPULatencyHolderName[kCpuDelayInterrupt];
 573 }
 574 }
 575 #endif
 576
 577 #if IOMATCHDEBUG
 578 static UInt64
 579 getDebugFlags( OSDictionary * props )
 580 {
 581         OSNumber *  debugProp;
 582         UInt64      debugFlags;
 583
 584         debugProp = OSDynamicCast( OSNumber,
 585             props->getObject( gIOKitDebugKey ));
 586         if (debugProp) {
 587                 debugFlags = debugProp->unsigned64BitValue();
 588         } else {
 589                 debugFlags = gIOKitDebug;
 590         }
 591
 592         return debugFlags;
 593 }
 594
 595 static UInt64
 596 getDebugFlags( IOService * inst )
 597 {
 598         OSObject *  prop;
 599         OSNumber *  debugProp;
 600         UInt64      debugFlags;
 601
 602         prop = inst->copyProperty(gIOKitDebugKey);
 603         debugProp = OSDynamicCast(OSNumber, prop);
 604         if (debugProp) {
 605                 debugFlags = debugProp->unsigned64BitValue();
 606         } else {
 607                 debugFlags = gIOKitDebug;
 608         }
 609
 610         OSSafeReleaseNULL(prop);
 611
 612         return debugFlags;
 613 }
 614 #endif
 615
 616 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 617
 618 // Probe a matched service and return an instance to be started.
 619 // The default score is from the property table, & may be altered
 620 // during probe to change the start order.
 621
 622 IOService *
 623 IOService::probe(   IOService * provider,
 624     SInt32    * score )
 625 {
 626         return this;
 627 }
 628
 629 bool
 630 IOService::start( IOService * provider )
 631 {
 632         return true;
 633 }
 634
 635 void
 636 IOService::stop( IOService * provider )
 637 {
 638         if (reserved->uvars && reserved->uvars->started && reserved->uvars->userServer) {
 639                 reserved->uvars->userServer->serviceStop(this, provider);
 640         }
 641 }
 642
 643 bool
 644 IOService::init( OSDictionary * dictionary )
 645 {
 646         bool ret;
 647
 648         ret = super::init(dictionary);
 649         if (!ret) {
 650                 return false;
 651         }
 652         if (reserved) {
 653                 return true;
 654         }
 655
 656         reserved = IONew(ExpansionData, 1);
 657         if (!reserved) {
 658                 return false;
 659         }
 660         bzero(reserved, sizeof(*reserved));
 661
 662         /*
 663          * TODO: Improve on this.  Previous efforts to more lazily allocate this
 664          * lock based on the presence of specifiers ran into issues as some
 665          * platforms set up the specifiers after IOService initialization.
 666          *
 667          * We may be able to get away with a global lock, as this should only be
 668          * contended by IOReporting clients and driver start/stop (unless a
 669          * driver wants to remove/add handlers in the course of normal operation,
 670          * which should be unlikely).
 671          */
 672         reserved->interruptStatisticsLock = IOLockAlloc();
 673         if (!reserved->interruptStatisticsLock) {
 674                 return false;
 675         }
 676
 677         return true;
 678 }
 679
 680 bool
 681 IOService::init( IORegistryEntry * from,
 682     const IORegistryPlane * inPlane )
 683 {
 684         bool ret;
 685
 686         ret = super::init(from, inPlane);
 687         if (!ret) {
 688                 return false;
 689         }
 690         if (reserved) {
 691                 return true;
 692         }
 693
 694         reserved = IONew(ExpansionData, 1);
 695         if (!reserved) {
 696                 return false;
 697         }
 698         bzero(reserved, sizeof(*reserved));
 699
 700         /*
 701          * TODO: Improve on this.  Previous efforts to more lazily allocate this
 702          * lock based on the presence of specifiers ran into issues as some
 703          * platforms set up the specifiers after IOService initialization.
 704          *
 705          * We may be able to get away with a global lock, as this should only be
 706          * contended by IOReporting clients and driver start/stop (unless a
 707          * driver wants to remove/add handlers in the course of normal operation,
 708          * which should be unlikely).
 709          */
 710         reserved->interruptStatisticsLock = IOLockAlloc();
 711         if (!reserved->interruptStatisticsLock) {
 712                 return false;
 713         }
 714
 715         return true;
 716 }
 717
 718 void
 719 IOService::free( void )
 720 {
 721         int i = 0;
 722         requireMaxBusStall(0);
 723         requireMaxInterruptDelay(0);
 724         if (getPropertyTable()) {
 725                 unregisterAllInterest();
 726         }
 727         PMfree();
 728
 729         if (reserved) {
 730                 if (reserved->interruptStatisticsArray) {
 731                         for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 732                                 if (reserved->interruptStatisticsArray[i].reporter) {
 733                                         reserved->interruptStatisticsArray[i].reporter->release();
 734                                 }
 735                         }
 736
 737                         IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 738                 }
 739
 740                 if (reserved->interruptStatisticsLock) {
 741                         IOLockFree(reserved->interruptStatisticsLock);
 742                 }
 743                 if (reserved->uvars && reserved->uvars->userServer) {
 744                         reserved->uvars->userServer->serviceFree(this);
 745                 }
 746                 IODelete(reserved, ExpansionData, 1);
 747         }
 748
 749         if (_numInterruptSources && _interruptSources) {
 750                 for (i = 0; i < _numInterruptSources; i++) {
 751                         void * block = _interruptSourcesPrivate(this)[i].vectorBlock;
 752                         if (block) {
 753                                 Block_release(block);
 754                         }
 755                 }
 756                 IOFree(_interruptSources,
 757                     _numInterruptSources * sizeofAllIOInterruptSource);
 758                 _interruptSources = NULL;
 759         }
 760
 761         super::free();
 762 }
 763
 764 /*
 765  * Attach in service plane
 766  */
 767 bool
 768 IOService::attach( IOService * provider )
 769 {
 770         bool         ok;
 771         uint32_t     count;
 772         AbsoluteTime deadline;
 773         int          waitResult = THREAD_AWAKENED;
 774         bool         wait, computeDeadline = true;
 775
 776         if (provider) {
 777                 if (gIOKitDebug & kIOLogAttach) {
 778                         LOG( "%s::attach(%s)\n", getName(),
 779                             provider->getName());
 780                 }
 781
 782                 ok   = false;
 783                 do{
 784                         wait = false;
 785                         provider->lockForArbitration();
 786                         if (provider->__state[0] & kIOServiceInactiveState) {
 787                                 ok = false;
 788                         } else {
 789                                 count = provider->getChildCount(gIOServicePlane);
 790                                 wait = (count > (kIOServiceBusyMax - 4));
 791                                 if (!wait) {
 792                                         ok = attachToParent(provider, gIOServicePlane);
 793                                 } else {
 794                                         IOLog("stalling for detach from %s\n", provider->getName());
 795                                         IOLockLock( gIOServiceBusyLock );
 796                                         provider->__state[1] |= kIOServiceWaitDetachState;
 797                                 }
 798                         }
 799                         provider->unlockForArbitration();
 800                         if (wait) {
 801                                 if (computeDeadline) {
 802                                         clock_interval_to_deadline(15, kSecondScale, &deadline);
 803                                         computeDeadline = false;
 804                                 }
 805                                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 806                                 IOLockUnlock( gIOServiceBusyLock );
 807                                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 808                                 wait = (waitResult != THREAD_TIMED_OUT);
 809                         }
 810                 }while (wait);
 811         } else {
 812                 gIOServiceRoot = this;
 813                 ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 814         }
 815
 816         if (ok && !__provider) {
 817                 (void) getProvider();
 818         }
 819
 820         return ok;
 821 }
 822
 823 IOService *
 824 IOService::getServiceRoot( void )
 825 {
 826         return gIOServiceRoot;
 827 }
 828
 829 void
 830 IOService::detach( IOService * provider )
 831 {
 832         IOService * newProvider = NULL;
 833         SInt32      busy;
 834         bool        adjParent;
 835
 836         if (gIOKitDebug & kIOLogAttach) {
 837                 LOG("%s::detach(%s)\n", getName(), provider->getName());
 838         }
 839
 840 #if !NO_KEXTD
 841         IOLockLock(gJobsLock);
 842         if (gIOMatchDeferList) {
 843                 auto idx = gIOMatchDeferList->getNextIndexOfObject(this, 0);
 844                 if (-1U != idx) {
 845                         gIOMatchDeferList->removeObject(idx);
 846                 }
 847         }
 848         if (IOServicePH::fMatchingDelayed) {
 849                 auto idx = IOServicePH::fMatchingDelayed->getNextIndexOfObject(this, 0);
 850                 if (-1U != idx) {
 851                         IOServicePH::fMatchingDelayed->removeObject(idx);
 852                 }
 853         }
 854         IOLockUnlock(gJobsLock);
 855 #endif /* NO_KEXTD */
 856
 857         lockForArbitration();
 858
 859         uint64_t regID1 = provider->getRegistryEntryID();
 860         uint64_t regID2 = getRegistryEntryID();
 861         IOServiceTrace(
 862                 IOSERVICE_DETACH,
 863                 (uintptr_t) regID1,
 864                 (uintptr_t) (regID1 >> 32),
 865                 (uintptr_t) regID2,
 866                 (uintptr_t) (regID2 >> 32));
 867
 868         adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 869             && (provider == getProvider()));
 870
 871         detachFromParent( provider, gIOServicePlane );
 872
 873         if (busy) {
 874                 newProvider = getProvider();
 875                 if (busy && (__state[1] & kIOServiceTermPhase3State) && (NULL == newProvider)) {
 876                         _adjustBusy( -busy );
 877                 }
 878         }
 879
 880         if (kIOServiceInactiveState & __state[0]) {
 881                 getMetaClass()->removeInstance(this);
 882                 IORemoveServicePlatformActions(this);
 883         }
 884
 885         unlockForArbitration();
 886
 887         if (newProvider && adjParent) {
 888                 newProvider->lockForArbitration();
 889                 newProvider->_adjustBusy(1);
 890                 newProvider->unlockForArbitration();
 891         }
 892
 893         // check for last client detach from a terminated service
 894         if (provider->lockForArbitration( true )) {
 895                 if (kIOServiceStartState & __state[1]) {
 896                         provider->scheduleTerminatePhase2();
 897                 }
 898                 if (adjParent) {
 899                         provider->_adjustBusy( -1 );
 900                 }
 901                 if ((provider->__state[1] & kIOServiceTermPhase3State)
 902                     && (NULL == provider->getClient())) {
 903                         provider->scheduleFinalize(false);
 904                 }
 905
 906                 IOLockLock( gIOServiceBusyLock );
 907                 if (kIOServiceWaitDetachState & provider->__state[1]) {
 908                         provider->__state[1] &= ~kIOServiceWaitDetachState;
 909                         thread_wakeup(&provider->__provider);
 910                 }
 911                 IOLockUnlock( gIOServiceBusyLock );
 912
 913                 provider->unlockForArbitration();
 914         }
 915 }
 916
 917 /*
 918  * Register instance - publish it for matching
 919  */
 920
 921 void
 922 IOService::registerService( IOOptionBits options )
 923 {
 924         char *              pathBuf;
 925         const char *        path;
 926         char *              skip;
 927         int                 len;
 928         enum { kMaxPathLen  = 256 };
 929         enum { kMaxChars    = 63 };
 930
 931         IORegistryEntry * parent = this;
 932         IORegistryEntry * root = getRegistryRoot();
 933         while (parent && (parent != root)) {
 934                 parent = parent->getParentEntry( gIOServicePlane);
 935         }
 936
 937         if (parent != root) {
 938                 IOLog("%s: not registry member at registerService()\n", getName());
 939                 return;
 940         }
 941
 942         // Allow the Platform Expert to adjust this node.
 943         if (gIOPlatform && (!gIOPlatform->platformAdjustService(this))) {
 944                 return;
 945         }
 946
 947         IOInstallServicePlatformActions(this);
 948
 949         if ((this != gIOResources)
 950             && (kIOLogRegister & gIOKitDebug)) {
 951                 pathBuf = (char *) IOMalloc( kMaxPathLen );
 952
 953                 IOLog( "Registering: " );
 954
 955                 len = kMaxPathLen;
 956                 if (pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 957                         path = pathBuf;
 958                         if (len > kMaxChars) {
 959                                 IOLog("..");
 960                                 len -= kMaxChars;
 961                                 path += len;
 962                                 if ((skip = strchr( path, '/'))) {
 963                                         path = skip;
 964                                 }
 965                         }
 966                 } else {
 967                         path = getName();
 968                 }
 969
 970                 IOLog( "%s\n", path );
 971
 972                 if (pathBuf) {
 973                         IOFree( pathBuf, kMaxPathLen );
 974                 }
 975         }
 976
 977         startMatching( options );
 978 }
 979
 980 void
 981 IOService::startMatching( IOOptionBits options )
 982 {
 983         IOService * provider;
 984         UInt32      prevBusy = 0;
 985         bool        needConfig;
 986         bool        needWake = false;
 987         bool        ok;
 988         bool        sync;
 989         bool        waitAgain;
 990
 991         lockForArbitration();
 992
 993         sync = (options & kIOServiceSynchronous)
 994             || ((provider = getProvider())
 995             && (provider->__state[1] & kIOServiceSynchronousState));
 996
 997         if (options & kIOServiceAsynchronous) {
 998                 sync = false;
 999         }
1000
1001         needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
1002             && (0 == (__state[0] & kIOServiceInactiveState));
1003
1004         __state[1] |= kIOServiceNeedConfigState;
1005
1006 //    __state[0] &= ~kIOServiceInactiveState;
1007
1008 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
1009 //                      OSKernelStackRemaining(), getName());
1010
1011         if (needConfig) {
1012                 needWake = (0 != (kIOServiceSyncPubState & __state[1]));
1013         }
1014
1015         if (sync) {
1016                 __state[1] |= kIOServiceSynchronousState;
1017         } else {
1018                 __state[1] &= ~kIOServiceSynchronousState;
1019         }
1020
1021         if (needConfig) {
1022                 prevBusy = _adjustBusy( 1 );
1023         }
1024
1025         unlockForArbitration();
1026
1027         if (needConfig) {
1028                 if (needWake) {
1029                         IOLockLock( gIOServiceBusyLock );
1030                         thread_wakeup((event_t) this /*&__state[1]*/ );
1031                         IOLockUnlock( gIOServiceBusyLock );
1032                 } else if (!sync || (kIOServiceAsynchronous & options)) {
1033                         ok = (NULL != _IOServiceJob::startJob( this, kMatchNubJob, options ));
1034                 } else {
1035                         do {
1036                                 if ((__state[1] & kIOServiceNeedConfigState)) {
1037                                         doServiceMatch( options );
1038                                 }
1039
1040                                 lockForArbitration();
1041                                 IOLockLock( gIOServiceBusyLock );
1042
1043                                 waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
1044                                     && (0 == (__state[0] & kIOServiceInactiveState)));
1045
1046                                 if (waitAgain) {
1047                                         __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
1048                                 } else {
1049                                         __state[1] &= ~kIOServiceSyncPubState;
1050                                 }
1051
1052                                 unlockForArbitration();
1053
1054                                 if (waitAgain) {
1055                                         assert_wait((event_t) this /*&__state[1]*/, THREAD_UNINT);
1056                                 }
1057
1058                                 IOLockUnlock( gIOServiceBusyLock );
1059                                 if (waitAgain) {
1060                                         thread_block(THREAD_CONTINUE_NULL);
1061                                 }
1062                         } while (waitAgain);
1063                 }
1064         }
1065 }
1066
1067
1068 void
1069 IOService::startDeferredMatches(void)
1070 {
1071 #if !NO_KEXTD
1072         OSArray * array;
1073
1074         IOLockLock(gJobsLock);
1075         array = gIOMatchDeferList;
1076         gIOMatchDeferList = NULL;
1077         IOLockUnlock(gJobsLock);
1078
1079         if (array) {
1080                 IOLog("deferred rematching count %d\n", array->getCount());
1081                 array->iterateObjects(^bool (OSObject * obj)
1082                 {
1083                         ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
1084                         return false;
1085                 });
1086                 array->release();
1087         }
1088 #endif /* !NO_KEXTD */
1089 }
1090
1091 void
1092 IOService::kextdLaunched(void)
1093 {
1094 #if !NO_KEXTD
1095         IOServiceTrace(IOSERVICE_KEXTD_READY, 0, 0, 0, 0);
1096         startDeferredMatches();
1097         getServiceRoot()->adjustBusy(-1);
1098         IOService::publishUserResource(gIOResourceIOKitKey);
1099 #endif /* !NO_KEXTD */
1100 }
1101
1102 IOReturn
1103 IOService::catalogNewDrivers( OSOrderedSet * newTables )
1104 {
1105         OSDictionary *      table;
1106         OSSet *             set;
1107         OSSet *             allSet = NULL;
1108         IOService *         service;
1109 #if IOMATCHDEBUG
1110         SInt32              count = 0;
1111 #endif
1112
1113         newTables->retain();
1114
1115         while ((table = (OSDictionary *) newTables->getFirstObject())) {
1116                 LOCKWRITENOTIFY();
1117                 set = (OSSet *) copyExistingServices( table,
1118                     kIOServiceRegisteredState,
1119                     kIOServiceExistingSet);
1120                 UNLOCKNOTIFY();
1121                 if (set) {
1122 #if IOMATCHDEBUG
1123                         count += set->getCount();
1124 #endif
1125                         if (allSet) {
1126                                 allSet->merge((const OSSet *) set);
1127                                 set->release();
1128                         } else {
1129                                 allSet = set;
1130                         }
1131                 }
1132
1133 #if IOMATCHDEBUG
1134                 if (getDebugFlags( table ) & kIOLogMatch) {
1135                         LOG("Matching service count = %ld\n", (long)count);
1136                 }
1137 #endif
1138                 newTables->removeObject(table);
1139         }
1140
1141         if (allSet) {
1142                 while ((service = (IOService *) allSet->getAnyObject())) {
1143                         service->startMatching(kIOServiceAsynchronous);
1144                         allSet->removeObject(service);
1145                 }
1146                 allSet->release();
1147         }
1148
1149         newTables->release();
1150
1151         return kIOReturnSuccess;
1152 }
1153
1154 _IOServiceJob *
1155 _IOServiceJob::startJob( IOService * nub, int type,
1156     IOOptionBits options )
1157 {
1158         _IOServiceJob *     job;
1159
1160         job = new _IOServiceJob;
1161         if (job && !job->init()) {
1162                 job->release();
1163                 job = NULL;
1164         }
1165
1166         if (job) {
1167                 job->type       = type;
1168                 job->nub        = nub;
1169                 job->options    = options;
1170                 nub->retain();          // thread will release()
1171                 pingConfig( job );
1172         }
1173
1174         return job;
1175 }
1176
1177 /*
1178  * Called on a registered service to see if it matches
1179  * a property table.
1180  */
1181
1182 bool
1183 IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
1184 {
1185         return matchPropertyTable(table);
1186 }
1187
1188 bool
1189 IOService::matchPropertyTable( OSDictionary * table )
1190 {
1191         return true;
1192 }
1193
1194 /*
1195  * Called on a matched service to allocate resources
1196  * before first driver is attached.
1197  */
1198
1199 IOReturn
1200 IOService::getResources( void )
1201 {
1202         return kIOReturnSuccess;
1203 }
1204
1205 /*
1206  * Client/provider accessors
1207  */
1208
1209 IOService *
1210 IOService::getProvider( void ) const
1211 {
1212         IOService * self = (IOService *) this;
1213         IOService * parent;
1214         SInt32      generation;
1215
1216         generation = getRegistryEntryGenerationCount();
1217         if (__providerGeneration == generation) {
1218                 return __provider;
1219         }
1220
1221         parent = (IOService *) getParentEntry( gIOServicePlane);
1222         if (parent == IORegistryEntry::getRegistryRoot()) {
1223                 /* root is not an IOService */
1224                 parent = NULL;
1225         }
1226
1227         self->__provider = parent;
1228         OSMemoryBarrier();
1229         // save the count from before call to getParentEntry()
1230         self->__providerGeneration = generation;
1231
1232         return parent;
1233 }
1234
1235 IOWorkLoop *
1236 IOService::getWorkLoop() const
1237 {
1238         IOService *provider = getProvider();
1239
1240         if (provider) {
1241                 return provider->getWorkLoop();
1242         } else {
1243                 return NULL;
1244         }
1245 }
1246
1247 OSIterator *
1248 IOService::getProviderIterator( void ) const
1249 {
1250         return getParentIterator( gIOServicePlane);
1251 }
1252
1253 IOService *
1254 IOService::getClient( void ) const
1255 {
1256         return (IOService *) getChildEntry( gIOServicePlane);
1257 }
1258
1259 OSIterator *
1260 IOService::getClientIterator( void ) const
1261 {
1262         return getChildIterator( gIOServicePlane);
1263 }
1264
1265 OSIterator *
1266 _IOOpenServiceIterator::iterator( OSIterator * _iter,
1267     const IOService * client,
1268     const IOService * provider )
1269 {
1270         _IOOpenServiceIterator * inst;
1271
1272         if (!_iter) {
1273                 return NULL;
1274         }
1275
1276         inst = new _IOOpenServiceIterator;
1277
1278         if (inst && !inst->init()) {
1279                 inst->release();
1280                 inst = NULL;
1281         }
1282         if (inst) {
1283                 inst->iter = _iter;
1284                 inst->client = client;
1285                 inst->provider = provider;
1286         }
1287
1288         return inst;
1289 }
1290
1291 void
1292 _IOOpenServiceIterator::free()
1293 {
1294         iter->release();
1295         if (last) {
1296                 last->unlockForArbitration();
1297         }
1298         OSIterator::free();
1299 }
1300
1301 OSObject *
1302 _IOOpenServiceIterator::getNextObject()
1303 {
1304         IOService * next;
1305
1306         if (last) {
1307                 last->unlockForArbitration();
1308         }
1309
1310         while ((next = (IOService *) iter->getNextObject())) {
1311                 next->lockForArbitration();
1312                 if ((client && (next->isOpen( client )))
1313                     || (provider && (provider->isOpen( next )))) {
1314                         break;
1315                 }
1316                 next->unlockForArbitration();
1317         }
1318
1319         last = next;
1320
1321         return next;
1322 }
1323
1324 bool
1325 _IOOpenServiceIterator::isValid()
1326 {
1327         return iter->isValid();
1328 }
1329
1330 void
1331 _IOOpenServiceIterator::reset()
1332 {
1333         if (last) {
1334                 last->unlockForArbitration();
1335                 last = NULL;
1336         }
1337         iter->reset();
1338 }
1339
1340 OSIterator *
1341 IOService::getOpenProviderIterator( void ) const
1342 {
1343         return _IOOpenServiceIterator::iterator( getProviderIterator(), this, NULL );
1344 }
1345
1346 OSIterator *
1347 IOService::getOpenClientIterator( void ) const
1348 {
1349         return _IOOpenServiceIterator::iterator( getClientIterator(), NULL, this );
1350 }
1351
1352
1353 IOReturn
1354 IOService::callPlatformFunction( const OSSymbol * functionName,
1355     bool waitForFunction,
1356     void *param1, void *param2,
1357     void *param3, void *param4 )
1358 {
1359         IOReturn  result = kIOReturnUnsupported;
1360         IOService *provider;
1361
1362         if (functionName == gIOPlatformQuiesceActionKey ||
1363             functionName == gIOPlatformActiveActionKey) {
1364                 /*
1365                  * Services which register for IOPlatformQuiesceAction / IOPlatformActiveAction
1366                  * must consume that event themselves, without passing it up to super/IOService.
1367                  */
1368                 if (gEnforceQuiesceSafety) {
1369                         panic("Class %s passed the quiesce/active action to IOService",
1370                             getMetaClass()->getClassName());
1371                 }
1372         }
1373
1374         if (gIOPlatformFunctionHandlerSet == functionName) {
1375 #if defined(__i386__) || defined(__x86_64__)
1376                 const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1377                 IOService *      target              = (IOService *) param2;
1378                 bool             enable              = (param3 != NULL);
1379
1380                 if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) {
1381                         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1382                 } else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) {
1383                         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1384                 }
1385 #endif /* defined(__i386__) || defined(__x86_64__) */
1386         }
1387
1388         if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1389                 result = provider->callPlatformFunction(functionName, waitForFunction,
1390                     param1, param2, param3, param4);
1391         }
1392
1393         return result;
1394 }
1395
1396 IOReturn
1397 IOService::callPlatformFunction( const char * functionName,
1398     bool waitForFunction,
1399     void *param1, void *param2,
1400     void *param3, void *param4 )
1401 {
1402         IOReturn result = kIOReturnNoMemory;
1403         const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1404
1405         if (functionSymbol != NULL) {
1406                 result = callPlatformFunction(functionSymbol, waitForFunction,
1407                     param1, param2, param3, param4);
1408                 functionSymbol->release();
1409         }
1410
1411         return result;
1412 }
1413
1414
1415 /*
1416  * Accessors for global services
1417  */
1418
1419 IOPlatformExpert *
1420 IOService::getPlatform( void )
1421 {
1422         return gIOPlatform;
1423 }
1424
1425 class IOPMrootDomain *
1426         IOService::getPMRootDomain( void )
1427 {
1428         return gIOPMRootDomain;
1429 }
1430
1431 IOService *
1432 IOService::getResourceService( void )
1433 {
1434         return gIOResources;
1435 }
1436
1437 void
1438 IOService::setPlatform( IOPlatformExpert * platform)
1439 {
1440         gIOPlatform = platform;
1441         gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1442         gIOUserResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1443
1444 #if defined(__i386__) || defined(__x86_64__)
1445
1446         static const char * keys[kCpuNumDelayTypes] = {
1447                 kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1448         const OSObject * objs[2];
1449         OSArray * array;
1450         uint32_t  idx;
1451
1452         for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
1453                 objs[0] = sCPULatencySet[idx];
1454                 objs[1] = sCPULatencyHolder[idx];
1455                 array   = OSArray::withObjects(objs, 2);
1456                 if (!array) {
1457                         break;
1458                 }
1459                 platform->setProperty(keys[idx], array);
1460                 array->release();
1461         }
1462 #endif /* defined(__i386__) || defined(__x86_64__) */
1463 }
1464
1465 void
1466 IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1467 {
1468         gIOPMRootDomain = rootDomain;
1469         publishResource(gIOResourceIOKitKey);
1470         IOServicePH::init(rootDomain);
1471 }
1472
1473 /*
1474  * Stacking change
1475  */
1476
1477 bool
1478 IOService::lockForArbitration( bool isSuccessRequired )
1479 {
1480         bool                          found;
1481         bool                          success;
1482         ArbitrationLockQueueElement * element;
1483         ArbitrationLockQueueElement * active;
1484         ArbitrationLockQueueElement * waiting;
1485
1486         enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1487
1488         // lock global access
1489         IOTakeLock( gArbitrationLockQueueLock );
1490
1491         // obtain an unused queue element
1492         if (!queue_empty( &gArbitrationLockQueueFree )) {
1493                 queue_remove_first( &gArbitrationLockQueueFree,
1494                     element,
1495                     ArbitrationLockQueueElement *,
1496                     link );
1497         } else {
1498                 element = IONew( ArbitrationLockQueueElement, 1 );
1499                 assert( element );
1500         }
1501
1502         // prepare the queue element
1503         element->thread   = IOThreadSelf();
1504         element->service  = this;
1505         element->count    = 1;
1506         element->required = isSuccessRequired;
1507         element->aborted  = false;
1508
1509         // determine whether this object is already locked (ie. on active queue)
1510         found = false;
1511         queue_iterate( &gArbitrationLockQueueActive,
1512             active,
1513             ArbitrationLockQueueElement *,
1514             link )
1515         {
1516                 if (active->service == element->service) {
1517                         found = true;
1518                         break;
1519                 }
1520         }
1521
1522         if (found) { // this object is already locked
1523                 // determine whether it is the same or a different thread trying to lock
1524                 if (active->thread != element->thread) { // it is a different thread
1525                         ArbitrationLockQueueElement * victim = NULL;
1526
1527                         // before placing this new thread on the waiting queue, we look for
1528                         // a deadlock cycle...
1529
1530                         while (1) {
1531                                 // determine whether the active thread holding the object we
1532                                 // want is waiting for another object to be unlocked
1533                                 found = false;
1534                                 queue_iterate( &gArbitrationLockQueueWaiting,
1535                                     waiting,
1536                                     ArbitrationLockQueueElement *,
1537                                     link )
1538                                 {
1539                                         if (waiting->thread == active->thread) {
1540                                                 assert( false == waiting->aborted );
1541                                                 found = true;
1542                                                 break;
1543                                         }
1544                                 }
1545
1546                                 if (found) { // yes, active thread waiting for another object
1547                                         // this may be a candidate for rejection if the required
1548                                         // flag is not set, should we detect a deadlock later on
1549                                         if (false == waiting->required) {
1550                                                 victim = waiting;
1551                                         }
1552
1553                                         // find the thread that is holding this other object, that
1554                                         // is blocking the active thread from proceeding (fun :-)
1555                                         found = false;
1556                                         queue_iterate( &gArbitrationLockQueueActive,
1557                                             active, // (reuse active queue element)
1558                                             ArbitrationLockQueueElement *,
1559                                             link )
1560                                         {
1561                                                 if (active->service == waiting->service) {
1562                                                         found = true;
1563                                                         break;
1564                                                 }
1565                                         }
1566
1567                                         // someone must be holding it or it wouldn't be waiting
1568                                         assert( found );
1569
1570                                         if (active->thread == element->thread) {
1571                                                 // doh, it's waiting for the thread that originated
1572                                                 // this whole lock (ie. current thread) -> deadlock
1573                                                 if (false == element->required) { // willing to fail?
1574                                                         // the originating thread doesn't have the required
1575                                                         // flag, so it can fail
1576                                                         success = false; // (fail originating lock request)
1577                                                         break; // (out of while)
1578                                                 } else { // originating thread is not willing to fail
1579                                                         // see if we came across a waiting thread that did
1580                                                         // not have the 'required' flag set: we'll fail it
1581                                                         if (victim) {
1582                                                                 // we do have a willing victim, fail it's lock
1583                                                                 victim->aborted = true;
1584
1585                                                                 // take the victim off the waiting queue
1586                                                                 queue_remove( &gArbitrationLockQueueWaiting,
1587                                                                     victim,
1588                                                                     ArbitrationLockQueueElement *,
1589                                                                     link );
1590
1591                                                                 // wake the victim
1592                                                                 IOLockWakeup( gArbitrationLockQueueLock,
1593                                                                     victim,
1594                                                                     /* one thread */ true );
1595
1596                                                                 // allow this thread to proceed (ie. wait)
1597                                                                 success = true; // (put request on wait queue)
1598                                                                 break; // (out of while)
1599                                                         } else {
1600                                                                 // all the waiting threads we came across in
1601                                                                 // finding this loop had the 'required' flag
1602                                                                 // set, so we've got a deadlock we can't avoid
1603                                                                 panic("I/O Kit: Unrecoverable deadlock.");
1604                                                         }
1605                                                 }
1606                                         } else {
1607                                                 // repeat while loop, redefining active thread to be the
1608                                                 // thread holding "this other object" (see above), and
1609                                                 // looking for threads waiting on it; note the active
1610                                                 // variable points to "this other object" already... so
1611                                                 // there nothing to do in this else clause.
1612                                         }
1613                                 } else { // no, active thread is not waiting for another object
1614                                         success = true; // (put request on wait queue)
1615                                         break; // (out of while)
1616                                 }
1617                         } // while forever
1618
1619                         if (success) { // put the request on the waiting queue?
1620                                 kern_return_t wait_result;
1621
1622                                 // place this thread on the waiting queue and put it to sleep;
1623                                 // we place it at the tail of the queue...
1624                                 queue_enter( &gArbitrationLockQueueWaiting,
1625                                     element,
1626                                     ArbitrationLockQueueElement *,
1627                                     link );
1628
1629                                 // declare that this thread will wait for a given event
1630 restart_sleep:                  wait_result = assert_wait( element,
1631                                     element->required ? THREAD_UNINT
1632                                     : THREAD_INTERRUPTIBLE );
1633
1634                                 // unlock global access
1635                                 IOUnlock( gArbitrationLockQueueLock );
1636
1637                                 // put thread to sleep, waiting for our event to fire...
1638                                 if (wait_result == THREAD_WAITING) {
1639                                         wait_result = thread_block(THREAD_CONTINUE_NULL);
1640                                 }
1641
1642
1643                                 // ...and we've been woken up; we might be in one of two states:
1644                                 // (a) we've been aborted and our queue element is not on
1645                                 //     any of the three queues, but is floating around
1646                                 // (b) we're allowed to proceed with the lock and we have
1647                                 //     already been moved from the waiting queue to the
1648                                 //     active queue.
1649                                 // ...plus a 3rd state, should the thread have been interrupted:
1650                                 // (c) we're still on the waiting queue
1651
1652                                 // determine whether we were interrupted out of our sleep
1653                                 if (THREAD_INTERRUPTED == wait_result) {
1654                                         // re-lock global access
1655                                         IOTakeLock( gArbitrationLockQueueLock );
1656
1657                                         // determine whether we're still on the waiting queue
1658                                         found = false;
1659                                         queue_iterate( &gArbitrationLockQueueWaiting,
1660                                             waiting, // (reuse waiting queue element)
1661                                             ArbitrationLockQueueElement *,
1662                                             link )
1663                                         {
1664                                                 if (waiting == element) {
1665                                                         found = true;
1666                                                         break;
1667                                                 }
1668                                         }
1669
1670                                         if (found) { // yes, we're still on the waiting queue
1671                                                 // determine whether we're willing to fail
1672                                                 if (false == element->required) {
1673                                                         // mark us as aborted
1674                                                         element->aborted = true;
1675
1676                                                         // take us off the waiting queue
1677                                                         queue_remove( &gArbitrationLockQueueWaiting,
1678                                                             element,
1679                                                             ArbitrationLockQueueElement *,
1680                                                             link );
1681                                                 } else { // we are not willing to fail
1682                                                         // ignore interruption, go back to sleep
1683                                                         goto restart_sleep;
1684                                                 }
1685                                         }
1686
1687                                         // unlock global access
1688                                         IOUnlock( gArbitrationLockQueueLock );
1689
1690                                         // proceed as though this were a normal wake up
1691                                         wait_result = THREAD_AWAKENED;
1692                                 }
1693
1694                                 assert( THREAD_AWAKENED == wait_result );
1695
1696                                 // determine whether we've been aborted while we were asleep
1697                                 if (element->aborted) {
1698                                         assert( false == element->required );
1699
1700                                         // re-lock global access
1701                                         IOTakeLock( gArbitrationLockQueueLock );
1702
1703                                         action = kPutOnFreeQueue;
1704                                         success = false;
1705                                 } else { // we weren't aborted, so we must be ready to go :-)
1706                                         // we've already been moved from waiting to active queue
1707                                         return true;
1708                                 }
1709                         } else { // the lock request is to be failed
1710                                 // return unused queue element to queue
1711                                 action = kPutOnFreeQueue;
1712                         }
1713                 } else { // it is the same thread, recursive access is allowed
1714                         // add one level of recursion
1715                         active->count++;
1716
1717                         // return unused queue element to queue
1718                         action = kPutOnFreeQueue;
1719                         success = true;
1720                 }
1721         } else { // this object is not already locked, so let this thread through
1722                 action = kPutOnActiveQueue;
1723                 success = true;
1724         }
1725
1726         // put the new element on a queue
1727         if (kPutOnActiveQueue == action) {
1728                 queue_enter( &gArbitrationLockQueueActive,
1729                     element,
1730                     ArbitrationLockQueueElement *,
1731                     link );
1732         } else if (kPutOnFreeQueue == action) {
1733                 queue_enter( &gArbitrationLockQueueFree,
1734                     element,
1735                     ArbitrationLockQueueElement *,
1736                     link );
1737         } else {
1738                 assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1739         }
1740
1741         // unlock global access
1742         IOUnlock( gArbitrationLockQueueLock );
1743
1744         return success;
1745 }
1746
1747 void
1748 IOService::unlockForArbitration( void )
1749 {
1750         bool                          found;
1751         ArbitrationLockQueueElement * element;
1752
1753         // lock global access
1754         IOTakeLock( gArbitrationLockQueueLock );
1755
1756         // find the lock element for this object (ie. on active queue)
1757         found = false;
1758         queue_iterate( &gArbitrationLockQueueActive,
1759             element,
1760             ArbitrationLockQueueElement *,
1761             link )
1762         {
1763                 if (element->service == this) {
1764                         found = true;
1765                         break;
1766                 }
1767         }
1768
1769         assert( found );
1770
1771         // determine whether the lock has been taken recursively
1772         if (element->count > 1) {
1773                 // undo one level of recursion
1774                 element->count--;
1775         } else {
1776                 // remove it from the active queue
1777                 queue_remove( &gArbitrationLockQueueActive,
1778                     element,
1779                     ArbitrationLockQueueElement *,
1780                     link );
1781
1782                 // put it on the free queue
1783                 queue_enter( &gArbitrationLockQueueFree,
1784                     element,
1785                     ArbitrationLockQueueElement *,
1786                     link );
1787
1788                 // determine whether a thread is waiting for object (head to tail scan)
1789                 found = false;
1790                 queue_iterate( &gArbitrationLockQueueWaiting,
1791                     element,
1792                     ArbitrationLockQueueElement *,
1793                     link )
1794                 {
1795                         if (element->service == this) {
1796                                 found = true;
1797                                 break;
1798                         }
1799                 }
1800
1801                 if (found) { // we found an interested thread on waiting queue
1802                         // remove it from the waiting queue
1803                         queue_remove( &gArbitrationLockQueueWaiting,
1804                             element,
1805                             ArbitrationLockQueueElement *,
1806                             link );
1807
1808                         // put it on the active queue
1809                         queue_enter( &gArbitrationLockQueueActive,
1810                             element,
1811                             ArbitrationLockQueueElement *,
1812                             link );
1813
1814                         // wake the waiting thread
1815                         IOLockWakeup( gArbitrationLockQueueLock,
1816                             element,
1817                             /* one thread */ true );
1818                 }
1819         }
1820
1821         // unlock global access
1822         IOUnlock( gArbitrationLockQueueLock );
1823 }
1824
1825 uint32_t
1826 IOService::isLockedForArbitration(IOService * service)
1827 {
1828 #if DEBUG_NOTIFIER_LOCKED
1829         uint32_t                      count;
1830         ArbitrationLockQueueElement * active;
1831
1832         // lock global access
1833         IOLockLock(gArbitrationLockQueueLock);
1834
1835         // determine whether this object is already locked (ie. on active queue)
1836         count = 0;
1837         queue_iterate(&gArbitrationLockQueueActive,
1838             active,
1839             ArbitrationLockQueueElement *,
1840             link)
1841         {
1842                 if ((active->thread == IOThreadSelf())
1843                     && (!service || (active->service == service))) {
1844                         count += 0x10000;
1845                         count += active->count;
1846                 }
1847         }
1848
1849         IOLockUnlock(gArbitrationLockQueueLock);
1850
1851         return count;
1852
1853 #else /* DEBUG_NOTIFIER_LOCKED */
1854
1855         return 0;
1856
1857 #endif /* DEBUG_NOTIFIER_LOCKED */
1858 }
1859
1860 void
1861 IOService::applyToProviders( IOServiceApplierFunction applier,
1862     void * context )
1863 {
1864         applyToParents((IORegistryEntryApplierFunction) applier,
1865             context, gIOServicePlane );
1866 }
1867
1868 void
1869 IOService::applyToClients( IOServiceApplierFunction applier,
1870     void * context )
1871 {
1872         applyToChildren((IORegistryEntryApplierFunction) applier,
1873             context, gIOServicePlane );
1874 }
1875
1876
1877 /*
1878  * Client messages
1879  */
1880
1881
1882 // send a message to a client or interested party of this service
1883 IOReturn
1884 IOService::messageClient( UInt32 type, OSObject * client,
1885     void * argument, vm_size_t argSize )
1886 {
1887         IOReturn                            ret;
1888         IOService *                         service;
1889         _IOServiceInterestNotifier *        notify;
1890
1891         if ((service = OSDynamicCast( IOService, client))) {
1892                 ret = service->message( type, this, argument );
1893         } else if ((notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1894                 _IOServiceNotifierInvocation invocation;
1895                 bool                     willNotify;
1896
1897                 invocation.thread = current_thread();
1898
1899                 LOCKWRITENOTIFY();
1900                 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1901
1902                 if (willNotify) {
1903                         queue_enter( &notify->handlerInvocations, &invocation,
1904                             _IOServiceNotifierInvocation *, link );
1905                 }
1906                 UNLOCKNOTIFY();
1907
1908                 if (willNotify) {
1909                         ret = (*notify->handler)( notify->target, notify->ref,
1910                             type, this, argument, argSize );
1911
1912                         LOCKWRITENOTIFY();
1913                         queue_remove( &notify->handlerInvocations, &invocation,
1914                             _IOServiceNotifierInvocation *, link );
1915                         if (kIOServiceNotifyWaiter & notify->state) {
1916                                 notify->state &= ~kIOServiceNotifyWaiter;
1917                                 WAKEUPNOTIFY( notify );
1918                         }
1919                         UNLOCKNOTIFY();
1920                 } else {
1921                         ret = kIOReturnSuccess;
1922                 }
1923         } else {
1924                 ret = kIOReturnBadArgument;
1925         }
1926
1927         return ret;
1928 }
1929
1930 static void
1931 applyToInterestNotifiers(const IORegistryEntry *target,
1932     const OSSymbol * typeOfInterest,
1933     OSObjectApplierFunction applier,
1934     void * context )
1935 {
1936         OSArray *  copyArray = NULL;
1937         OSObject * prop;
1938
1939         LOCKREADNOTIFY();
1940
1941         prop = target->copyProperty(typeOfInterest);
1942         IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
1943
1944         if (notifyList) {
1945                 copyArray = OSArray::withCapacity(1);
1946
1947                 // iterate over queue, entry is set to each element in the list
1948                 iterqueue(&notifyList->fCommandChain, entry) {
1949                         _IOServiceInterestNotifier * notify;
1950
1951                         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1952                         copyArray->setObject(notify);
1953                 }
1954         }
1955         UNLOCKNOTIFY();
1956
1957         if (copyArray) {
1958                 unsigned int    index;
1959                 OSObject *      next;
1960
1961                 for (index = 0; (next = copyArray->getObject( index )); index++) {
1962                         (*applier)(next, context);
1963                 }
1964                 copyArray->release();
1965         }
1966
1967         OSSafeReleaseNULL(prop);
1968 }
1969
1970 void
1971 IOService::applyToInterested( const OSSymbol * typeOfInterest,
1972     OSObjectApplierFunction applier,
1973     void * context )
1974 {
1975         if (gIOGeneralInterest == typeOfInterest) {
1976                 applyToClients((IOServiceApplierFunction) applier, context );
1977         }
1978         applyToInterestNotifiers(this, typeOfInterest, applier, context);
1979 }
1980
1981 struct MessageClientsContext {
1982         IOService * service;
1983         UInt32      type;
1984         void *      argument;
1985         vm_size_t   argSize;
1986         IOReturn    ret;
1987 };
1988
1989 static void
1990 messageClientsApplier( OSObject * object, void * ctx )
1991 {
1992         IOReturn                ret;
1993         MessageClientsContext * context = (MessageClientsContext *) ctx;
1994
1995         ret = context->service->messageClient( context->type,
1996             object, context->argument, context->argSize );
1997         if (kIOReturnSuccess != ret) {
1998                 context->ret = ret;
1999         }
2000 }
2001
2002 // send a message to all clients
2003 IOReturn
2004 IOService::messageClients( UInt32 type,
2005     void * argument, vm_size_t argSize )
2006 {
2007         MessageClientsContext       context;
2008
2009         context.service     = this;
2010         context.type        = type;
2011         context.argument    = argument;
2012         context.argSize     = argSize;
2013         context.ret         = kIOReturnSuccess;
2014
2015         applyToInterested( gIOGeneralInterest,
2016             &messageClientsApplier, &context );
2017
2018         return context.ret;
2019 }
2020
2021 IOReturn
2022 IOService::acknowledgeNotification( IONotificationRef notification,
2023     IOOptionBits response )
2024 {
2025         return kIOReturnUnsupported;
2026 }
2027
2028 IONotifier *
2029 IOService::registerInterest( const OSSymbol * typeOfInterest,
2030     IOServiceInterestHandler handler, void * target, void * ref )
2031 {
2032         _IOServiceInterestNotifier * notify = NULL;
2033         IOReturn rc = kIOReturnError;
2034
2035         notify = new _IOServiceInterestNotifier;
2036         if (!notify) {
2037                 return NULL;
2038         }
2039
2040         if (notify->init()) {
2041                 rc = registerInterestForNotifier(notify, typeOfInterest,
2042                     handler, target, ref);
2043         }
2044
2045         if (rc != kIOReturnSuccess) {
2046                 notify->release();
2047                 notify = NULL;
2048         }
2049
2050         return notify;
2051 }
2052
2053
2054
2055 static IOReturn
2056 IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
2057     UInt32 messageType, IOService * provider,
2058     void * messageArgument, vm_size_t argSize )
2059 {
2060         return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
2061 }
2062
2063 IONotifier *
2064 IOService::registerInterest(const OSSymbol * typeOfInterest,
2065     IOServiceInterestHandlerBlock handler)
2066 {
2067         IONotifier * notify;
2068         void       * block;
2069
2070         block = Block_copy(handler);
2071         if (!block) {
2072                 return NULL;
2073         }
2074
2075         notify = registerInterest(typeOfInterest, &IOServiceInterestHandlerToBlock, NULL, block);
2076
2077         if (!notify) {
2078                 Block_release(block);
2079         }
2080
2081         return notify;
2082 }
2083
2084 IOReturn
2085 IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
2086     IOServiceInterestHandler handler, void * target, void * ref )
2087 {
2088         IOReturn rc = kIOReturnSuccess;
2089         _IOServiceInterestNotifier  *notify = NULL;
2090
2091         if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify))) {
2092                 return kIOReturnBadArgument;
2093         }
2094
2095         notify->handler = handler;
2096         notify->target = target;
2097         notify->ref = ref;
2098
2099         if ((typeOfInterest != gIOGeneralInterest)
2100             && (typeOfInterest != gIOBusyInterest)
2101             && (typeOfInterest != gIOAppPowerStateInterest)
2102             && (typeOfInterest != gIOConsoleSecurityInterest)
2103             && (typeOfInterest != gIOPriorityPowerStateInterest)) {
2104                 return kIOReturnBadArgument;
2105         }
2106
2107         lockForArbitration();
2108         if (0 == (__state[0] & kIOServiceInactiveState)) {
2109                 notify->state = kIOServiceNotifyEnable;
2110
2111                 ////// queue
2112
2113                 LOCKWRITENOTIFY();
2114
2115                 // Get the head of the notifier linked list
2116                 IOCommand * notifyList;
2117                 OSObject  * obj = copyProperty( typeOfInterest );
2118                 if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
2119                         notifyList = OSTypeAlloc(IOCommand);
2120                         if (notifyList) {
2121                                 notifyList->init();
2122                                 bool ok = setProperty( typeOfInterest, notifyList);
2123                                 notifyList->release();
2124                                 if (!ok) {
2125                                         notifyList = NULL;
2126                                 }
2127                         }
2128                 }
2129                 if (obj) {
2130                         obj->release();
2131                 }
2132
2133                 if (notifyList) {
2134                         enqueue(&notifyList->fCommandChain, &notify->chain);
2135                         notify->retain(); // ref'ed while in list
2136                 }
2137
2138                 UNLOCKNOTIFY();
2139         } else {
2140                 rc = kIOReturnNotReady;
2141         }
2142         unlockForArbitration();
2143
2144         return rc;
2145 }
2146
2147 static void
2148 cleanInterestList( OSObject * head )
2149 {
2150         IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
2151         if (!notifyHead) {
2152                 return;
2153         }
2154
2155         LOCKWRITENOTIFY();
2156         while (queue_entry_t entry = dequeue(&notifyHead->fCommandChain)) {
2157                 queue_next(entry) = queue_prev(entry) = NULL;
2158
2159                 _IOServiceInterestNotifier * notify;
2160
2161                 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2162                 notify->release();
2163         }
2164         UNLOCKNOTIFY();
2165 }
2166
2167 void
2168 IOService::unregisterAllInterest( void )
2169 {
2170         OSObject * prop;
2171
2172         prop = copyProperty(gIOGeneralInterest);
2173         cleanInterestList(prop);
2174         OSSafeReleaseNULL(prop);
2175
2176         prop = copyProperty(gIOBusyInterest);
2177         cleanInterestList(prop);
2178         OSSafeReleaseNULL(prop);
2179
2180         prop = copyProperty(gIOAppPowerStateInterest);
2181         cleanInterestList(prop);
2182         OSSafeReleaseNULL(prop);
2183
2184         prop = copyProperty(gIOPriorityPowerStateInterest);
2185         cleanInterestList(prop);
2186         OSSafeReleaseNULL(prop);
2187
2188         prop = copyProperty(gIOConsoleSecurityInterest);
2189         cleanInterestList(prop);
2190         OSSafeReleaseNULL(prop);
2191 }
2192
2193 /*
2194  * _IOServiceInterestNotifier
2195  */
2196
2197 // wait for all threads, other than the current one,
2198 //  to exit the handler
2199
2200 void
2201 _IOServiceInterestNotifier::wait()
2202 {
2203         _IOServiceNotifierInvocation * next;
2204         bool doWait;
2205
2206         do {
2207                 doWait = false;
2208                 queue_iterate( &handlerInvocations, next,
2209                     _IOServiceNotifierInvocation *, link) {
2210                         if (next->thread != current_thread()) {
2211                                 doWait = true;
2212                                 break;
2213                         }
2214                 }
2215                 if (doWait) {
2216                         state |= kIOServiceNotifyWaiter;
2217                         SLEEPNOTIFY(this);
2218                 }
2219         } while (doWait);
2220 }
2221
2222 void
2223 _IOServiceInterestNotifier::free()
2224 {
2225         assert( queue_empty( &handlerInvocations ));
2226
2227         if (handler == &IOServiceInterestHandlerToBlock) {
2228                 Block_release(ref);
2229         }
2230
2231         OSObject::free();
2232 }
2233
2234 void
2235 _IOServiceInterestNotifier::remove()
2236 {
2237         LOCKWRITENOTIFY();
2238
2239         if (queue_next( &chain )) {
2240                 remqueue(&chain);
2241                 queue_next( &chain) = queue_prev( &chain) = NULL;
2242                 release();
2243         }
2244
2245         state &= ~kIOServiceNotifyEnable;
2246
2247         wait();
2248
2249         UNLOCKNOTIFY();
2250
2251         release();
2252 }
2253
2254 bool
2255 _IOServiceInterestNotifier::disable()
2256 {
2257         bool        ret;
2258
2259         LOCKWRITENOTIFY();
2260
2261         ret = (0 != (kIOServiceNotifyEnable & state));
2262         state &= ~kIOServiceNotifyEnable;
2263         if (ret) {
2264                 wait();
2265         }
2266
2267         UNLOCKNOTIFY();
2268
2269         return ret;
2270 }
2271
2272 void
2273 _IOServiceInterestNotifier::enable( bool was )
2274 {
2275         LOCKWRITENOTIFY();
2276         if (was) {
2277                 state |= kIOServiceNotifyEnable;
2278         } else {
2279                 state &= ~kIOServiceNotifyEnable;
2280         }
2281         UNLOCKNOTIFY();
2282 }
2283
2284 bool
2285 _IOServiceInterestNotifier::init()
2286 {
2287         queue_init( &handlerInvocations );
2288         return OSObject::init();
2289 }
2290 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2291
2292 /*
2293  * Termination
2294  */
2295
2296 #define tailQ(o)                setObject(o)
2297 #define headQ(o)                setObject(0, o)
2298 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2299
2300 static void
2301 _workLoopAction( IOWorkLoop::Action action,
2302     IOService * service,
2303     void * p0 = NULL, void * p1 = NULL,
2304     void * p2 = NULL, void * p3 = NULL )
2305 {
2306         IOWorkLoop * wl;
2307
2308         if ((wl = service->getWorkLoop())) {
2309                 wl->retain();
2310                 wl->runAction( action, service, p0, p1, p2, p3 );
2311                 wl->release();
2312         } else {
2313                 (*action)( service, p0, p1, p2, p3 );
2314         }
2315 }
2316
2317 bool
2318 IOService::requestTerminate( IOService * provider, IOOptionBits options )
2319 {
2320         bool ok;
2321
2322         // if its our only provider
2323         ok = isParent( provider, gIOServicePlane, true);
2324
2325         // -- compat
2326         if (ok) {
2327                 provider->terminateClient( this, options | kIOServiceRecursing );
2328                 ok = (0 != (kIOServiceInactiveState & __state[0]));
2329         }
2330         // --
2331
2332         return ok;
2333 }
2334
2335 bool
2336 IOService::terminatePhase1( IOOptionBits options )
2337 {
2338         IOService *  victim;
2339         IOService *  client;
2340         IOService *  rematchProvider;
2341         OSIterator * iter;
2342         OSArray *    makeInactive;
2343         OSArray *    waitingInactive;
2344         IOOptionBits callerOptions;
2345         int          waitResult = THREAD_AWAKENED;
2346         bool         wait;
2347         bool                 ok;
2348         bool                 didInactive;
2349         bool                 startPhase2 = false;
2350
2351         TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2352
2353         callerOptions = options;
2354         rematchProvider = NULL;
2355         uint64_t regID = getRegistryEntryID();
2356         IOServiceTrace(
2357                 IOSERVICE_TERMINATE_PHASE1,
2358                 (uintptr_t) regID,
2359                 (uintptr_t) (regID >> 32),
2360                 (uintptr_t) this,
2361                 (uintptr_t) options);
2362
2363         // -- compat
2364         if (options & kIOServiceRecursing) {
2365                 lockForArbitration();
2366                 if (0 == (kIOServiceInactiveState & __state[0])) {
2367                         __state[0] |= kIOServiceInactiveState;
2368                         __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2369                 }
2370                 unlockForArbitration();
2371
2372                 return true;
2373         }
2374         // --
2375
2376         makeInactive    = OSArray::withCapacity( 16 );
2377         waitingInactive = OSArray::withCapacity( 16 );
2378         if (!makeInactive || !waitingInactive) {
2379                 return false;
2380         }
2381
2382         victim = this;
2383         victim->retain();
2384
2385         while (victim) {
2386                 didInactive = victim->lockForArbitration( true );
2387                 if (didInactive) {
2388                         uint64_t regID1 = victim->getRegistryEntryID();
2389                         IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2390                             (uintptr_t) regID1,
2391                             (uintptr_t) (regID1 >> 32),
2392                             (uintptr_t) victim->__state[1],
2393                             (uintptr_t) 0);
2394
2395                         enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2396                         didInactive = (kRP1 == (victim->__state[1] & kRP1))
2397                             || (0 == (victim->__state[0] & kIOServiceInactiveState));
2398
2399                         if (!didInactive) {
2400                                 // a multiply attached IOService can be visited twice
2401                                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) {
2402                                         do{
2403                                                 IOLockLock(gIOServiceBusyLock);
2404                                                 wait = (victim->__state[1] & kIOServiceTermPhase1State);
2405                                                 if (wait) {
2406                                                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2407                                                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2408                                                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2409                                                         victim->unlockForArbitration();
2410                                                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2411                                                 }
2412                                                 IOLockUnlock(gIOServiceBusyLock);
2413                                                 if (wait) {
2414                                                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2415                                                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2416                                                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2417                                                         victim->lockForArbitration();
2418                                                 }
2419                                         }while (wait && (waitResult != THREAD_TIMED_OUT));
2420                                 }
2421                         } else {
2422                                 victim->__state[0] |= kIOServiceInactiveState;
2423                                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2424                                     | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2425                                 victim->__state[1] &= ~kIOServiceRecursing;
2426                                 victim->__state[1] |= kIOServiceTermPhase1State;
2427                                 waitingInactive->headQ(victim);
2428                                 if (victim == this) {
2429                                         if (kIOServiceTerminateNeedWillTerminate & options) {
2430                                                 victim->__state[1] |= kIOServiceNeedWillTerminate;
2431                                         }
2432                                 }
2433                                 victim->_adjustBusy( 1 );
2434
2435                                 if ((options & kIOServiceTerminateWithRematch) && (victim == this)) {
2436                                         OSObject     * obj;
2437                                         OSObject     * rematchProps;
2438                                         OSNumber     * num;
2439                                         uint32_t       count;
2440
2441                                         rematchProvider = getProvider();
2442                                         if (rematchProvider) {
2443                                                 obj = rematchProvider->copyProperty(gIORematchCountKey);
2444                                                 num = OSDynamicCast(OSNumber, obj);
2445                                                 count = 0;
2446                                                 if (num) {
2447                                                         count = num->unsigned32BitValue();
2448                                                         count++;
2449                                                 }
2450                                                 num = OSNumber::withNumber(count, 32);
2451                                                 rematchProvider->setProperty(gIORematchCountKey, num);
2452                                                 rematchProps = copyProperty(gIOMatchedPersonalityKey);
2453                                                 rematchProvider->setProperty(gIORematchPersonalityKey, rematchProps);
2454                                                 OSSafeReleaseNULL(num);
2455                                                 OSSafeReleaseNULL(rematchProps);
2456                                                 OSSafeReleaseNULL(obj);
2457                                         }
2458                                 }
2459                         }
2460                         victim->unlockForArbitration();
2461                 }
2462                 if (victim == this) {
2463                         options &= ~kIOServiceTerminateWithRematch;
2464                         startPhase2 = didInactive;
2465                 }
2466                 if (didInactive) {
2467                         OSArray * notifiers;
2468                         notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2469                         victim->invokeNotifiers(&notifiers);
2470
2471                         IOUserClient::destroyUserReferences( victim );
2472
2473                         iter = victim->getClientIterator();
2474                         if (iter) {
2475                                 while ((client = (IOService *) iter->getNextObject())) {
2476                                         TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2477                                             client->getName(), client->getRegistryEntryID(),
2478                                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2479                                         ok = client->requestTerminate( victim, options );
2480                                         TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2481                                             client->getName(), client->getRegistryEntryID(),
2482                                             victim->getName(), victim->getRegistryEntryID(), ok);
2483
2484                                         uint64_t regID1 = client->getRegistryEntryID();
2485                                         uint64_t regID2 = victim->getRegistryEntryID();
2486                                         IOServiceTrace(
2487                                                 (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2488                                                 : IOSERVICE_TERMINATE_REQUEST_FAIL),
2489                                                 (uintptr_t) regID1,
2490                                                 (uintptr_t) (regID1 >> 32),
2491                                                 (uintptr_t) regID2,
2492                                                 (uintptr_t) (regID2 >> 32));
2493
2494                                         if (ok) {
2495                                                 makeInactive->setObject( client );
2496                                         }
2497                                 }
2498                                 iter->release();
2499                         }
2500                 }
2501                 victim->release();
2502                 victim = (IOService *) makeInactive->getObject(0);
2503                 if (victim) {
2504                         victim->retain();
2505                         makeInactive->removeObject(0);
2506                 }
2507         }
2508         makeInactive->release();
2509
2510         while ((victim = (IOService *) waitingInactive->getObject(0))) {
2511                 victim->retain();
2512                 waitingInactive->removeObject(0);
2513
2514                 victim->lockForArbitration();
2515                 victim->__state[1] &= ~kIOServiceTermPhase1State;
2516                 if (kIOServiceTerm1WaiterState & victim->__state[1]) {
2517                         victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2518                         TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2519                         IOLockLock( gIOServiceBusyLock );
2520                         thread_wakeup((event_t) &victim->__state[1]);
2521                         IOLockUnlock( gIOServiceBusyLock );
2522                 }
2523                 victim->unlockForArbitration();
2524                 victim->release();
2525         }
2526         waitingInactive->release();
2527
2528         if (startPhase2) {
2529                 retain();
2530                 lockForArbitration();
2531                 scheduleTerminatePhase2(options);
2532                 unlockForArbitration();
2533                 release();
2534         }
2535
2536         if (rematchProvider) {
2537                 DKLOG(DKS " rematching after dext crash\n", DKN(rematchProvider));
2538                 rematchProvider->registerService();
2539         }
2540
2541         return true;
2542 }
2543
2544 void
2545 IOService::setTerminateDefer(IOService * provider, bool defer)
2546 {
2547         lockForArbitration();
2548         if (defer) {
2549                 __state[1] |= kIOServiceStartState;
2550         } else {
2551                 __state[1] &= ~kIOServiceStartState;
2552         }
2553         unlockForArbitration();
2554
2555         if (provider && !defer) {
2556                 provider->lockForArbitration();
2557                 provider->scheduleTerminatePhase2();
2558                 provider->unlockForArbitration();
2559         }
2560 }
2561
2562 // Must call this while holding gJobsLock
2563 void
2564 IOService::waitToBecomeTerminateThread(void)
2565 {
2566         IOLockAssert(gJobsLock, kIOLockAssertOwned);
2567         bool wait;
2568         do {
2569                 wait = (gIOTerminateThread != THREAD_NULL);
2570                 if (wait) {
2571                         IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2572                 }
2573         } while (wait);
2574         gIOTerminateThread = current_thread();
2575 }
2576
2577 // call with lockForArbitration
2578 void
2579 IOService::scheduleTerminatePhase2( IOOptionBits options )
2580 {
2581         AbsoluteTime        deadline;
2582         uint64_t            regID1;
2583         int                 waitResult = THREAD_AWAKENED;
2584         bool                wait = false, haveDeadline = false;
2585
2586         if (!(__state[0] & kIOServiceInactiveState)) {
2587                 return;
2588         }
2589
2590         regID1 = getRegistryEntryID();
2591         IOServiceTrace(
2592                 IOSERVICE_TERM_SCHED_PHASE2,
2593                 (uintptr_t) regID1,
2594                 (uintptr_t) (regID1 >> 32),
2595                 (uintptr_t) __state[1],
2596                 (uintptr_t) options);
2597
2598         if (__state[1] & kIOServiceTermPhase1State) {
2599                 return;
2600         }
2601
2602         retain();
2603         unlockForArbitration();
2604         options |= kIOServiceRequired;
2605         IOLockLock( gJobsLock );
2606
2607         if ((options & kIOServiceSynchronous)
2608             && (current_thread() != gIOTerminateThread)) {
2609                 waitToBecomeTerminateThread();
2610                 gIOTerminatePhase2List->setObject( this );
2611                 gIOTerminateWork++;
2612
2613                 do {
2614                         while (gIOTerminateWork) {
2615                                 terminateWorker( options );
2616                         }
2617                         wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2618                         if (wait) {
2619                                 /* wait for the victim to go non-busy */
2620                                 if (!haveDeadline) {
2621                                         clock_interval_to_deadline( 15, kSecondScale, &deadline );
2622                                         haveDeadline = true;
2623                                 }
2624                                 /* let others do work while we wait */
2625                                 gIOTerminateThread = NULL;
2626                                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2627                                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2628                                     deadline, THREAD_UNINT );
2629                                 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2630                                         IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous): THREAD_TIMED_OUT. "
2631                                             "Attempting to auto-resolve your deadlock. PLEASE FIX!\n", getName(), getRegistryEntryID());
2632                                 }
2633                                 waitToBecomeTerminateThread();
2634                         }
2635                 } while (gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2636
2637                 gIOTerminateThread = NULL;
2638                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2639         } else {
2640                 // ! kIOServiceSynchronous
2641
2642                 gIOTerminatePhase2List->setObject( this );
2643                 if (0 == gIOTerminateWork++) {
2644                         assert(gIOTerminateWorkerThread);
2645                         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2646                 }
2647         }
2648
2649         IOLockUnlock( gJobsLock );
2650         lockForArbitration();
2651         release();
2652 }
2653
2654 __attribute__((__noreturn__))
2655 void
2656 IOService::terminateThread( void * arg, wait_result_t waitResult )
2657 {
2658         // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2659         IOLockLock(gJobsLock);
2660         while (true) {
2661                 if (gIOTerminateThread != gIOTerminateWorkerThread) {
2662                         waitToBecomeTerminateThread();
2663                 }
2664
2665                 while (gIOTerminateWork) {
2666                         terminateWorker((uintptr_t)arg );
2667                 }
2668
2669                 gIOTerminateThread = NULL;
2670                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2671                 IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2672         }
2673 }
2674
2675 void
2676 IOService::scheduleStop( IOService * provider )
2677 {
2678         uint64_t regID1 = getRegistryEntryID();
2679         uint64_t regID2 = provider->getRegistryEntryID();
2680
2681         TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2682         IOServiceTrace(
2683                 IOSERVICE_TERMINATE_SCHEDULE_STOP,
2684                 (uintptr_t) regID1,
2685                 (uintptr_t) (regID1 >> 32),
2686                 (uintptr_t) regID2,
2687                 (uintptr_t) (regID2 >> 32));
2688
2689         IOLockLock( gJobsLock );
2690         gIOStopList->tailQ( this );
2691         gIOStopProviderList->tailQ( provider );
2692
2693         if (0 == gIOTerminateWork++) {
2694                 assert(gIOTerminateWorkerThread);
2695                 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2696         }
2697
2698         IOLockUnlock( gJobsLock );
2699 }
2700
2701 void
2702 IOService::scheduleFinalize(bool now)
2703 {
2704         uint64_t regID1 = getRegistryEntryID();
2705
2706         TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2707         IOServiceTrace(
2708                 IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2709                 (uintptr_t) regID1,
2710                 (uintptr_t) (regID1 >> 32),
2711                 0, 0);
2712
2713         if (now || IOUserClient::finalizeUserReferences(this)) {
2714                 IOLockLock( gJobsLock );
2715                 gIOFinalizeList->tailQ(this);
2716                 if (0 == gIOTerminateWork++) {
2717                         assert(gIOTerminateWorkerThread);
2718                         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2719                 }
2720                 IOLockUnlock( gJobsLock );
2721         }
2722 }
2723
2724 bool
2725 IOService::willTerminate( IOService * provider, IOOptionBits options )
2726 {
2727         if (reserved->uvars) {
2728                 IOUserServer::serviceWillTerminate(this, provider, options);
2729         }
2730         return true;
2731 }
2732
2733 bool
2734 IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2735 {
2736         if (reserved->uvars) {
2737                 IOUserServer::serviceDidTerminate(this, provider, options, defer);
2738         }
2739
2740         if (false == *defer) {
2741                 if (lockForArbitration( true )) {
2742                         if (false == provider->handleIsOpen( this )) {
2743                                 scheduleStop( provider );
2744                         }
2745                         // -- compat
2746                         else {
2747                                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2748                                 if (false == provider->handleIsOpen( this )) {
2749                                         scheduleStop( provider );
2750                                 }
2751                         }
2752                         // --
2753                         unlockForArbitration();
2754                 }
2755         }
2756
2757         return true;
2758 }
2759
2760 void
2761 IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2762     OSArray * doPhase2List,
2763     bool user,
2764     void *unused3 __unused)
2765 {
2766         OSIterator * iter;
2767         IOService *  client;
2768         bool         ok;
2769         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2770
2771         iter = victim->getClientIterator();
2772         if (iter) {
2773                 while ((client = (IOService *) iter->getNextObject())) {
2774                         if (user != (NULL != client->reserved->uvars)) {
2775                                 continue;
2776                         }
2777                         regID1 = client->getRegistryEntryID();
2778                         TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2779                             client->getName(), regID1,
2780                             victim->getName(), regID2, (long long)options);
2781                         IOServiceTrace(
2782                                 IOSERVICE_TERMINATE_WILL,
2783                                 (uintptr_t) regID1,
2784                                 (uintptr_t) (regID1 >> 32),
2785                                 (uintptr_t) regID2,
2786                                 (uintptr_t) (regID2 >> 32));
2787
2788                         ok = client->willTerminate( victim, options );
2789                         doPhase2List->tailQ( client );
2790                 }
2791                 iter->release();
2792         }
2793 }
2794
2795 void
2796 IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2797     void *unused1 __unused, void *unused2 __unused,
2798     void *unused3 __unused )
2799 {
2800         OSIterator * iter;
2801         IOService *  client;
2802         bool         defer;
2803         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2804
2805         victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2806
2807         iter = victim->getClientIterator();
2808         if (iter) {
2809                 while ((client = (IOService *) iter->getNextObject())) {
2810                         regID1 = client->getRegistryEntryID();
2811                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2812                             client->getName(), regID1,
2813                             victim->getName(), regID2, (long long)options);
2814                         defer = false;
2815                         client->didTerminate( victim, options, &defer );
2816
2817                         IOServiceTrace(
2818                                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2819                                 : IOSERVICE_TERMINATE_DID),
2820                                 (uintptr_t) regID1,
2821                                 (uintptr_t) (regID1 >> 32),
2822                                 (uintptr_t) regID2,
2823                                 (uintptr_t) (regID2 >> 32));
2824
2825                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2826                             client->getName(), regID1,
2827                             victim->getName(), regID2, defer);
2828                 }
2829                 iter->release();
2830         }
2831 }
2832
2833
2834 void
2835 IOService::actionWillStop( IOService * victim, IOOptionBits options,
2836     void *unused1 __unused, void *unused2 __unused,
2837     void *unused3 __unused )
2838 {
2839         OSIterator * iter;
2840         IOService *  provider;
2841         bool         ok;
2842         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2843
2844         iter = victim->getProviderIterator();
2845         if (iter) {
2846                 while ((provider = (IOService *) iter->getNextObject())) {
2847                         regID1 = provider->getRegistryEntryID();
2848                         TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2849                             victim->getName(), regID2,
2850                             provider->getName(), regID1, (long long)options);
2851                         IOServiceTrace(
2852                                 IOSERVICE_TERMINATE_WILL,
2853                                 (uintptr_t) regID2,
2854                                 (uintptr_t) (regID2 >> 32),
2855                                 (uintptr_t) regID1,
2856                                 (uintptr_t) (regID1 >> 32));
2857
2858                         ok = victim->willTerminate( provider, options );
2859                 }
2860                 iter->release();
2861         }
2862 }
2863
2864 void
2865 IOService::actionDidStop( IOService * victim, IOOptionBits options,
2866     void *unused1 __unused, void *unused2 __unused,
2867     void *unused3 __unused )
2868 {
2869         OSIterator * iter;
2870         IOService *  provider;
2871         bool defer = false;
2872         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2873
2874         iter = victim->getProviderIterator();
2875         if (iter) {
2876                 while ((provider = (IOService *) iter->getNextObject())) {
2877                         regID1 = provider->getRegistryEntryID();
2878                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2879                             victim->getName(), regID2,
2880                             provider->getName(), regID1, (long long)options);
2881                         victim->didTerminate( provider, options, &defer );
2882
2883                         IOServiceTrace(
2884                                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2885                                 : IOSERVICE_TERMINATE_DID),
2886                                 (uintptr_t) regID2,
2887                                 (uintptr_t) (regID2 >> 32),
2888                                 (uintptr_t) regID1,
2889                                 (uintptr_t) (regID1 >> 32));
2890
2891                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2892                             victim->getName(), regID2,
2893                             provider->getName(), regID1, defer);
2894                 }
2895                 iter->release();
2896         }
2897 }
2898
2899
2900 void
2901 IOService::actionFinalize( IOService * victim, IOOptionBits options,
2902     void *unused1 __unused, void *unused2 __unused,
2903     void *unused3 __unused )
2904 {
2905         uint64_t regID1 = victim->getRegistryEntryID();
2906         TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options);
2907         IOServiceTrace(
2908                 IOSERVICE_TERMINATE_FINALIZE,
2909                 (uintptr_t) regID1,
2910                 (uintptr_t) (regID1 >> 32),
2911                 0, 0);
2912
2913         victim->finalize( options );
2914 }
2915
2916 void
2917 IOService::actionStop( IOService * provider, IOService * client,
2918     void *unused1 __unused, void *unused2 __unused,
2919     void *unused3 __unused )
2920 {
2921         uint64_t regID1 = provider->getRegistryEntryID();
2922         uint64_t regID2 = client->getRegistryEntryID();
2923
2924         TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2925         IOServiceTrace(
2926                 IOSERVICE_TERMINATE_STOP,
2927                 (uintptr_t) regID1,
2928                 (uintptr_t) (regID1 >> 32),
2929                 (uintptr_t) regID2,
2930                 (uintptr_t) (regID2 >> 32));
2931
2932         client->stop( provider );
2933         if (provider->isOpen( client )) {
2934                 provider->close( client );
2935         }
2936
2937         TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2938         client->detach( provider );
2939 }
2940
2941 void
2942 IOService::terminateWorker( IOOptionBits options )
2943 {
2944         OSArray *           doPhase2List;
2945         OSArray *           didPhase2List;
2946         OSSet *             freeList;
2947         OSIterator *        iter;
2948         UInt32              workDone;
2949         IOService *         victim;
2950         IOService *         client;
2951         IOService *         provider;
2952         unsigned int        idx;
2953         bool                moreToDo;
2954         bool                doPhase2;
2955         bool                doPhase3;
2956
2957         options |= kIOServiceRequired;
2958
2959         doPhase2List  = OSArray::withCapacity( 16 );
2960         didPhase2List = OSArray::withCapacity( 16 );
2961         freeList      = OSSet::withCapacity( 16 );
2962         if ((NULL == doPhase2List) || (NULL == didPhase2List) || (NULL == freeList)) {
2963                 return;
2964         }
2965
2966         do {
2967                 workDone = gIOTerminateWork;
2968
2969                 while ((victim = (IOService *) gIOTerminatePhase2List->getObject(0))) {
2970                         victim->retain();
2971                         gIOTerminatePhase2List->removeObject(0);
2972                         IOLockUnlock( gJobsLock );
2973
2974                         uint64_t regID1 = victim->getRegistryEntryID();
2975                         IOServiceTrace(
2976                                 IOSERVICE_TERM_START_PHASE2,
2977                                 (uintptr_t) regID1,
2978                                 (uintptr_t) (regID1 >> 32),
2979                                 (uintptr_t) 0,
2980                                 (uintptr_t) 0);
2981
2982                         while (victim) {
2983                                 doPhase2 = victim->lockForArbitration( true );
2984                                 if (doPhase2) {
2985                                         doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2986                                         if (doPhase2) {
2987                                                 uint64_t regID1 = victim->getRegistryEntryID();
2988                                                 IOServiceTrace(
2989                                                         IOSERVICE_TERM_TRY_PHASE2,
2990                                                         (uintptr_t) regID1,
2991                                                         (uintptr_t) (regID1 >> 32),
2992                                                         (uintptr_t) victim->__state[1],
2993                                                         (uintptr_t) 0);
2994
2995                                                 doPhase2 = (0 == (victim->__state[1] &
2996                                                     (kIOServiceTermPhase1State
2997                                                     | kIOServiceTermPhase2State
2998                                                     | kIOServiceConfigState)));
2999
3000                                                 if (doPhase2 && (iter = victim->getClientIterator())) {
3001                                                         while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
3002                                                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
3003                                                                 if (!doPhase2) {
3004                                                                         uint64_t regID1 = client->getRegistryEntryID();
3005                                                                         IOServiceTrace(
3006                                                                                 IOSERVICE_TERM_UC_DEFER,
3007                                                                                 (uintptr_t) regID1,
3008                                                                                 (uintptr_t) (regID1 >> 32),
3009                                                                                 (uintptr_t) client->__state[1],
3010                                                                                 (uintptr_t) 0);
3011                                                                         TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
3012                                                                             victim->getName(), victim->getRegistryEntryID(),
3013                                                                             client->getName(), client->getRegistryEntryID());
3014                                                                 }
3015                                                         }
3016                                                         iter->release();
3017                                                 }
3018                                                 if (doPhase2) {
3019                                                         victim->__state[1] |= kIOServiceTermPhase2State;
3020                                                 }
3021                                         }
3022                                         victim->unlockForArbitration();
3023                                 }
3024                                 if (doPhase2) {
3025                                         if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3026                                                 if (NULL == victim->reserved->uvars) {
3027                                                         _workLoopAction((IOWorkLoop::Action) &actionWillStop,
3028                                                             victim, (void *)(uintptr_t) options);
3029                                                 } else {
3030                                                         actionWillStop(victim, options, NULL, NULL, NULL);
3031                                                 }
3032                                         }
3033
3034                                         OSArray * notifiers;
3035                                         notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
3036                                         victim->invokeNotifiers(&notifiers);
3037
3038                                         _workLoopAction((IOWorkLoop::Action) &actionWillTerminate,
3039                                             victim,
3040                                             (void *)(uintptr_t) options,
3041                                             (void *)(uintptr_t) doPhase2List,
3042                                             (void *)(uintptr_t) false);
3043
3044                                         actionWillTerminate(
3045                                                 victim, options, doPhase2List, true, NULL);
3046
3047                                         didPhase2List->headQ( victim );
3048                                 }
3049                                 victim->release();
3050                                 victim = (IOService *) doPhase2List->getObject(0);
3051                                 if (victim) {
3052                                         victim->retain();
3053                                         doPhase2List->removeObject(0);
3054                                 }
3055                         }
3056
3057                         while ((victim = (IOService *) didPhase2List->getObject(0))) {
3058                                 bool scheduleFinalize = false;
3059                                 if (victim->lockForArbitration( true )) {
3060                                         victim->__state[1] |= kIOServiceTermPhase3State;
3061                                         scheduleFinalize = (NULL == victim->getClient());
3062                                         victim->unlockForArbitration();
3063                                 }
3064                                 _workLoopAction((IOWorkLoop::Action) &actionDidTerminate,
3065                                     victim, (void *)(uintptr_t) options );
3066                                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3067                                         _workLoopAction((IOWorkLoop::Action) &actionDidStop,
3068                                             victim, (void *)(uintptr_t) options, NULL );
3069                                 }
3070                                 // no clients - will go to finalize
3071                                 if (scheduleFinalize) {
3072                                         victim->scheduleFinalize(false);
3073                                 }
3074                                 didPhase2List->removeObject(0);
3075                         }
3076                         IOLockLock( gJobsLock );
3077                 }
3078
3079                 // phase 3
3080                 do {
3081                         doPhase3 = false;
3082                         // finalize leaves
3083                         while ((victim = (IOService *) gIOFinalizeList->getObject(0))) {
3084                                 bool sendFinal = false;
3085                                 IOLockUnlock( gJobsLock );
3086                                 if (victim->lockForArbitration(true)) {
3087                                         sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
3088                                         if (sendFinal) {
3089                                                 victim->__state[1] |= kIOServiceFinalized;
3090                                         }
3091                                         victim->unlockForArbitration();
3092                                 }
3093                                 if (sendFinal) {
3094                                         _workLoopAction((IOWorkLoop::Action) &actionFinalize,
3095                                             victim, (void *)(uintptr_t) options );
3096                                 }
3097                                 IOLockLock( gJobsLock );
3098                                 // hold off free
3099                                 freeList->setObject( victim );
3100                                 // safe if finalize list is append only
3101                                 gIOFinalizeList->removeObject(0);
3102                         }
3103
3104                         for (idx = 0;
3105                             (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx));) {
3106                                 provider = (IOService *) gIOStopProviderList->getObject(idx);
3107                                 assert( provider );
3108
3109                                 uint64_t regID1 = provider->getRegistryEntryID();
3110                                 uint64_t regID2 = client->getRegistryEntryID();
3111
3112                                 if (!provider->isChild( client, gIOServicePlane )) {
3113                                         // may be multiply queued - nop it
3114                                         TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3115                                         IOServiceTrace(
3116                                                 IOSERVICE_TERMINATE_STOP_NOP,
3117                                                 (uintptr_t) regID1,
3118                                                 (uintptr_t) (regID1 >> 32),
3119                                                 (uintptr_t) regID2,
3120                                                 (uintptr_t) (regID2 >> 32));
3121                                 } else {
3122                                         // a terminated client is not ready for stop if it has clients, skip it
3123                                         bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
3124                                         IOLockUnlock( gJobsLock );
3125                                         if (deferStop && client->lockForArbitration(true)) {
3126                                                 deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
3127                                                 //deferStop = (!deferStop && (0 != client->getClient()));
3128                                                 //deferStop = (0 != client->getClient());
3129                                                 client->unlockForArbitration();
3130                                                 if (deferStop) {
3131                                                         TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
3132                                                         IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
3133                                                             (uintptr_t) regID1,
3134                                                             (uintptr_t) (regID1 >> 32),
3135                                                             (uintptr_t) regID2,
3136                                                             (uintptr_t) (regID2 >> 32));
3137
3138                                                         idx++;
3139                                                         IOLockLock( gJobsLock );
3140                                                         continue;
3141                                                 }
3142                                         }
3143                                         _workLoopAction((IOWorkLoop::Action) &actionStop,
3144                                             provider, (void *) client );
3145                                         IOLockLock( gJobsLock );
3146                                         // check the finalize list now
3147                                         doPhase3 = true;
3148                                 }
3149                                 // hold off free
3150                                 freeList->setObject( client );
3151                                 freeList->setObject( provider );
3152
3153                                 // safe if stop list is append only
3154                                 gIOStopList->removeObject( idx );
3155                                 gIOStopProviderList->removeObject( idx );
3156                                 idx = 0;
3157                         }
3158                 } while (doPhase3);
3159
3160                 gIOTerminateWork -= workDone;
3161                 moreToDo = (gIOTerminateWork != 0);
3162
3163                 if (!moreToDo) {
3164                         TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
3165                         IOServiceTrace(
3166                                 IOSERVICE_TERMINATE_DONE,
3167                                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
3168                 }
3169         } while (moreToDo);
3170
3171         IOLockUnlock( gJobsLock );
3172
3173         freeList->release();
3174         doPhase2List->release();
3175         didPhase2List->release();
3176
3177         IOLockLock( gJobsLock );
3178 }
3179
3180 bool
3181 IOService::finalize( IOOptionBits options )
3182 {
3183         OSIterator *  iter;
3184         IOService *   provider;
3185         uint64_t      regID1, regID2 = getRegistryEntryID();
3186
3187         iter = getProviderIterator();
3188         assert( iter );
3189
3190         if (iter) {
3191                 while ((provider = (IOService *) iter->getNextObject())) {
3192                         // -- compat
3193                         if (0 == (__state[1] & kIOServiceTermPhase3State)) {
3194                                 /* we come down here on programmatic terminate */
3195
3196                                 regID1 = provider->getRegistryEntryID();
3197                                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
3198                                 IOServiceTrace(
3199                                         IOSERVICE_TERMINATE_STOP,
3200                                         (uintptr_t) regID1,
3201                                         (uintptr_t) (regID1 >> 32),
3202                                         (uintptr_t) regID2,
3203                                         (uintptr_t) (regID2 >> 32));
3204
3205                                 stop( provider );
3206                                 if (provider->isOpen( this )) {
3207                                         provider->close( this );
3208                                 }
3209                                 detach( provider );
3210                         } else {
3211                                 //--
3212                                 if (provider->lockForArbitration( true )) {
3213                                         if (0 == (provider->__state[1] & kIOServiceTermPhase3State)) {
3214                                                 scheduleStop( provider );
3215                                         }
3216                                         provider->unlockForArbitration();
3217                                 }
3218                         }
3219                 }
3220                 iter->release();
3221         }
3222
3223         return true;
3224 }
3225
3226 #undef tailQ
3227 #undef headQ
3228
3229 /*
3230  * Terminate
3231  */
3232
3233 void
3234 IOService::doServiceTerminate( IOOptionBits options )
3235 {
3236 }
3237
3238 // a method in case someone needs to override it
3239 bool
3240 IOService::terminateClient( IOService * client, IOOptionBits options )
3241 {
3242         bool ok;
3243
3244         if (client->isParent( this, gIOServicePlane, true)) {
3245                 // we are the clients only provider
3246                 ok = client->terminate( options );
3247         } else {
3248                 ok = true;
3249         }
3250
3251         return ok;
3252 }
3253
3254 bool
3255 IOService::terminate( IOOptionBits options )
3256 {
3257         options |= kIOServiceTerminate;
3258
3259         return terminatePhase1( options );
3260 }
3261
3262 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3263
3264 /*
3265  * Open & close
3266  */
3267
3268 struct ServiceOpenMessageContext {
3269         IOService *  service;
3270         UInt32       type;
3271         IOService *  excludeClient;
3272         IOOptionBits options;
3273 };
3274
3275 static void
3276 serviceOpenMessageApplier( OSObject * object, void * ctx )
3277 {
3278         ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
3279
3280         if (object != context->excludeClient) {
3281                 context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
3282         }
3283 }
3284
3285 bool
3286 IOService::open(   IOService *     forClient,
3287     IOOptionBits    options,
3288     void *          arg )
3289 {
3290         bool                        ok;
3291         ServiceOpenMessageContext   context;
3292
3293         context.service             = this;
3294         context.type                = kIOMessageServiceIsAttemptingOpen;
3295         context.excludeClient       = forClient;
3296         context.options             = options;
3297
3298         applyToInterested( gIOGeneralInterest,
3299             &serviceOpenMessageApplier, &context );
3300
3301         if (false == lockForArbitration(false)) {
3302                 return false;
3303         }
3304
3305         ok = (0 == (__state[0] & kIOServiceInactiveState));
3306         if (ok) {
3307                 ok = handleOpen( forClient, options, arg );
3308         }
3309
3310         if (ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3311                 forClient->reserved->uvars->userServer->serviceOpen(this, forClient);
3312         }
3313
3314         unlockForArbitration();
3315
3316         return ok;
3317 }
3318
3319 void
3320 IOService::close(  IOService *     forClient,
3321     IOOptionBits    options )
3322 {
3323         bool                wasClosed;
3324         bool                last = false;
3325
3326         lockForArbitration();
3327
3328         wasClosed = handleIsOpen( forClient );
3329         if (wasClosed) {
3330                 handleClose( forClient, options );
3331                 last = (__state[1] & kIOServiceTermPhase3State);
3332
3333                 if (forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3334                         forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3335                 }
3336         }
3337
3338         unlockForArbitration();
3339
3340         if (last) {
3341                 forClient->scheduleStop( this );
3342         } else if (wasClosed) {
3343                 ServiceOpenMessageContext context;
3344
3345                 context.service         = this;
3346                 context.type            = kIOMessageServiceWasClosed;
3347                 context.excludeClient   = forClient;
3348                 context.options         = options;
3349
3350                 applyToInterested( gIOGeneralInterest,
3351                     &serviceOpenMessageApplier, &context );
3352         }
3353 }
3354
3355 bool
3356 IOService::isOpen( const IOService * forClient ) const
3357 {
3358         IOService * self = (IOService *) this;
3359         bool ok;
3360
3361         self->lockForArbitration();
3362
3363         ok = handleIsOpen( forClient );
3364
3365         self->unlockForArbitration();
3366
3367         return ok;
3368 }
3369
3370 bool
3371 IOService::handleOpen(     IOService *     forClient,
3372     IOOptionBits    options,
3373     void *          arg )
3374 {
3375         bool        ok;
3376
3377         ok = (NULL == __owner);
3378         if (ok) {
3379                 __owner = forClient;
3380         } else if (options & kIOServiceSeize) {
3381                 ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3382                     __owner, (void *)(uintptr_t) options ));
3383                 if (ok && (NULL == __owner)) {
3384                         __owner = forClient;
3385                 } else {
3386                         ok = false;
3387                 }
3388         }
3389         return ok;
3390 }
3391
3392 void
3393 IOService::handleClose(    IOService *     forClient,
3394     IOOptionBits    options )
3395 {
3396         if (__owner == forClient) {
3397                 __owner = NULL;
3398         }
3399 }
3400
3401 bool
3402 IOService::handleIsOpen(   const IOService * forClient ) const
3403 {
3404         if (forClient) {
3405                 return __owner == forClient;
3406         } else {
3407                 return __owner != forClient;
3408         }
3409 }
3410
3411 /*
3412  * Probing & starting
3413  */
3414 static SInt32
3415 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3416 {
3417         const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3418         const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3419         SInt32             val1;
3420         SInt32             val2;
3421
3422         val1 = 0;
3423         val2 = 0;
3424
3425         if (obj1) {
3426                 val1 = obj1->priority;
3427         }
3428
3429         if (obj2) {
3430                 val2 = obj2->priority;
3431         }
3432
3433         return val1 - val2;
3434 }
3435
3436 static SInt32
3437 IOServiceObjectOrder( const OSObject * entry, void * ref)
3438 {
3439         OSDictionary *      dict;
3440         IOService *         service;
3441         _IOServiceNotifier * notify;
3442         OSSymbol *          key = (OSSymbol *) ref;
3443         OSNumber *          offset;
3444         OSObject *          prop;
3445         SInt32              result;
3446
3447         prop = NULL;
3448         result = kIODefaultProbeScore;
3449         if ((dict = OSDynamicCast( OSDictionary, entry))) {
3450                 offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3451         } else if ((notify = OSDynamicCast( _IOServiceNotifier, entry))) {
3452                 return notify->priority;
3453         } else if ((service = OSDynamicCast( IOService, entry))) {
3454                 prop = service->copyProperty(key);
3455                 offset = OSDynamicCast(OSNumber, prop);
3456         } else {
3457                 assert( false );
3458                 offset = NULL;
3459         }
3460
3461         if (offset) {
3462                 result = offset->unsigned32BitValue();
3463         }
3464
3465         OSSafeReleaseNULL(prop);
3466
3467         return result;
3468 }
3469
3470 SInt32
3471 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3472 {
3473         const OSObject *    obj1 = (const OSObject *) inObj1;
3474         const OSObject *    obj2 = (const OSObject *) inObj2;
3475         SInt32               val1;
3476         SInt32               val2;
3477
3478         val1 = 0;
3479         val2 = 0;
3480
3481         if (obj1) {
3482                 val1 = IOServiceObjectOrder( obj1, ref );
3483         }
3484
3485         if (obj2) {
3486                 val2 = IOServiceObjectOrder( obj2, ref );
3487         }
3488
3489         return val1 - val2;
3490 }
3491
3492 IOService *
3493 IOService::copyClientWithCategory( const OSSymbol * category )
3494 {
3495         IOService *         service = NULL;
3496         OSIterator *        iter;
3497         const OSSymbol *    nextCat;
3498
3499         iter = getClientIterator();
3500         if (iter) {
3501                 while ((service = (IOService *) iter->getNextObject())) {
3502                         if (kIOServiceInactiveState & service->__state[0]) {
3503                                 continue;
3504                         }
3505                         nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3506                             service->getProperty( gIOMatchCategoryKey ));
3507                         if (category == nextCat) {
3508                                 service->retain();
3509                                 break;
3510                         }
3511                 }
3512                 iter->release();
3513         }
3514         return service;
3515 }
3516
3517 IOService *
3518 IOService::getClientWithCategory( const OSSymbol * category )
3519 {
3520         IOService *
3521             service = copyClientWithCategory(category);
3522         if (service) {
3523                 service->release();
3524         }
3525         return service;
3526 }
3527
3528 bool
3529 IOService::invokeNotifier( _IOServiceNotifier * notify )
3530 {
3531         _IOServiceNotifierInvocation invocation;
3532         bool                         willNotify;
3533         bool                         ret = true;
3534         invocation.thread = current_thread();
3535
3536 #if DEBUG_NOTIFIER_LOCKED
3537         uint32_t count;
3538         if ((count = isLockedForArbitration(0))) {
3539                 IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3540                 panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3541         }
3542 #endif /* DEBUG_NOTIFIER_LOCKED */
3543
3544         LOCKWRITENOTIFY();
3545         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3546
3547         if (willNotify) {
3548                 queue_enter( &notify->handlerInvocations, &invocation,
3549                     _IOServiceNotifierInvocation *, link );
3550         }
3551         UNLOCKNOTIFY();
3552
3553         if (willNotify) {
3554                 ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3555
3556                 LOCKWRITENOTIFY();
3557                 queue_remove( &notify->handlerInvocations, &invocation,
3558                     _IOServiceNotifierInvocation *, link );
3559                 if (kIOServiceNotifyWaiter & notify->state) {
3560                         notify->state &= ~kIOServiceNotifyWaiter;
3561                         WAKEUPNOTIFY( notify );
3562                 }
3563                 UNLOCKNOTIFY();
3564         }
3565
3566         return ret;
3567 }
3568
3569 bool
3570 IOService::invokeNotifiers(OSArray * willSend[])
3571 {
3572         OSArray *            array;
3573         _IOServiceNotifier * notify;
3574         bool                 ret = true;
3575
3576         array = *willSend;
3577         if (!array) {
3578                 return true;
3579         }
3580         *willSend = NULL;
3581
3582         for (unsigned int idx = 0;
3583             (notify = (_IOServiceNotifier *) array->getObject(idx));
3584             idx++) {
3585                 ret &= invokeNotifier(notify);
3586         }
3587         array->release();
3588
3589         return ret;
3590 }
3591
3592
3593 /*
3594  * Alloc and probe matching classes,
3595  * called on the provider instance
3596  */
3597
3598 void
3599 IOService::probeCandidates( OSOrderedSet * matches )
3600 {
3601         OSDictionary        *       match = NULL;
3602         OSSymbol            *       symbol;
3603         IOService           *       inst;
3604         IOService           *       newInst;
3605         OSDictionary        *       props;
3606         SInt32                      score;
3607         OSNumber            *       newPri;
3608         OSOrderedSet        *       familyMatches = NULL;
3609         OSOrderedSet        *       startList;
3610         OSSet               *       kexts = NULL;
3611         OSObject            *       kextRef;
3612
3613         OSDictionary        *       startDict = NULL;
3614         const OSSymbol      *       category;
3615         OSIterator          *       iter;
3616         _IOServiceNotifier  *       notify;
3617         OSObject            *       nextMatch = NULL;
3618         bool                        started;
3619         bool                        needReloc = false;
3620         bool                        matchDeferred = false;
3621 #if IOMATCHDEBUG
3622         SInt64                      debugFlags;
3623 #endif
3624         IOService           *       client = NULL;
3625         OSObject            *       prop1;
3626         OSObject            *       prop2;
3627         OSDictionary        *       rematchPersonality;
3628         OSNumber            *       num;
3629         uint32_t                    count;
3630         uint32_t                    dextCount;
3631         bool                        isDext;
3632         bool                        categoryConsumed;
3633
3634         prop2 = NULL;
3635         count = 0;
3636         prop1 = copyProperty(gIORematchPersonalityKey);
3637         rematchPersonality = OSDynamicCast(OSDictionary, prop1);
3638         if (rematchPersonality) {
3639                 prop2 = copyProperty(gIORematchCountKey);
3640                 num   = OSDynamicCast(OSNumber, prop2);
3641                 if (num) {
3642                         count = num->unsigned32BitValue();
3643                 }
3644                 OSSafeReleaseNULL(prop2);
3645         }
3646         dextCount = 0;
3647
3648         assert( matches );
3649         while (!needReloc
3650             && (nextMatch = matches->getFirstObject())) {
3651                 nextMatch->retain();
3652                 matches->removeObject(nextMatch);
3653
3654                 if ((notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3655                         if (0 == (__state[0] & kIOServiceInactiveState)) {
3656                                 invokeNotifier( notify );
3657                         }
3658                         nextMatch->release();
3659                         nextMatch = NULL;
3660                         continue;
3661                 } else if (!(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3662                         nextMatch->release();
3663                         nextMatch = NULL;
3664                         continue;
3665                 }
3666
3667                 props = NULL;
3668 #if IOMATCHDEBUG
3669                 debugFlags = getDebugFlags( match );
3670 #endif
3671
3672                 do {
3673                         isDext = (NULL != match->getObject(gIOUserServerNameKey));
3674                         if (isDext && !(kIODKEnable & gIODKDebug)) {
3675                                 continue;
3676                         }
3677
3678                         category = OSDynamicCast( OSSymbol,
3679                             match->getObject( gIOMatchCategoryKey ));
3680                         if (NULL == category) {
3681                                 category = gIODefaultMatchCategoryKey;
3682                         }
3683                         client = copyClientWithCategory(category);
3684
3685                         categoryConsumed = (client != NULL);
3686                         if (categoryConsumed) {
3687 #if IOMATCHDEBUG
3688                                 if ((debugFlags & kIOLogMatch) && (this != gIOResources)) {
3689                                         LOG("%s: match category %s exists\n", getName(),
3690                                             category->getCStringNoCopy());
3691                                 }
3692 #endif
3693                                 OSSafeReleaseNULL(client);
3694                                 if (!isDext) {
3695                                         break;
3696                                 }
3697                         }
3698
3699                         // create a copy now in case its modified during matching
3700                         props = OSDictionary::withDictionary(match, match->getCount());
3701                         if (NULL == props) {
3702                                 break;
3703                         }
3704                         props->setCapacityIncrement(1);
3705
3706                         // check the nub matches
3707                         if (false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) {
3708                                 break;
3709                         }
3710                         if (isDext) {
3711                                 dextCount++;
3712                                 if (categoryConsumed) {
3713                                         break;
3714                                 }
3715                         }
3716
3717                         if (rematchPersonality) {
3718                                 bool personalityMatch = match->isEqualTo(rematchPersonality);
3719                                 if (count > gIODextRelaunchMax) {
3720                                         personalityMatch = !personalityMatch;
3721                                 }
3722                                 if (!personalityMatch) {
3723                                         break;
3724                                 }
3725                         }
3726
3727                         // Check to see if driver reloc has been loaded.
3728                         needReloc = (false == gIOCatalogue->isModuleLoaded( match, &kextRef ));
3729                         if (needReloc) {
3730 #if IOMATCHDEBUG
3731                                 if (debugFlags & kIOLogCatalogue) {
3732                                         LOG("%s: stalling for module\n", getName());
3733                                 }
3734 #endif
3735                                 // If reloc hasn't been loaded, exit;
3736                                 // reprobing will occur after reloc has been loaded.
3737                                 break;
3738                         }
3739                         if (kextRef) {
3740                                 if (NULL == kexts) {
3741                                         kexts = OSSet::withCapacity(1);
3742                                 }
3743                                 if (kexts) {
3744                                         kexts->setObject(kextRef);
3745                                         kextRef->release();
3746                                 }
3747                         }
3748                         if (isDext) {
3749                                 // copy saved for rematchng
3750                                 props->setObject(gIOMatchedPersonalityKey, match);
3751                         }
3752                         // reorder on family matchPropertyTable score.
3753                         if (NULL == familyMatches) {
3754                                 familyMatches = OSOrderedSet::withCapacity( 1,
3755                                     IOServiceOrdering, (void *) gIOProbeScoreKey );
3756                         }
3757                         if (familyMatches) {
3758                                 familyMatches->setObject( props );
3759                         }
3760                 } while (false);
3761
3762                 OSSafeReleaseNULL(nextMatch);
3763                 OSSafeReleaseNULL(props);
3764         }
3765         matches->release();
3766         matches = NULL;
3767
3768         if (familyMatches) {
3769                 while (!needReloc
3770                     && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3771                         props->retain();
3772                         familyMatches->removeObject( props );
3773
3774                         inst = NULL;
3775                         newInst = NULL;
3776 #if IOMATCHDEBUG
3777                         debugFlags = getDebugFlags( props );
3778 #endif
3779                         do {
3780                                 symbol = OSDynamicCast( OSSymbol,
3781                                     props->getObject( gIOClassKey));
3782                                 if (!symbol) {
3783                                         continue;
3784                                 }
3785
3786                                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
3787
3788                                 // alloc the driver instance
3789                                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3790
3791                                 if (!inst || !OSDynamicCast(IOService, inst)) {
3792                                         IOLog("Couldn't alloc class \"%s\"\n",
3793                                             symbol->getCStringNoCopy());
3794                                         continue;
3795                                 }
3796
3797                                 // init driver instance
3798                                 if (!(inst->init( props ))) {
3799 #if IOMATCHDEBUG
3800                                         if (debugFlags & kIOLogStart) {
3801                                                 IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3802                                         }
3803 #endif
3804                                         continue;
3805                                 }
3806                                 if (__state[1] & kIOServiceSynchronousState) {
3807                                         inst->__state[1] |= kIOServiceSynchronousState;
3808                                 }
3809
3810                                 // give the driver the default match category if not specified
3811                                 category = OSDynamicCast( OSSymbol,
3812                                     props->getObject( gIOMatchCategoryKey ));
3813                                 if (NULL == category) {
3814                                         category = gIODefaultMatchCategoryKey;
3815                                 }
3816                                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3817                                 // attach driver instance
3818                                 if (!(inst->attach( this ))) {
3819                                         continue;
3820                                 }
3821
3822                                 // pass in score from property table
3823                                 score = familyMatches->orderObject( props );
3824
3825                                 // & probe the new driver instance
3826 #if IOMATCHDEBUG
3827                                 if (debugFlags & kIOLogProbe) {
3828                                         LOG("%s::probe(%s)\n",
3829                                             inst->getMetaClass()->getClassName(), getName());
3830                                 }
3831 #endif
3832
3833                                 newInst = inst->probe( this, &score );
3834                                 inst->detach( this );
3835                                 if (NULL == newInst) {
3836 #if IOMATCHDEBUG
3837                                         if (debugFlags & kIOLogProbe) {
3838                                                 IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3839                                         }
3840 #endif
3841                                         continue;
3842                                 }
3843
3844                                 // save the score
3845                                 newPri = OSNumber::withNumber( score, 32 );
3846                                 if (newPri) {
3847                                         newInst->setProperty( gIOProbeScoreKey, newPri );
3848                                         newPri->release();
3849                                 }
3850
3851                                 // add to start list for the match category
3852                                 if (NULL == startDict) {
3853                                         startDict = OSDictionary::withCapacity( 1 );
3854                                 }
3855                                 assert( startDict );
3856                                 startList = (OSOrderedSet *)
3857                                     startDict->getObject( category );
3858                                 if (NULL == startList) {
3859                                         startList = OSOrderedSet::withCapacity( 1,
3860                                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3861                                         if (startDict && startList) {
3862                                                 startDict->setObject( category, startList );
3863                                                 startList->release();
3864                                         }
3865                                 }
3866                                 assert( startList );
3867                                 if (startList) {
3868                                         startList->setObject( newInst );
3869                                 }
3870                         } while (false);
3871
3872                         props->release();
3873                         if (inst) {
3874                                 inst->release();
3875                         }
3876                 }
3877                 familyMatches->release();
3878                 familyMatches = NULL;
3879         }
3880
3881         // start the best (until success) of each category
3882
3883         iter = OSCollectionIterator::withCollection( startDict );
3884         if (iter) {
3885                 while ((category = (const OSSymbol *) iter->getNextObject())) {
3886                         startList = (OSOrderedSet *) startDict->getObject( category );
3887                         assert( startList );
3888                         if (!startList) {
3889                                 continue;
3890                         }
3891
3892                         started = false;
3893                         while (true // (!started)
3894                             && !matchDeferred
3895                             && (inst = (IOService *)startList->getFirstObject())) {
3896                                 inst->retain();
3897                                 startList->removeObject(inst);
3898
3899 #if IOMATCHDEBUG
3900                                 debugFlags = getDebugFlags( inst );
3901
3902                                 if (debugFlags & kIOLogStart) {
3903                                         if (started) {
3904                                                 LOG( "match category exists, skipping " );
3905                                         }
3906                                         LOG( "%s::start(%s) <%d>\n", inst->getName(),
3907                                             getName(), inst->getRetainCount());
3908                                 }
3909 #endif
3910                                 if (false == started) {
3911 #if !NO_KEXTD
3912                                         IOLockLock(gJobsLock);
3913                                         matchDeferred = (gIOMatchDeferList
3914                                             && (kOSBooleanTrue == inst->getProperty(gIOMatchDeferKey)));
3915                                         if (matchDeferred && (-1U == gIOMatchDeferList->getNextIndexOfObject(this, 0))) {
3916                                                 gIOMatchDeferList->setObject(this);
3917                                         }
3918                                         IOLockUnlock(gJobsLock);
3919                                         if (matchDeferred) {
3920                                                 symbol = OSDynamicCast(OSSymbol, inst->getProperty(gIOClassKey));
3921                                                 IOLog("%s(0x%qx): matching deferred by %s\n",
3922                                                     getName(), getRegistryEntryID(),
3923                                                     symbol ? symbol->getCStringNoCopy() : "");
3924                                                 // rematching will occur after kextd loads all plists
3925                                         }
3926 #endif
3927                                         if (!matchDeferred) {
3928                                                 started = startCandidate( inst );
3929 #if IOMATCHDEBUG
3930                                                 if ((debugFlags & kIOLogStart) && (false == started)) {
3931                                                         LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3932                                                             inst->getRetainCount());
3933                                                 }
3934 #endif
3935                                         }
3936                                 }
3937                                 inst->release();
3938                         }
3939                 }
3940                 iter->release();
3941         }
3942
3943         OSSafeReleaseNULL(prop1);
3944
3945         if (dextCount) {
3946                 num = OSNumber::withNumber(dextCount, 32);
3947                 setProperty(gIODEXTMatchCountKey, num);
3948                 OSSafeReleaseNULL(num);
3949         } else if (rematchPersonality) {
3950                 removeProperty(gIODEXTMatchCountKey);
3951         }
3952
3953         // now that instances are created, drop the refs on any kexts allowing unload
3954         if (kexts) {
3955                 OSKext::dropMatchingReferences(kexts);
3956                 OSSafeReleaseNULL(kexts);
3957         }
3958
3959         // adjust the busy count by +1 if matching is stalled for a module,
3960         // or -1 if a previously stalled matching is complete.
3961         lockForArbitration();
3962         SInt32 adjBusy = 0;
3963         uint64_t regID = getRegistryEntryID();
3964
3965         if (needReloc) {
3966                 adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3967                 if (adjBusy) {
3968                         IOServiceTrace(
3969                                 IOSERVICE_MODULESTALL,
3970                                 (uintptr_t) regID,
3971                                 (uintptr_t) (regID >> 32),
3972                                 (uintptr_t) this,
3973                                 0);
3974
3975                         __state[1] |= kIOServiceModuleStallState;
3976                 }
3977         } else if (__state[1] & kIOServiceModuleStallState) {
3978                 IOServiceTrace(
3979                         IOSERVICE_MODULEUNSTALL,
3980                         (uintptr_t) regID,
3981                         (uintptr_t) (regID >> 32),
3982                         (uintptr_t) this,
3983                         0);
3984
3985                 __state[1] &= ~kIOServiceModuleStallState;
3986                 adjBusy = -1;
3987         }
3988         if (adjBusy) {
3989                 _adjustBusy( adjBusy );
3990         }
3991         unlockForArbitration();
3992
3993         if (startDict) {
3994                 startDict->release();
3995         }
3996 }
3997
3998 /*
3999  * Wait for a IOUserServer to check in
4000  */
4001
4002 static
4003 __attribute__((noinline, not_tail_called))
4004 IOService *
4005 __WAITING_FOR_USER_SERVER__(OSDictionary * matching)
4006 {
4007         IOService * server;
4008         server = IOService::waitForMatchingService(matching, kIOUserServerCheckInTimeoutSecs * NSEC_PER_SEC);
4009         return server;
4010 }
4011
4012 void
4013 IOService::willShutdown()
4014 {
4015         gKextdWillTerminate = true;
4016 #if !NO_KEXTD
4017         getPlatform()->waitQuiet(30 * NSEC_PER_SEC);
4018 #endif
4019         OSKext::willShutdown();
4020 }
4021
4022 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4023
4024 void
4025 IOServicePH::init(IOPMrootDomain * root)
4026 {
4027         fUserServers     = OSArray::withCapacity(4);
4028         fMatchingWork    = OSArray::withCapacity(4);
4029
4030         assert(fUserServers && fMatchingWork);
4031
4032         fRootNotifier = root->registerInterest(
4033                 gIOPriorityPowerStateInterest, &IOServicePH::systemPowerChange, NULL, NULL);
4034
4035         assert(fRootNotifier);
4036 }
4037
4038 void
4039 IOServicePH::lock()
4040 {
4041         IOLockLock(gJobsLock);
4042 }
4043
4044 void
4045 IOServicePH::unlock()
4046 {
4047         IOLockUnlock(gJobsLock);
4048 }
4049
4050 void
4051 IOServicePH::serverAdd(IOUserServer * server)
4052 {
4053         uint32_t idx;
4054
4055         lock();
4056         idx = fUserServers->getNextIndexOfObject(server, 0);
4057         if (idx == -1U) {
4058                 fUserServers->setObject(server);
4059         }
4060         unlock();
4061 }
4062
4063 void
4064 IOServicePH::serverRemove(IOUserServer * server)
4065 {
4066         uint32_t idx;
4067
4068         lock();
4069         idx = fUserServers->getNextIndexOfObject(server, 0);
4070         if (idx != -1U) {
4071                 fUserServers->removeObject(idx);
4072         }
4073         unlock();
4074 }
4075
4076 void
4077 IOServicePH::serverAck(IOUserServer * server)
4078 {
4079         uint32_t    idx;
4080         IOService * ackTo;
4081         uint32_t    ackToRef;
4082
4083         ackTo = NULL;
4084         lock();
4085         if (server && fUserServersWait) {
4086                 idx = fUserServersWait->getNextIndexOfObject(server, 0);
4087                 if (idx != -1U) {
4088                         fUserServersWait->removeObject(idx);
4089                         if (0 == fUserServersWait->getCount()) {
4090                                 OSSafeReleaseNULL(fUserServersWait);
4091                         }
4092                 }
4093         }
4094         if (!fUserServersWait && !fMatchingWork->getCount()) {
4095                 ackTo             = fSystemPowerAckTo;
4096                 ackToRef          = fSystemPowerAckRef;
4097                 fSystemPowerAckTo = NULL;
4098         }
4099         unlock();
4100
4101         if (ackTo) {
4102                 DKLOG("allowPowerChange\n");
4103                 ackTo->allowPowerChange((uintptr_t) ackToRef);
4104         }
4105 }
4106
4107 bool
4108 IOServicePH::matchingStart(IOService * service)
4109 {
4110         uint32_t idx;
4111         bool ok;
4112
4113         lock();
4114         ok = !fSystemOff;
4115         if (ok) {
4116                 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4117                 if (idx == -1U) {
4118                         fMatchingWork->setObject(service);
4119                 }
4120         } else {
4121                 if (!fMatchingDelayed) {
4122                         fMatchingDelayed = OSArray::withObjects((const OSObject **) &service, 1, 1);
4123                 } else {
4124                         idx = fMatchingDelayed->getNextIndexOfObject(service, 0);
4125                         if (idx == -1U) {
4126                                 fMatchingDelayed->setObject(service);
4127                         }
4128                 }
4129         }
4130         unlock();
4131
4132         return ok;
4133 }
4134
4135 void
4136 IOServicePH::matchingEnd(IOService * service)
4137 {
4138         uint32_t idx;
4139         OSArray   * notifyServers;
4140         OSArray   * deferredMatches;
4141
4142         notifyServers   = NULL;
4143         deferredMatches = NULL;
4144
4145         lock();
4146
4147         if (service) {
4148                 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4149                 if (idx != -1U) {
4150                         fMatchingWork->removeObject(idx);
4151                 }
4152         }
4153
4154
4155         if ((fUserServerOff != fSystemOff) && fUserServers->getCount()) {
4156                 if (fSystemOff) {
4157                         if (0 == fMatchingWork->getCount()) {
4158                                 fUserServersWait = OSArray::withArray(fUserServers);
4159                                 notifyServers = OSArray::withArray(fUserServers);
4160                                 fUserServerOff = fSystemOff;
4161                         }
4162                 } else {
4163                         notifyServers = OSArray::withArray(fUserServers);
4164                         fUserServerOff = fSystemOff;
4165                 }
4166         }
4167
4168         if (!fSystemOff && fMatchingDelayed) {
4169                 deferredMatches = fMatchingDelayed;
4170                 fMatchingDelayed = NULL;
4171         }
4172
4173         unlock();
4174
4175         if (notifyServers) {
4176                 notifyServers->iterateObjects(^bool (OSObject * obj) {
4177                         IOUserServer * us;
4178                         us = (typeof(us))obj;
4179                         us->systemPower(fSystemOff);
4180                         return false;
4181                 });
4182                 OSSafeReleaseNULL(notifyServers);
4183         }
4184
4185         if (deferredMatches) {
4186                 DKLOG("sleep deferred rematching count %d\n", deferredMatches->getCount());
4187                 deferredMatches->iterateObjects(^bool (OSObject * obj)
4188                 {
4189                         ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
4190                         return false;
4191                 });
4192                 deferredMatches->release();
4193         }
4194
4195         serverAck(NULL);
4196 }
4197
4198 IOReturn
4199 IOServicePH::systemPowerChange(
4200         void * target,
4201         void * refCon,
4202         UInt32 messageType, IOService * service,
4203         void * messageArgument, vm_size_t argSize)
4204 {
4205         IOReturn                               ret;
4206         IOUserServer                         * us;
4207         IOPMSystemCapabilityChangeParameters * params;
4208
4209         us = NULL;
4210
4211         switch (messageType) {
4212         case kIOMessageSystemCapabilityChange:
4213
4214                 params = (typeof params)messageArgument;
4215
4216                 if (kIODKLogPM & gIODKDebug) {
4217                         IOLog("IOServicePH::kIOMessageSystemCapabilityChange: %s%s 0x%x->0x%x\n",
4218                             params->changeFlags & kIOPMSystemCapabilityWillChange ? "will" : "",
4219                             params->changeFlags & kIOPMSystemCapabilityDidChange ? "did" : "",
4220                             params->fromCapabilities,
4221                             params->toCapabilities);
4222                 }
4223
4224                 if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4225                     (params->fromCapabilities & kIOPMSystemCapabilityCPU) &&
4226                     ((params->toCapabilities & kIOPMSystemCapabilityCPU) == 0)) {
4227                         lock();
4228                         fSystemOff         = true;
4229                         fSystemPowerAckRef = params->notifyRef;
4230                         fSystemPowerAckTo  = service;
4231                         unlock();
4232
4233                         matchingEnd(NULL);
4234
4235                         params->maxWaitForReply = 60 * 1000 * 1000;
4236                         ret = kIOReturnSuccess;
4237                 } else if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4238                     ((params->fromCapabilities & kIOPMSystemCapabilityCPU) == 0) &&
4239                     (params->toCapabilities & kIOPMSystemCapabilityCPU)) {
4240                         lock();
4241                         fSystemOff = false;
4242                         unlock();
4243
4244                         matchingEnd(NULL);
4245
4246                         params->maxWaitForReply = 0;
4247                         ret                 = kIOReturnSuccess;
4248                 } else {
4249                         params->maxWaitForReply = 0;
4250                         ret                 = kIOReturnSuccess;
4251                 }
4252                 break;
4253
4254         default:
4255                 ret = kIOReturnUnsupported;
4256                 break;
4257         }
4258
4259         return ret;
4260 }
4261
4262 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4263
4264 /*
4265  * Start a previously attached & probed instance,
4266  * called on exporting object instance
4267  */
4268
4269 bool
4270 IOService::startCandidate( IOService * service )
4271 {
4272         bool                ok;
4273         OSObject          * obj;
4274         OSObject          * prop;
4275         IOUserServer      * userServer;
4276         bool                ph;
4277
4278         userServer = NULL;
4279         obj = service->copyProperty(gIOUserServerNameKey);
4280
4281         if (obj && (this == gIOResources)) {
4282                 ok = false;
4283         } else {
4284                 ok = service->attach( this );
4285         }
4286         if (!ok) {
4287                 return false;
4288         }
4289
4290         if ((this != gIOResources) && (this != gIOUserResources)) {
4291                 // stall for any nub resources
4292                 checkResources();
4293                 // stall for any driver resources
4294                 service->checkResources();
4295         }
4296         ph = false;
4297         {
4298                 OSString       * bundleID;
4299                 OSString       * serverName;
4300                 OSString       * str;
4301                 const OSSymbol * sym;
4302                 OSDictionary   * matching;
4303                 IOService      * server;
4304                 OSNumber       * serverTag;
4305                 uint64_t         entryID;
4306
4307                 if ((serverName = OSDynamicCast(OSString, obj))) {
4308                         obj       = service->copyProperty(gIOModuleIdentifierKey);
4309                         bundleID  = OSDynamicCast(OSString, obj);
4310                         entryID   = service->getRegistryEntryID();
4311                         serverTag = OSNumber::withNumber(entryID, 64);
4312
4313                         if (gKextdWillTerminate) {
4314                                 DKLOG("%s disabled in shutdown\n", serverName->getCStringNoCopy());
4315                                 service->detach(this);
4316                                 OSSafeReleaseNULL(obj);
4317                                 return false;
4318                         }
4319
4320                         ph = IOServicePH::matchingStart(this);
4321                         if (!ph) {
4322                                 DKLOG("%s deferred in sleep\n", serverName->getCStringNoCopy());
4323                                 service->detach(this);
4324                                 OSSafeReleaseNULL(obj);
4325                                 return false;
4326                         }
4327
4328                         prop = service->copyProperty(gIOUserClassKey);
4329                         str = OSDynamicCast(OSString, prop);
4330                         if (str) {
4331                                 service->setName(str);
4332                         }
4333                         OSSafeReleaseNULL(prop);
4334
4335                         if (!(kIODKDisableDextLaunch & gIODKDebug)) {
4336                                 OSKext::requestDaemonLaunch(bundleID, serverName, serverTag);
4337                         }
4338                         sym = OSSymbol::withString(serverName);
4339                         matching = serviceMatching(gIOUserServerClassKey);
4340                         propertyMatching(gIOUserServerNameKey, sym, matching);
4341                         if (!(kIODKDisableDextTag & gIODKDebug)) {
4342                                 propertyMatching(gIOUserServerTagKey, serverTag, matching);
4343                         }
4344
4345                         server = __WAITING_FOR_USER_SERVER__(matching);
4346                         matching->release();
4347                         OSSafeReleaseNULL(serverTag);
4348                         OSSafeReleaseNULL(serverName);
4349
4350                         userServer = OSDynamicCast(IOUserServer, server);
4351                         if (!userServer) {
4352                                 service->detach(this);
4353                                 IOServicePH::matchingEnd(this);
4354                                 DKLOG(DKS " user server timeout\n", DKN(service));
4355                                 return false;
4356                         }
4357
4358                         if (!(kIODKDisableCDHashChecking & gIODKDebug)) {
4359                                 if (!userServer->serviceMatchesCDHash(service)) {
4360                                         service->detach(this);
4361                                         IOServicePH::matchingEnd(this);
4362                                         userServer->exit("CDHash check failed");
4363                                         userServer->release();
4364                                         return false;
4365                                 }
4366                         }
4367                         OSKext *kext = OSKext::lookupKextWithIdentifier(bundleID);
4368                         if (!kext) {
4369                                 const char *name = bundleID->getCStringNoCopy();
4370                                 IOLog("%s Could not find OSKext for %s\n", __func__, name);
4371                                 goto skip_log;
4372                         }
4373
4374                         /*
4375                          * Used for logging
4376                          */
4377                         userServer->setTaskLoadTag(kext);
4378                         userServer->setDriverKitUUID(kext);
4379                         OSKext::OSKextLogDriverKitInfoLoad(kext);
4380 skip_log:
4381                         OSSafeReleaseNULL(bundleID);
4382                         OSSafeReleaseNULL(kext);
4383
4384                         if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4385                                 if (!userServer->checkEntitlements(this, service)) {
4386                                         service->detach(this);
4387                                         IOServicePH::matchingEnd(this);
4388                                         userServer->exit("Entitlements check failed");
4389                                         userServer->release();
4390                                         return false;
4391                                 }
4392                         }
4393
4394                         userServer->serviceAttach(service, this);
4395                 }
4396         }
4397
4398         AbsoluteTime startTime;
4399         AbsoluteTime endTime;
4400         UInt64       nano;
4401
4402         if (kIOLogStart & gIOKitDebug) {
4403                 clock_get_uptime(&startTime);
4404         }
4405
4406         ok = service->start(this);
4407
4408         if (kIOLogStart & gIOKitDebug) {
4409                 clock_get_uptime(&endTime);
4410
4411                 if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0) {
4412                         SUB_ABSOLUTETIME(&endTime, &startTime);
4413                         absolutetime_to_nanoseconds(endTime, &nano);
4414                         if (nano > 500000000ULL) {
4415                                 IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
4416                         }
4417                 }
4418         }
4419         if (userServer) {
4420                 userServer->serviceStarted(service, this, ok);
4421                 userServer->release();
4422         }
4423         if (!ok) {
4424                 service->detach( this );
4425         }
4426
4427         if (ph) {
4428                 IOServicePH::matchingEnd(this);
4429         }
4430
4431         return ok;
4432 }
4433
4434 void
4435 IOService::publishResource( const char * key, OSObject * value )
4436 {
4437         const OSSymbol *    sym;
4438
4439         if ((sym = OSSymbol::withCString( key))) {
4440                 publishResource( sym, value);
4441                 sym->release();
4442         }
4443 }
4444
4445 void
4446 IOService::publishResource( const OSSymbol * key, OSObject * value )
4447 {
4448         if (NULL == value) {
4449                 value = (OSObject *) gIOServiceKey;
4450         }
4451
4452         gIOResources->setProperty( key, value);
4453
4454         if (IORecursiveLockHaveLock( gNotificationLock)) {
4455                 return;
4456         }
4457
4458         gIOResourceGenerationCount++;
4459         gIOResources->registerService();
4460 }
4461
4462 void
4463 IOService::publishUserResource( const OSSymbol * key, OSObject * value )
4464 {
4465         if (NULL == value) {
4466                 value = (OSObject *) gIOServiceKey;
4467         }
4468
4469         gIOUserResources->setProperty( key, value);
4470
4471         if (IORecursiveLockHaveLock( gNotificationLock)) {
4472                 return;
4473         }
4474
4475         gIOResourceGenerationCount++;
4476         gIOUserResources->registerService();
4477 }
4478
4479 bool
4480 IOService::addNeededResource( const char * key )
4481 {
4482         OSObject *  resourcesProp;
4483         OSSet *     set;
4484         OSString *  newKey;
4485         bool ret;
4486
4487         resourcesProp = copyProperty( gIOResourceMatchKey );
4488         if (!resourcesProp) {
4489                 return false;
4490         }
4491
4492         newKey = OSString::withCString( key );
4493         if (!newKey) {
4494                 resourcesProp->release();
4495                 return false;
4496         }
4497
4498         set = OSDynamicCast( OSSet, resourcesProp );
4499         if (!set) {
4500                 set = OSSet::withCapacity( 1 );
4501                 if (set) {
4502                         set->setObject( resourcesProp );
4503                 }
4504         } else {
4505                 set->retain();
4506         }
4507
4508         set->setObject( newKey );
4509         newKey->release();
4510         ret = setProperty( gIOResourceMatchKey, set );
4511         set->release();
4512         resourcesProp->release();
4513
4514         return ret;
4515 }
4516
4517 bool
4518 IOService::checkResource( OSObject * matching )
4519 {
4520         OSString *          str;
4521         OSDictionary *      table;
4522
4523         if ((str = OSDynamicCast( OSString, matching ))) {
4524                 if (gIOResources->getProperty( str )) {
4525                         return true;
4526                 }
4527         }
4528
4529         if (str) {
4530                 table = resourceMatching( str );
4531         } else if ((table = OSDynamicCast( OSDictionary, matching ))) {
4532                 table->retain();
4533         } else {
4534                 IOLog("%s: Can't match using: %s\n", getName(),
4535                     matching->getMetaClass()->getClassName());
4536                 /* false would stall forever */
4537                 return true;
4538         }
4539
4540         if (gIOKitDebug & kIOLogConfig) {
4541                 LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
4542         }
4543
4544         waitForService( table );
4545
4546         if (gIOKitDebug & kIOLogConfig) {
4547                 LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
4548         }
4549
4550         return true;
4551 }
4552
4553 bool
4554 IOService::checkResources( void )
4555 {
4556         OSObject *          resourcesProp;
4557         OSSet *             set;
4558         OSIterator *        iter;
4559         bool                ok;
4560
4561         resourcesProp = copyProperty( gIOResourceMatchKey );
4562         if (NULL == resourcesProp) {
4563                 return true;
4564         }
4565
4566         if ((set = OSDynamicCast( OSSet, resourcesProp ))) {
4567                 iter = OSCollectionIterator::withCollection( set );
4568                 ok = (NULL != iter);
4569                 while (ok && (resourcesProp = iter->getNextObject())) {
4570                         ok = checkResource( resourcesProp );
4571                 }
4572                 if (iter) {
4573                         iter->release();
4574                 }
4575         } else {
4576                 ok = checkResource( resourcesProp );
4577         }
4578
4579         OSSafeReleaseNULL(resourcesProp);
4580
4581         return ok;
4582 }
4583
4584
4585 void
4586 _IOConfigThread::configThread( int configThreadId )
4587 {
4588         _IOConfigThread *   inst;
4589
4590         do {
4591                 if (!(inst = new _IOConfigThread)) {
4592                         continue;
4593                 }
4594                 if (!inst->init()) {
4595                         continue;
4596                 }
4597                 thread_t thread;
4598                 if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &thread)) {
4599                         continue;
4600                 }
4601
4602                 char threadName[MAXTHREADNAMESIZE];
4603                 snprintf(threadName, sizeof(threadName), "IOConfigThread_%d", configThreadId);
4604                 thread_set_thread_name(thread, threadName);
4605                 thread_deallocate(thread);
4606
4607                 return;
4608         } while (false);
4609
4610         if (inst) {
4611                 inst->release();
4612         }
4613
4614         return;
4615 }
4616
4617 void
4618 IOService::doServiceMatch( IOOptionBits options )
4619 {
4620         _IOServiceNotifier * notify;
4621         OSIterator *        iter;
4622         OSOrderedSet *      matches;
4623         OSArray *           resourceKeys = NULL;
4624         SInt32              catalogGeneration;
4625         bool                keepGuessing = true;
4626         bool                reRegistered = true;
4627         bool                didRegister;
4628         OSArray *           notifiers[2] = {NULL};
4629
4630 //    job->nub->deliverNotification( gIOPublishNotification,
4631 //                              kIOServiceRegisteredState, 0xffffffff );
4632
4633         while (keepGuessing) {
4634                 matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
4635                 // the matches list should always be created by findDrivers()
4636                 if (matches) {
4637                         lockForArbitration();
4638                         if (0 == (__state[0] & kIOServiceFirstPublishState)) {
4639                                 getMetaClass()->addInstance(this);
4640                                 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
4641                                     kIOServiceFirstPublishState, 0xffffffff );
4642                         }
4643                         LOCKREADNOTIFY();
4644                         __state[1] &= ~kIOServiceNeedConfigState;
4645                         __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
4646                         didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
4647                         __state[0] |= kIOServiceRegisteredState;
4648
4649                         keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
4650                         if (reRegistered && keepGuessing) {
4651                                 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
4652                                     gNotifications->getObject( gIOPublishNotification ));
4653                                 if (iter) {
4654                                         while ((notify = (_IOServiceNotifier *)
4655                                             iter->getNextObject())) {
4656                                                 if (matchPassive(notify->matching, 0)
4657                                                     && (kIOServiceNotifyEnable & notify->state)) {
4658                                                         matches->setObject( notify );
4659                                                 }
4660                                         }
4661                                         iter->release();
4662                                 }
4663                         }
4664
4665                         UNLOCKNOTIFY();
4666                         unlockForArbitration();
4667                         invokeNotifiers(&notifiers[0]);
4668
4669                         if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) {
4670                                 if ((this == gIOResources) || (this == gIOUserResources)) {
4671                                         if (resourceKeys) {
4672                                                 resourceKeys->release();
4673                                         }
4674                                         resourceKeys = copyPropertyKeys();
4675                                 }
4676                                 probeCandidates( matches );
4677                         } else {
4678                                 matches->release();
4679                         }
4680                 }
4681
4682                 lockForArbitration();
4683                 reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
4684                 keepGuessing =
4685                     (reRegistered || (catalogGeneration !=
4686                     gIOCatalogue->getGenerationCount()))
4687                     && (0 == (__state[0] & kIOServiceInactiveState));
4688
4689                 if (keepGuessing) {
4690                         unlockForArbitration();
4691                 }
4692         }
4693
4694         if ((0 == (__state[0] & kIOServiceInactiveState))
4695             && (0 == (__state[1] & kIOServiceModuleStallState))) {
4696                 if (resourceKeys) {
4697                         setProperty(gIOResourceMatchedKey, resourceKeys);
4698                 }
4699
4700                 notifiers[0] = copyNotifiers(gIOMatchedNotification,
4701                     kIOServiceMatchedState, 0xffffffff);
4702                 if (0 == (__state[0] & kIOServiceFirstMatchState)) {
4703                         notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
4704                             kIOServiceFirstMatchState, 0xffffffff);
4705                 }
4706         }
4707
4708         __state[1] &= ~kIOServiceConfigRunning;
4709         unlockForArbitration();
4710
4711         if (resourceKeys) {
4712                 resourceKeys->release();
4713         }
4714
4715         invokeNotifiers(&notifiers[0]);
4716         invokeNotifiers(&notifiers[1]);
4717
4718         lockForArbitration();
4719         __state[1] &= ~kIOServiceConfigState;
4720         scheduleTerminatePhase2();
4721
4722         _adjustBusy( -1 );
4723         unlockForArbitration();
4724 }
4725
4726 UInt32
4727 IOService::_adjustBusy( SInt32 delta )
4728 {
4729         IOService * next;
4730         UInt32      count;
4731         UInt32      result;
4732         bool        wasQuiet, nowQuiet, needWake;
4733
4734         next = this;
4735         result = __state[1] & kIOServiceBusyStateMask;
4736
4737         if (delta) {
4738                 do {
4739                         if (next != this) {
4740                                 next->lockForArbitration();
4741                         }
4742                         count = next->__state[1] & kIOServiceBusyStateMask;
4743                         wasQuiet = (0 == count);
4744                         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) {
4745                                 OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
4746                         } else {
4747                                 count += delta;
4748                         }
4749                         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
4750                         nowQuiet = (0 == count);
4751                         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
4752
4753                         if (needWake) {
4754                                 next->__state[1] &= ~kIOServiceBusyWaiterState;
4755                                 IOLockLock( gIOServiceBusyLock );
4756                                 thread_wakeup((event_t) next);
4757                                 IOLockUnlock( gIOServiceBusyLock );
4758                         }
4759                         if (next != this) {
4760                                 next->unlockForArbitration();
4761                         }
4762
4763                         if ((wasQuiet || nowQuiet)) {
4764                                 uint64_t regID = next->getRegistryEntryID();
4765                                 IOServiceTrace(
4766                                         ((wasQuiet /*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
4767                                         (uintptr_t) regID,
4768                                         (uintptr_t) (regID >> 32),
4769                                         (uintptr_t) next,
4770                                         0);
4771
4772                                 if (wasQuiet) {
4773                                         next->__timeBusy = mach_absolute_time();
4774                                 } else {
4775                                         next->__accumBusy += mach_absolute_time() - next->__timeBusy;
4776                                         next->__timeBusy = 0;
4777                                 }
4778
4779                                 MessageClientsContext context;
4780
4781                                 context.service  = next;
4782                                 context.type     = kIOMessageServiceBusyStateChange;
4783                                 context.argument = (void *) wasQuiet; /*nowBusy*/
4784                                 context.argSize  = 0;
4785
4786                                 applyToInterestNotifiers( next, gIOBusyInterest,
4787                                     &messageClientsApplier, &context );
4788
4789 #if !NO_KEXTD
4790                                 if (nowQuiet && (next == gIOServiceRoot)) {
4791                                         OSKext::considerUnloads();
4792                                         IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
4793                                 }
4794 #endif
4795                         }
4796
4797                         delta = nowQuiet ? -1 : +1;
4798                 } while ((wasQuiet || nowQuiet) && (next = next->getProvider()));
4799         }
4800
4801         return result;
4802 }
4803
4804 void
4805 IOService::adjustBusy( SInt32 delta )
4806 {
4807         lockForArbitration();
4808         _adjustBusy( delta );
4809         unlockForArbitration();
4810 }
4811
4812 uint64_t
4813 IOService::getAccumulatedBusyTime( void )
4814 {
4815         uint64_t accumBusy = __accumBusy;
4816         uint64_t timeBusy = __timeBusy;
4817         uint64_t nano;
4818
4819         do{
4820                 accumBusy = __accumBusy;
4821                 timeBusy  = __timeBusy;
4822                 if (timeBusy) {
4823                         accumBusy += mach_absolute_time() - timeBusy;
4824                 }
4825         }while (timeBusy != __timeBusy);
4826
4827         absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
4828
4829         return nano;
4830 }
4831
4832 UInt32
4833 IOService::getBusyState( void )
4834 {
4835         return __state[1] & kIOServiceBusyStateMask;
4836 }
4837
4838 IOReturn
4839 IOService::waitForState( UInt32 mask, UInt32 value,
4840     mach_timespec_t * timeout )
4841 {
4842         panic("waitForState");
4843         return kIOReturnUnsupported;
4844 }
4845
4846 IOReturn
4847 IOService::waitForState( UInt32 mask, UInt32 value,
4848     uint64_t timeout )
4849 {
4850         bool            wait;
4851         int             waitResult = THREAD_AWAKENED;
4852         bool            computeDeadline = true;
4853         AbsoluteTime    abstime;
4854
4855         do {
4856                 lockForArbitration();
4857                 IOLockLock( gIOServiceBusyLock );
4858                 wait = (value != (__state[1] & mask));
4859                 if (wait) {
4860                         __state[1] |= kIOServiceBusyWaiterState;
4861                         unlockForArbitration();
4862                         if (timeout != UINT64_MAX) {
4863                                 if (computeDeadline) {
4864                                         AbsoluteTime  nsinterval;
4865                                         nanoseconds_to_absolutetime(timeout, &nsinterval );
4866                                         clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
4867                                         computeDeadline = false;
4868                                 }
4869                                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
4870                         } else {
4871                                 assert_wait((event_t)this, THREAD_UNINT );
4872                         }
4873                 } else {
4874                         unlockForArbitration();
4875                 }
4876                 IOLockUnlock( gIOServiceBusyLock );
4877                 if (wait) {
4878                         waitResult = thread_block(THREAD_CONTINUE_NULL);
4879                 }
4880         } while (wait && (waitResult != THREAD_TIMED_OUT));
4881
4882         if (waitResult == THREAD_TIMED_OUT) {
4883                 return kIOReturnTimeout;
4884         } else {
4885                 return kIOReturnSuccess;
4886         }
4887 }
4888
4889 IOReturn
4890 IOService::waitQuiet( uint64_t timeout )
4891 {
4892         IOReturn ret;
4893         uint32_t loops;
4894         char *   string = NULL;
4895         char *   panicString = NULL;
4896         size_t   len;
4897         size_t   panicStringLen;
4898         uint64_t time;
4899         uint64_t nano;
4900         bool     kextdWait;
4901         bool     dopanic;
4902
4903 #if KASAN
4904         /*
4905          * On kasan kernels, everything takes longer, so double the number of
4906          * timeout extensions. This should help with issues like 41259215
4907          * where WindowServer was timing out waiting for kextd to get all the
4908          * kasan kexts loaded and started.
4909          */
4910         enum { kTimeoutExtensions = 8 };
4911 #else
4912         enum { kTimeoutExtensions = 4 };
4913 #endif
4914
4915         time = mach_absolute_time();
4916         kextdWait = false;
4917         for (loops = 0; loops < kTimeoutExtensions; loops++) {
4918                 ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
4919
4920                 if (loops && (kIOReturnSuccess == ret)) {
4921                         time = mach_absolute_time() - time;
4922                         absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
4923                         IOLog("busy extended ok[%d], (%llds, %llds)\n",
4924                             loops, timeout / 1000000000ULL, nano / 1000000000ULL);
4925                         break;
4926                 } else if (kIOReturnTimeout != ret) {
4927                         break;
4928                 } else if (timeout < (4100ull * NSEC_PER_SEC)) {
4929                         break;
4930                 }
4931
4932                 {
4933                         IORegistryIterator * iter;
4934                         OSOrderedSet       * set;
4935                         OSOrderedSet       * leaves;
4936                         IOService          * next;
4937                         IOService          * nextParent;
4938                         char               * s;
4939                         size_t               l;
4940
4941                         len = 256;
4942                         panicStringLen = 256;
4943                         if (!string) {
4944                                 string      = IONew(char, len);
4945                         }
4946                         if (!panicString) {
4947                                 panicString = IONew(char, panicStringLen);
4948                         }
4949                         set = NULL;
4950                         kextdWait = OSKext::isWaitingKextd();
4951                         iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
4952                         leaves = OSOrderedSet::withCapacity(4);
4953                         if (iter) {
4954                                 set = iter->iterateAll();
4955                         }
4956                         if (string && panicString && leaves && set) {
4957                                 string[0] = panicString[0] = 0;
4958                                 set->setObject(this);
4959                                 while ((next = (IOService *) set->getLastObject())) {
4960                                         if (next->getBusyState()) {
4961                                                 if (kIOServiceModuleStallState & next->__state[1]) {
4962                                                         kextdWait = true;
4963                                                 }
4964                                                 leaves->setObject(next);
4965                                                 nextParent = next;
4966                                                 while ((nextParent = nextParent->getProvider())) {
4967                                                         set->removeObject(nextParent);
4968                                                         leaves->removeObject(nextParent);
4969                                                 }
4970                                         }
4971                                         set->removeObject(next);
4972                                 }
4973                                 s = string;
4974                                 while ((next = (IOService *) leaves->getLastObject())) {
4975                                         l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
4976                                         if (l >= len) {
4977                                                 break;
4978                                         }
4979                                         s += l;
4980                                         len -= l;
4981                                         leaves->removeObject(next);
4982                                 }
4983                         }
4984                         OSSafeReleaseNULL(leaves);
4985                         OSSafeReleaseNULL(set);
4986                         OSSafeReleaseNULL(iter);
4987                 }
4988
4989                 dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug));
4990                 snprintf(panicString, panicStringLen,
4991                     "%s[%d], (%llds): %s",
4992                     kextdWait ? "kextd stall" : "busy timeout",
4993                     loops, timeout / 1000000000ULL,
4994                     string ? string : "");
4995                 IOLog("%s\n", panicString);
4996                 if (dopanic) {
4997                         panic("%s", panicString);
4998                 } else if (!loops) {
4999                         getPMRootDomain()->startSpinDump(1);
5000                 }
5001         }
5002
5003         if (string) {
5004                 IODelete(string, char, 256);
5005         }
5006         if (panicString) {
5007                 IODelete(panicString, char, panicStringLen);
5008         }
5009
5010         return ret;
5011 }
5012
5013 IOReturn
5014 IOService::waitQuiet( mach_timespec_t * timeout )
5015 {
5016         uint64_t    timeoutNS;
5017
5018         if (timeout) {
5019                 timeoutNS = timeout->tv_sec;
5020                 timeoutNS *= kSecondScale;
5021                 timeoutNS += timeout->tv_nsec;
5022         } else {
5023                 timeoutNS = UINT64_MAX;
5024         }
5025
5026         return waitQuiet(timeoutNS);
5027 }
5028
5029 bool
5030 IOService::serializeProperties( OSSerialize * s ) const
5031 {
5032 #if 0
5033         ((IOService *)this)->setProperty(((IOService *)this)->__state,
5034             sizeof(__state), "__state");
5035 #endif
5036         return super::serializeProperties(s);
5037 }
5038
5039
5040 void
5041 _IOConfigThread::main(void * arg, wait_result_t result)
5042 {
5043         _IOConfigThread * self = (_IOConfigThread *) arg;
5044         _IOServiceJob * job;
5045         IOService   *   nub;
5046         bool            alive = true;
5047         kern_return_t   kr;
5048         thread_precedence_policy_data_t precedence = { -1 };
5049
5050         kr = thread_policy_set(current_thread(),
5051             THREAD_PRECEDENCE_POLICY,
5052             (thread_policy_t) &precedence,
5053             THREAD_PRECEDENCE_POLICY_COUNT);
5054         if (KERN_SUCCESS != kr) {
5055                 IOLog("thread_policy_set(%d)\n", kr);
5056         }
5057
5058         do {
5059 //      randomDelay();
5060
5061                 semaphore_wait( gJobsSemaphore );
5062
5063                 IOTakeLock( gJobsLock );
5064                 job = (_IOServiceJob *) gJobs->getFirstObject();
5065                 job->retain();
5066                 gJobs->removeObject(job);
5067                 if (job) {
5068                         gOutstandingJobs--;
5069 //          gNumConfigThreads--;        // we're out of service
5070                         gNumWaitingThreads--; // we're out of service
5071                 }
5072                 IOUnlock( gJobsLock );
5073
5074                 if (job) {
5075                         nub = job->nub;
5076
5077                         if (gIOKitDebug & kIOLogConfig) {
5078                                 LOG("config(%p): starting on %s, %d\n",
5079                                     IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
5080                         }
5081
5082                         switch (job->type) {
5083                         case kMatchNubJob:
5084                                 nub->doServiceMatch( job->options );
5085                                 break;
5086
5087                         default:
5088                                 LOG("config(%p): strange type (%d)\n",
5089                                     IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
5090                                 break;
5091                         }
5092
5093                         nub->release();
5094                         job->release();
5095
5096                         IOTakeLock( gJobsLock );
5097                         alive = (gOutstandingJobs > gNumWaitingThreads);
5098                         if (alive) {
5099                                 gNumWaitingThreads++; // back in service
5100                         }
5101 //              gNumConfigThreads++;
5102                         else {
5103                                 if (0 == --gNumConfigThreads) {
5104 //                    IOLog("MATCH IDLE\n");
5105                                         IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
5106                                 }
5107                         }
5108                         IOUnlock( gJobsLock );
5109                 }
5110         } while (alive);
5111
5112         if (gIOKitDebug & kIOLogConfig) {
5113                 LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5114         }
5115
5116         self->release();
5117 }
5118
5119 IOReturn
5120 IOService::waitMatchIdle( UInt32 msToWait )
5121 {
5122         bool            wait;
5123         int             waitResult = THREAD_AWAKENED;
5124         bool            computeDeadline = true;
5125         AbsoluteTime    deadline;
5126
5127         IOLockLock( gJobsLock );
5128         do {
5129                 wait = (0 != gNumConfigThreads);
5130                 if (wait) {
5131                         if (msToWait) {
5132                                 if (computeDeadline) {
5133                                         clock_interval_to_deadline(
5134                                                 msToWait, kMillisecondScale, &deadline );
5135                                         computeDeadline = false;
5136                                 }
5137                                 waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
5138                                     deadline, THREAD_UNINT );
5139                         } else {
5140                                 waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
5141                                     THREAD_UNINT );
5142                         }
5143                 }
5144         } while (wait && (waitResult != THREAD_TIMED_OUT));
5145         IOLockUnlock( gJobsLock );
5146
5147         if (waitResult == THREAD_TIMED_OUT) {
5148                 return kIOReturnTimeout;
5149         } else {
5150                 return kIOReturnSuccess;
5151         }
5152 }
5153
5154 void
5155 IOService::cpusRunning(void)
5156 {
5157         gCPUsRunning = true;
5158 }
5159
5160 void
5161 _IOServiceJob::pingConfig( _IOServiceJob * job )
5162 {
5163         int         count;
5164         bool        create;
5165
5166         assert( job );
5167
5168         IOTakeLock( gJobsLock );
5169
5170         gOutstandingJobs++;
5171         gJobs->setLastObject( job );
5172
5173         count = gNumWaitingThreads;
5174 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
5175
5176         create = ((gOutstandingJobs > count)
5177             && ((gNumConfigThreads < kMaxConfigThreads)
5178             || (job->nub == gIOResources)
5179             || !gCPUsRunning));
5180         if (create) {
5181                 gNumConfigThreads++;
5182                 gNumWaitingThreads++;
5183         }
5184
5185         IOUnlock( gJobsLock );
5186
5187         job->release();
5188
5189         if (create) {
5190                 if (gIOKitDebug & kIOLogConfig) {
5191                         LOG("config(%d): creating\n", gNumConfigThreads - 1);
5192                 }
5193                 _IOConfigThread::configThread(gNumConfigThreads - 1);
5194         }
5195
5196         semaphore_signal( gJobsSemaphore );
5197 }
5198
5199 struct IOServiceMatchContext {
5200         OSDictionary * table;
5201         OSObject *     result;
5202         uint32_t       options;
5203         uint32_t       state;
5204         uint32_t       count;
5205         uint32_t       done;
5206 };
5207
5208 bool
5209 IOService::instanceMatch(const OSObject * entry, void * context)
5210 {
5211         IOServiceMatchContext * ctx = (typeof(ctx))context;
5212         IOService *    service = (typeof(service))entry;
5213         OSDictionary * table   = ctx->table;
5214         uint32_t       options = ctx->options;
5215         uint32_t       state   = ctx->state;
5216         uint32_t       done;
5217         bool           match;
5218
5219         done = 0;
5220         do{
5221                 match = ((state == (state & service->__state[0]))
5222                     && (0 == (service->__state[0] & kIOServiceInactiveState)));
5223                 if (!match) {
5224                         break;
5225                 }
5226                 ctx->count += table->getCount();
5227                 match = service->matchInternal(table, options, &done);
5228                 ctx->done += done;
5229         }while (false);
5230         if (!match) {
5231                 return false;
5232         }
5233
5234         if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) {
5235                 service->retain();
5236                 ctx->result = service;
5237                 return true;
5238         } else if (!ctx->result) {
5239                 ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
5240         } else {
5241                 ((OSSet *)ctx->result)->setObject(service);
5242         }
5243         return false;
5244 }
5245
5246 // internal - call with gNotificationLock
5247 OSObject *
5248 IOService::copyExistingServices( OSDictionary * matching,
5249     IOOptionBits inState, IOOptionBits options )
5250 {
5251         OSObject *   current = NULL;
5252         OSIterator * iter;
5253         IOService *  service;
5254         OSObject *   obj;
5255         OSString *   str;
5256
5257         if (!matching) {
5258                 return NULL;
5259         }
5260
5261 #if MATCH_DEBUG
5262         OSSerialize * s = OSSerialize::withCapacity(128);
5263         matching->serialize(s);
5264 #endif
5265
5266         if ((obj = matching->getObject(gIOProviderClassKey))
5267             && gIOResourcesKey
5268             && gIOResourcesKey->isEqualTo(obj)
5269             && (service = gIOResources)) {
5270                 if ((inState == (service->__state[0] & inState))
5271                     && (0 == (service->__state[0] & kIOServiceInactiveState))
5272                     && service->matchPassive(matching, options)) {
5273                         if (options & kIONotifyOnce) {
5274                                 service->retain();
5275                                 current = service;
5276                         } else {
5277                                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
5278                         }
5279                 }
5280         } else {
5281                 IOServiceMatchContext ctx;
5282                 ctx.table   = matching;
5283                 ctx.state   = inState;
5284                 ctx.count   = 0;
5285                 ctx.done    = 0;
5286                 ctx.options = options;
5287                 ctx.result  = NULL;
5288
5289                 if ((str = OSDynamicCast(OSString, obj))) {
5290                         const OSSymbol * sym = OSSymbol::withString(str);
5291                         OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
5292                         sym->release();
5293                 } else {
5294                         IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
5295                 }
5296
5297
5298                 current = ctx.result;
5299
5300                 options |= kIOServiceInternalDone | kIOServiceClassDone;
5301                 if (current && (ctx.done != ctx.count)) {
5302                         OSSet *
5303                             source = OSDynamicCast(OSSet, current);
5304                         current = NULL;
5305                         while ((service = (IOService *) source->getAnyObject())) {
5306                                 if (service->matchPassive(matching, options)) {
5307                                         if (options & kIONotifyOnce) {
5308                                                 service->retain();
5309                                                 current = service;
5310                                                 break;
5311                                         }
5312                                         if (current) {
5313                                                 ((OSSet *)current)->setObject( service );
5314                                         } else {
5315                                                 current = OSSet::withObjects(
5316                                                         (const OSObject **) &service, 1, 1 );
5317                                         }
5318                                 }
5319                                 source->removeObject(service);
5320                         }
5321                         source->release();
5322                 }
5323         }
5324
5325 #if MATCH_DEBUG
5326         {
5327                 OSObject * _current = 0;
5328
5329                 iter = IORegistryIterator::iterateOver( gIOServicePlane,
5330                     kIORegistryIterateRecursively );
5331                 if (iter) {
5332                         do {
5333                                 iter->reset();
5334                                 while ((service = (IOService *) iter->getNextObject())) {
5335                                         if ((inState == (service->__state[0] & inState))
5336                                             && (0 == (service->__state[0] & kIOServiceInactiveState))
5337                                             && service->matchPassive(matching, 0)) {
5338                                                 if (options & kIONotifyOnce) {
5339                                                         service->retain();
5340                                                         _current = service;
5341                                                         break;
5342                                                 }
5343                                                 if (_current) {
5344                                                         ((OSSet *)_current)->setObject( service );
5345                                                 } else {
5346                                                         _current = OSSet::withObjects(
5347                                                                 (const OSObject **) &service, 1, 1 );
5348                                                 }
5349                                         }
5350                                 }
5351                         } while (!service && !iter->isValid());
5352                         iter->release();
5353                 }
5354
5355                 if (((current != 0) != (_current != 0))
5356                     || (current && _current && !current->isEqualTo(_current))) {
5357                         OSSerialize * s1 = OSSerialize::withCapacity(128);
5358                         OSSerialize * s2 = OSSerialize::withCapacity(128);
5359                         current->serialize(s1);
5360                         _current->serialize(s2);
5361                         kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
5362                             IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
5363                         s1->release();
5364                         s2->release();
5365                 }
5366
5367                 if (_current) {
5368                         _current->release();
5369                 }
5370         }
5371
5372         s->release();
5373 #endif
5374
5375         if (current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
5376                 iter = OSCollectionIterator::withCollection((OSSet *)current );
5377                 current->release();
5378                 current = iter;
5379         }
5380
5381         return current;
5382 }
5383
5384 // public version
5385 OSIterator *
5386 IOService::getMatchingServices( OSDictionary * matching )
5387 {
5388         OSIterator *        iter;
5389
5390         // is a lock even needed?
5391         LOCKWRITENOTIFY();
5392
5393         iter = (OSIterator *) copyExistingServices( matching,
5394             kIOServiceMatchedState );
5395
5396         UNLOCKNOTIFY();
5397
5398         return iter;
5399 }
5400
5401 IOService *
5402 IOService::copyMatchingService( OSDictionary * matching )
5403 {
5404         IOService * service;
5405
5406         // is a lock even needed?
5407         LOCKWRITENOTIFY();
5408
5409         service = (IOService *) copyExistingServices( matching,
5410             kIOServiceMatchedState, kIONotifyOnce );
5411
5412         UNLOCKNOTIFY();
5413
5414         return service;
5415 }
5416
5417 struct _IOServiceMatchingNotificationHandlerRef {
5418         IOServiceNotificationHandler handler;
5419         void * ref;
5420 };
5421
5422 static bool
5423 _IOServiceMatchingNotificationHandler( void * target, void * refCon,
5424     IOService * newService,
5425     IONotifier * notifier )
5426 {
5427         return (*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService);
5428 }
5429
5430 // internal - call with gNotificationLock
5431 IONotifier *
5432 IOService::setNotification(
5433         const OSSymbol * type, OSDictionary * matching,
5434         IOServiceMatchingNotificationHandler handler, void * target, void * ref,
5435         SInt32 priority )
5436 {
5437         _IOServiceNotifier * notify = NULL;
5438         OSOrderedSet *      set;
5439
5440         if (!matching) {
5441                 return NULL;
5442         }
5443
5444         notify = new _IOServiceNotifier;
5445         if (notify && !notify->init()) {
5446                 notify->release();
5447                 notify = NULL;
5448         }
5449
5450         if (notify) {
5451                 notify->handler = handler;
5452                 notify->target = target;
5453                 notify->type = type;
5454                 notify->matching = matching;
5455                 matching->retain();
5456                 if (handler == &_IOServiceMatchingNotificationHandler) {
5457                         notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
5458                         notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
5459                 } else {
5460                         notify->ref = ref;
5461                 }
5462                 notify->priority = priority;
5463                 notify->state = kIOServiceNotifyEnable;
5464                 queue_init( &notify->handlerInvocations );
5465
5466                 ////// queue
5467
5468                 if (NULL == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
5469                         set = OSOrderedSet::withCapacity( 1,
5470                             IONotifyOrdering, NULL );
5471                         if (set) {
5472                                 gNotifications->setObject( type, set );
5473                                 set->release();
5474                         }
5475                 }
5476                 notify->whence = set;
5477                 if (set) {
5478                         set->setObject( notify );
5479                 }
5480         }
5481
5482         return notify;
5483 }
5484
5485 // internal - call with gNotificationLock
5486 IONotifier *
5487 IOService::doInstallNotification(
5488         const OSSymbol * type, OSDictionary * matching,
5489         IOServiceMatchingNotificationHandler handler,
5490         void * target, void * ref,
5491         SInt32 priority, OSIterator ** existing )
5492 {
5493         OSIterator *        exist;
5494         IONotifier *        notify;
5495         IOOptionBits        inState;
5496
5497         if (!matching) {
5498                 return NULL;
5499         }
5500
5501         if (type == gIOPublishNotification) {
5502                 inState = kIOServiceRegisteredState;
5503         } else if (type == gIOFirstPublishNotification) {
5504                 inState = kIOServiceFirstPublishState;
5505         } else if (type == gIOMatchedNotification) {
5506                 inState = kIOServiceMatchedState;
5507         } else if (type == gIOFirstMatchNotification) {
5508                 inState = kIOServiceFirstMatchState;
5509         } else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) {
5510                 inState = 0;
5511         } else {
5512                 return NULL;
5513         }
5514
5515         notify = setNotification( type, matching, handler, target, ref, priority );
5516
5517         if (inState) {
5518                 // get the current set
5519                 exist = (OSIterator *) copyExistingServices( matching, inState );
5520         } else {
5521                 exist = NULL;
5522         }
5523
5524         *existing = exist;
5525
5526         return notify;
5527 }
5528
5529 #if !defined(__LP64__)
5530 IONotifier *
5531 IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
5532     IOServiceNotificationHandler handler,
5533     void * target, void * refCon,
5534     SInt32 priority, OSIterator ** existing )
5535 {
5536         IONotifier * result;
5537         _IOServiceMatchingNotificationHandlerRef ref;
5538         ref.handler = handler;
5539         ref.ref     = refCon;
5540
5541         result = (_IOServiceNotifier *) installNotification( type, matching,
5542             &_IOServiceMatchingNotificationHandler,
5543             target, &ref, priority, existing );
5544         if (result) {
5545                 matching->release();
5546         }
5547
5548         return result;
5549 }
5550 #endif /* !defined(__LP64__) */
5551
5552
5553 IONotifier *
5554 IOService::installNotification(
5555         const OSSymbol * type, OSDictionary * matching,
5556         IOServiceMatchingNotificationHandler handler,
5557         void * target, void * ref,
5558         SInt32 priority, OSIterator ** existing )
5559 {
5560         IONotifier * notify;
5561
5562         LOCKWRITENOTIFY();
5563
5564         notify = doInstallNotification( type, matching, handler, target, ref,
5565             priority, existing );
5566
5567         // in case handler remove()s
5568         if (notify) {
5569                 notify->retain();
5570         }
5571
5572         UNLOCKNOTIFY();
5573
5574         return notify;
5575 }
5576
5577 IONotifier *
5578 IOService::addNotification(
5579         const OSSymbol * type, OSDictionary * matching,
5580         IOServiceNotificationHandler handler,
5581         void * target, void * refCon,
5582         SInt32 priority )
5583 {
5584         IONotifier * result;
5585         _IOServiceMatchingNotificationHandlerRef ref;
5586
5587         ref.handler = handler;
5588         ref.ref     = refCon;
5589
5590         result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
5591             target, &ref, priority);
5592
5593         if (result) {
5594                 matching->release();
5595         }
5596
5597         return result;
5598 }
5599
5600 IONotifier *
5601 IOService::addMatchingNotification(
5602         const OSSymbol * type, OSDictionary * matching,
5603         IOServiceMatchingNotificationHandler handler,
5604         void * target, void * ref,
5605         SInt32 priority )
5606 {
5607         OSIterator *                existing = NULL;
5608         IONotifier *                ret;
5609         _IOServiceNotifier *        notify;
5610         IOService *                 next;
5611
5612         ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
5613             handler, target, ref, priority, &existing );
5614         if (!ret) {
5615                 return NULL;
5616         }
5617
5618         // send notifications for existing set
5619         if (existing) {
5620                 while ((next = (IOService *) existing->getNextObject())) {
5621                         if (0 == (next->__state[0] & kIOServiceInactiveState)) {
5622                                 next->invokeNotifier( notify );
5623                         }
5624                 }
5625                 existing->release();
5626         }
5627
5628         LOCKWRITENOTIFY();
5629         bool removed = (NULL == notify->whence);
5630         notify->release();
5631         if (removed) {
5632                 ret = gIOServiceNullNotifier;
5633         }
5634         UNLOCKNOTIFY();
5635
5636         return ret;
5637 }
5638
5639 static bool
5640 IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
5641     IOService * newService,
5642     IONotifier * notifier )
5643 {
5644         return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
5645 }
5646
5647 IONotifier *
5648 IOService::addMatchingNotification(
5649         const OSSymbol * type, OSDictionary * matching,
5650         SInt32 priority,
5651         IOServiceMatchingNotificationHandlerBlock handler)
5652 {
5653         IONotifier * notify;
5654         void       * block;
5655
5656         block = Block_copy(handler);
5657         if (!block) {
5658                 return NULL;
5659         }
5660
5661         notify = addMatchingNotification(type, matching,
5662             &IOServiceMatchingNotificationHandlerToBlock, NULL, block, priority);
5663
5664         if (!notify) {
5665                 Block_release(block);
5666         }
5667
5668         return notify;
5669 }
5670
5671
5672 bool
5673 IOService::syncNotificationHandler(
5674         void * /* target */, void * ref,
5675         IOService * newService,
5676         IONotifier * notifier )
5677 {
5678         LOCKWRITENOTIFY();
5679         if (!*((IOService **) ref)) {
5680                 newService->retain();
5681                 (*(IOService **) ref) = newService;
5682                 WAKEUPNOTIFY(ref);
5683         }
5684         UNLOCKNOTIFY();
5685
5686         return false;
5687 }
5688
5689 IOService *
5690 IOService::waitForMatchingService( OSDictionary * matching,
5691     uint64_t timeout)
5692 {
5693         IONotifier *        notify = NULL;
5694         // priority doesn't help us much since we need a thread wakeup
5695         SInt32              priority = 0;
5696         IOService *         result;
5697
5698         if (!matching) {
5699                 return NULL;
5700         }
5701
5702         result = NULL;
5703
5704         LOCKWRITENOTIFY();
5705         do{
5706                 result = (IOService *) copyExistingServices( matching,
5707                     kIOServiceMatchedState, kIONotifyOnce );
5708                 if (result) {
5709                         break;
5710                 }
5711                 notify = IOService::setNotification( gIOMatchedNotification, matching,
5712                     &IOService::syncNotificationHandler, (void *) NULL,
5713                     &result, priority );
5714                 if (!notify) {
5715                         break;
5716                 }
5717                 if (UINT64_MAX != timeout) {
5718                         AbsoluteTime deadline;
5719                         nanoseconds_to_absolutetime(timeout, &deadline);
5720                         clock_absolutetime_interval_to_deadline(deadline, &deadline);
5721                         SLEEPNOTIFYTO(&result, deadline);
5722                 } else {
5723                         SLEEPNOTIFY(&result);
5724                 }
5725         }while (false);
5726
5727         UNLOCKNOTIFY();
5728
5729         if (notify) {
5730                 notify->remove(); // dequeues
5731         }
5732         return result;
5733 }
5734
5735 IOService *
5736 IOService::waitForService( OSDictionary * matching,
5737     mach_timespec_t * timeout )
5738 {
5739         IOService * result;
5740         uint64_t    timeoutNS;
5741
5742         if (timeout) {
5743                 timeoutNS = timeout->tv_sec;
5744                 timeoutNS *= kSecondScale;
5745                 timeoutNS += timeout->tv_nsec;
5746         } else {
5747                 timeoutNS = UINT64_MAX;
5748         }
5749
5750         result = waitForMatchingService(matching, timeoutNS);
5751
5752         matching->release();
5753         if (result) {
5754                 result->release();
5755         }
5756
5757         return result;
5758 }
5759
5760 __dead2
5761 void
5762 IOService::deliverNotification( const OSSymbol * type,
5763     IOOptionBits orNewState, IOOptionBits andNewState )
5764 {
5765         panic("deliverNotification");
5766 }
5767
5768 OSArray *
5769 IOService::copyNotifiers(const OSSymbol * type,
5770     IOOptionBits orNewState, IOOptionBits andNewState )
5771 {
5772         _IOServiceNotifier * notify;
5773         OSIterator *         iter;
5774         OSArray *            willSend = NULL;
5775
5776         lockForArbitration();
5777
5778         if ((0 == (__state[0] & kIOServiceInactiveState))
5779             || (type == gIOTerminatedNotification)
5780             || (type == gIOWillTerminateNotification)) {
5781                 LOCKREADNOTIFY();
5782
5783                 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
5784                     gNotifications->getObject( type ));
5785
5786                 if (iter) {
5787                         while ((notify = (_IOServiceNotifier *) iter->getNextObject())) {
5788                                 if (matchPassive(notify->matching, 0)
5789                                     && (kIOServiceNotifyEnable & notify->state)) {
5790                                         if (NULL == willSend) {
5791                                                 willSend = OSArray::withCapacity(8);
5792                                         }
5793                                         if (willSend) {
5794                                                 willSend->setObject( notify );
5795                                         }
5796                                 }
5797                         }
5798                         iter->release();
5799                 }
5800                 __state[0] = (__state[0] | orNewState) & andNewState;
5801                 UNLOCKNOTIFY();
5802         }
5803
5804         unlockForArbitration();
5805
5806         return willSend;
5807 }
5808
5809 IOOptionBits
5810 IOService::getState( void ) const
5811 {
5812         return __state[0];
5813 }
5814
5815 /*
5816  * Helpers to make matching objects for simple cases
5817  */
5818
5819 OSDictionary *
5820 IOService::serviceMatching( const OSString * name,
5821     OSDictionary * table )
5822 {
5823         const OSString *    str;
5824
5825         str = OSSymbol::withString(name);
5826         if (!str) {
5827                 return NULL;
5828         }
5829
5830         if (!table) {
5831                 table = OSDictionary::withCapacity( 2 );
5832         }
5833         if (table) {
5834                 table->setObject(gIOProviderClassKey, (OSObject *)str );
5835         }
5836         str->release();
5837
5838         return table;
5839 }
5840
5841 OSDictionary *
5842 IOService::serviceMatching( const char * name,
5843     OSDictionary * table )
5844 {
5845         const OSString *    str;
5846
5847         str = OSSymbol::withCString( name );
5848         if (!str) {
5849                 return NULL;
5850         }
5851
5852         table = serviceMatching( str, table );
5853         str->release();
5854         return table;
5855 }
5856
5857 OSDictionary *
5858 IOService::nameMatching( const OSString * name,
5859     OSDictionary * table )
5860 {
5861         if (!table) {
5862                 table = OSDictionary::withCapacity( 2 );
5863         }
5864         if (table) {
5865                 table->setObject( gIONameMatchKey, (OSObject *)name );
5866         }
5867
5868         return table;
5869 }
5870
5871 OSDictionary *
5872 IOService::nameMatching( const char * name,
5873     OSDictionary * table )
5874 {
5875         const OSString *    str;
5876
5877         str = OSSymbol::withCString( name );
5878         if (!str) {
5879                 return NULL;
5880         }
5881
5882         table = nameMatching( str, table );
5883         str->release();
5884         return table;
5885 }
5886
5887 OSDictionary *
5888 IOService::resourceMatching( const OSString * str,
5889     OSDictionary * table )
5890 {
5891         table = serviceMatching( gIOResourcesKey, table );
5892         if (table) {
5893                 table->setObject( gIOResourceMatchKey, (OSObject *) str );
5894         }
5895
5896         return table;
5897 }
5898
5899 OSDictionary *
5900 IOService::resourceMatching( const char * name,
5901     OSDictionary * table )
5902 {
5903         const OSSymbol *    str;
5904
5905         str = OSSymbol::withCString( name );
5906         if (!str) {
5907                 return NULL;
5908         }
5909
5910         table = resourceMatching( str, table );
5911         str->release();
5912
5913         return table;
5914 }
5915
5916 OSDictionary *
5917 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
5918     OSDictionary * table )
5919 {
5920         OSDictionary * properties;
5921
5922         properties = OSDictionary::withCapacity( 2 );
5923         if (!properties) {
5924                 return NULL;
5925         }
5926         properties->setObject( key, value );
5927
5928         if (!table) {
5929                 table = OSDictionary::withCapacity( 2 );
5930         }
5931         if (table) {
5932                 table->setObject( gIOPropertyMatchKey, properties );
5933         }
5934
5935         properties->release();
5936
5937         return table;
5938 }
5939
5940 OSDictionary *
5941 IOService::registryEntryIDMatching( uint64_t entryID,
5942     OSDictionary * table )
5943 {
5944         OSNumber *     num;
5945
5946         num = OSNumber::withNumber( entryID, 64 );
5947         if (!num) {
5948                 return NULL;
5949         }
5950
5951         if (!table) {
5952                 table = OSDictionary::withCapacity( 2 );
5953         }
5954         if (table) {
5955                 table->setObject( gIORegistryEntryIDKey, num );
5956         }
5957
5958         if (num) {
5959                 num->release();
5960         }
5961
5962         return table;
5963 }
5964
5965
5966 /*
5967  * _IOServiceNotifier
5968  */
5969
5970 // wait for all threads, other than the current one,
5971 //  to exit the handler
5972
5973 void
5974 _IOServiceNotifier::wait()
5975 {
5976         _IOServiceNotifierInvocation * next;
5977         bool doWait;
5978
5979         do {
5980                 doWait = false;
5981                 queue_iterate( &handlerInvocations, next,
5982                     _IOServiceNotifierInvocation *, link) {
5983                         if (next->thread != current_thread()) {
5984                                 doWait = true;
5985                                 break;
5986                         }
5987                 }
5988                 if (doWait) {
5989                         state |= kIOServiceNotifyWaiter;
5990                         SLEEPNOTIFY(this);
5991                 }
5992         } while (doWait);
5993 }
5994
5995 void
5996 _IOServiceNotifier::free()
5997 {
5998         assert( queue_empty( &handlerInvocations ));
5999
6000         if (handler == &IOServiceMatchingNotificationHandlerToBlock) {
6001                 Block_release(ref);
6002         }
6003
6004         OSObject::free();
6005 }
6006
6007 void
6008 _IOServiceNotifier::remove()
6009 {
6010         LOCKWRITENOTIFY();
6011
6012         if (whence) {
6013                 whence->removeObject((OSObject *) this );
6014                 whence = NULL;
6015         }
6016         if (matching) {
6017                 matching->release();
6018                 matching = NULL;
6019         }
6020
6021         state &= ~kIOServiceNotifyEnable;
6022
6023         wait();
6024
6025         UNLOCKNOTIFY();
6026
6027         release();
6028 }
6029
6030 bool
6031 _IOServiceNotifier::disable()
6032 {
6033         bool        ret;
6034
6035         LOCKWRITENOTIFY();
6036
6037         ret = (0 != (kIOServiceNotifyEnable & state));
6038         state &= ~kIOServiceNotifyEnable;
6039         if (ret) {
6040                 wait();
6041         }
6042
6043         UNLOCKNOTIFY();
6044
6045         return ret;
6046 }
6047
6048 void
6049 _IOServiceNotifier::enable( bool was )
6050 {
6051         LOCKWRITENOTIFY();
6052         if (was) {
6053                 state |= kIOServiceNotifyEnable;
6054         } else {
6055                 state &= ~kIOServiceNotifyEnable;
6056         }
6057         UNLOCKNOTIFY();
6058 }
6059
6060
6061 /*
6062  * _IOServiceNullNotifier
6063  */
6064
6065 void
6066 _IOServiceNullNotifier::taggedRetain(const void *tag) const
6067 {
6068 }
6069 void
6070 _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const
6071 {
6072 }
6073 void
6074 _IOServiceNullNotifier::free()
6075 {
6076 }
6077 void
6078 _IOServiceNullNotifier::wait()
6079 {
6080 }
6081 void
6082 _IOServiceNullNotifier::remove()
6083 {
6084 }
6085 void
6086 _IOServiceNullNotifier::enable(bool was)
6087 {
6088 }
6089 bool
6090 _IOServiceNullNotifier::disable()
6091 {
6092         return false;
6093 }
6094
6095 /*
6096  * IOResources
6097  */
6098
6099 IOService *
6100 IOResources::resources( void )
6101 {
6102         IOResources *       inst;
6103
6104         inst = new IOResources;
6105         if (inst && !inst->init()) {
6106                 inst->release();
6107                 inst = NULL;
6108         }
6109
6110         return inst;
6111 }
6112
6113 bool
6114 IOResources::init( OSDictionary * dictionary )
6115 {
6116         // Do super init first
6117         if (!IOService::init()) {
6118                 return false;
6119         }
6120
6121         // Allow PAL layer to publish a value
6122         const char *property_name;
6123         int property_value;
6124
6125         pal_get_resource_property( &property_name, &property_value );
6126
6127         if (property_name) {
6128                 OSNumber *num;
6129                 const OSSymbol *        sym;
6130
6131                 if ((num = OSNumber::withNumber(property_value, 32)) != NULL) {
6132                         if ((sym = OSSymbol::withCString( property_name)) != NULL) {
6133                                 this->setProperty( sym, num );
6134                                 sym->release();
6135                         }
6136                         num->release();
6137                 }
6138         }
6139
6140         return true;
6141 }
6142
6143 IOReturn
6144 IOResources::newUserClient(task_t owningTask, void * securityID,
6145     UInt32 type, OSDictionary * properties,
6146     IOUserClient ** handler)
6147 {
6148         return kIOReturnUnsupported;
6149 }
6150
6151 IOWorkLoop *
6152 IOResources::getWorkLoop() const
6153 {
6154         // If we are the resource root
6155         // then use the platform's workloop
6156         if (this == (IOResources *) gIOResources) {
6157                 return getPlatform()->getWorkLoop();
6158         } else {
6159                 return IOService::getWorkLoop();
6160         }
6161 }
6162
6163 static bool
6164 IOResourcesMatchPropertyTable(IOService * resources, OSDictionary * table)
6165 {
6166         OSObject *          prop;
6167         OSString *          str;
6168         OSSet *             set;
6169         OSIterator *        iter;
6170         OSObject *          obj;
6171         OSArray *           keys;
6172         bool                ok = true;
6173
6174         prop = table->getObject( gIOResourceMatchKey );
6175         str = OSDynamicCast( OSString, prop );
6176         if (str) {
6177                 ok = (NULL != resources->getProperty( str ));
6178         } else if ((set = OSDynamicCast( OSSet, prop))) {
6179                 iter = OSCollectionIterator::withCollection( set );
6180                 ok = (iter != NULL);
6181                 while (ok && (str = OSDynamicCast( OSString, iter->getNextObject()))) {
6182                         ok = (NULL != resources->getProperty( str ));
6183                 }
6184
6185                 if (iter) {
6186                         iter->release();
6187                 }
6188         } else if ((prop = table->getObject(gIOResourceMatchedKey))) {
6189                 obj = resources->copyProperty(gIOResourceMatchedKey);
6190                 keys = OSDynamicCast(OSArray, obj);
6191                 ok = false;
6192                 if (keys) {
6193                         // assuming OSSymbol
6194                         ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
6195                 }
6196                 OSSafeReleaseNULL(obj);
6197         }
6198
6199         return ok;
6200 }
6201
6202 bool
6203 IOResources::matchPropertyTable( OSDictionary * table )
6204 {
6205         return IOResourcesMatchPropertyTable(this, table);
6206 }
6207
6208 /*
6209  * IOUserResources
6210  */
6211
6212 IOService *
6213 IOUserResources::resources( void )
6214 {
6215         IOUserResources *       inst;
6216
6217         inst = OSTypeAlloc(IOUserResources);
6218         if (inst && !inst->init()) {
6219                 inst->release();
6220                 inst = NULL;
6221         }
6222
6223         return inst;
6224 }
6225
6226 bool
6227 IOUserResources::init( OSDictionary * dictionary )
6228 {
6229         // Do super init first
6230         if (!IOService::init()) {
6231                 return false;
6232         }
6233         return true;
6234 }
6235
6236 IOReturn
6237 IOUserResources::newUserClient(task_t owningTask, void * securityID,
6238     UInt32 type, OSDictionary * properties,
6239     IOUserClient ** handler)
6240 {
6241         return kIOReturnUnsupported;
6242 }
6243
6244 IOWorkLoop *
6245 IOUserResources::getWorkLoop() const
6246 {
6247         return getPlatform()->getWorkLoop();
6248 }
6249
6250 bool
6251 IOUserResources::matchPropertyTable( OSDictionary * table )
6252 {
6253         return IOResourcesMatchPropertyTable(this, table);
6254 }
6255
6256 // --
6257
6258 void
6259 IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
6260 {
6261         IOService::updateConsoleUsers(NULL, 0);
6262 }
6263
6264 void
6265 IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
6266 {
6267         IORegistryEntry * regEntry;
6268         OSObject *        locked = kOSBooleanFalse;
6269         uint32_t          idx;
6270         bool              publish;
6271         OSDictionary *    user;
6272         static IOMessage  sSystemPower;
6273         clock_sec_t       now = 0;
6274         clock_usec_t      microsecs;
6275
6276         regEntry = IORegistryEntry::getRegistryRoot();
6277
6278         if (!gIOChosenEntry) {
6279                 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
6280         }
6281
6282         IOLockLock(gIOConsoleUsersLock);
6283
6284         if (systemMessage) {
6285                 sSystemPower = systemMessage;
6286 #if HIBERNATION
6287                 if (kIOMessageSystemHasPoweredOn == systemMessage) {
6288                         uint32_t lockState = IOHibernateWasScreenLocked();
6289                         switch (lockState) {
6290                         case 0:
6291                                 break;
6292                         case kIOScreenLockLocked:
6293                         case kIOScreenLockFileVaultDialog:
6294                                 gIOConsoleBooterLockState = kOSBooleanTrue;
6295                                 break;
6296                         case kIOScreenLockNoLock:
6297                                 gIOConsoleBooterLockState = NULL;
6298                                 break;
6299                         case kIOScreenLockUnlocked:
6300                         default:
6301                                 gIOConsoleBooterLockState = kOSBooleanFalse;
6302                                 break;
6303                         }
6304                 }
6305 #endif /* HIBERNATION */
6306         }
6307
6308         if (consoleUsers) {
6309                 OSNumber * num = NULL;
6310                 bool       loginLocked = true;
6311
6312                 gIOConsoleLoggedIn = false;
6313                 for (idx = 0;
6314                     (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
6315                     idx++) {
6316                         gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
6317                             && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
6318
6319                         loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
6320                         if (!num) {
6321                                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
6322                         }
6323                 }
6324 #if HIBERNATION
6325                 if (!loginLocked) {
6326                         gIOConsoleBooterLockState = NULL;
6327                 }
6328                 IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
6329                     (num != NULL), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
6330                     gIOConsoleLoggedIn, loginLocked);
6331 #endif /* HIBERNATION */
6332                 gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
6333         }
6334
6335         if (!gIOConsoleLoggedIn
6336             || (kIOMessageSystemWillSleep == sSystemPower)
6337             || (kIOMessageSystemPagingOff == sSystemPower)) {
6338                 locked = kOSBooleanTrue;
6339         }
6340 #if HIBERNATION
6341         else if (gIOConsoleBooterLockState) {
6342                 locked = gIOConsoleBooterLockState;
6343         }
6344 #endif /* HIBERNATION */
6345         else if (gIOConsoleLockTime) {
6346                 clock_get_calendar_microtime(&now, &microsecs);
6347                 if (gIOConsoleLockTime > now) {
6348                         AbsoluteTime deadline;
6349                         clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
6350                         thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
6351                 } else {
6352                         locked = kOSBooleanTrue;
6353                 }
6354         }
6355
6356         publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
6357         if (publish) {
6358                 regEntry->setProperty(gIOConsoleLockedKey, locked);
6359                 if (consoleUsers) {
6360                         regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
6361                 }
6362                 OSIncrementAtomic( &gIOConsoleUsersSeed );
6363         }
6364
6365 #if HIBERNATION
6366         if (gIOChosenEntry) {
6367                 if (locked == kOSBooleanTrue) {
6368                         gIOScreenLockState = kIOScreenLockLocked;
6369                 } else if (gIOConsoleLockTime) {
6370                         gIOScreenLockState = kIOScreenLockUnlocked;
6371                 } else {
6372                         gIOScreenLockState = kIOScreenLockNoLock;
6373                 }
6374                 gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
6375
6376                 IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
6377                     gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != NULL), now, systemMessage);
6378         }
6379 #endif /* HIBERNATION */
6380
6381         IOLockUnlock(gIOConsoleUsersLock);
6382
6383         if (publish) {
6384                 publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
6385
6386                 MessageClientsContext context;
6387
6388                 context.service  = getServiceRoot();
6389                 context.type     = kIOMessageConsoleSecurityChange;
6390                 context.argument = (void *) regEntry;
6391                 context.argSize  = 0;
6392
6393                 applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
6394                     &messageClientsApplier, &context );
6395         }
6396 }
6397
6398 IOReturn
6399 IOResources::setProperties( OSObject * properties )
6400 {
6401         IOReturn                    err;
6402         const OSSymbol *            key;
6403         OSDictionary *              dict;
6404         OSCollectionIterator *      iter;
6405
6406         err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
6407         if (kIOReturnSuccess != err) {
6408                 return err;
6409         }
6410
6411         dict = OSDynamicCast(OSDictionary, properties);
6412         if (NULL == dict) {
6413                 return kIOReturnBadArgument;
6414         }
6415
6416         iter = OSCollectionIterator::withCollection( dict);
6417         if (NULL == iter) {
6418                 return kIOReturnBadArgument;
6419         }
6420
6421         while ((key = OSDynamicCast(OSSymbol, iter->getNextObject()))) {
6422                 if (gIOConsoleUsersKey == key) {
6423                         do{
6424                                 OSArray * consoleUsers;
6425                                 consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
6426                                 if (!consoleUsers) {
6427                                         continue;
6428                                 }
6429                                 IOService::updateConsoleUsers(consoleUsers, 0);
6430                         }while (false);
6431                 }
6432
6433                 publishResource( key, dict->getObject(key));
6434         }
6435
6436         iter->release();
6437
6438         return kIOReturnSuccess;
6439 }
6440
6441 /*
6442  * Helpers for matching dictionaries.
6443  * Keys existing in matching are checked in properties.
6444  * Keys may be a string or OSCollection of IOStrings
6445  */
6446
6447 bool
6448 IOService::compareProperty( OSDictionary * matching,
6449     const char *   key )
6450 {
6451         OSObject *  value;
6452         OSObject *  prop;
6453         bool        ok;
6454
6455         value = matching->getObject( key );
6456         if (value) {
6457                 prop = copyProperty(key);
6458                 ok = value->isEqualTo(prop);
6459                 if (prop) {
6460                         prop->release();
6461                 }
6462         } else {
6463                 ok = true;
6464         }
6465
6466         return ok;
6467 }
6468
6469
6470 bool
6471 IOService::compareProperty( OSDictionary *   matching,
6472     const OSString * key )
6473 {
6474         OSObject *  value;
6475         OSObject *  prop;
6476         bool        ok;
6477
6478         value = matching->getObject( key );
6479         if (value) {
6480                 prop = copyProperty(key);
6481                 ok = value->isEqualTo(prop);
6482                 if (prop) {
6483                         prop->release();
6484                 }
6485         } else {
6486                 ok = true;
6487         }
6488
6489         return ok;
6490 }
6491
6492 bool
6493 IOService::compareProperties( OSDictionary * matching,
6494     OSCollection * keys )
6495 {
6496         OSCollectionIterator *      iter;
6497         const OSString *            key;
6498         bool                        ok = true;
6499
6500         if (!matching || !keys) {
6501                 return false;
6502         }
6503
6504         iter = OSCollectionIterator::withCollection( keys );
6505
6506         if (iter) {
6507                 while (ok && (key = OSDynamicCast( OSString, iter->getNextObject()))) {
6508                         ok = compareProperty( matching, key );
6509                 }
6510
6511                 iter->release();
6512         }
6513         keys->release(); // !! consume a ref !!
6514
6515         return ok;
6516 }
6517
6518 /* Helper to add a location matching dict to the table */
6519
6520 OSDictionary *
6521 IOService::addLocation( OSDictionary * table )
6522 {
6523         OSDictionary *      dict;
6524
6525         if (!table) {
6526                 return NULL;
6527         }
6528
6529         dict = OSDictionary::withCapacity( 1 );
6530         if (dict) {
6531                 bool ok = table->setObject( gIOLocationMatchKey, dict );
6532                 dict->release();
6533                 if (!ok) {
6534                         dict = NULL;
6535                 }
6536         }
6537
6538         return dict;
6539 }
6540
6541 /*
6542  * Go looking for a provider to match a location dict.
6543  */
6544
6545 IOService *
6546 IOService::matchLocation( IOService * /* client */ )
6547 {
6548         IOService * parent;
6549
6550         parent = getProvider();
6551
6552         if (parent) {
6553                 parent = parent->matchLocation( this );
6554         }
6555
6556         return parent;
6557 }
6558
6559 bool
6560 IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
6561 {
6562         OSString *          matched;
6563         OSObject *          obj;
6564         OSString *          str;
6565         IORegistryEntry *   entry;
6566         OSNumber *          num;
6567         bool                match = true;
6568         bool                changesOK = (0 != (kIOServiceChangesOK & options));
6569         uint32_t            count;
6570         uint32_t            done;
6571
6572         do{
6573                 count = table->getCount();
6574                 done = 0;
6575
6576                 str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
6577                 if (str) {
6578                         done++;
6579                         match = ((kIOServiceClassDone & options) || (NULL != metaCast(str)));
6580 #if MATCH_DEBUG
6581                         match = (0 != metaCast( str ));
6582                         if ((kIOServiceClassDone & options) && !match) {
6583                                 panic("classDone");
6584                         }
6585 #endif
6586                         if ((!match) || (done == count)) {
6587                                 break;
6588                         }
6589                 }
6590
6591                 obj = table->getObject( gIONameMatchKey );
6592                 if (obj) {
6593                         done++;
6594                         match = compareNames( obj, changesOK ? &matched : NULL );
6595                         if (!match) {
6596                                 break;
6597                         }
6598                         if (changesOK && matched) {
6599                                 // leave a hint as to which name matched
6600                                 table->setObject( gIONameMatchedKey, matched );
6601                                 matched->release();
6602                         }
6603                         if (done == count) {
6604                                 break;
6605                         }
6606                 }
6607
6608                 str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
6609                 if (str) {
6610                         const OSSymbol * sym;
6611                         done++;
6612                         match = false;
6613                         sym = copyLocation();
6614                         if (sym) {
6615                                 match = sym->isEqualTo( str );
6616                                 sym->release();
6617                         }
6618                         if ((!match) || (done == count)) {
6619                                 break;
6620                         }
6621                 }
6622
6623                 obj = table->getObject( gIOPropertyMatchKey );
6624                 if (obj) {
6625                         OSDictionary * dict;
6626                         OSDictionary * nextDict;
6627                         OSIterator *   iter;
6628                         done++;
6629                         match = false;
6630                         dict = dictionaryWithProperties();
6631                         if (dict) {
6632                                 nextDict = OSDynamicCast( OSDictionary, obj);
6633                                 if (nextDict) {
6634                                         iter = NULL;
6635                                 } else {
6636                                         iter = OSCollectionIterator::withCollection(
6637                                                 OSDynamicCast(OSCollection, obj));
6638                                 }
6639
6640                                 while (nextDict
6641                                     || (iter && (NULL != (nextDict = OSDynamicCast(OSDictionary,
6642                                     iter->getNextObject()))))) {
6643                                         match = dict->isEqualTo( nextDict, nextDict);
6644                                         if (match) {
6645                                                 break;
6646                                         }
6647                                         nextDict = NULL;
6648                                 }
6649                                 dict->release();
6650                                 if (iter) {
6651                                         iter->release();
6652                                 }
6653                         }
6654                         if ((!match) || (done == count)) {
6655                                 break;
6656                         }
6657                 }
6658
6659                 obj = table->getObject( gIOPropertyExistsMatchKey );
6660                 if (obj) {
6661                         OSDictionary * dict;
6662                         OSString     * nextKey;
6663                         OSIterator *   iter;
6664                         done++;
6665                         match = false;
6666                         dict = dictionaryWithProperties();
6667                         if (dict) {
6668                                 nextKey = OSDynamicCast( OSString, obj);
6669                                 if (nextKey) {
6670                                         iter = NULL;
6671                                 } else {
6672                                         iter = OSCollectionIterator::withCollection(
6673                                                 OSDynamicCast(OSCollection, obj));
6674                                 }
6675
6676                                 while (nextKey
6677                                     || (iter && (NULL != (nextKey = OSDynamicCast(OSString,
6678                                     iter->getNextObject()))))) {
6679                                         match = (NULL != dict->getObject(nextKey));
6680                                         if (match) {
6681                                                 break;
6682                                         }
6683                                         nextKey = NULL;
6684                                 }
6685                                 dict->release();
6686                                 if (iter) {
6687                                         iter->release();
6688                                 }
6689                         }
6690                         if ((!match) || (done == count)) {
6691                                 break;
6692                         }
6693                 }
6694
6695                 str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
6696                 if (str) {
6697                         done++;
6698                         entry = IORegistryEntry::fromPath( str->getCStringNoCopy());
6699                         match = (this == entry);
6700                         if (entry) {
6701                                 entry->release();
6702                         }
6703                         if ((!match) || (done == count)) {
6704                                 break;
6705                         }
6706                 }
6707
6708                 num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
6709                 if (num) {
6710                         done++;
6711                         match = (getRegistryEntryID() == num->unsigned64BitValue());
6712                         if ((!match) || (done == count)) {
6713                                 break;
6714                         }
6715                 }
6716
6717                 num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
6718                 if (num) {
6719                         OSIterator *        iter;
6720                         IOService *         service = NULL;
6721                         UInt32              serviceCount = 0;
6722
6723                         done++;
6724                         iter = getClientIterator();
6725                         if (iter) {
6726                                 while ((service = (IOService *) iter->getNextObject())) {
6727                                         if (kIOServiceInactiveState & service->__state[0]) {
6728                                                 continue;
6729                                         }
6730                                         if (NULL == service->getProperty( gIOMatchCategoryKey )) {
6731                                                 continue;
6732                                         }
6733                                         ++serviceCount;
6734                                 }
6735                                 iter->release();
6736                         }
6737                         match = (serviceCount == num->unsigned32BitValue());
6738                         if ((!match) || (done == count)) {
6739                                 break;
6740                         }
6741                 }
6742
6743 #define propMatch(key)                                  \
6744         obj = table->getObject(key);                    \
6745         if (obj)                                        \
6746         {                                               \
6747             OSObject * prop;                            \
6748             done++;                                     \
6749             prop = copyProperty(key);                   \
6750             match = obj->isEqualTo(prop);               \
6751             if (prop) prop->release();                  \
6752             if ((!match) || (done == count)) break;     \
6753         }
6754                 propMatch(gIOBSDNameKey)
6755                 propMatch(gIOBSDMajorKey)
6756                 propMatch(gIOBSDMinorKey)
6757                 propMatch(gIOBSDUnitKey)
6758 #undef propMatch
6759         }while (false);
6760
6761         if (did) {
6762                 *did = done;
6763         }
6764         return match;
6765 }
6766
6767 bool
6768 IOService::passiveMatch( OSDictionary * table, bool changesOK )
6769 {
6770         return matchPassive(table, changesOK ? kIOServiceChangesOK : 0);
6771 }
6772
6773 bool
6774 IOService::matchPassive(OSDictionary * table, uint32_t options)
6775 {
6776         IOService *         where;
6777         OSDictionary *      nextTable;
6778         SInt32              score;
6779         OSNumber *          newPri;
6780         bool                match = true;
6781         bool                matchParent = false;
6782         uint32_t            count;
6783         uint32_t            done;
6784
6785         assert( table );
6786
6787 #if !CONFIG_EMBEDDED
6788         OSArray* aliasServiceRegIds = NULL;
6789         IOService* foundAlternateService = NULL;
6790 #endif
6791
6792 #if MATCH_DEBUG
6793         OSDictionary * root = table;
6794 #endif
6795
6796         where = this;
6797         do{
6798                 do{
6799                         count = table->getCount();
6800                         if (!(kIOServiceInternalDone & options)) {
6801                                 match = where->matchInternal(table, options, &done);
6802                                 // don't call family if we've done all the entries in the table
6803                                 if ((!match) || (done == count)) {
6804                                         break;
6805                                 }
6806                         }
6807
6808                         // pass in score from property table
6809                         score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
6810
6811                         // do family specific matching
6812                         match = where->matchPropertyTable( table, &score );
6813
6814                         if (!match) {
6815 #if IOMATCHDEBUG
6816                                 if (kIOLogMatch & getDebugFlags( table )) {
6817                                         LOG("%s: family specific matching fails\n", where->getName());
6818                                 }
6819 #endif
6820                                 break;
6821                         }
6822
6823                         if (kIOServiceChangesOK & options) {
6824                                 // save the score
6825                                 newPri = OSNumber::withNumber( score, 32 );
6826                                 if (newPri) {
6827                                         table->setObject( gIOProbeScoreKey, newPri );
6828                                         newPri->release();
6829                                 }
6830                         }
6831
6832                         options = 0;
6833                         matchParent = false;
6834
6835                         nextTable = OSDynamicCast(OSDictionary,
6836                             table->getObject( gIOParentMatchKey ));
6837                         if (nextTable) {
6838                                 // look for a matching entry anywhere up to root
6839                                 match = false;
6840                                 matchParent = true;
6841                                 table = nextTable;
6842                                 break;
6843                         }
6844
6845                         table = OSDynamicCast(OSDictionary,
6846                             table->getObject( gIOLocationMatchKey ));
6847                         if (table) {
6848                                 // look for a matching entry at matchLocation()
6849                                 match = false;
6850                                 where = where->getProvider();
6851                                 if (where && (where = where->matchLocation(where))) {
6852                                         continue;
6853                                 }
6854                         }
6855                         break;
6856                 }while (true);
6857
6858                 if (match == true) {
6859                         break;
6860                 }
6861
6862                 if (matchParent == true) {
6863 #if !CONFIG_EMBEDDED
6864                         // check if service has an alias to search its other "parents" if a parent match isn't found
6865                         OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
6866                         OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
6867                         if (alternateRegistryID != NULL) {
6868                                 if (aliasServiceRegIds == NULL) {
6869                                         aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
6870                                 }
6871                                 aliasServiceRegIds->setObject(alternateRegistryID);
6872                         }
6873                         OSSafeReleaseNULL(prop);
6874 #endif
6875                 } else {
6876                         break;
6877                 }
6878
6879                 where = where->getProvider();
6880 #if !CONFIG_EMBEDDED
6881                 if (where == NULL) {
6882                         // there were no matching parent services, check to see if there are aliased services that have a matching parent
6883                         if (aliasServiceRegIds != NULL) {
6884                                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
6885                                 if (numAliasedServices != 0) {
6886                                         OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
6887                                         if (alternateRegistryID != NULL) {
6888                                                 OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
6889                                                 aliasServiceRegIds->removeObject(numAliasedServices - 1);
6890                                                 if (alternateMatchingDict != NULL) {
6891                                                         OSSafeReleaseNULL(foundAlternateService);
6892                                                         foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
6893                                                         alternateMatchingDict->release();
6894                                                         if (foundAlternateService != NULL) {
6895                                                                 where = foundAlternateService;
6896                                                         }
6897                                                 }
6898                                         }
6899                                 }
6900                         }
6901                 }
6902 #endif
6903         }while (where != NULL);
6904
6905 #if !CONFIG_EMBEDDED
6906         OSSafeReleaseNULL(foundAlternateService);
6907         OSSafeReleaseNULL(aliasServiceRegIds);
6908 #endif
6909
6910 #if MATCH_DEBUG
6911         if (where != this) {
6912                 OSSerialize * s = OSSerialize::withCapacity(128);
6913                 root->serialize(s);
6914                 kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
6915                 s->release();
6916         }
6917 #endif
6918
6919         return match;
6920 }
6921
6922
6923 IOReturn
6924 IOService::newUserClient( task_t owningTask, void * securityID,
6925     UInt32 type, OSDictionary * properties,
6926     IOUserClient ** handler )
6927 {
6928         const OSSymbol *userClientClass = NULL;
6929         IOUserClient *client;
6930         OSObject *prop;
6931         OSObject *temp;
6932
6933         if (reserved && reserved->uvars && reserved->uvars->userServer) {
6934                 return reserved->uvars->userServer->serviceNewUserClient(this, owningTask, securityID, type, properties, handler);
6935         }
6936
6937         if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) {
6938                 return kIOReturnSuccess;
6939         }
6940
6941         // First try my own properties for a user client class name
6942         prop = copyProperty(gIOUserClientClassKey);
6943         if (prop) {
6944                 if (OSDynamicCast(OSSymbol, prop)) {
6945                         userClientClass = (const OSSymbol *) prop;
6946                 } else if (OSDynamicCast(OSString, prop)) {
6947                         userClientClass = OSSymbol::withString((OSString *) prop);
6948                         if (userClientClass) {
6949                                 setProperty(gIOUserClientClassKey,
6950                                     (OSObject *) userClientClass);
6951                         }
6952                 }
6953         }
6954
6955         // Didn't find one so lets just bomb out now without further ado.
6956         if (!userClientClass) {
6957                 OSSafeReleaseNULL(prop);
6958                 return kIOReturnUnsupported;
6959         }
6960
6961         // This reference is consumed by the IOServiceOpen call
6962         temp = OSMetaClass::allocClassWithName(userClientClass);
6963         OSSafeReleaseNULL(prop);
6964         if (!temp) {
6965                 return kIOReturnNoMemory;
6966         }
6967
6968         if (OSDynamicCast(IOUserClient, temp)) {
6969                 client = (IOUserClient *) temp;
6970         } else {
6971                 temp->release();
6972                 return kIOReturnUnsupported;
6973         }
6974
6975         if (!client->initWithTask(owningTask, securityID, type, properties)) {
6976                 client->release();
6977                 return kIOReturnBadArgument;
6978         }
6979
6980         if (!client->attach(this)) {
6981                 client->release();
6982                 return kIOReturnUnsupported;
6983         }
6984
6985         if (!client->start(this)) {
6986                 client->detach(this);
6987                 client->release();
6988                 return kIOReturnUnsupported;
6989         }
6990
6991         *handler = client;
6992         return kIOReturnSuccess;
6993 }
6994
6995 IOReturn
6996 IOService::newUserClient( task_t owningTask, void * securityID,
6997     UInt32 type, IOUserClient ** handler )
6998 {
6999         return kIOReturnUnsupported;
7000 }
7001
7002 IOReturn
7003 IOService::requestProbe( IOOptionBits options )
7004 {
7005         return kIOReturnUnsupported;
7006 }
7007
7008 /*
7009  * Convert an IOReturn to text. Subclasses which add additional
7010  * IOReturn's should override this method and call
7011  * super::stringFromReturn if the desired value is not found.
7012  */
7013
7014 const char *
7015 IOService::stringFromReturn( IOReturn rtn )
7016 {
7017         static const IONamedValue IOReturn_values[] = {
7018                 {kIOReturnSuccess, "success"                           },
7019                 {kIOReturnError, "general error"                     },
7020                 {kIOReturnNoMemory, "memory allocation error"           },
7021                 {kIOReturnNoResources, "resource shortage"                 },
7022                 {kIOReturnIPCError, "Mach IPC failure"                  },
7023                 {kIOReturnNoDevice, "no such device"                    },
7024                 {kIOReturnNotPrivileged, "privilege violation"               },
7025                 {kIOReturnBadArgument, "invalid argument"                  },
7026                 {kIOReturnLockedRead, "device is read locked"             },
7027                 {kIOReturnLockedWrite, "device is write locked"            },
7028                 {kIOReturnExclusiveAccess, "device is exclusive access"        },
7029                 {kIOReturnBadMessageID, "bad IPC message ID"                },
7030                 {kIOReturnUnsupported, "unsupported function"              },
7031                 {kIOReturnVMError, "virtual memory error"              },
7032                 {kIOReturnInternalError, "internal driver error"             },
7033                 {kIOReturnIOError, "I/O error"                         },
7034                 {kIOReturnCannotLock, "cannot acquire lock"               },
7035                 {kIOReturnNotOpen, "device is not open"                },
7036                 {kIOReturnNotReadable, "device is not readable"            },
7037                 {kIOReturnNotWritable, "device is not writeable"           },
7038                 {kIOReturnNotAligned, "alignment error"                   },
7039                 {kIOReturnBadMedia, "media error"                       },
7040                 {kIOReturnStillOpen, "device is still open"              },
7041                 {kIOReturnRLDError, "rld failure"                       },
7042                 {kIOReturnDMAError, "DMA failure"                       },
7043                 {kIOReturnBusy, "device is busy"                    },
7044                 {kIOReturnTimeout, "I/O timeout"                       },
7045                 {kIOReturnOffline, "device is offline"                 },
7046                 {kIOReturnNotReady, "device is not ready"               },
7047                 {kIOReturnNotAttached, "device/channel is not attached"    },
7048                 {kIOReturnNoChannels, "no DMA channels available"         },
7049                 {kIOReturnNoSpace, "no space for data"                 },
7050                 {kIOReturnPortExists, "device port already exists"        },
7051                 {kIOReturnCannotWire, "cannot wire physical memory"       },
7052                 {kIOReturnNoInterrupt, "no interrupt attached"             },
7053                 {kIOReturnNoFrames, "no DMA frames enqueued"            },
7054                 {kIOReturnMessageTooLarge, "message is too large"              },
7055                 {kIOReturnNotPermitted, "operation is not permitted"        },
7056                 {kIOReturnNoPower, "device is without power"           },
7057                 {kIOReturnNoMedia, "media is not present"              },
7058                 {kIOReturnUnformattedMedia, "media is not formatted"            },
7059                 {kIOReturnUnsupportedMode, "unsupported mode"                  },
7060                 {kIOReturnUnderrun, "data underrun"                     },
7061                 {kIOReturnOverrun, "data overrun"                      },
7062                 {kIOReturnDeviceError, "device error"                      },
7063                 {kIOReturnNoCompletion, "no completion routine"             },
7064                 {kIOReturnAborted, "operation was aborted"             },
7065                 {kIOReturnNoBandwidth, "bus bandwidth would be exceeded"   },
7066                 {kIOReturnNotResponding, "device is not responding"          },
7067                 {kIOReturnInvalid, "unanticipated driver error"        },
7068                 {0, NULL                                }
7069         };
7070
7071         return IOFindNameForValue(rtn, IOReturn_values);
7072 }
7073
7074 /*
7075  * Convert an IOReturn to an errno.
7076  */
7077 int
7078 IOService::errnoFromReturn( IOReturn rtn )
7079 {
7080         if (unix_err(err_get_code(rtn)) == rtn) {
7081                 return err_get_code(rtn);
7082         }
7083
7084         switch (rtn) {
7085         // (obvious match)
7086         case kIOReturnSuccess:
7087                 return 0;
7088         case kIOReturnNoMemory:
7089                 return ENOMEM;
7090         case kIOReturnNoDevice:
7091                 return ENXIO;
7092         case kIOReturnVMError:
7093                 return EFAULT;
7094         case kIOReturnNotPermitted:
7095                 return EPERM;
7096         case kIOReturnNotPrivileged:
7097                 return EACCES;
7098         case kIOReturnIOError:
7099                 return EIO;
7100         case kIOReturnNotWritable:
7101                 return EROFS;
7102         case kIOReturnBadArgument:
7103                 return EINVAL;
7104         case kIOReturnUnsupported:
7105                 return ENOTSUP;
7106         case kIOReturnBusy:
7107                 return EBUSY;
7108         case kIOReturnNoPower:
7109                 return EPWROFF;
7110         case kIOReturnDeviceError:
7111                 return EDEVERR;
7112         case kIOReturnTimeout:
7113                 return ETIMEDOUT;
7114         case kIOReturnMessageTooLarge:
7115                 return EMSGSIZE;
7116         case kIOReturnNoSpace:
7117                 return ENOSPC;
7118         case kIOReturnCannotLock:
7119                 return ENOLCK;
7120
7121         // (best match)
7122         case kIOReturnBadMessageID:
7123         case kIOReturnNoCompletion:
7124         case kIOReturnNotAligned:
7125                 return EINVAL;
7126         case kIOReturnNotReady:
7127                 return EBUSY;
7128         case kIOReturnRLDError:
7129                 return EBADMACHO;
7130         case kIOReturnPortExists:
7131         case kIOReturnStillOpen:
7132                 return EEXIST;
7133         case kIOReturnExclusiveAccess:
7134         case kIOReturnLockedRead:
7135         case kIOReturnLockedWrite:
7136         case kIOReturnNotOpen:
7137         case kIOReturnNotReadable:
7138                 return EACCES;
7139         case kIOReturnCannotWire:
7140         case kIOReturnNoResources:
7141                 return ENOMEM;
7142         case kIOReturnAborted:
7143         case kIOReturnOffline:
7144         case kIOReturnNotResponding:
7145                 return EBUSY;
7146         case kIOReturnBadMedia:
7147         case kIOReturnNoMedia:
7148         case kIOReturnNotAttached:
7149         case kIOReturnUnformattedMedia:
7150                 return ENXIO; // (media error)
7151         case kIOReturnDMAError:
7152         case kIOReturnOverrun:
7153         case kIOReturnUnderrun:
7154                 return EIO; // (transfer error)
7155         case kIOReturnNoBandwidth:
7156         case kIOReturnNoChannels:
7157         case kIOReturnNoFrames:
7158         case kIOReturnNoInterrupt:
7159                 return EIO; // (hardware error)
7160         case kIOReturnError:
7161         case kIOReturnInternalError:
7162         case kIOReturnInvalid:
7163                 return EIO; // (generic error)
7164         case kIOReturnIPCError:
7165                 return EIO; // (ipc error)
7166         default:
7167                 return EIO; // (all other errors)
7168         }
7169 }
7170
7171 IOReturn
7172 IOService::message( UInt32 type, IOService * provider,
7173     void * argument )
7174 {
7175         /*
7176          * Generic entry point for calls from the provider.  A return value of
7177          * kIOReturnSuccess indicates that the message was received, and where
7178          * applicable, that it was successful.
7179          */
7180
7181         return kIOReturnUnsupported;
7182 }
7183
7184 /*
7185  * Device memory
7186  */
7187
7188 IOItemCount
7189 IOService::getDeviceMemoryCount( void )
7190 {
7191         OSArray *           array;
7192         IOItemCount         count;
7193
7194         array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
7195         if (array) {
7196                 count = array->getCount();
7197         } else {
7198                 count = 0;
7199         }
7200
7201         return count;
7202 }
7203
7204 IODeviceMemory *
7205 IOService::getDeviceMemoryWithIndex( unsigned int index )
7206 {
7207         OSArray *           array;
7208         IODeviceMemory *    range;
7209
7210         array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
7211         if (array) {
7212                 range = (IODeviceMemory *) array->getObject( index );
7213         } else {
7214                 range = NULL;
7215         }
7216
7217         return range;
7218 }
7219
7220 IOMemoryMap *
7221 IOService::mapDeviceMemoryWithIndex( unsigned int index,
7222     IOOptionBits options )
7223 {
7224         IODeviceMemory *    range;
7225         IOMemoryMap *       map;
7226
7227         range = getDeviceMemoryWithIndex( index );
7228         if (range) {
7229                 map = range->map( options );
7230         } else {
7231                 map = NULL;
7232         }
7233
7234         return map;
7235 }
7236
7237 OSArray *
7238 IOService::getDeviceMemory( void )
7239 {
7240         return OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
7241 }
7242
7243
7244 void
7245 IOService::setDeviceMemory( OSArray * array )
7246 {
7247         setProperty( gIODeviceMemoryKey, array);
7248 }
7249
7250 /*
7251  * For machines where the transfers on an I/O bus can stall because
7252  * the CPU is in an idle mode, These APIs allow a driver to specify
7253  * the maximum bus stall that they can handle.  0 indicates no limit.
7254  */
7255 void
7256 IOService::
7257 setCPUSnoopDelay(UInt32 __unused ns)
7258 {
7259 #if defined(__i386__) || defined(__x86_64__)
7260         ml_set_maxsnoop(ns);
7261 #endif /* defined(__i386__) || defined(__x86_64__) */
7262 }
7263
7264 UInt32
7265 IOService::
7266 getCPUSnoopDelay()
7267 {
7268 #if defined(__i386__) || defined(__x86_64__)
7269         return ml_get_maxsnoop();
7270 #else
7271         return 0;
7272 #endif /* defined(__i386__) || defined(__x86_64__) */
7273 }
7274
7275 #if defined(__i386__) || defined(__x86_64__)
7276 static void
7277 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
7278 {
7279         static const UInt kNoReplace = -1U; // Must be an illegal index
7280         UInt replace = kNoReplace;
7281         bool setCpuDelay = false;
7282
7283         IORecursiveLockLock(sCpuDelayLock);
7284
7285         UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
7286         CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
7287         IOService * holder = NULL;
7288
7289         if (ns) {
7290                 const CpuDelayEntry ne = {service, ns, delayType};
7291                 holder = service;
7292                 // Set maximum delay.
7293                 for (UInt i = 0; i < count; i++) {
7294                         IOService *thisService = entries[i].fService;
7295                         bool sameType = (delayType == entries[i].fDelayType);
7296                         if ((service == thisService) && sameType) {
7297                                 replace = i;
7298                         } else if (!thisService) {
7299                                 if (kNoReplace == replace) {
7300                                         replace = i;
7301                                 }
7302                         } else if (sameType) {
7303                                 const UInt32 thisMax = entries[i].fMaxDelay;
7304                                 if (thisMax < ns) {
7305                                         ns = thisMax;
7306                                         holder = thisService;
7307                                 }
7308                         }
7309                 }
7310
7311                 setCpuDelay = true;
7312                 if (kNoReplace == replace) {
7313                         sCpuDelayData->appendBytes(&ne, sizeof(ne));
7314                 } else {
7315                         entries[replace] = ne;
7316                 }
7317         } else {
7318                 ns = -1U; // Set to max unsigned, i.e. no restriction
7319
7320                 for (UInt i = 0; i < count; i++) {
7321                         // Clear a maximum delay.
7322                         IOService *thisService = entries[i].fService;
7323                         if (thisService && (delayType == entries[i].fDelayType)) {
7324                                 UInt32 thisMax = entries[i].fMaxDelay;
7325                                 if (service == thisService) {
7326                                         replace = i;
7327                                 } else if (thisMax < ns) {
7328                                         ns = thisMax;
7329                                         holder = thisService;
7330                                 }
7331                         }
7332                 }
7333
7334                 // Check if entry found
7335                 if (kNoReplace != replace) {
7336                         entries[replace].fService = NULL; // Null the entry
7337                         setCpuDelay = true;
7338                 }
7339         }
7340
7341         if (setCpuDelay) {
7342                 if (holder && debug_boot_arg) {
7343                         strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
7344                 }
7345
7346                 // Must be safe to call from locked context
7347                 if (delayType == kCpuDelayBusStall) {
7348                         ml_set_maxbusdelay(ns);
7349                 } else if (delayType == kCpuDelayInterrupt) {
7350                         ml_set_maxintdelay(ns);
7351                 }
7352                 sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
7353                 sCPULatencySet[delayType]->setValue(ns);
7354
7355                 OSArray * handlers = sCpuLatencyHandlers[delayType];
7356                 IOService * target;
7357                 if (handlers) {
7358                         for (unsigned int idx = 0;
7359                             (target = (IOService *) handlers->getObject(idx));
7360                             idx++) {
7361                                 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
7362                                     (void *) (uintptr_t) ns, holder,
7363                                     NULL, NULL);
7364                         }
7365                 }
7366         }
7367
7368         IORecursiveLockUnlock(sCpuDelayLock);
7369 }
7370
7371 static IOReturn
7372 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
7373 {
7374         IOReturn result = kIOReturnNotFound;
7375         OSArray * array;
7376         unsigned int idx;
7377
7378         IORecursiveLockLock(sCpuDelayLock);
7379
7380         do{
7381                 if (enable && !sCpuLatencyHandlers[delayType]) {
7382                         sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
7383                 }
7384                 array = sCpuLatencyHandlers[delayType];
7385                 if (!array) {
7386                         break;
7387                 }
7388                 idx = array->getNextIndexOfObject(target, 0);
7389                 if (!enable) {
7390                         if (-1U != idx) {
7391                                 array->removeObject(idx);
7392                                 result = kIOReturnSuccess;
7393                         }
7394                 } else {
7395                         if (-1U != idx) {
7396                                 result = kIOReturnExclusiveAccess;
7397                                 break;
7398                         }
7399                         array->setObject(target);
7400
7401                         UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
7402                         CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
7403                         UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction
7404                         IOService * holder = NULL;
7405
7406                         for (UInt i = 0; i < count; i++) {
7407                                 if (entries[i].fService
7408                                     && (delayType == entries[i].fDelayType)
7409                                     && (entries[i].fMaxDelay < ns)) {
7410                                         ns = entries[i].fMaxDelay;
7411                                         holder = entries[i].fService;
7412                                 }
7413                         }
7414                         target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
7415                             (void *) (uintptr_t) ns, holder,
7416                             NULL, NULL);
7417                         result = kIOReturnSuccess;
7418                 }
7419         }while (false);
7420
7421         IORecursiveLockUnlock(sCpuDelayLock);
7422
7423         return result;
7424 }
7425
7426 #endif /* defined(__i386__) || defined(__x86_64__) */
7427
7428 void
7429 IOService::
7430 requireMaxBusStall(UInt32 __unused ns)
7431 {
7432 #if defined(__i386__) || defined(__x86_64__)
7433         requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
7434 #endif
7435 }
7436
7437 void
7438 IOService::
7439 requireMaxInterruptDelay(uint32_t __unused ns)
7440 {
7441 #if defined(__i386__) || defined(__x86_64__)
7442         requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
7443 #endif
7444 }
7445
7446 /*
7447  * Device interrupts
7448  */
7449
7450 IOReturn
7451 IOService::resolveInterrupt(IOService *nub, int source)
7452 {
7453         IOInterruptController *interruptController;
7454         OSArray               *array;
7455         OSData                *data;
7456         OSSymbol              *interruptControllerName;
7457         long                  numSources;
7458         IOInterruptSource     *interruptSources;
7459
7460         // Get the parents list from the nub.
7461         array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
7462         if (array == NULL) {
7463                 return kIOReturnNoResources;
7464         }
7465
7466         // Allocate space for the IOInterruptSources if needed... then return early.
7467         if (nub->_interruptSources == NULL) {
7468                 numSources = array->getCount();
7469                 interruptSources = (IOInterruptSource *)IOMalloc(
7470                         numSources * sizeofAllIOInterruptSource);
7471                 if (interruptSources == NULL) {
7472                         return kIOReturnNoMemory;
7473                 }
7474
7475                 bzero(interruptSources, numSources * sizeofAllIOInterruptSource);
7476
7477                 nub->_numInterruptSources = numSources;
7478                 nub->_interruptSources = interruptSources;
7479                 return kIOReturnSuccess;
7480         }
7481
7482         interruptControllerName = OSDynamicCast(OSSymbol, array->getObject(source));
7483         if (interruptControllerName == NULL) {
7484                 return kIOReturnNoResources;
7485         }
7486
7487         interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
7488         if (interruptController == NULL) {
7489                 return kIOReturnNoResources;
7490         }
7491
7492         // Get the interrupt numbers from the nub.
7493         array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
7494         if (array == NULL) {
7495                 return kIOReturnNoResources;
7496         }
7497         data = OSDynamicCast(OSData, array->getObject(source));
7498         if (data == NULL) {
7499                 return kIOReturnNoResources;
7500         }
7501
7502         // Set the interruptController and interruptSource in the nub's table.
7503         interruptSources = nub->_interruptSources;
7504         interruptSources[source].interruptController = interruptController;
7505         interruptSources[source].vectorData = data;
7506
7507         return kIOReturnSuccess;
7508 }
7509
7510 IOReturn
7511 IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
7512 {
7513         IOReturn ret;
7514
7515         /* Make sure the _interruptSources are set */
7516         if (_interruptSources == NULL) {
7517                 ret = resolveInterrupt(this, source);
7518                 if (ret != kIOReturnSuccess) {
7519                         return ret;
7520                 }
7521         }
7522
7523         /* Make sure the local source number is valid */
7524         if ((source < 0) || (source >= _numInterruptSources)) {
7525                 return kIOReturnNoInterrupt;
7526         }
7527
7528         /* Look up the contoller for the local source */
7529         *interruptController = _interruptSources[source].interruptController;
7530
7531         if (*interruptController == NULL) {
7532                 if (!resolve) {
7533                         return kIOReturnNoInterrupt;
7534                 }
7535
7536                 /* Try to resolve the interrupt */
7537                 ret = resolveInterrupt(this, source);
7538                 if (ret != kIOReturnSuccess) {
7539                         return ret;
7540                 }
7541
7542                 *interruptController = _interruptSources[source].interruptController;
7543         }
7544
7545         return kIOReturnSuccess;
7546 }
7547
7548 IOReturn
7549 IOService::registerInterrupt(int source, OSObject *target,
7550     IOInterruptAction handler,
7551     void *refCon)
7552 {
7553         IOInterruptController *interruptController;
7554         IOReturn              ret;
7555
7556         ret = lookupInterrupt(source, true, &interruptController);
7557         if (ret != kIOReturnSuccess) {
7558                 return ret;
7559         }
7560
7561         /* Register the source */
7562         return interruptController->registerInterrupt(this, source, target,
7563                    (IOInterruptHandler)handler,
7564                    refCon);
7565 }
7566
7567 static void
7568 IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
7569     IOService * nub, int source )
7570 {
7571         ((IOInterruptActionBlock)(refCon))(nub, source);
7572 }
7573
7574 IOReturn
7575 IOService::registerInterruptBlock(int source, OSObject *target,
7576     IOInterruptActionBlock handler)
7577 {
7578         IOReturn ret;
7579         void   * block;
7580
7581         block = Block_copy(handler);
7582         if (!block) {
7583                 return kIOReturnNoMemory;
7584         }
7585
7586         ret = registerInterrupt(source, target, &IOServiceInterruptActionToBlock, block);
7587         if (kIOReturnSuccess != ret) {
7588                 Block_release(block);
7589                 return ret;
7590         }
7591         _interruptSourcesPrivate(this)[source].vectorBlock = block;
7592
7593         return ret;
7594 }
7595
7596 IOReturn
7597 IOService::unregisterInterrupt(int source)
7598 {
7599         IOReturn              ret;
7600         IOInterruptController *interruptController;
7601         void                  *block;
7602
7603         ret = lookupInterrupt(source, false, &interruptController);
7604         if (ret != kIOReturnSuccess) {
7605                 return ret;
7606         }
7607
7608         /* Unregister the source */
7609         block = _interruptSourcesPrivate(this)[source].vectorBlock;
7610         ret = interruptController->unregisterInterrupt(this, source);
7611         if ((kIOReturnSuccess == ret) && (block = _interruptSourcesPrivate(this)[source].vectorBlock)) {
7612                 _interruptSourcesPrivate(this)[source].vectorBlock = NULL;
7613                 Block_release(block);
7614         }
7615
7616         return ret;
7617 }
7618
7619 IOReturn
7620 IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
7621 {
7622         IOReportLegend * legend = NULL;
7623         IOInterruptAccountingData * oldValue = NULL;
7624         IOInterruptAccountingReporter * newArray = NULL;
7625         char subgroupName[64];
7626         int newArraySize = 0;
7627         int i = 0;
7628
7629         if (source < 0) {
7630                 return kIOReturnBadArgument;
7631         }
7632
7633         /*
7634          * We support statistics on a maximum of 256 interrupts per nub; if a nub
7635          * has more than 256 interrupt specifiers associated with it, and tries
7636          * to register a high interrupt index with interrupt accounting, panic.
7637          * Having more than 256 interrupts associated with a single nub is
7638          * probably a sign that something fishy is going on.
7639          */
7640         if (source > IA_INDEX_MAX) {
7641                 panic("addInterruptStatistics called for an excessively large index (%d)", source);
7642         }
7643
7644         /*
7645          * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
7646          * leaving it as is because the likelihood of contention where we are
7647          * actually growing the array is minimal (we would realistically need
7648          * to be starting a driver for the first time, with an IOReporting
7649          * client already in place).  Nonetheless, cleanup that can be done
7650          * to adhere to best practices; it'll make the code more complicated,
7651          * unfortunately.
7652          */
7653         IOLockLock(reserved->interruptStatisticsLock);
7654
7655         /*
7656          * Lazily allocate the statistics array.
7657          */
7658         if (!reserved->interruptStatisticsArray) {
7659                 reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
7660                 assert(reserved->interruptStatisticsArray);
7661                 reserved->interruptStatisticsArrayCount = 1;
7662                 bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
7663         }
7664
7665         if (source >= reserved->interruptStatisticsArrayCount) {
7666                 /*
7667                  * We're still within the range of supported indices, but we are out
7668                  * of space in the current array.  Do a nasty realloc (because
7669                  * IORealloc isn't a thing) here.  We'll double the size with each
7670                  * reallocation.
7671                  *
7672                  * Yes, the "next power of 2" could be more efficient; but this will
7673                  * be invoked incredibly rarely.  Who cares.
7674                  */
7675                 newArraySize = (reserved->interruptStatisticsArrayCount << 1);
7676
7677                 while (newArraySize <= source) {
7678                         newArraySize = (newArraySize << 1);
7679                 }
7680                 newArray = IONew(IOInterruptAccountingReporter, newArraySize);
7681
7682                 assert(newArray);
7683
7684                 /*
7685                  * TODO: This even zeroes the memory it is about to overwrite.
7686                  * Shameful; fix it.  Not particularly high impact, however.
7687                  */
7688                 bzero(newArray, newArraySize * sizeof(*newArray));
7689                 memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
7690                 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
7691                 reserved->interruptStatisticsArray = newArray;
7692                 reserved->interruptStatisticsArrayCount = newArraySize;
7693         }
7694
7695         if (!reserved->interruptStatisticsArray[source].reporter) {
7696                 /*
7697                  * We don't have a reporter associated with this index yet, so we
7698                  * need to create one.
7699                  */
7700                 /*
7701                  * TODO: Some statistics do in fact have common units (time); should this be
7702                  * split into separate reporters to communicate this?
7703                  */
7704                 reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
7705
7706                 /*
7707                  * Each statistic is given an identifier based on the interrupt index (which
7708                  * should be unique relative to any single nub) and the statistic involved.
7709                  * We should now have a sane (small and positive) index, so start
7710                  * constructing the channels for statistics.
7711                  */
7712                 for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
7713                         /*
7714                          * TODO: Currently, this does not add channels for disabled statistics.
7715                          * Will this be confusing for clients?  If so, we should just add the
7716                          * channels; we can avoid updating the channels even if they exist.
7717                          */
7718                         if (IA_GET_STATISTIC_ENABLED(i)) {
7719                                 reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
7720                         }
7721                 }
7722
7723                 /*
7724                  * We now need to add the legend for this reporter to the registry.
7725                  */
7726                 OSObject * prop = copyProperty(kIOReportLegendKey);
7727                 legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
7728                 OSSafeReleaseNULL(prop);
7729
7730                 /*
7731                  * Note that while we compose the subgroup name, we do not need to
7732                  * manage its lifecycle (the reporter will handle this).
7733                  */
7734                 snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
7735                 subgroupName[sizeof(subgroupName) - 1] = 0;
7736                 legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
7737                 setProperty(kIOReportLegendKey, legend->getLegend());
7738                 legend->release();
7739
7740                 /*
7741                  * TODO: Is this a good idea?  Probably not; my assumption is it opts
7742                  * all entities who register interrupts into public disclosure of all
7743                  * IOReporting channels.  Unfortunately, this appears to be as fine
7744                  * grain as it gets.
7745                  */
7746                 setProperty(kIOReportLegendPublicKey, true);
7747         }
7748
7749         /*
7750          * Don't stomp existing entries.  If we are about to, panic; this
7751          * probably means we failed to tear down our old interrupt source
7752          * correctly.
7753          */
7754         oldValue = reserved->interruptStatisticsArray[source].statistics;
7755
7756         if (oldValue) {
7757                 panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
7758         }
7759
7760         reserved->interruptStatisticsArray[source].statistics = statistics;
7761
7762         /*
7763          * Inherit the reporter values for each statistic.  The target may
7764          * be torn down as part of the runtime of the service (especially
7765          * for sleep/wake), so we inherit in order to avoid having values
7766          * reset for no apparent reason.  Our statistics are ultimately
7767          * tied to the index and the sevice, not to an individual target,
7768          * so we should maintain them accordingly.
7769          */
7770         interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
7771
7772         IOLockUnlock(reserved->interruptStatisticsLock);
7773
7774         return kIOReturnSuccess;
7775 }
7776
7777 IOReturn
7778 IOService::removeInterruptStatistics(int source)
7779 {
7780         IOInterruptAccountingData * value = NULL;
7781
7782         if (source < 0) {
7783                 return kIOReturnBadArgument;
7784         }
7785
7786         IOLockLock(reserved->interruptStatisticsLock);
7787
7788         /*
7789          * We dynamically grow the statistics array, so an excessively
7790          * large index value has NEVER been registered.  This either
7791          * means our cap on the array size is too small (unlikely), or
7792          * that we have been passed a corrupt index (this must be passed
7793          * the plain index into the interrupt specifier list).
7794          */
7795         if (source >= reserved->interruptStatisticsArrayCount) {
7796                 panic("removeInterruptStatistics called for index %d, which was never registered", source);
7797         }
7798
7799         assert(reserved->interruptStatisticsArray);
7800
7801         /*
7802          * If there is no existing entry, we are most likely trying to
7803          * free an interrupt owner twice, or we have corrupted the
7804          * index value.
7805          */
7806         value = reserved->interruptStatisticsArray[source].statistics;
7807
7808         if (!value) {
7809                 panic("removeInterruptStatistics called for empty index %d", source);
7810         }
7811
7812         /*
7813          * We update the statistics, so that any delta with the reporter
7814          * state is not lost.
7815          */
7816         interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
7817         reserved->interruptStatisticsArray[source].statistics = NULL;
7818         IOLockUnlock(reserved->interruptStatisticsLock);
7819
7820         return kIOReturnSuccess;
7821 }
7822
7823 IOReturn
7824 IOService::getInterruptType(int source, int *interruptType)
7825 {
7826         IOInterruptController *interruptController;
7827         IOReturn              ret;
7828
7829         ret = lookupInterrupt(source, true, &interruptController);
7830         if (ret != kIOReturnSuccess) {
7831                 return ret;
7832         }
7833
7834         /* Return the type */
7835         return interruptController->getInterruptType(this, source, interruptType);
7836 }
7837
7838 IOReturn
7839 IOService::enableInterrupt(int source)
7840 {
7841         IOInterruptController *interruptController;
7842         IOReturn              ret;
7843
7844         ret = lookupInterrupt(source, false, &interruptController);
7845         if (ret != kIOReturnSuccess) {
7846                 return ret;
7847         }
7848
7849         /* Enable the source */
7850         return interruptController->enableInterrupt(this, source);
7851 }
7852
7853 IOReturn
7854 IOService::disableInterrupt(int source)
7855 {
7856         IOInterruptController *interruptController;
7857         IOReturn              ret;
7858
7859         ret = lookupInterrupt(source, false, &interruptController);
7860         if (ret != kIOReturnSuccess) {
7861                 return ret;
7862         }
7863
7864         /* Disable the source */
7865         return interruptController->disableInterrupt(this, source);
7866 }
7867
7868 IOReturn
7869 IOService::causeInterrupt(int source)
7870 {
7871         IOInterruptController *interruptController;
7872         IOReturn              ret;
7873
7874         ret = lookupInterrupt(source, false, &interruptController);
7875         if (ret != kIOReturnSuccess) {
7876                 return ret;
7877         }
7878
7879         /* Cause an interrupt for the source */
7880         return interruptController->causeInterrupt(this, source);
7881 }
7882
7883 IOReturn
7884 IOService::configureReport(IOReportChannelList    *channelList,
7885     IOReportConfigureAction action,
7886     void                   *result,
7887     void                   *destination)
7888 {
7889         unsigned cnt;
7890
7891         for (cnt = 0; cnt < channelList->nchannels; cnt++) {
7892                 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
7893                         if (pwrMgt) {
7894                                 configurePowerStatesReport(action, result);
7895                         } else {
7896                                 return kIOReturnUnsupported;
7897                         }
7898                 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
7899                         if (pwrMgt) {
7900                                 configureSimplePowerReport(action, result);
7901                         } else {
7902                                 return kIOReturnUnsupported;
7903                         }
7904                 }
7905         }
7906
7907         IOLockLock(reserved->interruptStatisticsLock);
7908
7909         /* The array count is signed (because the interrupt indices are signed), hence the cast */
7910         for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
7911                 if (reserved->interruptStatisticsArray[cnt].reporter) {
7912                         /*
7913                          * If the reporter is currently associated with the statistics
7914                          * for an event source, we may need to update the reporter.
7915                          */
7916                         if (reserved->interruptStatisticsArray[cnt].statistics) {
7917                                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
7918                         }
7919
7920                         reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
7921                 }
7922         }
7923
7924         IOLockUnlock(reserved->interruptStatisticsLock);
7925
7926         return kIOReturnSuccess;
7927 }
7928
7929 IOReturn
7930 IOService::updateReport(IOReportChannelList      *channelList,
7931     IOReportUpdateAction      action,
7932     void                     *result,
7933     void                     *destination)
7934 {
7935         unsigned cnt;
7936
7937         for (cnt = 0; cnt < channelList->nchannels; cnt++) {
7938                 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
7939                         if (pwrMgt) {
7940                                 updatePowerStatesReport(action, result, destination);
7941                         } else {
7942                                 return kIOReturnUnsupported;
7943                         }
7944                 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
7945                         if (pwrMgt) {
7946                                 updateSimplePowerReport(action, result, destination);
7947                         } else {
7948                                 return kIOReturnUnsupported;
7949                         }
7950                 }
7951         }
7952
7953         IOLockLock(reserved->interruptStatisticsLock);
7954
7955         /* The array count is signed (because the interrupt indices are signed), hence the cast */
7956         for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
7957                 if (reserved->interruptStatisticsArray[cnt].reporter) {
7958                         /*
7959                          * If the reporter is currently associated with the statistics
7960                          * for an event source, we need to update the reporter.
7961                          */
7962                         if (reserved->interruptStatisticsArray[cnt].statistics) {
7963                                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
7964                         }
7965
7966                         reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
7967                 }
7968         }
7969
7970         IOLockUnlock(reserved->interruptStatisticsLock);
7971
7972         return kIOReturnSuccess;
7973 }
7974
7975 uint64_t
7976 IOService::getAuthorizationID( void )
7977 {
7978         return reserved->authorizationID;
7979 }
7980
7981 IOReturn
7982 IOService::setAuthorizationID( uint64_t authorizationID )
7983 {
7984         OSObject * entitlement;
7985         IOReturn status;
7986
7987         entitlement = IOUserClient::copyClientEntitlement( current_task(), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
7988
7989         if (entitlement) {
7990                 if (entitlement == kOSBooleanTrue) {
7991                         reserved->authorizationID = authorizationID;
7992
7993                         status = kIOReturnSuccess;
7994                 } else {
7995                         status = kIOReturnNotPrivileged;
7996                 }
7997
7998                 entitlement->release();
7999         } else {
8000                 status = kIOReturnNotPrivileged;
8001         }
8002
8003         return status;
8004 }
8005
8006 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
8007
8008
8009 #if __LP64__
8010 OSMetaClassDefineReservedUsed(IOService, 0);
8011 OSMetaClassDefineReservedUsed(IOService, 1);
8012 OSMetaClassDefineReservedUnused(IOService, 2);
8013 OSMetaClassDefineReservedUnused(IOService, 3);
8014 OSMetaClassDefineReservedUnused(IOService, 4);
8015 OSMetaClassDefineReservedUnused(IOService, 5);
8016 OSMetaClassDefineReservedUnused(IOService, 6);
8017 OSMetaClassDefineReservedUnused(IOService, 7);
8018 #else
8019 OSMetaClassDefineReservedUsed(IOService, 0);
8020 OSMetaClassDefineReservedUsed(IOService, 1);
8021 OSMetaClassDefineReservedUsed(IOService, 2);
8022 OSMetaClassDefineReservedUsed(IOService, 3);
8023 OSMetaClassDefineReservedUsed(IOService, 4);
8024 OSMetaClassDefineReservedUsed(IOService, 5);
8025 OSMetaClassDefineReservedUsed(IOService, 6);
8026 OSMetaClassDefineReservedUsed(IOService, 7);
8027 #endif
8028 OSMetaClassDefineReservedUnused(IOService, 8);
8029 OSMetaClassDefineReservedUnused(IOService, 9);
8030 OSMetaClassDefineReservedUnused(IOService, 10);
8031 OSMetaClassDefineReservedUnused(IOService, 11);
8032 OSMetaClassDefineReservedUnused(IOService, 12);
8033 OSMetaClassDefineReservedUnused(IOService, 13);
8034 OSMetaClassDefineReservedUnused(IOService, 14);
8035 OSMetaClassDefineReservedUnused(IOService, 15);
8036 OSMetaClassDefineReservedUnused(IOService, 16);
8037 OSMetaClassDefineReservedUnused(IOService, 17);
8038 OSMetaClassDefineReservedUnused(IOService, 18);
8039 OSMetaClassDefineReservedUnused(IOService, 19);
8040 OSMetaClassDefineReservedUnused(IOService, 20);
8041 OSMetaClassDefineReservedUnused(IOService, 21);
8042 OSMetaClassDefineReservedUnused(IOService, 22);
8043 OSMetaClassDefineReservedUnused(IOService, 23);
8044 OSMetaClassDefineReservedUnused(IOService, 24);
8045 OSMetaClassDefineReservedUnused(IOService, 25);
8046 OSMetaClassDefineReservedUnused(IOService, 26);
8047 OSMetaClassDefineReservedUnused(IOService, 27);
8048 OSMetaClassDefineReservedUnused(IOService, 28);
8049 OSMetaClassDefineReservedUnused(IOService, 29);
8050 OSMetaClassDefineReservedUnused(IOService, 30);
8051 OSMetaClassDefineReservedUnused(IOService, 31);
8052 OSMetaClassDefineReservedUnused(IOService, 32);
8053 OSMetaClassDefineReservedUnused(IOService, 33);
8054 OSMetaClassDefineReservedUnused(IOService, 34);
8055 OSMetaClassDefineReservedUnused(IOService, 35);
8056 OSMetaClassDefineReservedUnused(IOService, 36);
8057 OSMetaClassDefineReservedUnused(IOService, 37);
8058 OSMetaClassDefineReservedUnused(IOService, 38);
8059 OSMetaClassDefineReservedUnused(IOService, 39);
8060 OSMetaClassDefineReservedUnused(IOService, 40);
8061 OSMetaClassDefineReservedUnused(IOService, 41);
8062 OSMetaClassDefineReservedUnused(IOService, 42);
8063 OSMetaClassDefineReservedUnused(IOService, 43);
8064 OSMetaClassDefineReservedUnused(IOService, 44);
8065 OSMetaClassDefineReservedUnused(IOService, 45);
8066 OSMetaClassDefineReservedUnused(IOService, 46);
8067 OSMetaClassDefineReservedUnused(IOService, 47);