iokit/Kernel/IOService.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/system.h>
  30 #include <IOKit/IOService.h>
  31 #include <libkern/OSDebug.h>
  32 #include <libkern/c++/OSContainers.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/c++/OSUnserialize.h>
  35 #include <libkern/c++/OSKext.h>
  36 #include <libkern/Block.h>
  37 #include <IOKit/IOCatalogue.h>
  38 #include <IOKit/IOCommand.h>
  39 #include <IOKit/IODeviceTreeSupport.h>
  40 #include <IOKit/IODeviceMemory.h>
  41 #include <IOKit/IOInterrupts.h>
  42 #include <IOKit/IOInterruptController.h>
  43 #include <IOKit/IOPlatformExpert.h>
  44 #include <IOKit/IOMessage.h>
  45 #include <IOKit/IOLib.h>
  46 #include <IOKit/IOKitKeysPrivate.h>
  47 #include <IOKit/IOBSD.h>
  48 #include <IOKit/IOUserClient.h>
  49 #include <IOKit/IOUserServer.h>
  50 #include <IOKit/IOWorkLoop.h>
  51 #include <IOKit/IOTimeStamp.h>
  52 #include <IOKit/IOHibernatePrivate.h>
  53 #include <IOKit/IOInterruptAccountingPrivate.h>
  54 #include <IOKit/IOKernelReporters.h>
  55 #include <IOKit/AppleKeyStoreInterface.h>
  56 #include <IOKit/pwr_mgt/RootDomain.h>
  57 #include <IOKit/IOCPU.h>
  58 #include <mach/sync_policy.h>
  59 #include <mach/thread_info.h>
  60 #include <IOKit/assert.h>
  61 #include <sys/errno.h>
  62 #include <sys/kdebug.h>
  63 #include <string.h>
  64
  65 #include <machine/pal_routines.h>
  66
  67 #define LOG kprintf
  68 //#define LOG IOLog
  69 #define MATCH_DEBUG     0
  70 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
  71
  72 // disabled since lockForArbitration() can be held externally
  73 #define DEBUG_NOTIFIER_LOCKED   0
  74
  75 enum{
  76         kIOUserServerCheckInTimeoutSecs = 120ULL
  77 };
  78
  79 #include "IOServicePrivate.h"
  80 #include "IOKitKernelInternal.h"
  81
  82 // take lockForArbitration before LOCKNOTIFY
  83
  84 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  85
  86 #define super IORegistryEntry
  87
  88 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
  89
  90 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
  91 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
  92
  93 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
  94
  95 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
  96
  97 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
  98
  99 OSDefineMetaClassAndStructors(IOResources, IOService)
 100 OSDefineMetaClassAndStructors(IOUserResources, IOService)
 101
 102 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
 103
 104 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
 105
 106 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 107
 108 static IOPlatformExpert *       gIOPlatform;
 109 static class IOPMrootDomain *   gIOPMRootDomain;
 110 const IORegistryPlane *         gIOServicePlane;
 111 const IORegistryPlane *         gIOPowerPlane;
 112 const OSSymbol *                gIODeviceMemoryKey;
 113 const OSSymbol *                gIOInterruptControllersKey;
 114 const OSSymbol *                gIOInterruptSpecifiersKey;
 115
 116 const OSSymbol *                gIOResourcesKey;
 117 const OSSymbol *                gIOUserResourcesKey;
 118 const OSSymbol *                gIOResourceMatchKey;
 119 const OSSymbol *                gIOResourceMatchedKey;
 120 const OSSymbol *                gIOResourceIOKitKey;
 121
 122 const OSSymbol *                gIOProviderClassKey;
 123 const OSSymbol *                gIONameMatchKey;
 124 const OSSymbol *                gIONameMatchedKey;
 125 const OSSymbol *                gIOPropertyMatchKey;
 126 const OSSymbol *                gIOPropertyExistsMatchKey;
 127 const OSSymbol *                gIOLocationMatchKey;
 128 const OSSymbol *                gIOParentMatchKey;
 129 const OSSymbol *                gIOPathMatchKey;
 130 const OSSymbol *                gIOMatchCategoryKey;
 131 const OSSymbol *                gIODefaultMatchCategoryKey;
 132 const OSSymbol *                gIOMatchedServiceCountKey;
 133 const OSSymbol *                gIOMatchedPersonalityKey;
 134 const OSSymbol *                gIORematchPersonalityKey;
 135 const OSSymbol *                gIORematchCountKey;
 136 const OSSymbol *                gIODEXTMatchCountKey;
 137 #if !CONFIG_EMBEDDED
 138 const OSSymbol *                gIOServiceLegacyMatchingRegistryIDKey;
 139 #endif
 140
 141 const OSSymbol *                gIOMapperIDKey;
 142 const OSSymbol *                gIOUserClientClassKey;
 143
 144 const OSSymbol *                gIOUserClassKey;
 145 const OSSymbol *                gIOUserServerClassKey;
 146 const OSSymbol *                gIOUserServerNameKey;
 147 const OSSymbol *                gIOUserServerTagKey;
 148 const OSSymbol *                gIOUserServerCDHashKey;
 149 const OSSymbol *                gIOUserUserClientKey;
 150
 151 const OSSymbol *                gIOKitDebugKey;
 152
 153 const OSSymbol *                gIOCommandPoolSizeKey;
 154
 155 const OSSymbol *                gIOConsoleLockedKey;
 156 const OSSymbol *                gIOConsoleUsersKey;
 157 const OSSymbol *                gIOConsoleSessionUIDKey;
 158 const OSSymbol *                gIOConsoleSessionAuditIDKey;
 159 const OSSymbol *                gIOConsoleUsersSeedKey;
 160 const OSSymbol *                gIOConsoleSessionOnConsoleKey;
 161 const OSSymbol *                gIOConsoleSessionLoginDoneKey;
 162 const OSSymbol *                gIOConsoleSessionSecureInputPIDKey;
 163 const OSSymbol *                gIOConsoleSessionScreenLockedTimeKey;
 164 const OSSymbol *                gIOConsoleSessionScreenIsLockedKey;
 165 clock_sec_t                     gIOConsoleLockTime;
 166 static bool                     gIOConsoleLoggedIn;
 167 #if HIBERNATION
 168 static OSBoolean *              gIOConsoleBooterLockState;
 169 static uint32_t                 gIOScreenLockState;
 170 #endif
 171 static IORegistryEntry *        gIOChosenEntry;
 172
 173 static int                      gIOResourceGenerationCount;
 174
 175 const OSSymbol *                gIOServiceKey;
 176 const OSSymbol *                gIOPublishNotification;
 177 const OSSymbol *                gIOFirstPublishNotification;
 178 const OSSymbol *                gIOMatchedNotification;
 179 const OSSymbol *                gIOFirstMatchNotification;
 180 const OSSymbol *                gIOTerminatedNotification;
 181 const OSSymbol *                gIOWillTerminateNotification;
 182
 183 const OSSymbol *                gIOServiceDEXTEntitlementsKey;
 184 const OSSymbol *                gIODriverKitEntitlementKey;
 185 const OSSymbol *                gIODriverKitUserClientEntitlementsKey;
 186 const OSSymbol *                gIOMatchDeferKey;
 187
 188 const OSSymbol *                gIOGeneralInterest;
 189 const OSSymbol *                gIOBusyInterest;
 190 const OSSymbol *                gIOAppPowerStateInterest;
 191 const OSSymbol *                gIOPriorityPowerStateInterest;
 192 const OSSymbol *                gIOConsoleSecurityInterest;
 193
 194 const OSSymbol *                gIOBSDKey;
 195 const OSSymbol *                gIOBSDNameKey;
 196 const OSSymbol *                gIOBSDMajorKey;
 197 const OSSymbol *                gIOBSDMinorKey;
 198 const OSSymbol *                gIOBSDUnitKey;
 199
 200 const  OSSymbol *               gAKSGetKey;
 201 #if defined(__i386__) || defined(__x86_64__)
 202 const OSSymbol *                gIOCreateEFIDevicePathSymbol;
 203 #endif
 204
 205 static OSDictionary *           gNotifications;
 206 static IORecursiveLock *        gNotificationLock;
 207
 208 static IOService *              gIOResources;
 209 static IOService *              gIOUserResources;
 210 static IOService *              gIOServiceRoot;
 211
 212 static OSOrderedSet *           gJobs;
 213 static semaphore_port_t         gJobsSemaphore;
 214 static IOLock *                 gJobsLock;
 215 static int                      gOutstandingJobs;
 216 static int                      gNumConfigThreads;
 217 static int                      gNumWaitingThreads;
 218 static IOLock *                 gIOServiceBusyLock;
 219 bool                            gCPUsRunning;
 220 bool                                                    gKextdWillTerminate;
 221
 222 static thread_t                 gIOTerminateThread;
 223 static thread_t                 gIOTerminateWorkerThread;
 224 static UInt32                   gIOTerminateWork;
 225 static OSArray *                gIOTerminatePhase2List;
 226 static OSArray *                gIOStopList;
 227 static OSArray *                gIOStopProviderList;
 228 static OSArray *                gIOFinalizeList;
 229
 230 #if !NO_KEXTD
 231 static OSArray *                gIOMatchDeferList;
 232 #endif
 233
 234 static SInt32                   gIOConsoleUsersSeed;
 235 static OSData *                 gIOConsoleUsersSeedValue;
 236
 237 extern const OSSymbol *         gIODTPHandleKey;
 238
 239 const OSSymbol *                gIOPlatformFunctionHandlerSet;
 240
 241
 242 static IOLock *                 gIOConsoleUsersLock;
 243 static thread_call_t            gIOConsoleLockCallout;
 244 static IONotifier *             gIOServiceNullNotifier;
 245
 246 static uint32_t                 gIODextRelaunchMax = 1000;
 247
 248 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 249
 250 #define LOCKREADNOTIFY()        \
 251     IORecursiveLockLock( gNotificationLock )
 252 #define LOCKWRITENOTIFY()       \
 253     IORecursiveLockLock( gNotificationLock )
 254 #define LOCKWRITE2READNOTIFY()
 255 #define UNLOCKNOTIFY()          \
 256     IORecursiveLockUnlock( gNotificationLock )
 257 #define SLEEPNOTIFY(event) \
 258     IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
 259 #define SLEEPNOTIFYTO(event, deadline) \
 260     IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
 261 #define WAKEUPNOTIFY(event) \
 262         IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
 263
 264 #define randomDelay()   \
 265         int del = read_processor_clock();                               \
 266         del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff;      \
 267         IOSleep( del );
 268
 269 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 270
 271 #define queue_element(entry, element, type, field) do { \
 272         vm_address_t __ele = (vm_address_t) (entry);    \
 273         __ele -= -4 + ((size_t)(&((type) 4)->field));   \
 274         (element) = (type) __ele;                       \
 275     } while(0)
 276
 277 #define iterqueue(que, elt)                             \
 278         for (queue_entry_t elt = queue_first(que);      \
 279              !queue_end(que, elt);                      \
 280              elt = queue_next(elt))
 281
 282 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 283
 284 struct IOInterruptAccountingReporter {
 285         IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
 286         IOInterruptAccountingData * statistics; /* The live statistics values, if any */
 287 };
 288
 289 struct ArbitrationLockQueueElement {
 290         queue_chain_t link;
 291         IOThread      thread;
 292         IOService *   service;
 293         unsigned      count;
 294         bool          required;
 295         bool          aborted;
 296 };
 297
 298 static queue_head_t gArbitrationLockQueueActive;
 299 static queue_head_t gArbitrationLockQueueWaiting;
 300 static queue_head_t gArbitrationLockQueueFree;
 301 static IOLock *     gArbitrationLockQueueLock;
 302
 303 bool
 304 IOService::isInactive( void ) const
 305 {
 306         return 0 != (kIOServiceInactiveState & getState());
 307 }
 308
 309 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 310
 311 #if defined(__i386__) || defined(__x86_64__)
 312
 313 // Only used by the intel implementation of
 314 //     IOService::requireMaxBusStall(UInt32 ns)
 315 //     IOService::requireMaxInterruptDelay(uint32_t ns)
 316 struct CpuDelayEntry {
 317         IOService * fService;
 318         UInt32      fMaxDelay;
 319         UInt32      fDelayType;
 320 };
 321
 322 enum {
 323         kCpuDelayBusStall, kCpuDelayInterrupt,
 324         kCpuNumDelayTypes
 325 };
 326
 327 static OSData          *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
 328 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
 329 static OSArray         *sCpuLatencyHandlers[kCpuNumDelayTypes];
 330 const OSSymbol         *sCPULatencyFunctionName[kCpuNumDelayTypes];
 331 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
 332 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
 333 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
 334
 335 static void
 336 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
 337 static IOReturn
 338 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
 339
 340 #endif /* defined(__i386__) || defined(__x86_64__) */
 341
 342 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 343
 344 namespace IOServicePH
 345 {
 346 IONotifier          * fRootNotifier;
 347 OSArray             * fUserServers;
 348 OSArray             * fUserServersWait;
 349 OSArray             * fMatchingWork;
 350 OSArray             * fMatchingDelayed;
 351 IOService           * fSystemPowerAckTo;
 352 uint32_t              fSystemPowerAckRef;
 353 uint8_t               fSystemOff;
 354 uint8_t               fUserServerOff;
 355
 356 void lock();
 357 void unlock();
 358
 359 void init(IOPMrootDomain * root);
 360
 361 IOReturn systemPowerChange(
 362         void * target,
 363         void * refCon,
 364         UInt32 messageType, IOService * service,
 365         void * messageArgument, vm_size_t argSize);
 366
 367 bool matchingStart(IOService * service);
 368 void matchingEnd(IOService * service);
 369 };
 370
 371 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 372
 373 void
 374 IOService::initialize( void )
 375 {
 376         kern_return_t       err;
 377
 378         gIOServicePlane     = IORegistryEntry::makePlane( kIOServicePlane );
 379         gIOPowerPlane       = IORegistryEntry::makePlane( kIOPowerPlane );
 380
 381         gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
 382         gIONameMatchKey     = OSSymbol::withCStringNoCopy( kIONameMatchKey );
 383         gIONameMatchedKey   = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
 384         gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
 385         gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
 386         gIOPathMatchKey     = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
 387         gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
 388         gIOParentMatchKey   = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
 389
 390         gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
 391         gIODefaultMatchCategoryKey  = OSSymbol::withCStringNoCopy(
 392                 kIODefaultMatchCategoryKey );
 393         gIOMatchedServiceCountKey   = OSSymbol::withCStringNoCopy(
 394                 kIOMatchedServiceCountKey );
 395         gIOMatchedPersonalityKey = OSSymbol::withCStringNoCopy(
 396                 kIOMatchedPersonalityKey );
 397         gIORematchPersonalityKey = OSSymbol::withCStringNoCopy(
 398                 kIORematchPersonalityKey );
 399         gIORematchCountKey = OSSymbol::withCStringNoCopy(
 400                 kIORematchCountKey );
 401         gIODEXTMatchCountKey = OSSymbol::withCStringNoCopy(
 402                 kIODEXTMatchCountKey );
 403
 404 #if !CONFIG_EMBEDDED
 405         gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
 406                 kIOServiceLegacyMatchingRegistryIDKey );
 407 #endif
 408
 409         PE_parse_boot_argn("dextrelaunch", &gIODextRelaunchMax, sizeof(gIODextRelaunchMax));
 410
 411         gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
 412
 413         gIOUserClassKey       = OSSymbol::withCStringNoCopy(kIOUserClassKey);
 414
 415         gIOUserServerClassKey  = OSSymbol::withCStringNoCopy(kIOUserServerClassKey);
 416         gIOUserServerNameKey   = OSSymbol::withCStringNoCopy(kIOUserServerNameKey);
 417         gIOUserServerTagKey    = OSSymbol::withCStringNoCopy(kIOUserServerTagKey);
 418         gIOUserServerCDHashKey = OSSymbol::withCStringNoCopy(kIOUserServerCDHashKey);
 419         gIOUserUserClientKey   = OSSymbol::withCStringNoCopy(kIOUserUserClientKey);
 420
 421         gIOResourcesKey       = OSSymbol::withCStringNoCopy( kIOResourcesClass );
 422         gIOResourceMatchKey   = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
 423         gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
 424         gIOResourceIOKitKey   = OSSymbol::withCStringNoCopy("IOKit");
 425
 426         gIODeviceMemoryKey  = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
 427         gIOInterruptControllersKey
 428                 = OSSymbol::withCStringNoCopy("IOInterruptControllers");
 429         gIOInterruptSpecifiersKey
 430                 = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
 431
 432         gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
 433
 434         gIOKitDebugKey      = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
 435
 436         gIOCommandPoolSizeKey       = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
 437
 438         gIOGeneralInterest          = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
 439         gIOBusyInterest             = OSSymbol::withCStringNoCopy( kIOBusyInterest );
 440         gIOAppPowerStateInterest    = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
 441         gIOPriorityPowerStateInterest       = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
 442         gIOConsoleSecurityInterest  = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
 443
 444         gIOBSDKey      = OSSymbol::withCStringNoCopy(kIOBSDKey);
 445         gIOBSDNameKey  = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
 446         gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
 447         gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
 448         gIOBSDUnitKey  = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
 449
 450         gNotifications              = OSDictionary::withCapacity( 1 );
 451         gIOPublishNotification      = OSSymbol::withCStringNoCopy(
 452                 kIOPublishNotification );
 453         gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
 454                 kIOFirstPublishNotification );
 455         gIOMatchedNotification      = OSSymbol::withCStringNoCopy(
 456                 kIOMatchedNotification );
 457         gIOFirstMatchNotification   = OSSymbol::withCStringNoCopy(
 458                 kIOFirstMatchNotification );
 459         gIOTerminatedNotification   = OSSymbol::withCStringNoCopy(
 460                 kIOTerminatedNotification );
 461         gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
 462                 kIOWillTerminateNotification );
 463         gIOServiceKey               = OSSymbol::withCStringNoCopy( kIOServiceClass);
 464
 465
 466         gIOConsoleLockedKey         = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
 467         gIOConsoleUsersKey          = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
 468         gIOConsoleSessionUIDKey     = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
 469         gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
 470
 471         gIOConsoleUsersSeedKey               = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
 472         gIOConsoleSessionOnConsoleKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
 473         gIOConsoleSessionLoginDoneKey        = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
 474         gIOConsoleSessionSecureInputPIDKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
 475         gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
 476         gIOConsoleSessionScreenIsLockedKey   = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
 477
 478         gIOConsoleUsersSeedValue           = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 479
 480         gIOServiceDEXTEntitlementsKey           = OSSymbol::withCStringNoCopy( kIOServiceDEXTEntitlementsKey );
 481         gIODriverKitEntitlementKey             = OSSymbol::withCStringNoCopy( kIODriverKitEntitlementKey );
 482         gIODriverKitUserClientEntitlementsKey   = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementsKey );
 483         gIOMatchDeferKey                        = OSSymbol::withCStringNoCopy( kIOMatchDeferKey );
 484
 485         gIOPlatformFunctionHandlerSet               = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
 486 #if defined(__i386__) || defined(__x86_64__)
 487         sCPULatencyFunctionName[kCpuDelayBusStall]  = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
 488         sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
 489         uint32_t  idx;
 490         for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
 491                 sCPULatencySet[idx]    = OSNumber::withNumber(-1U, 32);
 492                 sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
 493                 assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
 494         }
 495         gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 496 #endif
 497         gNotificationLock           = IORecursiveLockAlloc();
 498
 499         gAKSGetKey                   = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
 500
 501         assert( gIOServicePlane && gIODeviceMemoryKey
 502             && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
 503             && gIOResourcesKey && gNotifications && gNotificationLock
 504             && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
 505             && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
 506             && gIOPublishNotification && gIOMatchedNotification
 507             && gIOTerminatedNotification && gIOServiceKey
 508             && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
 509             && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
 510             && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
 511
 512         gJobsLock   = IOLockAlloc();
 513         gJobs       = OSOrderedSet::withCapacity( 10 );
 514
 515         gIOServiceBusyLock = IOLockAlloc();
 516
 517         gIOConsoleUsersLock = IOLockAlloc();
 518
 519         err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
 520
 521         gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
 522
 523         IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
 524
 525         assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
 526             && gIOConsoleLockCallout && (err == KERN_SUCCESS));
 527
 528         gIOResources = IOResources::resources();
 529         gIOUserResources = IOUserResources::resources();
 530         assert( gIOResources && gIOUserResources );
 531
 532         gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
 533         assert(gIOServiceNullNotifier);
 534
 535         gArbitrationLockQueueLock = IOLockAlloc();
 536         queue_init(&gArbitrationLockQueueActive);
 537         queue_init(&gArbitrationLockQueueWaiting);
 538         queue_init(&gArbitrationLockQueueFree);
 539
 540         assert( gArbitrationLockQueueLock );
 541
 542         gIOTerminatePhase2List = OSArray::withCapacity( 2 );
 543         gIOStopList            = OSArray::withCapacity( 16 );
 544         gIOStopProviderList    = OSArray::withCapacity( 16 );
 545         gIOFinalizeList        = OSArray::withCapacity( 16 );
 546 #if !NO_KEXTD
 547         gIOMatchDeferList      = OSArray::withCapacity( 16 );
 548 #endif
 549         assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
 550
 551         // worker thread that is responsible for terminating / cleaning up threads
 552         kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
 553         assert(gIOTerminateWorkerThread);
 554         thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
 555 }
 556
 557 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 558
 559 #if defined(__i386__) || defined(__x86_64__)
 560 extern "C" {
 561 const char *getCpuDelayBusStallHolderName(void);
 562 const char *
 563 getCpuDelayBusStallHolderName(void)
 564 {
 565         return sCPULatencyHolderName[kCpuDelayBusStall];
 566 }
 567
 568 const char *getCpuInterruptDelayHolderName(void);
 569 const char *
 570 getCpuInterruptDelayHolderName(void)
 571 {
 572         return sCPULatencyHolderName[kCpuDelayInterrupt];
 573 }
 574 }
 575 #endif
 576
 577 #if IOMATCHDEBUG
 578 static UInt64
 579 getDebugFlags( OSDictionary * props )
 580 {
 581         OSNumber *  debugProp;
 582         UInt64      debugFlags;
 583
 584         debugProp = OSDynamicCast( OSNumber,
 585             props->getObject( gIOKitDebugKey ));
 586         if (debugProp) {
 587                 debugFlags = debugProp->unsigned64BitValue();
 588         } else {
 589                 debugFlags = gIOKitDebug;
 590         }
 591
 592         return debugFlags;
 593 }
 594
 595 static UInt64
 596 getDebugFlags( IOService * inst )
 597 {
 598         OSObject *  prop;
 599         OSNumber *  debugProp;
 600         UInt64      debugFlags;
 601
 602         prop = inst->copyProperty(gIOKitDebugKey);
 603         debugProp = OSDynamicCast(OSNumber, prop);
 604         if (debugProp) {
 605                 debugFlags = debugProp->unsigned64BitValue();
 606         } else {
 607                 debugFlags = gIOKitDebug;
 608         }
 609
 610         OSSafeReleaseNULL(prop);
 611
 612         return debugFlags;
 613 }
 614 #endif
 615
 616 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 617
 618 // Probe a matched service and return an instance to be started.
 619 // The default score is from the property table, & may be altered
 620 // during probe to change the start order.
 621
 622 IOService *
 623 IOService::probe(   IOService * provider,
 624     SInt32    * score )
 625 {
 626         return this;
 627 }
 628
 629 bool
 630 IOService::start( IOService * provider )
 631 {
 632         return true;
 633 }
 634
 635 void
 636 IOService::stop( IOService * provider )
 637 {
 638         if (reserved->uvars && reserved->uvars->started && reserved->uvars->userServer) {
 639                 reserved->uvars->userServer->serviceStop(this, provider);
 640         }
 641 }
 642
 643 bool
 644 IOService::init( OSDictionary * dictionary )
 645 {
 646         bool ret;
 647
 648         ret = super::init(dictionary);
 649         if (!ret) {
 650                 return false;
 651         }
 652         if (reserved) {
 653                 return true;
 654         }
 655
 656         reserved = IONew(ExpansionData, 1);
 657         if (!reserved) {
 658                 return false;
 659         }
 660         bzero(reserved, sizeof(*reserved));
 661
 662         /*
 663          * TODO: Improve on this.  Previous efforts to more lazily allocate this
 664          * lock based on the presence of specifiers ran into issues as some
 665          * platforms set up the specifiers after IOService initialization.
 666          *
 667          * We may be able to get away with a global lock, as this should only be
 668          * contended by IOReporting clients and driver start/stop (unless a
 669          * driver wants to remove/add handlers in the course of normal operation,
 670          * which should be unlikely).
 671          */
 672         reserved->interruptStatisticsLock = IOLockAlloc();
 673         if (!reserved->interruptStatisticsLock) {
 674                 return false;
 675         }
 676
 677         return true;
 678 }
 679
 680 bool
 681 IOService::init( IORegistryEntry * from,
 682     const IORegistryPlane * inPlane )
 683 {
 684         bool ret;
 685
 686         ret = super::init(from, inPlane);
 687         if (!ret) {
 688                 return false;
 689         }
 690         if (reserved) {
 691                 return true;
 692         }
 693
 694         reserved = IONew(ExpansionData, 1);
 695         if (!reserved) {
 696                 return false;
 697         }
 698         bzero(reserved, sizeof(*reserved));
 699
 700         /*
 701          * TODO: Improve on this.  Previous efforts to more lazily allocate this
 702          * lock based on the presence of specifiers ran into issues as some
 703          * platforms set up the specifiers after IOService initialization.
 704          *
 705          * We may be able to get away with a global lock, as this should only be
 706          * contended by IOReporting clients and driver start/stop (unless a
 707          * driver wants to remove/add handlers in the course of normal operation,
 708          * which should be unlikely).
 709          */
 710         reserved->interruptStatisticsLock = IOLockAlloc();
 711         if (!reserved->interruptStatisticsLock) {
 712                 return false;
 713         }
 714
 715         return true;
 716 }
 717
 718 void
 719 IOService::free( void )
 720 {
 721         int i = 0;
 722         requireMaxBusStall(0);
 723         requireMaxInterruptDelay(0);
 724         if (getPropertyTable()) {
 725                 unregisterAllInterest();
 726         }
 727         PMfree();
 728
 729         if (reserved) {
 730                 if (reserved->interruptStatisticsArray) {
 731                         for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
 732                                 if (reserved->interruptStatisticsArray[i].reporter) {
 733                                         reserved->interruptStatisticsArray[i].reporter->release();
 734                                 }
 735                         }
 736
 737                         IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
 738                 }
 739
 740                 if (reserved->interruptStatisticsLock) {
 741                         IOLockFree(reserved->interruptStatisticsLock);
 742                 }
 743                 if (reserved->uvars && reserved->uvars->userServer) {
 744                         reserved->uvars->userServer->serviceFree(this);
 745                 }
 746                 IODelete(reserved, ExpansionData, 1);
 747         }
 748
 749         if (_numInterruptSources && _interruptSources) {
 750                 for (i = 0; i < _numInterruptSources; i++) {
 751                         void * block = _interruptSourcesPrivate(this)[i].vectorBlock;
 752                         if (block) {
 753                                 Block_release(block);
 754                         }
 755                 }
 756                 IOFree(_interruptSources,
 757                     _numInterruptSources * sizeofAllIOInterruptSource);
 758                 _interruptSources = NULL;
 759         }
 760
 761         super::free();
 762 }
 763
 764 /*
 765  * Attach in service plane
 766  */
 767 bool
 768 IOService::attach( IOService * provider )
 769 {
 770         bool         ok;
 771         uint32_t     count;
 772         AbsoluteTime deadline;
 773         int          waitResult = THREAD_AWAKENED;
 774         bool         wait, computeDeadline = true;
 775
 776         if (provider) {
 777                 if (gIOKitDebug & kIOLogAttach) {
 778                         LOG( "%s::attach(%s)\n", getName(),
 779                             provider->getName());
 780                 }
 781
 782                 ok   = false;
 783                 do{
 784                         wait = false;
 785                         provider->lockForArbitration();
 786                         if (provider->__state[0] & kIOServiceInactiveState) {
 787                                 ok = false;
 788                         } else {
 789                                 count = provider->getChildCount(gIOServicePlane);
 790                                 wait = (count > (kIOServiceBusyMax - 4));
 791                                 if (!wait) {
 792                                         ok = attachToParent(provider, gIOServicePlane);
 793                                 } else {
 794                                         IOLog("stalling for detach from %s\n", provider->getName());
 795                                         IOLockLock( gIOServiceBusyLock );
 796                                         provider->__state[1] |= kIOServiceWaitDetachState;
 797                                 }
 798                         }
 799                         provider->unlockForArbitration();
 800                         if (wait) {
 801                                 if (computeDeadline) {
 802                                         clock_interval_to_deadline(15, kSecondScale, &deadline);
 803                                         computeDeadline = false;
 804                                 }
 805                                 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
 806                                 IOLockUnlock( gIOServiceBusyLock );
 807                                 waitResult = thread_block(THREAD_CONTINUE_NULL);
 808                                 wait = (waitResult != THREAD_TIMED_OUT);
 809                         }
 810                 }while (wait);
 811         } else {
 812                 gIOServiceRoot = this;
 813                 ok = attachToParent( getRegistryRoot(), gIOServicePlane);
 814         }
 815
 816         if (ok && !__provider) {
 817                 (void) getProvider();
 818         }
 819
 820         return ok;
 821 }
 822
 823 IOService *
 824 IOService::getServiceRoot( void )
 825 {
 826         return gIOServiceRoot;
 827 }
 828
 829 void
 830 IOService::detach( IOService * provider )
 831 {
 832         IOService * newProvider = NULL;
 833         SInt32      busy;
 834         bool        adjParent;
 835
 836         if (gIOKitDebug & kIOLogAttach) {
 837                 LOG("%s::detach(%s)\n", getName(), provider->getName());
 838         }
 839
 840 #if !NO_KEXTD
 841         IOLockLock(gJobsLock);
 842         if (gIOMatchDeferList) {
 843                 auto idx = gIOMatchDeferList->getNextIndexOfObject(this, 0);
 844                 if (-1U != idx) {
 845                         gIOMatchDeferList->removeObject(idx);
 846                 }
 847         }
 848         if (IOServicePH::fMatchingDelayed) {
 849                 auto idx = IOServicePH::fMatchingDelayed->getNextIndexOfObject(this, 0);
 850                 if (-1U != idx) {
 851                         IOServicePH::fMatchingDelayed->removeObject(idx);
 852                 }
 853         }
 854         IOLockUnlock(gJobsLock);
 855 #endif /* NO_KEXTD */
 856
 857         lockForArbitration();
 858
 859         uint64_t regID1 = provider->getRegistryEntryID();
 860         uint64_t regID2 = getRegistryEntryID();
 861         IOServiceTrace(
 862                 IOSERVICE_DETACH,
 863                 (uintptr_t) regID1,
 864                 (uintptr_t) (regID1 >> 32),
 865                 (uintptr_t) regID2,
 866                 (uintptr_t) (regID2 >> 32));
 867
 868         adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
 869             && (provider == getProvider()));
 870
 871         detachFromParent( provider, gIOServicePlane );
 872
 873         if (busy) {
 874                 newProvider = getProvider();
 875                 if (busy && (__state[1] & kIOServiceTermPhase3State) && (NULL == newProvider)) {
 876                         _adjustBusy( -busy );
 877                 }
 878         }
 879
 880         if (kIOServiceInactiveState & __state[0]) {
 881                 getMetaClass()->removeInstance(this);
 882                 IORemoveServicePlatformActions(this);
 883         }
 884
 885         unlockForArbitration();
 886
 887         if (newProvider && adjParent) {
 888                 newProvider->lockForArbitration();
 889                 newProvider->_adjustBusy(1);
 890                 newProvider->unlockForArbitration();
 891         }
 892
 893         // check for last client detach from a terminated service
 894         if (provider->lockForArbitration( true )) {
 895                 if (kIOServiceStartState & __state[1]) {
 896                         provider->scheduleTerminatePhase2();
 897                 }
 898                 if (adjParent) {
 899                         provider->_adjustBusy( -1 );
 900                 }
 901                 if ((provider->__state[1] & kIOServiceTermPhase3State)
 902                     && (NULL == provider->getClient())) {
 903                         provider->scheduleFinalize(false);
 904                 }
 905
 906                 IOLockLock( gIOServiceBusyLock );
 907                 if (kIOServiceWaitDetachState & provider->__state[1]) {
 908                         provider->__state[1] &= ~kIOServiceWaitDetachState;
 909                         thread_wakeup(&provider->__provider);
 910                 }
 911                 IOLockUnlock( gIOServiceBusyLock );
 912
 913                 provider->unlockForArbitration();
 914         }
 915 }
 916
 917 /*
 918  * Register instance - publish it for matching
 919  */
 920
 921 void
 922 IOService::registerService( IOOptionBits options )
 923 {
 924         char *              pathBuf;
 925         const char *        path;
 926         char *              skip;
 927         int                 len;
 928         enum { kMaxPathLen  = 256 };
 929         enum { kMaxChars    = 63 };
 930
 931         IORegistryEntry * parent = this;
 932         IORegistryEntry * root = getRegistryRoot();
 933         while (parent && (parent != root)) {
 934                 parent = parent->getParentEntry( gIOServicePlane);
 935         }
 936
 937         if (parent != root) {
 938                 IOLog("%s: not registry member at registerService()\n", getName());
 939                 return;
 940         }
 941
 942         // Allow the Platform Expert to adjust this node.
 943         if (gIOPlatform && (!gIOPlatform->platformAdjustService(this))) {
 944                 return;
 945         }
 946
 947         IOInstallServicePlatformActions(this);
 948
 949         if ((this != gIOResources)
 950             && (kIOLogRegister & gIOKitDebug)) {
 951                 pathBuf = (char *) IOMalloc( kMaxPathLen );
 952
 953                 IOLog( "Registering: " );
 954
 955                 len = kMaxPathLen;
 956                 if (pathBuf && getPath( pathBuf, &len, gIOServicePlane)) {
 957                         path = pathBuf;
 958                         if (len > kMaxChars) {
 959                                 IOLog("..");
 960                                 len -= kMaxChars;
 961                                 path += len;
 962                                 if ((skip = strchr( path, '/'))) {
 963                                         path = skip;
 964                                 }
 965                         }
 966                 } else {
 967                         path = getName();
 968                 }
 969
 970                 IOLog( "%s\n", path );
 971
 972                 if (pathBuf) {
 973                         IOFree( pathBuf, kMaxPathLen );
 974                 }
 975         }
 976
 977         startMatching( options );
 978 }
 979
 980 void
 981 IOService::startMatching( IOOptionBits options )
 982 {
 983         IOService * provider;
 984         UInt32      prevBusy = 0;
 985         bool        needConfig;
 986         bool        needWake = false;
 987         bool        ok;
 988         bool        sync;
 989         bool        waitAgain;
 990
 991         lockForArbitration();
 992
 993         sync = (options & kIOServiceSynchronous)
 994             || ((provider = getProvider())
 995             && (provider->__state[1] & kIOServiceSynchronousState));
 996
 997         if (options & kIOServiceAsynchronous) {
 998                 sync = false;
 999         }
1000
1001         needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
1002             && (0 == (__state[0] & kIOServiceInactiveState));
1003
1004         __state[1] |= kIOServiceNeedConfigState;
1005
1006 //    __state[0] &= ~kIOServiceInactiveState;
1007
1008 //    if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
1009 //                      OSKernelStackRemaining(), getName());
1010
1011         if (needConfig) {
1012                 needWake = (0 != (kIOServiceSyncPubState & __state[1]));
1013         }
1014
1015         if (sync) {
1016                 __state[1] |= kIOServiceSynchronousState;
1017         } else {
1018                 __state[1] &= ~kIOServiceSynchronousState;
1019         }
1020
1021         if (needConfig) {
1022                 prevBusy = _adjustBusy( 1 );
1023         }
1024
1025         unlockForArbitration();
1026
1027         if (needConfig) {
1028                 if (needWake) {
1029                         IOLockLock( gIOServiceBusyLock );
1030                         thread_wakeup((event_t) this /*&__state[1]*/ );
1031                         IOLockUnlock( gIOServiceBusyLock );
1032                 } else if (!sync || (kIOServiceAsynchronous & options)) {
1033                         ok = (NULL != _IOServiceJob::startJob( this, kMatchNubJob, options ));
1034                 } else {
1035                         do {
1036                                 if ((__state[1] & kIOServiceNeedConfigState)) {
1037                                         doServiceMatch( options );
1038                                 }
1039
1040                                 lockForArbitration();
1041                                 IOLockLock( gIOServiceBusyLock );
1042
1043                                 waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
1044                                     && (0 == (__state[0] & kIOServiceInactiveState)));
1045
1046                                 if (waitAgain) {
1047                                         __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
1048                                 } else {
1049                                         __state[1] &= ~kIOServiceSyncPubState;
1050                                 }
1051
1052                                 unlockForArbitration();
1053
1054                                 if (waitAgain) {
1055                                         assert_wait((event_t) this /*&__state[1]*/, THREAD_UNINT);
1056                                 }
1057
1058                                 IOLockUnlock( gIOServiceBusyLock );
1059                                 if (waitAgain) {
1060                                         thread_block(THREAD_CONTINUE_NULL);
1061                                 }
1062                         } while (waitAgain);
1063                 }
1064         }
1065 }
1066
1067
1068 void
1069 IOService::startDeferredMatches(void)
1070 {
1071 #if !NO_KEXTD
1072         OSArray * array;
1073
1074         IOLockLock(gJobsLock);
1075         array = gIOMatchDeferList;
1076         gIOMatchDeferList = NULL;
1077         IOLockUnlock(gJobsLock);
1078
1079         if (array) {
1080                 IOLog("deferred rematching count %d\n", array->getCount());
1081                 array->iterateObjects(^bool (OSObject * obj)
1082                 {
1083                         ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
1084                         return false;
1085                 });
1086                 array->release();
1087         }
1088 #endif /* !NO_KEXTD */
1089 }
1090
1091 void
1092 IOService::kextdLaunched(void)
1093 {
1094 #if !NO_KEXTD
1095         IOServiceTrace(IOSERVICE_KEXTD_READY, 0, 0, 0, 0);
1096         startDeferredMatches();
1097         getServiceRoot()->adjustBusy(-1);
1098         IOService::publishUserResource(gIOResourceIOKitKey);
1099 #endif /* !NO_KEXTD */
1100 }
1101
1102 IOReturn
1103 IOService::catalogNewDrivers( OSOrderedSet * newTables )
1104 {
1105         OSDictionary *      table;
1106         OSSet *             set;
1107         OSSet *             allSet = NULL;
1108         IOService *         service;
1109 #if IOMATCHDEBUG
1110         SInt32              count = 0;
1111 #endif
1112
1113         newTables->retain();
1114
1115         while ((table = (OSDictionary *) newTables->getFirstObject())) {
1116                 LOCKWRITENOTIFY();
1117                 set = (OSSet *) copyExistingServices( table,
1118                     kIOServiceRegisteredState,
1119                     kIOServiceExistingSet);
1120                 UNLOCKNOTIFY();
1121                 if (set) {
1122 #if IOMATCHDEBUG
1123                         count += set->getCount();
1124 #endif
1125                         if (allSet) {
1126                                 allSet->merge((const OSSet *) set);
1127                                 set->release();
1128                         } else {
1129                                 allSet = set;
1130                         }
1131                 }
1132
1133 #if IOMATCHDEBUG
1134                 if (getDebugFlags( table ) & kIOLogMatch) {
1135                         LOG("Matching service count = %ld\n", (long)count);
1136                 }
1137 #endif
1138                 newTables->removeObject(table);
1139         }
1140
1141         if (allSet) {
1142                 while ((service = (IOService *) allSet->getAnyObject())) {
1143                         service->startMatching(kIOServiceAsynchronous);
1144                         allSet->removeObject(service);
1145                 }
1146                 allSet->release();
1147         }
1148
1149         newTables->release();
1150
1151         return kIOReturnSuccess;
1152 }
1153
1154 _IOServiceJob *
1155 _IOServiceJob::startJob( IOService * nub, int type,
1156     IOOptionBits options )
1157 {
1158         _IOServiceJob *     job;
1159
1160         job = new _IOServiceJob;
1161         if (job && !job->init()) {
1162                 job->release();
1163                 job = NULL;
1164         }
1165
1166         if (job) {
1167                 job->type       = type;
1168                 job->nub        = nub;
1169                 job->options    = options;
1170                 nub->retain();          // thread will release()
1171                 pingConfig( job );
1172         }
1173
1174         return job;
1175 }
1176
1177 /*
1178  * Called on a registered service to see if it matches
1179  * a property table.
1180  */
1181
1182 bool
1183 IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
1184 {
1185         return matchPropertyTable(table);
1186 }
1187
1188 bool
1189 IOService::matchPropertyTable( OSDictionary * table )
1190 {
1191         return true;
1192 }
1193
1194 /*
1195  * Called on a matched service to allocate resources
1196  * before first driver is attached.
1197  */
1198
1199 IOReturn
1200 IOService::getResources( void )
1201 {
1202         return kIOReturnSuccess;
1203 }
1204
1205 /*
1206  * Client/provider accessors
1207  */
1208
1209 IOService *
1210 IOService::getProvider( void ) const
1211 {
1212         IOService * self = (IOService *) this;
1213         IOService * parent;
1214         SInt32      generation;
1215
1216         generation = getRegistryEntryGenerationCount();
1217         if (__providerGeneration == generation) {
1218                 return __provider;
1219         }
1220
1221         parent = (IOService *) getParentEntry( gIOServicePlane);
1222         if (parent == IORegistryEntry::getRegistryRoot()) {
1223                 /* root is not an IOService */
1224                 parent = NULL;
1225         }
1226
1227         self->__provider = parent;
1228         OSMemoryBarrier();
1229         // save the count from before call to getParentEntry()
1230         self->__providerGeneration = generation;
1231
1232         return parent;
1233 }
1234
1235 IOWorkLoop *
1236 IOService::getWorkLoop() const
1237 {
1238         IOService *provider = getProvider();
1239
1240         if (provider) {
1241                 return provider->getWorkLoop();
1242         } else {
1243                 return NULL;
1244         }
1245 }
1246
1247 OSIterator *
1248 IOService::getProviderIterator( void ) const
1249 {
1250         return getParentIterator( gIOServicePlane);
1251 }
1252
1253 IOService *
1254 IOService::getClient( void ) const
1255 {
1256         return (IOService *) getChildEntry( gIOServicePlane);
1257 }
1258
1259 OSIterator *
1260 IOService::getClientIterator( void ) const
1261 {
1262         return getChildIterator( gIOServicePlane);
1263 }
1264
1265 OSIterator *
1266 _IOOpenServiceIterator::iterator( OSIterator * _iter,
1267     const IOService * client,
1268     const IOService * provider )
1269 {
1270         _IOOpenServiceIterator * inst;
1271
1272         if (!_iter) {
1273                 return NULL;
1274         }
1275
1276         inst = new _IOOpenServiceIterator;
1277
1278         if (inst && !inst->init()) {
1279                 inst->release();
1280                 inst = NULL;
1281         }
1282         if (inst) {
1283                 inst->iter = _iter;
1284                 inst->client = client;
1285                 inst->provider = provider;
1286         }
1287
1288         return inst;
1289 }
1290
1291 void
1292 _IOOpenServiceIterator::free()
1293 {
1294         iter->release();
1295         if (last) {
1296                 last->unlockForArbitration();
1297         }
1298         OSIterator::free();
1299 }
1300
1301 OSObject *
1302 _IOOpenServiceIterator::getNextObject()
1303 {
1304         IOService * next;
1305
1306         if (last) {
1307                 last->unlockForArbitration();
1308         }
1309
1310         while ((next = (IOService *) iter->getNextObject())) {
1311                 next->lockForArbitration();
1312                 if ((client && (next->isOpen( client )))
1313                     || (provider && (provider->isOpen( next )))) {
1314                         break;
1315                 }
1316                 next->unlockForArbitration();
1317         }
1318
1319         last = next;
1320
1321         return next;
1322 }
1323
1324 bool
1325 _IOOpenServiceIterator::isValid()
1326 {
1327         return iter->isValid();
1328 }
1329
1330 void
1331 _IOOpenServiceIterator::reset()
1332 {
1333         if (last) {
1334                 last->unlockForArbitration();
1335                 last = NULL;
1336         }
1337         iter->reset();
1338 }
1339
1340 OSIterator *
1341 IOService::getOpenProviderIterator( void ) const
1342 {
1343         return _IOOpenServiceIterator::iterator( getProviderIterator(), this, NULL );
1344 }
1345
1346 OSIterator *
1347 IOService::getOpenClientIterator( void ) const
1348 {
1349         return _IOOpenServiceIterator::iterator( getClientIterator(), NULL, this );
1350 }
1351
1352
1353 IOReturn
1354 IOService::callPlatformFunction( const OSSymbol * functionName,
1355     bool waitForFunction,
1356     void *param1, void *param2,
1357     void *param3, void *param4 )
1358 {
1359         IOReturn  result = kIOReturnUnsupported;
1360         IOService *provider;
1361
1362         if (functionName == gIOPlatformQuiesceActionKey ||
1363             functionName == gIOPlatformActiveActionKey) {
1364                 /*
1365                  * Services which register for IOPlatformQuiesceAction / IOPlatformActiveAction
1366                  * must consume that event themselves, without passing it up to super/IOService.
1367                  */
1368                 if (gEnforceQuiesceSafety) {
1369                         panic("Class %s passed the quiesce/active action to IOService",
1370                             getMetaClass()->getClassName());
1371                 }
1372         }
1373
1374         if (gIOPlatformFunctionHandlerSet == functionName) {
1375 #if defined(__i386__) || defined(__x86_64__)
1376                 const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1377                 IOService *      target              = (IOService *) param2;
1378                 bool             enable              = (param3 != NULL);
1379
1380                 if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) {
1381                         result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1382                 } else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) {
1383                         result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1384                 }
1385 #endif /* defined(__i386__) || defined(__x86_64__) */
1386         }
1387
1388         if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1389                 result = provider->callPlatformFunction(functionName, waitForFunction,
1390                     param1, param2, param3, param4);
1391         }
1392
1393         return result;
1394 }
1395
1396 IOReturn
1397 IOService::callPlatformFunction( const char * functionName,
1398     bool waitForFunction,
1399     void *param1, void *param2,
1400     void *param3, void *param4 )
1401 {
1402         IOReturn result = kIOReturnNoMemory;
1403         const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1404
1405         if (functionSymbol != NULL) {
1406                 result = callPlatformFunction(functionSymbol, waitForFunction,
1407                     param1, param2, param3, param4);
1408                 functionSymbol->release();
1409         }
1410
1411         return result;
1412 }
1413
1414
1415 /*
1416  * Accessors for global services
1417  */
1418
1419 IOPlatformExpert *
1420 IOService::getPlatform( void )
1421 {
1422         return gIOPlatform;
1423 }
1424
1425 class IOPMrootDomain *
1426         IOService::getPMRootDomain( void )
1427 {
1428         return gIOPMRootDomain;
1429 }
1430
1431 IOService *
1432 IOService::getResourceService( void )
1433 {
1434         return gIOResources;
1435 }
1436
1437 void
1438 IOService::setPlatform( IOPlatformExpert * platform)
1439 {
1440         gIOPlatform = platform;
1441         gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1442         gIOUserResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1443
1444 #if defined(__i386__) || defined(__x86_64__)
1445
1446         static const char * keys[kCpuNumDelayTypes] = {
1447                 kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay };
1448         const OSObject * objs[2];
1449         OSArray * array;
1450         uint32_t  idx;
1451
1452         for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
1453                 objs[0] = sCPULatencySet[idx];
1454                 objs[1] = sCPULatencyHolder[idx];
1455                 array   = OSArray::withObjects(objs, 2);
1456                 if (!array) {
1457                         break;
1458                 }
1459                 platform->setProperty(keys[idx], array);
1460                 array->release();
1461         }
1462 #endif /* defined(__i386__) || defined(__x86_64__) */
1463 }
1464
1465 void
1466 IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1467 {
1468         gIOPMRootDomain = rootDomain;
1469         publishResource(gIOResourceIOKitKey);
1470         IOServicePH::init(rootDomain);
1471 }
1472
1473 /*
1474  * Stacking change
1475  */
1476
1477 bool
1478 IOService::lockForArbitration( bool isSuccessRequired )
1479 {
1480         bool                          found;
1481         bool                          success;
1482         ArbitrationLockQueueElement * element;
1483         ArbitrationLockQueueElement * active;
1484         ArbitrationLockQueueElement * waiting;
1485
1486         enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1487
1488         // lock global access
1489         IOTakeLock( gArbitrationLockQueueLock );
1490
1491         // obtain an unused queue element
1492         if (!queue_empty( &gArbitrationLockQueueFree )) {
1493                 queue_remove_first( &gArbitrationLockQueueFree,
1494                     element,
1495                     ArbitrationLockQueueElement *,
1496                     link );
1497         } else {
1498                 element = IONew( ArbitrationLockQueueElement, 1 );
1499                 assert( element );
1500         }
1501
1502         // prepare the queue element
1503         element->thread   = IOThreadSelf();
1504         element->service  = this;
1505         element->count    = 1;
1506         element->required = isSuccessRequired;
1507         element->aborted  = false;
1508
1509         // determine whether this object is already locked (ie. on active queue)
1510         found = false;
1511         queue_iterate( &gArbitrationLockQueueActive,
1512             active,
1513             ArbitrationLockQueueElement *,
1514             link )
1515         {
1516                 if (active->service == element->service) {
1517                         found = true;
1518                         break;
1519                 }
1520         }
1521
1522         if (found) { // this object is already locked
1523                 // determine whether it is the same or a different thread trying to lock
1524                 if (active->thread != element->thread) { // it is a different thread
1525                         ArbitrationLockQueueElement * victim = NULL;
1526
1527                         // before placing this new thread on the waiting queue, we look for
1528                         // a deadlock cycle...
1529
1530                         while (1) {
1531                                 // determine whether the active thread holding the object we
1532                                 // want is waiting for another object to be unlocked
1533                                 found = false;
1534                                 queue_iterate( &gArbitrationLockQueueWaiting,
1535                                     waiting,
1536                                     ArbitrationLockQueueElement *,
1537                                     link )
1538                                 {
1539                                         if (waiting->thread == active->thread) {
1540                                                 assert( false == waiting->aborted );
1541                                                 found = true;
1542                                                 break;
1543                                         }
1544                                 }
1545
1546                                 if (found) { // yes, active thread waiting for another object
1547                                         // this may be a candidate for rejection if the required
1548                                         // flag is not set, should we detect a deadlock later on
1549                                         if (false == waiting->required) {
1550                                                 victim = waiting;
1551                                         }
1552
1553                                         // find the thread that is holding this other object, that
1554                                         // is blocking the active thread from proceeding (fun :-)
1555                                         found = false;
1556                                         queue_iterate( &gArbitrationLockQueueActive,
1557                                             active, // (reuse active queue element)
1558                                             ArbitrationLockQueueElement *,
1559                                             link )
1560                                         {
1561                                                 if (active->service == waiting->service) {
1562                                                         found = true;
1563                                                         break;
1564                                                 }
1565                                         }
1566
1567                                         // someone must be holding it or it wouldn't be waiting
1568                                         assert( found );
1569
1570                                         if (active->thread == element->thread) {
1571                                                 // doh, it's waiting for the thread that originated
1572                                                 // this whole lock (ie. current thread) -> deadlock
1573                                                 if (false == element->required) { // willing to fail?
1574                                                         // the originating thread doesn't have the required
1575                                                         // flag, so it can fail
1576                                                         success = false; // (fail originating lock request)
1577                                                         break; // (out of while)
1578                                                 } else { // originating thread is not willing to fail
1579                                                         // see if we came across a waiting thread that did
1580                                                         // not have the 'required' flag set: we'll fail it
1581                                                         if (victim) {
1582                                                                 // we do have a willing victim, fail it's lock
1583                                                                 victim->aborted = true;
1584
1585                                                                 // take the victim off the waiting queue
1586                                                                 queue_remove( &gArbitrationLockQueueWaiting,
1587                                                                     victim,
1588                                                                     ArbitrationLockQueueElement *,
1589                                                                     link );
1590
1591                                                                 // wake the victim
1592                                                                 IOLockWakeup( gArbitrationLockQueueLock,
1593                                                                     victim,
1594                                                                     /* one thread */ true );
1595
1596                                                                 // allow this thread to proceed (ie. wait)
1597                                                                 success = true; // (put request on wait queue)
1598                                                                 break; // (out of while)
1599                                                         } else {
1600                                                                 // all the waiting threads we came across in
1601                                                                 // finding this loop had the 'required' flag
1602                                                                 // set, so we've got a deadlock we can't avoid
1603                                                                 panic("I/O Kit: Unrecoverable deadlock.");
1604                                                         }
1605                                                 }
1606                                         } else {
1607                                                 // repeat while loop, redefining active thread to be the
1608                                                 // thread holding "this other object" (see above), and
1609                                                 // looking for threads waiting on it; note the active
1610                                                 // variable points to "this other object" already... so
1611                                                 // there nothing to do in this else clause.
1612                                         }
1613                                 } else { // no, active thread is not waiting for another object
1614                                         success = true; // (put request on wait queue)
1615                                         break; // (out of while)
1616                                 }
1617                         } // while forever
1618
1619                         if (success) { // put the request on the waiting queue?
1620                                 kern_return_t wait_result;
1621
1622                                 // place this thread on the waiting queue and put it to sleep;
1623                                 // we place it at the tail of the queue...
1624                                 queue_enter( &gArbitrationLockQueueWaiting,
1625                                     element,
1626                                     ArbitrationLockQueueElement *,
1627                                     link );
1628
1629                                 // declare that this thread will wait for a given event
1630 restart_sleep:                  wait_result = assert_wait( element,
1631                                     element->required ? THREAD_UNINT
1632                                     : THREAD_INTERRUPTIBLE );
1633
1634                                 // unlock global access
1635                                 IOUnlock( gArbitrationLockQueueLock );
1636
1637                                 // put thread to sleep, waiting for our event to fire...
1638                                 if (wait_result == THREAD_WAITING) {
1639                                         wait_result = thread_block(THREAD_CONTINUE_NULL);
1640                                 }
1641
1642
1643                                 // ...and we've been woken up; we might be in one of two states:
1644                                 // (a) we've been aborted and our queue element is not on
1645                                 //     any of the three queues, but is floating around
1646                                 // (b) we're allowed to proceed with the lock and we have
1647                                 //     already been moved from the waiting queue to the
1648                                 //     active queue.
1649                                 // ...plus a 3rd state, should the thread have been interrupted:
1650                                 // (c) we're still on the waiting queue
1651
1652                                 // determine whether we were interrupted out of our sleep
1653                                 if (THREAD_INTERRUPTED == wait_result) {
1654                                         // re-lock global access
1655                                         IOTakeLock( gArbitrationLockQueueLock );
1656
1657                                         // determine whether we're still on the waiting queue
1658                                         found = false;
1659                                         queue_iterate( &gArbitrationLockQueueWaiting,
1660                                             waiting, // (reuse waiting queue element)
1661                                             ArbitrationLockQueueElement *,
1662                                             link )
1663                                         {
1664                                                 if (waiting == element) {
1665                                                         found = true;
1666                                                         break;
1667                                                 }
1668                                         }
1669
1670                                         if (found) { // yes, we're still on the waiting queue
1671                                                 // determine whether we're willing to fail
1672                                                 if (false == element->required) {
1673                                                         // mark us as aborted
1674                                                         element->aborted = true;
1675
1676                                                         // take us off the waiting queue
1677                                                         queue_remove( &gArbitrationLockQueueWaiting,
1678                                                             element,
1679                                                             ArbitrationLockQueueElement *,
1680                                                             link );
1681                                                 } else { // we are not willing to fail
1682                                                         // ignore interruption, go back to sleep
1683                                                         goto restart_sleep;
1684                                                 }
1685                                         }
1686
1687                                         // unlock global access
1688                                         IOUnlock( gArbitrationLockQueueLock );
1689
1690                                         // proceed as though this were a normal wake up
1691                                         wait_result = THREAD_AWAKENED;
1692                                 }
1693
1694                                 assert( THREAD_AWAKENED == wait_result );
1695
1696                                 // determine whether we've been aborted while we were asleep
1697                                 if (element->aborted) {
1698                                         assert( false == element->required );
1699
1700                                         // re-lock global access
1701                                         IOTakeLock( gArbitrationLockQueueLock );
1702
1703                                         action = kPutOnFreeQueue;
1704                                         success = false;
1705                                 } else { // we weren't aborted, so we must be ready to go :-)
1706                                         // we've already been moved from waiting to active queue
1707                                         return true;
1708                                 }
1709                         } else { // the lock request is to be failed
1710                                 // return unused queue element to queue
1711                                 action = kPutOnFreeQueue;
1712                         }
1713                 } else { // it is the same thread, recursive access is allowed
1714                         // add one level of recursion
1715                         active->count++;
1716
1717                         // return unused queue element to queue
1718                         action = kPutOnFreeQueue;
1719                         success = true;
1720                 }
1721         } else { // this object is not already locked, so let this thread through
1722                 action = kPutOnActiveQueue;
1723                 success = true;
1724         }
1725
1726         // put the new element on a queue
1727         if (kPutOnActiveQueue == action) {
1728                 queue_enter( &gArbitrationLockQueueActive,
1729                     element,
1730                     ArbitrationLockQueueElement *,
1731                     link );
1732         } else if (kPutOnFreeQueue == action) {
1733                 queue_enter( &gArbitrationLockQueueFree,
1734                     element,
1735                     ArbitrationLockQueueElement *,
1736                     link );
1737         } else {
1738                 assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1739         }
1740
1741         // unlock global access
1742         IOUnlock( gArbitrationLockQueueLock );
1743
1744         return success;
1745 }
1746
1747 void
1748 IOService::unlockForArbitration( void )
1749 {
1750         bool                          found;
1751         ArbitrationLockQueueElement * element;
1752
1753         // lock global access
1754         IOTakeLock( gArbitrationLockQueueLock );
1755
1756         // find the lock element for this object (ie. on active queue)
1757         found = false;
1758         queue_iterate( &gArbitrationLockQueueActive,
1759             element,
1760             ArbitrationLockQueueElement *,
1761             link )
1762         {
1763                 if (element->service == this) {
1764                         found = true;
1765                         break;
1766                 }
1767         }
1768
1769         assert( found );
1770
1771         // determine whether the lock has been taken recursively
1772         if (element->count > 1) {
1773                 // undo one level of recursion
1774                 element->count--;
1775         } else {
1776                 // remove it from the active queue
1777                 queue_remove( &gArbitrationLockQueueActive,
1778                     element,
1779                     ArbitrationLockQueueElement *,
1780                     link );
1781
1782                 // put it on the free queue
1783                 queue_enter( &gArbitrationLockQueueFree,
1784                     element,
1785                     ArbitrationLockQueueElement *,
1786                     link );
1787
1788                 // determine whether a thread is waiting for object (head to tail scan)
1789                 found = false;
1790                 queue_iterate( &gArbitrationLockQueueWaiting,
1791                     element,
1792                     ArbitrationLockQueueElement *,
1793                     link )
1794                 {
1795                         if (element->service == this) {
1796                                 found = true;
1797                                 break;
1798                         }
1799                 }
1800
1801                 if (found) { // we found an interested thread on waiting queue
1802                         // remove it from the waiting queue
1803                         queue_remove( &gArbitrationLockQueueWaiting,
1804                             element,
1805                             ArbitrationLockQueueElement *,
1806                             link );
1807
1808                         // put it on the active queue
1809                         queue_enter( &gArbitrationLockQueueActive,
1810                             element,
1811                             ArbitrationLockQueueElement *,
1812                             link );
1813
1814                         // wake the waiting thread
1815                         IOLockWakeup( gArbitrationLockQueueLock,
1816                             element,
1817                             /* one thread */ true );
1818                 }
1819         }
1820
1821         // unlock global access
1822         IOUnlock( gArbitrationLockQueueLock );
1823 }
1824
1825 uint32_t
1826 IOService::isLockedForArbitration(IOService * service)
1827 {
1828 #if DEBUG_NOTIFIER_LOCKED
1829         uint32_t                      count;
1830         ArbitrationLockQueueElement * active;
1831
1832         // lock global access
1833         IOLockLock(gArbitrationLockQueueLock);
1834
1835         // determine whether this object is already locked (ie. on active queue)
1836         count = 0;
1837         queue_iterate(&gArbitrationLockQueueActive,
1838             active,
1839             ArbitrationLockQueueElement *,
1840             link)
1841         {
1842                 if ((active->thread == IOThreadSelf())
1843                     && (!service || (active->service == service))) {
1844                         count += 0x10000;
1845                         count += active->count;
1846                 }
1847         }
1848
1849         IOLockUnlock(gArbitrationLockQueueLock);
1850
1851         return count;
1852
1853 #else /* DEBUG_NOTIFIER_LOCKED */
1854
1855         return 0;
1856
1857 #endif /* DEBUG_NOTIFIER_LOCKED */
1858 }
1859
1860 void
1861 IOService::applyToProviders( IOServiceApplierFunction applier,
1862     void * context )
1863 {
1864         applyToParents((IORegistryEntryApplierFunction) applier,
1865             context, gIOServicePlane );
1866 }
1867
1868 void
1869 IOService::applyToClients( IOServiceApplierFunction applier,
1870     void * context )
1871 {
1872         applyToChildren((IORegistryEntryApplierFunction) applier,
1873             context, gIOServicePlane );
1874 }
1875
1876
1877 /*
1878  * Client messages
1879  */
1880
1881
1882 // send a message to a client or interested party of this service
1883 IOReturn
1884 IOService::messageClient( UInt32 type, OSObject * client,
1885     void * argument, vm_size_t argSize )
1886 {
1887         IOReturn                            ret;
1888         IOService *                         service;
1889         _IOServiceInterestNotifier *        notify;
1890
1891         if ((service = OSDynamicCast( IOService, client))) {
1892                 ret = service->message( type, this, argument );
1893         } else if ((notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
1894                 _IOServiceNotifierInvocation invocation;
1895                 bool                     willNotify;
1896
1897                 invocation.thread = current_thread();
1898
1899                 LOCKWRITENOTIFY();
1900                 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
1901
1902                 if (willNotify) {
1903                         queue_enter( &notify->handlerInvocations, &invocation,
1904                             _IOServiceNotifierInvocation *, link );
1905                 }
1906                 UNLOCKNOTIFY();
1907
1908                 if (willNotify) {
1909                         ret = (*notify->handler)( notify->target, notify->ref,
1910                             type, this, argument, argSize );
1911
1912                         LOCKWRITENOTIFY();
1913                         queue_remove( &notify->handlerInvocations, &invocation,
1914                             _IOServiceNotifierInvocation *, link );
1915                         if (kIOServiceNotifyWaiter & notify->state) {
1916                                 notify->state &= ~kIOServiceNotifyWaiter;
1917                                 WAKEUPNOTIFY( notify );
1918                         }
1919                         UNLOCKNOTIFY();
1920                 } else {
1921                         ret = kIOReturnSuccess;
1922                 }
1923         } else {
1924                 ret = kIOReturnBadArgument;
1925         }
1926
1927         return ret;
1928 }
1929
1930 static void
1931 applyToInterestNotifiers(const IORegistryEntry *target,
1932     const OSSymbol * typeOfInterest,
1933     OSObjectApplierFunction applier,
1934     void * context )
1935 {
1936         OSArray *  copyArray = NULL;
1937         OSObject * prop;
1938
1939         LOCKREADNOTIFY();
1940
1941         prop = target->copyProperty(typeOfInterest);
1942         IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
1943
1944         if (notifyList) {
1945                 copyArray = OSArray::withCapacity(1);
1946
1947                 // iterate over queue, entry is set to each element in the list
1948                 iterqueue(&notifyList->fCommandChain, entry) {
1949                         _IOServiceInterestNotifier * notify;
1950
1951                         queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
1952                         copyArray->setObject(notify);
1953                 }
1954         }
1955         UNLOCKNOTIFY();
1956
1957         if (copyArray) {
1958                 unsigned int    index;
1959                 OSObject *      next;
1960
1961                 for (index = 0; (next = copyArray->getObject( index )); index++) {
1962                         (*applier)(next, context);
1963                 }
1964                 copyArray->release();
1965         }
1966
1967         OSSafeReleaseNULL(prop);
1968 }
1969
1970 void
1971 IOService::applyToInterested( const OSSymbol * typeOfInterest,
1972     OSObjectApplierFunction applier,
1973     void * context )
1974 {
1975         if (gIOGeneralInterest == typeOfInterest) {
1976                 applyToClients((IOServiceApplierFunction) applier, context );
1977         }
1978         applyToInterestNotifiers(this, typeOfInterest, applier, context);
1979 }
1980
1981 struct MessageClientsContext {
1982         IOService * service;
1983         UInt32      type;
1984         void *      argument;
1985         vm_size_t   argSize;
1986         IOReturn    ret;
1987 };
1988
1989 static void
1990 messageClientsApplier( OSObject * object, void * ctx )
1991 {
1992         IOReturn                ret;
1993         MessageClientsContext * context = (MessageClientsContext *) ctx;
1994
1995         ret = context->service->messageClient( context->type,
1996             object, context->argument, context->argSize );
1997         if (kIOReturnSuccess != ret) {
1998                 context->ret = ret;
1999         }
2000 }
2001
2002 // send a message to all clients
2003 IOReturn
2004 IOService::messageClients( UInt32 type,
2005     void * argument, vm_size_t argSize )
2006 {
2007         MessageClientsContext       context;
2008
2009         context.service     = this;
2010         context.type        = type;
2011         context.argument    = argument;
2012         context.argSize     = argSize;
2013         context.ret         = kIOReturnSuccess;
2014
2015         applyToInterested( gIOGeneralInterest,
2016             &messageClientsApplier, &context );
2017
2018         return context.ret;
2019 }
2020
2021 IOReturn
2022 IOService::acknowledgeNotification( IONotificationRef notification,
2023     IOOptionBits response )
2024 {
2025         return kIOReturnUnsupported;
2026 }
2027
2028 IONotifier *
2029 IOService::registerInterest( const OSSymbol * typeOfInterest,
2030     IOServiceInterestHandler handler, void * target, void * ref )
2031 {
2032         _IOServiceInterestNotifier * notify = NULL;
2033         IOReturn rc = kIOReturnError;
2034
2035         notify = new _IOServiceInterestNotifier;
2036         if (!notify) {
2037                 return NULL;
2038         }
2039
2040         if (notify->init()) {
2041                 rc = registerInterestForNotifier(notify, typeOfInterest,
2042                     handler, target, ref);
2043         }
2044
2045         if (rc != kIOReturnSuccess) {
2046                 notify->release();
2047                 notify = NULL;
2048         }
2049
2050         return notify;
2051 }
2052
2053
2054
2055 static IOReturn
2056 IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
2057     UInt32 messageType, IOService * provider,
2058     void * messageArgument, vm_size_t argSize )
2059 {
2060         return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
2061 }
2062
2063 IONotifier *
2064 IOService::registerInterest(const OSSymbol * typeOfInterest,
2065     IOServiceInterestHandlerBlock handler)
2066 {
2067         IONotifier * notify;
2068         void       * block;
2069
2070         block = Block_copy(handler);
2071         if (!block) {
2072                 return NULL;
2073         }
2074
2075         notify = registerInterest(typeOfInterest, &IOServiceInterestHandlerToBlock, NULL, block);
2076
2077         if (!notify) {
2078                 Block_release(block);
2079         }
2080
2081         return notify;
2082 }
2083
2084 IOReturn
2085 IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
2086     IOServiceInterestHandler handler, void * target, void * ref )
2087 {
2088         IOReturn rc = kIOReturnSuccess;
2089         _IOServiceInterestNotifier  *notify = NULL;
2090
2091         if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify))) {
2092                 return kIOReturnBadArgument;
2093         }
2094
2095         notify->handler = handler;
2096         notify->target = target;
2097         notify->ref = ref;
2098
2099         if ((typeOfInterest != gIOGeneralInterest)
2100             && (typeOfInterest != gIOBusyInterest)
2101             && (typeOfInterest != gIOAppPowerStateInterest)
2102             && (typeOfInterest != gIOConsoleSecurityInterest)
2103             && (typeOfInterest != gIOPriorityPowerStateInterest)) {
2104                 return kIOReturnBadArgument;
2105         }
2106
2107         lockForArbitration();
2108         if (0 == (__state[0] & kIOServiceInactiveState)) {
2109                 notify->state = kIOServiceNotifyEnable;
2110
2111                 ////// queue
2112
2113                 LOCKWRITENOTIFY();
2114
2115                 // Get the head of the notifier linked list
2116                 IOCommand * notifyList;
2117                 OSObject  * obj = copyProperty( typeOfInterest );
2118                 if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
2119                         notifyList = OSTypeAlloc(IOCommand);
2120                         if (notifyList) {
2121                                 notifyList->init();
2122                                 bool ok = setProperty( typeOfInterest, notifyList);
2123                                 notifyList->release();
2124                                 if (!ok) {
2125                                         notifyList = NULL;
2126                                 }
2127                         }
2128                 }
2129                 if (obj) {
2130                         obj->release();
2131                 }
2132
2133                 if (notifyList) {
2134                         enqueue(&notifyList->fCommandChain, &notify->chain);
2135                         notify->retain(); // ref'ed while in list
2136                 }
2137
2138                 UNLOCKNOTIFY();
2139         } else {
2140                 rc = kIOReturnNotReady;
2141         }
2142         unlockForArbitration();
2143
2144         return rc;
2145 }
2146
2147 static void
2148 cleanInterestList( OSObject * head )
2149 {
2150         IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
2151         if (!notifyHead) {
2152                 return;
2153         }
2154
2155         LOCKWRITENOTIFY();
2156         while (queue_entry_t entry = dequeue(&notifyHead->fCommandChain)) {
2157                 queue_next(entry) = queue_prev(entry) = NULL;
2158
2159                 _IOServiceInterestNotifier * notify;
2160
2161                 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2162                 notify->release();
2163         }
2164         UNLOCKNOTIFY();
2165 }
2166
2167 void
2168 IOService::unregisterAllInterest( void )
2169 {
2170         OSObject * prop;
2171
2172         prop = copyProperty(gIOGeneralInterest);
2173         cleanInterestList(prop);
2174         OSSafeReleaseNULL(prop);
2175
2176         prop = copyProperty(gIOBusyInterest);
2177         cleanInterestList(prop);
2178         OSSafeReleaseNULL(prop);
2179
2180         prop = copyProperty(gIOAppPowerStateInterest);
2181         cleanInterestList(prop);
2182         OSSafeReleaseNULL(prop);
2183
2184         prop = copyProperty(gIOPriorityPowerStateInterest);
2185         cleanInterestList(prop);
2186         OSSafeReleaseNULL(prop);
2187
2188         prop = copyProperty(gIOConsoleSecurityInterest);
2189         cleanInterestList(prop);
2190         OSSafeReleaseNULL(prop);
2191 }
2192
2193 /*
2194  * _IOServiceInterestNotifier
2195  */
2196
2197 // wait for all threads, other than the current one,
2198 //  to exit the handler
2199
2200 void
2201 _IOServiceInterestNotifier::wait()
2202 {
2203         _IOServiceNotifierInvocation * next;
2204         bool doWait;
2205
2206         do {
2207                 doWait = false;
2208                 queue_iterate( &handlerInvocations, next,
2209                     _IOServiceNotifierInvocation *, link) {
2210                         if (next->thread != current_thread()) {
2211                                 doWait = true;
2212                                 break;
2213                         }
2214                 }
2215                 if (doWait) {
2216                         state |= kIOServiceNotifyWaiter;
2217                         SLEEPNOTIFY(this);
2218                 }
2219         } while (doWait);
2220 }
2221
2222 void
2223 _IOServiceInterestNotifier::free()
2224 {
2225         assert( queue_empty( &handlerInvocations ));
2226
2227         if (handler == &IOServiceInterestHandlerToBlock) {
2228                 Block_release(ref);
2229         }
2230
2231         OSObject::free();
2232 }
2233
2234 void
2235 _IOServiceInterestNotifier::remove()
2236 {
2237         LOCKWRITENOTIFY();
2238
2239         if (queue_next( &chain )) {
2240                 remqueue(&chain);
2241                 queue_next( &chain) = queue_prev( &chain) = NULL;
2242                 release();
2243         }
2244
2245         state &= ~kIOServiceNotifyEnable;
2246
2247         wait();
2248
2249         UNLOCKNOTIFY();
2250
2251         release();
2252 }
2253
2254 bool
2255 _IOServiceInterestNotifier::disable()
2256 {
2257         bool        ret;
2258
2259         LOCKWRITENOTIFY();
2260
2261         ret = (0 != (kIOServiceNotifyEnable & state));
2262         state &= ~kIOServiceNotifyEnable;
2263         if (ret) {
2264                 wait();
2265         }
2266
2267         UNLOCKNOTIFY();
2268
2269         return ret;
2270 }
2271
2272 void
2273 _IOServiceInterestNotifier::enable( bool was )
2274 {
2275         LOCKWRITENOTIFY();
2276         if (was) {
2277                 state |= kIOServiceNotifyEnable;
2278         } else {
2279                 state &= ~kIOServiceNotifyEnable;
2280         }
2281         UNLOCKNOTIFY();
2282 }
2283
2284 bool
2285 _IOServiceInterestNotifier::init()
2286 {
2287         queue_init( &handlerInvocations );
2288         return OSObject::init();
2289 }
2290 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2291
2292 /*
2293  * Termination
2294  */
2295
2296 #define tailQ(o)                setObject(o)
2297 #define headQ(o)                setObject(0, o)
2298 #define TLOG(fmt, args...)      { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2299
2300 static void
2301 _workLoopAction( IOWorkLoop::Action action,
2302     IOService * service,
2303     void * p0 = NULL, void * p1 = NULL,
2304     void * p2 = NULL, void * p3 = NULL )
2305 {
2306         IOWorkLoop * wl;
2307
2308         if ((wl = service->getWorkLoop())) {
2309                 wl->retain();
2310                 wl->runAction( action, service, p0, p1, p2, p3 );
2311                 wl->release();
2312         } else {
2313                 (*action)( service, p0, p1, p2, p3 );
2314         }
2315 }
2316
2317 bool
2318 IOService::requestTerminate( IOService * provider, IOOptionBits options )
2319 {
2320         bool ok;
2321
2322         // if its our only provider
2323         ok = isParent( provider, gIOServicePlane, true);
2324
2325         // -- compat
2326         if (ok) {
2327                 provider->terminateClient( this, options | kIOServiceRecursing );
2328                 ok = (0 != (kIOServiceInactiveState & __state[0]));
2329         }
2330         // --
2331
2332         return ok;
2333 }
2334
2335 bool
2336 IOService::terminatePhase1( IOOptionBits options )
2337 {
2338         IOService *  victim;
2339         IOService *  client;
2340         IOService *  rematchProvider;
2341         OSIterator * iter;
2342         OSArray *    makeInactive;
2343         OSArray *    waitingInactive;
2344         IOOptionBits callerOptions;
2345         int          waitResult = THREAD_AWAKENED;
2346         bool         wait;
2347         bool                 ok;
2348         bool                 didInactive;
2349         bool                 startPhase2 = false;
2350
2351         TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2352
2353         callerOptions = options;
2354         rematchProvider = NULL;
2355         uint64_t regID = getRegistryEntryID();
2356         IOServiceTrace(
2357                 IOSERVICE_TERMINATE_PHASE1,
2358                 (uintptr_t) regID,
2359                 (uintptr_t) (regID >> 32),
2360                 (uintptr_t) this,
2361                 (uintptr_t) options);
2362
2363         // -- compat
2364         if (options & kIOServiceRecursing) {
2365                 lockForArbitration();
2366                 if (0 == (kIOServiceInactiveState & __state[0])) {
2367                         __state[0] |= kIOServiceInactiveState;
2368                         __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2369                 }
2370                 unlockForArbitration();
2371
2372                 return true;
2373         }
2374         // --
2375
2376         makeInactive    = OSArray::withCapacity( 16 );
2377         waitingInactive = OSArray::withCapacity( 16 );
2378         if (!makeInactive || !waitingInactive) {
2379                 return false;
2380         }
2381
2382         victim = this;
2383         victim->retain();
2384
2385         while (victim) {
2386                 didInactive = victim->lockForArbitration( true );
2387                 if (didInactive) {
2388                         uint64_t regID1 = victim->getRegistryEntryID();
2389                         IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2390                             (uintptr_t) regID1,
2391                             (uintptr_t) (regID1 >> 32),
2392                             (uintptr_t) victim->__state[1],
2393                             (uintptr_t) 0);
2394
2395                         enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2396                         didInactive = (kRP1 == (victim->__state[1] & kRP1))
2397                             || (0 == (victim->__state[0] & kIOServiceInactiveState));
2398
2399                         if (!didInactive) {
2400                                 // a multiply attached IOService can be visited twice
2401                                 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) {
2402                                         do{
2403                                                 IOLockLock(gIOServiceBusyLock);
2404                                                 wait = (victim->__state[1] & kIOServiceTermPhase1State);
2405                                                 if (wait) {
2406                                                         TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2407                                                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2408                                                         victim->__state[1] |= kIOServiceTerm1WaiterState;
2409                                                         victim->unlockForArbitration();
2410                                                         assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2411                                                 }
2412                                                 IOLockUnlock(gIOServiceBusyLock);
2413                                                 if (wait) {
2414                                                         waitResult = thread_block(THREAD_CONTINUE_NULL);
2415                                                         TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2416                                                             getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2417                                                         victim->lockForArbitration();
2418                                                 }
2419                                         }while (wait && (waitResult != THREAD_TIMED_OUT));
2420                                 }
2421                         } else {
2422                                 victim->__state[0] |= kIOServiceInactiveState;
2423                                 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2424                                     | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2425                                 victim->__state[1] &= ~kIOServiceRecursing;
2426                                 victim->__state[1] |= kIOServiceTermPhase1State;
2427                                 waitingInactive->headQ(victim);
2428                                 if (victim == this) {
2429                                         if (kIOServiceTerminateNeedWillTerminate & options) {
2430                                                 victim->__state[1] |= kIOServiceNeedWillTerminate;
2431                                         }
2432                                 }
2433                                 victim->_adjustBusy( 1 );
2434
2435                                 if ((options & kIOServiceTerminateWithRematch) && (victim == this)) {
2436                                         OSObject     * obj;
2437                                         OSObject     * rematchProps;
2438                                         OSNumber     * num;
2439                                         uint32_t       count;
2440
2441                                         rematchProvider = getProvider();
2442                                         if (rematchProvider) {
2443                                                 obj = rematchProvider->copyProperty(gIORematchCountKey);
2444                                                 num = OSDynamicCast(OSNumber, obj);
2445                                                 count = 0;
2446                                                 if (num) {
2447                                                         count = num->unsigned32BitValue();
2448                                                         count++;
2449                                                 }
2450                                                 num = OSNumber::withNumber(count, 32);
2451                                                 rematchProvider->setProperty(gIORematchCountKey, num);
2452                                                 rematchProps = copyProperty(gIOMatchedPersonalityKey);
2453                                                 rematchProvider->setProperty(gIORematchPersonalityKey, rematchProps);
2454                                                 OSSafeReleaseNULL(num);
2455                                                 OSSafeReleaseNULL(rematchProps);
2456                                                 OSSafeReleaseNULL(obj);
2457                                         }
2458                                 }
2459                         }
2460                         victim->unlockForArbitration();
2461                 }
2462                 if (victim == this) {
2463                         options &= ~kIOServiceTerminateWithRematch;
2464                         startPhase2 = didInactive;
2465                 }
2466                 if (didInactive) {
2467                         OSArray * notifiers;
2468                         notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2469                         victim->invokeNotifiers(&notifiers);
2470
2471                         IOUserClient::destroyUserReferences( victim );
2472
2473                         iter = victim->getClientIterator();
2474                         if (iter) {
2475                                 while ((client = (IOService *) iter->getNextObject())) {
2476                                         TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2477                                             client->getName(), client->getRegistryEntryID(),
2478                                             victim->getName(), victim->getRegistryEntryID(), (long long)options);
2479                                         ok = client->requestTerminate( victim, options );
2480                                         TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2481                                             client->getName(), client->getRegistryEntryID(),
2482                                             victim->getName(), victim->getRegistryEntryID(), ok);
2483
2484                                         uint64_t regID1 = client->getRegistryEntryID();
2485                                         uint64_t regID2 = victim->getRegistryEntryID();
2486                                         IOServiceTrace(
2487                                                 (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2488                                                 : IOSERVICE_TERMINATE_REQUEST_FAIL),
2489                                                 (uintptr_t) regID1,
2490                                                 (uintptr_t) (regID1 >> 32),
2491                                                 (uintptr_t) regID2,
2492                                                 (uintptr_t) (regID2 >> 32));
2493
2494                                         if (ok) {
2495                                                 makeInactive->setObject( client );
2496                                         }
2497                                 }
2498                                 iter->release();
2499                         }
2500                 }
2501                 victim->release();
2502                 victim = (IOService *) makeInactive->getObject(0);
2503                 if (victim) {
2504                         victim->retain();
2505                         makeInactive->removeObject(0);
2506                 }
2507         }
2508         makeInactive->release();
2509
2510         while ((victim = (IOService *) waitingInactive->getObject(0))) {
2511                 victim->retain();
2512                 waitingInactive->removeObject(0);
2513
2514                 victim->lockForArbitration();
2515                 victim->__state[1] &= ~kIOServiceTermPhase1State;
2516                 if (kIOServiceTerm1WaiterState & victim->__state[1]) {
2517                         victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2518                         TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2519                         IOLockLock( gIOServiceBusyLock );
2520                         thread_wakeup((event_t) &victim->__state[1]);
2521                         IOLockUnlock( gIOServiceBusyLock );
2522                 }
2523                 victim->unlockForArbitration();
2524                 victim->release();
2525         }
2526         waitingInactive->release();
2527
2528         if (startPhase2) {
2529                 retain();
2530                 lockForArbitration();
2531                 scheduleTerminatePhase2(options);
2532                 unlockForArbitration();
2533                 release();
2534         }
2535
2536         if (rematchProvider) {
2537                 DKLOG(DKS " rematching after dext crash\n", DKN(rematchProvider));
2538                 rematchProvider->registerService();
2539         }
2540
2541         return true;
2542 }
2543
2544 void
2545 IOService::setTerminateDefer(IOService * provider, bool defer)
2546 {
2547         lockForArbitration();
2548         if (defer) {
2549                 __state[1] |= kIOServiceStartState;
2550         } else {
2551                 __state[1] &= ~kIOServiceStartState;
2552         }
2553         unlockForArbitration();
2554
2555         if (provider && !defer) {
2556                 provider->lockForArbitration();
2557                 provider->scheduleTerminatePhase2();
2558                 provider->unlockForArbitration();
2559         }
2560 }
2561
2562 // Must call this while holding gJobsLock
2563 void
2564 IOService::waitToBecomeTerminateThread(void)
2565 {
2566         IOLockAssert(gJobsLock, kIOLockAssertOwned);
2567         bool wait;
2568         do {
2569                 wait = (gIOTerminateThread != THREAD_NULL);
2570                 if (wait) {
2571                         IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
2572                 }
2573         } while (wait);
2574         gIOTerminateThread = current_thread();
2575 }
2576
2577 // call with lockForArbitration
2578 void
2579 IOService::scheduleTerminatePhase2( IOOptionBits options )
2580 {
2581         AbsoluteTime        deadline;
2582         uint64_t            regID1;
2583         int                 waitResult = THREAD_AWAKENED;
2584         bool                wait = false, haveDeadline = false;
2585
2586         if (!(__state[0] & kIOServiceInactiveState)) {
2587                 return;
2588         }
2589
2590         regID1 = getRegistryEntryID();
2591         IOServiceTrace(
2592                 IOSERVICE_TERM_SCHED_PHASE2,
2593                 (uintptr_t) regID1,
2594                 (uintptr_t) (regID1 >> 32),
2595                 (uintptr_t) __state[1],
2596                 (uintptr_t) options);
2597
2598         if (__state[1] & kIOServiceTermPhase1State) {
2599                 return;
2600         }
2601
2602         retain();
2603         unlockForArbitration();
2604         options |= kIOServiceRequired;
2605         IOLockLock( gJobsLock );
2606
2607         if ((options & kIOServiceSynchronous)
2608             && (current_thread() != gIOTerminateThread)) {
2609                 waitToBecomeTerminateThread();
2610                 gIOTerminatePhase2List->setObject( this );
2611                 gIOTerminateWork++;
2612
2613                 do {
2614                         while (gIOTerminateWork) {
2615                                 terminateWorker( options );
2616                         }
2617                         wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2618                         if (wait) {
2619                                 /* wait for the victim to go non-busy */
2620                                 if (!haveDeadline) {
2621                                         clock_interval_to_deadline( 15, kSecondScale, &deadline );
2622                                         haveDeadline = true;
2623                                 }
2624                                 /* let others do work while we wait */
2625                                 gIOTerminateThread = NULL;
2626                                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2627                                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2628                                     deadline, THREAD_UNINT );
2629                                 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2630                                         IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous): THREAD_TIMED_OUT. "
2631                                             "Attempting to auto-resolve your deadlock. PLEASE FIX!\n", getName(), getRegistryEntryID());
2632                                 }
2633                                 waitToBecomeTerminateThread();
2634                         }
2635                 } while (gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2636
2637                 gIOTerminateThread = NULL;
2638                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2639         } else {
2640                 // ! kIOServiceSynchronous
2641
2642                 gIOTerminatePhase2List->setObject( this );
2643                 if (0 == gIOTerminateWork++) {
2644                         assert(gIOTerminateWorkerThread);
2645                         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2646                 }
2647         }
2648
2649         IOLockUnlock( gJobsLock );
2650         lockForArbitration();
2651         release();
2652 }
2653
2654 __attribute__((__noreturn__))
2655 void
2656 IOService::terminateThread( void * arg, wait_result_t waitResult )
2657 {
2658         // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2659         IOLockLock(gJobsLock);
2660         while (true) {
2661                 if (gIOTerminateThread != gIOTerminateWorkerThread) {
2662                         waitToBecomeTerminateThread();
2663                 }
2664
2665                 while (gIOTerminateWork) {
2666                         terminateWorker((uintptr_t)arg );
2667                 }
2668
2669                 gIOTerminateThread = NULL;
2670                 IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
2671                 IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2672         }
2673 }
2674
2675 void
2676 IOService::scheduleStop( IOService * provider )
2677 {
2678         uint64_t regID1 = getRegistryEntryID();
2679         uint64_t regID2 = provider->getRegistryEntryID();
2680
2681         TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2682         IOServiceTrace(
2683                 IOSERVICE_TERMINATE_SCHEDULE_STOP,
2684                 (uintptr_t) regID1,
2685                 (uintptr_t) (regID1 >> 32),
2686                 (uintptr_t) regID2,
2687                 (uintptr_t) (regID2 >> 32));
2688
2689         IOLockLock( gJobsLock );
2690         gIOStopList->tailQ( this );
2691         gIOStopProviderList->tailQ( provider );
2692
2693         if (0 == gIOTerminateWork++) {
2694                 assert(gIOTerminateWorkerThread);
2695                 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2696         }
2697
2698         IOLockUnlock( gJobsLock );
2699 }
2700
2701 void
2702 IOService::scheduleFinalize(bool now)
2703 {
2704         uint64_t regID1 = getRegistryEntryID();
2705
2706         TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2707         IOServiceTrace(
2708                 IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2709                 (uintptr_t) regID1,
2710                 (uintptr_t) (regID1 >> 32),
2711                 0, 0);
2712
2713         if (now || IOUserClient::finalizeUserReferences(this)) {
2714                 IOLockLock( gJobsLock );
2715                 gIOFinalizeList->tailQ(this);
2716                 if (0 == gIOTerminateWork++) {
2717                         assert(gIOTerminateWorkerThread);
2718                         IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2719                 }
2720                 IOLockUnlock( gJobsLock );
2721         }
2722 }
2723
2724 bool
2725 IOService::willTerminate( IOService * provider, IOOptionBits options )
2726 {
2727         if (reserved->uvars) {
2728                 IOUserServer::serviceWillTerminate(this, provider, options);
2729         }
2730         return true;
2731 }
2732
2733 bool
2734 IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2735 {
2736         if (reserved->uvars) {
2737                 IOUserServer::serviceDidTerminate(this, provider, options, defer);
2738         }
2739
2740         if (false == *defer) {
2741                 if (lockForArbitration( true )) {
2742                         if (false == provider->handleIsOpen( this )) {
2743                                 scheduleStop( provider );
2744                         }
2745                         // -- compat
2746                         else {
2747                                 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2748                                 if (false == provider->handleIsOpen( this )) {
2749                                         scheduleStop( provider );
2750                                 }
2751                         }
2752                         // --
2753                         unlockForArbitration();
2754                 }
2755         }
2756
2757         return true;
2758 }
2759
2760 void
2761 IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2762     OSArray * doPhase2List,
2763     bool user,
2764     void *unused3 __unused)
2765 {
2766         OSIterator * iter;
2767         IOService *  client;
2768         bool         ok;
2769         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2770
2771         iter = victim->getClientIterator();
2772         if (iter) {
2773                 while ((client = (IOService *) iter->getNextObject())) {
2774                         if (user != (NULL != client->reserved->uvars)) {
2775                                 continue;
2776                         }
2777                         regID1 = client->getRegistryEntryID();
2778                         TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2779                             client->getName(), regID1,
2780                             victim->getName(), regID2, (long long)options);
2781                         IOServiceTrace(
2782                                 IOSERVICE_TERMINATE_WILL,
2783                                 (uintptr_t) regID1,
2784                                 (uintptr_t) (regID1 >> 32),
2785                                 (uintptr_t) regID2,
2786                                 (uintptr_t) (regID2 >> 32));
2787
2788                         ok = client->willTerminate( victim, options );
2789                         doPhase2List->tailQ( client );
2790                 }
2791                 iter->release();
2792         }
2793 }
2794
2795 void
2796 IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
2797     void *unused1 __unused, void *unused2 __unused,
2798     void *unused3 __unused )
2799 {
2800         OSIterator * iter;
2801         IOService *  client;
2802         bool         defer;
2803         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2804
2805         victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
2806
2807         iter = victim->getClientIterator();
2808         if (iter) {
2809                 while ((client = (IOService *) iter->getNextObject())) {
2810                         regID1 = client->getRegistryEntryID();
2811                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2812                             client->getName(), regID1,
2813                             victim->getName(), regID2, (long long)options);
2814                         defer = false;
2815                         client->didTerminate( victim, options, &defer );
2816
2817                         IOServiceTrace(
2818                                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2819                                 : IOSERVICE_TERMINATE_DID),
2820                                 (uintptr_t) regID1,
2821                                 (uintptr_t) (regID1 >> 32),
2822                                 (uintptr_t) regID2,
2823                                 (uintptr_t) (regID2 >> 32));
2824
2825                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2826                             client->getName(), regID1,
2827                             victim->getName(), regID2, defer);
2828                 }
2829                 iter->release();
2830         }
2831 }
2832
2833
2834 void
2835 IOService::actionWillStop( IOService * victim, IOOptionBits options,
2836     void *unused1 __unused, void *unused2 __unused,
2837     void *unused3 __unused )
2838 {
2839         OSIterator * iter;
2840         IOService *  provider;
2841         bool         ok;
2842         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2843
2844         iter = victim->getProviderIterator();
2845         if (iter) {
2846                 while ((provider = (IOService *) iter->getNextObject())) {
2847                         regID1 = provider->getRegistryEntryID();
2848                         TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2849                             victim->getName(), regID2,
2850                             provider->getName(), regID1, (long long)options);
2851                         IOServiceTrace(
2852                                 IOSERVICE_TERMINATE_WILL,
2853                                 (uintptr_t) regID2,
2854                                 (uintptr_t) (regID2 >> 32),
2855                                 (uintptr_t) regID1,
2856                                 (uintptr_t) (regID1 >> 32));
2857
2858                         ok = victim->willTerminate( provider, options );
2859                 }
2860                 iter->release();
2861         }
2862 }
2863
2864 void
2865 IOService::actionDidStop( IOService * victim, IOOptionBits options,
2866     void *unused1 __unused, void *unused2 __unused,
2867     void *unused3 __unused )
2868 {
2869         OSIterator * iter;
2870         IOService *  provider;
2871         bool defer = false;
2872         uint64_t     regID1, regID2 = victim->getRegistryEntryID();
2873
2874         iter = victim->getProviderIterator();
2875         if (iter) {
2876                 while ((provider = (IOService *) iter->getNextObject())) {
2877                         regID1 = provider->getRegistryEntryID();
2878                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
2879                             victim->getName(), regID2,
2880                             provider->getName(), regID1, (long long)options);
2881                         victim->didTerminate( provider, options, &defer );
2882
2883                         IOServiceTrace(
2884                                 (defer ? IOSERVICE_TERMINATE_DID_DEFER
2885                                 : IOSERVICE_TERMINATE_DID),
2886                                 (uintptr_t) regID2,
2887                                 (uintptr_t) (regID2 >> 32),
2888                                 (uintptr_t) regID1,
2889                                 (uintptr_t) (regID1 >> 32));
2890
2891                         TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
2892                             victim->getName(), regID2,
2893                             provider->getName(), regID1, defer);
2894                 }
2895                 iter->release();
2896         }
2897 }
2898
2899
2900 void
2901 IOService::actionFinalize( IOService * victim, IOOptionBits options,
2902     void *unused1 __unused, void *unused2 __unused,
2903     void *unused3 __unused )
2904 {
2905         uint64_t regID1 = victim->getRegistryEntryID();
2906         TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options);
2907         IOServiceTrace(
2908                 IOSERVICE_TERMINATE_FINALIZE,
2909                 (uintptr_t) regID1,
2910                 (uintptr_t) (regID1 >> 32),
2911                 0, 0);
2912
2913         victim->finalize( options );
2914 }
2915
2916 void
2917 IOService::actionStop( IOService * provider, IOService * client,
2918     void *unused1 __unused, void *unused2 __unused,
2919     void *unused3 __unused )
2920 {
2921         uint64_t regID1 = provider->getRegistryEntryID();
2922         uint64_t regID2 = client->getRegistryEntryID();
2923
2924         TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2925         IOServiceTrace(
2926                 IOSERVICE_TERMINATE_STOP,
2927                 (uintptr_t) regID1,
2928                 (uintptr_t) (regID1 >> 32),
2929                 (uintptr_t) regID2,
2930                 (uintptr_t) (regID2 >> 32));
2931
2932         client->stop( provider );
2933         if (provider->isOpen( client )) {
2934                 provider->close( client );
2935         }
2936
2937         TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
2938         client->detach( provider );
2939 }
2940
2941 void
2942 IOService::terminateWorker( IOOptionBits options )
2943 {
2944         OSArray *           doPhase2List;
2945         OSArray *           didPhase2List;
2946         OSSet *             freeList;
2947         OSIterator *        iter;
2948         UInt32              workDone;
2949         IOService *         victim;
2950         IOService *         client;
2951         IOService *         provider;
2952         unsigned int        idx;
2953         bool                moreToDo;
2954         bool                doPhase2;
2955         bool                doPhase3;
2956
2957         options |= kIOServiceRequired;
2958
2959         doPhase2List  = OSArray::withCapacity( 16 );
2960         didPhase2List = OSArray::withCapacity( 16 );
2961         freeList      = OSSet::withCapacity( 16 );
2962         if ((NULL == doPhase2List) || (NULL == didPhase2List) || (NULL == freeList)) {
2963                 return;
2964         }
2965
2966         do {
2967                 workDone = gIOTerminateWork;
2968
2969                 while ((victim = (IOService *) gIOTerminatePhase2List->getObject(0))) {
2970                         victim->retain();
2971                         gIOTerminatePhase2List->removeObject(0);
2972                         IOLockUnlock( gJobsLock );
2973
2974                         uint64_t regID1 = victim->getRegistryEntryID();
2975                         IOServiceTrace(
2976                                 IOSERVICE_TERM_START_PHASE2,
2977                                 (uintptr_t) regID1,
2978                                 (uintptr_t) (regID1 >> 32),
2979                                 (uintptr_t) 0,
2980                                 (uintptr_t) 0);
2981
2982                         while (victim) {
2983                                 doPhase2 = victim->lockForArbitration( true );
2984                                 if (doPhase2) {
2985                                         doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
2986                                         if (doPhase2) {
2987                                                 uint64_t regID1 = victim->getRegistryEntryID();
2988                                                 IOServiceTrace(
2989                                                         IOSERVICE_TERM_TRY_PHASE2,
2990                                                         (uintptr_t) regID1,
2991                                                         (uintptr_t) (regID1 >> 32),
2992                                                         (uintptr_t) victim->__state[1],
2993                                                         (uintptr_t) 0);
2994
2995                                                 doPhase2 = (0 == (victim->__state[1] &
2996                                                     (kIOServiceTermPhase1State
2997                                                     | kIOServiceTermPhase2State
2998                                                     | kIOServiceConfigState)));
2999
3000                                                 if (doPhase2 && (iter = victim->getClientIterator())) {
3001                                                         while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
3002                                                                 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
3003                                                                 if (!doPhase2) {
3004                                                                         uint64_t regID1 = client->getRegistryEntryID();
3005                                                                         IOServiceTrace(
3006                                                                                 IOSERVICE_TERM_UC_DEFER,
3007                                                                                 (uintptr_t) regID1,
3008                                                                                 (uintptr_t) (regID1 >> 32),
3009                                                                                 (uintptr_t) client->__state[1],
3010                                                                                 (uintptr_t) 0);
3011                                                                         TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
3012                                                                             victim->getName(), victim->getRegistryEntryID(),
3013                                                                             client->getName(), client->getRegistryEntryID());
3014                                                                 }
3015                                                         }
3016                                                         iter->release();
3017                                                 }
3018                                                 if (doPhase2) {
3019                                                         victim->__state[1] |= kIOServiceTermPhase2State;
3020                                                 }
3021                                         }
3022                                         victim->unlockForArbitration();
3023                                 }
3024                                 if (doPhase2) {
3025                                         if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3026                                                 _workLoopAction((IOWorkLoop::Action) &actionWillStop,
3027                                                     victim, (void *)(uintptr_t) options, NULL );
3028                                         }
3029
3030                                         OSArray * notifiers;
3031                                         notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
3032                                         victim->invokeNotifiers(&notifiers);
3033
3034                                         _workLoopAction((IOWorkLoop::Action) &actionWillTerminate,
3035                                             victim,
3036                                             (void *)(uintptr_t) options,
3037                                             (void *)(uintptr_t) doPhase2List,
3038                                             (void *)(uintptr_t) false);
3039
3040                                         actionWillTerminate(
3041                                                 victim, options, doPhase2List, true, NULL);
3042
3043                                         didPhase2List->headQ( victim );
3044                                 }
3045                                 victim->release();
3046                                 victim = (IOService *) doPhase2List->getObject(0);
3047                                 if (victim) {
3048                                         victim->retain();
3049                                         doPhase2List->removeObject(0);
3050                                 }
3051                         }
3052
3053                         while ((victim = (IOService *) didPhase2List->getObject(0))) {
3054                                 bool scheduleFinalize = false;
3055                                 if (victim->lockForArbitration( true )) {
3056                                         victim->__state[1] |= kIOServiceTermPhase3State;
3057                                         scheduleFinalize = (NULL == victim->getClient());
3058                                         victim->unlockForArbitration();
3059                                 }
3060                                 _workLoopAction((IOWorkLoop::Action) &actionDidTerminate,
3061                                     victim, (void *)(uintptr_t) options );
3062                                 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3063                                         _workLoopAction((IOWorkLoop::Action) &actionDidStop,
3064                                             victim, (void *)(uintptr_t) options, NULL );
3065                                 }
3066                                 // no clients - will go to finalize
3067                                 if (scheduleFinalize) {
3068                                         victim->scheduleFinalize(false);
3069                                 }
3070                                 didPhase2List->removeObject(0);
3071                         }
3072                         IOLockLock( gJobsLock );
3073                 }
3074
3075                 // phase 3
3076                 do {
3077                         doPhase3 = false;
3078                         // finalize leaves
3079                         while ((victim = (IOService *) gIOFinalizeList->getObject(0))) {
3080                                 bool sendFinal = false;
3081                                 IOLockUnlock( gJobsLock );
3082                                 if (victim->lockForArbitration(true)) {
3083                                         sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
3084                                         if (sendFinal) {
3085                                                 victim->__state[1] |= kIOServiceFinalized;
3086                                         }
3087                                         victim->unlockForArbitration();
3088                                 }
3089                                 if (sendFinal) {
3090                                         _workLoopAction((IOWorkLoop::Action) &actionFinalize,
3091                                             victim, (void *)(uintptr_t) options );
3092                                 }
3093                                 IOLockLock( gJobsLock );
3094                                 // hold off free
3095                                 freeList->setObject( victim );
3096                                 // safe if finalize list is append only
3097                                 gIOFinalizeList->removeObject(0);
3098                         }
3099
3100                         for (idx = 0;
3101                             (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx));) {
3102                                 provider = (IOService *) gIOStopProviderList->getObject(idx);
3103                                 assert( provider );
3104
3105                                 uint64_t regID1 = provider->getRegistryEntryID();
3106                                 uint64_t regID2 = client->getRegistryEntryID();
3107
3108                                 if (!provider->isChild( client, gIOServicePlane )) {
3109                                         // may be multiply queued - nop it
3110                                         TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3111                                         IOServiceTrace(
3112                                                 IOSERVICE_TERMINATE_STOP_NOP,
3113                                                 (uintptr_t) regID1,
3114                                                 (uintptr_t) (regID1 >> 32),
3115                                                 (uintptr_t) regID2,
3116                                                 (uintptr_t) (regID2 >> 32));
3117                                 } else {
3118                                         // a terminated client is not ready for stop if it has clients, skip it
3119                                         bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
3120                                         IOLockUnlock( gJobsLock );
3121                                         if (deferStop && client->lockForArbitration(true)) {
3122                                                 deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
3123                                                 //deferStop = (!deferStop && (0 != client->getClient()));
3124                                                 //deferStop = (0 != client->getClient());
3125                                                 client->unlockForArbitration();
3126                                                 if (deferStop) {
3127                                                         TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
3128                                                         IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
3129                                                             (uintptr_t) regID1,
3130                                                             (uintptr_t) (regID1 >> 32),
3131                                                             (uintptr_t) regID2,
3132                                                             (uintptr_t) (regID2 >> 32));
3133
3134                                                         idx++;
3135                                                         IOLockLock( gJobsLock );
3136                                                         continue;
3137                                                 }
3138                                         }
3139                                         _workLoopAction((IOWorkLoop::Action) &actionStop,
3140                                             provider, (void *) client );
3141                                         IOLockLock( gJobsLock );
3142                                         // check the finalize list now
3143                                         doPhase3 = true;
3144                                 }
3145                                 // hold off free
3146                                 freeList->setObject( client );
3147                                 freeList->setObject( provider );
3148
3149                                 // safe if stop list is append only
3150                                 gIOStopList->removeObject( idx );
3151                                 gIOStopProviderList->removeObject( idx );
3152                                 idx = 0;
3153                         }
3154                 } while (doPhase3);
3155
3156                 gIOTerminateWork -= workDone;
3157                 moreToDo = (gIOTerminateWork != 0);
3158
3159                 if (!moreToDo) {
3160                         TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
3161                         IOServiceTrace(
3162                                 IOSERVICE_TERMINATE_DONE,
3163                                 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
3164                 }
3165         } while (moreToDo);
3166
3167         IOLockUnlock( gJobsLock );
3168
3169         freeList->release();
3170         doPhase2List->release();
3171         didPhase2List->release();
3172
3173         IOLockLock( gJobsLock );
3174 }
3175
3176 bool
3177 IOService::finalize( IOOptionBits options )
3178 {
3179         OSIterator *  iter;
3180         IOService *   provider;
3181         uint64_t      regID1, regID2 = getRegistryEntryID();
3182
3183         iter = getProviderIterator();
3184         assert( iter );
3185
3186         if (iter) {
3187                 while ((provider = (IOService *) iter->getNextObject())) {
3188                         // -- compat
3189                         if (0 == (__state[1] & kIOServiceTermPhase3State)) {
3190                                 /* we come down here on programmatic terminate */
3191
3192                                 regID1 = provider->getRegistryEntryID();
3193                                 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
3194                                 IOServiceTrace(
3195                                         IOSERVICE_TERMINATE_STOP,
3196                                         (uintptr_t) regID1,
3197                                         (uintptr_t) (regID1 >> 32),
3198                                         (uintptr_t) regID2,
3199                                         (uintptr_t) (regID2 >> 32));
3200
3201                                 stop( provider );
3202                                 if (provider->isOpen( this )) {
3203                                         provider->close( this );
3204                                 }
3205                                 detach( provider );
3206                         } else {
3207                                 //--
3208                                 if (provider->lockForArbitration( true )) {
3209                                         if (0 == (provider->__state[1] & kIOServiceTermPhase3State)) {
3210                                                 scheduleStop( provider );
3211                                         }
3212                                         provider->unlockForArbitration();
3213                                 }
3214                         }
3215                 }
3216                 iter->release();
3217         }
3218
3219         return true;
3220 }
3221
3222 #undef tailQ
3223 #undef headQ
3224
3225 /*
3226  * Terminate
3227  */
3228
3229 void
3230 IOService::doServiceTerminate( IOOptionBits options )
3231 {
3232 }
3233
3234 // a method in case someone needs to override it
3235 bool
3236 IOService::terminateClient( IOService * client, IOOptionBits options )
3237 {
3238         bool ok;
3239
3240         if (client->isParent( this, gIOServicePlane, true)) {
3241                 // we are the clients only provider
3242                 ok = client->terminate( options );
3243         } else {
3244                 ok = true;
3245         }
3246
3247         return ok;
3248 }
3249
3250 bool
3251 IOService::terminate( IOOptionBits options )
3252 {
3253         options |= kIOServiceTerminate;
3254
3255         return terminatePhase1( options );
3256 }
3257
3258 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3259
3260 /*
3261  * Open & close
3262  */
3263
3264 struct ServiceOpenMessageContext {
3265         IOService *  service;
3266         UInt32       type;
3267         IOService *  excludeClient;
3268         IOOptionBits options;
3269 };
3270
3271 static void
3272 serviceOpenMessageApplier( OSObject * object, void * ctx )
3273 {
3274         ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
3275
3276         if (object != context->excludeClient) {
3277                 context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
3278         }
3279 }
3280
3281 bool
3282 IOService::open(   IOService *     forClient,
3283     IOOptionBits    options,
3284     void *          arg )
3285 {
3286         bool                        ok;
3287         ServiceOpenMessageContext   context;
3288
3289         context.service             = this;
3290         context.type                = kIOMessageServiceIsAttemptingOpen;
3291         context.excludeClient       = forClient;
3292         context.options             = options;
3293
3294         applyToInterested( gIOGeneralInterest,
3295             &serviceOpenMessageApplier, &context );
3296
3297         if (false == lockForArbitration(false)) {
3298                 return false;
3299         }
3300
3301         ok = (0 == (__state[0] & kIOServiceInactiveState));
3302         if (ok) {
3303                 ok = handleOpen( forClient, options, arg );
3304         }
3305
3306         if (ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3307                 forClient->reserved->uvars->userServer->serviceOpen(this, forClient);
3308         }
3309
3310         unlockForArbitration();
3311
3312         return ok;
3313 }
3314
3315 void
3316 IOService::close(  IOService *     forClient,
3317     IOOptionBits    options )
3318 {
3319         bool                wasClosed;
3320         bool                last = false;
3321
3322         lockForArbitration();
3323
3324         wasClosed = handleIsOpen( forClient );
3325         if (wasClosed) {
3326                 handleClose( forClient, options );
3327                 last = (__state[1] & kIOServiceTermPhase3State);
3328
3329                 if (forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3330                         forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3331                 }
3332         }
3333
3334         unlockForArbitration();
3335
3336         if (last) {
3337                 forClient->scheduleStop( this );
3338         } else if (wasClosed) {
3339                 ServiceOpenMessageContext context;
3340
3341                 context.service         = this;
3342                 context.type            = kIOMessageServiceWasClosed;
3343                 context.excludeClient   = forClient;
3344                 context.options         = options;
3345
3346                 applyToInterested( gIOGeneralInterest,
3347                     &serviceOpenMessageApplier, &context );
3348         }
3349 }
3350
3351 bool
3352 IOService::isOpen( const IOService * forClient ) const
3353 {
3354         IOService * self = (IOService *) this;
3355         bool ok;
3356
3357         self->lockForArbitration();
3358
3359         ok = handleIsOpen( forClient );
3360
3361         self->unlockForArbitration();
3362
3363         return ok;
3364 }
3365
3366 bool
3367 IOService::handleOpen(     IOService *     forClient,
3368     IOOptionBits    options,
3369     void *          arg )
3370 {
3371         bool        ok;
3372
3373         ok = (NULL == __owner);
3374         if (ok) {
3375                 __owner = forClient;
3376         } else if (options & kIOServiceSeize) {
3377                 ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3378                     __owner, (void *)(uintptr_t) options ));
3379                 if (ok && (NULL == __owner)) {
3380                         __owner = forClient;
3381                 } else {
3382                         ok = false;
3383                 }
3384         }
3385         return ok;
3386 }
3387
3388 void
3389 IOService::handleClose(    IOService *     forClient,
3390     IOOptionBits    options )
3391 {
3392         if (__owner == forClient) {
3393                 __owner = NULL;
3394         }
3395 }
3396
3397 bool
3398 IOService::handleIsOpen(   const IOService * forClient ) const
3399 {
3400         if (forClient) {
3401                 return __owner == forClient;
3402         } else {
3403                 return __owner != forClient;
3404         }
3405 }
3406
3407 /*
3408  * Probing & starting
3409  */
3410 static SInt32
3411 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3412 {
3413         const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3414         const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3415         SInt32             val1;
3416         SInt32             val2;
3417
3418         val1 = 0;
3419         val2 = 0;
3420
3421         if (obj1) {
3422                 val1 = obj1->priority;
3423         }
3424
3425         if (obj2) {
3426                 val2 = obj2->priority;
3427         }
3428
3429         return val1 - val2;
3430 }
3431
3432 static SInt32
3433 IOServiceObjectOrder( const OSObject * entry, void * ref)
3434 {
3435         OSDictionary *      dict;
3436         IOService *         service;
3437         _IOServiceNotifier * notify;
3438         OSSymbol *          key = (OSSymbol *) ref;
3439         OSNumber *          offset;
3440         OSObject *          prop;
3441         SInt32              result;
3442
3443         prop = NULL;
3444         result = kIODefaultProbeScore;
3445         if ((dict = OSDynamicCast( OSDictionary, entry))) {
3446                 offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3447         } else if ((notify = OSDynamicCast( _IOServiceNotifier, entry))) {
3448                 return notify->priority;
3449         } else if ((service = OSDynamicCast( IOService, entry))) {
3450                 prop = service->copyProperty(key);
3451                 offset = OSDynamicCast(OSNumber, prop);
3452         } else {
3453                 assert( false );
3454                 offset = NULL;
3455         }
3456
3457         if (offset) {
3458                 result = offset->unsigned32BitValue();
3459         }
3460
3461         OSSafeReleaseNULL(prop);
3462
3463         return result;
3464 }
3465
3466 SInt32
3467 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3468 {
3469         const OSObject *    obj1 = (const OSObject *) inObj1;
3470         const OSObject *    obj2 = (const OSObject *) inObj2;
3471         SInt32               val1;
3472         SInt32               val2;
3473
3474         val1 = 0;
3475         val2 = 0;
3476
3477         if (obj1) {
3478                 val1 = IOServiceObjectOrder( obj1, ref );
3479         }
3480
3481         if (obj2) {
3482                 val2 = IOServiceObjectOrder( obj2, ref );
3483         }
3484
3485         return val1 - val2;
3486 }
3487
3488 IOService *
3489 IOService::copyClientWithCategory( const OSSymbol * category )
3490 {
3491         IOService *         service = NULL;
3492         OSIterator *        iter;
3493         const OSSymbol *    nextCat;
3494
3495         iter = getClientIterator();
3496         if (iter) {
3497                 while ((service = (IOService *) iter->getNextObject())) {
3498                         if (kIOServiceInactiveState & service->__state[0]) {
3499                                 continue;
3500                         }
3501                         nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3502                             service->getProperty( gIOMatchCategoryKey ));
3503                         if (category == nextCat) {
3504                                 service->retain();
3505                                 break;
3506                         }
3507                 }
3508                 iter->release();
3509         }
3510         return service;
3511 }
3512
3513 IOService *
3514 IOService::getClientWithCategory( const OSSymbol * category )
3515 {
3516         IOService *
3517             service = copyClientWithCategory(category);
3518         if (service) {
3519                 service->release();
3520         }
3521         return service;
3522 }
3523
3524 bool
3525 IOService::invokeNotifier( _IOServiceNotifier * notify )
3526 {
3527         _IOServiceNotifierInvocation invocation;
3528         bool                         willNotify;
3529         bool                         ret = true;
3530         invocation.thread = current_thread();
3531
3532 #if DEBUG_NOTIFIER_LOCKED
3533         uint32_t count;
3534         if ((count = isLockedForArbitration(0))) {
3535                 IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3536                 panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3537         }
3538 #endif /* DEBUG_NOTIFIER_LOCKED */
3539
3540         LOCKWRITENOTIFY();
3541         willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3542
3543         if (willNotify) {
3544                 queue_enter( &notify->handlerInvocations, &invocation,
3545                     _IOServiceNotifierInvocation *, link );
3546         }
3547         UNLOCKNOTIFY();
3548
3549         if (willNotify) {
3550                 ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3551
3552                 LOCKWRITENOTIFY();
3553                 queue_remove( &notify->handlerInvocations, &invocation,
3554                     _IOServiceNotifierInvocation *, link );
3555                 if (kIOServiceNotifyWaiter & notify->state) {
3556                         notify->state &= ~kIOServiceNotifyWaiter;
3557                         WAKEUPNOTIFY( notify );
3558                 }
3559                 UNLOCKNOTIFY();
3560         }
3561
3562         return ret;
3563 }
3564
3565 bool
3566 IOService::invokeNotifiers(OSArray * willSend[])
3567 {
3568         OSArray *            array;
3569         _IOServiceNotifier * notify;
3570         bool                 ret = true;
3571
3572         array = *willSend;
3573         if (!array) {
3574                 return true;
3575         }
3576         *willSend = NULL;
3577
3578         for (unsigned int idx = 0;
3579             (notify = (_IOServiceNotifier *) array->getObject(idx));
3580             idx++) {
3581                 ret &= invokeNotifier(notify);
3582         }
3583         array->release();
3584
3585         return ret;
3586 }
3587
3588
3589 /*
3590  * Alloc and probe matching classes,
3591  * called on the provider instance
3592  */
3593
3594 void
3595 IOService::probeCandidates( OSOrderedSet * matches )
3596 {
3597         OSDictionary        *       match = NULL;
3598         OSSymbol            *       symbol;
3599         IOService           *       inst;
3600         IOService           *       newInst;
3601         OSDictionary        *       props;
3602         SInt32                      score;
3603         OSNumber            *       newPri;
3604         OSOrderedSet        *       familyMatches = NULL;
3605         OSOrderedSet        *       startList;
3606         OSSet               *       kexts = NULL;
3607         OSObject            *       kextRef;
3608
3609         OSDictionary        *       startDict = NULL;
3610         const OSSymbol      *       category;
3611         OSIterator          *       iter;
3612         _IOServiceNotifier  *       notify;
3613         OSObject            *       nextMatch = NULL;
3614         bool                        started;
3615         bool                        needReloc = false;
3616         bool                        matchDeferred = false;
3617 #if IOMATCHDEBUG
3618         SInt64                      debugFlags;
3619 #endif
3620         IOService           *       client = NULL;
3621         OSObject            *       prop1;
3622         OSObject            *       prop2;
3623         OSDictionary        *       rematchPersonality;
3624         OSNumber            *       num;
3625         uint32_t                    count;
3626         uint32_t                    dextCount;
3627         bool                        isDext;
3628         bool                        categoryConsumed;
3629
3630         prop2 = NULL;
3631         count = 0;
3632         prop1 = copyProperty(gIORematchPersonalityKey);
3633         rematchPersonality = OSDynamicCast(OSDictionary, prop1);
3634         if (rematchPersonality) {
3635                 prop2 = copyProperty(gIORematchCountKey);
3636                 num   = OSDynamicCast(OSNumber, prop2);
3637                 if (num) {
3638                         count = num->unsigned32BitValue();
3639                 }
3640                 OSSafeReleaseNULL(prop2);
3641         }
3642         dextCount = 0;
3643
3644         assert( matches );
3645         while (!needReloc
3646             && (nextMatch = matches->getFirstObject())) {
3647                 nextMatch->retain();
3648                 matches->removeObject(nextMatch);
3649
3650                 if ((notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3651                         if (0 == (__state[0] & kIOServiceInactiveState)) {
3652                                 invokeNotifier( notify );
3653                         }
3654                         nextMatch->release();
3655                         nextMatch = NULL;
3656                         continue;
3657                 } else if (!(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3658                         nextMatch->release();
3659                         nextMatch = NULL;
3660                         continue;
3661                 }
3662
3663                 props = NULL;
3664 #if IOMATCHDEBUG
3665                 debugFlags = getDebugFlags( match );
3666 #endif
3667
3668                 do {
3669                         isDext = (NULL != match->getObject(gIOUserServerNameKey));
3670                         if (isDext && !(kIODKEnable & gIODKDebug)) {
3671                                 continue;
3672                         }
3673
3674                         category = OSDynamicCast( OSSymbol,
3675                             match->getObject( gIOMatchCategoryKey ));
3676                         if (NULL == category) {
3677                                 category = gIODefaultMatchCategoryKey;
3678                         }
3679                         client = copyClientWithCategory(category);
3680
3681                         categoryConsumed = (client != NULL);
3682                         if (categoryConsumed) {
3683 #if IOMATCHDEBUG
3684                                 if ((debugFlags & kIOLogMatch) && (this != gIOResources)) {
3685                                         LOG("%s: match category %s exists\n", getName(),
3686                                             category->getCStringNoCopy());
3687                                 }
3688 #endif
3689                                 OSSafeReleaseNULL(client);
3690                                 if (!isDext) {
3691                                         break;
3692                                 }
3693                         }
3694
3695                         // create a copy now in case its modified during matching
3696                         props = OSDictionary::withDictionary(match, match->getCount());
3697                         if (NULL == props) {
3698                                 break;
3699                         }
3700                         props->setCapacityIncrement(1);
3701
3702                         // check the nub matches
3703                         if (false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) {
3704                                 break;
3705                         }
3706                         if (isDext) {
3707                                 dextCount++;
3708                                 if (categoryConsumed) {
3709                                         break;
3710                                 }
3711                         }
3712
3713                         if (rematchPersonality) {
3714                                 bool personalityMatch = match->isEqualTo(rematchPersonality);
3715                                 if (count > gIODextRelaunchMax) {
3716                                         personalityMatch = !personalityMatch;
3717                                 }
3718                                 if (!personalityMatch) {
3719                                         break;
3720                                 }
3721                         }
3722
3723                         // Check to see if driver reloc has been loaded.
3724                         needReloc = (false == gIOCatalogue->isModuleLoaded( match, &kextRef ));
3725                         if (needReloc) {
3726 #if IOMATCHDEBUG
3727                                 if (debugFlags & kIOLogCatalogue) {
3728                                         LOG("%s: stalling for module\n", getName());
3729                                 }
3730 #endif
3731                                 // If reloc hasn't been loaded, exit;
3732                                 // reprobing will occur after reloc has been loaded.
3733                                 break;
3734                         }
3735                         if (kextRef) {
3736                                 if (NULL == kexts) {
3737                                         kexts = OSSet::withCapacity(1);
3738                                 }
3739                                 if (kexts) {
3740                                         kexts->setObject(kextRef);
3741                                         kextRef->release();
3742                                 }
3743                         }
3744                         if (isDext) {
3745                                 // copy saved for rematchng
3746                                 props->setObject(gIOMatchedPersonalityKey, match);
3747                         }
3748                         // reorder on family matchPropertyTable score.
3749                         if (NULL == familyMatches) {
3750                                 familyMatches = OSOrderedSet::withCapacity( 1,
3751                                     IOServiceOrdering, (void *) gIOProbeScoreKey );
3752                         }
3753                         if (familyMatches) {
3754                                 familyMatches->setObject( props );
3755                         }
3756                 } while (false);
3757
3758                 OSSafeReleaseNULL(nextMatch);
3759                 OSSafeReleaseNULL(props);
3760         }
3761         matches->release();
3762         matches = NULL;
3763
3764         if (familyMatches) {
3765                 while (!needReloc
3766                     && (props = (OSDictionary *) familyMatches->getFirstObject())) {
3767                         props->retain();
3768                         familyMatches->removeObject( props );
3769
3770                         inst = NULL;
3771                         newInst = NULL;
3772 #if IOMATCHDEBUG
3773                         debugFlags = getDebugFlags( props );
3774 #endif
3775                         do {
3776                                 symbol = OSDynamicCast( OSSymbol,
3777                                     props->getObject( gIOClassKey));
3778                                 if (!symbol) {
3779                                         continue;
3780                                 }
3781
3782                                 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
3783
3784                                 // alloc the driver instance
3785                                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
3786
3787                                 if (!inst || !OSDynamicCast(IOService, inst)) {
3788                                         IOLog("Couldn't alloc class \"%s\"\n",
3789                                             symbol->getCStringNoCopy());
3790                                         continue;
3791                                 }
3792
3793                                 // init driver instance
3794                                 if (!(inst->init( props ))) {
3795 #if IOMATCHDEBUG
3796                                         if (debugFlags & kIOLogStart) {
3797                                                 IOLog("%s::init fails\n", symbol->getCStringNoCopy());
3798                                         }
3799 #endif
3800                                         continue;
3801                                 }
3802                                 if (__state[1] & kIOServiceSynchronousState) {
3803                                         inst->__state[1] |= kIOServiceSynchronousState;
3804                                 }
3805
3806                                 // give the driver the default match category if not specified
3807                                 category = OSDynamicCast( OSSymbol,
3808                                     props->getObject( gIOMatchCategoryKey ));
3809                                 if (NULL == category) {
3810                                         category = gIODefaultMatchCategoryKey;
3811                                 }
3812                                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
3813                                 // attach driver instance
3814                                 if (!(inst->attach( this ))) {
3815                                         continue;
3816                                 }
3817
3818                                 // pass in score from property table
3819                                 score = familyMatches->orderObject( props );
3820
3821                                 // & probe the new driver instance
3822 #if IOMATCHDEBUG
3823                                 if (debugFlags & kIOLogProbe) {
3824                                         LOG("%s::probe(%s)\n",
3825                                             inst->getMetaClass()->getClassName(), getName());
3826                                 }
3827 #endif
3828
3829                                 newInst = inst->probe( this, &score );
3830                                 inst->detach( this );
3831                                 if (NULL == newInst) {
3832 #if IOMATCHDEBUG
3833                                         if (debugFlags & kIOLogProbe) {
3834                                                 IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
3835                                         }
3836 #endif
3837                                         continue;
3838                                 }
3839
3840                                 // save the score
3841                                 newPri = OSNumber::withNumber( score, 32 );
3842                                 if (newPri) {
3843                                         newInst->setProperty( gIOProbeScoreKey, newPri );
3844                                         newPri->release();
3845                                 }
3846
3847                                 // add to start list for the match category
3848                                 if (NULL == startDict) {
3849                                         startDict = OSDictionary::withCapacity( 1 );
3850                                 }
3851                                 assert( startDict );
3852                                 startList = (OSOrderedSet *)
3853                                     startDict->getObject( category );
3854                                 if (NULL == startList) {
3855                                         startList = OSOrderedSet::withCapacity( 1,
3856                                             IOServiceOrdering, (void *) gIOProbeScoreKey );
3857                                         if (startDict && startList) {
3858                                                 startDict->setObject( category, startList );
3859                                                 startList->release();
3860                                         }
3861                                 }
3862                                 assert( startList );
3863                                 if (startList) {
3864                                         startList->setObject( newInst );
3865                                 }
3866                         } while (false);
3867
3868                         props->release();
3869                         if (inst) {
3870                                 inst->release();
3871                         }
3872                 }
3873                 familyMatches->release();
3874                 familyMatches = NULL;
3875         }
3876
3877         // start the best (until success) of each category
3878
3879         iter = OSCollectionIterator::withCollection( startDict );
3880         if (iter) {
3881                 while ((category = (const OSSymbol *) iter->getNextObject())) {
3882                         startList = (OSOrderedSet *) startDict->getObject( category );
3883                         assert( startList );
3884                         if (!startList) {
3885                                 continue;
3886                         }
3887
3888                         started = false;
3889                         while (true // (!started)
3890                             && !matchDeferred
3891                             && (inst = (IOService *)startList->getFirstObject())) {
3892                                 inst->retain();
3893                                 startList->removeObject(inst);
3894
3895 #if IOMATCHDEBUG
3896                                 debugFlags = getDebugFlags( inst );
3897
3898                                 if (debugFlags & kIOLogStart) {
3899                                         if (started) {
3900                                                 LOG( "match category exists, skipping " );
3901                                         }
3902                                         LOG( "%s::start(%s) <%d>\n", inst->getName(),
3903                                             getName(), inst->getRetainCount());
3904                                 }
3905 #endif
3906                                 if (false == started) {
3907 #if !NO_KEXTD
3908                                         IOLockLock(gJobsLock);
3909                                         matchDeferred = (gIOMatchDeferList
3910                                             && (kOSBooleanTrue == inst->getProperty(gIOMatchDeferKey)));
3911                                         if (matchDeferred && (-1U == gIOMatchDeferList->getNextIndexOfObject(this, 0))) {
3912                                                 gIOMatchDeferList->setObject(this);
3913                                         }
3914                                         IOLockUnlock(gJobsLock);
3915                                         if (matchDeferred) {
3916                                                 symbol = OSDynamicCast(OSSymbol, inst->getProperty(gIOClassKey));
3917                                                 IOLog("%s(0x%qx): matching deferred by %s\n",
3918                                                     getName(), getRegistryEntryID(),
3919                                                     symbol ? symbol->getCStringNoCopy() : "");
3920                                                 // rematching will occur after kextd loads all plists
3921                                         }
3922 #endif
3923                                         if (!matchDeferred) {
3924                                                 started = startCandidate( inst );
3925 #if IOMATCHDEBUG
3926                                                 if ((debugFlags & kIOLogStart) && (false == started)) {
3927                                                         LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
3928                                                             inst->getRetainCount());
3929                                                 }
3930 #endif
3931                                         }
3932                                 }
3933                                 inst->release();
3934                         }
3935                 }
3936                 iter->release();
3937         }
3938
3939         OSSafeReleaseNULL(prop1);
3940
3941         if (dextCount) {
3942                 num = OSNumber::withNumber(dextCount, 32);
3943                 setProperty(gIODEXTMatchCountKey, num);
3944                 OSSafeReleaseNULL(num);
3945         } else if (rematchPersonality) {
3946                 removeProperty(gIODEXTMatchCountKey);
3947         }
3948
3949         // now that instances are created, drop the refs on any kexts allowing unload
3950         if (kexts) {
3951                 OSKext::dropMatchingReferences(kexts);
3952                 OSSafeReleaseNULL(kexts);
3953         }
3954
3955         // adjust the busy count by +1 if matching is stalled for a module,
3956         // or -1 if a previously stalled matching is complete.
3957         lockForArbitration();
3958         SInt32 adjBusy = 0;
3959         uint64_t regID = getRegistryEntryID();
3960
3961         if (needReloc) {
3962                 adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
3963                 if (adjBusy) {
3964                         IOServiceTrace(
3965                                 IOSERVICE_MODULESTALL,
3966                                 (uintptr_t) regID,
3967                                 (uintptr_t) (regID >> 32),
3968                                 (uintptr_t) this,
3969                                 0);
3970
3971                         __state[1] |= kIOServiceModuleStallState;
3972                 }
3973         } else if (__state[1] & kIOServiceModuleStallState) {
3974                 IOServiceTrace(
3975                         IOSERVICE_MODULEUNSTALL,
3976                         (uintptr_t) regID,
3977                         (uintptr_t) (regID >> 32),
3978                         (uintptr_t) this,
3979                         0);
3980
3981                 __state[1] &= ~kIOServiceModuleStallState;
3982                 adjBusy = -1;
3983         }
3984         if (adjBusy) {
3985                 _adjustBusy( adjBusy );
3986         }
3987         unlockForArbitration();
3988
3989         if (startDict) {
3990                 startDict->release();
3991         }
3992 }
3993
3994 /*
3995  * Wait for a IOUserServer to check in
3996  */
3997
3998 static
3999 __attribute__((noinline, not_tail_called))
4000 IOService *
4001 __WAITING_FOR_USER_SERVER__(OSDictionary * matching)
4002 {
4003         IOService * server;
4004         server = IOService::waitForMatchingService(matching, kIOUserServerCheckInTimeoutSecs * NSEC_PER_SEC);
4005         return server;
4006 }
4007
4008 void
4009 IOService::willShutdown()
4010 {
4011         gKextdWillTerminate = true;
4012 #if !NO_KEXTD
4013         getPlatform()->waitQuiet(30 * NSEC_PER_SEC);
4014 #endif
4015         OSKext::willShutdown();
4016 }
4017
4018 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4019
4020 void
4021 IOServicePH::init(IOPMrootDomain * root)
4022 {
4023         fUserServers     = OSArray::withCapacity(4);
4024         fMatchingWork    = OSArray::withCapacity(4);
4025
4026         assert(fUserServers && fMatchingWork);
4027
4028         fRootNotifier = root->registerInterest(
4029                 gIOPriorityPowerStateInterest, &IOServicePH::systemPowerChange, NULL, NULL);
4030
4031         assert(fRootNotifier);
4032 }
4033
4034 void
4035 IOServicePH::lock()
4036 {
4037         IOLockLock(gJobsLock);
4038 }
4039
4040 void
4041 IOServicePH::unlock()
4042 {
4043         IOLockUnlock(gJobsLock);
4044 }
4045
4046 void
4047 IOServicePH::serverAdd(IOUserServer * server)
4048 {
4049         uint32_t idx;
4050
4051         lock();
4052         idx = fUserServers->getNextIndexOfObject(server, 0);
4053         if (idx == -1U) {
4054                 fUserServers->setObject(server);
4055         }
4056         unlock();
4057 }
4058
4059 void
4060 IOServicePH::serverRemove(IOUserServer * server)
4061 {
4062         uint32_t idx;
4063
4064         lock();
4065         idx = fUserServers->getNextIndexOfObject(server, 0);
4066         if (idx != -1U) {
4067                 fUserServers->removeObject(idx);
4068         }
4069         unlock();
4070 }
4071
4072 void
4073 IOServicePH::serverAck(IOUserServer * server)
4074 {
4075         uint32_t    idx;
4076         IOService * ackTo;
4077         uint32_t    ackToRef;
4078
4079         ackTo = NULL;
4080         lock();
4081         if (server && fUserServersWait) {
4082                 idx = fUserServersWait->getNextIndexOfObject(server, 0);
4083                 if (idx != -1U) {
4084                         fUserServersWait->removeObject(idx);
4085                         if (0 == fUserServersWait->getCount()) {
4086                                 OSSafeReleaseNULL(fUserServersWait);
4087                         }
4088                 }
4089         }
4090         if (!fUserServersWait && !fMatchingWork->getCount()) {
4091                 ackTo             = fSystemPowerAckTo;
4092                 ackToRef          = fSystemPowerAckRef;
4093                 fSystemPowerAckTo = NULL;
4094         }
4095         unlock();
4096
4097         if (ackTo) {
4098                 DKLOG("allowPowerChange\n");
4099                 ackTo->allowPowerChange((uintptr_t) ackToRef);
4100         }
4101 }
4102
4103 bool
4104 IOServicePH::matchingStart(IOService * service)
4105 {
4106         uint32_t idx;
4107         bool ok;
4108
4109         lock();
4110         ok = !fSystemOff;
4111         if (ok) {
4112                 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4113                 if (idx == -1U) {
4114                         fMatchingWork->setObject(service);
4115                 }
4116         } else {
4117                 if (!fMatchingDelayed) {
4118                         fMatchingDelayed = OSArray::withObjects((const OSObject **) &service, 1, 1);
4119                 } else {
4120                         idx = fMatchingDelayed->getNextIndexOfObject(service, 0);
4121                         if (idx == -1U) {
4122                                 fMatchingDelayed->setObject(service);
4123                         }
4124                 }
4125         }
4126         unlock();
4127
4128         return ok;
4129 }
4130
4131 void
4132 IOServicePH::matchingEnd(IOService * service)
4133 {
4134         uint32_t idx;
4135         OSArray   * notifyServers;
4136         OSArray   * deferredMatches;
4137
4138         notifyServers   = NULL;
4139         deferredMatches = NULL;
4140
4141         lock();
4142
4143         if (service) {
4144                 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4145                 if (idx != -1U) {
4146                         fMatchingWork->removeObject(idx);
4147                 }
4148         }
4149
4150
4151         if ((fUserServerOff != fSystemOff) && fUserServers->getCount()) {
4152                 if (fSystemOff) {
4153                         if (0 == fMatchingWork->getCount()) {
4154                                 fUserServersWait = OSArray::withArray(fUserServers);
4155                                 notifyServers = OSArray::withArray(fUserServers);
4156                                 fUserServerOff = fSystemOff;
4157                         }
4158                 } else {
4159                         notifyServers = OSArray::withArray(fUserServers);
4160                         fUserServerOff = fSystemOff;
4161                 }
4162         }
4163
4164         if (!fSystemOff && fMatchingDelayed) {
4165                 deferredMatches = fMatchingDelayed;
4166                 fMatchingDelayed = NULL;
4167         }
4168
4169         unlock();
4170
4171         if (notifyServers) {
4172                 notifyServers->iterateObjects(^bool (OSObject * obj) {
4173                         IOUserServer * us;
4174                         us = (typeof(us))obj;
4175                         us->systemPower(fSystemOff);
4176                         return false;
4177                 });
4178                 OSSafeReleaseNULL(notifyServers);
4179         }
4180
4181         if (deferredMatches) {
4182                 DKLOG("sleep deferred rematching count %d\n", deferredMatches->getCount());
4183                 deferredMatches->iterateObjects(^bool (OSObject * obj)
4184                 {
4185                         ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
4186                         return false;
4187                 });
4188                 deferredMatches->release();
4189         }
4190
4191         serverAck(NULL);
4192 }
4193
4194 IOReturn
4195 IOServicePH::systemPowerChange(
4196         void * target,
4197         void * refCon,
4198         UInt32 messageType, IOService * service,
4199         void * messageArgument, vm_size_t argSize)
4200 {
4201         IOReturn                               ret;
4202         IOUserServer                         * us;
4203         IOPMSystemCapabilityChangeParameters * params;
4204
4205         us = NULL;
4206
4207         switch (messageType) {
4208         case kIOMessageSystemCapabilityChange:
4209
4210                 params = (typeof params)messageArgument;
4211
4212                 if (kIODKLogPM & gIODKDebug) {
4213                         IOLog("IOServicePH::kIOMessageSystemCapabilityChange: %s%s 0x%x->0x%x\n",
4214                             params->changeFlags & kIOPMSystemCapabilityWillChange ? "will" : "",
4215                             params->changeFlags & kIOPMSystemCapabilityDidChange ? "did" : "",
4216                             params->fromCapabilities,
4217                             params->toCapabilities);
4218                 }
4219
4220                 if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4221                     (params->fromCapabilities & kIOPMSystemCapabilityCPU) &&
4222                     ((params->toCapabilities & kIOPMSystemCapabilityCPU) == 0)) {
4223                         lock();
4224                         fSystemOff         = true;
4225                         fSystemPowerAckRef = params->notifyRef;
4226                         fSystemPowerAckTo  = service;
4227                         unlock();
4228
4229                         matchingEnd(NULL);
4230
4231                         params->maxWaitForReply = 60 * 1000 * 1000;
4232                         ret = kIOReturnSuccess;
4233                 } else if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4234                     ((params->fromCapabilities & kIOPMSystemCapabilityCPU) == 0) &&
4235                     (params->toCapabilities & kIOPMSystemCapabilityCPU)) {
4236                         lock();
4237                         fSystemOff = false;
4238                         unlock();
4239
4240                         matchingEnd(NULL);
4241
4242                         params->maxWaitForReply = 0;
4243                         ret                 = kIOReturnSuccess;
4244                 } else {
4245                         params->maxWaitForReply = 0;
4246                         ret                 = kIOReturnSuccess;
4247                 }
4248                 break;
4249
4250         default:
4251                 ret = kIOReturnUnsupported;
4252                 break;
4253         }
4254
4255         return ret;
4256 }
4257
4258 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4259
4260 /*
4261  * Start a previously attached & probed instance,
4262  * called on exporting object instance
4263  */
4264
4265 bool
4266 IOService::startCandidate( IOService * service )
4267 {
4268         bool                ok;
4269         OSObject          * obj;
4270         OSObject          * prop;
4271         IOUserServer      * userServer;
4272         bool                ph;
4273
4274         userServer = NULL;
4275         obj = service->copyProperty(gIOUserServerNameKey);
4276
4277         if (obj && (this == gIOResources)) {
4278                 ok = false;
4279         } else {
4280                 ok = service->attach( this );
4281         }
4282         if (!ok) {
4283                 return false;
4284         }
4285
4286         if ((this != gIOResources) && (this != gIOUserResources)) {
4287                 // stall for any nub resources
4288                 checkResources();
4289                 // stall for any driver resources
4290                 service->checkResources();
4291         }
4292         ph = false;
4293         {
4294                 OSString       * bundleID;
4295                 OSString       * serverName;
4296                 OSString       * str;
4297                 const OSSymbol * sym;
4298                 OSDictionary   * matching;
4299                 IOService      * server;
4300                 OSNumber       * serverTag;
4301                 uint64_t         entryID;
4302
4303                 if ((serverName = OSDynamicCast(OSString, obj))) {
4304                         obj       = service->copyProperty(gIOModuleIdentifierKey);
4305                         bundleID  = OSDynamicCast(OSString, obj);
4306                         entryID   = service->getRegistryEntryID();
4307                         serverTag = OSNumber::withNumber(entryID, 64);
4308
4309                         if (gKextdWillTerminate) {
4310                                 DKLOG("%s disabled in shutdown\n", serverName->getCStringNoCopy());
4311                                 service->detach(this);
4312                                 OSSafeReleaseNULL(obj);
4313                                 return false;
4314                         }
4315
4316                         ph = IOServicePH::matchingStart(this);
4317                         if (!ph) {
4318                                 DKLOG("%s deferred in sleep\n", serverName->getCStringNoCopy());
4319                                 service->detach(this);
4320                                 OSSafeReleaseNULL(obj);
4321                                 return false;
4322                         }
4323
4324                         prop = service->copyProperty(gIOUserClassKey);
4325                         str = OSDynamicCast(OSString, prop);
4326                         if (str) {
4327                                 service->setName(str);
4328                         }
4329                         OSSafeReleaseNULL(prop);
4330
4331                         if (!(kIODKDisableDextLaunch & gIODKDebug)) {
4332                                 OSKext::requestDaemonLaunch(bundleID, serverName, serverTag);
4333                         }
4334                         sym = OSSymbol::withString(serverName);
4335                         matching = serviceMatching(gIOUserServerClassKey);
4336                         propertyMatching(gIOUserServerNameKey, sym, matching);
4337                         if (!(kIODKDisableDextTag & gIODKDebug)) {
4338                                 propertyMatching(gIOUserServerTagKey, serverTag, matching);
4339                         }
4340
4341                         server = __WAITING_FOR_USER_SERVER__(matching);
4342                         matching->release();
4343                         OSSafeReleaseNULL(serverTag);
4344                         OSSafeReleaseNULL(serverName);
4345
4346                         userServer = OSDynamicCast(IOUserServer, server);
4347                         if (!userServer) {
4348                                 service->detach(this);
4349                                 IOServicePH::matchingEnd(this);
4350                                 DKLOG(DKS " user server timeout\n", DKN(service));
4351                                 return false;
4352                         }
4353
4354                         if (!(kIODKDisableCDHashChecking & gIODKDebug)) {
4355                                 if (!userServer->serviceMatchesCDHash(service)) {
4356                                         service->detach(this);
4357                                         IOServicePH::matchingEnd(this);
4358                                         userServer->exit("CDHash check failed");
4359                                         userServer->release();
4360                                         return false;
4361                                 }
4362                         }
4363                         OSKext *kext = OSKext::lookupKextWithIdentifier(bundleID);
4364                         if (!kext) {
4365                                 const char *name = bundleID->getCStringNoCopy();
4366                                 IOLog("%s Could not find OSKext for %s\n", __func__, name);
4367                                 goto skip_log;
4368                         }
4369
4370                         /*
4371                          * Used for logging
4372                          */
4373                         userServer->setTaskLoadTag(kext);
4374                         userServer->setDriverKitUUID(kext);
4375                         OSKext::OSKextLogDriverKitInfoLoad(kext);
4376 skip_log:
4377                         OSSafeReleaseNULL(bundleID);
4378                         OSSafeReleaseNULL(kext);
4379
4380                         if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4381                                 if (!userServer->checkEntitlements(this, service)) {
4382                                         service->detach(this);
4383                                         IOServicePH::matchingEnd(this);
4384                                         userServer->exit("Entitlements check failed");
4385                                         userServer->release();
4386                                         return false;
4387                                 }
4388                         }
4389
4390                         userServer->serviceAttach(service, this);
4391                 }
4392         }
4393
4394         AbsoluteTime startTime;
4395         AbsoluteTime endTime;
4396         UInt64       nano;
4397
4398         if (kIOLogStart & gIOKitDebug) {
4399                 clock_get_uptime(&startTime);
4400         }
4401
4402         ok = service->start(this);
4403
4404         if (kIOLogStart & gIOKitDebug) {
4405                 clock_get_uptime(&endTime);
4406
4407                 if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0) {
4408                         SUB_ABSOLUTETIME(&endTime, &startTime);
4409                         absolutetime_to_nanoseconds(endTime, &nano);
4410                         if (nano > 500000000ULL) {
4411                                 IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
4412                         }
4413                 }
4414         }
4415         if (userServer) {
4416                 userServer->serviceStarted(service, this, ok);
4417                 userServer->release();
4418         }
4419         if (!ok) {
4420                 service->detach( this );
4421         }
4422
4423         if (ph) {
4424                 IOServicePH::matchingEnd(this);
4425         }
4426
4427         return ok;
4428 }
4429
4430 void
4431 IOService::publishResource( const char * key, OSObject * value )
4432 {
4433         const OSSymbol *    sym;
4434
4435         if ((sym = OSSymbol::withCString( key))) {
4436                 publishResource( sym, value);
4437                 sym->release();
4438         }
4439 }
4440
4441 void
4442 IOService::publishResource( const OSSymbol * key, OSObject * value )
4443 {
4444         if (NULL == value) {
4445                 value = (OSObject *) gIOServiceKey;
4446         }
4447
4448         gIOResources->setProperty( key, value);
4449
4450         if (IORecursiveLockHaveLock( gNotificationLock)) {
4451                 return;
4452         }
4453
4454         gIOResourceGenerationCount++;
4455         gIOResources->registerService();
4456 }
4457
4458 void
4459 IOService::publishUserResource( const OSSymbol * key, OSObject * value )
4460 {
4461         if (NULL == value) {
4462                 value = (OSObject *) gIOServiceKey;
4463         }
4464
4465         gIOUserResources->setProperty( key, value);
4466
4467         if (IORecursiveLockHaveLock( gNotificationLock)) {
4468                 return;
4469         }
4470
4471         gIOResourceGenerationCount++;
4472         gIOUserResources->registerService();
4473 }
4474
4475 bool
4476 IOService::addNeededResource( const char * key )
4477 {
4478         OSObject *  resourcesProp;
4479         OSSet *     set;
4480         OSString *  newKey;
4481         bool ret;
4482
4483         resourcesProp = copyProperty( gIOResourceMatchKey );
4484         if (!resourcesProp) {
4485                 return false;
4486         }
4487
4488         newKey = OSString::withCString( key );
4489         if (!newKey) {
4490                 resourcesProp->release();
4491                 return false;
4492         }
4493
4494         set = OSDynamicCast( OSSet, resourcesProp );
4495         if (!set) {
4496                 set = OSSet::withCapacity( 1 );
4497                 if (set) {
4498                         set->setObject( resourcesProp );
4499                 }
4500         } else {
4501                 set->retain();
4502         }
4503
4504         set->setObject( newKey );
4505         newKey->release();
4506         ret = setProperty( gIOResourceMatchKey, set );
4507         set->release();
4508         resourcesProp->release();
4509
4510         return ret;
4511 }
4512
4513 bool
4514 IOService::checkResource( OSObject * matching )
4515 {
4516         OSString *          str;
4517         OSDictionary *      table;
4518
4519         if ((str = OSDynamicCast( OSString, matching ))) {
4520                 if (gIOResources->getProperty( str )) {
4521                         return true;
4522                 }
4523         }
4524
4525         if (str) {
4526                 table = resourceMatching( str );
4527         } else if ((table = OSDynamicCast( OSDictionary, matching ))) {
4528                 table->retain();
4529         } else {
4530                 IOLog("%s: Can't match using: %s\n", getName(),
4531                     matching->getMetaClass()->getClassName());
4532                 /* false would stall forever */
4533                 return true;
4534         }
4535
4536         if (gIOKitDebug & kIOLogConfig) {
4537                 LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
4538         }
4539
4540         waitForService( table );
4541
4542         if (gIOKitDebug & kIOLogConfig) {
4543                 LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
4544         }
4545
4546         return true;
4547 }
4548
4549 bool
4550 IOService::checkResources( void )
4551 {
4552         OSObject *          resourcesProp;
4553         OSSet *             set;
4554         OSIterator *        iter;
4555         bool                ok;
4556
4557         resourcesProp = copyProperty( gIOResourceMatchKey );
4558         if (NULL == resourcesProp) {
4559                 return true;
4560         }
4561
4562         if ((set = OSDynamicCast( OSSet, resourcesProp ))) {
4563                 iter = OSCollectionIterator::withCollection( set );
4564                 ok = (NULL != iter);
4565                 while (ok && (resourcesProp = iter->getNextObject())) {
4566                         ok = checkResource( resourcesProp );
4567                 }
4568                 if (iter) {
4569                         iter->release();
4570                 }
4571         } else {
4572                 ok = checkResource( resourcesProp );
4573         }
4574
4575         OSSafeReleaseNULL(resourcesProp);
4576
4577         return ok;
4578 }
4579
4580
4581 void
4582 _IOConfigThread::configThread( int configThreadId )
4583 {
4584         _IOConfigThread *   inst;
4585
4586         do {
4587                 if (!(inst = new _IOConfigThread)) {
4588                         continue;
4589                 }
4590                 if (!inst->init()) {
4591                         continue;
4592                 }
4593                 thread_t thread;
4594                 if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &thread)) {
4595                         continue;
4596                 }
4597
4598                 char threadName[MAXTHREADNAMESIZE];
4599                 snprintf(threadName, sizeof(threadName), "IOConfigThread_%d", configThreadId);
4600                 thread_set_thread_name(thread, threadName);
4601                 thread_deallocate(thread);
4602
4603                 return;
4604         } while (false);
4605
4606         if (inst) {
4607                 inst->release();
4608         }
4609
4610         return;
4611 }
4612
4613 void
4614 IOService::doServiceMatch( IOOptionBits options )
4615 {
4616         _IOServiceNotifier * notify;
4617         OSIterator *        iter;
4618         OSOrderedSet *      matches;
4619         OSArray *           resourceKeys = NULL;
4620         SInt32              catalogGeneration;
4621         bool                keepGuessing = true;
4622         bool                reRegistered = true;
4623         bool                didRegister;
4624         OSArray *           notifiers[2] = {NULL};
4625
4626 //    job->nub->deliverNotification( gIOPublishNotification,
4627 //                              kIOServiceRegisteredState, 0xffffffff );
4628
4629         while (keepGuessing) {
4630                 matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
4631                 // the matches list should always be created by findDrivers()
4632                 if (matches) {
4633                         lockForArbitration();
4634                         if (0 == (__state[0] & kIOServiceFirstPublishState)) {
4635                                 getMetaClass()->addInstance(this);
4636                                 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
4637                                     kIOServiceFirstPublishState, 0xffffffff );
4638                         }
4639                         LOCKREADNOTIFY();
4640                         __state[1] &= ~kIOServiceNeedConfigState;
4641                         __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
4642                         didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
4643                         __state[0] |= kIOServiceRegisteredState;
4644
4645                         keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
4646                         if (reRegistered && keepGuessing) {
4647                                 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
4648                                     gNotifications->getObject( gIOPublishNotification ));
4649                                 if (iter) {
4650                                         while ((notify = (_IOServiceNotifier *)
4651                                             iter->getNextObject())) {
4652                                                 if (matchPassive(notify->matching, 0)
4653                                                     && (kIOServiceNotifyEnable & notify->state)) {
4654                                                         matches->setObject( notify );
4655                                                 }
4656                                         }
4657                                         iter->release();
4658                                 }
4659                         }
4660
4661                         UNLOCKNOTIFY();
4662                         unlockForArbitration();
4663                         invokeNotifiers(&notifiers[0]);
4664
4665                         if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) {
4666                                 if ((this == gIOResources) || (this == gIOUserResources)) {
4667                                         if (resourceKeys) {
4668                                                 resourceKeys->release();
4669                                         }
4670                                         resourceKeys = copyPropertyKeys();
4671                                 }
4672                                 probeCandidates( matches );
4673                         } else {
4674                                 matches->release();
4675                         }
4676                 }
4677
4678                 lockForArbitration();
4679                 reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
4680                 keepGuessing =
4681                     (reRegistered || (catalogGeneration !=
4682                     gIOCatalogue->getGenerationCount()))
4683                     && (0 == (__state[0] & kIOServiceInactiveState));
4684
4685                 if (keepGuessing) {
4686                         unlockForArbitration();
4687                 }
4688         }
4689
4690         if ((0 == (__state[0] & kIOServiceInactiveState))
4691             && (0 == (__state[1] & kIOServiceModuleStallState))) {
4692                 if (resourceKeys) {
4693                         setProperty(gIOResourceMatchedKey, resourceKeys);
4694                 }
4695
4696                 notifiers[0] = copyNotifiers(gIOMatchedNotification,
4697                     kIOServiceMatchedState, 0xffffffff);
4698                 if (0 == (__state[0] & kIOServiceFirstMatchState)) {
4699                         notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
4700                             kIOServiceFirstMatchState, 0xffffffff);
4701                 }
4702         }
4703
4704         __state[1] &= ~kIOServiceConfigRunning;
4705         unlockForArbitration();
4706
4707         if (resourceKeys) {
4708                 resourceKeys->release();
4709         }
4710
4711         invokeNotifiers(&notifiers[0]);
4712         invokeNotifiers(&notifiers[1]);
4713
4714         lockForArbitration();
4715         __state[1] &= ~kIOServiceConfigState;
4716         scheduleTerminatePhase2();
4717
4718         _adjustBusy( -1 );
4719         unlockForArbitration();
4720 }
4721
4722 UInt32
4723 IOService::_adjustBusy( SInt32 delta )
4724 {
4725         IOService * next;
4726         UInt32      count;
4727         UInt32      result;
4728         bool        wasQuiet, nowQuiet, needWake;
4729
4730         next = this;
4731         result = __state[1] & kIOServiceBusyStateMask;
4732
4733         if (delta) {
4734                 do {
4735                         if (next != this) {
4736                                 next->lockForArbitration();
4737                         }
4738                         count = next->__state[1] & kIOServiceBusyStateMask;
4739                         wasQuiet = (0 == count);
4740                         if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) {
4741                                 OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta);
4742                         } else {
4743                                 count += delta;
4744                         }
4745                         next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
4746                         nowQuiet = (0 == count);
4747                         needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
4748
4749                         if (needWake) {
4750                                 next->__state[1] &= ~kIOServiceBusyWaiterState;
4751                                 IOLockLock( gIOServiceBusyLock );
4752                                 thread_wakeup((event_t) next);
4753                                 IOLockUnlock( gIOServiceBusyLock );
4754                         }
4755                         if (next != this) {
4756                                 next->unlockForArbitration();
4757                         }
4758
4759                         if ((wasQuiet || nowQuiet)) {
4760                                 uint64_t regID = next->getRegistryEntryID();
4761                                 IOServiceTrace(
4762                                         ((wasQuiet /*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
4763                                         (uintptr_t) regID,
4764                                         (uintptr_t) (regID >> 32),
4765                                         (uintptr_t) next,
4766                                         0);
4767
4768                                 if (wasQuiet) {
4769                                         next->__timeBusy = mach_absolute_time();
4770                                 } else {
4771                                         next->__accumBusy += mach_absolute_time() - next->__timeBusy;
4772                                         next->__timeBusy = 0;
4773                                 }
4774
4775                                 MessageClientsContext context;
4776
4777                                 context.service  = next;
4778                                 context.type     = kIOMessageServiceBusyStateChange;
4779                                 context.argument = (void *) wasQuiet; /*nowBusy*/
4780                                 context.argSize  = 0;
4781
4782                                 applyToInterestNotifiers( next, gIOBusyInterest,
4783                                     &messageClientsApplier, &context );
4784
4785 #if !NO_KEXTD
4786                                 if (nowQuiet && (next == gIOServiceRoot)) {
4787                                         OSKext::considerUnloads();
4788                                         IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
4789                                 }
4790 #endif
4791                         }
4792
4793                         delta = nowQuiet ? -1 : +1;
4794                 } while ((wasQuiet || nowQuiet) && (next = next->getProvider()));
4795         }
4796
4797         return result;
4798 }
4799
4800 void
4801 IOService::adjustBusy( SInt32 delta )
4802 {
4803         lockForArbitration();
4804         _adjustBusy( delta );
4805         unlockForArbitration();
4806 }
4807
4808 uint64_t
4809 IOService::getAccumulatedBusyTime( void )
4810 {
4811         uint64_t accumBusy = __accumBusy;
4812         uint64_t timeBusy = __timeBusy;
4813         uint64_t nano;
4814
4815         do{
4816                 accumBusy = __accumBusy;
4817                 timeBusy  = __timeBusy;
4818                 if (timeBusy) {
4819                         accumBusy += mach_absolute_time() - timeBusy;
4820                 }
4821         }while (timeBusy != __timeBusy);
4822
4823         absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
4824
4825         return nano;
4826 }
4827
4828 UInt32
4829 IOService::getBusyState( void )
4830 {
4831         return __state[1] & kIOServiceBusyStateMask;
4832 }
4833
4834 IOReturn
4835 IOService::waitForState( UInt32 mask, UInt32 value,
4836     mach_timespec_t * timeout )
4837 {
4838         panic("waitForState");
4839         return kIOReturnUnsupported;
4840 }
4841
4842 IOReturn
4843 IOService::waitForState( UInt32 mask, UInt32 value,
4844     uint64_t timeout )
4845 {
4846         bool            wait;
4847         int             waitResult = THREAD_AWAKENED;
4848         bool            computeDeadline = true;
4849         AbsoluteTime    abstime;
4850
4851         do {
4852                 lockForArbitration();
4853                 IOLockLock( gIOServiceBusyLock );
4854                 wait = (value != (__state[1] & mask));
4855                 if (wait) {
4856                         __state[1] |= kIOServiceBusyWaiterState;
4857                         unlockForArbitration();
4858                         if (timeout != UINT64_MAX) {
4859                                 if (computeDeadline) {
4860                                         AbsoluteTime  nsinterval;
4861                                         nanoseconds_to_absolutetime(timeout, &nsinterval );
4862                                         clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
4863                                         computeDeadline = false;
4864                                 }
4865                                 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
4866                         } else {
4867                                 assert_wait((event_t)this, THREAD_UNINT );
4868                         }
4869                 } else {
4870                         unlockForArbitration();
4871                 }
4872                 IOLockUnlock( gIOServiceBusyLock );
4873                 if (wait) {
4874                         waitResult = thread_block(THREAD_CONTINUE_NULL);
4875                 }
4876         } while (wait && (waitResult != THREAD_TIMED_OUT));
4877
4878         if (waitResult == THREAD_TIMED_OUT) {
4879                 return kIOReturnTimeout;
4880         } else {
4881                 return kIOReturnSuccess;
4882         }
4883 }
4884
4885 IOReturn
4886 IOService::waitQuiet( uint64_t timeout )
4887 {
4888         IOReturn ret;
4889         uint32_t loops;
4890         char *   string = NULL;
4891         char *   panicString = NULL;
4892         size_t   len;
4893         size_t   panicStringLen;
4894         uint64_t time;
4895         uint64_t nano;
4896         bool     kextdWait;
4897         bool     dopanic;
4898
4899 #if KASAN
4900         /*
4901          * On kasan kernels, everything takes longer, so double the number of
4902          * timeout extensions. This should help with issues like 41259215
4903          * where WindowServer was timing out waiting for kextd to get all the
4904          * kasan kexts loaded and started.
4905          */
4906         enum { kTimeoutExtensions = 8 };
4907 #else
4908         enum { kTimeoutExtensions = 4 };
4909 #endif
4910
4911         time = mach_absolute_time();
4912         kextdWait = false;
4913         for (loops = 0; loops < kTimeoutExtensions; loops++) {
4914                 ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
4915
4916                 if (loops && (kIOReturnSuccess == ret)) {
4917                         time = mach_absolute_time() - time;
4918                         absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
4919                         IOLog("busy extended ok[%d], (%llds, %llds)\n",
4920                             loops, timeout / 1000000000ULL, nano / 1000000000ULL);
4921                         break;
4922                 } else if (kIOReturnTimeout != ret) {
4923                         break;
4924                 } else if (timeout < (4100ull * NSEC_PER_SEC)) {
4925                         break;
4926                 }
4927
4928                 {
4929                         IORegistryIterator * iter;
4930                         OSOrderedSet       * set;
4931                         OSOrderedSet       * leaves;
4932                         IOService          * next;
4933                         IOService          * nextParent;
4934                         char               * s;
4935                         size_t               l;
4936
4937                         len = 256;
4938                         panicStringLen = 256;
4939                         if (!string) {
4940                                 string      = IONew(char, len);
4941                         }
4942                         if (!panicString) {
4943                                 panicString = IONew(char, panicStringLen);
4944                         }
4945                         set = NULL;
4946                         kextdWait = OSKext::isWaitingKextd();
4947                         iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
4948                         leaves = OSOrderedSet::withCapacity(4);
4949                         if (iter) {
4950                                 set = iter->iterateAll();
4951                         }
4952                         if (string && panicString && leaves && set) {
4953                                 string[0] = panicString[0] = 0;
4954                                 set->setObject(this);
4955                                 while ((next = (IOService *) set->getLastObject())) {
4956                                         if (next->getBusyState()) {
4957                                                 if (kIOServiceModuleStallState & next->__state[1]) {
4958                                                         kextdWait = true;
4959                                                 }
4960                                                 leaves->setObject(next);
4961                                                 nextParent = next;
4962                                                 while ((nextParent = nextParent->getProvider())) {
4963                                                         set->removeObject(nextParent);
4964                                                         leaves->removeObject(nextParent);
4965                                                 }
4966                                         }
4967                                         set->removeObject(next);
4968                                 }
4969                                 s = string;
4970                                 while ((next = (IOService *) leaves->getLastObject())) {
4971                                         l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName());
4972                                         if (l >= len) {
4973                                                 break;
4974                                         }
4975                                         s += l;
4976                                         len -= l;
4977                                         leaves->removeObject(next);
4978                                 }
4979                         }
4980                         OSSafeReleaseNULL(leaves);
4981                         OSSafeReleaseNULL(set);
4982                         OSSafeReleaseNULL(iter);
4983                 }
4984
4985                 dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug));
4986                 snprintf(panicString, panicStringLen,
4987                     "%s[%d], (%llds): %s",
4988                     kextdWait ? "kextd stall" : "busy timeout",
4989                     loops, timeout / 1000000000ULL,
4990                     string ? string : "");
4991                 IOLog("%s\n", panicString);
4992                 if (dopanic) {
4993                         panic("%s", panicString);
4994                 } else if (!loops) {
4995                         getPMRootDomain()->startSpinDump(1);
4996                 }
4997         }
4998
4999         if (string) {
5000                 IODelete(string, char, 256);
5001         }
5002         if (panicString) {
5003                 IODelete(panicString, char, panicStringLen);
5004         }
5005
5006         return ret;
5007 }
5008
5009 IOReturn
5010 IOService::waitQuiet( mach_timespec_t * timeout )
5011 {
5012         uint64_t    timeoutNS;
5013
5014         if (timeout) {
5015                 timeoutNS = timeout->tv_sec;
5016                 timeoutNS *= kSecondScale;
5017                 timeoutNS += timeout->tv_nsec;
5018         } else {
5019                 timeoutNS = UINT64_MAX;
5020         }
5021
5022         return waitQuiet(timeoutNS);
5023 }
5024
5025 bool
5026 IOService::serializeProperties( OSSerialize * s ) const
5027 {
5028 #if 0
5029         ((IOService *)this)->setProperty(((IOService *)this)->__state,
5030             sizeof(__state), "__state");
5031 #endif
5032         return super::serializeProperties(s);
5033 }
5034
5035
5036 void
5037 _IOConfigThread::main(void * arg, wait_result_t result)
5038 {
5039         _IOConfigThread * self = (_IOConfigThread *) arg;
5040         _IOServiceJob * job;
5041         IOService   *   nub;
5042         bool            alive = true;
5043         kern_return_t   kr;
5044         thread_precedence_policy_data_t precedence = { -1 };
5045
5046         kr = thread_policy_set(current_thread(),
5047             THREAD_PRECEDENCE_POLICY,
5048             (thread_policy_t) &precedence,
5049             THREAD_PRECEDENCE_POLICY_COUNT);
5050         if (KERN_SUCCESS != kr) {
5051                 IOLog("thread_policy_set(%d)\n", kr);
5052         }
5053
5054         do {
5055 //      randomDelay();
5056
5057                 semaphore_wait( gJobsSemaphore );
5058
5059                 IOTakeLock( gJobsLock );
5060                 job = (_IOServiceJob *) gJobs->getFirstObject();
5061                 job->retain();
5062                 gJobs->removeObject(job);
5063                 if (job) {
5064                         gOutstandingJobs--;
5065 //          gNumConfigThreads--;        // we're out of service
5066                         gNumWaitingThreads--; // we're out of service
5067                 }
5068                 IOUnlock( gJobsLock );
5069
5070                 if (job) {
5071                         nub = job->nub;
5072
5073                         if (gIOKitDebug & kIOLogConfig) {
5074                                 LOG("config(%p): starting on %s, %d\n",
5075                                     IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
5076                         }
5077
5078                         switch (job->type) {
5079                         case kMatchNubJob:
5080                                 nub->doServiceMatch( job->options );
5081                                 break;
5082
5083                         default:
5084                                 LOG("config(%p): strange type (%d)\n",
5085                                     IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
5086                                 break;
5087                         }
5088
5089                         nub->release();
5090                         job->release();
5091
5092                         IOTakeLock( gJobsLock );
5093                         alive = (gOutstandingJobs > gNumWaitingThreads);
5094                         if (alive) {
5095                                 gNumWaitingThreads++; // back in service
5096                         }
5097 //              gNumConfigThreads++;
5098                         else {
5099                                 if (0 == --gNumConfigThreads) {
5100 //                    IOLog("MATCH IDLE\n");
5101                                         IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
5102                                 }
5103                         }
5104                         IOUnlock( gJobsLock );
5105                 }
5106         } while (alive);
5107
5108         if (gIOKitDebug & kIOLogConfig) {
5109                 LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5110         }
5111
5112         self->release();
5113 }
5114
5115 IOReturn
5116 IOService::waitMatchIdle( UInt32 msToWait )
5117 {
5118         bool            wait;
5119         int             waitResult = THREAD_AWAKENED;
5120         bool            computeDeadline = true;
5121         AbsoluteTime    deadline;
5122
5123         IOLockLock( gJobsLock );
5124         do {
5125                 wait = (0 != gNumConfigThreads);
5126                 if (wait) {
5127                         if (msToWait) {
5128                                 if (computeDeadline) {
5129                                         clock_interval_to_deadline(
5130                                                 msToWait, kMillisecondScale, &deadline );
5131                                         computeDeadline = false;
5132                                 }
5133                                 waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
5134                                     deadline, THREAD_UNINT );
5135                         } else {
5136                                 waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
5137                                     THREAD_UNINT );
5138                         }
5139                 }
5140         } while (wait && (waitResult != THREAD_TIMED_OUT));
5141         IOLockUnlock( gJobsLock );
5142
5143         if (waitResult == THREAD_TIMED_OUT) {
5144                 return kIOReturnTimeout;
5145         } else {
5146                 return kIOReturnSuccess;
5147         }
5148 }
5149
5150 void
5151 IOService::cpusRunning(void)
5152 {
5153         gCPUsRunning = true;
5154 }
5155
5156 void
5157 _IOServiceJob::pingConfig( _IOServiceJob * job )
5158 {
5159         int         count;
5160         bool        create;
5161
5162         assert( job );
5163
5164         IOTakeLock( gJobsLock );
5165
5166         gOutstandingJobs++;
5167         gJobs->setLastObject( job );
5168
5169         count = gNumWaitingThreads;
5170 //    if( gNumConfigThreads) count++;// assume we're called from a config thread
5171
5172         create = ((gOutstandingJobs > count)
5173             && ((gNumConfigThreads < kMaxConfigThreads)
5174             || (job->nub == gIOResources)
5175             || !gCPUsRunning));
5176         if (create) {
5177                 gNumConfigThreads++;
5178                 gNumWaitingThreads++;
5179         }
5180
5181         IOUnlock( gJobsLock );
5182
5183         job->release();
5184
5185         if (create) {
5186                 if (gIOKitDebug & kIOLogConfig) {
5187                         LOG("config(%d): creating\n", gNumConfigThreads - 1);
5188                 }
5189                 _IOConfigThread::configThread(gNumConfigThreads - 1);
5190         }
5191
5192         semaphore_signal( gJobsSemaphore );
5193 }
5194
5195 struct IOServiceMatchContext {
5196         OSDictionary * table;
5197         OSObject *     result;
5198         uint32_t       options;
5199         uint32_t       state;
5200         uint32_t       count;
5201         uint32_t       done;
5202 };
5203
5204 bool
5205 IOService::instanceMatch(const OSObject * entry, void * context)
5206 {
5207         IOServiceMatchContext * ctx = (typeof(ctx))context;
5208         IOService *    service = (typeof(service))entry;
5209         OSDictionary * table   = ctx->table;
5210         uint32_t       options = ctx->options;
5211         uint32_t       state   = ctx->state;
5212         uint32_t       done;
5213         bool           match;
5214
5215         done = 0;
5216         do{
5217                 match = ((state == (state & service->__state[0]))
5218                     && (0 == (service->__state[0] & kIOServiceInactiveState)));
5219                 if (!match) {
5220                         break;
5221                 }
5222                 ctx->count += table->getCount();
5223                 match = service->matchInternal(table, options, &done);
5224                 ctx->done += done;
5225         }while (false);
5226         if (!match) {
5227                 return false;
5228         }
5229
5230         if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) {
5231                 service->retain();
5232                 ctx->result = service;
5233                 return true;
5234         } else if (!ctx->result) {
5235                 ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
5236         } else {
5237                 ((OSSet *)ctx->result)->setObject(service);
5238         }
5239         return false;
5240 }
5241
5242 // internal - call with gNotificationLock
5243 OSObject *
5244 IOService::copyExistingServices( OSDictionary * matching,
5245     IOOptionBits inState, IOOptionBits options )
5246 {
5247         OSObject *   current = NULL;
5248         OSIterator * iter;
5249         IOService *  service;
5250         OSObject *   obj;
5251         OSString *   str;
5252
5253         if (!matching) {
5254                 return NULL;
5255         }
5256
5257 #if MATCH_DEBUG
5258         OSSerialize * s = OSSerialize::withCapacity(128);
5259         matching->serialize(s);
5260 #endif
5261
5262         if ((obj = matching->getObject(gIOProviderClassKey))
5263             && gIOResourcesKey
5264             && gIOResourcesKey->isEqualTo(obj)
5265             && (service = gIOResources)) {
5266                 if ((inState == (service->__state[0] & inState))
5267                     && (0 == (service->__state[0] & kIOServiceInactiveState))
5268                     && service->matchPassive(matching, options)) {
5269                         if (options & kIONotifyOnce) {
5270                                 service->retain();
5271                                 current = service;
5272                         } else {
5273                                 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
5274                         }
5275                 }
5276         } else {
5277                 IOServiceMatchContext ctx;
5278                 ctx.table   = matching;
5279                 ctx.state   = inState;
5280                 ctx.count   = 0;
5281                 ctx.done    = 0;
5282                 ctx.options = options;
5283                 ctx.result  = NULL;
5284
5285                 if ((str = OSDynamicCast(OSString, obj))) {
5286                         const OSSymbol * sym = OSSymbol::withString(str);
5287                         OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
5288                         sym->release();
5289                 } else {
5290                         IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
5291                 }
5292
5293
5294                 current = ctx.result;
5295
5296                 options |= kIOServiceInternalDone | kIOServiceClassDone;
5297                 if (current && (ctx.done != ctx.count)) {
5298                         OSSet *
5299                             source = OSDynamicCast(OSSet, current);
5300                         current = NULL;
5301                         while ((service = (IOService *) source->getAnyObject())) {
5302                                 if (service->matchPassive(matching, options)) {
5303                                         if (options & kIONotifyOnce) {
5304                                                 service->retain();
5305                                                 current = service;
5306                                                 break;
5307                                         }
5308                                         if (current) {
5309                                                 ((OSSet *)current)->setObject( service );
5310                                         } else {
5311                                                 current = OSSet::withObjects(
5312                                                         (const OSObject **) &service, 1, 1 );
5313                                         }
5314                                 }
5315                                 source->removeObject(service);
5316                         }
5317                         source->release();
5318                 }
5319         }
5320
5321 #if MATCH_DEBUG
5322         {
5323                 OSObject * _current = 0;
5324
5325                 iter = IORegistryIterator::iterateOver( gIOServicePlane,
5326                     kIORegistryIterateRecursively );
5327                 if (iter) {
5328                         do {
5329                                 iter->reset();
5330                                 while ((service = (IOService *) iter->getNextObject())) {
5331                                         if ((inState == (service->__state[0] & inState))
5332                                             && (0 == (service->__state[0] & kIOServiceInactiveState))
5333                                             && service->matchPassive(matching, 0)) {
5334                                                 if (options & kIONotifyOnce) {
5335                                                         service->retain();
5336                                                         _current = service;
5337                                                         break;
5338                                                 }
5339                                                 if (_current) {
5340                                                         ((OSSet *)_current)->setObject( service );
5341                                                 } else {
5342                                                         _current = OSSet::withObjects(
5343                                                                 (const OSObject **) &service, 1, 1 );
5344                                                 }
5345                                         }
5346                                 }
5347                         } while (!service && !iter->isValid());
5348                         iter->release();
5349                 }
5350
5351                 if (((current != 0) != (_current != 0))
5352                     || (current && _current && !current->isEqualTo(_current))) {
5353                         OSSerialize * s1 = OSSerialize::withCapacity(128);
5354                         OSSerialize * s2 = OSSerialize::withCapacity(128);
5355                         current->serialize(s1);
5356                         _current->serialize(s2);
5357                         kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
5358                             IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
5359                         s1->release();
5360                         s2->release();
5361                 }
5362
5363                 if (_current) {
5364                         _current->release();
5365                 }
5366         }
5367
5368         s->release();
5369 #endif
5370
5371         if (current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
5372                 iter = OSCollectionIterator::withCollection((OSSet *)current );
5373                 current->release();
5374                 current = iter;
5375         }
5376
5377         return current;
5378 }
5379
5380 // public version
5381 OSIterator *
5382 IOService::getMatchingServices( OSDictionary * matching )
5383 {
5384         OSIterator *        iter;
5385
5386         // is a lock even needed?
5387         LOCKWRITENOTIFY();
5388
5389         iter = (OSIterator *) copyExistingServices( matching,
5390             kIOServiceMatchedState );
5391
5392         UNLOCKNOTIFY();
5393
5394         return iter;
5395 }
5396
5397 IOService *
5398 IOService::copyMatchingService( OSDictionary * matching )
5399 {
5400         IOService * service;
5401
5402         // is a lock even needed?
5403         LOCKWRITENOTIFY();
5404
5405         service = (IOService *) copyExistingServices( matching,
5406             kIOServiceMatchedState, kIONotifyOnce );
5407
5408         UNLOCKNOTIFY();
5409
5410         return service;
5411 }
5412
5413 struct _IOServiceMatchingNotificationHandlerRef {
5414         IOServiceNotificationHandler handler;
5415         void * ref;
5416 };
5417
5418 static bool
5419 _IOServiceMatchingNotificationHandler( void * target, void * refCon,
5420     IOService * newService,
5421     IONotifier * notifier )
5422 {
5423         return (*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService);
5424 }
5425
5426 // internal - call with gNotificationLock
5427 IONotifier *
5428 IOService::setNotification(
5429         const OSSymbol * type, OSDictionary * matching,
5430         IOServiceMatchingNotificationHandler handler, void * target, void * ref,
5431         SInt32 priority )
5432 {
5433         _IOServiceNotifier * notify = NULL;
5434         OSOrderedSet *      set;
5435
5436         if (!matching) {
5437                 return NULL;
5438         }
5439
5440         notify = new _IOServiceNotifier;
5441         if (notify && !notify->init()) {
5442                 notify->release();
5443                 notify = NULL;
5444         }
5445
5446         if (notify) {
5447                 notify->handler = handler;
5448                 notify->target = target;
5449                 notify->type = type;
5450                 notify->matching = matching;
5451                 matching->retain();
5452                 if (handler == &_IOServiceMatchingNotificationHandler) {
5453                         notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
5454                         notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
5455                 } else {
5456                         notify->ref = ref;
5457                 }
5458                 notify->priority = priority;
5459                 notify->state = kIOServiceNotifyEnable;
5460                 queue_init( &notify->handlerInvocations );
5461
5462                 ////// queue
5463
5464                 if (NULL == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
5465                         set = OSOrderedSet::withCapacity( 1,
5466                             IONotifyOrdering, NULL );
5467                         if (set) {
5468                                 gNotifications->setObject( type, set );
5469                                 set->release();
5470                         }
5471                 }
5472                 notify->whence = set;
5473                 if (set) {
5474                         set->setObject( notify );
5475                 }
5476         }
5477
5478         return notify;
5479 }
5480
5481 // internal - call with gNotificationLock
5482 IONotifier *
5483 IOService::doInstallNotification(
5484         const OSSymbol * type, OSDictionary * matching,
5485         IOServiceMatchingNotificationHandler handler,
5486         void * target, void * ref,
5487         SInt32 priority, OSIterator ** existing )
5488 {
5489         OSIterator *        exist;
5490         IONotifier *        notify;
5491         IOOptionBits        inState;
5492
5493         if (!matching) {
5494                 return NULL;
5495         }
5496
5497         if (type == gIOPublishNotification) {
5498                 inState = kIOServiceRegisteredState;
5499         } else if (type == gIOFirstPublishNotification) {
5500                 inState = kIOServiceFirstPublishState;
5501         } else if (type == gIOMatchedNotification) {
5502                 inState = kIOServiceMatchedState;
5503         } else if (type == gIOFirstMatchNotification) {
5504                 inState = kIOServiceFirstMatchState;
5505         } else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) {
5506                 inState = 0;
5507         } else {
5508                 return NULL;
5509         }
5510
5511         notify = setNotification( type, matching, handler, target, ref, priority );
5512
5513         if (inState) {
5514                 // get the current set
5515                 exist = (OSIterator *) copyExistingServices( matching, inState );
5516         } else {
5517                 exist = NULL;
5518         }
5519
5520         *existing = exist;
5521
5522         return notify;
5523 }
5524
5525 #if !defined(__LP64__)
5526 IONotifier *
5527 IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
5528     IOServiceNotificationHandler handler,
5529     void * target, void * refCon,
5530     SInt32 priority, OSIterator ** existing )
5531 {
5532         IONotifier * result;
5533         _IOServiceMatchingNotificationHandlerRef ref;
5534         ref.handler = handler;
5535         ref.ref     = refCon;
5536
5537         result = (_IOServiceNotifier *) installNotification( type, matching,
5538             &_IOServiceMatchingNotificationHandler,
5539             target, &ref, priority, existing );
5540         if (result) {
5541                 matching->release();
5542         }
5543
5544         return result;
5545 }
5546 #endif /* !defined(__LP64__) */
5547
5548
5549 IONotifier *
5550 IOService::installNotification(
5551         const OSSymbol * type, OSDictionary * matching,
5552         IOServiceMatchingNotificationHandler handler,
5553         void * target, void * ref,
5554         SInt32 priority, OSIterator ** existing )
5555 {
5556         IONotifier * notify;
5557
5558         LOCKWRITENOTIFY();
5559
5560         notify = doInstallNotification( type, matching, handler, target, ref,
5561             priority, existing );
5562
5563         // in case handler remove()s
5564         if (notify) {
5565                 notify->retain();
5566         }
5567
5568         UNLOCKNOTIFY();
5569
5570         return notify;
5571 }
5572
5573 IONotifier *
5574 IOService::addNotification(
5575         const OSSymbol * type, OSDictionary * matching,
5576         IOServiceNotificationHandler handler,
5577         void * target, void * refCon,
5578         SInt32 priority )
5579 {
5580         IONotifier * result;
5581         _IOServiceMatchingNotificationHandlerRef ref;
5582
5583         ref.handler = handler;
5584         ref.ref     = refCon;
5585
5586         result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
5587             target, &ref, priority);
5588
5589         if (result) {
5590                 matching->release();
5591         }
5592
5593         return result;
5594 }
5595
5596 IONotifier *
5597 IOService::addMatchingNotification(
5598         const OSSymbol * type, OSDictionary * matching,
5599         IOServiceMatchingNotificationHandler handler,
5600         void * target, void * ref,
5601         SInt32 priority )
5602 {
5603         OSIterator *                existing = NULL;
5604         IONotifier *                ret;
5605         _IOServiceNotifier *        notify;
5606         IOService *                 next;
5607
5608         ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
5609             handler, target, ref, priority, &existing );
5610         if (!ret) {
5611                 return NULL;
5612         }
5613
5614         // send notifications for existing set
5615         if (existing) {
5616                 while ((next = (IOService *) existing->getNextObject())) {
5617                         if (0 == (next->__state[0] & kIOServiceInactiveState)) {
5618                                 next->invokeNotifier( notify );
5619                         }
5620                 }
5621                 existing->release();
5622         }
5623
5624         LOCKWRITENOTIFY();
5625         bool removed = (NULL == notify->whence);
5626         notify->release();
5627         if (removed) {
5628                 ret = gIOServiceNullNotifier;
5629         }
5630         UNLOCKNOTIFY();
5631
5632         return ret;
5633 }
5634
5635 static bool
5636 IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
5637     IOService * newService,
5638     IONotifier * notifier )
5639 {
5640         return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
5641 }
5642
5643 IONotifier *
5644 IOService::addMatchingNotification(
5645         const OSSymbol * type, OSDictionary * matching,
5646         SInt32 priority,
5647         IOServiceMatchingNotificationHandlerBlock handler)
5648 {
5649         IONotifier * notify;
5650         void       * block;
5651
5652         block = Block_copy(handler);
5653         if (!block) {
5654                 return NULL;
5655         }
5656
5657         notify = addMatchingNotification(type, matching,
5658             &IOServiceMatchingNotificationHandlerToBlock, NULL, block, priority);
5659
5660         if (!notify) {
5661                 Block_release(block);
5662         }
5663
5664         return notify;
5665 }
5666
5667
5668 bool
5669 IOService::syncNotificationHandler(
5670         void * /* target */, void * ref,
5671         IOService * newService,
5672         IONotifier * notifier )
5673 {
5674         LOCKWRITENOTIFY();
5675         if (!*((IOService **) ref)) {
5676                 newService->retain();
5677                 (*(IOService **) ref) = newService;
5678                 WAKEUPNOTIFY(ref);
5679         }
5680         UNLOCKNOTIFY();
5681
5682         return false;
5683 }
5684
5685 IOService *
5686 IOService::waitForMatchingService( OSDictionary * matching,
5687     uint64_t timeout)
5688 {
5689         IONotifier *        notify = NULL;
5690         // priority doesn't help us much since we need a thread wakeup
5691         SInt32              priority = 0;
5692         IOService *         result;
5693
5694         if (!matching) {
5695                 return NULL;
5696         }
5697
5698         result = NULL;
5699
5700         LOCKWRITENOTIFY();
5701         do{
5702                 result = (IOService *) copyExistingServices( matching,
5703                     kIOServiceMatchedState, kIONotifyOnce );
5704                 if (result) {
5705                         break;
5706                 }
5707                 notify = IOService::setNotification( gIOMatchedNotification, matching,
5708                     &IOService::syncNotificationHandler, (void *) NULL,
5709                     &result, priority );
5710                 if (!notify) {
5711                         break;
5712                 }
5713                 if (UINT64_MAX != timeout) {
5714                         AbsoluteTime deadline;
5715                         nanoseconds_to_absolutetime(timeout, &deadline);
5716                         clock_absolutetime_interval_to_deadline(deadline, &deadline);
5717                         SLEEPNOTIFYTO(&result, deadline);
5718                 } else {
5719                         SLEEPNOTIFY(&result);
5720                 }
5721         }while (false);
5722
5723         UNLOCKNOTIFY();
5724
5725         if (notify) {
5726                 notify->remove(); // dequeues
5727         }
5728         return result;
5729 }
5730
5731 IOService *
5732 IOService::waitForService( OSDictionary * matching,
5733     mach_timespec_t * timeout )
5734 {
5735         IOService * result;
5736         uint64_t    timeoutNS;
5737
5738         if (timeout) {
5739                 timeoutNS = timeout->tv_sec;
5740                 timeoutNS *= kSecondScale;
5741                 timeoutNS += timeout->tv_nsec;
5742         } else {
5743                 timeoutNS = UINT64_MAX;
5744         }
5745
5746         result = waitForMatchingService(matching, timeoutNS);
5747
5748         matching->release();
5749         if (result) {
5750                 result->release();
5751         }
5752
5753         return result;
5754 }
5755
5756 __dead2
5757 void
5758 IOService::deliverNotification( const OSSymbol * type,
5759     IOOptionBits orNewState, IOOptionBits andNewState )
5760 {
5761         panic("deliverNotification");
5762 }
5763
5764 OSArray *
5765 IOService::copyNotifiers(const OSSymbol * type,
5766     IOOptionBits orNewState, IOOptionBits andNewState )
5767 {
5768         _IOServiceNotifier * notify;
5769         OSIterator *         iter;
5770         OSArray *            willSend = NULL;
5771
5772         lockForArbitration();
5773
5774         if ((0 == (__state[0] & kIOServiceInactiveState))
5775             || (type == gIOTerminatedNotification)
5776             || (type == gIOWillTerminateNotification)) {
5777                 LOCKREADNOTIFY();
5778
5779                 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
5780                     gNotifications->getObject( type ));
5781
5782                 if (iter) {
5783                         while ((notify = (_IOServiceNotifier *) iter->getNextObject())) {
5784                                 if (matchPassive(notify->matching, 0)
5785                                     && (kIOServiceNotifyEnable & notify->state)) {
5786                                         if (NULL == willSend) {
5787                                                 willSend = OSArray::withCapacity(8);
5788                                         }
5789                                         if (willSend) {
5790                                                 willSend->setObject( notify );
5791                                         }
5792                                 }
5793                         }
5794                         iter->release();
5795                 }
5796                 __state[0] = (__state[0] | orNewState) & andNewState;
5797                 UNLOCKNOTIFY();
5798         }
5799
5800         unlockForArbitration();
5801
5802         return willSend;
5803 }
5804
5805 IOOptionBits
5806 IOService::getState( void ) const
5807 {
5808         return __state[0];
5809 }
5810
5811 /*
5812  * Helpers to make matching objects for simple cases
5813  */
5814
5815 OSDictionary *
5816 IOService::serviceMatching( const OSString * name,
5817     OSDictionary * table )
5818 {
5819         const OSString *    str;
5820
5821         str = OSSymbol::withString(name);
5822         if (!str) {
5823                 return NULL;
5824         }
5825
5826         if (!table) {
5827                 table = OSDictionary::withCapacity( 2 );
5828         }
5829         if (table) {
5830                 table->setObject(gIOProviderClassKey, (OSObject *)str );
5831         }
5832         str->release();
5833
5834         return table;
5835 }
5836
5837 OSDictionary *
5838 IOService::serviceMatching( const char * name,
5839     OSDictionary * table )
5840 {
5841         const OSString *    str;
5842
5843         str = OSSymbol::withCString( name );
5844         if (!str) {
5845                 return NULL;
5846         }
5847
5848         table = serviceMatching( str, table );
5849         str->release();
5850         return table;
5851 }
5852
5853 OSDictionary *
5854 IOService::nameMatching( const OSString * name,
5855     OSDictionary * table )
5856 {
5857         if (!table) {
5858                 table = OSDictionary::withCapacity( 2 );
5859         }
5860         if (table) {
5861                 table->setObject( gIONameMatchKey, (OSObject *)name );
5862         }
5863
5864         return table;
5865 }
5866
5867 OSDictionary *
5868 IOService::nameMatching( const char * name,
5869     OSDictionary * table )
5870 {
5871         const OSString *    str;
5872
5873         str = OSSymbol::withCString( name );
5874         if (!str) {
5875                 return NULL;
5876         }
5877
5878         table = nameMatching( str, table );
5879         str->release();
5880         return table;
5881 }
5882
5883 OSDictionary *
5884 IOService::resourceMatching( const OSString * str,
5885     OSDictionary * table )
5886 {
5887         table = serviceMatching( gIOResourcesKey, table );
5888         if (table) {
5889                 table->setObject( gIOResourceMatchKey, (OSObject *) str );
5890         }
5891
5892         return table;
5893 }
5894
5895 OSDictionary *
5896 IOService::resourceMatching( const char * name,
5897     OSDictionary * table )
5898 {
5899         const OSSymbol *    str;
5900
5901         str = OSSymbol::withCString( name );
5902         if (!str) {
5903                 return NULL;
5904         }
5905
5906         table = resourceMatching( str, table );
5907         str->release();
5908
5909         return table;
5910 }
5911
5912 OSDictionary *
5913 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
5914     OSDictionary * table )
5915 {
5916         OSDictionary * properties;
5917
5918         properties = OSDictionary::withCapacity( 2 );
5919         if (!properties) {
5920                 return NULL;
5921         }
5922         properties->setObject( key, value );
5923
5924         if (!table) {
5925                 table = OSDictionary::withCapacity( 2 );
5926         }
5927         if (table) {
5928                 table->setObject( gIOPropertyMatchKey, properties );
5929         }
5930
5931         properties->release();
5932
5933         return table;
5934 }
5935
5936 OSDictionary *
5937 IOService::registryEntryIDMatching( uint64_t entryID,
5938     OSDictionary * table )
5939 {
5940         OSNumber *     num;
5941
5942         num = OSNumber::withNumber( entryID, 64 );
5943         if (!num) {
5944                 return NULL;
5945         }
5946
5947         if (!table) {
5948                 table = OSDictionary::withCapacity( 2 );
5949         }
5950         if (table) {
5951                 table->setObject( gIORegistryEntryIDKey, num );
5952         }
5953
5954         if (num) {
5955                 num->release();
5956         }
5957
5958         return table;
5959 }
5960
5961
5962 /*
5963  * _IOServiceNotifier
5964  */
5965
5966 // wait for all threads, other than the current one,
5967 //  to exit the handler
5968
5969 void
5970 _IOServiceNotifier::wait()
5971 {
5972         _IOServiceNotifierInvocation * next;
5973         bool doWait;
5974
5975         do {
5976                 doWait = false;
5977                 queue_iterate( &handlerInvocations, next,
5978                     _IOServiceNotifierInvocation *, link) {
5979                         if (next->thread != current_thread()) {
5980                                 doWait = true;
5981                                 break;
5982                         }
5983                 }
5984                 if (doWait) {
5985                         state |= kIOServiceNotifyWaiter;
5986                         SLEEPNOTIFY(this);
5987                 }
5988         } while (doWait);
5989 }
5990
5991 void
5992 _IOServiceNotifier::free()
5993 {
5994         assert( queue_empty( &handlerInvocations ));
5995
5996         if (handler == &IOServiceMatchingNotificationHandlerToBlock) {
5997                 Block_release(ref);
5998         }
5999
6000         OSObject::free();
6001 }
6002
6003 void
6004 _IOServiceNotifier::remove()
6005 {
6006         LOCKWRITENOTIFY();
6007
6008         if (whence) {
6009                 whence->removeObject((OSObject *) this );
6010                 whence = NULL;
6011         }
6012         if (matching) {
6013                 matching->release();
6014                 matching = NULL;
6015         }
6016
6017         state &= ~kIOServiceNotifyEnable;
6018
6019         wait();
6020
6021         UNLOCKNOTIFY();
6022
6023         release();
6024 }
6025
6026 bool
6027 _IOServiceNotifier::disable()
6028 {
6029         bool        ret;
6030
6031         LOCKWRITENOTIFY();
6032
6033         ret = (0 != (kIOServiceNotifyEnable & state));
6034         state &= ~kIOServiceNotifyEnable;
6035         if (ret) {
6036                 wait();
6037         }
6038
6039         UNLOCKNOTIFY();
6040
6041         return ret;
6042 }
6043
6044 void
6045 _IOServiceNotifier::enable( bool was )
6046 {
6047         LOCKWRITENOTIFY();
6048         if (was) {
6049                 state |= kIOServiceNotifyEnable;
6050         } else {
6051                 state &= ~kIOServiceNotifyEnable;
6052         }
6053         UNLOCKNOTIFY();
6054 }
6055
6056
6057 /*
6058  * _IOServiceNullNotifier
6059  */
6060
6061 void
6062 _IOServiceNullNotifier::taggedRetain(const void *tag) const
6063 {
6064 }
6065 void
6066 _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const
6067 {
6068 }
6069 void
6070 _IOServiceNullNotifier::free()
6071 {
6072 }
6073 void
6074 _IOServiceNullNotifier::wait()
6075 {
6076 }
6077 void
6078 _IOServiceNullNotifier::remove()
6079 {
6080 }
6081 void
6082 _IOServiceNullNotifier::enable(bool was)
6083 {
6084 }
6085 bool
6086 _IOServiceNullNotifier::disable()
6087 {
6088         return false;
6089 }
6090
6091 /*
6092  * IOResources
6093  */
6094
6095 IOService *
6096 IOResources::resources( void )
6097 {
6098         IOResources *       inst;
6099
6100         inst = new IOResources;
6101         if (inst && !inst->init()) {
6102                 inst->release();
6103                 inst = NULL;
6104         }
6105
6106         return inst;
6107 }
6108
6109 bool
6110 IOResources::init( OSDictionary * dictionary )
6111 {
6112         // Do super init first
6113         if (!IOService::init()) {
6114                 return false;
6115         }
6116
6117         // Allow PAL layer to publish a value
6118         const char *property_name;
6119         int property_value;
6120
6121         pal_get_resource_property( &property_name, &property_value );
6122
6123         if (property_name) {
6124                 OSNumber *num;
6125                 const OSSymbol *        sym;
6126
6127                 if ((num = OSNumber::withNumber(property_value, 32)) != NULL) {
6128                         if ((sym = OSSymbol::withCString( property_name)) != NULL) {
6129                                 this->setProperty( sym, num );
6130                                 sym->release();
6131                         }
6132                         num->release();
6133                 }
6134         }
6135
6136         return true;
6137 }
6138
6139 IOReturn
6140 IOResources::newUserClient(task_t owningTask, void * securityID,
6141     UInt32 type, OSDictionary * properties,
6142     IOUserClient ** handler)
6143 {
6144         return kIOReturnUnsupported;
6145 }
6146
6147 IOWorkLoop *
6148 IOResources::getWorkLoop() const
6149 {
6150         // If we are the resource root
6151         // then use the platform's workloop
6152         if (this == (IOResources *) gIOResources) {
6153                 return getPlatform()->getWorkLoop();
6154         } else {
6155                 return IOService::getWorkLoop();
6156         }
6157 }
6158
6159 static bool
6160 IOResourcesMatchPropertyTable(IOService * resources, OSDictionary * table)
6161 {
6162         OSObject *          prop;
6163         OSString *          str;
6164         OSSet *             set;
6165         OSIterator *        iter;
6166         OSObject *          obj;
6167         OSArray *           keys;
6168         bool                ok = true;
6169
6170         prop = table->getObject( gIOResourceMatchKey );
6171         str = OSDynamicCast( OSString, prop );
6172         if (str) {
6173                 ok = (NULL != resources->getProperty( str ));
6174         } else if ((set = OSDynamicCast( OSSet, prop))) {
6175                 iter = OSCollectionIterator::withCollection( set );
6176                 ok = (iter != NULL);
6177                 while (ok && (str = OSDynamicCast( OSString, iter->getNextObject()))) {
6178                         ok = (NULL != resources->getProperty( str ));
6179                 }
6180
6181                 if (iter) {
6182                         iter->release();
6183                 }
6184         } else if ((prop = table->getObject(gIOResourceMatchedKey))) {
6185                 obj = resources->copyProperty(gIOResourceMatchedKey);
6186                 keys = OSDynamicCast(OSArray, obj);
6187                 ok = false;
6188                 if (keys) {
6189                         // assuming OSSymbol
6190                         ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
6191                 }
6192                 OSSafeReleaseNULL(obj);
6193         }
6194
6195         return ok;
6196 }
6197
6198 bool
6199 IOResources::matchPropertyTable( OSDictionary * table )
6200 {
6201         return IOResourcesMatchPropertyTable(this, table);
6202 }
6203
6204 /*
6205  * IOUserResources
6206  */
6207
6208 IOService *
6209 IOUserResources::resources( void )
6210 {
6211         IOUserResources *       inst;
6212
6213         inst = OSTypeAlloc(IOUserResources);
6214         if (inst && !inst->init()) {
6215                 inst->release();
6216                 inst = NULL;
6217         }
6218
6219         return inst;
6220 }
6221
6222 bool
6223 IOUserResources::init( OSDictionary * dictionary )
6224 {
6225         // Do super init first
6226         if (!IOService::init()) {
6227                 return false;
6228         }
6229         return true;
6230 }
6231
6232 IOReturn
6233 IOUserResources::newUserClient(task_t owningTask, void * securityID,
6234     UInt32 type, OSDictionary * properties,
6235     IOUserClient ** handler)
6236 {
6237         return kIOReturnUnsupported;
6238 }
6239
6240 IOWorkLoop *
6241 IOUserResources::getWorkLoop() const
6242 {
6243         return getPlatform()->getWorkLoop();
6244 }
6245
6246 bool
6247 IOUserResources::matchPropertyTable( OSDictionary * table )
6248 {
6249         return IOResourcesMatchPropertyTable(this, table);
6250 }
6251
6252 // --
6253
6254 void
6255 IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
6256 {
6257         IOService::updateConsoleUsers(NULL, 0);
6258 }
6259
6260 void
6261 IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage)
6262 {
6263         IORegistryEntry * regEntry;
6264         OSObject *        locked = kOSBooleanFalse;
6265         uint32_t          idx;
6266         bool              publish;
6267         OSDictionary *    user;
6268         static IOMessage  sSystemPower;
6269         clock_sec_t       now = 0;
6270         clock_usec_t      microsecs;
6271
6272         regEntry = IORegistryEntry::getRegistryRoot();
6273
6274         if (!gIOChosenEntry) {
6275                 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
6276         }
6277
6278         IOLockLock(gIOConsoleUsersLock);
6279
6280         if (systemMessage) {
6281                 sSystemPower = systemMessage;
6282 #if HIBERNATION
6283                 if (kIOMessageSystemHasPoweredOn == systemMessage) {
6284                         uint32_t lockState = IOHibernateWasScreenLocked();
6285                         switch (lockState) {
6286                         case 0:
6287                                 break;
6288                         case kIOScreenLockLocked:
6289                         case kIOScreenLockFileVaultDialog:
6290                                 gIOConsoleBooterLockState = kOSBooleanTrue;
6291                                 break;
6292                         case kIOScreenLockNoLock:
6293                                 gIOConsoleBooterLockState = NULL;
6294                                 break;
6295                         case kIOScreenLockUnlocked:
6296                         default:
6297                                 gIOConsoleBooterLockState = kOSBooleanFalse;
6298                                 break;
6299                         }
6300                 }
6301 #endif /* HIBERNATION */
6302         }
6303
6304         if (consoleUsers) {
6305                 OSNumber * num = NULL;
6306                 bool       loginLocked = true;
6307
6308                 gIOConsoleLoggedIn = false;
6309                 for (idx = 0;
6310                     (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
6311                     idx++) {
6312                         gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
6313                             && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
6314
6315                         loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
6316                         if (!num) {
6317                                 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
6318                         }
6319                 }
6320 #if HIBERNATION
6321                 if (!loginLocked) {
6322                         gIOConsoleBooterLockState = NULL;
6323                 }
6324                 IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
6325                     (num != NULL), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
6326                     gIOConsoleLoggedIn, loginLocked);
6327 #endif /* HIBERNATION */
6328                 gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
6329         }
6330
6331         if (!gIOConsoleLoggedIn
6332             || (kIOMessageSystemWillSleep == sSystemPower)
6333             || (kIOMessageSystemPagingOff == sSystemPower)) {
6334                 locked = kOSBooleanTrue;
6335         }
6336 #if HIBERNATION
6337         else if (gIOConsoleBooterLockState) {
6338                 locked = gIOConsoleBooterLockState;
6339         }
6340 #endif /* HIBERNATION */
6341         else if (gIOConsoleLockTime) {
6342                 clock_get_calendar_microtime(&now, &microsecs);
6343                 if (gIOConsoleLockTime > now) {
6344                         AbsoluteTime deadline;
6345                         clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline);
6346                         thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
6347                 } else {
6348                         locked = kOSBooleanTrue;
6349                 }
6350         }
6351
6352         publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
6353         if (publish) {
6354                 regEntry->setProperty(gIOConsoleLockedKey, locked);
6355                 if (consoleUsers) {
6356                         regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
6357                 }
6358                 OSIncrementAtomic( &gIOConsoleUsersSeed );
6359         }
6360
6361 #if HIBERNATION
6362         if (gIOChosenEntry) {
6363                 if (locked == kOSBooleanTrue) {
6364                         gIOScreenLockState = kIOScreenLockLocked;
6365                 } else if (gIOConsoleLockTime) {
6366                         gIOScreenLockState = kIOScreenLockUnlocked;
6367                 } else {
6368                         gIOScreenLockState = kIOScreenLockNoLock;
6369                 }
6370                 gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
6371
6372                 IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
6373                     gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != NULL), now, systemMessage);
6374         }
6375 #endif /* HIBERNATION */
6376
6377         IOLockUnlock(gIOConsoleUsersLock);
6378
6379         if (publish) {
6380                 publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
6381
6382                 MessageClientsContext context;
6383
6384                 context.service  = getServiceRoot();
6385                 context.type     = kIOMessageConsoleSecurityChange;
6386                 context.argument = (void *) regEntry;
6387                 context.argSize  = 0;
6388
6389                 applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
6390                     &messageClientsApplier, &context );
6391         }
6392 }
6393
6394 IOReturn
6395 IOResources::setProperties( OSObject * properties )
6396 {
6397         IOReturn                    err;
6398         const OSSymbol *            key;
6399         OSDictionary *              dict;
6400         OSCollectionIterator *      iter;
6401
6402         err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
6403         if (kIOReturnSuccess != err) {
6404                 return err;
6405         }
6406
6407         dict = OSDynamicCast(OSDictionary, properties);
6408         if (NULL == dict) {
6409                 return kIOReturnBadArgument;
6410         }
6411
6412         iter = OSCollectionIterator::withCollection( dict);
6413         if (NULL == iter) {
6414                 return kIOReturnBadArgument;
6415         }
6416
6417         while ((key = OSDynamicCast(OSSymbol, iter->getNextObject()))) {
6418                 if (gIOConsoleUsersKey == key) {
6419                         do{
6420                                 OSArray * consoleUsers;
6421                                 consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
6422                                 if (!consoleUsers) {
6423                                         continue;
6424                                 }
6425                                 IOService::updateConsoleUsers(consoleUsers, 0);
6426                         }while (false);
6427                 }
6428
6429                 publishResource( key, dict->getObject(key));
6430         }
6431
6432         iter->release();
6433
6434         return kIOReturnSuccess;
6435 }
6436
6437 /*
6438  * Helpers for matching dictionaries.
6439  * Keys existing in matching are checked in properties.
6440  * Keys may be a string or OSCollection of IOStrings
6441  */
6442
6443 bool
6444 IOService::compareProperty( OSDictionary * matching,
6445     const char *   key )
6446 {
6447         OSObject *  value;
6448         OSObject *  prop;
6449         bool        ok;
6450
6451         value = matching->getObject( key );
6452         if (value) {
6453                 prop = copyProperty(key);
6454                 ok = value->isEqualTo(prop);
6455                 if (prop) {
6456                         prop->release();
6457                 }
6458         } else {
6459                 ok = true;
6460         }
6461
6462         return ok;
6463 }
6464
6465
6466 bool
6467 IOService::compareProperty( OSDictionary *   matching,
6468     const OSString * key )
6469 {
6470         OSObject *  value;
6471         OSObject *  prop;
6472         bool        ok;
6473
6474         value = matching->getObject( key );
6475         if (value) {
6476                 prop = copyProperty(key);
6477                 ok = value->isEqualTo(prop);
6478                 if (prop) {
6479                         prop->release();
6480                 }
6481         } else {
6482                 ok = true;
6483         }
6484
6485         return ok;
6486 }
6487
6488 bool
6489 IOService::compareProperties( OSDictionary * matching,
6490     OSCollection * keys )
6491 {
6492         OSCollectionIterator *      iter;
6493         const OSString *            key;
6494         bool                        ok = true;
6495
6496         if (!matching || !keys) {
6497                 return false;
6498         }
6499
6500         iter = OSCollectionIterator::withCollection( keys );
6501
6502         if (iter) {
6503                 while (ok && (key = OSDynamicCast( OSString, iter->getNextObject()))) {
6504                         ok = compareProperty( matching, key );
6505                 }
6506
6507                 iter->release();
6508         }
6509         keys->release(); // !! consume a ref !!
6510
6511         return ok;
6512 }
6513
6514 /* Helper to add a location matching dict to the table */
6515
6516 OSDictionary *
6517 IOService::addLocation( OSDictionary * table )
6518 {
6519         OSDictionary *      dict;
6520
6521         if (!table) {
6522                 return NULL;
6523         }
6524
6525         dict = OSDictionary::withCapacity( 1 );
6526         if (dict) {
6527                 bool ok = table->setObject( gIOLocationMatchKey, dict );
6528                 dict->release();
6529                 if (!ok) {
6530                         dict = NULL;
6531                 }
6532         }
6533
6534         return dict;
6535 }
6536
6537 /*
6538  * Go looking for a provider to match a location dict.
6539  */
6540
6541 IOService *
6542 IOService::matchLocation( IOService * /* client */ )
6543 {
6544         IOService * parent;
6545
6546         parent = getProvider();
6547
6548         if (parent) {
6549                 parent = parent->matchLocation( this );
6550         }
6551
6552         return parent;
6553 }
6554
6555 bool
6556 IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
6557 {
6558         OSString *          matched;
6559         OSObject *          obj;
6560         OSString *          str;
6561         IORegistryEntry *   entry;
6562         OSNumber *          num;
6563         bool                match = true;
6564         bool                changesOK = (0 != (kIOServiceChangesOK & options));
6565         uint32_t            count;
6566         uint32_t            done;
6567
6568         do{
6569                 count = table->getCount();
6570                 done = 0;
6571
6572                 str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
6573                 if (str) {
6574                         done++;
6575                         match = ((kIOServiceClassDone & options) || (NULL != metaCast(str)));
6576 #if MATCH_DEBUG
6577                         match = (0 != metaCast( str ));
6578                         if ((kIOServiceClassDone & options) && !match) {
6579                                 panic("classDone");
6580                         }
6581 #endif
6582                         if ((!match) || (done == count)) {
6583                                 break;
6584                         }
6585                 }
6586
6587                 obj = table->getObject( gIONameMatchKey );
6588                 if (obj) {
6589                         done++;
6590                         match = compareNames( obj, changesOK ? &matched : NULL );
6591                         if (!match) {
6592                                 break;
6593                         }
6594                         if (changesOK && matched) {
6595                                 // leave a hint as to which name matched
6596                                 table->setObject( gIONameMatchedKey, matched );
6597                                 matched->release();
6598                         }
6599                         if (done == count) {
6600                                 break;
6601                         }
6602                 }
6603
6604                 str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
6605                 if (str) {
6606                         const OSSymbol * sym;
6607                         done++;
6608                         match = false;
6609                         sym = copyLocation();
6610                         if (sym) {
6611                                 match = sym->isEqualTo( str );
6612                                 sym->release();
6613                         }
6614                         if ((!match) || (done == count)) {
6615                                 break;
6616                         }
6617                 }
6618
6619                 obj = table->getObject( gIOPropertyMatchKey );
6620                 if (obj) {
6621                         OSDictionary * dict;
6622                         OSDictionary * nextDict;
6623                         OSIterator *   iter;
6624                         done++;
6625                         match = false;
6626                         dict = dictionaryWithProperties();
6627                         if (dict) {
6628                                 nextDict = OSDynamicCast( OSDictionary, obj);
6629                                 if (nextDict) {
6630                                         iter = NULL;
6631                                 } else {
6632                                         iter = OSCollectionIterator::withCollection(
6633                                                 OSDynamicCast(OSCollection, obj));
6634                                 }
6635
6636                                 while (nextDict
6637                                     || (iter && (NULL != (nextDict = OSDynamicCast(OSDictionary,
6638                                     iter->getNextObject()))))) {
6639                                         match = dict->isEqualTo( nextDict, nextDict);
6640                                         if (match) {
6641                                                 break;
6642                                         }
6643                                         nextDict = NULL;
6644                                 }
6645                                 dict->release();
6646                                 if (iter) {
6647                                         iter->release();
6648                                 }
6649                         }
6650                         if ((!match) || (done == count)) {
6651                                 break;
6652                         }
6653                 }
6654
6655                 obj = table->getObject( gIOPropertyExistsMatchKey );
6656                 if (obj) {
6657                         OSDictionary * dict;
6658                         OSString     * nextKey;
6659                         OSIterator *   iter;
6660                         done++;
6661                         match = false;
6662                         dict = dictionaryWithProperties();
6663                         if (dict) {
6664                                 nextKey = OSDynamicCast( OSString, obj);
6665                                 if (nextKey) {
6666                                         iter = NULL;
6667                                 } else {
6668                                         iter = OSCollectionIterator::withCollection(
6669                                                 OSDynamicCast(OSCollection, obj));
6670                                 }
6671
6672                                 while (nextKey
6673                                     || (iter && (NULL != (nextKey = OSDynamicCast(OSString,
6674                                     iter->getNextObject()))))) {
6675                                         match = (NULL != dict->getObject(nextKey));
6676                                         if (match) {
6677                                                 break;
6678                                         }
6679                                         nextKey = NULL;
6680                                 }
6681                                 dict->release();
6682                                 if (iter) {
6683                                         iter->release();
6684                                 }
6685                         }
6686                         if ((!match) || (done == count)) {
6687                                 break;
6688                         }
6689                 }
6690
6691                 str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
6692                 if (str) {
6693                         done++;
6694                         entry = IORegistryEntry::fromPath( str->getCStringNoCopy());
6695                         match = (this == entry);
6696                         if (entry) {
6697                                 entry->release();
6698                         }
6699                         if ((!match) || (done == count)) {
6700                                 break;
6701                         }
6702                 }
6703
6704                 num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
6705                 if (num) {
6706                         done++;
6707                         match = (getRegistryEntryID() == num->unsigned64BitValue());
6708                         if ((!match) || (done == count)) {
6709                                 break;
6710                         }
6711                 }
6712
6713                 num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
6714                 if (num) {
6715                         OSIterator *        iter;
6716                         IOService *         service = NULL;
6717                         UInt32              serviceCount = 0;
6718
6719                         done++;
6720                         iter = getClientIterator();
6721                         if (iter) {
6722                                 while ((service = (IOService *) iter->getNextObject())) {
6723                                         if (kIOServiceInactiveState & service->__state[0]) {
6724                                                 continue;
6725                                         }
6726                                         if (NULL == service->getProperty( gIOMatchCategoryKey )) {
6727                                                 continue;
6728                                         }
6729                                         ++serviceCount;
6730                                 }
6731                                 iter->release();
6732                         }
6733                         match = (serviceCount == num->unsigned32BitValue());
6734                         if ((!match) || (done == count)) {
6735                                 break;
6736                         }
6737                 }
6738
6739 #define propMatch(key)                                  \
6740         obj = table->getObject(key);                    \
6741         if (obj)                                        \
6742         {                                               \
6743             OSObject * prop;                            \
6744             done++;                                     \
6745             prop = copyProperty(key);                   \
6746             match = obj->isEqualTo(prop);               \
6747             if (prop) prop->release();                  \
6748             if ((!match) || (done == count)) break;     \
6749         }
6750                 propMatch(gIOBSDNameKey)
6751                 propMatch(gIOBSDMajorKey)
6752                 propMatch(gIOBSDMinorKey)
6753                 propMatch(gIOBSDUnitKey)
6754 #undef propMatch
6755         }while (false);
6756
6757         if (did) {
6758                 *did = done;
6759         }
6760         return match;
6761 }
6762
6763 bool
6764 IOService::passiveMatch( OSDictionary * table, bool changesOK )
6765 {
6766         return matchPassive(table, changesOK ? kIOServiceChangesOK : 0);
6767 }
6768
6769 bool
6770 IOService::matchPassive(OSDictionary * table, uint32_t options)
6771 {
6772         IOService *         where;
6773         OSDictionary *      nextTable;
6774         SInt32              score;
6775         OSNumber *          newPri;
6776         bool                match = true;
6777         bool                matchParent = false;
6778         uint32_t            count;
6779         uint32_t            done;
6780
6781         assert( table );
6782
6783 #if !CONFIG_EMBEDDED
6784         OSArray* aliasServiceRegIds = NULL;
6785         IOService* foundAlternateService = NULL;
6786 #endif
6787
6788 #if MATCH_DEBUG
6789         OSDictionary * root = table;
6790 #endif
6791
6792         where = this;
6793         do{
6794                 do{
6795                         count = table->getCount();
6796                         if (!(kIOServiceInternalDone & options)) {
6797                                 match = where->matchInternal(table, options, &done);
6798                                 // don't call family if we've done all the entries in the table
6799                                 if ((!match) || (done == count)) {
6800                                         break;
6801                                 }
6802                         }
6803
6804                         // pass in score from property table
6805                         score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
6806
6807                         // do family specific matching
6808                         match = where->matchPropertyTable( table, &score );
6809
6810                         if (!match) {
6811 #if IOMATCHDEBUG
6812                                 if (kIOLogMatch & getDebugFlags( table )) {
6813                                         LOG("%s: family specific matching fails\n", where->getName());
6814                                 }
6815 #endif
6816                                 break;
6817                         }
6818
6819                         if (kIOServiceChangesOK & options) {
6820                                 // save the score
6821                                 newPri = OSNumber::withNumber( score, 32 );
6822                                 if (newPri) {
6823                                         table->setObject( gIOProbeScoreKey, newPri );
6824                                         newPri->release();
6825                                 }
6826                         }
6827
6828                         options = 0;
6829                         matchParent = false;
6830
6831                         nextTable = OSDynamicCast(OSDictionary,
6832                             table->getObject( gIOParentMatchKey ));
6833                         if (nextTable) {
6834                                 // look for a matching entry anywhere up to root
6835                                 match = false;
6836                                 matchParent = true;
6837                                 table = nextTable;
6838                                 break;
6839                         }
6840
6841                         table = OSDynamicCast(OSDictionary,
6842                             table->getObject( gIOLocationMatchKey ));
6843                         if (table) {
6844                                 // look for a matching entry at matchLocation()
6845                                 match = false;
6846                                 where = where->getProvider();
6847                                 if (where && (where = where->matchLocation(where))) {
6848                                         continue;
6849                                 }
6850                         }
6851                         break;
6852                 }while (true);
6853
6854                 if (match == true) {
6855                         break;
6856                 }
6857
6858                 if (matchParent == true) {
6859 #if !CONFIG_EMBEDDED
6860                         // check if service has an alias to search its other "parents" if a parent match isn't found
6861                         OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
6862                         OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
6863                         if (alternateRegistryID != NULL) {
6864                                 if (aliasServiceRegIds == NULL) {
6865                                         aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
6866                                 }
6867                                 aliasServiceRegIds->setObject(alternateRegistryID);
6868                         }
6869                         OSSafeReleaseNULL(prop);
6870 #endif
6871                 } else {
6872                         break;
6873                 }
6874
6875                 where = where->getProvider();
6876 #if !CONFIG_EMBEDDED
6877                 if (where == NULL) {
6878                         // there were no matching parent services, check to see if there are aliased services that have a matching parent
6879                         if (aliasServiceRegIds != NULL) {
6880                                 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
6881                                 if (numAliasedServices != 0) {
6882                                         OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
6883                                         if (alternateRegistryID != NULL) {
6884                                                 OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
6885                                                 aliasServiceRegIds->removeObject(numAliasedServices - 1);
6886                                                 if (alternateMatchingDict != NULL) {
6887                                                         OSSafeReleaseNULL(foundAlternateService);
6888                                                         foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
6889                                                         alternateMatchingDict->release();
6890                                                         if (foundAlternateService != NULL) {
6891                                                                 where = foundAlternateService;
6892                                                         }
6893                                                 }
6894                                         }
6895                                 }
6896                         }
6897                 }
6898 #endif
6899         }while (where != NULL);
6900
6901 #if !CONFIG_EMBEDDED
6902         OSSafeReleaseNULL(foundAlternateService);
6903         OSSafeReleaseNULL(aliasServiceRegIds);
6904 #endif
6905
6906 #if MATCH_DEBUG
6907         if (where != this) {
6908                 OSSerialize * s = OSSerialize::withCapacity(128);
6909                 root->serialize(s);
6910                 kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
6911                 s->release();
6912         }
6913 #endif
6914
6915         return match;
6916 }
6917
6918
6919 IOReturn
6920 IOService::newUserClient( task_t owningTask, void * securityID,
6921     UInt32 type, OSDictionary * properties,
6922     IOUserClient ** handler )
6923 {
6924         const OSSymbol *userClientClass = NULL;
6925         IOUserClient *client;
6926         OSObject *prop;
6927         OSObject *temp;
6928
6929         if (reserved && reserved->uvars && reserved->uvars->userServer) {
6930                 return reserved->uvars->userServer->serviceNewUserClient(this, owningTask, securityID, type, properties, handler);
6931         }
6932
6933         if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) {
6934                 return kIOReturnSuccess;
6935         }
6936
6937         // First try my own properties for a user client class name
6938         prop = copyProperty(gIOUserClientClassKey);
6939         if (prop) {
6940                 if (OSDynamicCast(OSSymbol, prop)) {
6941                         userClientClass = (const OSSymbol *) prop;
6942                 } else if (OSDynamicCast(OSString, prop)) {
6943                         userClientClass = OSSymbol::withString((OSString *) prop);
6944                         if (userClientClass) {
6945                                 setProperty(gIOUserClientClassKey,
6946                                     (OSObject *) userClientClass);
6947                         }
6948                 }
6949         }
6950
6951         // Didn't find one so lets just bomb out now without further ado.
6952         if (!userClientClass) {
6953                 OSSafeReleaseNULL(prop);
6954                 return kIOReturnUnsupported;
6955         }
6956
6957         // This reference is consumed by the IOServiceOpen call
6958         temp = OSMetaClass::allocClassWithName(userClientClass);
6959         OSSafeReleaseNULL(prop);
6960         if (!temp) {
6961                 return kIOReturnNoMemory;
6962         }
6963
6964         if (OSDynamicCast(IOUserClient, temp)) {
6965                 client = (IOUserClient *) temp;
6966         } else {
6967                 temp->release();
6968                 return kIOReturnUnsupported;
6969         }
6970
6971         if (!client->initWithTask(owningTask, securityID, type, properties)) {
6972                 client->release();
6973                 return kIOReturnBadArgument;
6974         }
6975
6976         if (!client->attach(this)) {
6977                 client->release();
6978                 return kIOReturnUnsupported;
6979         }
6980
6981         if (!client->start(this)) {
6982                 client->detach(this);
6983                 client->release();
6984                 return kIOReturnUnsupported;
6985         }
6986
6987         *handler = client;
6988         return kIOReturnSuccess;
6989 }
6990
6991 IOReturn
6992 IOService::newUserClient( task_t owningTask, void * securityID,
6993     UInt32 type, IOUserClient ** handler )
6994 {
6995         return kIOReturnUnsupported;
6996 }
6997
6998 IOReturn
6999 IOService::requestProbe( IOOptionBits options )
7000 {
7001         return kIOReturnUnsupported;
7002 }
7003
7004 /*
7005  * Convert an IOReturn to text. Subclasses which add additional
7006  * IOReturn's should override this method and call
7007  * super::stringFromReturn if the desired value is not found.
7008  */
7009
7010 const char *
7011 IOService::stringFromReturn( IOReturn rtn )
7012 {
7013         static const IONamedValue IOReturn_values[] = {
7014                 {kIOReturnSuccess, "success"                           },
7015                 {kIOReturnError, "general error"                     },
7016                 {kIOReturnNoMemory, "memory allocation error"           },
7017                 {kIOReturnNoResources, "resource shortage"                 },
7018                 {kIOReturnIPCError, "Mach IPC failure"                  },
7019                 {kIOReturnNoDevice, "no such device"                    },
7020                 {kIOReturnNotPrivileged, "privilege violation"               },
7021                 {kIOReturnBadArgument, "invalid argument"                  },
7022                 {kIOReturnLockedRead, "device is read locked"             },
7023                 {kIOReturnLockedWrite, "device is write locked"            },
7024                 {kIOReturnExclusiveAccess, "device is exclusive access"        },
7025                 {kIOReturnBadMessageID, "bad IPC message ID"                },
7026                 {kIOReturnUnsupported, "unsupported function"              },
7027                 {kIOReturnVMError, "virtual memory error"              },
7028                 {kIOReturnInternalError, "internal driver error"             },
7029                 {kIOReturnIOError, "I/O error"                         },
7030                 {kIOReturnCannotLock, "cannot acquire lock"               },
7031                 {kIOReturnNotOpen, "device is not open"                },
7032                 {kIOReturnNotReadable, "device is not readable"            },
7033                 {kIOReturnNotWritable, "device is not writeable"           },
7034                 {kIOReturnNotAligned, "alignment error"                   },
7035                 {kIOReturnBadMedia, "media error"                       },
7036                 {kIOReturnStillOpen, "device is still open"              },
7037                 {kIOReturnRLDError, "rld failure"                       },
7038                 {kIOReturnDMAError, "DMA failure"                       },
7039                 {kIOReturnBusy, "device is busy"                    },
7040                 {kIOReturnTimeout, "I/O timeout"                       },
7041                 {kIOReturnOffline, "device is offline"                 },
7042                 {kIOReturnNotReady, "device is not ready"               },
7043                 {kIOReturnNotAttached, "device/channel is not attached"    },
7044                 {kIOReturnNoChannels, "no DMA channels available"         },
7045                 {kIOReturnNoSpace, "no space for data"                 },
7046                 {kIOReturnPortExists, "device port already exists"        },
7047                 {kIOReturnCannotWire, "cannot wire physical memory"       },
7048                 {kIOReturnNoInterrupt, "no interrupt attached"             },
7049                 {kIOReturnNoFrames, "no DMA frames enqueued"            },
7050                 {kIOReturnMessageTooLarge, "message is too large"              },
7051                 {kIOReturnNotPermitted, "operation is not permitted"        },
7052                 {kIOReturnNoPower, "device is without power"           },
7053                 {kIOReturnNoMedia, "media is not present"              },
7054                 {kIOReturnUnformattedMedia, "media is not formatted"            },
7055                 {kIOReturnUnsupportedMode, "unsupported mode"                  },
7056                 {kIOReturnUnderrun, "data underrun"                     },
7057                 {kIOReturnOverrun, "data overrun"                      },
7058                 {kIOReturnDeviceError, "device error"                      },
7059                 {kIOReturnNoCompletion, "no completion routine"             },
7060                 {kIOReturnAborted, "operation was aborted"             },
7061                 {kIOReturnNoBandwidth, "bus bandwidth would be exceeded"   },
7062                 {kIOReturnNotResponding, "device is not responding"          },
7063                 {kIOReturnInvalid, "unanticipated driver error"        },
7064                 {0, NULL                                }
7065         };
7066
7067         return IOFindNameForValue(rtn, IOReturn_values);
7068 }
7069
7070 /*
7071  * Convert an IOReturn to an errno.
7072  */
7073 int
7074 IOService::errnoFromReturn( IOReturn rtn )
7075 {
7076         if (unix_err(err_get_code(rtn)) == rtn) {
7077                 return err_get_code(rtn);
7078         }
7079
7080         switch (rtn) {
7081         // (obvious match)
7082         case kIOReturnSuccess:
7083                 return 0;
7084         case kIOReturnNoMemory:
7085                 return ENOMEM;
7086         case kIOReturnNoDevice:
7087                 return ENXIO;
7088         case kIOReturnVMError:
7089                 return EFAULT;
7090         case kIOReturnNotPermitted:
7091                 return EPERM;
7092         case kIOReturnNotPrivileged:
7093                 return EACCES;
7094         case kIOReturnIOError:
7095                 return EIO;
7096         case kIOReturnNotWritable:
7097                 return EROFS;
7098         case kIOReturnBadArgument:
7099                 return EINVAL;
7100         case kIOReturnUnsupported:
7101                 return ENOTSUP;
7102         case kIOReturnBusy:
7103                 return EBUSY;
7104         case kIOReturnNoPower:
7105                 return EPWROFF;
7106         case kIOReturnDeviceError:
7107                 return EDEVERR;
7108         case kIOReturnTimeout:
7109                 return ETIMEDOUT;
7110         case kIOReturnMessageTooLarge:
7111                 return EMSGSIZE;
7112         case kIOReturnNoSpace:
7113                 return ENOSPC;
7114         case kIOReturnCannotLock:
7115                 return ENOLCK;
7116
7117         // (best match)
7118         case kIOReturnBadMessageID:
7119         case kIOReturnNoCompletion:
7120         case kIOReturnNotAligned:
7121                 return EINVAL;
7122         case kIOReturnNotReady:
7123                 return EBUSY;
7124         case kIOReturnRLDError:
7125                 return EBADMACHO;
7126         case kIOReturnPortExists:
7127         case kIOReturnStillOpen:
7128                 return EEXIST;
7129         case kIOReturnExclusiveAccess:
7130         case kIOReturnLockedRead:
7131         case kIOReturnLockedWrite:
7132         case kIOReturnNotOpen:
7133         case kIOReturnNotReadable:
7134                 return EACCES;
7135         case kIOReturnCannotWire:
7136         case kIOReturnNoResources:
7137                 return ENOMEM;
7138         case kIOReturnAborted:
7139         case kIOReturnOffline:
7140         case kIOReturnNotResponding:
7141                 return EBUSY;
7142         case kIOReturnBadMedia:
7143         case kIOReturnNoMedia:
7144         case kIOReturnNotAttached:
7145         case kIOReturnUnformattedMedia:
7146                 return ENXIO; // (media error)
7147         case kIOReturnDMAError:
7148         case kIOReturnOverrun:
7149         case kIOReturnUnderrun:
7150                 return EIO; // (transfer error)
7151         case kIOReturnNoBandwidth:
7152         case kIOReturnNoChannels:
7153         case kIOReturnNoFrames:
7154         case kIOReturnNoInterrupt:
7155                 return EIO; // (hardware error)
7156         case kIOReturnError:
7157         case kIOReturnInternalError:
7158         case kIOReturnInvalid:
7159                 return EIO; // (generic error)
7160         case kIOReturnIPCError:
7161                 return EIO; // (ipc error)
7162         default:
7163                 return EIO; // (all other errors)
7164         }
7165 }
7166
7167 IOReturn
7168 IOService::message( UInt32 type, IOService * provider,
7169     void * argument )
7170 {
7171         /*
7172          * Generic entry point for calls from the provider.  A return value of
7173          * kIOReturnSuccess indicates that the message was received, and where
7174          * applicable, that it was successful.
7175          */
7176
7177         return kIOReturnUnsupported;
7178 }
7179
7180 /*
7181  * Device memory
7182  */
7183
7184 IOItemCount
7185 IOService::getDeviceMemoryCount( void )
7186 {
7187         OSArray *           array;
7188         IOItemCount         count;
7189
7190         array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
7191         if (array) {
7192                 count = array->getCount();
7193         } else {
7194                 count = 0;
7195         }
7196
7197         return count;
7198 }
7199
7200 IODeviceMemory *
7201 IOService::getDeviceMemoryWithIndex( unsigned int index )
7202 {
7203         OSArray *           array;
7204         IODeviceMemory *    range;
7205
7206         array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
7207         if (array) {
7208                 range = (IODeviceMemory *) array->getObject( index );
7209         } else {
7210                 range = NULL;
7211         }
7212
7213         return range;
7214 }
7215
7216 IOMemoryMap *
7217 IOService::mapDeviceMemoryWithIndex( unsigned int index,
7218     IOOptionBits options )
7219 {
7220         IODeviceMemory *    range;
7221         IOMemoryMap *       map;
7222
7223         range = getDeviceMemoryWithIndex( index );
7224         if (range) {
7225                 map = range->map( options );
7226         } else {
7227                 map = NULL;
7228         }
7229
7230         return map;
7231 }
7232
7233 OSArray *
7234 IOService::getDeviceMemory( void )
7235 {
7236         return OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
7237 }
7238
7239
7240 void
7241 IOService::setDeviceMemory( OSArray * array )
7242 {
7243         setProperty( gIODeviceMemoryKey, array);
7244 }
7245
7246 /*
7247  * For machines where the transfers on an I/O bus can stall because
7248  * the CPU is in an idle mode, These APIs allow a driver to specify
7249  * the maximum bus stall that they can handle.  0 indicates no limit.
7250  */
7251 void
7252 IOService::
7253 setCPUSnoopDelay(UInt32 __unused ns)
7254 {
7255 #if defined(__i386__) || defined(__x86_64__)
7256         ml_set_maxsnoop(ns);
7257 #endif /* defined(__i386__) || defined(__x86_64__) */
7258 }
7259
7260 UInt32
7261 IOService::
7262 getCPUSnoopDelay()
7263 {
7264 #if defined(__i386__) || defined(__x86_64__)
7265         return ml_get_maxsnoop();
7266 #else
7267         return 0;
7268 #endif /* defined(__i386__) || defined(__x86_64__) */
7269 }
7270
7271 #if defined(__i386__) || defined(__x86_64__)
7272 static void
7273 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
7274 {
7275         static const UInt kNoReplace = -1U; // Must be an illegal index
7276         UInt replace = kNoReplace;
7277         bool setCpuDelay = false;
7278
7279         IORecursiveLockLock(sCpuDelayLock);
7280
7281         UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
7282         CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
7283         IOService * holder = NULL;
7284
7285         if (ns) {
7286                 const CpuDelayEntry ne = {service, ns, delayType};
7287                 holder = service;
7288                 // Set maximum delay.
7289                 for (UInt i = 0; i < count; i++) {
7290                         IOService *thisService = entries[i].fService;
7291                         bool sameType = (delayType == entries[i].fDelayType);
7292                         if ((service == thisService) && sameType) {
7293                                 replace = i;
7294                         } else if (!thisService) {
7295                                 if (kNoReplace == replace) {
7296                                         replace = i;
7297                                 }
7298                         } else if (sameType) {
7299                                 const UInt32 thisMax = entries[i].fMaxDelay;
7300                                 if (thisMax < ns) {
7301                                         ns = thisMax;
7302                                         holder = thisService;
7303                                 }
7304                         }
7305                 }
7306
7307                 setCpuDelay = true;
7308                 if (kNoReplace == replace) {
7309                         sCpuDelayData->appendBytes(&ne, sizeof(ne));
7310                 } else {
7311                         entries[replace] = ne;
7312                 }
7313         } else {
7314                 ns = -1U; // Set to max unsigned, i.e. no restriction
7315
7316                 for (UInt i = 0; i < count; i++) {
7317                         // Clear a maximum delay.
7318                         IOService *thisService = entries[i].fService;
7319                         if (thisService && (delayType == entries[i].fDelayType)) {
7320                                 UInt32 thisMax = entries[i].fMaxDelay;
7321                                 if (service == thisService) {
7322                                         replace = i;
7323                                 } else if (thisMax < ns) {
7324                                         ns = thisMax;
7325                                         holder = thisService;
7326                                 }
7327                         }
7328                 }
7329
7330                 // Check if entry found
7331                 if (kNoReplace != replace) {
7332                         entries[replace].fService = NULL; // Null the entry
7333                         setCpuDelay = true;
7334                 }
7335         }
7336
7337         if (setCpuDelay) {
7338                 if (holder && debug_boot_arg) {
7339                         strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
7340                 }
7341
7342                 // Must be safe to call from locked context
7343                 if (delayType == kCpuDelayBusStall) {
7344                         ml_set_maxbusdelay(ns);
7345                 } else if (delayType == kCpuDelayInterrupt) {
7346                         ml_set_maxintdelay(ns);
7347                 }
7348                 sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
7349                 sCPULatencySet[delayType]->setValue(ns);
7350
7351                 OSArray * handlers = sCpuLatencyHandlers[delayType];
7352                 IOService * target;
7353                 if (handlers) {
7354                         for (unsigned int idx = 0;
7355                             (target = (IOService *) handlers->getObject(idx));
7356                             idx++) {
7357                                 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
7358                                     (void *) (uintptr_t) ns, holder,
7359                                     NULL, NULL);
7360                         }
7361                 }
7362         }
7363
7364         IORecursiveLockUnlock(sCpuDelayLock);
7365 }
7366
7367 static IOReturn
7368 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
7369 {
7370         IOReturn result = kIOReturnNotFound;
7371         OSArray * array;
7372         unsigned int idx;
7373
7374         IORecursiveLockLock(sCpuDelayLock);
7375
7376         do{
7377                 if (enable && !sCpuLatencyHandlers[delayType]) {
7378                         sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
7379                 }
7380                 array = sCpuLatencyHandlers[delayType];
7381                 if (!array) {
7382                         break;
7383                 }
7384                 idx = array->getNextIndexOfObject(target, 0);
7385                 if (!enable) {
7386                         if (-1U != idx) {
7387                                 array->removeObject(idx);
7388                                 result = kIOReturnSuccess;
7389                         }
7390                 } else {
7391                         if (-1U != idx) {
7392                                 result = kIOReturnExclusiveAccess;
7393                                 break;
7394                         }
7395                         array->setObject(target);
7396
7397                         UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
7398                         CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
7399                         UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction
7400                         IOService * holder = NULL;
7401
7402                         for (UInt i = 0; i < count; i++) {
7403                                 if (entries[i].fService
7404                                     && (delayType == entries[i].fDelayType)
7405                                     && (entries[i].fMaxDelay < ns)) {
7406                                         ns = entries[i].fMaxDelay;
7407                                         holder = entries[i].fService;
7408                                 }
7409                         }
7410                         target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
7411                             (void *) (uintptr_t) ns, holder,
7412                             NULL, NULL);
7413                         result = kIOReturnSuccess;
7414                 }
7415         }while (false);
7416
7417         IORecursiveLockUnlock(sCpuDelayLock);
7418
7419         return result;
7420 }
7421
7422 #endif /* defined(__i386__) || defined(__x86_64__) */
7423
7424 void
7425 IOService::
7426 requireMaxBusStall(UInt32 __unused ns)
7427 {
7428 #if defined(__i386__) || defined(__x86_64__)
7429         requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
7430 #endif
7431 }
7432
7433 void
7434 IOService::
7435 requireMaxInterruptDelay(uint32_t __unused ns)
7436 {
7437 #if defined(__i386__) || defined(__x86_64__)
7438         requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
7439 #endif
7440 }
7441
7442 /*
7443  * Device interrupts
7444  */
7445
7446 IOReturn
7447 IOService::resolveInterrupt(IOService *nub, int source)
7448 {
7449         IOInterruptController *interruptController;
7450         OSArray               *array;
7451         OSData                *data;
7452         OSSymbol              *interruptControllerName;
7453         long                  numSources;
7454         IOInterruptSource     *interruptSources;
7455
7456         // Get the parents list from the nub.
7457         array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
7458         if (array == NULL) {
7459                 return kIOReturnNoResources;
7460         }
7461
7462         // Allocate space for the IOInterruptSources if needed... then return early.
7463         if (nub->_interruptSources == NULL) {
7464                 numSources = array->getCount();
7465                 interruptSources = (IOInterruptSource *)IOMalloc(
7466                         numSources * sizeofAllIOInterruptSource);
7467                 if (interruptSources == NULL) {
7468                         return kIOReturnNoMemory;
7469                 }
7470
7471                 bzero(interruptSources, numSources * sizeofAllIOInterruptSource);
7472
7473                 nub->_numInterruptSources = numSources;
7474                 nub->_interruptSources = interruptSources;
7475                 return kIOReturnSuccess;
7476         }
7477
7478         interruptControllerName = OSDynamicCast(OSSymbol, array->getObject(source));
7479         if (interruptControllerName == NULL) {
7480                 return kIOReturnNoResources;
7481         }
7482
7483         interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
7484         if (interruptController == NULL) {
7485                 return kIOReturnNoResources;
7486         }
7487
7488         // Get the interrupt numbers from the nub.
7489         array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
7490         if (array == NULL) {
7491                 return kIOReturnNoResources;
7492         }
7493         data = OSDynamicCast(OSData, array->getObject(source));
7494         if (data == NULL) {
7495                 return kIOReturnNoResources;
7496         }
7497
7498         // Set the interruptController and interruptSource in the nub's table.
7499         interruptSources = nub->_interruptSources;
7500         interruptSources[source].interruptController = interruptController;
7501         interruptSources[source].vectorData = data;
7502
7503         return kIOReturnSuccess;
7504 }
7505
7506 IOReturn
7507 IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
7508 {
7509         IOReturn ret;
7510
7511         /* Make sure the _interruptSources are set */
7512         if (_interruptSources == NULL) {
7513                 ret = resolveInterrupt(this, source);
7514                 if (ret != kIOReturnSuccess) {
7515                         return ret;
7516                 }
7517         }
7518
7519         /* Make sure the local source number is valid */
7520         if ((source < 0) || (source >= _numInterruptSources)) {
7521                 return kIOReturnNoInterrupt;
7522         }
7523
7524         /* Look up the contoller for the local source */
7525         *interruptController = _interruptSources[source].interruptController;
7526
7527         if (*interruptController == NULL) {
7528                 if (!resolve) {
7529                         return kIOReturnNoInterrupt;
7530                 }
7531
7532                 /* Try to resolve the interrupt */
7533                 ret = resolveInterrupt(this, source);
7534                 if (ret != kIOReturnSuccess) {
7535                         return ret;
7536                 }
7537
7538                 *interruptController = _interruptSources[source].interruptController;
7539         }
7540
7541         return kIOReturnSuccess;
7542 }
7543
7544 IOReturn
7545 IOService::registerInterrupt(int source, OSObject *target,
7546     IOInterruptAction handler,
7547     void *refCon)
7548 {
7549         IOInterruptController *interruptController;
7550         IOReturn              ret;
7551
7552         ret = lookupInterrupt(source, true, &interruptController);
7553         if (ret != kIOReturnSuccess) {
7554                 return ret;
7555         }
7556
7557         /* Register the source */
7558         return interruptController->registerInterrupt(this, source, target,
7559                    (IOInterruptHandler)handler,
7560                    refCon);
7561 }
7562
7563 static void
7564 IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
7565     IOService * nub, int source )
7566 {
7567         ((IOInterruptActionBlock)(refCon))(nub, source);
7568 }
7569
7570 IOReturn
7571 IOService::registerInterruptBlock(int source, OSObject *target,
7572     IOInterruptActionBlock handler)
7573 {
7574         IOReturn ret;
7575         void   * block;
7576
7577         block = Block_copy(handler);
7578         if (!block) {
7579                 return kIOReturnNoMemory;
7580         }
7581
7582         ret = registerInterrupt(source, target, &IOServiceInterruptActionToBlock, block);
7583         if (kIOReturnSuccess != ret) {
7584                 Block_release(block);
7585                 return ret;
7586         }
7587         _interruptSourcesPrivate(this)[source].vectorBlock = block;
7588
7589         return ret;
7590 }
7591
7592 IOReturn
7593 IOService::unregisterInterrupt(int source)
7594 {
7595         IOReturn              ret;
7596         IOInterruptController *interruptController;
7597         void                  *block;
7598
7599         ret = lookupInterrupt(source, false, &interruptController);
7600         if (ret != kIOReturnSuccess) {
7601                 return ret;
7602         }
7603
7604         /* Unregister the source */
7605         block = _interruptSourcesPrivate(this)[source].vectorBlock;
7606         ret = interruptController->unregisterInterrupt(this, source);
7607         if ((kIOReturnSuccess == ret) && (block = _interruptSourcesPrivate(this)[source].vectorBlock)) {
7608                 _interruptSourcesPrivate(this)[source].vectorBlock = NULL;
7609                 Block_release(block);
7610         }
7611
7612         return ret;
7613 }
7614
7615 IOReturn
7616 IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
7617 {
7618         IOReportLegend * legend = NULL;
7619         IOInterruptAccountingData * oldValue = NULL;
7620         IOInterruptAccountingReporter * newArray = NULL;
7621         char subgroupName[64];
7622         int newArraySize = 0;
7623         int i = 0;
7624
7625         if (source < 0) {
7626                 return kIOReturnBadArgument;
7627         }
7628
7629         /*
7630          * We support statistics on a maximum of 256 interrupts per nub; if a nub
7631          * has more than 256 interrupt specifiers associated with it, and tries
7632          * to register a high interrupt index with interrupt accounting, panic.
7633          * Having more than 256 interrupts associated with a single nub is
7634          * probably a sign that something fishy is going on.
7635          */
7636         if (source > IA_INDEX_MAX) {
7637                 panic("addInterruptStatistics called for an excessively large index (%d)", source);
7638         }
7639
7640         /*
7641          * TODO: This is ugly (wrapping a lock around an allocation).  I'm only
7642          * leaving it as is because the likelihood of contention where we are
7643          * actually growing the array is minimal (we would realistically need
7644          * to be starting a driver for the first time, with an IOReporting
7645          * client already in place).  Nonetheless, cleanup that can be done
7646          * to adhere to best practices; it'll make the code more complicated,
7647          * unfortunately.
7648          */
7649         IOLockLock(reserved->interruptStatisticsLock);
7650
7651         /*
7652          * Lazily allocate the statistics array.
7653          */
7654         if (!reserved->interruptStatisticsArray) {
7655                 reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
7656                 assert(reserved->interruptStatisticsArray);
7657                 reserved->interruptStatisticsArrayCount = 1;
7658                 bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
7659         }
7660
7661         if (source >= reserved->interruptStatisticsArrayCount) {
7662                 /*
7663                  * We're still within the range of supported indices, but we are out
7664                  * of space in the current array.  Do a nasty realloc (because
7665                  * IORealloc isn't a thing) here.  We'll double the size with each
7666                  * reallocation.
7667                  *
7668                  * Yes, the "next power of 2" could be more efficient; but this will
7669                  * be invoked incredibly rarely.  Who cares.
7670                  */
7671                 newArraySize = (reserved->interruptStatisticsArrayCount << 1);
7672
7673                 while (newArraySize <= source) {
7674                         newArraySize = (newArraySize << 1);
7675                 }
7676                 newArray = IONew(IOInterruptAccountingReporter, newArraySize);
7677
7678                 assert(newArray);
7679
7680                 /*
7681                  * TODO: This even zeroes the memory it is about to overwrite.
7682                  * Shameful; fix it.  Not particularly high impact, however.
7683                  */
7684                 bzero(newArray, newArraySize * sizeof(*newArray));
7685                 memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
7686                 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
7687                 reserved->interruptStatisticsArray = newArray;
7688                 reserved->interruptStatisticsArrayCount = newArraySize;
7689         }
7690
7691         if (!reserved->interruptStatisticsArray[source].reporter) {
7692                 /*
7693                  * We don't have a reporter associated with this index yet, so we
7694                  * need to create one.
7695                  */
7696                 /*
7697                  * TODO: Some statistics do in fact have common units (time); should this be
7698                  * split into separate reporters to communicate this?
7699                  */
7700                 reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
7701
7702                 /*
7703                  * Each statistic is given an identifier based on the interrupt index (which
7704                  * should be unique relative to any single nub) and the statistic involved.
7705                  * We should now have a sane (small and positive) index, so start
7706                  * constructing the channels for statistics.
7707                  */
7708                 for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
7709                         /*
7710                          * TODO: Currently, this does not add channels for disabled statistics.
7711                          * Will this be confusing for clients?  If so, we should just add the
7712                          * channels; we can avoid updating the channels even if they exist.
7713                          */
7714                         if (IA_GET_STATISTIC_ENABLED(i)) {
7715                                 reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
7716                         }
7717                 }
7718
7719                 /*
7720                  * We now need to add the legend for this reporter to the registry.
7721                  */
7722                 OSObject * prop = copyProperty(kIOReportLegendKey);
7723                 legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
7724                 OSSafeReleaseNULL(prop);
7725
7726                 /*
7727                  * Note that while we compose the subgroup name, we do not need to
7728                  * manage its lifecycle (the reporter will handle this).
7729                  */
7730                 snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
7731                 subgroupName[sizeof(subgroupName) - 1] = 0;
7732                 legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
7733                 setProperty(kIOReportLegendKey, legend->getLegend());
7734                 legend->release();
7735
7736                 /*
7737                  * TODO: Is this a good idea?  Probably not; my assumption is it opts
7738                  * all entities who register interrupts into public disclosure of all
7739                  * IOReporting channels.  Unfortunately, this appears to be as fine
7740                  * grain as it gets.
7741                  */
7742                 setProperty(kIOReportLegendPublicKey, true);
7743         }
7744
7745         /*
7746          * Don't stomp existing entries.  If we are about to, panic; this
7747          * probably means we failed to tear down our old interrupt source
7748          * correctly.
7749          */
7750         oldValue = reserved->interruptStatisticsArray[source].statistics;
7751
7752         if (oldValue) {
7753                 panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
7754         }
7755
7756         reserved->interruptStatisticsArray[source].statistics = statistics;
7757
7758         /*
7759          * Inherit the reporter values for each statistic.  The target may
7760          * be torn down as part of the runtime of the service (especially
7761          * for sleep/wake), so we inherit in order to avoid having values
7762          * reset for no apparent reason.  Our statistics are ultimately
7763          * tied to the index and the sevice, not to an individual target,
7764          * so we should maintain them accordingly.
7765          */
7766         interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
7767
7768         IOLockUnlock(reserved->interruptStatisticsLock);
7769
7770         return kIOReturnSuccess;
7771 }
7772
7773 IOReturn
7774 IOService::removeInterruptStatistics(int source)
7775 {
7776         IOInterruptAccountingData * value = NULL;
7777
7778         if (source < 0) {
7779                 return kIOReturnBadArgument;
7780         }
7781
7782         IOLockLock(reserved->interruptStatisticsLock);
7783
7784         /*
7785          * We dynamically grow the statistics array, so an excessively
7786          * large index value has NEVER been registered.  This either
7787          * means our cap on the array size is too small (unlikely), or
7788          * that we have been passed a corrupt index (this must be passed
7789          * the plain index into the interrupt specifier list).
7790          */
7791         if (source >= reserved->interruptStatisticsArrayCount) {
7792                 panic("removeInterruptStatistics called for index %d, which was never registered", source);
7793         }
7794
7795         assert(reserved->interruptStatisticsArray);
7796
7797         /*
7798          * If there is no existing entry, we are most likely trying to
7799          * free an interrupt owner twice, or we have corrupted the
7800          * index value.
7801          */
7802         value = reserved->interruptStatisticsArray[source].statistics;
7803
7804         if (!value) {
7805                 panic("removeInterruptStatistics called for empty index %d", source);
7806         }
7807
7808         /*
7809          * We update the statistics, so that any delta with the reporter
7810          * state is not lost.
7811          */
7812         interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
7813         reserved->interruptStatisticsArray[source].statistics = NULL;
7814         IOLockUnlock(reserved->interruptStatisticsLock);
7815
7816         return kIOReturnSuccess;
7817 }
7818
7819 IOReturn
7820 IOService::getInterruptType(int source, int *interruptType)
7821 {
7822         IOInterruptController *interruptController;
7823         IOReturn              ret;
7824
7825         ret = lookupInterrupt(source, true, &interruptController);
7826         if (ret != kIOReturnSuccess) {
7827                 return ret;
7828         }
7829
7830         /* Return the type */
7831         return interruptController->getInterruptType(this, source, interruptType);
7832 }
7833
7834 IOReturn
7835 IOService::enableInterrupt(int source)
7836 {
7837         IOInterruptController *interruptController;
7838         IOReturn              ret;
7839
7840         ret = lookupInterrupt(source, false, &interruptController);
7841         if (ret != kIOReturnSuccess) {
7842                 return ret;
7843         }
7844
7845         /* Enable the source */
7846         return interruptController->enableInterrupt(this, source);
7847 }
7848
7849 IOReturn
7850 IOService::disableInterrupt(int source)
7851 {
7852         IOInterruptController *interruptController;
7853         IOReturn              ret;
7854
7855         ret = lookupInterrupt(source, false, &interruptController);
7856         if (ret != kIOReturnSuccess) {
7857                 return ret;
7858         }
7859
7860         /* Disable the source */
7861         return interruptController->disableInterrupt(this, source);
7862 }
7863
7864 IOReturn
7865 IOService::causeInterrupt(int source)
7866 {
7867         IOInterruptController *interruptController;
7868         IOReturn              ret;
7869
7870         ret = lookupInterrupt(source, false, &interruptController);
7871         if (ret != kIOReturnSuccess) {
7872                 return ret;
7873         }
7874
7875         /* Cause an interrupt for the source */
7876         return interruptController->causeInterrupt(this, source);
7877 }
7878
7879 IOReturn
7880 IOService::configureReport(IOReportChannelList    *channelList,
7881     IOReportConfigureAction action,
7882     void                   *result,
7883     void                   *destination)
7884 {
7885         unsigned cnt;
7886
7887         for (cnt = 0; cnt < channelList->nchannels; cnt++) {
7888                 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
7889                         if (pwrMgt) {
7890                                 configurePowerStatesReport(action, result);
7891                         } else {
7892                                 return kIOReturnUnsupported;
7893                         }
7894                 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
7895                         if (pwrMgt) {
7896                                 configureSimplePowerReport(action, result);
7897                         } else {
7898                                 return kIOReturnUnsupported;
7899                         }
7900                 }
7901         }
7902
7903         IOLockLock(reserved->interruptStatisticsLock);
7904
7905         /* The array count is signed (because the interrupt indices are signed), hence the cast */
7906         for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
7907                 if (reserved->interruptStatisticsArray[cnt].reporter) {
7908                         /*
7909                          * If the reporter is currently associated with the statistics
7910                          * for an event source, we may need to update the reporter.
7911                          */
7912                         if (reserved->interruptStatisticsArray[cnt].statistics) {
7913                                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
7914                         }
7915
7916                         reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
7917                 }
7918         }
7919
7920         IOLockUnlock(reserved->interruptStatisticsLock);
7921
7922         return kIOReturnSuccess;
7923 }
7924
7925 IOReturn
7926 IOService::updateReport(IOReportChannelList      *channelList,
7927     IOReportUpdateAction      action,
7928     void                     *result,
7929     void                     *destination)
7930 {
7931         unsigned cnt;
7932
7933         for (cnt = 0; cnt < channelList->nchannels; cnt++) {
7934                 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
7935                         if (pwrMgt) {
7936                                 updatePowerStatesReport(action, result, destination);
7937                         } else {
7938                                 return kIOReturnUnsupported;
7939                         }
7940                 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
7941                         if (pwrMgt) {
7942                                 updateSimplePowerReport(action, result, destination);
7943                         } else {
7944                                 return kIOReturnUnsupported;
7945                         }
7946                 }
7947         }
7948
7949         IOLockLock(reserved->interruptStatisticsLock);
7950
7951         /* The array count is signed (because the interrupt indices are signed), hence the cast */
7952         for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
7953                 if (reserved->interruptStatisticsArray[cnt].reporter) {
7954                         /*
7955                          * If the reporter is currently associated with the statistics
7956                          * for an event source, we need to update the reporter.
7957                          */
7958                         if (reserved->interruptStatisticsArray[cnt].statistics) {
7959                                 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
7960                         }
7961
7962                         reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
7963                 }
7964         }
7965
7966         IOLockUnlock(reserved->interruptStatisticsLock);
7967
7968         return kIOReturnSuccess;
7969 }
7970
7971 uint64_t
7972 IOService::getAuthorizationID( void )
7973 {
7974         return reserved->authorizationID;
7975 }
7976
7977 IOReturn
7978 IOService::setAuthorizationID( uint64_t authorizationID )
7979 {
7980         OSObject * entitlement;
7981         IOReturn status;
7982
7983         entitlement = IOUserClient::copyClientEntitlement( current_task(), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
7984
7985         if (entitlement) {
7986                 if (entitlement == kOSBooleanTrue) {
7987                         reserved->authorizationID = authorizationID;
7988
7989                         status = kIOReturnSuccess;
7990                 } else {
7991                         status = kIOReturnNotPrivileged;
7992                 }
7993
7994                 entitlement->release();
7995         } else {
7996                 status = kIOReturnNotPrivileged;
7997         }
7998
7999         return status;
8000 }
8001
8002 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
8003
8004
8005 #if __LP64__
8006 OSMetaClassDefineReservedUsed(IOService, 0);
8007 OSMetaClassDefineReservedUsed(IOService, 1);
8008 OSMetaClassDefineReservedUnused(IOService, 2);
8009 OSMetaClassDefineReservedUnused(IOService, 3);
8010 OSMetaClassDefineReservedUnused(IOService, 4);
8011 OSMetaClassDefineReservedUnused(IOService, 5);
8012 OSMetaClassDefineReservedUnused(IOService, 6);
8013 OSMetaClassDefineReservedUnused(IOService, 7);
8014 #else
8015 OSMetaClassDefineReservedUsed(IOService, 0);
8016 OSMetaClassDefineReservedUsed(IOService, 1);
8017 OSMetaClassDefineReservedUsed(IOService, 2);
8018 OSMetaClassDefineReservedUsed(IOService, 3);
8019 OSMetaClassDefineReservedUsed(IOService, 4);
8020 OSMetaClassDefineReservedUsed(IOService, 5);
8021 OSMetaClassDefineReservedUsed(IOService, 6);
8022 OSMetaClassDefineReservedUsed(IOService, 7);
8023 #endif
8024 OSMetaClassDefineReservedUnused(IOService, 8);
8025 OSMetaClassDefineReservedUnused(IOService, 9);
8026 OSMetaClassDefineReservedUnused(IOService, 10);
8027 OSMetaClassDefineReservedUnused(IOService, 11);
8028 OSMetaClassDefineReservedUnused(IOService, 12);
8029 OSMetaClassDefineReservedUnused(IOService, 13);
8030 OSMetaClassDefineReservedUnused(IOService, 14);
8031 OSMetaClassDefineReservedUnused(IOService, 15);
8032 OSMetaClassDefineReservedUnused(IOService, 16);
8033 OSMetaClassDefineReservedUnused(IOService, 17);
8034 OSMetaClassDefineReservedUnused(IOService, 18);
8035 OSMetaClassDefineReservedUnused(IOService, 19);
8036 OSMetaClassDefineReservedUnused(IOService, 20);
8037 OSMetaClassDefineReservedUnused(IOService, 21);
8038 OSMetaClassDefineReservedUnused(IOService, 22);
8039 OSMetaClassDefineReservedUnused(IOService, 23);
8040 OSMetaClassDefineReservedUnused(IOService, 24);
8041 OSMetaClassDefineReservedUnused(IOService, 25);
8042 OSMetaClassDefineReservedUnused(IOService, 26);
8043 OSMetaClassDefineReservedUnused(IOService, 27);
8044 OSMetaClassDefineReservedUnused(IOService, 28);
8045 OSMetaClassDefineReservedUnused(IOService, 29);
8046 OSMetaClassDefineReservedUnused(IOService, 30);
8047 OSMetaClassDefineReservedUnused(IOService, 31);
8048 OSMetaClassDefineReservedUnused(IOService, 32);
8049 OSMetaClassDefineReservedUnused(IOService, 33);
8050 OSMetaClassDefineReservedUnused(IOService, 34);
8051 OSMetaClassDefineReservedUnused(IOService, 35);
8052 OSMetaClassDefineReservedUnused(IOService, 36);
8053 OSMetaClassDefineReservedUnused(IOService, 37);
8054 OSMetaClassDefineReservedUnused(IOService, 38);
8055 OSMetaClassDefineReservedUnused(IOService, 39);
8056 OSMetaClassDefineReservedUnused(IOService, 40);
8057 OSMetaClassDefineReservedUnused(IOService, 41);
8058 OSMetaClassDefineReservedUnused(IOService, 42);
8059 OSMetaClassDefineReservedUnused(IOService, 43);
8060 OSMetaClassDefineReservedUnused(IOService, 44);
8061 OSMetaClassDefineReservedUnused(IOService, 45);
8062 OSMetaClassDefineReservedUnused(IOService, 46);
8063 OSMetaClassDefineReservedUnused(IOService, 47);