X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/55e303ae13a4cf49d70f2294092726f2fffb9ef2..d26ffc64f583ab2d29df48f13518685602bc8832:/iokit/Kernel/IOService.cpp diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp index 5dafe2216..5d4be71a6 100644 --- a/iokit/Kernel/IOService.cpp +++ b/iokit/Kernel/IOService.cpp @@ -1,16 +1,19 @@ /* - * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1998-2016 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ - * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -20,15 +23,19 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include +#include #include +#include #include #include +#include +#include #include #include #include @@ -39,14 +46,31 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include + +#include + +#define LOG kprintf +//#define LOG IOLog +#define MATCH_DEBUG 0 +#define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x))) -//#define LOG kprintf -#define LOG IOLog +// disabled since lockForArbitration() can be held externally +#define DEBUG_NOTIFIER_LOCKED 0 #include "IOServicePrivate.h" +#include "IOKitKernelInternal.h" // take lockForArbitration before LOCKNOTIFY @@ -57,6 +81,7 @@ OSDefineMetaClassAndStructors(IOService, IORegistryEntry) OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier) +OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier) OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier) @@ -82,25 +107,45 @@ const OSSymbol * gIOInterruptSpecifiersKey; const OSSymbol * gIOResourcesKey; const OSSymbol * gIOResourceMatchKey; +const OSSymbol * gIOResourceMatchedKey; const OSSymbol * gIOProviderClassKey; const OSSymbol * gIONameMatchKey; const OSSymbol * gIONameMatchedKey; const OSSymbol * gIOPropertyMatchKey; +const OSSymbol * gIOPropertyExistsMatchKey; const OSSymbol * gIOLocationMatchKey; const OSSymbol * gIOParentMatchKey; const OSSymbol * gIOPathMatchKey; const OSSymbol * gIOMatchCategoryKey; const OSSymbol * gIODefaultMatchCategoryKey; const OSSymbol * gIOMatchedServiceCountKey; +#if !CONFIG_EMBEDDED +const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey; +#endif +const OSSymbol * gIOMapperIDKey; const OSSymbol * gIOUserClientClassKey; const OSSymbol * gIOKitDebugKey; const OSSymbol * gIOCommandPoolSizeKey; +const OSSymbol * gIOConsoleLockedKey; const OSSymbol * gIOConsoleUsersKey; const OSSymbol * gIOConsoleSessionUIDKey; +const OSSymbol * gIOConsoleSessionAuditIDKey; const OSSymbol * gIOConsoleUsersSeedKey; +const OSSymbol * gIOConsoleSessionOnConsoleKey; +const OSSymbol * gIOConsoleSessionLoginDoneKey; +const OSSymbol * gIOConsoleSessionSecureInputPIDKey; +const OSSymbol * gIOConsoleSessionScreenLockedTimeKey; +const OSSymbol * gIOConsoleSessionScreenIsLockedKey; +clock_sec_t gIOConsoleLockTime; +static bool gIOConsoleLoggedIn; +#if HIBERNATION +static OSBoolean * gIOConsoleBooterLockState; +static uint32_t gIOScreenLockState; +#endif +static IORegistryEntry * gIOChosenEntry; static int gIOResourceGenerationCount; @@ -110,11 +155,24 @@ const OSSymbol * gIOFirstPublishNotification; const OSSymbol * gIOMatchedNotification; const OSSymbol * gIOFirstMatchNotification; const OSSymbol * gIOTerminatedNotification; +const OSSymbol * gIOWillTerminateNotification; const OSSymbol * gIOGeneralInterest; const OSSymbol * gIOBusyInterest; const OSSymbol * gIOAppPowerStateInterest; const OSSymbol * gIOPriorityPowerStateInterest; +const OSSymbol * gIOConsoleSecurityInterest; + +const OSSymbol * gIOBSDKey; +const OSSymbol * gIOBSDNameKey; +const OSSymbol * gIOBSDMajorKey; +const OSSymbol * gIOBSDMinorKey; +const OSSymbol * gIOBSDUnitKey; + +const OSSymbol * gAKSGetKey; +#if defined(__i386__) || defined(__x86_64__) +const OSSymbol * gIOCreateEFIDevicePathSymbol; +#endif static OSDictionary * gNotifications; static IORecursiveLock * gNotificationLock; @@ -129,8 +187,10 @@ static int gOutstandingJobs; static int gNumConfigThreads; static int gNumWaitingThreads; static IOLock * gIOServiceBusyLock; +bool gCPUsRunning; static thread_t gIOTerminateThread; +static thread_t gIOTerminateWorkerThread; static UInt32 gIOTerminateWork; static OSArray * gIOTerminatePhase2List; static OSArray * gIOStopList; @@ -140,6 +200,14 @@ static OSArray * gIOFinalizeList; static SInt32 gIOConsoleUsersSeed; static OSData * gIOConsoleUsersSeedValue; +extern const OSSymbol * gIODTPHandleKey; + +const OSSymbol * gIOPlatformFunctionHandlerSet; + +static IOLock * gIOConsoleUsersLock; +static thread_call_t gIOConsoleLockCallout; +static IONotifier * gIOServiceNullNotifier; + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define LOCKREADNOTIFY() \ @@ -151,6 +219,8 @@ static OSData * gIOConsoleUsersSeedValue; IORecursiveLockUnlock( gNotificationLock ) #define SLEEPNOTIFY(event) \ IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT ) +#define SLEEPNOTIFYTO(event, deadline) \ + IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT ) #define WAKEUPNOTIFY(event) \ IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false ) @@ -161,6 +231,24 @@ static OSData * gIOConsoleUsersSeedValue; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#define queue_element(entry, element, type, field) do { \ + vm_address_t __ele = (vm_address_t) (entry); \ + __ele -= -4 + ((size_t)(&((type) 4)->field)); \ + (element) = (type) __ele; \ + } while(0) + +#define iterqueue(que, elt) \ + for (queue_entry_t elt = queue_first(que); \ + !queue_end(que, elt); \ + elt = queue_next(elt)) + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +struct IOInterruptAccountingReporter { + IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */ + IOInterruptAccountingData * statistics; /* The live statistics values, if any */ +}; + struct ArbitrationLockQueueElement { queue_chain_t link; IOThread thread; @@ -180,6 +268,40 @@ bool IOService::isInactive( void ) const /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#if defined(__i386__) || defined(__x86_64__) + +// Only used by the intel implementation of +// IOService::requireMaxBusStall(UInt32 ns) +// IOService::requireMaxInterruptDelay(uint32_t ns) +struct CpuDelayEntry +{ + IOService * fService; + UInt32 fMaxDelay; + UInt32 fDelayType; +}; + +enum { + kCpuDelayBusStall, kCpuDelayInterrupt, + kCpuNumDelayTypes +}; + +static OSData *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry)); +static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc(); +static OSArray *sCpuLatencyHandlers[kCpuNumDelayTypes]; +const OSSymbol *sCPULatencyFunctionName[kCpuNumDelayTypes]; +static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes]; +static char sCPULatencyHolderName[kCpuNumDelayTypes][128]; +static OSNumber * sCPULatencySet[kCpuNumDelayTypes]; + +static void +requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType); +static IOReturn +setLatencyHandler(UInt32 delayType, IOService * target, bool enable); + +#endif /* defined(__i386__) || defined(__x86_64__) */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + void IOService::initialize( void ) { kern_return_t err; @@ -191,6 +313,7 @@ void IOService::initialize( void ) gIONameMatchKey = OSSymbol::withCStringNoCopy( kIONameMatchKey ); gIONameMatchedKey = OSSymbol::withCStringNoCopy( kIONameMatchedKey ); gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey ); + gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey ); gIOPathMatchKey = OSSymbol::withCStringNoCopy( kIOPathMatchKey ); gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey ); gIOParentMatchKey = OSSymbol::withCStringNoCopy( kIOParentMatchKey ); @@ -200,11 +323,16 @@ void IOService::initialize( void ) kIODefaultMatchCategoryKey ); gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy( kIOMatchedServiceCountKey ); +#if !CONFIG_EMBEDDED + gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy( + kIOServiceLegacyMatchingRegistryIDKey ); +#endif gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey ); - gIOResourcesKey = OSSymbol::withCStringNoCopy( kIOResourcesClass ); - gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey ); + gIOResourcesKey = OSSymbol::withCStringNoCopy( kIOResourcesClass ); + gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey ); + gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey ); gIODeviceMemoryKey = OSSymbol::withCStringNoCopy( "IODeviceMemory" ); gIOInterruptControllersKey @@ -212,6 +340,8 @@ void IOService::initialize( void ) gIOInterruptSpecifiersKey = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers"); + gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey); + gIOKitDebugKey = OSSymbol::withCStringNoCopy( kIOKitDebugKey ); gIOCommandPoolSizeKey = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey ); @@ -220,6 +350,13 @@ void IOService::initialize( void ) gIOBusyInterest = OSSymbol::withCStringNoCopy( kIOBusyInterest ); gIOAppPowerStateInterest = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest ); gIOPriorityPowerStateInterest = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest ); + gIOConsoleSecurityInterest = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest ); + + gIOBSDKey = OSSymbol::withCStringNoCopy(kIOBSDKey); + gIOBSDNameKey = OSSymbol::withCStringNoCopy(kIOBSDNameKey); + gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey); + gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey); + gIOBSDUnitKey = OSSymbol::withCStringNoCopy(kIOBSDUnitKey); gNotifications = OSDictionary::withCapacity( 1 ); gIOPublishNotification = OSSymbol::withCStringNoCopy( @@ -232,15 +369,41 @@ void IOService::initialize( void ) kIOFirstMatchNotification ); gIOTerminatedNotification = OSSymbol::withCStringNoCopy( kIOTerminatedNotification ); + gIOWillTerminateNotification = OSSymbol::withCStringNoCopy( + kIOWillTerminateNotification ); gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass); + gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey); gIOConsoleUsersKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey); gIOConsoleSessionUIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey); - gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersSeedKey); - gIOConsoleUsersSeedValue = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed)); - + gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey); + + gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey); + gIOConsoleSessionOnConsoleKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey); + gIOConsoleSessionLoginDoneKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey); + gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey); + gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey); + gIOConsoleSessionScreenIsLockedKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey); + + gIOConsoleUsersSeedValue = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed)); + + gIOPlatformFunctionHandlerSet = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet); +#if defined(__i386__) || defined(__x86_64__) + sCPULatencyFunctionName[kCpuDelayBusStall] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay); + sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay); + uint32_t idx; + for (idx = 0; idx < kCpuNumDelayTypes; idx++) + { + sCPULatencySet[idx] = OSNumber::withNumber(-1U, 32); + sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64); + assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]); + } + gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); +#endif gNotificationLock = IORecursiveLockAlloc(); + gAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY); + assert( gIOServicePlane && gIODeviceMemoryKey && gIOInterruptControllersKey && gIOInterruptSpecifiersKey && gIOResourcesKey && gNotifications && gNotificationLock @@ -249,6 +412,7 @@ void IOService::initialize( void ) && gIOPublishNotification && gIOMatchedNotification && gIOTerminatedNotification && gIOServiceKey && gIOConsoleUsersKey && gIOConsoleSessionUIDKey + && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue); gJobsLock = IOLockAlloc(); @@ -256,13 +420,23 @@ void IOService::initialize( void ) gIOServiceBusyLock = IOLockAlloc(); + gIOConsoleUsersLock = IOLockAlloc(); + err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0); - assert( gIOServiceBusyLock && gJobs && gJobsLock && (err == KERN_SUCCESS) ); + gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL); + + IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue); + + assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock + && gIOConsoleLockCallout && (err == KERN_SUCCESS) ); gIOResources = IOResources::resources(); assert( gIOResources ); + gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier); + assert(gIOServiceNullNotifier); + gArbitrationLockQueueLock = IOLockAlloc(); queue_init(&gArbitrationLockQueueActive); queue_init(&gArbitrationLockQueueWaiting); @@ -275,10 +449,31 @@ void IOService::initialize( void ) gIOStopProviderList = OSArray::withCapacity( 16 ); gIOFinalizeList = OSArray::withCapacity( 16 ); assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList ); + + // worker thread that is responsible for terminating / cleaning up threads + kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread); + assert(gIOTerminateWorkerThread); + thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread"); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#if defined(__i386__) || defined(__x86_64__) +extern "C" { + +const char *getCpuDelayBusStallHolderName(void); +const char *getCpuDelayBusStallHolderName(void) { + return sCPULatencyHolderName[kCpuDelayBusStall]; +} + +const char *getCpuInterruptDelayHolderName(void); +const char *getCpuInterruptDelayHolderName(void) { + return sCPULatencyHolderName[kCpuDelayInterrupt]; +} + +} +#endif + #if IOMATCHDEBUG static UInt64 getDebugFlags( OSDictionary * props ) { @@ -294,6 +489,24 @@ static UInt64 getDebugFlags( OSDictionary * props ) return( debugFlags ); } + +static UInt64 getDebugFlags( IOService * inst ) +{ + OSObject * prop; + OSNumber * debugProp; + UInt64 debugFlags; + + prop = inst->copyProperty(gIOKitDebugKey); + debugProp = OSDynamicCast(OSNumber, prop); + if( debugProp) + debugFlags = debugProp->unsigned64BitValue(); + else + debugFlags = gIOKitDebug; + + OSSafeReleaseNULL(prop); + + return( debugFlags ); +} #endif /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -317,11 +530,93 @@ void IOService::stop( IOService * provider ) { } +bool IOService::init( OSDictionary * dictionary ) +{ + bool ret; + + ret = super::init(dictionary); + if (!ret) return (false); + if (reserved) return (true); + + reserved = IONew(ExpansionData, 1); + if (!reserved) return (false); + bzero(reserved, sizeof(*reserved)); + + /* + * TODO: Improve on this. Previous efforts to more lazily allocate this + * lock based on the presence of specifiers ran into issues as some + * platforms set up the specifiers after IOService initialization. + * + * We may be able to get away with a global lock, as this should only be + * contended by IOReporting clients and driver start/stop (unless a + * driver wants to remove/add handlers in the course of normal operation, + * which should be unlikely). + */ + reserved->interruptStatisticsLock = IOLockAlloc(); + if (!reserved->interruptStatisticsLock) return (false); + + return (true); +} + +bool IOService::init( IORegistryEntry * from, + const IORegistryPlane * inPlane ) +{ + bool ret; + + ret = super::init(from, inPlane); + if (!ret) return (false); + if (reserved) return (true); + + reserved = IONew(ExpansionData, 1); + if (!reserved) return (false); + bzero(reserved, sizeof(*reserved)); + + /* + * TODO: Improve on this. Previous efforts to more lazily allocate this + * lock based on the presence of specifiers ran into issues as some + * platforms set up the specifiers after IOService initialization. + * + * We may be able to get away with a global lock, as this should only be + * contended by IOReporting clients and driver start/stop (unless a + * driver wants to remove/add handlers in the course of normal operation, + * which should be unlikely). + */ + reserved->interruptStatisticsLock = IOLockAlloc(); + if (!reserved->interruptStatisticsLock) return (false); + + return (true); +} + void IOService::free( void ) { + int i = 0; + requireMaxBusStall(0); + requireMaxInterruptDelay(0); if( getPropertyTable()) unregisterAllInterest(); PMfree(); + + if (reserved) { + if (reserved->interruptStatisticsArray) { + for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) { + if (reserved->interruptStatisticsArray[i].reporter) + reserved->interruptStatisticsArray[i].reporter->release(); + } + + IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount); + } + + if (reserved->interruptStatisticsLock) + IOLockFree(reserved->interruptStatisticsLock); + IODelete(reserved, ExpansionData, 1); + } + + if (_numInterruptSources && _interruptSources) + { + IOFree(_interruptSources, _numInterruptSources * sizeof(IOInterruptSource)); + _interruptSources = 0; + } + super::free(); } @@ -330,7 +625,11 @@ void IOService::free( void ) */ bool IOService::attach( IOService * provider ) { - bool ok; + bool ok; + uint32_t count; + AbsoluteTime deadline; + int waitResult = THREAD_AWAKENED; + bool wait, computeDeadline = true; if( provider) { @@ -338,18 +637,47 @@ bool IOService::attach( IOService * provider ) LOG( "%s::attach(%s)\n", getName(), provider->getName()); - provider->lockForArbitration(); - if( provider->__state[0] & kIOServiceInactiveState) - ok = false; - else - ok = attachToParent( provider, gIOServicePlane); - provider->unlockForArbitration(); + ok = false; + do + { + wait = false; + provider->lockForArbitration(); + if (provider->__state[0] & kIOServiceInactiveState) ok = false; + else + { + count = provider->getChildCount(gIOServicePlane); + wait = (count > (kIOServiceBusyMax - 4)); + if (!wait) ok = attachToParent(provider, gIOServicePlane); + else + { + IOLog("stalling for detach from %s\n", provider->getName()); + IOLockLock( gIOServiceBusyLock ); + provider->__state[1] |= kIOServiceWaitDetachState; + } + } + provider->unlockForArbitration(); + if (wait) + { + if (computeDeadline) + { + clock_interval_to_deadline(15, kSecondScale, &deadline); + computeDeadline = false; + } + assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline); + IOLockUnlock( gIOServiceBusyLock ); + waitResult = thread_block(THREAD_CONTINUE_NULL); + wait = (waitResult != THREAD_TIMED_OUT); + } + } + while (wait); } else { gIOServiceRoot = this; ok = attachToParent( getRegistryRoot(), gIOServicePlane); } + if (ok && !__provider) (void) getProvider(); + return( ok ); } @@ -369,6 +697,15 @@ void IOService::detach( IOService * provider ) lockForArbitration(); + uint64_t regID1 = provider->getRegistryEntryID(); + uint64_t regID2 = getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_DETACH, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask)) && (provider == getProvider())); @@ -380,6 +717,11 @@ void IOService::detach( IOService * provider ) _adjustBusy( -busy ); } + if (kIOServiceInactiveState & __state[0]) { + getMetaClass()->removeInstance(this); + IORemoveServicePlatformActions(this); + } + unlockForArbitration(); if( newProvider) { @@ -389,13 +731,26 @@ void IOService::detach( IOService * provider ) } // check for last client detach from a terminated service - if( provider->lockForArbitration( true )) { - if( adjParent) - provider->_adjustBusy( -1 ); + if( provider->lockForArbitration( true )) + { + if (kIOServiceStartState & __state[1]) + { + provider->scheduleTerminatePhase2(); + } + if( adjParent) provider->_adjustBusy( -1 ); if( (provider->__state[1] & kIOServiceTermPhase3State) && (0 == provider->getClient())) { - provider->scheduleFinalize(); + provider->scheduleFinalize(false); } + + IOLockLock( gIOServiceBusyLock ); + if (kIOServiceWaitDetachState & provider->__state[1]) + { + provider->__state[1] &= ~kIOServiceWaitDetachState; + thread_wakeup(&provider->__provider); + } + IOLockUnlock( gIOServiceBusyLock ); + provider->unlockForArbitration(); } } @@ -427,6 +782,8 @@ void IOService::registerService( IOOptionBits options ) if( gIOPlatform && (!gIOPlatform->platformAdjustService(this))) return; + IOInstallServicePlatformActions(this); + if( (this != gIOResources) && (kIOLogRegister & gIOKitDebug)) { @@ -473,8 +830,10 @@ void IOService::startMatching( IOOptionBits options ) || ((provider = getProvider()) && (provider->__state[1] & kIOServiceSynchronousState)); + if ( options & kIOServiceAsynchronous ) + sync = false; - needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState))) + needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning))) && (0 == (__state[0] & kIOServiceInactiveState)); __state[1] |= kIOServiceNeedConfigState; @@ -485,7 +844,6 @@ void IOService::startMatching( IOOptionBits options ) // OSKernelStackRemaining(), getName()); if( needConfig) { - prevBusy = _adjustBusy( 1 ); needWake = (0 != (kIOServiceSyncPubState & __state[1])); } @@ -494,6 +852,8 @@ void IOService::startMatching( IOOptionBits options ) else __state[1] &= ~kIOServiceSynchronousState; + if( needConfig) prevBusy = _adjustBusy( 1 ); + unlockForArbitration(); if( needConfig) { @@ -515,7 +875,9 @@ void IOService::startMatching( IOOptionBits options ) lockForArbitration(); IOLockLock( gIOServiceBusyLock ); - waitAgain = (prevBusy != (__state[1] & kIOServiceBusyStateMask)); + waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask)) + && (0 == (__state[0] & kIOServiceInactiveState))); + if( waitAgain) __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState; else @@ -537,7 +899,8 @@ void IOService::startMatching( IOOptionBits options ) IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables ) { OSDictionary * table; - OSIterator * iter; + OSSet * set; + OSSet * allSet = 0; IOService * service; #if IOMATCHDEBUG SInt32 count = 0; @@ -547,24 +910,37 @@ IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables ) while( (table = (OSDictionary *) newTables->getFirstObject())) { - LOCKWRITENOTIFY(); - iter = (OSIterator *) getExistingServices( table, - kIOServiceRegisteredState ); - UNLOCKNOTIFY(); - if( iter) { - while( (service = (IOService *) iter->getNextObject())) { - service->startMatching(kIOServiceAsynchronous); + LOCKWRITENOTIFY(); + set = (OSSet *) copyExistingServices( table, + kIOServiceRegisteredState, + kIOServiceExistingSet); + UNLOCKNOTIFY(); + if( set) { + #if IOMATCHDEBUG - count++; + count += set->getCount(); #endif - } - iter->release(); - } + if (allSet) { + allSet->merge((const OSSet *) set); + set->release(); + } + else + allSet = set; + } + #if IOMATCHDEBUG - if( getDebugFlags( table ) & kIOLogMatch) - LOG("Matching service count = %ld\n", count); + if( getDebugFlags( table ) & kIOLogMatch) + LOG("Matching service count = %ld\n", (long)count); #endif - newTables->removeObject(table); + newTables->removeObject(table); + } + + if (allSet) { + while( (service = (IOService *) allSet->getAnyObject())) { + service->startMatching(kIOServiceAsynchronous); + allSet->removeObject(service); + } + allSet->release(); } newTables->release(); @@ -629,10 +1005,9 @@ IOService * IOService::getProvider( void ) const IOService * parent; SInt32 generation; - parent = __provider; - generation = getGenerationCount(); + generation = getRegistryEntryGenerationCount(); if( __providerGeneration == generation) - return( parent ); + return( __provider ); parent = (IOService *) getParentEntry( gIOServicePlane); if( parent == IORegistryEntry::getRegistryRoot()) @@ -640,7 +1015,8 @@ IOService * IOService::getProvider( void ) const parent = 0; self->__provider = parent; - // save the count before getParentEntry() + OSMemoryBarrier(); + // save the count from before call to getParentEntry() self->__providerGeneration = generation; return( parent ); @@ -755,9 +1131,23 @@ IOReturn IOService::callPlatformFunction( const OSSymbol * functionName, void *param3, void *param4 ) { IOReturn result = kIOReturnUnsupported; - IOService *provider = getProvider(); - - if (provider != 0) { + IOService *provider; + + if (gIOPlatformFunctionHandlerSet == functionName) + { +#if defined(__i386__) || defined(__x86_64__) + const OSSymbol * functionHandlerName = (const OSSymbol *) param1; + IOService * target = (IOService *) param2; + bool enable = (param3 != 0); + + if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) + result = setLatencyHandler(kCpuDelayBusStall, target, enable); + else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) + result = setLatencyHandler(kCpuDelayInterrupt, target, enable); +#endif /* defined(__i386__) || defined(__x86_64__) */ + } + + if ((kIOReturnUnsupported == result) && (provider = getProvider())) { result = provider->callPlatformFunction(functionName, waitForFunction, param1, param2, param3, param4); } @@ -806,6 +1196,25 @@ void IOService::setPlatform( IOPlatformExpert * platform) { gIOPlatform = platform; gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane ); + +#if defined(__i386__) || defined(__x86_64__) + + static const char * keys[kCpuNumDelayTypes] = { + kIOPlatformMaxBusDelay, kIOPlatformMaxInterruptDelay }; + const OSObject * objs[2]; + OSArray * array; + uint32_t idx; + + for (idx = 0; idx < kCpuNumDelayTypes; idx++) + { + objs[0] = sCPULatencySet[idx]; + objs[1] = sCPULatencyHolder[idx]; + array = OSArray::withObjects(objs, 2); + if (!array) break; + platform->setProperty(keys[idx], array); + array->release(); + } +#endif /* defined(__i386__) || defined(__x86_64__) */ } void IOService::setPMRootDomain( class IOPMrootDomain * rootDomain) @@ -1183,6 +1592,41 @@ void IOService::unlockForArbitration( void ) IOUnlock( gArbitrationLockQueueLock ); } +uint32_t IOService::isLockedForArbitration(IOService * service) +{ +#if DEBUG_NOTIFIER_LOCKED + uint32_t count; + ArbitrationLockQueueElement * active; + + // lock global access + IOLockLock(gArbitrationLockQueueLock); + + // determine whether this object is already locked (ie. on active queue) + count = 0; + queue_iterate(&gArbitrationLockQueueActive, + active, + ArbitrationLockQueueElement *, + link) + { + if ((active->thread == IOThreadSelf()) + && (!service || (active->service == service))) + { + count += 0x10000; + count += active->count; + } + } + + IOLockUnlock(gArbitrationLockQueueLock); + + return (count); + +#else /* DEBUG_NOTIFIER_LOCKED */ + + return (0); + +#endif /* DEBUG_NOTIFIER_LOCKED */ +} + void IOService::applyToProviders( IOServiceApplierFunction applier, void * context ) { @@ -1253,32 +1697,52 @@ IOReturn IOService::messageClient( UInt32 type, OSObject * client, return( ret ); } +static void +applyToInterestNotifiers(const IORegistryEntry *target, + const OSSymbol * typeOfInterest, + OSObjectApplierFunction applier, + void * context ) +{ + OSArray * copyArray = 0; + OSObject * prop; + + LOCKREADNOTIFY(); + + prop = target->copyProperty(typeOfInterest); + IOCommand *notifyList = OSDynamicCast(IOCommand, prop); + + if( notifyList) { + copyArray = OSArray::withCapacity(1); + + // iterate over queue, entry is set to each element in the list + iterqueue(¬ifyList->fCommandChain, entry) { + _IOServiceInterestNotifier * notify; + + queue_element(entry, notify, _IOServiceInterestNotifier *, chain); + copyArray->setObject(notify); + } + } + UNLOCKNOTIFY(); + + if( copyArray) { + unsigned int index; + OSObject * next; + + for( index = 0; (next = copyArray->getObject( index )); index++) + (*applier)(next, context); + copyArray->release(); + } + + OSSafeReleaseNULL(prop); +} + void IOService::applyToInterested( const OSSymbol * typeOfInterest, OSObjectApplierFunction applier, void * context ) { - OSArray * array; - unsigned int index; - OSObject * next; - OSArray * copyArray; - - applyToClients( (IOServiceApplierFunction) applier, context ); - - LOCKREADNOTIFY(); - array = OSDynamicCast( OSArray, getProperty( typeOfInterest )); - if( array) { - copyArray = OSArray::withArray( array ); - UNLOCKNOTIFY(); - if( copyArray) { - for( index = 0; - (next = copyArray->getObject( index )); - index++) { - (*applier)(next, context); - } - copyArray->release(); - } - } else - UNLOCKNOTIFY(); + if (gIOGeneralInterest == typeOfInterest) + applyToClients( (IOServiceApplierFunction) applier, context ); + applyToInterestNotifiers(this, typeOfInterest, applier, context); } struct MessageClientsContext { @@ -1328,75 +1792,122 @@ IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest, IOServiceInterestHandler handler, void * target, void * ref ) { _IOServiceInterestNotifier * notify = 0; - OSArray * set; + IOReturn rc = kIOReturnError; + + notify = new _IOServiceInterestNotifier; + if (!notify) return NULL; + + if(notify->init()) { + rc = registerInterestForNotifier(notify, typeOfInterest, + handler, target, ref); + } + + if (rc != kIOReturnSuccess) { + notify->release(); + notify = 0; + } + + return( notify ); +} + +IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest, + IOServiceInterestHandler handler, void * target, void * ref ) +{ + IOReturn rc = kIOReturnSuccess; + _IOServiceInterestNotifier *notify = 0; if( (typeOfInterest != gIOGeneralInterest) && (typeOfInterest != gIOBusyInterest) && (typeOfInterest != gIOAppPowerStateInterest) + && (typeOfInterest != gIOConsoleSecurityInterest) && (typeOfInterest != gIOPriorityPowerStateInterest)) - return( 0 ); + return( kIOReturnBadArgument ); + + if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify))) + return( kIOReturnBadArgument ); lockForArbitration(); if( 0 == (__state[0] & kIOServiceInactiveState)) { - notify = new _IOServiceInterestNotifier; - if( notify && !notify->init()) { - notify->release(); - notify = 0; - } + notify->handler = handler; + notify->target = target; + notify->ref = ref; + notify->state = kIOServiceNotifyEnable; - if( notify) { - notify->handler = handler; - notify->target = target; - notify->ref = ref; - notify->state = kIOServiceNotifyEnable; - queue_init( ¬ify->handlerInvocations ); + ////// queue - ////// queue + LOCKWRITENOTIFY(); - LOCKWRITENOTIFY(); - if( 0 == (set = (OSArray *) getProperty( typeOfInterest ))) { - set = OSArray::withCapacity( 1 ); - if( set) { - setProperty( typeOfInterest, set ); - set->release(); - } + // Get the head of the notifier linked list + IOCommand * notifyList; + OSObject * obj = copyProperty( typeOfInterest ); + if (!(notifyList = OSDynamicCast(IOCommand, obj))) { + notifyList = OSTypeAlloc(IOCommand); + if (notifyList) { + notifyList->init(); + bool ok = setProperty( typeOfInterest, notifyList); + notifyList->release(); + if (!ok) notifyList = 0; } - notify->whence = set; - if( set) - set->setObject( notify ); - UNLOCKNOTIFY(); } + if (obj) obj->release(); + + if (notifyList) { + enqueue(¬ifyList->fCommandChain, ¬ify->chain); + notify->retain(); // ref'ed while in list + } + + UNLOCKNOTIFY(); + } + else { + rc = kIOReturnNotReady; } unlockForArbitration(); - return( notify ); + return rc; } -static void cleanInterestArray( OSObject * object ) +static void cleanInterestList( OSObject * head ) { - OSArray * array; - unsigned int index; - _IOServiceInterestNotifier * next; - - if( (array = OSDynamicCast( OSArray, object))) { - LOCKWRITENOTIFY(); - for( index = 0; - (next = (_IOServiceInterestNotifier *) - array->getObject( index )); - index++) { - next->whence = 0; - } - UNLOCKNOTIFY(); + IOCommand *notifyHead = OSDynamicCast(IOCommand, head); + if (!notifyHead) + return; + + LOCKWRITENOTIFY(); + while ( queue_entry_t entry = dequeue(¬ifyHead->fCommandChain) ) { + queue_next(entry) = queue_prev(entry) = 0; + + _IOServiceInterestNotifier * notify; + + queue_element(entry, notify, _IOServiceInterestNotifier *, chain); + notify->release(); } + UNLOCKNOTIFY(); } void IOService::unregisterAllInterest( void ) { - cleanInterestArray( getProperty( gIOGeneralInterest )); - cleanInterestArray( getProperty( gIOBusyInterest )); - cleanInterestArray( getProperty( gIOAppPowerStateInterest )); - cleanInterestArray( getProperty( gIOPriorityPowerStateInterest )); + OSObject * prop; + + prop = copyProperty(gIOGeneralInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOBusyInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOAppPowerStateInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOPriorityPowerStateInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOConsoleSecurityInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); } /* @@ -1438,10 +1949,10 @@ void _IOServiceInterestNotifier::remove() { LOCKWRITENOTIFY(); - if( whence) { - whence->removeObject(whence->getNextIndexOfObject( - (OSObject *) this, 0 )); - whence = 0; + if( queue_next( &chain )) { + remqueue(&chain); + queue_next( &chain) = queue_prev( &chain) = 0; + release(); } state &= ~kIOServiceNotifyEnable; @@ -1479,6 +1990,11 @@ void _IOServiceInterestNotifier::enable( bool was ) UNLOCKNOTIFY(); } +bool _IOServiceInterestNotifier::init() +{ + queue_init( &handlerInvocations ); + return (OSObject::init()); +} /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* @@ -1487,9 +2003,9 @@ void _IOServiceInterestNotifier::enable( bool was ) #define tailQ(o) setObject(o) #define headQ(o) setObject(0, o) -#define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) IOLog(fmt, ## args); } +#define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }} -inline void _workLoopAction( IOWorkLoop::Action action, +static void _workLoopAction( IOWorkLoop::Action action, IOService * service, void * p0 = 0, void * p1 = 0, void * p2 = 0, void * p3 = 0 ) @@ -1514,7 +2030,7 @@ bool IOService::requestTerminate( IOService * provider, IOOptionBits options ) // -- compat if( ok) { provider->terminateClient( this, options | kIOServiceRecursing ); - ok = (0 != (__state[1] & kIOServiceRecursing)); + ok = (0 != (kIOServiceInactiveState & __state[0])); } // -- @@ -1523,59 +2039,137 @@ bool IOService::requestTerminate( IOService * provider, IOOptionBits options ) bool IOService::terminatePhase1( IOOptionBits options ) { - IOService * victim; - IOService * client; - OSIterator * iter; - OSArray * makeInactive; - bool ok; - bool didInactive; - bool startPhase2 = false; - - TLOG("%s::terminatePhase1(%08lx)\n", getName(), options); + IOService * victim; + IOService * client; + OSIterator * iter; + OSArray * makeInactive; + OSArray * waitingInactive; + int waitResult = THREAD_AWAKENED; + bool wait; + bool ok; + bool didInactive; + bool startPhase2 = false; + + TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options); + + uint64_t regID = getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_PHASE1, + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) this, + (uintptr_t) options); // -- compat if( options & kIOServiceRecursing) { - __state[1] |= kIOServiceRecursing; + lockForArbitration(); + if (0 == (kIOServiceInactiveState & __state[0])) + { + __state[0] |= kIOServiceInactiveState; + __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State; + } + unlockForArbitration(); + return( true ); } // -- - makeInactive = OSArray::withCapacity( 16 ); - if( !makeInactive) - return( false ); + makeInactive = OSArray::withCapacity( 16 ); + waitingInactive = OSArray::withCapacity( 16 ); + if(!makeInactive || !waitingInactive) return( false ); victim = this; victim->retain(); - while( victim ) { - + while( victim ) + { didInactive = victim->lockForArbitration( true ); - if( didInactive) { - didInactive = (0 == (victim->__state[0] & kIOServiceInactiveState)); - if( didInactive) { - victim->__state[0] |= kIOServiceInactiveState; - victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState - | kIOServiceFirstPublishState | kIOServiceFirstMatchState); - victim->_adjustBusy( 1 ); - } + if( didInactive) + { + uint64_t regID1 = victim->getRegistryEntryID(); + IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) victim->__state[1], + (uintptr_t) 0); + + enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State }; + didInactive = (kRP1 == (victim->__state[1] & kRP1)) + || (0 == (victim->__state[0] & kIOServiceInactiveState)); + + if (!didInactive) + { + // a multiply attached IOService can be visited twice + if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) do + { + IOLockLock(gIOServiceBusyLock); + wait = (victim->__state[1] & kIOServiceTermPhase1State); + if( wait) { + TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n", + getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID()); + victim->__state[1] |= kIOServiceTerm1WaiterState; + victim->unlockForArbitration(); + assert_wait((event_t)&victim->__state[1], THREAD_UNINT); + } + IOLockUnlock(gIOServiceBusyLock); + if( wait) { + waitResult = thread_block(THREAD_CONTINUE_NULL); + TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n", + getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID()); + victim->lockForArbitration(); + } + } + while (wait && (waitResult != THREAD_TIMED_OUT)); + } + else + { + victim->__state[0] |= kIOServiceInactiveState; + victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState + | kIOServiceFirstPublishState | kIOServiceFirstMatchState); + victim->__state[1] &= ~kIOServiceRecursing; + victim->__state[1] |= kIOServiceTermPhase1State; + waitingInactive->headQ(victim); + if (victim == this) + { + if (kIOServiceTerminateNeedWillTerminate & options) + { + victim->__state[1] |= kIOServiceNeedWillTerminate; + } + } + victim->_adjustBusy( 1 ); + } victim->unlockForArbitration(); } - if( victim == this) - startPhase2 = didInactive; - if( didInactive) { + if( victim == this) startPhase2 = didInactive; + if (didInactive) + { + OSArray * notifiers; + notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff); + victim->invokeNotifiers(¬ifiers); - victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff ); IOUserClient::destroyUserReferences( victim ); - victim->unregisterAllInterest(); iter = victim->getClientIterator(); if( iter) { while( (client = (IOService *) iter->getNextObject())) { - TLOG("%s::requestTerminate(%s, %08lx)\n", - client->getName(), victim->getName(), options); + TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n", + client->getName(), client->getRegistryEntryID(), + victim->getName(), victim->getRegistryEntryID(), (long long)options); ok = client->requestTerminate( victim, options ); - TLOG("%s::requestTerminate(%s, ok = %d)\n", - client->getName(), victim->getName(), ok); + TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n", + client->getName(), client->getRegistryEntryID(), + victim->getName(), victim->getRegistryEntryID(), ok); + + uint64_t regID1 = client->getRegistryEntryID(); + uint64_t regID2 = victim->getRegistryEntryID(); + IOServiceTrace( + (ok ? IOSERVICE_TERMINATE_REQUEST_OK + : IOSERVICE_TERMINATE_REQUEST_FAIL), + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + if( ok) makeInactive->setObject( client ); } @@ -1589,39 +2183,98 @@ bool IOService::terminatePhase1( IOOptionBits options ) makeInactive->removeObject(0); } } - makeInactive->release(); + while ((victim = (IOService *) waitingInactive->getObject(0))) + { + victim->retain(); + waitingInactive->removeObject(0); + + victim->lockForArbitration(); + victim->__state[1] &= ~kIOServiceTermPhase1State; + if (kIOServiceTerm1WaiterState & victim->__state[1]) + { + victim->__state[1] &= ~kIOServiceTerm1WaiterState; + TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID()); + IOLockLock( gIOServiceBusyLock ); + thread_wakeup( (event_t) &victim->__state[1]); + IOLockUnlock( gIOServiceBusyLock ); + } + victim->unlockForArbitration(); + victim->release(); + } + waitingInactive->release(); + if( startPhase2) - scheduleTerminatePhase2( options ); + { + retain(); + lockForArbitration(); + scheduleTerminatePhase2(options); + unlockForArbitration(); + release(); + } return( true ); } +void IOService::setTerminateDefer(IOService * provider, bool defer) +{ + lockForArbitration(); + if (defer) __state[1] |= kIOServiceStartState; + else __state[1] &= ~kIOServiceStartState; + unlockForArbitration(); + + if (provider && !defer) + { + provider->lockForArbitration(); + provider->scheduleTerminatePhase2(); + provider->unlockForArbitration(); + } +} + +// Must call this while holding gJobsLock +void IOService::waitToBecomeTerminateThread(void) +{ + IOLockAssert(gJobsLock, kIOLockAssertOwned); + bool wait; + do { + wait = (gIOTerminateThread != THREAD_NULL); + if (wait) { + IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT); + } + } while (wait); + gIOTerminateThread = current_thread(); +} + +// call with lockForArbitration void IOService::scheduleTerminatePhase2( IOOptionBits options ) { AbsoluteTime deadline; - int waitResult; - bool wait, haveDeadline = false; + uint64_t regID1; + int waitResult = THREAD_AWAKENED; + bool wait = false, haveDeadline = false; - options |= kIOServiceRequired; + if (!(__state[0] & kIOServiceInactiveState)) return; - retain(); + regID1 = getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERM_SCHED_PHASE2, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) __state[1], + (uintptr_t) options); + if (__state[1] & kIOServiceTermPhase1State) return; + + retain(); + unlockForArbitration(); + options |= kIOServiceRequired; IOLockLock( gJobsLock ); if( (options & kIOServiceSynchronous) && (current_thread() != gIOTerminateThread)) { - do { - wait = (gIOTerminateThread != 0); - if( wait) { - // wait to become the terminate thread - IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT); - } - } while( wait ); - - gIOTerminateThread = current_thread(); + waitToBecomeTerminateThread(); gIOTerminatePhase2List->setObject( this ); gIOTerminateWork++; @@ -1630,17 +2283,20 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) terminateWorker( options ); wait = (0 != (__state[1] & kIOServiceBusyStateMask)); if( wait) { - // wait for the victim to go non-busy + /* wait for the victim to go non-busy */ if( !haveDeadline) { clock_interval_to_deadline( 15, kSecondScale, &deadline ); haveDeadline = true; } + /* let others do work while we wait */ + gIOTerminateThread = 0; + IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false); waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork, deadline, THREAD_UNINT ); - if( waitResult == THREAD_TIMED_OUT) { - TLOG("%s::terminate(kIOServiceSynchronous) timeout", getName()); - } else - thread_cancel_timer(); + if (__improbable(waitResult == THREAD_TIMED_OUT)) { + panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID()); + } + waitToBecomeTerminateThread(); } } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT))); @@ -1652,64 +2308,81 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) gIOTerminatePhase2List->setObject( this ); if( 0 == gIOTerminateWork++) { - if( !gIOTerminateThread) - gIOTerminateThread = IOCreateThread( &terminateThread, (void *) options ); - else - IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); - } + assert(gIOTerminateWorkerThread); + IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false ); + } } IOLockUnlock( gJobsLock ); - + lockForArbitration(); release(); } -void IOService::terminateThread( void * arg ) +__attribute__((__noreturn__)) +void IOService::terminateThread( void * arg, wait_result_t waitResult ) { - IOLockLock( gJobsLock ); - - while (gIOTerminateWork) - terminateWorker( (IOOptionBits) arg ); + // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once + IOLockLock(gJobsLock); + while (true) { + if (gIOTerminateThread != gIOTerminateWorkerThread) { + waitToBecomeTerminateThread(); + } - gIOTerminateThread = 0; - IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false); + while (gIOTerminateWork) + terminateWorker( (uintptr_t)arg ); - IOLockUnlock( gJobsLock ); + gIOTerminateThread = 0; + IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false); + IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT); + } } void IOService::scheduleStop( IOService * provider ) { - TLOG("%s::scheduleStop(%s)\n", getName(), provider->getName()); + uint64_t regID1 = getRegistryEntryID(); + uint64_t regID2 = provider->getRegistryEntryID(); + + TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2); + IOServiceTrace( + IOSERVICE_TERMINATE_SCHEDULE_STOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); IOLockLock( gJobsLock ); gIOStopList->tailQ( this ); gIOStopProviderList->tailQ( provider ); if( 0 == gIOTerminateWork++) { - if( !gIOTerminateThread) - gIOTerminateThread = IOCreateThread( &terminateThread, (void *) 0 ); - else - IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); + assert(gIOTerminateWorkerThread); + IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false ); } IOLockUnlock( gJobsLock ); } -void IOService::scheduleFinalize( void ) +void IOService::scheduleFinalize(bool now) { - TLOG("%s::scheduleFinalize\n", getName()); + uint64_t regID1 = getRegistryEntryID(); - IOLockLock( gJobsLock ); - gIOFinalizeList->tailQ( this ); + TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1); + IOServiceTrace( + IOSERVICE_TERMINATE_SCHEDULE_FINALIZE, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + 0, 0); - if( 0 == gIOTerminateWork++) { - if( !gIOTerminateThread) - gIOTerminateThread = IOCreateThread( &terminateThread, (void *) 0 ); - else - IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); + if (now || IOUserClient::finalizeUserReferences(this)) + { + IOLockLock( gJobsLock ); + gIOFinalizeList->tailQ(this); + if( 0 == gIOTerminateWork++) { + assert(gIOTerminateWorkerThread); + IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false ); + } + IOLockUnlock( gJobsLock ); } - - IOLockUnlock( gJobsLock ); } bool IOService::willTerminate( IOService * provider, IOOptionBits options ) @@ -1726,7 +2399,7 @@ bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * scheduleStop( provider ); // -- compat else { - message( kIOMessageServiceIsRequestingClose, provider, (void *) options ); + message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options ); if( false == provider->handleIsOpen( this )) scheduleStop( provider ); } @@ -1739,17 +2412,30 @@ bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * } void IOService::actionWillTerminate( IOService * victim, IOOptionBits options, - OSArray * doPhase2List ) + OSArray * doPhase2List, + void *unused2 __unused, + void *unused3 __unused ) { OSIterator * iter; IOService * client; bool ok; + uint64_t regID1, regID2 = victim->getRegistryEntryID(); iter = victim->getClientIterator(); if( iter) { while( (client = (IOService *) iter->getNextObject())) { - TLOG("%s::willTerminate(%s, %08lx)\n", - client->getName(), victim->getName(), options); + + regID1 = client->getRegistryEntryID(); + TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n", + client->getName(), regID1, + victim->getName(), regID2, (long long)options); + IOServiceTrace( + IOSERVICE_TERMINATE_WILL, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + ok = client->willTerminate( victim, options ); doPhase2List->tailQ( client ); } @@ -1757,40 +2443,146 @@ void IOService::actionWillTerminate( IOService * victim, IOOptionBits options, } } -void IOService::actionDidTerminate( IOService * victim, IOOptionBits options ) +void IOService::actionDidTerminate( IOService * victim, IOOptionBits options, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) { OSIterator * iter; IOService * client; - bool defer = false; + bool defer; + uint64_t regID1, regID2 = victim->getRegistryEntryID(); - victim->messageClients( kIOMessageServiceIsTerminated, (void *) options ); + victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options ); iter = victim->getClientIterator(); if( iter) { while( (client = (IOService *) iter->getNextObject())) { - TLOG("%s::didTerminate(%s, %08lx)\n", - client->getName(), victim->getName(), options); + + regID1 = client->getRegistryEntryID(); + TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n", + client->getName(), regID1, + victim->getName(), regID2, (long long)options); + defer = false; client->didTerminate( victim, options, &defer ); - TLOG("%s::didTerminate(%s, defer %d)\n", - client->getName(), victim->getName(), defer); + + IOServiceTrace( + (defer ? IOSERVICE_TERMINATE_DID_DEFER + : IOSERVICE_TERMINATE_DID), + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + + TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n", + client->getName(), regID1, + victim->getName(), regID2, defer); + } + iter->release(); + } +} + + +void IOService::actionWillStop( IOService * victim, IOOptionBits options, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) +{ + OSIterator * iter; + IOService * provider; + bool ok; + uint64_t regID1, regID2 = victim->getRegistryEntryID(); + + iter = victim->getProviderIterator(); + if( iter) { + while( (provider = (IOService *) iter->getNextObject())) { + + regID1 = provider->getRegistryEntryID(); + TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n", + victim->getName(), regID2, + provider->getName(), regID1, (long long)options); + IOServiceTrace( + IOSERVICE_TERMINATE_WILL, + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32), + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32)); + + ok = victim->willTerminate( provider, options ); + } + iter->release(); + } +} + +void IOService::actionDidStop( IOService * victim, IOOptionBits options, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) +{ + OSIterator * iter; + IOService * provider; + bool defer = false; + uint64_t regID1, regID2 = victim->getRegistryEntryID(); + + iter = victim->getProviderIterator(); + if( iter) { + while( (provider = (IOService *) iter->getNextObject())) { + + regID1 = provider->getRegistryEntryID(); + TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n", + victim->getName(), regID2, + provider->getName(), regID1, (long long)options); + victim->didTerminate( provider, options, &defer ); + + IOServiceTrace( + (defer ? IOSERVICE_TERMINATE_DID_DEFER + : IOSERVICE_TERMINATE_DID), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32), + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32)); + + TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n", + victim->getName(), regID2, + provider->getName(), regID1, defer); } iter->release(); } } -void IOService::actionFinalize( IOService * victim, IOOptionBits options ) + +void IOService::actionFinalize( IOService * victim, IOOptionBits options, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) { - TLOG("%s::finalize(%08lx)\n", victim->getName(), options); + uint64_t regID1 = victim->getRegistryEntryID(); + TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options); + IOServiceTrace( + IOSERVICE_TERMINATE_FINALIZE, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + 0, 0); + victim->finalize( options ); } -void IOService::actionStop( IOService * provider, IOService * client ) +void IOService::actionStop( IOService * provider, IOService * client, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) { - TLOG("%s::stop(%s)\n", client->getName(), provider->getName()); + uint64_t regID1 = provider->getRegistryEntryID(); + uint64_t regID2 = client->getRegistryEntryID(); + + TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1); + IOServiceTrace( + IOSERVICE_TERMINATE_STOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + client->stop( provider ); if( provider->isOpen( client )) provider->close( client ); - TLOG("%s::detach(%s)\n", client->getName(), provider->getName()); + + TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1); client->detach( provider ); } @@ -1799,6 +2591,7 @@ void IOService::terminateWorker( IOOptionBits options ) OSArray * doPhase2List; OSArray * didPhase2List; OSSet * freeList; + OSIterator * iter; UInt32 workDone; IOService * victim; IOService * client; @@ -1825,29 +2618,72 @@ void IOService::terminateWorker( IOOptionBits options ) gIOTerminatePhase2List->removeObject(0); IOLockUnlock( gJobsLock ); + uint64_t regID1 = victim->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERM_START_PHASE2, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) 0, + (uintptr_t) 0); + while( victim ) { doPhase2 = victim->lockForArbitration( true ); if( doPhase2) { doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0])); if( doPhase2) { - doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State)) - && (0 == (victim->__state[1] & kIOServiceConfigState)); + + uint64_t regID1 = victim->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERM_TRY_PHASE2, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) victim->__state[1], + (uintptr_t) 0); + + doPhase2 = (0 == (victim->__state[1] & + (kIOServiceTermPhase1State + | kIOServiceTermPhase2State + | kIOServiceConfigState))); + + if (doPhase2 && (iter = victim->getClientIterator())) { + while (doPhase2 && (client = (IOService *) iter->getNextObject())) { + doPhase2 = (0 == (client->__state[1] & kIOServiceStartState)); + if (!doPhase2) + { + uint64_t regID1 = client->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERM_UC_DEFER, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) client->__state[1], + (uintptr_t) 0); + TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n", + victim->getName(), victim->getRegistryEntryID(), + client->getName(), client->getRegistryEntryID()); + } + } + iter->release(); + } if( doPhase2) victim->__state[1] |= kIOServiceTermPhase2State; } victim->unlockForArbitration(); } if( doPhase2) { - if( 0 == victim->getClient()) { - // no clients - will go to finalize - IOLockLock( gJobsLock ); - gIOFinalizeList->tailQ( victim ); - IOLockUnlock( gJobsLock ); - } else { - _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate, - victim, (void *) options, (void *) doPhase2List ); - } + + if (kIOServiceNeedWillTerminate & victim->__state[1]) { + _workLoopAction( (IOWorkLoop::Action) &actionWillStop, + victim, (void *)(uintptr_t) options, NULL ); + } + + OSArray * notifiers; + notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff); + victim->invokeNotifiers(¬ifiers); + + _workLoopAction( (IOWorkLoop::Action) &actionWillTerminate, + victim, (void *)(uintptr_t) options, (void *)(uintptr_t) doPhase2List ); + didPhase2List->headQ( victim ); } victim->release(); @@ -1859,13 +2695,20 @@ void IOService::terminateWorker( IOOptionBits options ) } while( (victim = (IOService *) didPhase2List->getObject(0)) ) { - - if( victim->lockForArbitration( true )) { + bool scheduleFinalize = false; + if( victim->lockForArbitration( true )) { victim->__state[1] |= kIOServiceTermPhase3State; + scheduleFinalize = (0 == victim->getClient()); victim->unlockForArbitration(); } _workLoopAction( (IOWorkLoop::Action) &actionDidTerminate, - victim, (void *) options ); + victim, (void *)(uintptr_t) options ); + if (kIOServiceNeedWillTerminate & victim->__state[1]) { + _workLoopAction( (IOWorkLoop::Action) &actionDidStop, + victim, (void *)(uintptr_t) options, NULL ); + } + // no clients - will go to finalize + if (scheduleFinalize) victim->scheduleFinalize(false); didPhase2List->removeObject(0); } IOLockLock( gJobsLock ); @@ -1876,10 +2719,17 @@ void IOService::terminateWorker( IOOptionBits options ) doPhase3 = false; // finalize leaves while( (victim = (IOService *) gIOFinalizeList->getObject(0))) { - + bool sendFinal = false; IOLockUnlock( gJobsLock ); - _workLoopAction( (IOWorkLoop::Action) &actionFinalize, - victim, (void *) options ); + if (victim->lockForArbitration(true)) { + sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized)); + if (sendFinal) victim->__state[1] |= kIOServiceFinalized; + victim->unlockForArbitration(); + } + if (sendFinal) { + _workLoopAction( (IOWorkLoop::Action) &actionFinalize, + victim, (void *)(uintptr_t) options ); + } IOLockLock( gJobsLock ); // hold off free freeList->setObject( victim ); @@ -1892,19 +2742,42 @@ void IOService::terminateWorker( IOOptionBits options ) provider = (IOService *) gIOStopProviderList->getObject(idx); assert( provider ); + + uint64_t regID1 = provider->getRegistryEntryID(); + uint64_t regID2 = client->getRegistryEntryID(); if( !provider->isChild( client, gIOServicePlane )) { // may be multiply queued - nop it - TLOG("%s::nop stop(%s)\n", client->getName(), provider->getName()); + TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1); + IOServiceTrace( + IOSERVICE_TERMINATE_STOP_NOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + } else { - // not ready for stop if it has clients, skip it - if( (client->__state[1] & kIOServiceTermPhase3State) && client->getClient()) { - TLOG("%s::defer stop(%s)\n", client->getName(), provider->getName()); - idx++; - continue; - } - + // a terminated client is not ready for stop if it has clients, skip it + bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0])); IOLockUnlock( gJobsLock ); + if (deferStop && client->lockForArbitration(true)) { + deferStop = (0 == (client->__state[1] & kIOServiceFinalized)); + //deferStop = (!deferStop && (0 != client->getClient())); + //deferStop = (0 != client->getClient()); + client->unlockForArbitration(); + if (deferStop) { + TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2); + IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + + idx++; + IOLockLock( gJobsLock ); + continue; + } + } _workLoopAction( (IOWorkLoop::Action) &actionStop, provider, (void *) client ); IOLockLock( gJobsLock ); @@ -1928,6 +2801,9 @@ void IOService::terminateWorker( IOOptionBits options ) if( !moreToDo) { TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount()); + IOServiceTrace( + IOSERVICE_TERMINATE_DONE, + (uintptr_t) gIOStopList->getCount(), 0, 0, 0); } } while( moreToDo ); @@ -1943,8 +2819,9 @@ void IOService::terminateWorker( IOOptionBits options ) bool IOService::finalize( IOOptionBits options ) { - OSIterator * iter; - IOService * provider; + OSIterator * iter; + IOService * provider; + uint64_t regID1, regID2 = getRegistryEntryID(); iter = getProviderIterator(); assert( iter ); @@ -1955,6 +2832,16 @@ bool IOService::finalize( IOOptionBits options ) // -- compat if( 0 == (__state[1] & kIOServiceTermPhase3State)) { /* we come down here on programmatic terminate */ + + regID1 = provider->getRegistryEntryID(); + TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1); + IOServiceTrace( + IOSERVICE_TERMINATE_STOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + stop( provider ); if( provider->isOpen( this )) provider->close( this ); @@ -2025,7 +2912,7 @@ static void serviceOpenMessageApplier( OSObject * object, void * ctx ) ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx; if( object != context->excludeClient) - context->service->messageClient( context->type, object, (void *) context->options ); + context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options ); } bool IOService::open( IOService * forClient, @@ -2114,7 +3001,7 @@ bool IOService::handleOpen( IOService * forClient, else if( options & kIOServiceSeize ) { ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose, - __owner, (void *) options )); + __owner, (void *)(uintptr_t) options )); if( ok && (0 == __owner )) __owner = forClient; else @@ -2167,23 +3054,30 @@ static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref) _IOServiceNotifier * notify; OSSymbol * key = (OSSymbol *) ref; OSNumber * offset; + OSObject * prop; + SInt32 result; - if( (notify = OSDynamicCast( _IOServiceNotifier, entry))) + prop = 0; + result = kIODefaultProbeScore; + if( (dict = OSDynamicCast( OSDictionary, entry))) + offset = OSDynamicCast(OSNumber, dict->getObject( key )); + else if( (notify = OSDynamicCast( _IOServiceNotifier, entry))) return( notify->priority ); - else if( (service = OSDynamicCast( IOService, entry))) - offset = OSDynamicCast(OSNumber, service->getProperty( key )); - else if( (dict = OSDynamicCast( OSDictionary, entry))) - offset = OSDynamicCast(OSNumber, dict->getObject( key )); + { + prop = service->copyProperty(key); + offset = OSDynamicCast(OSNumber, prop); + } else { assert( false ); offset = 0; } - if( offset) - return( (SInt32) offset->unsigned32BitValue()); - else - return( kIODefaultProbeScore ); + if (offset) result = offset->unsigned32BitValue(); + + OSSafeReleaseNULL(prop); + + return (result); } SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref ) @@ -2205,7 +3099,7 @@ SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase return ( val1 - val2 ); } -IOService * IOService::getClientWithCategory( const OSSymbol * category ) +IOService * IOService::copyClientWithCategory( const OSSymbol * category ) { IOService * service = 0; OSIterator * iter; @@ -2219,21 +3113,41 @@ IOService * IOService::getClientWithCategory( const OSSymbol * category ) nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol, service->getProperty( gIOMatchCategoryKey )); if( category == nextCat) + { + service->retain(); break; + } } iter->release(); } return( service ); } -bool IOService::invokeNotifer( _IOServiceNotifier * notify ) +IOService * IOService::getClientWithCategory( const OSSymbol * category ) +{ + IOService * + service = copyClientWithCategory(category); + if (service) + service->release(); + return (service); +} + +bool IOService::invokeNotifier( _IOServiceNotifier * notify ) { _IOServiceNotifierInvocation invocation; bool willNotify; bool ret = true; - invocation.thread = current_thread(); +#if DEBUG_NOTIFIER_LOCKED + uint32_t count; + if ((count = isLockedForArbitration(0))) + { + IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count); + panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count); + } +#endif /* DEBUG_NOTIFIER_LOCKED */ + LOCKWRITENOTIFY(); willNotify = (0 != (kIOServiceNotifyEnable & notify->state)); @@ -2245,7 +3159,7 @@ bool IOService::invokeNotifer( _IOServiceNotifier * notify ) if( willNotify) { - ret = (*notify->handler)( notify->target, notify->ref, this ); + ret = (*notify->handler)(notify->target, notify->ref, this, notify); LOCKWRITENOTIFY(); queue_remove( ¬ify->handlerInvocations, &invocation, @@ -2260,6 +3174,27 @@ bool IOService::invokeNotifer( _IOServiceNotifier * notify ) return( ret ); } +bool IOService::invokeNotifiers(OSArray ** willSend) +{ + OSArray * array; + _IOServiceNotifier * notify; + bool ret = true; + + array = *willSend; + if (!array) return (true); + *willSend = 0; + + for( unsigned int idx = 0; + (notify = (_IOServiceNotifier *) array->getObject(idx)); + idx++) { + ret &= invokeNotifier(notify); + } + array->release(); + + return (ret); +} + + /* * Alloc and probe matching classes, * called on the provider instance @@ -2286,6 +3221,8 @@ void IOService::probeCandidates( OSOrderedSet * matches ) #if IOMATCHDEBUG SInt64 debugFlags; #endif + IOService * client = NULL; + assert( matches ); while( !needReloc && (nextMatch = matches->getFirstObject())) { @@ -2295,10 +3232,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) { - lockForArbitration(); - if( 0 == (__state[0] & kIOServiceInactiveState)) - invokeNotifer( notify ); - unlockForArbitration(); + if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify ); nextMatch->release(); nextMatch = 0; continue; @@ -2319,15 +3253,19 @@ void IOService::probeCandidates( OSOrderedSet * matches ) match->getObject( gIOMatchCategoryKey )); if( 0 == category) category = gIODefaultMatchCategoryKey; - - if( getClientWithCategory( category )) { + + if( (client = copyClientWithCategory(category)) ) { #if IOMATCHDEBUG - if( debugFlags & kIOLogMatch) + if( (debugFlags & kIOLogMatch) && (this != gIOResources)) LOG("%s: match category %s exists\n", getName(), category->getCStringNoCopy()); #endif nextMatch->release(); nextMatch = 0; + + client->release(); + client = NULL; + continue; } @@ -2338,7 +3276,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) props->setCapacityIncrement(1); // check the nub matches - if( false == passiveMatch( props, true )) + if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) continue; // Check to see if driver reloc has been loaded. @@ -2391,10 +3329,12 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( !symbol) continue; + //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props)); + // alloc the driver instance inst = (IOService *) OSMetaClass::allocClassWithName( symbol); - if( !inst) { + if( !inst || !OSDynamicCast(IOService, inst)) { IOLog("Couldn't alloc class \"%s\"\n", symbol->getCStringNoCopy()); continue; @@ -2417,7 +3357,6 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( 0 == category) category = gIODefaultMatchCategoryKey; inst->setProperty( gIOMatchCategoryKey, (OSObject *) category ); - // attach driver instance if( !(inst->attach( this ))) continue; @@ -2496,7 +3435,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) startList->removeObject(inst); #if IOMATCHDEBUG - debugFlags = getDebugFlags( inst->getPropertyTable() ); + debugFlags = getDebugFlags( inst ); if( debugFlags & kIOLogStart) { if( started) @@ -2519,16 +3458,35 @@ void IOService::probeCandidates( OSOrderedSet * matches ) } - // adjust the busy count by -1 if matching is stalled for a module, - // or +1 if a previously stalled matching is complete. + // adjust the busy count by +1 if matching is stalled for a module, + // or -1 if a previously stalled matching is complete. lockForArbitration(); SInt32 adjBusy = 0; + uint64_t regID = getRegistryEntryID(); + if( needReloc) { adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1; - if( adjBusy) + if( adjBusy) { + + IOServiceTrace( + IOSERVICE_MODULESTALL, + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) this, + 0); + __state[1] |= kIOServiceModuleStallState; + } } else if( __state[1] & kIOServiceModuleStallState) { + + IOServiceTrace( + IOSERVICE_MODULEUNSTALL, + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) this, + 0); + __state[1] &= ~kIOServiceModuleStallState; adjBusy = -1; } @@ -2551,11 +3509,15 @@ bool IOService::startCandidate( IOService * service ) ok = service->attach( this ); - if( ok) { - // stall for any nub resources - checkResources(); - // stall for any driver resources - service->checkResources(); + if( ok) + { + if (this != gIOResources) + { + // stall for any nub resources + checkResources(); + // stall for any driver resources + service->checkResources(); + } AbsoluteTime startTime; AbsoluteTime endTime; @@ -2575,7 +3537,7 @@ bool IOService::startCandidate( IOService * service ) SUB_ABSOLUTETIME(&endTime, &startTime); absolutetime_to_nanoseconds(endTime, &nano); if (nano > 500000000ULL) - IOLog("%s::start took %ld ms\n", service->getName(), (UInt32)(nano / 1000000ULL)); + IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL)); } } if( !ok) @@ -2584,11 +3546,6 @@ bool IOService::startCandidate( IOService * service ) return( ok ); } -IOService * IOService::resources( void ) -{ - return( gIOResources ); -} - void IOService::publishResource( const char * key, OSObject * value ) { const OSSymbol * sym; @@ -2615,22 +3572,26 @@ void IOService::publishResource( const OSSymbol * key, OSObject * value ) bool IOService::addNeededResource( const char * key ) { - OSObject * resources; + OSObject * resourcesProp; OSSet * set; OSString * newKey; bool ret; - resources = getProperty( gIOResourceMatchKey ); + resourcesProp = copyProperty( gIOResourceMatchKey ); + if (!resourcesProp) return(false); newKey = OSString::withCString( key ); - if( (0 == resources) || (0 == newKey)) + if (!newKey) + { + resourcesProp->release(); return( false); + } - set = OSDynamicCast( OSSet, resources ); + set = OSDynamicCast( OSSet, resourcesProp ); if( !set) { set = OSSet::withCapacity( 1 ); if( set) - set->setObject( resources ); + set->setObject( resourcesProp ); } else set->retain(); @@ -2639,6 +3600,7 @@ bool IOService::addNeededResource( const char * key ) newKey->release(); ret = setProperty( gIOResourceMatchKey, set ); set->release(); + resourcesProp->release(); return( ret ); } @@ -2665,44 +3627,46 @@ bool IOService::checkResource( OSObject * matching ) } if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): stalling %s\n", (int) IOThreadSelf(), getName()); + LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName()); waitForService( table ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): waking\n", (int) IOThreadSelf() ); + LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) ); return( true ); } bool IOService::checkResources( void ) { - OSObject * resources; + OSObject * resourcesProp; OSSet * set; OSIterator * iter; bool ok; - resources = getProperty( gIOResourceMatchKey ); - if( 0 == resources) + resourcesProp = copyProperty( gIOResourceMatchKey ); + if( 0 == resourcesProp) return( true ); - if( (set = OSDynamicCast( OSSet, resources ))) { + if( (set = OSDynamicCast( OSSet, resourcesProp ))) { iter = OSCollectionIterator::withCollection( set ); ok = (0 != iter); - while( ok && (resources = iter->getNextObject()) ) - ok = checkResource( resources ); + while( ok && (resourcesProp = iter->getNextObject()) ) + ok = checkResource( resourcesProp ); if( iter) iter->release(); } else - ok = checkResource( resources ); + ok = checkResource( resourcesProp ); + + OSSafeReleaseNULL(resourcesProp); return( ok ); } -_IOConfigThread * _IOConfigThread::configThread( void ) +void _IOConfigThread::configThread( void ) { _IOConfigThread * inst; @@ -2711,22 +3675,23 @@ _IOConfigThread * _IOConfigThread::configThread( void ) continue; if( !inst->init()) continue; - if( !(inst->thread = IOCreateThread - ( (IOThreadFunc) &_IOConfigThread::main, inst ))) + thread_t unused; + if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused)) continue; - return( inst ); + return; } while( false); if( inst) inst->release(); - return( 0 ); + return; } void _IOConfigThread::free( void ) { + thread_deallocate(current_thread()); OSObject::free(); } @@ -2735,9 +3700,12 @@ void IOService::doServiceMatch( IOOptionBits options ) _IOServiceNotifier * notify; OSIterator * iter; OSOrderedSet * matches; + OSArray * resourceKeys = 0; SInt32 catalogGeneration; bool keepGuessing = true; bool reRegistered = true; + bool didRegister; + OSArray * notifiers[2] = {0}; // job->nub->deliverNotification( gIOPublishNotification, // kIOServiceRegisteredState, 0xffffffff ); @@ -2749,23 +3717,26 @@ void IOService::doServiceMatch( IOOptionBits options ) if( matches) { lockForArbitration(); - if( 0 == (__state[0] & kIOServiceFirstPublishState)) - deliverNotification( gIOFirstPublishNotification, + if( 0 == (__state[0] & kIOServiceFirstPublishState)) { + getMetaClass()->addInstance(this); + notifiers[0] = copyNotifiers(gIOFirstPublishNotification, kIOServiceFirstPublishState, 0xffffffff ); + } LOCKREADNOTIFY(); __state[1] &= ~kIOServiceNeedConfigState; - __state[1] |= kIOServiceConfigState; + __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning; + didRegister = (0 == (kIOServiceRegisteredState & __state[0])); __state[0] |= kIOServiceRegisteredState; - if( reRegistered && (0 == (__state[0] & kIOServiceInactiveState))) { - + keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState)); + if (reRegistered && keepGuessing) { iter = OSCollectionIterator::withCollection( (OSOrderedSet *) gNotifications->getObject( gIOPublishNotification ) ); if( iter) { while((notify = (_IOServiceNotifier *) iter->getNextObject())) { - if( passiveMatch( notify->matching ) + if( matchPassive(notify->matching, 0) && (kIOServiceNotifyEnable & notify->state)) matches->setObject( notify ); } @@ -2775,9 +3746,17 @@ void IOService::doServiceMatch( IOOptionBits options ) UNLOCKNOTIFY(); unlockForArbitration(); + invokeNotifiers(¬ifiers[0]); - if( matches->getCount() && (kIOReturnSuccess == getResources())) + if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) + { + if (this == gIOResources) + { + if (resourceKeys) resourceKeys->release(); + resourceKeys = copyPropertyKeys(); + } probeCandidates( matches ); + } else matches->release(); } @@ -2795,16 +3774,27 @@ void IOService::doServiceMatch( IOOptionBits options ) if( (0 == (__state[0] & kIOServiceInactiveState)) && (0 == (__state[1] & kIOServiceModuleStallState)) ) { - deliverNotification( gIOMatchedNotification, - kIOServiceMatchedState, 0xffffffff ); + + if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys); + + notifiers[0] = copyNotifiers(gIOMatchedNotification, + kIOServiceMatchedState, 0xffffffff); if( 0 == (__state[0] & kIOServiceFirstMatchState)) - deliverNotification( gIOFirstMatchNotification, - kIOServiceFirstMatchState, 0xffffffff ); + notifiers[1] = copyNotifiers(gIOFirstMatchNotification, + kIOServiceFirstMatchState, 0xffffffff); } + __state[1] &= ~kIOServiceConfigRunning; + unlockForArbitration(); + + if (resourceKeys) resourceKeys->release(); + + invokeNotifiers(¬ifiers[0]); + invokeNotifiers(¬ifiers[1]); + + lockForArbitration(); __state[1] &= ~kIOServiceConfigState; - if( __state[0] & kIOServiceInactiveState) - scheduleTerminatePhase2(); + scheduleTerminatePhase2(); _adjustBusy( -1 ); unlockForArbitration(); @@ -2824,11 +3814,13 @@ UInt32 IOService::_adjustBusy( SInt32 delta ) if( next != this) next->lockForArbitration(); count = next->__state[1] & kIOServiceBusyStateMask; - assert( count < kIOServiceBusyMax); wasQuiet = (0 == count); - assert( (!wasQuiet) || (delta > 0)); - next->__state[1] += delta; - nowQuiet = (0 == (next->__state[1] & kIOServiceBusyStateMask)); + if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) + OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta); + else + count += delta; + next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count; + nowQuiet = (0 == count); needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1])); if( needWake) { @@ -2841,24 +3833,41 @@ UInt32 IOService::_adjustBusy( SInt32 delta ) next->unlockForArbitration(); if( (wasQuiet || nowQuiet) ) { - OSArray * array; - unsigned int index; - OSObject * interested; - - array = OSDynamicCast( OSArray, next->getProperty( gIOBusyInterest )); - if( array) { - LOCKREADNOTIFY(); - for( index = 0; - (interested = array->getObject( index )); - index++) { - next->messageClient(kIOMessageServiceBusyStateChange, - interested, (void *) wasQuiet /* busy now */); - } - UNLOCKNOTIFY(); - } - if( nowQuiet && (next == gIOServiceRoot)) - OSMetaClass::considerUnloads(); + uint64_t regID = next->getRegistryEntryID(); + IOServiceTrace( + ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY), + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) next, + 0); + + if (wasQuiet) + { + next->__timeBusy = mach_absolute_time(); + } + else + { + next->__accumBusy += mach_absolute_time() - next->__timeBusy; + next->__timeBusy = 0; + } + + MessageClientsContext context; + + context.service = next; + context.type = kIOMessageServiceBusyStateChange; + context.argument = (void *) wasQuiet; /*nowBusy*/ + context.argSize = 0; + + applyToInterestNotifiers( next, gIOBusyInterest, + &messageClientsApplier, &context ); + +#if !NO_KEXTD + if( nowQuiet && (next == gIOServiceRoot)) { + OSKext::considerUnloads(); + IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0); + } +#endif } delta = nowQuiet ? -1 : +1; @@ -2875,13 +3884,40 @@ void IOService::adjustBusy( SInt32 delta ) unlockForArbitration(); } +uint64_t IOService::getAccumulatedBusyTime( void ) +{ + uint64_t accumBusy = __accumBusy; + uint64_t timeBusy = __timeBusy; + uint64_t nano; + + do + { + accumBusy = __accumBusy; + timeBusy = __timeBusy; + if (timeBusy) + accumBusy += mach_absolute_time() - timeBusy; + } + while (timeBusy != __timeBusy); + + absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano); + + return (nano); +} + UInt32 IOService::getBusyState( void ) { return( __state[1] & kIOServiceBusyStateMask ); } IOReturn IOService::waitForState( UInt32 mask, UInt32 value, - mach_timespec_t * timeout ) + mach_timespec_t * timeout ) +{ + panic("waitForState"); + return (kIOReturnUnsupported); +} + +IOReturn IOService::waitForState( UInt32 mask, UInt32 value, + uint64_t timeout ) { bool wait; int waitResult = THREAD_AWAKENED; @@ -2895,29 +3931,22 @@ IOReturn IOService::waitForState( UInt32 mask, UInt32 value, if( wait) { __state[1] |= kIOServiceBusyWaiterState; unlockForArbitration(); - assert_wait( (event_t) this, THREAD_UNINT ); - if( timeout ) { + if( timeout != UINT64_MAX ) { if( computeDeadline ) { AbsoluteTime nsinterval; - clock_interval_to_absolutetime_interval( - timeout->tv_sec, kSecondScale, &abstime ); - clock_interval_to_absolutetime_interval( - timeout->tv_nsec, kNanosecondScale, &nsinterval ); - ADD_ABSOLUTETIME( &abstime, &nsinterval ); - clock_absolutetime_interval_to_deadline( - abstime, &abstime ); + nanoseconds_to_absolutetime(timeout, &nsinterval ); + clock_absolutetime_interval_to_deadline(nsinterval, &abstime); computeDeadline = false; } - thread_set_timer_deadline( abstime ); + assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime)); } + else + assert_wait((event_t)this, THREAD_UNINT ); } else unlockForArbitration(); IOLockUnlock( gIOServiceBusyLock ); - if( wait) { + if( wait) waitResult = thread_block(THREAD_CONTINUE_NULL); - if( timeout && (waitResult != THREAD_TIMED_OUT)) - thread_cancel_timer(); - } } while( wait && (waitResult != THREAD_TIMED_OUT)); @@ -2927,9 +3956,121 @@ IOReturn IOService::waitForState( UInt32 mask, UInt32 value, return( kIOReturnSuccess ); } +IOReturn IOService::waitQuiet( uint64_t timeout ) +{ + IOReturn ret; + uint32_t loops; + char * string = NULL; + char * panicString = NULL; + size_t len; + size_t panicStringLen; + uint64_t time; + uint64_t nano; + bool kextdWait; + bool dopanic; + + enum { kTimeoutExtensions = 4 }; + + time = mach_absolute_time(); + kextdWait = false; + for (loops = 0; loops < kTimeoutExtensions; loops++) + { + ret = waitForState( kIOServiceBusyStateMask, 0, timeout ); + + if (loops && (kIOReturnSuccess == ret)) + { + time = mach_absolute_time() - time; + absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano); + IOLog("busy extended ok[%d], (%llds, %llds)\n", + loops, timeout / 1000000000ULL, nano / 1000000000ULL); + break; + } + else if (kIOReturnTimeout != ret) break; + else if (timeout < 41000000000) break; + + { + IORegistryIterator * iter; + OSOrderedSet * set; + OSOrderedSet * leaves; + IOService * next; + IOService * nextParent; + char * s; + size_t l; + + len = 256; + panicStringLen = 256; + if (!string) string = IONew(char, len); + if (!panicString) panicString = IONew(char, panicStringLen); + set = NULL; + kextdWait = OSKext::isWaitingKextd(); + iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively); + leaves = OSOrderedSet::withCapacity(4); + if (iter) set = iter->iterateAll(); + if (string && panicString && leaves && set) + { + string[0] = panicString[0] = 0; + set->setObject(this); + while ((next = (IOService *) set->getLastObject())) + { + if (next->getBusyState()) + { + if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true; + leaves->setObject(next); + nextParent = next; + while ((nextParent = nextParent->getProvider())) + { + set->removeObject(nextParent); + leaves->removeObject(nextParent); + } + } + set->removeObject(next); + } + s = string; + while ((next = (IOService *) leaves->getLastObject())) + { + l = snprintf(s, len, "%s'%s'", ((s == string) ? "" : ", "), next->getName()); + if (l >= len) break; + s += l; + len -= l; + leaves->removeObject(next); + } + } + OSSafeReleaseNULL(leaves); + OSSafeReleaseNULL(set); + OSSafeReleaseNULL(iter); + } + + dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug)); + snprintf(panicString, panicStringLen, + "%s[%d], (%llds): %s", + kextdWait ? "kextd stall" : "busy timeout", + loops, timeout / 1000000000ULL, + string ? string : ""); + IOLog("%s\n", panicString); + if (dopanic) panic("%s", panicString); + else if (!loops) getPMRootDomain()->startSpinDump(1); + } + + if (string) IODelete(string, char, 256); + if (panicString) IODelete(panicString, char, panicStringLen); + + return (ret); +} + IOReturn IOService::waitQuiet( mach_timespec_t * timeout ) { - return( waitForState( kIOServiceBusyStateMask, 0, timeout )); + uint64_t timeoutNS; + + if (timeout) + { + timeoutNS = timeout->tv_sec; + timeoutNS *= kSecondScale; + timeoutNS += timeout->tv_nsec; + } + else + timeoutNS = UINT64_MAX; + + return (waitQuiet(timeoutNS)); } bool IOService::serializeProperties( OSSerialize * s ) const @@ -2942,11 +4083,21 @@ bool IOService::serializeProperties( OSSerialize * s ) const } -void _IOConfigThread::main( _IOConfigThread * self ) +void _IOConfigThread::main(void * arg, wait_result_t result) { - _IOServiceJob * job; - IOService * nub; - bool alive = true; + _IOConfigThread * self = (_IOConfigThread *) arg; + _IOServiceJob * job; + IOService * nub; + bool alive = true; + kern_return_t kr; + thread_precedence_policy_data_t precedence = { -1 }; + + kr = thread_policy_set(current_thread(), + THREAD_PRECEDENCE_POLICY, + (thread_policy_t) &precedence, + THREAD_PRECEDENCE_POLICY_COUNT); + if (KERN_SUCCESS != kr) + IOLog("thread_policy_set(%d)\n", kr); do { @@ -2970,8 +4121,8 @@ void _IOConfigThread::main( _IOConfigThread * self ) nub = job->nub; if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): starting on %s, %d\n", - (int) IOThreadSelf(), job->nub->getName(), job->type); + LOG("config(%p): starting on %s, %d\n", + IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type); switch( job->type) { @@ -2980,8 +4131,8 @@ void _IOConfigThread::main( _IOConfigThread * self ) break; default: - LOG("config(%x): strange type (%d)\n", - (int) IOThreadSelf(), job->type ); + LOG("config(%p): strange type (%d)\n", + IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type ); break; } @@ -3005,7 +4156,7 @@ void _IOConfigThread::main( _IOConfigThread * self ) } while( alive ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): terminating\n", (int) IOThreadSelf() ); + LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) ); self->release(); } @@ -3015,7 +4166,7 @@ IOReturn IOService::waitMatchIdle( UInt32 msToWait ) bool wait; int waitResult = THREAD_AWAKENED; bool computeDeadline = true; - AbsoluteTime abstime; + AbsoluteTime deadline; IOLockLock( gJobsLock ); do { @@ -3023,14 +4174,12 @@ IOReturn IOService::waitMatchIdle( UInt32 msToWait ) if( wait) { if( msToWait) { if( computeDeadline ) { - clock_interval_to_absolutetime_interval( - msToWait, kMillisecondScale, &abstime ); - clock_absolutetime_interval_to_deadline( - abstime, &abstime ); + clock_interval_to_deadline( + msToWait, kMillisecondScale, &deadline ); computeDeadline = false; } waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads, - abstime, THREAD_UNINT ); + deadline, THREAD_UNINT ); } else { waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads, THREAD_UNINT ); @@ -3045,6 +4194,11 @@ IOReturn IOService::waitMatchIdle( UInt32 msToWait ) return( kIOReturnSuccess ); } +void IOService::cpusRunning(void) +{ + gCPUsRunning = true; +} + void _IOServiceJob::pingConfig( _IOServiceJob * job ) { int count; @@ -3061,7 +4215,9 @@ void _IOServiceJob::pingConfig( _IOServiceJob * job ) // if( gNumConfigThreads) count++;// assume we're called from a config thread create = ( (gOutstandingJobs > count) - && (gNumConfigThreads < kMaxConfigThreads) ); + && ((gNumConfigThreads < kMaxConfigThreads) + || (job->nub == gIOResources) + || !gCPUsRunning)); if( create) { gNumConfigThreads++; gNumWaitingThreads++; @@ -3080,44 +4236,200 @@ void _IOServiceJob::pingConfig( _IOServiceJob * job ) semaphore_signal( gJobsSemaphore ); } +struct IOServiceMatchContext +{ + OSDictionary * table; + OSObject * result; + uint32_t options; + uint32_t state; + uint32_t count; + uint32_t done; +}; -// internal - call with gNotificationLock -OSObject * IOService::getExistingServices( OSDictionary * matching, - IOOptionBits inState, IOOptionBits options ) +bool IOService::instanceMatch(const OSObject * entry, void * context) { - OSObject * current = 0; - OSIterator * iter; - IOService * service; + IOServiceMatchContext * ctx = (typeof(ctx)) context; + IOService * service = (typeof(service)) entry; + OSDictionary * table = ctx->table; + uint32_t options = ctx->options; + uint32_t state = ctx->state; + uint32_t done; + bool match; - if( !matching) - return( 0 ); + done = 0; + do + { + match = ((state == (state & service->__state[0])) + && (0 == (service->__state[0] & kIOServiceInactiveState))); + if (!match) break; + ctx->count += table->getCount(); + match = service->matchInternal(table, options, &done); + ctx->done += done; + } + while (false); + if (!match) + return (false); - iter = IORegistryIterator::iterateOver( gIOServicePlane, - kIORegistryIterateRecursively ); - if( iter) { - do { - iter->reset(); - while( (service = (IOService *) iter->getNextObject())) { - if( (inState == (service->__state[0] & inState)) - && (0 == (service->__state[0] & kIOServiceInactiveState)) - && service->passiveMatch( matching )) { - - if( options & kIONotifyOnce) { - current = service; - break; - } - if( current) - ((OSSet *)current)->setObject( service ); - else - current = OSSet::withObjects( + if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) + { + service->retain(); + ctx->result = service; + return (true); + } + else if (!ctx->result) + { + ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1); + } + else + { + ((OSSet *)ctx->result)->setObject(service); + } + return (false); +} + +// internal - call with gNotificationLock +OSObject * IOService::copyExistingServices( OSDictionary * matching, + IOOptionBits inState, IOOptionBits options ) +{ + OSObject * current = 0; + OSIterator * iter; + IOService * service; + OSObject * obj; + OSString * str; + + if( !matching) + return( 0 ); + +#if MATCH_DEBUG + OSSerialize * s = OSSerialize::withCapacity(128); + matching->serialize(s); +#endif + + if((obj = matching->getObject(gIOProviderClassKey)) + && gIOResourcesKey + && gIOResourcesKey->isEqualTo(obj) + && (service = gIOResources)) + { + if( (inState == (service->__state[0] & inState)) + && (0 == (service->__state[0] & kIOServiceInactiveState)) + && service->matchPassive(matching, options)) + { + if( options & kIONotifyOnce) + { + service->retain(); + current = service; + } + else + current = OSSet::withObjects((const OSObject **) &service, 1, 1 ); + } + } + else + { + IOServiceMatchContext ctx; + ctx.table = matching; + ctx.state = inState; + ctx.count = 0; + ctx.done = 0; + ctx.options = options; + ctx.result = 0; + + if ((str = OSDynamicCast(OSString, obj))) + { + const OSSymbol * sym = OSSymbol::withString(str); + OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx); + sym->release(); + } + else + { + IOService::gMetaClass.applyToInstances(instanceMatch, &ctx); + } + + + current = ctx.result; + + options |= kIOServiceInternalDone | kIOServiceClassDone; + if (current && (ctx.done != ctx.count)) + { + OSSet * + source = OSDynamicCast(OSSet, current); + current = 0; + while ((service = (IOService *) source->getAnyObject())) + { + if (service->matchPassive(matching, options)) + { + if( options & kIONotifyOnce) + { + service->retain(); + current = service; + break; + } + if( current) + { + ((OSSet *)current)->setObject( service ); + } + else + { + current = OSSet::withObjects( (const OSObject **) &service, 1, 1 ); - } - } - } while( !service && !iter->isValid()); - iter->release(); + } + } + source->removeObject(service); + } + source->release(); + } } - if( current && (0 == (options & kIONotifyOnce))) { +#if MATCH_DEBUG + { + OSObject * _current = 0; + + iter = IORegistryIterator::iterateOver( gIOServicePlane, + kIORegistryIterateRecursively ); + if( iter) { + do { + iter->reset(); + while( (service = (IOService *) iter->getNextObject())) { + if( (inState == (service->__state[0] & inState)) + && (0 == (service->__state[0] & kIOServiceInactiveState)) + && service->matchPassive(matching, 0)) { + + if( options & kIONotifyOnce) { + service->retain(); + _current = service; + break; + } + if( _current) + ((OSSet *)_current)->setObject( service ); + else + _current = OSSet::withObjects( + (const OSObject **) &service, 1, 1 ); + } + } + } while( !service && !iter->isValid()); + iter->release(); + } + + + if ( ((current != 0) != (_current != 0)) + || (current && _current && !current->isEqualTo(_current))) + { + OSSerialize * s1 = OSSerialize::withCapacity(128); + OSSerialize * s2 = OSSerialize::withCapacity(128); + current->serialize(s1); + _current->serialize(s2); + kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current), + IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text()); + s1->release(); + s2->release(); + } + + if (_current) _current->release(); + } + + s->release(); +#endif + + if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) { iter = OSCollectionIterator::withCollection( (OSSet *)current ); current->release(); current = iter; @@ -3134,7 +4446,7 @@ OSIterator * IOService::getMatchingServices( OSDictionary * matching ) // is a lock even needed? LOCKWRITENOTIFY(); - iter = (OSIterator *) getExistingServices( matching, + iter = (OSIterator *) copyExistingServices( matching, kIOServiceMatchedState ); UNLOCKNOTIFY(); @@ -3142,11 +4454,38 @@ OSIterator * IOService::getMatchingServices( OSDictionary * matching ) return( iter ); } +IOService * IOService::copyMatchingService( OSDictionary * matching ) +{ + IOService * service; + + // is a lock even needed? + LOCKWRITENOTIFY(); + + service = (IOService *) copyExistingServices( matching, + kIOServiceMatchedState, kIONotifyOnce ); + + UNLOCKNOTIFY(); + + return( service ); +} + +struct _IOServiceMatchingNotificationHandlerRef +{ + IOServiceNotificationHandler handler; + void * ref; +}; + +static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon, + IOService * newService, + IONotifier * notifier ) +{ + return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService)); +} // internal - call with gNotificationLock IONotifier * IOService::setNotification( const OSSymbol * type, OSDictionary * matching, - IOServiceNotificationHandler handler, void * target, void * ref, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority ) { _IOServiceNotifier * notify = 0; @@ -3162,10 +4501,18 @@ IONotifier * IOService::setNotification( } if( notify) { - notify->matching = matching; - notify->handler = handler; + notify->handler = handler; notify->target = target; - notify->ref = ref; + notify->type = type; + notify->matching = matching; + matching->retain(); + if (handler == &_IOServiceMatchingNotificationHandler) + { + notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler; + notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref; + } + else + notify->ref = ref; notify->priority = priority; notify->state = kIOServiceNotifyEnable; queue_init( ¬ify->handlerInvocations ); @@ -3191,7 +4538,7 @@ IONotifier * IOService::setNotification( // internal - call with gNotificationLock IONotifier * IOService::doInstallNotification( const OSSymbol * type, OSDictionary * matching, - IOServiceNotificationHandler handler, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority, OSIterator ** existing ) { @@ -3208,10 +4555,13 @@ IONotifier * IOService::doInstallNotification( else if( type == gIOFirstPublishNotification) inState = kIOServiceFirstPublishState; - else if( (type == gIOMatchedNotification) - || (type == gIOFirstMatchNotification)) + else if (type == gIOMatchedNotification) inState = kIOServiceMatchedState; - else if( type == gIOTerminatedNotification) + + else if (type == gIOFirstMatchNotification) + inState = kIOServiceFirstMatchState; + + else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) inState = 0; else return( 0 ); @@ -3220,7 +4570,7 @@ IONotifier * IOService::doInstallNotification( if( inState) // get the current set - exist = (OSIterator *) getExistingServices( matching, inState ); + exist = (OSIterator *) copyExistingServices( matching, inState ); else exist = 0; @@ -3229,20 +4579,44 @@ IONotifier * IOService::doInstallNotification( return( notify ); } +#if !defined(__LP64__) +IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching, + IOServiceNotificationHandler handler, + void * target, void * refCon, + SInt32 priority, OSIterator ** existing ) +{ + IONotifier * result; + _IOServiceMatchingNotificationHandlerRef ref; + ref.handler = handler; + ref.ref = refCon; + + result = (_IOServiceNotifier *) installNotification( type, matching, + &_IOServiceMatchingNotificationHandler, + target, &ref, priority, existing ); + if (result) + matching->release(); + + return (result); +} +#endif /* !defined(__LP64__) */ + IONotifier * IOService::installNotification( const OSSymbol * type, OSDictionary * matching, - IOServiceNotificationHandler handler, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority, OSIterator ** existing ) { - IONotifier * notify; + IONotifier * notify; LOCKWRITENOTIFY(); notify = doInstallNotification( type, matching, handler, target, ref, priority, existing ); + // in case handler remove()s + if (notify) notify->retain(); + UNLOCKNOTIFY(); return( notify ); @@ -3251,108 +4625,158 @@ IONotifier * IOService::installNotification( IONotifier * IOService::addNotification( const OSSymbol * type, OSDictionary * matching, IOServiceNotificationHandler handler, + void * target, void * refCon, + SInt32 priority ) +{ + IONotifier * result; + _IOServiceMatchingNotificationHandlerRef ref; + + ref.handler = handler; + ref.ref = refCon; + + result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler, + target, &ref, priority); + + if (result) + matching->release(); + + return (result); +} + +IONotifier * IOService::addMatchingNotification( + const OSSymbol * type, OSDictionary * matching, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority ) { - OSIterator * existing; + OSIterator * existing = NULL; + IONotifier * ret; _IOServiceNotifier * notify; IOService * next; - notify = (_IOServiceNotifier *) installNotification( type, matching, + ret = notify = (_IOServiceNotifier *) installNotification( type, matching, handler, target, ref, priority, &existing ); + if (!ret) return (0); // send notifications for existing set - if( existing) { - - notify->retain(); // in case handler remove()s - while( (next = (IOService *) existing->getNextObject())) { - - next->lockForArbitration(); + if (existing) + { + while( (next = (IOService *) existing->getNextObject())) + { if( 0 == (next->__state[0] & kIOServiceInactiveState)) - next->invokeNotifer( notify ); - next->unlockForArbitration(); + { + next->invokeNotifier( notify ); + } } - notify->release(); existing->release(); } - return( notify ); -} + LOCKWRITENOTIFY(); + bool removed = (0 == notify->whence); + notify->release(); + if (removed) ret = gIOServiceNullNotifier; + UNLOCKNOTIFY(); -struct SyncNotifyVars { - semaphore_port_t waitHere; - IOService * result; -}; + return( ret ); +} bool IOService::syncNotificationHandler( void * /* target */, void * ref, - IOService * newService ) + IOService * newService, + IONotifier * notifier ) { - // result may get written more than once before the - // notification is removed! - ((SyncNotifyVars *) ref)->result = newService; - semaphore_signal( ((SyncNotifyVars *) ref)->waitHere ); + LOCKWRITENOTIFY(); + if (!*((IOService **) ref)) + { + newService->retain(); + (*(IOService **) ref) = newService; + WAKEUPNOTIFY(ref); + } + UNLOCKNOTIFY(); return( false ); } -IOService * IOService::waitForService( OSDictionary * matching, - mach_timespec_t * timeout ) +IOService * IOService::waitForMatchingService( OSDictionary * matching, + uint64_t timeout) { IONotifier * notify = 0; // priority doesn't help us much since we need a thread wakeup SInt32 priority = 0; - SyncNotifyVars state; - kern_return_t err = kIOReturnBadArgument; + IOService * result; - if( !matching) + if (!matching) return( 0 ); - state.waitHere = 0; - state.result = 0; + result = NULL; LOCKWRITENOTIFY(); - - do { - - state.result = (IOService *) getExistingServices( matching, + do + { + result = (IOService *) copyExistingServices( matching, kIOServiceMatchedState, kIONotifyOnce ); - if( state.result) - continue; - - err = semaphore_create( kernel_task, &state.waitHere, - SYNC_POLICY_FIFO, 0 ); - if( KERN_SUCCESS != err) - continue; - + if (result) + break; notify = IOService::setNotification( gIOMatchedNotification, matching, &IOService::syncNotificationHandler, (void *) 0, - (void *) &state, priority ); - - } while( false ); - - UNLOCKNOTIFY(); - - if( notify) { - if( timeout) - err = semaphore_timedwait( state.waitHere, *timeout ); + &result, priority ); + if (!notify) + break; + if (UINT64_MAX != timeout) + { + AbsoluteTime deadline; + nanoseconds_to_absolutetime(timeout, &deadline); + clock_absolutetime_interval_to_deadline(deadline, &deadline); + SLEEPNOTIFYTO(&result, deadline); + } else - err = semaphore_wait( state.waitHere ); + { + SLEEPNOTIFY(&result); + } } + while( false ); - if( notify) + UNLOCKNOTIFY(); + + if (notify) notify->remove(); // dequeues + + return( result ); +} + +IOService * IOService::waitForService( OSDictionary * matching, + mach_timespec_t * timeout ) +{ + IOService * result; + uint64_t timeoutNS; + + if (timeout) + { + timeoutNS = timeout->tv_sec; + timeoutNS *= kSecondScale; + timeoutNS += timeout->tv_nsec; + } else - matching->release(); - if( state.waitHere) - semaphore_destroy( kernel_task, state.waitHere ); + timeoutNS = UINT64_MAX; + + result = waitForMatchingService(matching, timeoutNS); - return( state.result ); + matching->release(); + if (result) + result->release(); + + return (result); } void IOService::deliverNotification( const OSSymbol * type, IOOptionBits orNewState, IOOptionBits andNewState ) +{ + panic("deliverNotification"); +} + +OSArray * IOService::copyNotifiers(const OSSymbol * type, + IOOptionBits orNewState, IOOptionBits andNewState ) { _IOServiceNotifier * notify; OSIterator * iter; @@ -3361,7 +4785,8 @@ void IOService::deliverNotification( const OSSymbol * type, lockForArbitration(); if( (0 == (__state[0] & kIOServiceInactiveState)) - || (type == gIOTerminatedNotification)) { + || (type == gIOTerminatedNotification) + || (type == gIOWillTerminateNotification)) { LOCKREADNOTIFY(); @@ -3371,7 +4796,7 @@ void IOService::deliverNotification( const OSSymbol * type, if( iter) { while( (notify = (_IOServiceNotifier *) iter->getNextObject())) { - if( passiveMatch( notify->matching) + if( matchPassive(notify->matching, 0) && (kIOServiceNotifyEnable & notify->state)) { if( 0 == willSend) willSend = OSArray::withCapacity(8); @@ -3381,21 +4806,14 @@ void IOService::deliverNotification( const OSSymbol * type, } iter->release(); } - __state[0] = (__state[0] | orNewState) & andNewState; - UNLOCKNOTIFY(); } - if( willSend) { - for( unsigned int idx = 0; - (notify = (_IOServiceNotifier *) willSend->getObject(idx)); - idx++) { - invokeNotifer( notify ); - } - willSend->release(); - } unlockForArbitration(); + + return (willSend); + } IOOptionBits IOService::getState( void ) const @@ -3410,10 +4828,18 @@ IOOptionBits IOService::getState( void ) const OSDictionary * IOService::serviceMatching( const OSString * name, OSDictionary * table ) { + + const OSString * str; + + str = OSSymbol::withString(name); + if( !str) + return( 0 ); + if( !table) table = OSDictionary::withCapacity( 2 ); if( table) - table->setObject(gIOProviderClassKey, (OSObject *)name ); + table->setObject(gIOProviderClassKey, (OSObject *)str ); + str->release(); return( table ); } @@ -3482,6 +4908,47 @@ OSDictionary * IOService::resourceMatching( const char * name, return( table ); } +OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value, + OSDictionary * table ) +{ + OSDictionary * properties; + + properties = OSDictionary::withCapacity( 2 ); + if( !properties) + return( 0 ); + properties->setObject( key, value ); + + if( !table) + table = OSDictionary::withCapacity( 2 ); + if( table) + table->setObject( gIOPropertyMatchKey, properties ); + + properties->release(); + + return( table ); +} + +OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID, + OSDictionary * table ) +{ + OSNumber * num; + + num = OSNumber::withNumber( entryID, 64 ); + if( !num) + return( 0 ); + + if( !table) + table = OSDictionary::withCapacity( 2 ); + if( table) + table->setObject( gIORegistryEntryIDKey, num ); + + if (num) + num->release(); + + return( table ); +} + + /* * _IOServiceNotifier */ @@ -3565,6 +5032,19 @@ void _IOServiceNotifier::enable( bool was ) UNLOCKNOTIFY(); } + +/* + * _IOServiceNullNotifier + */ + +void _IOServiceNullNotifier::taggedRetain(const void *tag) const {} +void _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const {} +void _IOServiceNullNotifier::free() {} +void _IOServiceNullNotifier::wait() {} +void _IOServiceNullNotifier::remove() {} +void _IOServiceNullNotifier::enable(bool was) {} +bool _IOServiceNullNotifier::disable() { return(false); } + /* * IOResources */ @@ -3582,10 +5062,45 @@ IOService * IOResources::resources( void ) return( inst ); } +bool IOResources::init( OSDictionary * dictionary ) +{ + // Do super init first + if ( !IOService::init() ) + return false; + + // Allow PAL layer to publish a value + const char *property_name; + int property_value; + + pal_get_resource_property( &property_name, &property_value ); + + if( property_name ) { + OSNumber *num; + const OSSymbol * sym; + + if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) { + if( (sym = OSSymbol::withCString( property_name)) != 0 ) { + this->setProperty( sym, num ); + sym->release(); + } + num->release(); + } + } + + return true; +} + +IOReturn IOResources::newUserClient(task_t owningTask, void * securityID, + UInt32 type, OSDictionary * properties, + IOUserClient ** handler) +{ + return( kIOReturnUnsupported ); +} + IOWorkLoop * IOResources::getWorkLoop() const { - // If we are the resource root then bringe over to the - // platform to get its workloop + // If we are the resource root + // then use the platform's workloop if (this == (IOResources *) gIOResources) return getPlatform()->getWorkLoop(); else @@ -3598,7 +5113,9 @@ bool IOResources::matchPropertyTable( OSDictionary * table ) OSString * str; OSSet * set; OSIterator * iter; - bool ok = false; + OSObject * obj; + OSArray * keys; + bool ok = true; prop = table->getObject( gIOResourceMatchKey ); str = OSDynamicCast( OSString, prop ); @@ -3615,10 +5132,169 @@ bool IOResources::matchPropertyTable( OSDictionary * table ) if( iter) iter->release(); } + else if ((prop = table->getObject(gIOResourceMatchedKey))) + { + obj = copyProperty(gIOResourceMatchedKey); + keys = OSDynamicCast(OSArray, obj); + ok = false; + if (keys) + { + // assuming OSSymbol + ok = ((-1U) != keys->getNextIndexOfObject(prop, 0)); + } + OSSafeReleaseNULL(obj); + } return( ok ); } +void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1) +{ + IOService::updateConsoleUsers(NULL, 0); +} + +void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage) +{ + IORegistryEntry * regEntry; + OSObject * locked = kOSBooleanFalse; + uint32_t idx; + bool publish; + OSDictionary * user; + static IOMessage sSystemPower; + clock_sec_t now = 0; + clock_usec_t microsecs; + + regEntry = IORegistryEntry::getRegistryRoot(); + + if (!gIOChosenEntry) + gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); + + IOLockLock(gIOConsoleUsersLock); + + if (systemMessage) + { + sSystemPower = systemMessage; +#if HIBERNATION + if (kIOMessageSystemHasPoweredOn == systemMessage) + { + uint32_t lockState = IOHibernateWasScreenLocked(); + switch (lockState) + { + case 0: + break; + case kIOScreenLockLocked: + case kIOScreenLockFileVaultDialog: + gIOConsoleBooterLockState = kOSBooleanTrue; + break; + case kIOScreenLockNoLock: + gIOConsoleBooterLockState = 0; + break; + case kIOScreenLockUnlocked: + default: + gIOConsoleBooterLockState = kOSBooleanFalse; + break; + } + } +#endif /* HIBERNATION */ + } + + if (consoleUsers) + { + OSNumber * num = 0; + bool loginLocked = true; + + gIOConsoleLoggedIn = false; + for (idx = 0; + (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx))); + idx++) + { + gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey)) + && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey))); + + loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey)); + if (!num) + { + num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey)); + } + } +#if HIBERNATION + if (!loginLocked) gIOConsoleBooterLockState = 0; + IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n", + (num != 0), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0), + gIOConsoleLoggedIn, loginLocked); +#endif /* HIBERNATION */ + gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0; + } + + if (!gIOConsoleLoggedIn + || (kIOMessageSystemWillSleep == sSystemPower) + || (kIOMessageSystemPagingOff == sSystemPower)) + { + locked = kOSBooleanTrue; + } +#if HIBERNATION + else if (gIOConsoleBooterLockState) + { + locked = gIOConsoleBooterLockState; + } +#endif /* HIBERNATION */ + else if (gIOConsoleLockTime) + { + clock_get_calendar_microtime(&now, µsecs); + if (gIOConsoleLockTime > now) + { + AbsoluteTime deadline; + clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline); + thread_call_enter_delayed(gIOConsoleLockCallout, deadline); + } + else + { + locked = kOSBooleanTrue; + } + } + + publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey))); + if (publish) + { + regEntry->setProperty(gIOConsoleLockedKey, locked); + if (consoleUsers) + { + regEntry->setProperty(gIOConsoleUsersKey, consoleUsers); + } + OSIncrementAtomic( &gIOConsoleUsersSeed ); + } + +#if HIBERNATION + if (gIOChosenEntry) + { + if (locked == kOSBooleanTrue) gIOScreenLockState = kIOScreenLockLocked; + else if (gIOConsoleLockTime) gIOScreenLockState = kIOScreenLockUnlocked; + else gIOScreenLockState = kIOScreenLockNoLock; + gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState)); + + IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n", + gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != 0), now, systemMessage); + } +#endif /* HIBERNATION */ + + IOLockUnlock(gIOConsoleUsersLock); + + if (publish) + { + publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue ); + + MessageClientsContext context; + + context.service = getServiceRoot(); + context.type = kIOMessageConsoleSecurityChange; + context.argument = (void *) regEntry; + context.argSize = 0; + + applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest, + &messageClientsApplier, &context ); + } +} + IOReturn IOResources::setProperties( OSObject * properties ) { IOReturn err; @@ -3638,15 +5314,17 @@ IOReturn IOResources::setProperties( OSObject * properties ) if( 0 == iter) return( kIOReturnBadArgument); - while( (key = OSDynamicCast(OSSymbol, iter->getNextObject()))) { - - if (gIOConsoleUsersKey == key) + while( (key = OSDynamicCast(OSSymbol, iter->getNextObject()))) + { + if (gIOConsoleUsersKey == key) do { - IORegistryEntry::getRegistryRoot()->setProperty(key, dict->getObject(key)); - OSIncrementAtomic( &gIOConsoleUsersSeed ); - publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue ); - continue; + OSArray * consoleUsers; + consoleUsers = OSDynamicCast(OSArray, dict->getObject(key)); + if (!consoleUsers) + continue; + IOService::updateConsoleUsers(consoleUsers, 0); } + while (false); publishResource( key, dict->getObject(key) ); } @@ -3666,11 +5344,16 @@ bool IOService::compareProperty( OSDictionary * matching, const char * key ) { OSObject * value; + OSObject * prop; bool ok; value = matching->getObject( key ); if( value) - ok = value->isEqualTo( getProperty( key )); + { + prop = copyProperty(key); + ok = value->isEqualTo(prop); + if (prop) prop->release(); + } else ok = true; @@ -3682,11 +5365,16 @@ bool IOService::compareProperty( OSDictionary * matching, const OSString * key ) { OSObject * value; + OSObject * prop; bool ok; value = matching->getObject( key ); if( value) - ok = value->isEqualTo( getProperty( key )); + { + prop = copyProperty(key); + ok = value->isEqualTo(prop); + if (prop) prop->release(); + } else ok = true; @@ -3747,141 +5435,229 @@ IOService * IOService::matchLocation( IOService * /* client */ ) if( parent) parent = parent->matchLocation( this ); - return( parent ); + return( parent ); +} + +bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did) +{ + OSString * matched; + OSObject * obj; + OSString * str; + IORegistryEntry * entry; + OSNumber * num; + bool match = true; + bool changesOK = (0 != (kIOServiceChangesOK & options)); + uint32_t count; + uint32_t done; + + do + { + count = table->getCount(); + done = 0; + + str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey)); + if (str) { + done++; + match = ((kIOServiceClassDone & options) || (0 != metaCast(str))); +#if MATCH_DEBUG + match = (0 != metaCast( str )); + if ((kIOServiceClassDone & options) && !match) panic("classDone"); +#endif + if ((!match) || (done == count)) break; + } + + obj = table->getObject( gIONameMatchKey ); + if( obj) { + done++; + match = compareNames( obj, changesOK ? &matched : 0 ); + if (!match) break; + if( changesOK && matched) { + // leave a hint as to which name matched + table->setObject( gIONameMatchedKey, matched ); + matched->release(); + } + if (done == count) break; + } + + str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey )); + if (str) + { + const OSSymbol * sym; + done++; + match = false; + sym = copyLocation(); + if (sym) { + match = sym->isEqualTo( str ); + sym->release(); + } + if ((!match) || (done == count)) break; + } + + obj = table->getObject( gIOPropertyMatchKey ); + if( obj) + { + OSDictionary * dict; + OSDictionary * nextDict; + OSIterator * iter; + done++; + match = false; + dict = dictionaryWithProperties(); + if( dict) { + nextDict = OSDynamicCast( OSDictionary, obj); + if( nextDict) + iter = 0; + else + iter = OSCollectionIterator::withCollection( + OSDynamicCast(OSCollection, obj)); + + while( nextDict + || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary, + iter->getNextObject()))))) { + match = dict->isEqualTo( nextDict, nextDict); + if( match) + break; + nextDict = 0; + } + dict->release(); + if( iter) + iter->release(); + } + if ((!match) || (done == count)) break; + } + + obj = table->getObject( gIOPropertyExistsMatchKey ); + if( obj) + { + OSDictionary * dict; + OSString * nextKey; + OSIterator * iter; + done++; + match = false; + dict = dictionaryWithProperties(); + if( dict) { + nextKey = OSDynamicCast( OSString, obj); + if( nextKey) + iter = 0; + else + iter = OSCollectionIterator::withCollection( + OSDynamicCast(OSCollection, obj)); + + while( nextKey + || (iter && (0 != (nextKey = OSDynamicCast(OSString, + iter->getNextObject()))))) { + match = (0 != dict->getObject(nextKey)); + if( match) + break; + nextKey = 0; + } + dict->release(); + if( iter) + iter->release(); + } + if ((!match) || (done == count)) break; + } + + str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey )); + if( str) { + done++; + entry = IORegistryEntry::fromPath( str->getCStringNoCopy() ); + match = (this == entry); + if( entry) + entry->release(); + if ((!match) || (done == count)) break; + } + + num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey )); + if (num) { + done++; + match = (getRegistryEntryID() == num->unsigned64BitValue()); + if ((!match) || (done == count)) break; + } + + num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey )); + if( num) + { + OSIterator * iter; + IOService * service = 0; + UInt32 serviceCount = 0; + + done++; + iter = getClientIterator(); + if( iter) { + while( (service = (IOService *) iter->getNextObject())) { + if( kIOServiceInactiveState & service->__state[0]) + continue; + if( 0 == service->getProperty( gIOMatchCategoryKey )) + continue; + ++serviceCount; + } + iter->release(); + } + match = (serviceCount == num->unsigned32BitValue()); + if ((!match) || (done == count)) break; + } + +#define propMatch(key) \ + obj = table->getObject(key); \ + if (obj) \ + { \ + OSObject * prop; \ + done++; \ + prop = copyProperty(key); \ + match = obj->isEqualTo(prop); \ + if (prop) prop->release(); \ + if ((!match) || (done == count)) break; \ + } + propMatch(gIOBSDNameKey) + propMatch(gIOBSDMajorKey) + propMatch(gIOBSDMinorKey) + propMatch(gIOBSDUnitKey) +#undef propMatch + } + while (false); + + if (did) *did = done; + return (match); } bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) +{ + return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0)); +} + +bool IOService::matchPassive(OSDictionary * table, uint32_t options) { IOService * where; - OSString * matched; - OSObject * obj; - OSString * str; - IORegistryEntry * entry; - OSNumber * num; + OSDictionary * nextTable; SInt32 score; OSNumber * newPri; bool match = true; bool matchParent = false; - UInt32 done; + uint32_t count; + uint32_t done; assert( table ); - where = this; - - do { - do { - done = 0; - - str = OSDynamicCast( OSString, table->getObject( gIOProviderClassKey)); - if( str) { - done++; - match = (0 != where->metaCast( str )); - if( !match) - break; - } - - obj = table->getObject( gIONameMatchKey ); - if( obj) { - done++; - match = where->compareNames( obj, changesOK ? &matched : 0 ); - if( !match) - break; - if( changesOK && matched) { - // leave a hint as to which name matched - table->setObject( gIONameMatchedKey, matched ); - matched->release(); - } - } - - str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey )); - if( str) { - - const OSSymbol * sym; - - done++; - match = false; - sym = where->copyLocation(); - if( sym) { - match = sym->isEqualTo( str ); - sym->release(); - } - if( !match) - break; - } - - obj = table->getObject( gIOPropertyMatchKey ); - if( obj) { - - OSDictionary * dict; - OSDictionary * nextDict; - OSIterator * iter; - - done++; - match = false; - dict = where->dictionaryWithProperties(); - if( dict) { - nextDict = OSDynamicCast( OSDictionary, obj); - if( nextDict) - iter = 0; - else - iter = OSCollectionIterator::withCollection( - OSDynamicCast(OSCollection, obj)); - - while( nextDict - || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary, - iter->getNextObject()))))) { - match = dict->isEqualTo( nextDict, nextDict); - if( match) - break; - nextDict = 0; - } - dict->release(); - if( iter) - iter->release(); - } - if( !match) - break; - } - - str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey )); - if( str) { - done++; - entry = IORegistryEntry::fromPath( str->getCStringNoCopy() ); - match = (where == entry); - if( entry) - entry->release(); - if( !match) - break; - } - - num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey )); - if( num) { - - OSIterator * iter; - IOService * service = 0; - UInt32 serviceCount = 0; +#if !CONFIG_EMBEDDED + OSArray* aliasServiceRegIds = NULL; + IOService* foundAlternateService = NULL; +#endif - done++; - iter = where->getClientIterator(); - if( iter) { - while( (service = (IOService *) iter->getNextObject())) { - if( kIOServiceInactiveState & service->__state[0]) - continue; - if( 0 == service->getProperty( gIOMatchCategoryKey )) - continue; - ++serviceCount; - } - iter->release(); - } - match = (serviceCount == num->unsigned32BitValue()); - if( !match) - break; - } +#if MATCH_DEBUG + OSDictionary * root = table; +#endif - if( done == table->getCount()) { - // don't call family if we've done all the entries in the table - matchParent = false; - break; + where = this; + do + { + do + { + count = table->getCount(); + if (!(kIOServiceInternalDone & options)) + { + match = where->matchInternal(table, options, &done); + // don't call family if we've done all the entries in the table + if ((!match) || (done == count)) break; } // pass in score from property table @@ -3898,7 +5674,7 @@ bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) break; } - if( changesOK) { + if (kIOServiceChangesOK & options) { // save the score newPri = OSNumber::withNumber( score, 32 ); if( newPri) { @@ -3907,37 +5683,95 @@ bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) } } - if( !(match = where->compareProperty( table, kIOBSDNameKey ))) - break; - + options = 0; matchParent = false; - obj = OSDynamicCast( OSDictionary, + nextTable = OSDynamicCast(OSDictionary, table->getObject( gIOParentMatchKey )); - if( obj) { + if( nextTable) { + // look for a matching entry anywhere up to root match = false; matchParent = true; - table = (OSDictionary *) obj; + table = nextTable; break; } - table = OSDynamicCast( OSDictionary, + table = OSDynamicCast(OSDictionary, table->getObject( gIOLocationMatchKey )); - if( table) { + if (table) { + // look for a matching entry at matchLocation() match = false; where = where->getProvider(); - if( where) - where = where->matchLocation( where ); + if (where && (where = where->matchLocation(where))) continue; + } + break; + } + while (true); + + if(match == true) { + break; + } + + if(matchParent == true) { +#if !CONFIG_EMBEDDED + // check if service has an alias to search its other "parents" if a parent match isn't found + OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey); + OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop); + if(alternateRegistryID != NULL) { + if(aliasServiceRegIds == NULL) + { + aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID)); + } + aliasServiceRegIds->setObject(alternateRegistryID); } + OSSafeReleaseNULL(prop); +#endif + } + else { + break; + } - } while( table && where ); + where = where->getProvider(); +#if !CONFIG_EMBEDDED + if(where == NULL) { + // there were no matching parent services, check to see if there are aliased services that have a matching parent + if(aliasServiceRegIds != NULL) { + unsigned int numAliasedServices = aliasServiceRegIds->getCount(); + if(numAliasedServices != 0) { + OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1)); + if(alternateRegistryID != NULL) { + OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue()); + aliasServiceRegIds->removeObject(numAliasedServices - 1); + if(alternateMatchingDict != NULL) { + OSSafeReleaseNULL(foundAlternateService); + foundAlternateService = IOService::copyMatchingService(alternateMatchingDict); + alternateMatchingDict->release(); + if(foundAlternateService != NULL) { + where = foundAlternateService; + } + } + } + } + } + } +#endif + } + while( where != NULL ); - } while( matchParent && (where = where->getProvider()) ); +#if !CONFIG_EMBEDDED + OSSafeReleaseNULL(foundAlternateService); + OSSafeReleaseNULL(aliasServiceRegIds); +#endif - if( kIOLogMatch & gIOKitDebug) - if( where != this) - LOG("match parent @ %s = %d\n", - where->getName(), match ); +#if MATCH_DEBUG + if (where != this) + { + OSSerialize * s = OSSerialize::withCapacity(128); + root->serialize(s); + kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text()); + s->release(); + } +#endif return( match ); } @@ -3949,26 +5783,35 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID, { const OSSymbol *userClientClass = 0; IOUserClient *client; + OSObject *prop; OSObject *temp; + if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) + return kIOReturnSuccess; + // First try my own properties for a user client class name - temp = getProperty(gIOUserClientClassKey); - if (temp) { - if (OSDynamicCast(OSSymbol, temp)) - userClientClass = (const OSSymbol *) temp; - else if (OSDynamicCast(OSString, temp)) { - userClientClass = OSSymbol::withString((OSString *) temp); + prop = copyProperty(gIOUserClientClassKey); + if (prop) { + if (OSDynamicCast(OSSymbol, prop)) + userClientClass = (const OSSymbol *) prop; + else if (OSDynamicCast(OSString, prop)) { + userClientClass = OSSymbol::withString((OSString *) prop); if (userClientClass) - setProperty(kIOUserClientClassKey, + setProperty(gIOUserClientClassKey, (OSObject *) userClientClass); } } // Didn't find one so lets just bomb out now without further ado. if (!userClientClass) + { + OSSafeReleaseNULL(prop); return kIOReturnUnsupported; + } + // This reference is consumed by the IOServiceOpen call temp = OSMetaClass::allocClassWithName(userClientClass); + OSSafeReleaseNULL(prop); if (!temp) return kIOReturnNoMemory; @@ -4002,7 +5845,7 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID, IOReturn IOService::newUserClient( task_t owningTask, void * securityID, UInt32 type, IOUserClient ** handler ) { - return( newUserClient( owningTask, securityID, type, 0, handler )); + return( kIOReturnUnsupported ); } IOReturn IOService::requestProbe( IOOptionBits options ) @@ -4080,6 +5923,9 @@ const char * IOService::stringFromReturn( IOReturn rtn ) */ int IOService::errnoFromReturn( IOReturn rtn ) { + if (unix_err(err_get_code(rtn)) == rtn) + return err_get_code(rtn); + switch(rtn) { // (obvious match) case kIOReturnSuccess: @@ -4101,7 +5947,7 @@ int IOService::errnoFromReturn( IOReturn rtn ) case kIOReturnBadArgument: return(EINVAL); case kIOReturnUnsupported: - return(EOPNOTSUPP); + return(ENOTSUP); case kIOReturnBusy: return(EBUSY); case kIOReturnNoPower: @@ -4132,7 +5978,6 @@ int IOService::errnoFromReturn( IOReturn rtn ) case kIOReturnExclusiveAccess: case kIOReturnLockedRead: case kIOReturnLockedWrite: - case kIOReturnNotAttached: case kIOReturnNotOpen: case kIOReturnNotReadable: return(EACCES); @@ -4145,6 +5990,7 @@ int IOService::errnoFromReturn( IOReturn rtn ) return(EBUSY); case kIOReturnBadMedia: case kIOReturnNoMedia: + case kIOReturnNotAttached: case kIOReturnUnformattedMedia: return(ENXIO); // (media error) case kIOReturnDMAError: @@ -4237,6 +6083,206 @@ void IOService::setDeviceMemory( OSArray * array ) setProperty( gIODeviceMemoryKey, array); } +/* + * For machines where the transfers on an I/O bus can stall because + * the CPU is in an idle mode, These APIs allow a driver to specify + * the maximum bus stall that they can handle. 0 indicates no limit. + */ +void IOService:: +setCPUSnoopDelay(UInt32 __unused ns) +{ +#if defined(__i386__) || defined(__x86_64__) + ml_set_maxsnoop(ns); +#endif /* defined(__i386__) || defined(__x86_64__) */ +} + +UInt32 IOService:: +getCPUSnoopDelay() +{ +#if defined(__i386__) || defined(__x86_64__) + return ml_get_maxsnoop(); +#else + return 0; +#endif /* defined(__i386__) || defined(__x86_64__) */ +} + +#if defined(__i386__) || defined(__x86_64__) +static void +requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType) +{ + static const UInt kNoReplace = -1U; // Must be an illegal index + UInt replace = kNoReplace; + bool setCpuDelay = false; + + IORecursiveLockLock(sCpuDelayLock); + + UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry); + CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy(); + IOService * holder = NULL; + + if (ns) { + const CpuDelayEntry ne = {service, ns, delayType}; + holder = service; + // Set maximum delay. + for (UInt i = 0; i < count; i++) { + IOService *thisService = entries[i].fService; + bool sameType = (delayType == entries[i].fDelayType); + if ((service == thisService) && sameType) + replace = i; + else if (!thisService) { + if (kNoReplace == replace) + replace = i; + } + else if (sameType) { + const UInt32 thisMax = entries[i].fMaxDelay; + if (thisMax < ns) + { + ns = thisMax; + holder = thisService; + } + } + } + + setCpuDelay = true; + if (kNoReplace == replace) + sCpuDelayData->appendBytes(&ne, sizeof(ne)); + else + entries[replace] = ne; + } + else { + ns = -1U; // Set to max unsigned, i.e. no restriction + + for (UInt i = 0; i < count; i++) { + // Clear a maximum delay. + IOService *thisService = entries[i].fService; + if (thisService && (delayType == entries[i].fDelayType)) { + UInt32 thisMax = entries[i].fMaxDelay; + if (service == thisService) + replace = i; + else if (thisMax < ns) { + ns = thisMax; + holder = thisService; + } + } + } + + // Check if entry found + if (kNoReplace != replace) { + entries[replace].fService = 0; // Null the entry + setCpuDelay = true; + } + } + + if (setCpuDelay) + { + if (holder && debug_boot_arg) { + strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType])); + } + + // Must be safe to call from locked context + if (delayType == kCpuDelayBusStall) + { + ml_set_maxbusdelay(ns); + } + else if (delayType == kCpuDelayInterrupt) + { + ml_set_maxintdelay(ns); + } + sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0); + sCPULatencySet [delayType]->setValue(ns); + + OSArray * handlers = sCpuLatencyHandlers[delayType]; + IOService * target; + if (handlers) for (unsigned int idx = 0; + (target = (IOService *) handlers->getObject(idx)); + idx++) + { + target->callPlatformFunction(sCPULatencyFunctionName[delayType], false, + (void *) (uintptr_t) ns, holder, + NULL, NULL); + } + } + + IORecursiveLockUnlock(sCpuDelayLock); +} + +static IOReturn +setLatencyHandler(UInt32 delayType, IOService * target, bool enable) +{ + IOReturn result = kIOReturnNotFound; + OSArray * array; + unsigned int idx; + + IORecursiveLockLock(sCpuDelayLock); + + do + { + if (enable && !sCpuLatencyHandlers[delayType]) + sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4); + array = sCpuLatencyHandlers[delayType]; + if (!array) + break; + idx = array->getNextIndexOfObject(target, 0); + if (!enable) + { + if (-1U != idx) + { + array->removeObject(idx); + result = kIOReturnSuccess; + } + } + else + { + if (-1U != idx) { + result = kIOReturnExclusiveAccess; + break; + } + array->setObject(target); + + UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry); + CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy(); + UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction + IOService * holder = NULL; + + for (UInt i = 0; i < count; i++) { + if (entries[i].fService + && (delayType == entries[i].fDelayType) + && (entries[i].fMaxDelay < ns)) { + ns = entries[i].fMaxDelay; + holder = entries[i].fService; + } + } + target->callPlatformFunction(sCPULatencyFunctionName[delayType], false, + (void *) (uintptr_t) ns, holder, + NULL, NULL); + result = kIOReturnSuccess; + } + } + while (false); + + IORecursiveLockUnlock(sCpuDelayLock); + + return (result); +} + +#endif /* defined(__i386__) || defined(__x86_64__) */ + +void IOService:: +requireMaxBusStall(UInt32 __unused ns) +{ +#if defined(__i386__) || defined(__x86_64__) + requireMaxCpuDelay(this, ns, kCpuDelayBusStall); +#endif +} + +void IOService:: +requireMaxInterruptDelay(uint32_t __unused ns) +{ +#if defined(__i386__) || defined(__x86_64__) + requireMaxCpuDelay(this, ns, kCpuDelayInterrupt); +#endif +} + /* * Device interrupts */ @@ -4345,6 +6391,206 @@ IOReturn IOService::unregisterInterrupt(int source) return interruptController->unregisterInterrupt(this, source); } +IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source) +{ + IOReportLegend * legend = NULL; + IOInterruptAccountingData * oldValue = NULL; + IOInterruptAccountingReporter * newArray = NULL; + char subgroupName[64]; + int newArraySize = 0; + int i = 0; + + if (source < 0) { + return kIOReturnBadArgument; + } + + /* + * We support statistics on a maximum of 256 interrupts per nub; if a nub + * has more than 256 interrupt specifiers associated with it, and tries + * to register a high interrupt index with interrupt accounting, panic. + * Having more than 256 interrupts associated with a single nub is + * probably a sign that something fishy is going on. + */ + if (source > IA_INDEX_MAX) { + panic("addInterruptStatistics called for an excessively large index (%d)", source); + } + + /* + * TODO: This is ugly (wrapping a lock around an allocation). I'm only + * leaving it as is because the likelihood of contention where we are + * actually growing the array is minimal (we would realistically need + * to be starting a driver for the first time, with an IOReporting + * client already in place). Nonetheless, cleanup that can be done + * to adhere to best practices; it'll make the code more complicated, + * unfortunately. + */ + IOLockLock(reserved->interruptStatisticsLock); + + /* + * Lazily allocate the statistics array. + */ + if (!reserved->interruptStatisticsArray) { + reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1); + assert(reserved->interruptStatisticsArray); + reserved->interruptStatisticsArrayCount = 1; + bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray)); + } + + if (source >= reserved->interruptStatisticsArrayCount) { + /* + * We're still within the range of supported indices, but we are out + * of space in the current array. Do a nasty realloc (because + * IORealloc isn't a thing) here. We'll double the size with each + * reallocation. + * + * Yes, the "next power of 2" could be more efficient; but this will + * be invoked incredibly rarely. Who cares. + */ + newArraySize = (reserved->interruptStatisticsArrayCount << 1); + + while (newArraySize <= source) + newArraySize = (newArraySize << 1); + newArray = IONew(IOInterruptAccountingReporter, newArraySize); + + assert(newArray); + + /* + * TODO: This even zeroes the memory it is about to overwrite. + * Shameful; fix it. Not particularly high impact, however. + */ + bzero(newArray, newArraySize * sizeof(*newArray)); + memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray)); + IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount); + reserved->interruptStatisticsArray = newArray; + reserved->interruptStatisticsArrayCount = newArraySize; + } + + if (!reserved->interruptStatisticsArray[source].reporter) { + /* + * We don't have a reporter associated with this index yet, so we + * need to create one. + */ + /* + * TODO: Some statistics do in fact have common units (time); should this be + * split into separate reporters to communicate this? + */ + reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone); + + /* + * Each statistic is given an identifier based on the interrupt index (which + * should be unique relative to any single nub) and the statistic involved. + * We should now have a sane (small and positive) index, so start + * constructing the channels for statistics. + */ + for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) { + /* + * TODO: Currently, this does not add channels for disabled statistics. + * Will this be confusing for clients? If so, we should just add the + * channels; we can avoid updating the channels even if they exist. + */ + if (IA_GET_STATISTIC_ENABLED(i)) + reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]); + } + + /* + * We now need to add the legend for this reporter to the registry. + */ + OSObject * prop = copyProperty(kIOReportLegendKey); + legend = IOReportLegend::with(OSDynamicCast(OSArray, prop)); + OSSafeReleaseNULL(prop); + + /* + * Note that while we compose the subgroup name, we do not need to + * manage its lifecycle (the reporter will handle this). + */ + snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source); + subgroupName[sizeof(subgroupName) - 1] = 0; + legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName); + setProperty(kIOReportLegendKey, legend->getLegend()); + legend->release(); + + /* + * TODO: Is this a good idea? Probably not; my assumption is it opts + * all entities who register interrupts into public disclosure of all + * IOReporting channels. Unfortunately, this appears to be as fine + * grain as it gets. + */ + setProperty(kIOReportLegendPublicKey, true); + } + + /* + * Don't stomp existing entries. If we are about to, panic; this + * probably means we failed to tear down our old interrupt source + * correctly. + */ + oldValue = reserved->interruptStatisticsArray[source].statistics; + + if (oldValue) { + panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source); + } + + reserved->interruptStatisticsArray[source].statistics = statistics; + + /* + * Inherit the reporter values for each statistic. The target may + * be torn down as part of the runtime of the service (especially + * for sleep/wake), so we inherit in order to avoid having values + * reset for no apparent reason. Our statistics are ultimately + * tied to the index and the sevice, not to an individual target, + * so we should maintain them accordingly. + */ + interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter); + + IOLockUnlock(reserved->interruptStatisticsLock); + + return kIOReturnSuccess; +} + +IOReturn IOService::removeInterruptStatistics(int source) +{ + IOInterruptAccountingData * value = NULL; + + if (source < 0) { + return kIOReturnBadArgument; + } + + IOLockLock(reserved->interruptStatisticsLock); + + /* + * We dynamically grow the statistics array, so an excessively + * large index value has NEVER been registered. This either + * means our cap on the array size is too small (unlikely), or + * that we have been passed a corrupt index (this must be passed + * the plain index into the interrupt specifier list). + */ + if (source >= reserved->interruptStatisticsArrayCount) { + panic("removeInterruptStatistics called for index %d, which was never registered", source); + } + + assert(reserved->interruptStatisticsArray); + + /* + * If there is no existing entry, we are most likely trying to + * free an interrupt owner twice, or we have corrupted the + * index value. + */ + value = reserved->interruptStatisticsArray[source].statistics; + + if (!value) { + panic("removeInterruptStatistics called for empty index %d", source); + } + + /* + * We update the statistics, so that any delta with the reporter + * state is not lost. + */ + interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter); + reserved->interruptStatisticsArray[source].statistics = NULL; + IOLockUnlock(reserved->interruptStatisticsLock); + + return kIOReturnSuccess; +} + IOReturn IOService::getInterruptType(int source, int *interruptType) { IOInterruptController *interruptController; @@ -4393,15 +6639,138 @@ IOReturn IOService::causeInterrupt(int source) return interruptController->causeInterrupt(this, source); } +IOReturn IOService::configureReport(IOReportChannelList *channelList, + IOReportConfigureAction action, + void *result, + void *destination) +{ + unsigned cnt; + + for (cnt = 0; cnt < channelList->nchannels; cnt++) { + if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) { + if (pwrMgt) configurePowerStatesReport(action, result); + else return kIOReturnUnsupported; + } + else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) { + if (pwrMgt) configureSimplePowerReport(action, result); + else return kIOReturnUnsupported; + } + } + + IOLockLock(reserved->interruptStatisticsLock); + + /* The array count is signed (because the interrupt indices are signed), hence the cast */ + for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) { + if (reserved->interruptStatisticsArray[cnt].reporter) { + /* + * If the reporter is currently associated with the statistics + * for an event source, we may need to update the reporter. + */ + if (reserved->interruptStatisticsArray[cnt].statistics) + interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter); + + reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination); + } + } + + IOLockUnlock(reserved->interruptStatisticsLock); + + return kIOReturnSuccess; +} + +IOReturn IOService::updateReport(IOReportChannelList *channelList, + IOReportUpdateAction action, + void *result, + void *destination) +{ + unsigned cnt; + + for (cnt = 0; cnt < channelList->nchannels; cnt++) { + if ( channelList->channels[cnt].channel_id == kPMPowerStatesChID ) { + if (pwrMgt) updatePowerStatesReport(action, result, destination); + else return kIOReturnUnsupported; + } + else if ( channelList->channels[cnt].channel_id == kPMCurrStateChID ) { + if (pwrMgt) updateSimplePowerReport(action, result, destination); + else return kIOReturnUnsupported; + } + } + + IOLockLock(reserved->interruptStatisticsLock); + + /* The array count is signed (because the interrupt indices are signed), hence the cast */ + for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) { + if (reserved->interruptStatisticsArray[cnt].reporter) { + /* + * If the reporter is currently associated with the statistics + * for an event source, we need to update the reporter. + */ + if (reserved->interruptStatisticsArray[cnt].statistics) + interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter); + + reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination); + } + } + + IOLockUnlock(reserved->interruptStatisticsLock); + + return kIOReturnSuccess; +} + +uint64_t IOService::getAuthorizationID( void ) +{ + return reserved->authorizationID; +} + +IOReturn IOService::setAuthorizationID( uint64_t authorizationID ) +{ + OSObject * entitlement; + IOReturn status; + + entitlement = IOUserClient::copyClientEntitlement( current_task( ), "com.apple.private.iokit.IOServiceSetAuthorizationID" ); + + if ( entitlement ) + { + if ( entitlement == kOSBooleanTrue ) + { + reserved->authorizationID = authorizationID; + + status = kIOReturnSuccess; + } + else + { + status = kIOReturnNotPrivileged; + } + + entitlement->release( ); + } + else + { + status = kIOReturnNotPrivileged; + } + + return status; +} + +#if __LP64__ OSMetaClassDefineReservedUsed(IOService, 0); OSMetaClassDefineReservedUsed(IOService, 1); -OSMetaClassDefineReservedUsed(IOService, 2); - +OSMetaClassDefineReservedUnused(IOService, 2); OSMetaClassDefineReservedUnused(IOService, 3); OSMetaClassDefineReservedUnused(IOService, 4); OSMetaClassDefineReservedUnused(IOService, 5); OSMetaClassDefineReservedUnused(IOService, 6); OSMetaClassDefineReservedUnused(IOService, 7); +#else +OSMetaClassDefineReservedUsed(IOService, 0); +OSMetaClassDefineReservedUsed(IOService, 1); +OSMetaClassDefineReservedUsed(IOService, 2); +OSMetaClassDefineReservedUsed(IOService, 3); +OSMetaClassDefineReservedUsed(IOService, 4); +OSMetaClassDefineReservedUsed(IOService, 5); +OSMetaClassDefineReservedUsed(IOService, 6); +OSMetaClassDefineReservedUsed(IOService, 7); +#endif OSMetaClassDefineReservedUnused(IOService, 8); OSMetaClassDefineReservedUnused(IOService, 9); OSMetaClassDefineReservedUnused(IOService, 10); @@ -4442,19 +6811,3 @@ OSMetaClassDefineReservedUnused(IOService, 44); OSMetaClassDefineReservedUnused(IOService, 45); OSMetaClassDefineReservedUnused(IOService, 46); OSMetaClassDefineReservedUnused(IOService, 47); -OSMetaClassDefineReservedUnused(IOService, 48); -OSMetaClassDefineReservedUnused(IOService, 49); -OSMetaClassDefineReservedUnused(IOService, 50); -OSMetaClassDefineReservedUnused(IOService, 51); -OSMetaClassDefineReservedUnused(IOService, 52); -OSMetaClassDefineReservedUnused(IOService, 53); -OSMetaClassDefineReservedUnused(IOService, 54); -OSMetaClassDefineReservedUnused(IOService, 55); -OSMetaClassDefineReservedUnused(IOService, 56); -OSMetaClassDefineReservedUnused(IOService, 57); -OSMetaClassDefineReservedUnused(IOService, 58); -OSMetaClassDefineReservedUnused(IOService, 59); -OSMetaClassDefineReservedUnused(IOService, 60); -OSMetaClassDefineReservedUnused(IOService, 61); -OSMetaClassDefineReservedUnused(IOService, 62); -OSMetaClassDefineReservedUnused(IOService, 63);