X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/c0fea4742e91338fffdcf79f86a7c1d5e2b97eb1..99c3a10404e5d1ef94397ab4df5a8b74711fc4d3:/iokit/Kernel/IOService.cpp diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp index 6fbc9e75e..0eabf83f9 100644 --- a/iokit/Kernel/IOService.cpp +++ b/iokit/Kernel/IOService.cpp @@ -1,32 +1,41 @@ /* - * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1998-2009 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include +#include #include +#include #include #include #include +#include #include #include #include @@ -37,14 +46,20 @@ #include #include #include +#include +#include #include #include #include -//#define LOG kprintf -#define LOG IOLog +#include + +#define LOG kprintf +//#define LOG IOLog +#define MATCH_DEBUG 0 #include "IOServicePrivate.h" +#include "IOKitKernelInternal.h" // take lockForArbitration before LOCKNOTIFY @@ -91,14 +106,26 @@ const OSSymbol * gIOMatchCategoryKey; const OSSymbol * gIODefaultMatchCategoryKey; const OSSymbol * gIOMatchedServiceCountKey; +const OSSymbol * gIOMapperIDKey; const OSSymbol * gIOUserClientClassKey; const OSSymbol * gIOKitDebugKey; const OSSymbol * gIOCommandPoolSizeKey; +const OSSymbol * gIOConsoleLockedKey; const OSSymbol * gIOConsoleUsersKey; const OSSymbol * gIOConsoleSessionUIDKey; +const OSSymbol * gIOConsoleSessionAuditIDKey; const OSSymbol * gIOConsoleUsersSeedKey; +const OSSymbol * gIOConsoleSessionOnConsoleKey; +const OSSymbol * gIOConsoleSessionLoginDoneKey; +const OSSymbol * gIOConsoleSessionSecureInputPIDKey; +const OSSymbol * gIOConsoleSessionScreenLockedTimeKey; + +clock_sec_t gIOConsoleLockTime; +static bool gIOConsoleLoggedIn; +static uint32_t gIOScreenLockState; +static IORegistryEntry * gIOChosenEntry; static int gIOResourceGenerationCount; @@ -113,6 +140,7 @@ const OSSymbol * gIOGeneralInterest; const OSSymbol * gIOBusyInterest; const OSSymbol * gIOAppPowerStateInterest; const OSSymbol * gIOPriorityPowerStateInterest; +const OSSymbol * gIOConsoleSecurityInterest; static OSDictionary * gNotifications; static IORecursiveLock * gNotificationLock; @@ -138,6 +166,18 @@ static OSArray * gIOFinalizeList; static SInt32 gIOConsoleUsersSeed; static OSData * gIOConsoleUsersSeedValue; +extern const OSSymbol * gIODTPHandleKey; + +const OSSymbol * gIOPlatformSleepActionKey; +const OSSymbol * gIOPlatformWakeActionKey; +const OSSymbol * gIOPlatformQuiesceActionKey; +const OSSymbol * gIOPlatformActiveActionKey; + +const OSSymbol * gIOPlatformFunctionHandlerSet; + +static IOLock * gIOConsoleUsersLock; +static thread_call_t gIOConsoleLockCallout; + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define LOCKREADNOTIFY() \ @@ -149,6 +189,8 @@ static OSData * gIOConsoleUsersSeedValue; IORecursiveLockUnlock( gNotificationLock ) #define SLEEPNOTIFY(event) \ IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT ) +#define SLEEPNOTIFYTO(event, deadline) \ + IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT ) #define WAKEUPNOTIFY(event) \ IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false ) @@ -191,19 +233,34 @@ bool IOService::isInactive( void ) const /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) // Only used by the intel implementation of -// IOService::requireMaxBusStall(UInt32 __unused ns) -struct BusStallEntry +// IOService::requireMaxBusStall(UInt32 ns) +// IOService::requireMaxInterruptDelay(uint32_t ns) +struct CpuDelayEntry { - const IOService *fService; - UInt32 fMaxDelay; + IOService * fService; + UInt32 fMaxDelay; + UInt32 fDelayType; +}; + +enum { + kCpuDelayBusStall, kCpuDelayInterrupt, + kCpuNumDelayTypes }; -static OSData *sBusStall = OSData::withCapacity(8 * sizeof(BusStallEntry)); -static IOLock *sBusStallLock = IOLockAlloc(); -#endif /* __i386__ */ +static OSData *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry)); +static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc(); +static OSArray *sCpuLatencyHandlers[kCpuNumDelayTypes]; +const OSSymbol *sCPULatencyFunctionName[kCpuNumDelayTypes]; + +static void +requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType); +static IOReturn +setLatencyHandler(UInt32 delayType, IOService * target, bool enable); + +#endif /* defined(__i386__) || defined(__x86_64__) */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -239,6 +296,8 @@ void IOService::initialize( void ) gIOInterruptSpecifiersKey = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers"); + gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey); + gIOKitDebugKey = OSSymbol::withCStringNoCopy( kIOKitDebugKey ); gIOCommandPoolSizeKey = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey ); @@ -247,6 +306,7 @@ void IOService::initialize( void ) gIOBusyInterest = OSSymbol::withCStringNoCopy( kIOBusyInterest ); gIOAppPowerStateInterest = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest ); gIOPriorityPowerStateInterest = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest ); + gIOConsoleSecurityInterest = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest ); gNotifications = OSDictionary::withCapacity( 1 ); gIOPublishNotification = OSSymbol::withCStringNoCopy( @@ -261,11 +321,29 @@ void IOService::initialize( void ) kIOTerminatedNotification ); gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass); + gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey); gIOConsoleUsersKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey); gIOConsoleSessionUIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey); - gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersSeedKey); - gIOConsoleUsersSeedValue = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed)); + gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey); + + gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey); + gIOConsoleSessionOnConsoleKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey); + gIOConsoleSessionLoginDoneKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey); + gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey); + gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey); + gIOConsoleUsersSeedValue = OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed)); + + gIOPlatformSleepActionKey = OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey); + gIOPlatformWakeActionKey = OSSymbol::withCStringNoCopy(kIOPlatformWakeActionKey); + gIOPlatformQuiesceActionKey = OSSymbol::withCStringNoCopy(kIOPlatformQuiesceActionKey); + gIOPlatformActiveActionKey = OSSymbol::withCStringNoCopy(kIOPlatformActiveActionKey); + + gIOPlatformFunctionHandlerSet = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet); +#if defined(__i386__) || defined(__x86_64__) + sCPULatencyFunctionName[kCpuDelayBusStall] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay); + sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay); +#endif gNotificationLock = IORecursiveLockAlloc(); assert( gIOServicePlane && gIODeviceMemoryKey @@ -276,6 +354,7 @@ void IOService::initialize( void ) && gIOPublishNotification && gIOMatchedNotification && gIOTerminatedNotification && gIOServiceKey && gIOConsoleUsersKey && gIOConsoleSessionUIDKey + && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue); gJobsLock = IOLockAlloc(); @@ -283,9 +362,16 @@ void IOService::initialize( void ) gIOServiceBusyLock = IOLockAlloc(); + gIOConsoleUsersLock = IOLockAlloc(); + err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0); - assert( gIOServiceBusyLock && gJobs && gJobsLock && (err == KERN_SUCCESS) ); + gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL); + + IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue); + + assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock + && gIOConsoleLockCallout && (err == KERN_SUCCESS) ); gIOResources = IOResources::resources(); assert( gIOResources ); @@ -347,6 +433,7 @@ void IOService::stop( IOService * provider ) void IOService::free( void ) { requireMaxBusStall(0); + requireMaxInterruptDelay(0); if( getPropertyTable()) unregisterAllInterest(); PMfree(); @@ -378,6 +465,8 @@ bool IOService::attach( IOService * provider ) ok = attachToParent( getRegistryRoot(), gIOServicePlane); } + if (ok && !__provider) (void) getProvider(); + return( ok ); } @@ -408,6 +497,10 @@ void IOService::detach( IOService * provider ) _adjustBusy( -busy ); } + if (kIOServiceInactiveState & __state[0]) { + getMetaClass()->removeInstance(this); + } + unlockForArbitration(); if( newProvider) { @@ -515,7 +608,6 @@ void IOService::startMatching( IOOptionBits options ) // OSKernelStackRemaining(), getName()); if( needConfig) { - prevBusy = _adjustBusy( 1 ); needWake = (0 != (kIOServiceSyncPubState & __state[1])); } @@ -528,6 +620,8 @@ void IOService::startMatching( IOOptionBits options ) if( needConfig) { + prevBusy = _adjustBusy( 1 ); + if( needWake) { IOLockLock( gIOServiceBusyLock ); thread_wakeup( (event_t) this/*&__state[1]*/ ); @@ -545,7 +639,9 @@ void IOService::startMatching( IOOptionBits options ) lockForArbitration(); IOLockLock( gIOServiceBusyLock ); - waitAgain = (prevBusy < (__state[1] & kIOServiceBusyStateMask)); + waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask)) + && (0 == (__state[0] & kIOServiceInactiveState))); + if( waitAgain) __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState; else @@ -578,37 +674,37 @@ IOReturn IOService::catalogNewDrivers( OSOrderedSet * newTables ) while( (table = (OSDictionary *) newTables->getFirstObject())) { - LOCKWRITENOTIFY(); - set = (OSSet *) getExistingServices( table, + LOCKWRITENOTIFY(); + set = (OSSet *) copyExistingServices( table, kIOServiceRegisteredState, kIOServiceExistingSet); - UNLOCKNOTIFY(); - if( set) { + UNLOCKNOTIFY(); + if( set) { #if IOMATCHDEBUG - count += set->getCount(); + count += set->getCount(); #endif - if (allSet) { - allSet->merge((const OSSet *) set); - set->release(); - } - else - allSet = set; - } + if (allSet) { + allSet->merge((const OSSet *) set); + set->release(); + } + else + allSet = set; + } #if IOMATCHDEBUG - if( getDebugFlags( table ) & kIOLogMatch) - LOG("Matching service count = %ld\n", count); + if( getDebugFlags( table ) & kIOLogMatch) + LOG("Matching service count = %ld\n", (long)count); #endif - newTables->removeObject(table); + newTables->removeObject(table); } if (allSet) { - while( (service = (IOService *) allSet->getAnyObject())) { - service->startMatching(kIOServiceAsynchronous); - allSet->removeObject(service); - } - allSet->release(); + while( (service = (IOService *) allSet->getAnyObject())) { + service->startMatching(kIOServiceAsynchronous); + allSet->removeObject(service); + } + allSet->release(); } newTables->release(); @@ -673,10 +769,9 @@ IOService * IOService::getProvider( void ) const IOService * parent; SInt32 generation; - parent = __provider; generation = getGenerationCount(); if( __providerGeneration == generation) - return( parent ); + return( __provider ); parent = (IOService *) getParentEntry( gIOServicePlane); if( parent == IORegistryEntry::getRegistryRoot()) @@ -684,7 +779,8 @@ IOService * IOService::getProvider( void ) const parent = 0; self->__provider = parent; - // save the count before getParentEntry() + OSMemoryBarrier(); + // save the count from before call to getParentEntry() self->__providerGeneration = generation; return( parent ); @@ -799,9 +895,23 @@ IOReturn IOService::callPlatformFunction( const OSSymbol * functionName, void *param3, void *param4 ) { IOReturn result = kIOReturnUnsupported; - IOService *provider = getProvider(); - - if (provider != 0) { + IOService *provider; + + if (gIOPlatformFunctionHandlerSet == functionName) + { +#if defined(__i386__) || defined(__x86_64__) + const OSSymbol * functionHandlerName = (const OSSymbol *) param1; + IOService * target = (IOService *) param2; + bool enable = (param3 != 0); + + if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) + result = setLatencyHandler(kCpuDelayBusStall, target, enable); + else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) + result = setLatencyHandler(kCpuDelayInterrupt, target, enable); +#endif /* defined(__i386__) || defined(__x86_64__) */ + } + + if ((kIOReturnUnsupported == result) && (provider = getProvider())) { result = provider->callPlatformFunction(functionName, waitForFunction, param1, param2, param3, param4); } @@ -1337,7 +1447,8 @@ void IOService::applyToInterested( const OSSymbol * typeOfInterest, OSObjectApplierFunction applier, void * context ) { - applyToClients( (IOServiceApplierFunction) applier, context ); + if (gIOGeneralInterest == typeOfInterest) + applyToClients( (IOServiceApplierFunction) applier, context ); applyToInterestNotifiers(this, typeOfInterest, applier, context); } @@ -1392,6 +1503,7 @@ IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest, if( (typeOfInterest != gIOGeneralInterest) && (typeOfInterest != gIOBusyInterest) && (typeOfInterest != gIOAppPowerStateInterest) + && (typeOfInterest != gIOConsoleSecurityInterest) && (typeOfInterest != gIOPriorityPowerStateInterest)) return( 0 ); @@ -1463,6 +1575,7 @@ void IOService::unregisterAllInterest( void ) cleanInterestList( getProperty( gIOBusyInterest )); cleanInterestList( getProperty( gIOAppPowerStateInterest )); cleanInterestList( getProperty( gIOPriorityPowerStateInterest )); + cleanInterestList( getProperty( gIOConsoleSecurityInterest )); } /* @@ -1505,7 +1618,7 @@ void _IOServiceInterestNotifier::remove() LOCKWRITENOTIFY(); if( queue_next( &chain )) { - remqueue( 0, &chain); + remqueue(&chain); queue_next( &chain) = queue_prev( &chain) = 0; release(); } @@ -1553,9 +1666,9 @@ void _IOServiceInterestNotifier::enable( bool was ) #define tailQ(o) setObject(o) #define headQ(o) setObject(0, o) -#define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) IOLog(fmt, ## args); } +#define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }} -inline void _workLoopAction( IOWorkLoop::Action action, +static void _workLoopAction( IOWorkLoop::Action action, IOService * service, void * p0 = 0, void * p1 = 0, void * p2 = 0, void * p3 = 0 ) @@ -1589,15 +1702,25 @@ bool IOService::requestTerminate( IOService * provider, IOOptionBits options ) bool IOService::terminatePhase1( IOOptionBits options ) { - IOService * victim; - IOService * client; - OSIterator * iter; - OSArray * makeInactive; - bool ok; - bool didInactive; - bool startPhase2 = false; - - TLOG("%s::terminatePhase1(%08lx)\n", getName(), options); + IOService * victim; + IOService * client; + OSIterator * iter; + OSArray * makeInactive; + int waitResult = THREAD_AWAKENED; + bool wait; + bool ok; + bool didInactive; + bool startPhase2 = false; + + TLOG("%s::terminatePhase1(%08llx)\n", getName(), (long long)options); + + uint64_t regID = getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_PHASE1, + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) this, + (uintptr_t) options); // -- compat if( options & kIOServiceRecursing) { @@ -1615,16 +1738,38 @@ bool IOService::terminatePhase1( IOOptionBits options ) while( victim ) { - didInactive = victim->lockForArbitration( true ); + didInactive = victim->lockForArbitration( true ); if( didInactive) { didInactive = (0 == (victim->__state[0] & kIOServiceInactiveState)); if( didInactive) { victim->__state[0] |= kIOServiceInactiveState; victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState | kIOServiceFirstPublishState | kIOServiceFirstMatchState); + + if (victim == this) + victim->__state[1] |= kIOServiceTermPhase1State; + victim->_adjustBusy( 1 ); - } - victim->unlockForArbitration(); + + } else if (victim != this) do { + + IOLockLock(gIOServiceBusyLock); + wait = (victim->__state[1] & kIOServiceTermPhase1State); + if( wait) { + TLOG("%s::waitPhase1(%s)\n", getName(), victim->getName()); + victim->__state[1] |= kIOServiceTerm1WaiterState; + victim->unlockForArbitration(); + assert_wait((event_t)&victim->__state[1], THREAD_UNINT); + } + IOLockUnlock(gIOServiceBusyLock); + if( wait) { + waitResult = thread_block(THREAD_CONTINUE_NULL); + TLOG("%s::did waitPhase1(%s)\n", getName(), victim->getName()); + victim->lockForArbitration(); + } + } while( wait && (waitResult != THREAD_TIMED_OUT)); + + victim->unlockForArbitration(); } if( victim == this) startPhase2 = didInactive; @@ -1636,11 +1781,22 @@ bool IOService::terminatePhase1( IOOptionBits options ) iter = victim->getClientIterator(); if( iter) { while( (client = (IOService *) iter->getNextObject())) { - TLOG("%s::requestTerminate(%s, %08lx)\n", - client->getName(), victim->getName(), options); + TLOG("%s::requestTerminate(%s, %08llx)\n", + client->getName(), victim->getName(), (long long)options); ok = client->requestTerminate( victim, options ); TLOG("%s::requestTerminate(%s, ok = %d)\n", client->getName(), victim->getName(), ok); + + uint64_t regID1 = client->getRegistryEntryID(); + uint64_t regID2 = victim->getRegistryEntryID(); + IOServiceTrace( + (ok ? IOSERVICE_TERMINATE_REQUEST_OK + : IOSERVICE_TERMINATE_REQUEST_FAIL), + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + if( ok) makeInactive->setObject( client ); } @@ -1658,8 +1814,21 @@ bool IOService::terminatePhase1( IOOptionBits options ) makeInactive->release(); if( startPhase2) - scheduleTerminatePhase2( options ); + { + lockForArbitration(); + __state[1] &= ~kIOServiceTermPhase1State; + if (kIOServiceTerm1WaiterState & __state[1]) + { + __state[1] &= ~kIOServiceTerm1WaiterState; + TLOG("%s::wakePhase1\n", getName()); + IOLockLock( gIOServiceBusyLock ); + thread_wakeup( (event_t) &__state[1]); + IOLockUnlock( gIOServiceBusyLock ); + } + unlockForArbitration(); + scheduleTerminatePhase2( options ); + } return( true ); } @@ -1703,8 +1872,8 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork, deadline, THREAD_UNINT ); if( waitResult == THREAD_TIMED_OUT) { - TLOG("%s::terminate(kIOServiceSynchronous) timeout", getName()); - } + IOLog("%s::terminate(kIOServiceSynchronous) timeout\n", getName()); + } } } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT))); @@ -1717,7 +1886,7 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) gIOTerminatePhase2List->setObject( this ); if( 0 == gIOTerminateWork++) { if( !gIOTerminateThread) - gIOTerminateThread = IOCreateThread( &terminateThread, (void *) options ); + kernel_thread_start(&terminateThread, (void *) options, &gIOTerminateThread); else IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); } @@ -1728,13 +1897,14 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) release(); } -void IOService::terminateThread( void * arg ) +void IOService::terminateThread( void * arg, wait_result_t waitResult ) { IOLockLock( gJobsLock ); while (gIOTerminateWork) - terminateWorker( (IOOptionBits) arg ); + terminateWorker( (uintptr_t) arg ); + thread_deallocate(gIOTerminateThread); gIOTerminateThread = 0; IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false); @@ -1745,13 +1915,22 @@ void IOService::scheduleStop( IOService * provider ) { TLOG("%s::scheduleStop(%s)\n", getName(), provider->getName()); + uint64_t regID1 = getRegistryEntryID(); + uint64_t regID2 = provider->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_SCHEDULE_STOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + IOLockLock( gJobsLock ); gIOStopList->tailQ( this ); gIOStopProviderList->tailQ( provider ); if( 0 == gIOTerminateWork++) { if( !gIOTerminateThread) - gIOTerminateThread = IOCreateThread( &terminateThread, (void *) 0 ); + kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread); else IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); } @@ -1763,12 +1942,19 @@ void IOService::scheduleFinalize( void ) { TLOG("%s::scheduleFinalize\n", getName()); + uint64_t regID1 = getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_SCHEDULE_FINALIZE, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + 0, 0); + IOLockLock( gJobsLock ); gIOFinalizeList->tailQ( this ); if( 0 == gIOTerminateWork++) { if( !gIOTerminateThread) - gIOTerminateThread = IOCreateThread( &terminateThread, (void *) 0 ); + kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread); else IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); } @@ -1803,7 +1989,9 @@ bool IOService::didTerminate( IOService * provider, IOOptionBits options, bool * } void IOService::actionWillTerminate( IOService * victim, IOOptionBits options, - OSArray * doPhase2List ) + OSArray * doPhase2List, + void *unused2 __unused, + void *unused3 __unused ) { OSIterator * iter; IOService * client; @@ -1812,8 +2000,18 @@ void IOService::actionWillTerminate( IOService * victim, IOOptionBits options, iter = victim->getClientIterator(); if( iter) { while( (client = (IOService *) iter->getNextObject())) { - TLOG("%s::willTerminate(%s, %08lx)\n", - client->getName(), victim->getName(), options); + TLOG("%s::willTerminate(%s, %08llx)\n", + client->getName(), victim->getName(), (long long)options); + + uint64_t regID1 = client->getRegistryEntryID(); + uint64_t regID2 = victim->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_WILL, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + ok = client->willTerminate( victim, options ); doPhase2List->tailQ( client ); } @@ -1821,7 +2019,9 @@ void IOService::actionWillTerminate( IOService * victim, IOOptionBits options, } } -void IOService::actionDidTerminate( IOService * victim, IOOptionBits options ) +void IOService::actionDidTerminate( IOService * victim, IOOptionBits options, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) { OSIterator * iter; IOService * client; @@ -1832,9 +2032,20 @@ void IOService::actionDidTerminate( IOService * victim, IOOptionBits options ) iter = victim->getClientIterator(); if( iter) { while( (client = (IOService *) iter->getNextObject())) { - TLOG("%s::didTerminate(%s, %08lx)\n", - client->getName(), victim->getName(), options); + TLOG("%s::didTerminate(%s, %08llx)\n", + client->getName(), victim->getName(), (long long)options); client->didTerminate( victim, options, &defer ); + + uint64_t regID1 = client->getRegistryEntryID(); + uint64_t regID2 = victim->getRegistryEntryID(); + IOServiceTrace( + (defer ? IOSERVICE_TERMINATE_DID_DEFER + : IOSERVICE_TERMINATE_DID), + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + TLOG("%s::didTerminate(%s, defer %d)\n", client->getName(), victim->getName(), defer); } @@ -1842,15 +2053,37 @@ void IOService::actionDidTerminate( IOService * victim, IOOptionBits options ) } } -void IOService::actionFinalize( IOService * victim, IOOptionBits options ) +void IOService::actionFinalize( IOService * victim, IOOptionBits options, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) { - TLOG("%s::finalize(%08lx)\n", victim->getName(), options); + TLOG("%s::finalize(%08llx)\n", victim->getName(), (long long)options); + + uint64_t regID1 = victim->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_FINALIZE, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + 0, 0); + victim->finalize( options ); } -void IOService::actionStop( IOService * provider, IOService * client ) +void IOService::actionStop( IOService * provider, IOService * client, + void *unused1 __unused, void *unused2 __unused, + void *unused3 __unused ) { TLOG("%s::stop(%s)\n", client->getName(), provider->getName()); + + uint64_t regID1 = provider->getRegistryEntryID(); + uint64_t regID2 = client->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_STOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + client->stop( provider ); if( provider->isOpen( client )) provider->close( client ); @@ -1960,10 +2193,30 @@ void IOService::terminateWorker( IOOptionBits options ) if( !provider->isChild( client, gIOServicePlane )) { // may be multiply queued - nop it TLOG("%s::nop stop(%s)\n", client->getName(), provider->getName()); + + uint64_t regID1 = provider->getRegistryEntryID(); + uint64_t regID2 = client->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_STOP_NOP, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + } else { - // not ready for stop if it has clients, skip it - if( (client->__state[1] & kIOServiceTermPhase3State) && client->getClient()) { + // a terminated client is not ready for stop if it has clients, skip it + if( (kIOServiceInactiveState & client->__state[0]) && client->getClient()) { TLOG("%s::defer stop(%s)\n", client->getName(), provider->getName()); + + uint64_t regID1 = provider->getRegistryEntryID(); + uint64_t regID2 = client->getRegistryEntryID(); + IOServiceTrace( + IOSERVICE_TERMINATE_STOP_DEFER, + (uintptr_t) regID1, + (uintptr_t) (regID1 >> 32), + (uintptr_t) regID2, + (uintptr_t) (regID2 >> 32)); + idx++; continue; } @@ -1992,6 +2245,9 @@ void IOService::terminateWorker( IOOptionBits options ) if( !moreToDo) { TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount()); + IOServiceTrace( + IOSERVICE_TERMINATE_DONE, + (uintptr_t) gIOStopList->getCount(), 0, 0, 0); } } while( moreToDo ); @@ -2232,13 +2488,13 @@ static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref) OSSymbol * key = (OSSymbol *) ref; OSNumber * offset; - if( (notify = OSDynamicCast( _IOServiceNotifier, entry))) + if( (dict = OSDynamicCast( OSDictionary, entry))) + offset = OSDynamicCast(OSNumber, dict->getObject( key )); + else if( (notify = OSDynamicCast( _IOServiceNotifier, entry))) return( notify->priority ); else if( (service = OSDynamicCast( IOService, entry))) offset = OSDynamicCast(OSNumber, service->getProperty( key )); - else if( (dict = OSDynamicCast( OSDictionary, entry))) - offset = OSDynamicCast(OSNumber, dict->getObject( key )); else { assert( false ); offset = 0; @@ -2269,7 +2525,7 @@ SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase return ( val1 - val2 ); } -IOService * IOService::getClientWithCategory( const OSSymbol * category ) +IOService * IOService::copyClientWithCategory( const OSSymbol * category ) { IOService * service = 0; OSIterator * iter; @@ -2283,13 +2539,25 @@ IOService * IOService::getClientWithCategory( const OSSymbol * category ) nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol, service->getProperty( gIOMatchCategoryKey )); if( category == nextCat) + { + service->retain(); break; + } } iter->release(); } return( service ); } +IOService * IOService::getClientWithCategory( const OSSymbol * category ) +{ + IOService * + service = copyClientWithCategory(category); + if (service) + service->release(); + return (service); +} + bool IOService::invokeNotifer( _IOServiceNotifier * notify ) { _IOServiceNotifierInvocation invocation; @@ -2309,7 +2577,7 @@ bool IOService::invokeNotifer( _IOServiceNotifier * notify ) if( willNotify) { - ret = (*notify->handler)( notify->target, notify->ref, this ); + ret = (*notify->handler)(notify->target, notify->ref, this, notify); LOCKWRITENOTIFY(); queue_remove( ¬ify->handlerInvocations, &invocation, @@ -2350,6 +2618,8 @@ void IOService::probeCandidates( OSOrderedSet * matches ) #if IOMATCHDEBUG SInt64 debugFlags; #endif + IOService * client = NULL; + assert( matches ); while( !needReloc && (nextMatch = matches->getFirstObject())) { @@ -2383,8 +2653,8 @@ void IOService::probeCandidates( OSOrderedSet * matches ) match->getObject( gIOMatchCategoryKey )); if( 0 == category) category = gIODefaultMatchCategoryKey; - - if( getClientWithCategory( category )) { + + if( (client = copyClientWithCategory(category)) ) { #if IOMATCHDEBUG if( debugFlags & kIOLogMatch) LOG("%s: match category %s exists\n", getName(), @@ -2392,6 +2662,10 @@ void IOService::probeCandidates( OSOrderedSet * matches ) #endif nextMatch->release(); nextMatch = 0; + + client->release(); + client = NULL; + continue; } @@ -2402,7 +2676,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) props->setCapacityIncrement(1); // check the nub matches - if( false == passiveMatch( props, true )) + if( false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) continue; // Check to see if driver reloc has been loaded. @@ -2455,6 +2729,8 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( !symbol) continue; + //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), symbol, props); + // alloc the driver instance inst = (IOService *) OSMetaClass::allocClassWithName( symbol); @@ -2481,7 +2757,6 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( 0 == category) category = gIODefaultMatchCategoryKey; inst->setProperty( gIOMatchCategoryKey, (OSObject *) category ); - // attach driver instance if( !(inst->attach( this ))) continue; @@ -2583,16 +2858,35 @@ void IOService::probeCandidates( OSOrderedSet * matches ) } - // adjust the busy count by -1 if matching is stalled for a module, - // or +1 if a previously stalled matching is complete. + // adjust the busy count by +1 if matching is stalled for a module, + // or -1 if a previously stalled matching is complete. lockForArbitration(); SInt32 adjBusy = 0; + uint64_t regID = getRegistryEntryID(); + if( needReloc) { adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1; - if( adjBusy) + if( adjBusy) { + + IOServiceTrace( + IOSERVICE_MODULESTALL, + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) this, + 0); + __state[1] |= kIOServiceModuleStallState; + } } else if( __state[1] & kIOServiceModuleStallState) { + + IOServiceTrace( + IOSERVICE_MODULEUNSTALL, + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) this, + 0); + __state[1] &= ~kIOServiceModuleStallState; adjBusy = -1; } @@ -2643,7 +2937,7 @@ bool IOService::startCandidate( IOService * service ) SUB_ABSOLUTETIME(&endTime, &startTime); absolutetime_to_nanoseconds(endTime, &nano); if (nano > 500000000ULL) - IOLog("%s::start took %ld ms\n", service->getName(), (UInt32)(nano / 1000000ULL)); + IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL)); } } if( !ok) @@ -2652,11 +2946,6 @@ bool IOService::startCandidate( IOService * service ) return( ok ); } -IOService * IOService::resources( void ) -{ - return( gIOResources ); -} - void IOService::publishResource( const char * key, OSObject * value ) { const OSSymbol * sym; @@ -2733,12 +3022,12 @@ bool IOService::checkResource( OSObject * matching ) } if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): stalling %s\n", (int) IOThreadSelf(), getName()); + LOG("config(%p): stalling %s\n", IOThreadSelf(), getName()); waitForService( table ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): waking\n", (int) IOThreadSelf() ); + LOG("config(%p): waking\n", IOThreadSelf() ); return( true ); } @@ -2779,7 +3068,8 @@ void _IOConfigThread::configThread( void ) continue; if( !inst->init()) continue; - if( !(IOCreateThread((IOThreadFunc) &_IOConfigThread::main, inst ))) + thread_t unused; + if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &unused)) continue; return; @@ -2794,6 +3084,7 @@ void _IOConfigThread::configThread( void ) void _IOConfigThread::free( void ) { + thread_deallocate(current_thread()); OSObject::free(); } @@ -2805,6 +3096,7 @@ void IOService::doServiceMatch( IOOptionBits options ) SInt32 catalogGeneration; bool keepGuessing = true; bool reRegistered = true; + bool didRegister; // job->nub->deliverNotification( gIOPublishNotification, // kIOServiceRegisteredState, 0xffffffff ); @@ -2822,17 +3114,18 @@ void IOService::doServiceMatch( IOOptionBits options ) LOCKREADNOTIFY(); __state[1] &= ~kIOServiceNeedConfigState; __state[1] |= kIOServiceConfigState; + didRegister = (0 == (kIOServiceRegisteredState & __state[0])); __state[0] |= kIOServiceRegisteredState; - if( reRegistered && (0 == (__state[0] & kIOServiceInactiveState))) { - + keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState)); + if (reRegistered && keepGuessing) { iter = OSCollectionIterator::withCollection( (OSOrderedSet *) gNotifications->getObject( gIOPublishNotification ) ); if( iter) { while((notify = (_IOServiceNotifier *) iter->getNextObject())) { - if( passiveMatch( notify->matching ) + if( matchPassive(notify->matching, 0) && (kIOServiceNotifyEnable & notify->state)) matches->setObject( notify ); } @@ -2841,9 +3134,12 @@ void IOService::doServiceMatch( IOOptionBits options ) } UNLOCKNOTIFY(); + if (didRegister) { + getMetaClass()->addInstance(this); + } unlockForArbitration(); - if( matches->getCount() && (kIOReturnSuccess == getResources())) + if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) probeCandidates( matches ); else matches->release(); @@ -2891,11 +3187,13 @@ UInt32 IOService::_adjustBusy( SInt32 delta ) if( next != this) next->lockForArbitration(); count = next->__state[1] & kIOServiceBusyStateMask; - assert( count < kIOServiceBusyMax); wasQuiet = (0 == count); - assert( (!wasQuiet) || (delta > 0)); - next->__state[1] += delta; - nowQuiet = (0 == (next->__state[1] & kIOServiceBusyStateMask)); + if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) + OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), count, delta); + else + count += delta; + next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count; + nowQuiet = (0 == count); needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1])); if( needWake) { @@ -2908,18 +3206,41 @@ UInt32 IOService::_adjustBusy( SInt32 delta ) next->unlockForArbitration(); if( (wasQuiet || nowQuiet) ) { + uint64_t regID = next->getRegistryEntryID(); + + IOServiceTrace( + ((wasQuiet/*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY), + (uintptr_t) regID, + (uintptr_t) (regID >> 32), + (uintptr_t) next, + 0); + + if (wasQuiet) + { + next->__timeBusy = mach_absolute_time(); + } + else + { + next->__accumBusy += mach_absolute_time() - next->__timeBusy; + next->__timeBusy = 0; + } + MessageClientsContext context; context.service = next; context.type = kIOMessageServiceBusyStateChange; - context.argument = (void *) wasQuiet; // busy now + context.argument = (void *) wasQuiet; /*nowBusy*/ context.argSize = 0; applyToInterestNotifiers( next, gIOBusyInterest, &messageClientsApplier, &context ); - if( nowQuiet && (next == gIOServiceRoot)) - OSMetaClass::considerUnloads(); +#if !NO_KEXTD + if( nowQuiet && (next == gIOServiceRoot)) { + OSKext::considerUnloads(); + IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0); + } +#endif } delta = nowQuiet ? -1 : +1; @@ -2936,13 +3257,40 @@ void IOService::adjustBusy( SInt32 delta ) unlockForArbitration(); } +uint64_t IOService::getAccumulatedBusyTime( void ) +{ + uint64_t accumBusy = __accumBusy; + uint64_t timeBusy = __timeBusy; + uint64_t nano; + + do + { + accumBusy = __accumBusy; + timeBusy = __timeBusy; + if (timeBusy) + accumBusy += mach_absolute_time() - timeBusy; + } + while (timeBusy != __timeBusy); + + absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano); + + return (nano); +} + UInt32 IOService::getBusyState( void ) { return( __state[1] & kIOServiceBusyStateMask ); } IOReturn IOService::waitForState( UInt32 mask, UInt32 value, - mach_timespec_t * timeout ) + mach_timespec_t * timeout ) +{ + panic("waitForState"); + return (kIOReturnUnsupported); +} + +IOReturn IOService::waitForState( UInt32 mask, UInt32 value, + uint64_t timeout ) { bool wait; int waitResult = THREAD_AWAKENED; @@ -2956,23 +3304,17 @@ IOReturn IOService::waitForState( UInt32 mask, UInt32 value, if( wait) { __state[1] |= kIOServiceBusyWaiterState; unlockForArbitration(); - if( timeout ) { + if( timeout != UINT64_MAX ) { if( computeDeadline ) { AbsoluteTime nsinterval; - clock_interval_to_absolutetime_interval( - timeout->tv_sec, kSecondScale, &abstime ); - clock_interval_to_absolutetime_interval( - timeout->tv_nsec, kNanosecondScale, &nsinterval ); - ADD_ABSOLUTETIME( &abstime, &nsinterval ); - clock_absolutetime_interval_to_deadline( - abstime, &abstime ); + nanoseconds_to_absolutetime(timeout, &nsinterval ); + clock_absolutetime_interval_to_deadline(nsinterval, &abstime); computeDeadline = false; } - - assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime)); + assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime)); } - else - assert_wait((event_t)this, THREAD_UNINT ); + else + assert_wait((event_t)this, THREAD_UNINT ); } else unlockForArbitration(); IOLockUnlock( gIOServiceBusyLock ); @@ -2987,11 +3329,27 @@ IOReturn IOService::waitForState( UInt32 mask, UInt32 value, return( kIOReturnSuccess ); } -IOReturn IOService::waitQuiet( mach_timespec_t * timeout ) +IOReturn IOService::waitQuiet( uint64_t timeout ) { return( waitForState( kIOServiceBusyStateMask, 0, timeout )); } +IOReturn IOService::waitQuiet( mach_timespec_t * timeout ) +{ + uint64_t timeoutNS; + + if (timeout) + { + timeoutNS = timeout->tv_sec; + timeoutNS *= kSecondScale; + timeoutNS += timeout->tv_nsec; + } + else + timeoutNS = UINT64_MAX; + + return( waitForState( kIOServiceBusyStateMask, 0, timeoutNS )); +} + bool IOService::serializeProperties( OSSerialize * s ) const { #if 0 @@ -3002,11 +3360,21 @@ bool IOService::serializeProperties( OSSerialize * s ) const } -void _IOConfigThread::main( _IOConfigThread * self ) +void _IOConfigThread::main(void * arg, wait_result_t result) { - _IOServiceJob * job; - IOService * nub; - bool alive = true; + _IOConfigThread * self = (_IOConfigThread *) arg; + _IOServiceJob * job; + IOService * nub; + bool alive = true; + kern_return_t kr; + thread_precedence_policy_data_t precedence = { -1 }; + + kr = thread_policy_set(current_thread(), + THREAD_PRECEDENCE_POLICY, + (thread_policy_t) &precedence, + THREAD_PRECEDENCE_POLICY_COUNT); + if (KERN_SUCCESS != kr) + IOLog("thread_policy_set(%d)\n", kr); do { @@ -3030,8 +3398,8 @@ void _IOConfigThread::main( _IOConfigThread * self ) nub = job->nub; if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): starting on %s, %d\n", - (int) IOThreadSelf(), job->nub->getName(), job->type); + LOG("config(%p): starting on %s, %d\n", + IOThreadSelf(), job->nub->getName(), job->type); switch( job->type) { @@ -3040,8 +3408,8 @@ void _IOConfigThread::main( _IOConfigThread * self ) break; default: - LOG("config(%x): strange type (%d)\n", - (int) IOThreadSelf(), job->type ); + LOG("config(%p): strange type (%d)\n", + IOThreadSelf(), job->type ); break; } @@ -3065,7 +3433,7 @@ void _IOConfigThread::main( _IOConfigThread * self ) } while( alive ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%x): terminating\n", (int) IOThreadSelf() ); + LOG("config(%p): terminating\n", IOThreadSelf() ); self->release(); } @@ -3075,7 +3443,7 @@ IOReturn IOService::waitMatchIdle( UInt32 msToWait ) bool wait; int waitResult = THREAD_AWAKENED; bool computeDeadline = true; - AbsoluteTime abstime; + AbsoluteTime deadline; IOLockLock( gJobsLock ); do { @@ -3083,14 +3451,12 @@ IOReturn IOService::waitMatchIdle( UInt32 msToWait ) if( wait) { if( msToWait) { if( computeDeadline ) { - clock_interval_to_absolutetime_interval( - msToWait, kMillisecondScale, &abstime ); - clock_absolutetime_interval_to_deadline( - abstime, &abstime ); + clock_interval_to_deadline( + msToWait, kMillisecondScale, &deadline ); computeDeadline = false; } waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads, - abstime, THREAD_UNINT ); + deadline, THREAD_UNINT ); } else { waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads, THREAD_UNINT ); @@ -3140,37 +3506,153 @@ void _IOServiceJob::pingConfig( _IOServiceJob * job ) semaphore_signal( gJobsSemaphore ); } +struct IOServiceMatchContext +{ + OSDictionary * table; + OSObject * result; + uint32_t options; + uint32_t state; + uint32_t count; + uint32_t done; +}; + +bool IOService::instanceMatch(const OSObject * entry, void * context) +{ + IOServiceMatchContext * ctx = (typeof(ctx)) context; + IOService * service = (typeof(service)) entry; + OSDictionary * table = ctx->table; + uint32_t options = ctx->options; + uint32_t state = ctx->state; + uint32_t done; + bool match; + + done = 0; + do + { + match = ((state == (state & service->__state[0])) + && (0 == (service->__state[0] & kIOServiceInactiveState))); + if (!match) break; + ctx->count += table->getCount(); + match = service->matchInternal(table, options, &done); + ctx->done += done; + } + while (false); + if (!match) + return (false); + + if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) + { + service->retain(); + ctx->result = service; + return (true); + } + else if (!ctx->result) + { + ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1); + } + else + { + ((OSSet *)ctx->result)->setObject(service); + } + return (false); +} + // internal - call with gNotificationLock -OSObject * IOService::getExistingServices( OSDictionary * matching, +OSObject * IOService::copyExistingServices( OSDictionary * matching, IOOptionBits inState, IOOptionBits options ) { - OSObject * current = 0; - OSIterator * iter; - IOService * service; - OSObject * obj; + OSObject * current = 0; + OSIterator * iter; + IOService * service; + OSObject * obj; + OSString * str; if( !matching) return( 0 ); - if(true - && (obj = matching->getObject(gIOProviderClassKey)) +#if MATCH_DEBUG + OSSerialize * s = OSSerialize::withCapacity(128); + matching->serialize(s); +#endif + + if((obj = matching->getObject(gIOProviderClassKey)) && gIOResourcesKey && gIOResourcesKey->isEqualTo(obj) && (service = gIOResources)) { if( (inState == (service->__state[0] & inState)) && (0 == (service->__state[0] & kIOServiceInactiveState)) - && service->passiveMatch( matching )) + && service->matchPassive(matching, options)) { if( options & kIONotifyOnce) + { + service->retain(); current = service; + } else - current = OSSet::withObjects( - (const OSObject **) &service, 1, 1 ); + current = OSSet::withObjects((const OSObject **) &service, 1, 1 ); } } else { + IOServiceMatchContext ctx; + ctx.table = matching; + ctx.state = inState; + ctx.count = 0; + ctx.done = 0; + ctx.options = options; + ctx.result = 0; + + if ((str = OSDynamicCast(OSString, obj))) + { + const OSSymbol * sym = OSSymbol::withString(str); + OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx); + sym->release(); + } + else + { + IOService::gMetaClass.applyToInstances(instanceMatch, &ctx); + } + + + current = ctx.result; + + options |= kIOServiceInternalDone | kIOServiceClassDone; + if (current && (ctx.done != ctx.count)) + { + OSSet * + source = OSDynamicCast(OSSet, current); + current = 0; + while ((service = (IOService *) source->getAnyObject())) + { + if (service->matchPassive(matching, options)) + { + if( options & kIONotifyOnce) + { + service->retain(); + current = service; + break; + } + if( current) + { + ((OSSet *)current)->setObject( service ); + } + else + { + current = OSSet::withObjects( + (const OSObject **) &service, 1, 1 ); + } + } + source->removeObject(service); + } + source->release(); + } + } + +#if MATCH_DEBUG + { + OSObject * _current = 0; + iter = IORegistryIterator::iterateOver( gIOServicePlane, kIORegistryIterateRecursively ); if( iter) { @@ -3179,23 +3661,42 @@ OSObject * IOService::getExistingServices( OSDictionary * matching, while( (service = (IOService *) iter->getNextObject())) { if( (inState == (service->__state[0] & inState)) && (0 == (service->__state[0] & kIOServiceInactiveState)) - && service->passiveMatch( matching )) { + && service->matchPassive(matching, 0)) { if( options & kIONotifyOnce) { - current = service; + service->retain(); + _current = service; break; } - if( current) - ((OSSet *)current)->setObject( service ); + if( _current) + ((OSSet *)_current)->setObject( service ); else - current = OSSet::withObjects( + _current = OSSet::withObjects( (const OSObject **) &service, 1, 1 ); } } } while( !service && !iter->isValid()); iter->release(); } - } + + + if ( ((current != 0) != (_current != 0)) + || (current && _current && !current->isEqualTo(_current))) + { + OSSerialize * s1 = OSSerialize::withCapacity(128); + OSSerialize * s2 = OSSerialize::withCapacity(128); + current->serialize(s1); + _current->serialize(s2); + kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", current, _current, s->text(), s1->text(), s2->text()); + s1->release(); + s2->release(); + } + + if (_current) _current->release(); + } + + s->release(); +#endif if( current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) { iter = OSCollectionIterator::withCollection( (OSSet *)current ); @@ -3214,7 +3715,7 @@ OSIterator * IOService::getMatchingServices( OSDictionary * matching ) // is a lock even needed? LOCKWRITENOTIFY(); - iter = (OSIterator *) getExistingServices( matching, + iter = (OSIterator *) copyExistingServices( matching, kIOServiceMatchedState ); UNLOCKNOTIFY(); @@ -3222,11 +3723,38 @@ OSIterator * IOService::getMatchingServices( OSDictionary * matching ) return( iter ); } +IOService * IOService::copyMatchingService( OSDictionary * matching ) +{ + IOService * service; + + // is a lock even needed? + LOCKWRITENOTIFY(); + + service = (IOService *) copyExistingServices( matching, + kIOServiceMatchedState, kIONotifyOnce ); + + UNLOCKNOTIFY(); + + return( service ); +} + +struct _IOServiceMatchingNotificationHandlerRef +{ + IOServiceNotificationHandler handler; + void * ref; +}; + +static bool _IOServiceMatchingNotificationHandler( void * target, void * refCon, + IOService * newService, + IONotifier * notifier ) +{ + return ((*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService)); +} // internal - call with gNotificationLock IONotifier * IOService::setNotification( const OSSymbol * type, OSDictionary * matching, - IOServiceNotificationHandler handler, void * target, void * ref, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority ) { _IOServiceNotifier * notify = 0; @@ -3242,10 +3770,17 @@ IONotifier * IOService::setNotification( } if( notify) { - notify->matching = matching; - notify->handler = handler; + notify->handler = handler; notify->target = target; - notify->ref = ref; + notify->matching = matching; + matching->retain(); + if (handler == &_IOServiceMatchingNotificationHandler) + { + notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler; + notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref; + } + else + notify->ref = ref; notify->priority = priority; notify->state = kIOServiceNotifyEnable; queue_init( ¬ify->handlerInvocations ); @@ -3271,7 +3806,7 @@ IONotifier * IOService::setNotification( // internal - call with gNotificationLock IONotifier * IOService::doInstallNotification( const OSSymbol * type, OSDictionary * matching, - IOServiceNotificationHandler handler, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority, OSIterator ** existing ) { @@ -3300,7 +3835,7 @@ IONotifier * IOService::doInstallNotification( if( inState) // get the current set - exist = (OSIterator *) getExistingServices( matching, inState ); + exist = (OSIterator *) copyExistingServices( matching, inState ); else exist = 0; @@ -3309,14 +3844,35 @@ IONotifier * IOService::doInstallNotification( return( notify ); } +#if !defined(__LP64__) +IONotifier * IOService::installNotification(const OSSymbol * type, OSDictionary * matching, + IOServiceNotificationHandler handler, + void * target, void * refCon, + SInt32 priority, OSIterator ** existing ) +{ + IONotifier * result; + _IOServiceMatchingNotificationHandlerRef ref; + ref.handler = handler; + ref.ref = refCon; + + result = (_IOServiceNotifier *) installNotification( type, matching, + &_IOServiceMatchingNotificationHandler, + target, &ref, priority, existing ); + if (result) + matching->release(); + + return (result); +} +#endif /* !defined(__LP64__) */ + IONotifier * IOService::installNotification( const OSSymbol * type, OSDictionary * matching, - IOServiceNotificationHandler handler, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority, OSIterator ** existing ) { - IONotifier * notify; + IONotifier * notify; LOCKWRITENOTIFY(); @@ -3331,10 +3887,31 @@ IONotifier * IOService::installNotification( IONotifier * IOService::addNotification( const OSSymbol * type, OSDictionary * matching, IOServiceNotificationHandler handler, + void * target, void * refCon, + SInt32 priority ) +{ + IONotifier * result; + _IOServiceMatchingNotificationHandlerRef ref; + + ref.handler = handler; + ref.ref = refCon; + + result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler, + target, &ref, priority); + + if (result) + matching->release(); + + return (result); +} + +IONotifier * IOService::addMatchingNotification( + const OSSymbol * type, OSDictionary * matching, + IOServiceMatchingNotificationHandler handler, void * target, void * ref, SInt32 priority ) { - OSIterator * existing; + OSIterator * existing = NULL; _IOServiceNotifier * notify; IOService * next; @@ -3359,76 +3936,93 @@ IONotifier * IOService::addNotification( return( notify ); } -struct SyncNotifyVars { - semaphore_port_t waitHere; - IOService * result; -}; - bool IOService::syncNotificationHandler( void * /* target */, void * ref, - IOService * newService ) + IOService * newService, + IONotifier * notifier ) { - // result may get written more than once before the - // notification is removed! - ((SyncNotifyVars *) ref)->result = newService; - semaphore_signal( ((SyncNotifyVars *) ref)->waitHere ); + LOCKWRITENOTIFY(); + if (!*((IOService **) ref)) + { + newService->retain(); + (*(IOService **) ref) = newService; + WAKEUPNOTIFY(ref); + } + UNLOCKNOTIFY(); return( false ); } -IOService * IOService::waitForService( OSDictionary * matching, - mach_timespec_t * timeout ) +IOService * IOService::waitForMatchingService( OSDictionary * matching, + uint64_t timeout) { IONotifier * notify = 0; // priority doesn't help us much since we need a thread wakeup SInt32 priority = 0; - SyncNotifyVars state; - kern_return_t err = kIOReturnBadArgument; + IOService * result; - if( !matching) + if (!matching) return( 0 ); - state.waitHere = 0; - state.result = 0; + result = NULL; LOCKWRITENOTIFY(); - - do { - - state.result = (IOService *) getExistingServices( matching, + do + { + result = (IOService *) copyExistingServices( matching, kIOServiceMatchedState, kIONotifyOnce ); - if( state.result) - continue; - - err = semaphore_create( kernel_task, &state.waitHere, - SYNC_POLICY_FIFO, 0 ); - if( KERN_SUCCESS != err) - continue; - + if (result) + break; notify = IOService::setNotification( gIOMatchedNotification, matching, &IOService::syncNotificationHandler, (void *) 0, - (void *) &state, priority ); - - } while( false ); - - UNLOCKNOTIFY(); - - if( notify) { - if( timeout) - err = semaphore_timedwait( state.waitHere, *timeout ); + &result, priority ); + if (!notify) + break; + if (UINT64_MAX != timeout) + { + AbsoluteTime deadline; + nanoseconds_to_absolutetime(timeout, &deadline); + clock_absolutetime_interval_to_deadline(deadline, &deadline); + SLEEPNOTIFYTO(&result, deadline); + } else - err = semaphore_wait( state.waitHere ); + { + SLEEPNOTIFY(&result); + } } + while( false ); - if( notify) + UNLOCKNOTIFY(); + + if (notify) notify->remove(); // dequeues + + return( result ); +} + +IOService * IOService::waitForService( OSDictionary * matching, + mach_timespec_t * timeout ) +{ + IOService * result; + uint64_t timeoutNS; + + if (timeout) + { + timeoutNS = timeout->tv_sec; + timeoutNS *= kSecondScale; + timeoutNS += timeout->tv_nsec; + } else - matching->release(); - if( state.waitHere) - semaphore_destroy( kernel_task, state.waitHere ); + timeoutNS = UINT64_MAX; + + result = waitForMatchingService(matching, timeoutNS); - return( state.result ); + matching->release(); + if (result) + result->release(); + + return (result); } void IOService::deliverNotification( const OSSymbol * type, @@ -3451,7 +4045,7 @@ void IOService::deliverNotification( const OSSymbol * type, if( iter) { while( (notify = (_IOServiceNotifier *) iter->getNextObject())) { - if( passiveMatch( notify->matching) + if( matchPassive(notify->matching, 0) && (kIOServiceNotifyEnable & notify->state)) { if( 0 == willSend) willSend = OSArray::withCapacity(8); @@ -3490,10 +4084,18 @@ IOOptionBits IOService::getState( void ) const OSDictionary * IOService::serviceMatching( const OSString * name, OSDictionary * table ) { + + const OSString * str; + + str = OSSymbol::withString(name); + if( !str) + return( 0 ); + if( !table) table = OSDictionary::withCapacity( 2 ); if( table) - table->setObject(gIOProviderClassKey, (OSObject *)name ); + table->setObject(gIOProviderClassKey, (OSObject *)str ); + str->release(); return( table ); } @@ -3562,6 +4164,47 @@ OSDictionary * IOService::resourceMatching( const char * name, return( table ); } +OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value, + OSDictionary * table ) +{ + OSDictionary * properties; + + properties = OSDictionary::withCapacity( 2 ); + if( !properties) + return( 0 ); + properties->setObject( key, value ); + + if( !table) + table = OSDictionary::withCapacity( 2 ); + if( table) + table->setObject( gIOPropertyMatchKey, properties ); + + properties->release(); + + return( table ); +} + +OSDictionary * IOService::registryEntryIDMatching( uint64_t entryID, + OSDictionary * table ) +{ + OSNumber * num; + + num = OSNumber::withNumber( entryID, 64 ); + if( !num) + return( 0 ); + + if( !table) + table = OSDictionary::withCapacity( 2 ); + if( table) + table->setObject( gIORegistryEntryIDKey, num ); + + if (num) + num->release(); + + return( table ); +} + + /* * _IOServiceNotifier */ @@ -3662,10 +4305,38 @@ IOService * IOResources::resources( void ) return( inst ); } +bool IOResources::init( OSDictionary * dictionary ) +{ + // Do super init first + if ( !super::init() ) + return false; + + // Allow PAL layer to publish a value + const char *property_name; + int property_value; + + pal_get_resource_property( &property_name, &property_value ); + + if( property_name ) { + OSNumber *num; + const OSSymbol * sym; + + if( (num = OSNumber::withNumber(property_value, 32)) != 0 ) { + if( (sym = OSSymbol::withCString( property_name)) != 0 ) { + this->setProperty( sym, num ); + sym->release(); + } + num->release(); + } + } + + return true; +} + IOWorkLoop * IOResources::getWorkLoop() const { - // If we are the resource root then bringe over to the - // platform to get its workloop + // If we are the resource root + // then use the platform's workloop if (this == (IOResources *) gIOResources) return getPlatform()->getWorkLoop(); else @@ -3699,6 +4370,123 @@ bool IOResources::matchPropertyTable( OSDictionary * table ) return( ok ); } +void IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1) +{ + IOService::updateConsoleUsers(NULL, 0); +} + +void IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage) +{ + IORegistryEntry * regEntry; + OSObject * locked = kOSBooleanFalse; + uint32_t idx; + bool publish; + OSDictionary * user; + static IOMessage sSystemPower; + + regEntry = IORegistryEntry::getRegistryRoot(); + + if (!gIOChosenEntry) + gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); + + IOLockLock(gIOConsoleUsersLock); + + if (systemMessage) + { + sSystemPower = systemMessage; +#if HIBERNATION + if ((kIOMessageSystemHasPoweredOn == systemMessage) && IOHibernateWasScreenLocked()) + { + locked = kOSBooleanTrue; + } +#endif /* HIBERNATION */ + } + + if (consoleUsers) + { + OSNumber * num = 0; + gIOConsoleLoggedIn = false; + for (idx = 0; + (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx))); + idx++) + { + gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey)) + && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey))); + if (!num) + { + num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey)); + } + } + gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0; + } + + if (!gIOConsoleLoggedIn + || (kIOMessageSystemWillSleep == sSystemPower) + || (kIOMessageSystemPagingOff == sSystemPower)) + { + locked = kOSBooleanTrue; + } + else if (gIOConsoleLockTime) + { + clock_sec_t now; + clock_usec_t microsecs; + + clock_get_calendar_microtime(&now, µsecs); + if (gIOConsoleLockTime > now) + { + AbsoluteTime deadline; + clock_interval_to_deadline(gIOConsoleLockTime - now, kSecondScale, &deadline); + thread_call_enter_delayed(gIOConsoleLockCallout, deadline); + } + else + { + locked = kOSBooleanTrue; + } + } + + publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey))); + if (publish) + { + regEntry->setProperty(gIOConsoleLockedKey, locked); + if (consoleUsers) + { + regEntry->setProperty(gIOConsoleUsersKey, consoleUsers); + } + OSIncrementAtomic( &gIOConsoleUsersSeed ); + } + +#if HIBERNATION + if (gIOChosenEntry) + { + uint32_t screenLockState; + + if (locked == kOSBooleanTrue) screenLockState = kIOScreenLockLocked; + else if (gIOConsoleLockTime) screenLockState = kIOScreenLockUnlocked; + else screenLockState = kIOScreenLockNoLock; + + if (screenLockState != gIOScreenLockState) gIOChosenEntry->setProperty(kIOScreenLockStateKey, &screenLockState, sizeof(screenLockState)); + gIOScreenLockState = screenLockState; + } +#endif /* HIBERNATION */ + + IOLockUnlock(gIOConsoleUsersLock); + + if (publish) + { + publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue ); + + MessageClientsContext context; + + context.service = getServiceRoot(); + context.type = kIOMessageConsoleSecurityChange; + context.argument = (void *) regEntry; + context.argSize = 0; + + applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest, + &messageClientsApplier, &context ); + } +} + IOReturn IOResources::setProperties( OSObject * properties ) { IOReturn err; @@ -3718,15 +4506,17 @@ IOReturn IOResources::setProperties( OSObject * properties ) if( 0 == iter) return( kIOReturnBadArgument); - while( (key = OSDynamicCast(OSSymbol, iter->getNextObject()))) { - - if (gIOConsoleUsersKey == key) + while( (key = OSDynamicCast(OSSymbol, iter->getNextObject()))) + { + if (gIOConsoleUsersKey == key) do { - IORegistryEntry::getRegistryRoot()->setProperty(key, dict->getObject(key)); - OSIncrementAtomic( &gIOConsoleUsersSeed ); - publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue ); - continue; + OSArray * consoleUsers; + consoleUsers = OSDynamicCast(OSArray, dict->getObject(key)); + if (!consoleUsers) + continue; + IOService::updateConsoleUsers(consoleUsers, 0); } + while (false); publishResource( key, dict->getObject(key) ); } @@ -3830,138 +4620,188 @@ IOService * IOService::matchLocation( IOService * /* client */ ) return( parent ); } -bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) +bool IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did) { - IOService * where; OSString * matched; OSObject * obj; OSString * str; IORegistryEntry * entry; OSNumber * num; - SInt32 score; - OSNumber * newPri; bool match = true; - bool matchParent = false; - UInt32 done; + bool changesOK = (0 != (kIOServiceChangesOK & options)); + uint32_t count; + uint32_t done; - assert( table ); + do + { + count = table->getCount(); + done = 0; + str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey)); + if (str) { + done++; + match = ((kIOServiceClassDone & options) || (0 != metaCast(str))); +#if MATCH_DEBUG + match = (0 != metaCast( str )); + if ((kIOServiceClassDone & options) && !match) panic("classDone"); +#endif + if ((!match) || (done == count)) break; + } - where = this; + obj = table->getObject( gIONameMatchKey ); + if( obj) { + done++; + match = compareNames( obj, changesOK ? &matched : 0 ); + if (!match) break; + if( changesOK && matched) { + // leave a hint as to which name matched + table->setObject( gIONameMatchedKey, matched ); + matched->release(); + } + if (done == count) break; + } - do { - do { - done = 0; - - str = OSDynamicCast( OSString, table->getObject( gIOProviderClassKey)); - if( str) { - done++; - match = (0 != where->metaCast( str )); - if( !match) - break; - } + str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey )); + if (str) + { + const OSSymbol * sym; + done++; + match = false; + sym = copyLocation(); + if (sym) { + match = sym->isEqualTo( str ); + sym->release(); + } + if ((!match) || (done == count)) break; + } - obj = table->getObject( gIONameMatchKey ); - if( obj) { - done++; - match = where->compareNames( obj, changesOK ? &matched : 0 ); - if( !match) - break; - if( changesOK && matched) { - // leave a hint as to which name matched - table->setObject( gIONameMatchedKey, matched ); - matched->release(); - } - } + obj = table->getObject( gIOPropertyMatchKey ); + if( obj) + { + OSDictionary * dict; + OSDictionary * nextDict; + OSIterator * iter; + done++; + match = false; + dict = dictionaryWithProperties(); + if( dict) { + nextDict = OSDynamicCast( OSDictionary, obj); + if( nextDict) + iter = 0; + else + iter = OSCollectionIterator::withCollection( + OSDynamicCast(OSCollection, obj)); + + while( nextDict + || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary, + iter->getNextObject()))))) { + match = dict->isEqualTo( nextDict, nextDict); + if( match) + break; + nextDict = 0; + } + dict->release(); + if( iter) + iter->release(); + } + if ((!match) || (done == count)) break; + } - str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey )); - if( str) { + str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey )); + if( str) { + done++; + entry = IORegistryEntry::fromPath( str->getCStringNoCopy() ); + match = (this == entry); + if( entry) + entry->release(); + if ((!match) || (done == count)) break; + } - const OSSymbol * sym; + num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey )); + if (num) { + done++; + match = (getRegistryEntryID() == num->unsigned64BitValue()); + if ((!match) || (done == count)) break; + } - done++; - match = false; - sym = where->copyLocation(); - if( sym) { - match = sym->isEqualTo( str ); - sym->release(); - } - if( !match) - break; - } + num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey )); + if( num) + { + OSIterator * iter; + IOService * service = 0; + UInt32 serviceCount = 0; - obj = table->getObject( gIOPropertyMatchKey ); - if( obj) { + done++; + iter = getClientIterator(); + if( iter) { + while( (service = (IOService *) iter->getNextObject())) { + if( kIOServiceInactiveState & service->__state[0]) + continue; + if( 0 == service->getProperty( gIOMatchCategoryKey )) + continue; + ++serviceCount; + } + iter->release(); + } + match = (serviceCount == num->unsigned32BitValue()); + if ((!match) || (done == count)) break; + } - OSDictionary * dict; - OSDictionary * nextDict; - OSIterator * iter; +#define propMatch(key) \ + obj = table->getObject(key); \ + if (obj) \ + { \ + OSObject * prop; \ + done++; \ + prop = copyProperty(key); \ + match = obj->isEqualTo(prop); \ + if (prop) prop->release(); \ + if ((!match) || (done == count)) break; \ + } + propMatch(kIOBSDNameKey) + propMatch(kIOBSDMajorKey) + propMatch(kIOBSDMinorKey) + propMatch(kIOBSDUnitKey) +#undef propMatch + } + while (false); - done++; - match = false; - dict = where->dictionaryWithProperties(); - if( dict) { - nextDict = OSDynamicCast( OSDictionary, obj); - if( nextDict) - iter = 0; - else - iter = OSCollectionIterator::withCollection( - OSDynamicCast(OSCollection, obj)); - - while( nextDict - || (iter && (0 != (nextDict = OSDynamicCast(OSDictionary, - iter->getNextObject()))))) { - match = dict->isEqualTo( nextDict, nextDict); - if( match) - break; - nextDict = 0; - } - dict->release(); - if( iter) - iter->release(); - } - if( !match) - break; - } + if (did) *did = done; + return (match); +} - str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey )); - if( str) { - done++; - entry = IORegistryEntry::fromPath( str->getCStringNoCopy() ); - match = (where == entry); - if( entry) - entry->release(); - if( !match) - break; - } +bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) +{ + return (matchPassive(table, changesOK ? kIOServiceChangesOK : 0)); +} - num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey )); - if( num) { +bool IOService::matchPassive(OSDictionary * table, uint32_t options) +{ + IOService * where; + OSDictionary * nextTable; + SInt32 score; + OSNumber * newPri; + bool match = true; + bool matchParent = false; + uint32_t count; + uint32_t done; - OSIterator * iter; - IOService * service = 0; - UInt32 serviceCount = 0; + assert( table ); - done++; - iter = where->getClientIterator(); - if( iter) { - while( (service = (IOService *) iter->getNextObject())) { - if( kIOServiceInactiveState & service->__state[0]) - continue; - if( 0 == service->getProperty( gIOMatchCategoryKey )) - continue; - ++serviceCount; - } - iter->release(); - } - match = (serviceCount == num->unsigned32BitValue()); - if( !match) - break; - } +#if MATCH_DEBUG + OSDictionary * root = table; +#endif - if( done == table->getCount()) { - // don't call family if we've done all the entries in the table - matchParent = false; - break; + where = this; + do + { + do + { + count = table->getCount(); + if (!(kIOServiceInternalDone & options)) + { + match = where->matchInternal(table, options, &done); + // don't call family if we've done all the entries in the table + if ((!match) || (done == count)) break; } // pass in score from property table @@ -3978,7 +4818,7 @@ bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) break; } - if( changesOK) { + if (kIOServiceChangesOK & options) { // save the score newPri = OSNumber::withNumber( score, 32 ); if( newPri) { @@ -3987,37 +4827,42 @@ bool IOService::passiveMatch( OSDictionary * table, bool changesOK ) } } - if( !(match = where->compareProperty( table, kIOBSDNameKey ))) - break; - + options = 0; matchParent = false; - obj = OSDynamicCast( OSDictionary, + nextTable = OSDynamicCast(OSDictionary, table->getObject( gIOParentMatchKey )); - if( obj) { + if( nextTable) { + // look for a matching entry anywhere up to root match = false; matchParent = true; - table = (OSDictionary *) obj; + table = nextTable; break; } - table = OSDynamicCast( OSDictionary, + table = OSDynamicCast(OSDictionary, table->getObject( gIOLocationMatchKey )); - if( table) { + if (table) { + // look for a matching entry at matchLocation() match = false; where = where->getProvider(); - if( where) - where = where->matchLocation( where ); + if (where && (where = where->matchLocation(where))) continue; } + break; + } + while (true); + } + while( matchParent && (!match) && (where = where->getProvider()) ); - } while( table && where ); - - } while( matchParent && (where = where->getProvider()) ); - - if( kIOLogMatch & gIOKitDebug) - if( where != this) - LOG("match parent @ %s = %d\n", - where->getName(), match ); +#if MATCH_DEBUG + if (where != this) + { + OSSerialize * s = OSSerialize::withCapacity(128); + root->serialize(s); + kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text()); + s->release(); + } +#endif return( match ); } @@ -4216,7 +5061,6 @@ int IOService::errnoFromReturn( IOReturn rtn ) case kIOReturnExclusiveAccess: case kIOReturnLockedRead: case kIOReturnLockedWrite: - case kIOReturnNotAttached: case kIOReturnNotOpen: case kIOReturnNotReadable: return(EACCES); @@ -4229,6 +5073,7 @@ int IOService::errnoFromReturn( IOReturn rtn ) return(EBUSY); case kIOReturnBadMedia: case kIOReturnNoMedia: + case kIOReturnNotAttached: case kIOReturnUnformattedMedia: return(ENXIO); // (media error) case kIOReturnDMAError: @@ -4329,82 +5174,190 @@ void IOService::setDeviceMemory( OSArray * array ) void IOService:: setCPUSnoopDelay(UInt32 __unused ns) { -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) ml_set_maxsnoop(ns); -#endif /* __i386__ */ +#endif /* defined(__i386__) || defined(__x86_64__) */ } UInt32 IOService:: getCPUSnoopDelay() { -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) return ml_get_maxsnoop(); #else return 0; -#endif /* __i386__ */ +#endif /* defined(__i386__) || defined(__x86_64__) */ } -void IOService:: -requireMaxBusStall(UInt32 __unused ns) +#if defined(__i386__) || defined(__x86_64__) +static void +requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType) { -#if __i386__ static const UInt kNoReplace = -1U; // Must be an illegal index UInt replace = kNoReplace; + bool setCpuDelay = false; - IOLockLock(sBusStallLock); + IORecursiveLockLock(sCpuDelayLock); - UInt count = sBusStall->getLength() / sizeof(BusStallEntry); - BusStallEntry *entries = (BusStallEntry *) sBusStall->getBytesNoCopy(); + UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry); + CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy(); + IOService * holder = NULL; if (ns) { - const BusStallEntry ne = {this, ns}; - - // Set Maximum bus delay. - for (UInt i = 0; i < count; i++) { - const IOService *thisService = entries[i].fService; - if (this == thisService) - replace = i; - else if (!thisService) { - if (kNoReplace == replace) - replace = i; - } - else { - const UInt32 thisMax = entries[i].fMaxDelay; - if (thisMax < ns) - ns = thisMax; - } - } - - // Must be safe to call from locked context - ml_set_maxbusdelay(ns); - - if (kNoReplace == replace) - sBusStall->appendBytes(&ne, sizeof(ne)); - else - entries[replace] = ne; + const CpuDelayEntry ne = {service, ns, delayType}; + holder = service; + // Set maximum delay. + for (UInt i = 0; i < count; i++) { + IOService *thisService = entries[i].fService; + bool sameType = (delayType == entries[i].fDelayType); + if ((service == thisService) && sameType) + replace = i; + else if (!thisService) { + if (kNoReplace == replace) + replace = i; + } + else if (sameType) { + const UInt32 thisMax = entries[i].fMaxDelay; + if (thisMax < ns) + { + ns = thisMax; + holder = thisService; + } + } + } + + setCpuDelay = true; + if (kNoReplace == replace) + sCpuDelayData->appendBytes(&ne, sizeof(ne)); + else + entries[replace] = ne; } else { - ns = -1U; // Set to max unsigned, i.e. no restriction - - for (UInt i = 0; i < count; i++) { - // Clear a maximum bus delay. - const IOService *thisService = entries[i].fService; - UInt32 thisMax = entries[i].fMaxDelay; - if (this == thisService) - replace = i; - else if (thisService && thisMax < ns) - ns = thisMax; + ns = -1U; // Set to max unsigned, i.e. no restriction + + for (UInt i = 0; i < count; i++) { + // Clear a maximum delay. + IOService *thisService = entries[i].fService; + if (thisService && (delayType == entries[i].fDelayType)) { + UInt32 thisMax = entries[i].fMaxDelay; + if (service == thisService) + replace = i; + else if (thisMax < ns) { + ns = thisMax; + holder = thisService; + } + } + } + + // Check if entry found + if (kNoReplace != replace) { + entries[replace].fService = 0; // Null the entry + setCpuDelay = true; + } + } + + if (setCpuDelay) + { + // Must be safe to call from locked context + if (delayType == kCpuDelayBusStall) + { + ml_set_maxbusdelay(ns); + } + else if (delayType == kCpuDelayInterrupt) + { + ml_set_maxintdelay(ns); + } + + OSArray * handlers = sCpuLatencyHandlers[delayType]; + IOService * target; + if (handlers) for (unsigned int idx = 0; + (target = (IOService *) handlers->getObject(idx)); + idx++) + { + target->callPlatformFunction(sCPULatencyFunctionName[delayType], false, + (void *) (uintptr_t) ns, holder, + NULL, NULL); } + } + + IORecursiveLockUnlock(sCpuDelayLock); +} - // Check if entry found - if (kNoReplace != replace) { - entries[replace].fService = 0; // Null the entry - ml_set_maxbusdelay(ns); +static IOReturn +setLatencyHandler(UInt32 delayType, IOService * target, bool enable) +{ + IOReturn result = kIOReturnNotFound; + OSArray * array; + unsigned int idx; + + IORecursiveLockLock(sCpuDelayLock); + + do + { + if (enable && !sCpuLatencyHandlers[delayType]) + sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4); + array = sCpuLatencyHandlers[delayType]; + if (!array) + break; + idx = array->getNextIndexOfObject(target, 0); + if (!enable) + { + if (-1U != idx) + { + array->removeObject(idx); + result = kIOReturnSuccess; + } + } + else + { + if (-1U != idx) { + result = kIOReturnExclusiveAccess; + break; + } + array->setObject(target); + + UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry); + CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy(); + UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction + IOService * holder = NULL; + + for (UInt i = 0; i < count; i++) { + if (entries[i].fService + && (delayType == entries[i].fDelayType) + && (entries[i].fMaxDelay < ns)) { + ns = entries[i].fMaxDelay; + holder = entries[i].fService; + } + } + target->callPlatformFunction(sCPULatencyFunctionName[delayType], false, + (void *) (uintptr_t) ns, holder, + NULL, NULL); + result = kIOReturnSuccess; } } + while (false); + + IORecursiveLockUnlock(sCpuDelayLock); + + return (result); +} + +#endif /* defined(__i386__) || defined(__x86_64__) */ + +void IOService:: +requireMaxBusStall(UInt32 __unused ns) +{ +#if defined(__i386__) || defined(__x86_64__) + requireMaxCpuDelay(this, ns, kCpuDelayBusStall); +#endif +} - IOLockUnlock(sBusStallLock); -#endif /* __i386__ */ +void IOService:: +requireMaxInterruptDelay(uint32_t __unused ns) +{ +#if defined(__i386__) || defined(__x86_64__) + requireMaxCpuDelay(this, ns, kCpuDelayInterrupt); +#endif } /* @@ -4563,13 +5516,21 @@ IOReturn IOService::causeInterrupt(int source) return interruptController->causeInterrupt(this, source); } +#if __LP64__ +OSMetaClassDefineReservedUnused(IOService, 0); +OSMetaClassDefineReservedUnused(IOService, 1); +OSMetaClassDefineReservedUnused(IOService, 2); +OSMetaClassDefineReservedUnused(IOService, 3); +OSMetaClassDefineReservedUnused(IOService, 4); +OSMetaClassDefineReservedUnused(IOService, 5); +#else OSMetaClassDefineReservedUsed(IOService, 0); OSMetaClassDefineReservedUsed(IOService, 1); OSMetaClassDefineReservedUsed(IOService, 2); OSMetaClassDefineReservedUsed(IOService, 3); - -OSMetaClassDefineReservedUnused(IOService, 4); -OSMetaClassDefineReservedUnused(IOService, 5); +OSMetaClassDefineReservedUsed(IOService, 4); +OSMetaClassDefineReservedUsed(IOService, 5); +#endif OSMetaClassDefineReservedUnused(IOService, 6); OSMetaClassDefineReservedUnused(IOService, 7); OSMetaClassDefineReservedUnused(IOService, 8); @@ -4612,22 +5573,3 @@ OSMetaClassDefineReservedUnused(IOService, 44); OSMetaClassDefineReservedUnused(IOService, 45); OSMetaClassDefineReservedUnused(IOService, 46); OSMetaClassDefineReservedUnused(IOService, 47); - -#ifdef __ppc__ -OSMetaClassDefineReservedUnused(IOService, 48); -OSMetaClassDefineReservedUnused(IOService, 49); -OSMetaClassDefineReservedUnused(IOService, 50); -OSMetaClassDefineReservedUnused(IOService, 51); -OSMetaClassDefineReservedUnused(IOService, 52); -OSMetaClassDefineReservedUnused(IOService, 53); -OSMetaClassDefineReservedUnused(IOService, 54); -OSMetaClassDefineReservedUnused(IOService, 55); -OSMetaClassDefineReservedUnused(IOService, 56); -OSMetaClassDefineReservedUnused(IOService, 57); -OSMetaClassDefineReservedUnused(IOService, 58); -OSMetaClassDefineReservedUnused(IOService, 59); -OSMetaClassDefineReservedUnused(IOService, 60); -OSMetaClassDefineReservedUnused(IOService, 61); -OSMetaClassDefineReservedUnused(IOService, 62); -OSMetaClassDefineReservedUnused(IOService, 63); -#endif