X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..55e303ae13a4cf49d70f2294092726f2fffb9ef2:/osfmk/kern/lock.c?ds=sidebyside diff --git a/osfmk/kern/lock.c b/osfmk/kern/lock.c index 383bf774b..ef600c40f 100644 --- a/osfmk/kern/lock.c +++ b/osfmk/kern/lock.c @@ -3,19 +3,22 @@ * * @APPLE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ */ @@ -63,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -77,9 +81,10 @@ #ifdef __ppc__ #include -#include #endif +#include + #define ANY_LOCK_DEBUG (USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG) /* @@ -210,9 +215,13 @@ usimple_lock_init( usimple_lock_t l, etap_event_t event) { +#ifndef MACHINE_SIMPLE_LOCK USLDBG(usld_lock_init(l, event)); ETAPCALL(etap_simplelock_init((l),(event))); hw_lock_init(&l->interlock); +#else + simple_lock_init((simple_lock_t)l,event); +#endif } @@ -227,6 +236,7 @@ void usimple_lock( usimple_lock_t l) { +#ifndef MACHINE_SIMPLE_LOCK int i; pc_t pc; #if ETAP_LOCK_TRACE @@ -243,45 +253,14 @@ usimple_lock( ETAP_TIME_CLEAR(start_wait_time); #endif /* ETAP_LOCK_TRACE */ -#ifdef __ppc__ - if(!hw_lock_to(&l->interlock, LockTimeOut)) { /* Try to get the lock with a timeout */ - + if(!hw_lock_to(&l->interlock, LockTimeOut)) /* Try to get the lock with a timeout */ panic("simple lock deadlock detection - l=%08X, cpu=%d, ret=%08X", l, cpu_number(), pc); -#else /* __ppc__ */ - while (!hw_lock_try(&l->interlock)) { - ETAPCALL(if (no_miss_info++ == 0) - start_wait_time = etap_simplelock_miss(l)); - while (hw_lock_held(&l->interlock)) { - /* - * Spin watching the lock value in cache, - * without consuming external bus cycles. - * On most SMP architectures, the atomic - * instruction(s) used by hw_lock_try - * cost much, much more than an ordinary - * memory read. - */ -#if USLOCK_DEBUG - if (count++ > max_lock_loops -#if MACH_KDB && NCPUS > 1 - && l != &kdb_lock -#endif /* MACH_KDB && NCPUS > 1 */ - ) { - if (l == &printf_lock) { - return; - } - mp_disable_preemption(); - panic("simple lock deadlock detection - l=%08X (=%08X), cpu=%d, ret=%08X", - l, *hw_lock_addr(l->interlock), cpu_number(), pc); - count = 0; - mp_enable_preemption(); - } -#endif /* USLOCK_DEBUG */ - } -#endif /* 0 */ - } ETAPCALL(etap_simplelock_hold(l, pc, start_wait_time)); USLDBG(usld_lock_post(l, pc)); +#else + simple_lock((simple_lock_t)l); +#endif } @@ -296,6 +275,7 @@ void usimple_unlock( usimple_lock_t l) { +#ifndef MACHINE_SIMPLE_LOCK pc_t pc; // checkNMI(); /* (TEST/DEBUG) */ @@ -303,7 +283,13 @@ usimple_unlock( OBTAIN_PC(pc, l); USLDBG(usld_unlock(l, pc)); ETAPCALL(etap_simplelock_unlock(l)); +#ifdef __ppc__ + sync(); +#endif hw_lock_unlock(&l->interlock); +#else + simple_unlock_rwmb((simple_lock_t)l); +#endif } @@ -323,6 +309,7 @@ unsigned int usimple_lock_try( usimple_lock_t l) { +#ifndef MACHINE_SIMPLE_LOCK pc_t pc; unsigned int success; etap_time_t zero_time; @@ -335,6 +322,9 @@ usimple_lock_try( ETAPCALL(etap_simplelock_hold(l, pc, zero_time)); } return success; +#else + return(simple_lock_try((simple_lock_t)l)); +#endif } #if ETAP_LOCK_TRACE @@ -1084,8 +1074,9 @@ lock_write( ETAP_SET_REASON(current_thread(), BLOCKED_ON_COMPLEX_LOCK); thread_sleep_simple_lock((event_t) l, - simple_lock_addr(l->interlock), FALSE); - simple_lock(&l->interlock); + simple_lock_addr(l->interlock), + THREAD_UNINT); + /* interlock relocked */ } } l->want_write = TRUE; @@ -1119,8 +1110,9 @@ lock_write( ETAP_SET_REASON(current_thread(), BLOCKED_ON_COMPLEX_LOCK); thread_sleep_simple_lock((event_t) l, - simple_lock_addr(l->interlock), FALSE); - simple_lock(&l->interlock); + simple_lock_addr(l->interlock), + THREAD_UNINT); + /* interlock relocked */ } } @@ -1303,8 +1295,9 @@ lock_read( if (l->can_sleep && (l->want_write || l->want_upgrade)) { l->waiting = TRUE; thread_sleep_simple_lock((event_t) l, - simple_lock_addr(l->interlock), FALSE); - simple_lock(&l->interlock); + simple_lock_addr(l->interlock), + THREAD_UNINT); + /* interlock relocked */ } } @@ -1454,8 +1447,9 @@ lock_read_to_write( if (l->can_sleep && l->read_count != 0) { l->waiting = TRUE; thread_sleep_simple_lock((event_t) l, - simple_lock_addr(l->interlock), FALSE); - simple_lock(&l->interlock); + simple_lock_addr(l->interlock), + THREAD_UNINT); + /* interlock relocked */ } } @@ -1715,35 +1709,245 @@ mutex_free( kfree((vm_offset_t)m, sizeof(mutex_t)); } - /* - * mutex_lock_wait: Invoked if the assembler routine mutex_lock () fails - * because the mutex is already held by another thread. Called with the - * interlock locked and returns with the interlock unlocked. + * mutex_lock_wait + * + * Invoked in order to wait on contention. + * + * Called with the interlock locked and + * returns it unlocked. */ - void mutex_lock_wait ( - mutex_t * m) + mutex_t *mutex, + thread_t holder) { - m->waiters++; - ETAP_SET_REASON(current_thread(), BLOCKED_ON_MUTEX_LOCK); - thread_sleep_interlock ((event_t) m, &m->interlock, THREAD_UNINT); + thread_t self = current_thread(); +#if !defined(i386) + integer_t priority; + spl_t s = splsched(); + + priority = self->sched_pri; + if (priority < self->priority) + priority = self->priority; + if (priority > MINPRI_KERNEL) + priority = MINPRI_KERNEL; + else + if (priority < BASEPRI_DEFAULT) + priority = BASEPRI_DEFAULT; + + assert(holder->thread == holder); /* XXX */ + thread_lock(holder); + if (mutex->promoted_pri == 0) + holder->promotions++; + if (holder->priority < MINPRI_KERNEL) { + holder->sched_mode |= TH_MODE_PROMOTED; + if ( mutex->promoted_pri < priority && + holder->sched_pri < priority ) { + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, + holder->sched_pri, priority, (int)holder, (int)mutex, 0); + + set_sched_pri(holder, priority); + } + } + thread_unlock(holder); + splx(s); + + if (mutex->promoted_pri < priority) + mutex->promoted_pri = priority; +#endif + + if (self->pending_promoter[self->pending_promoter_index] == NULL) { + self->pending_promoter[self->pending_promoter_index] = mutex; + mutex->waiters++; + } + else + if (self->pending_promoter[self->pending_promoter_index] != mutex) { + self->pending_promoter[++self->pending_promoter_index] = mutex; + mutex->waiters++; + } + + assert_wait(mutex, THREAD_UNINT); + interlock_unlock(&mutex->interlock); + + thread_block(THREAD_CONTINUE_NULL); } /* - * mutex_unlock_wakeup: Invoked if the assembler routine mutex_unlock () - * fails because there are thread(s) waiting for this mutex. Called and - * returns with the interlock locked. + * mutex_lock_acquire + * + * Invoked on acquiring the mutex when there is + * contention. + * + * Returns the current number of waiters. + * + * Called with the interlock locked. */ +int +mutex_lock_acquire( + mutex_t *mutex) +{ + thread_t thread = current_thread(); + + if (thread->pending_promoter[thread->pending_promoter_index] == mutex) { + thread->pending_promoter[thread->pending_promoter_index] = NULL; + if (thread->pending_promoter_index > 0) + thread->pending_promoter_index--; + mutex->waiters--; + } + +#if !defined(i386) + if (mutex->waiters > 0) { + integer_t priority = mutex->promoted_pri; + spl_t s = splsched(); + + thread_lock(thread); + thread->promotions++; + if (thread->priority < MINPRI_KERNEL) { + thread->sched_mode |= TH_MODE_PROMOTED; + if (thread->sched_pri < priority) { + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, + thread->sched_pri, priority, 0, (int)mutex, 0); + + set_sched_pri(thread, priority); + } + } + thread_unlock(thread); + splx(s); + } + else + mutex->promoted_pri = 0; +#endif + + return (mutex->waiters); +} +/* + * mutex_unlock_wakeup + * + * Invoked on unlock when there is contention. + * + * Called with the interlock locked. + */ void mutex_unlock_wakeup ( - mutex_t * m) + mutex_t *mutex, + thread_t holder) +{ +#if !defined(i386) + thread_t thread = current_thread(); + + if (thread->top_act != holder) + panic("mutex_unlock_wakeup: mutex %x holder %x\n", mutex, holder); + + if (thread->promotions > 0) { + spl_t s = splsched(); + + thread_lock(thread); + if ( --thread->promotions == 0 && + (thread->sched_mode & TH_MODE_PROMOTED) ) { + thread->sched_mode &= ~TH_MODE_PROMOTED; + if (thread->sched_mode & TH_MODE_ISDEPRESSED) { + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | DBG_FUNC_NONE, + thread->sched_pri, DEPRESSPRI, 0, (int)mutex, 0); + + set_sched_pri(thread, DEPRESSPRI); + } + else { + if (thread->priority < thread->sched_pri) { + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | + DBG_FUNC_NONE, + thread->sched_pri, thread->priority, + 0, (int)mutex, 0); + } + + compute_priority(thread, FALSE); + } + } + thread_unlock(thread); + splx(s); + } +#endif + + assert(mutex->waiters > 0); + thread_wakeup_one(mutex); +} + +boolean_t +mutex_preblock_wait( + mutex_t *mutex, + thread_t thread, + thread_t holder) { - assert(m->waiters); - m->waiters--; - thread_wakeup_one ((event_t) m); + wait_result_t wresult; + integer_t priority; + wait_queue_t wq; + + assert(holder == NULL || holder->thread == holder); + + wq = wait_event_wait_queue((event_t)mutex); + if (!wait_queue_lock_try(wq)) + return (FALSE); + + if (holder != NULL && !thread_lock_try(holder)) { + wait_queue_unlock(wq); + return (FALSE); + } + + wresult = wait_queue_assert_wait64_locked(wq, (uint32_t)mutex, + THREAD_UNINT, thread); + wait_queue_unlock(wq); + assert(wresult == THREAD_WAITING); + + priority = thread->sched_pri; + if (priority < thread->priority) + priority = thread->priority; + if (priority > MINPRI_KERNEL) + priority = MINPRI_KERNEL; + else + if (priority < BASEPRI_DEFAULT) + priority = BASEPRI_DEFAULT; + + if (holder != NULL) { + if (mutex->promoted_pri == 0) + holder->promotions++; + if (holder->priority < MINPRI_KERNEL) { + holder->sched_mode |= TH_MODE_PROMOTED; + if ( mutex->promoted_pri < priority && + holder->sched_pri < priority ) { + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, + holder->sched_pri, priority, + (int)holder, (int)mutex, 0); + + set_sched_pri(holder, priority); + } + } + thread_unlock(holder); + } + + if (mutex->promoted_pri < priority) + mutex->promoted_pri = priority; + + if (thread->pending_promoter[thread->pending_promoter_index] == NULL) { + thread->pending_promoter[thread->pending_promoter_index] = mutex; + mutex->waiters++; + } + else + if (thread->pending_promoter[thread->pending_promoter_index] != mutex) { + thread->pending_promoter[++thread->pending_promoter_index] = mutex; + mutex->waiters++; + } + + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_PREBLOCK_MUTEX) | DBG_FUNC_NONE, + (int)thread, thread->sched_pri, (int)mutex, 0, 0); + + return (TRUE); } /* @@ -1753,13 +1957,15 @@ mutex_unlock_wakeup ( void mutex_pause(void) { - int wait_result; + wait_result_t wait_result; + + wait_result = assert_wait_timeout( 1, THREAD_UNINT); + assert(wait_result == THREAD_WAITING); - assert_wait_timeout( 1, THREAD_INTERRUPTIBLE); ETAP_SET_REASON(current_thread(), BLOCKED_ON_MUTEX_LOCK); - wait_result = thread_block((void (*)(void))0); - if (wait_result != THREAD_TIMED_OUT) - thread_cancel_timer(); + + wait_result = thread_block(THREAD_CONTINUE_NULL); + assert(wait_result == THREAD_TIMED_OUT); } #if MACH_KDB