/*
* Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
*
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the
+ * License may not be used to create, or enable the creation or
+ * redistribution of, unlawful or unlicensed copies of an Apple operating
+ * system, or to circumvent, violate, or enable the circumvention or
+ * violation of, any terms of an Apple operating system software license
+ * agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
#include <ddb/db_print.h>
#endif /* MACH_KDB */
-#include <i386/machine_cpu.h>
+#ifdef __ppc__
+#include <ppc/Firmware.h>
+#endif
#include <sys/kdebug.h>
#define LCK_RW_LCK_SH_TO_EX1_CODE 0x104
#define LCK_RW_LCK_EX_TO_SH_CODE 0x105
-#define LCK_MTX_LCK_SPIN 0x200
#define ANY_LOCK_DEBUG (USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG)
lck_spin_try_lock(
lck_spin_t *lck)
{
- return(usimple_lock_try((usimple_lock_t) lck));
+ usimple_lock_try((usimple_lock_t) lck);
}
/*
lock_t *l,
boolean_t can_sleep,
__unused unsigned short tag,
- __unused unsigned short tag1)
+ unsigned short tag1)
{
- hw_lock_init(&l->interlock);
+ (void) memset((void *) l, 0, sizeof(lock_t));
+
+ simple_lock_init(&l->interlock, tag1);
l->want_write = FALSE;
l->want_upgrade = FALSE;
l->read_count = 0;
l->can_sleep = can_sleep;
- l->lck_rw_tag = tag;
}
lock_write(
register lock_t * l)
{
- lck_rw_lock_exclusive(l);
+ register int i;
+ boolean_t lock_miss = FALSE;
+#if MACH_LDEBUG
+ int decrementer;
+#endif /* MACH_LDEBUG */
+
+ simple_lock(&l->interlock);
+
+#if MACH_LDEBUG
+ decrementer = DECREMENTER_TIMEOUT;
+#endif /* MACH_LDEBUG */
+
+ /*
+ * Try to acquire the want_write bit.
+ */
+ while (l->want_write) {
+ if (!lock_miss) {
+ lock_miss = TRUE;
+ }
+
+ i = lock_wait_time[l->can_sleep ? 1 : 0];
+ if (i != 0) {
+ simple_unlock(&l->interlock);
+#if MACH_LDEBUG
+ if (!--decrementer)
+ Debugger("timeout - want_write");
+#endif /* MACH_LDEBUG */
+ while (--i != 0 && l->want_write)
+ continue;
+ simple_lock(&l->interlock);
+ }
+
+ if (l->can_sleep && l->want_write) {
+ l->waiting = TRUE;
+ thread_sleep_simple_lock((event_t) l,
+ simple_lock_addr(l->interlock),
+ THREAD_UNINT);
+ /* interlock relocked */
+ }
+ }
+ l->want_write = TRUE;
+
+ /* Wait for readers (and upgrades) to finish */
+
+#if MACH_LDEBUG
+ decrementer = DECREMENTER_TIMEOUT;
+#endif /* MACH_LDEBUG */
+ while ((l->read_count != 0) || l->want_upgrade) {
+ if (!lock_miss) {
+ lock_miss = TRUE;
+ }
+
+ i = lock_wait_time[l->can_sleep ? 1 : 0];
+ if (i != 0) {
+ simple_unlock(&l->interlock);
+#if MACH_LDEBUG
+ if (!--decrementer)
+ Debugger("timeout - wait for readers");
+#endif /* MACH_LDEBUG */
+ while (--i != 0 && (l->read_count != 0 ||
+ l->want_upgrade))
+ continue;
+ simple_lock(&l->interlock);
+ }
+
+ if (l->can_sleep && (l->read_count != 0 || l->want_upgrade)) {
+ l->waiting = TRUE;
+ thread_sleep_simple_lock((event_t) l,
+ simple_lock_addr(l->interlock),
+ THREAD_UNINT);
+ /* interlock relocked */
+ }
+ }
+
+ simple_unlock(&l->interlock);
}
void
lock_done(
register lock_t * l)
{
- (void) lck_rw_done(l);
+ boolean_t do_wakeup = FALSE;
+
+
+ simple_lock(&l->interlock);
+
+ if (l->read_count != 0) {
+ l->read_count--;
+ }
+ else
+ if (l->want_upgrade) {
+ l->want_upgrade = FALSE;
+ }
+ else {
+ l->want_write = FALSE;
+ }
+
+ /*
+ * There is no reason to wakeup a waiting thread
+ * if the read-count is non-zero. Consider:
+ * we must be dropping a read lock
+ * threads are waiting only if one wants a write lock
+ * if there are still readers, they can't proceed
+ */
+
+ if (l->waiting && (l->read_count == 0)) {
+ l->waiting = FALSE;
+ do_wakeup = TRUE;
+ }
+
+ simple_unlock(&l->interlock);
+
+ if (do_wakeup)
+ thread_wakeup((event_t) l);
}
void
lock_read(
register lock_t * l)
{
- lck_rw_lock_shared(l);
+ register int i;
+#if MACH_LDEBUG
+ int decrementer;
+#endif /* MACH_LDEBUG */
+
+ simple_lock(&l->interlock);
+
+#if MACH_LDEBUG
+ decrementer = DECREMENTER_TIMEOUT;
+#endif /* MACH_LDEBUG */
+ while (l->want_write || l->want_upgrade) {
+ i = lock_wait_time[l->can_sleep ? 1 : 0];
+
+ if (i != 0) {
+ simple_unlock(&l->interlock);
+#if MACH_LDEBUG
+ if (!--decrementer)
+ Debugger("timeout - wait no writers");
+#endif /* MACH_LDEBUG */
+ while (--i != 0 && (l->want_write || l->want_upgrade))
+ continue;
+ simple_lock(&l->interlock);
+ }
+
+ if (l->can_sleep && (l->want_write || l->want_upgrade)) {
+ l->waiting = TRUE;
+ thread_sleep_simple_lock((event_t) l,
+ simple_lock_addr(l->interlock),
+ THREAD_UNINT);
+ /* interlock relocked */
+ }
+ }
+
+ l->read_count++;
+
+ simple_unlock(&l->interlock);
}
lock_read_to_write(
register lock_t * l)
{
- return lck_rw_lock_shared_to_exclusive(l);
+ register int i;
+ boolean_t do_wakeup = FALSE;
+#if MACH_LDEBUG
+ int decrementer;
+#endif /* MACH_LDEBUG */
+
+ simple_lock(&l->interlock);
+
+ l->read_count--;
+
+ if (l->want_upgrade) {
+ /*
+ * Someone else has requested upgrade.
+ * Since we've released a read lock, wake
+ * him up.
+ */
+ if (l->waiting && (l->read_count == 0)) {
+ l->waiting = FALSE;
+ do_wakeup = TRUE;
+ }
+
+ simple_unlock(&l->interlock);
+
+ if (do_wakeup)
+ thread_wakeup((event_t) l);
+ return (TRUE);
+ }
+
+ l->want_upgrade = TRUE;
+
+#if MACH_LDEBUG
+ decrementer = DECREMENTER_TIMEOUT;
+#endif /* MACH_LDEBUG */
+ while (l->read_count != 0) {
+ i = lock_wait_time[l->can_sleep ? 1 : 0];
+
+ if (i != 0) {
+ simple_unlock(&l->interlock);
+#if MACH_LDEBUG
+ if (!--decrementer)
+ Debugger("timeout - read_count");
+#endif /* MACH_LDEBUG */
+ while (--i != 0 && l->read_count != 0)
+ continue;
+ simple_lock(&l->interlock);
+ }
+
+ if (l->can_sleep && l->read_count != 0) {
+ l->waiting = TRUE;
+ thread_sleep_simple_lock((event_t) l,
+ simple_lock_addr(l->interlock),
+ THREAD_UNINT);
+ /* interlock relocked */
+ }
+ }
+
+ simple_unlock(&l->interlock);
+
+ return (FALSE);
}
void
lock_write_to_read(
register lock_t * l)
{
- lck_rw_lock_exclusive_to_shared(l);
+ boolean_t do_wakeup = FALSE;
+
+ simple_lock(&l->interlock);
+
+ l->read_count++;
+ if (l->want_upgrade)
+ l->want_upgrade = FALSE;
+ else
+ l->want_write = FALSE;
+
+ if (l->waiting) {
+ l->waiting = FALSE;
+ do_wakeup = TRUE;
+ }
+
+ simple_unlock(&l->interlock);
+
+ if (do_wakeup)
+ thread_wakeup((event_t) l);
+}
+
+
+#if 0 /* Unused */
+/*
+ * Routine: lock_try_write
+ * Function:
+ * Tries to get a write lock.
+ *
+ * Returns FALSE if the lock is not held on return.
+ */
+
+boolean_t
+lock_try_write(
+ register lock_t * l)
+{
+ pc_t pc;
+
+ simple_lock(&l->interlock);
+
+ if (l->want_write || l->want_upgrade || l->read_count) {
+ /*
+ * Can't get lock.
+ */
+ simple_unlock(&l->interlock);
+ return(FALSE);
+ }
+
+ /*
+ * Have lock.
+ */
+
+ l->want_write = TRUE;
+
+ simple_unlock(&l->interlock);
+
+ return(TRUE);
}
+/*
+ * Routine: lock_try_read
+ * Function:
+ * Tries to get a read lock.
+ *
+ * Returns FALSE if the lock is not held on return.
+ */
+
+boolean_t
+lock_try_read(
+ register lock_t * l)
+{
+ pc_t pc;
+
+ simple_lock(&l->interlock);
+
+ if (l->want_write || l->want_upgrade) {
+ simple_unlock(&l->interlock);
+ return(FALSE);
+ }
+
+ l->read_count++;
+
+ simple_unlock(&l->interlock);
+
+ return(TRUE);
+}
+#endif /* Unused */
/*
lck_rw_init(
lck_rw_t *lck,
lck_grp_t *grp,
- lck_attr_t *attr)
-{
- lck_attr_t *lck_attr = (attr != LCK_ATTR_NULL) ?
- attr : &LockDefaultLckAttr;
+ __unused lck_attr_t *attr) {
hw_lock_init(&lck->interlock);
lck->want_write = FALSE;
lck->read_count = 0;
lck->can_sleep = TRUE;
lck->lck_rw_tag = 0;
- lck->read_priority = (lck_attr->lck_attr_val &
- LCK_ATTR_RW_SHARED_PRIORITY) != 0;
lck_grp_reference(grp);
lck_grp_lckcnt_incr(grp, LCK_TYPE_RW);
ml_set_interrupts_enabled(istate);
}
-
-/*
- * This inline is used when busy-waiting for an rw lock.
- * If interrupts were disabled when the lock primitive was called,
- * we poll the IPI handler for pending tlb flushes.
- * XXX This is a hack to avoid deadlocking on the pmap_system_lock.
- */
-static inline void
-lck_rw_lock_pause(boolean_t interrupts_enabled)
-{
- if (!interrupts_enabled)
- handle_pending_TLB_flushes();
- cpu_pause();
-}
-
/*
* Routine: lck_rw_lock_exclusive
*/
Debugger("timeout - want_write");
#endif /* MACH_LDEBUG */
while (--i != 0 && lck->want_write)
- lck_rw_lock_pause(istate);
+ continue;
istate = lck_interlock_lock(lck);
}
#endif /* MACH_LDEBUG */
while (--i != 0 && (lck->read_count != 0 ||
lck->want_upgrade))
- lck_rw_lock_pause(istate);
+ continue;
istate = lck_interlock_lock(lck);
}
#if MACH_LDEBUG
decrementer = DECREMENTER_TIMEOUT;
#endif /* MACH_LDEBUG */
- while ((lck->want_write && (lck->read_priority ?
- lck->read_count == 0 : TRUE)) ||
- lck->want_upgrade) {
-
+ while (lck->want_write || lck->want_upgrade) {
i = lock_wait_time[lck->can_sleep ? 1 : 0];
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
if (!--decrementer)
Debugger("timeout - wait no writers");
#endif /* MACH_LDEBUG */
- while (--i != 0 &&
- ((lck->want_write && (lck->read_priority ?
- lck->read_count == 0 : TRUE)) ||
- lck->want_upgrade))
- lck_rw_lock_pause(istate);
+ while (--i != 0 && (lck->want_write || lck->want_upgrade))
+ continue;
istate = lck_interlock_lock(lck);
}
- if (lck->can_sleep &&
- ((lck->want_write && (lck->read_priority ?
- lck->read_count == 0 : TRUE)) ||
- lck->want_upgrade)) {
+ if (lck->can_sleep && (lck->want_write || lck->want_upgrade)) {
lck->waiting = TRUE;
res = assert_wait((event_t) lck, THREAD_UNINT);
if (res == THREAD_WAITING) {
Debugger("timeout - read_count");
#endif /* MACH_LDEBUG */
while (--i != 0 && lck->read_count != 0)
- lck_rw_lock_pause(istate);
+ continue;
istate = lck_interlock_lock(lck);
}
{
}
-/*
- * Routine: lck_mtx_lock_spin
- *
- * Invoked trying to acquire a mutex when there is contention but
- * the holder is running on another processor. We spin for up to a maximum
- * time waiting for the lock to be released.
- *
- * Called with the interlock unlocked.
- */
-void
-lck_mtx_lock_spin(
- lck_mtx_t *lck)
-{
- thread_t holder;
- lck_mtx_t *mutex;
- uint64_t deadline;
-
- if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
- mutex = lck;
- else
- mutex = &lck->lck_mtx_ptr->lck_mtx;
-
- KERNEL_DEBUG(
- MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_SPIN) | DBG_FUNC_START,
- (int)lck, (int)mutex->lck_mtx_locked, 0, 0, 0);
-
- deadline = mach_absolute_time() + MutexSpin;
- /*
- * Spin while:
- * - mutex is locked, and
- * - owner is running on another processor, and
- * - owner is not is the idle delay, and
- * - we haven't spun for long enough.
- */
- while ((holder = (thread_t) mutex->lck_mtx_locked) != NULL &&
- (holder->machine.specFlags & OnProc) != 0 &&
- (holder->options & TH_OPT_DELAYIDLE) == 0 &&
- mach_absolute_time() < deadline)
- cpu_pause();
-}
-
-/*
- * Called from assembly code when a mutex interlock is held.
- * We spin here re-checking the interlock but panic if we timeout.
- * Note: here with interrupts disabled.
- */
-void
-lck_mtx_interlock_spin(
- lck_mtx_t *lck)
-{
- lck_mtx_t *mutex;
- uint64_t deadline;
-
- if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
- mutex = lck;
- else
- mutex = &lck->lck_mtx_ptr->lck_mtx;
-
- deadline = mach_absolute_time() + LockTimeOut;
- while (mutex->lck_mtx_ilk != 0) {
- cpu_pause();
- if (mach_absolute_time() > deadline)
- panic("interlock timeout for mutex %p", lck);
- }
-
-}
-
#if MACH_KDB
void db_show_one_lock(lock_t *);
db_printf("%swaiting, %scan_sleep\n",
lock->waiting ? "" : "!", lock->can_sleep ? "" : "!");
db_printf("Interlock:\n");
- db_show_one_simple_lock((db_expr_t) ((vm_offset_t)simple_lock_addr(lock->interlock)),
+ db_show_one_simple_lock((db_expr_t)simple_lock_addr(lock->interlock),
TRUE, (db_expr_t)0, (char *)0);
}
* fashion.
*/
-const char *simple_lock_labels = "ENTRY ILK THREAD DURATION CALLER";
-const char *mutex_labels = "ENTRY LOCKED WAITERS THREAD CALLER";
+char *simple_lock_labels = "ENTRY ILK THREAD DURATION CALLER";
+char *mutex_labels = "ENTRY LOCKED WAITERS THREAD CALLER";
void
db_show_one_simple_lock (
db_expr_t addr,
boolean_t have_addr,
- __unused db_expr_t count,
- __unused char * modif)
+ db_expr_t count,
+ char * modif)
{
- simple_lock_t saddr = (simple_lock_t) ((vm_offset_t) addr);
+ simple_lock_t saddr = (simple_lock_t)addr;
if (saddr == (simple_lock_t)0 || !have_addr) {
db_error ("No simple_lock\n");
db_show_one_mutex (
db_expr_t addr,
boolean_t have_addr,
- __unused db_expr_t count,
- __unused char * modif)
+ db_expr_t count,
+ char * modif)
{
- mutex_t * maddr = (mutex_t *)((vm_offset_t) addr);
+ mutex_t * maddr = (mutex_t *)addr;
if (maddr == (mutex_t *)0 || !have_addr)
db_error ("No mutex\n");
projects / apple / xnu.git / blobdiff