+#if MACH_LDEBUG
+ movl %ecx,M_THREAD
+ movl B_PC,%ecx
+ movl %ecx,M_PC
+#endif
+ cmpw $0,M_WAITERS /* are there any waiters? */
+ jne Lmt_waiters /* yes, more work to do */
+Lmt_return:
+ xorl %eax,%eax
+ movl %eax,M_ILK
+ popf /* restore interrupt state */
+
+ movl $1,%eax
+ leave
+#if CONFIG_DTRACE
+ LOCKSTAT_LABEL(_mutex_try_lockstat_patch_point)
+ ret
+ /* inherit the lock pointer in %edx from above */
+ LOCKSTAT_RECORD(LS_MUTEX_TRY_LOCK_ACQUIRE, %edx)
+ movl $1,%eax
+#endif
+ ret
+
+Lmt_waiters:
+ pushl %edx /* save mutex address */
+ pushl %edx
+ call EXT(lck_mtx_lock_acquire)
+ addl $4,%esp
+ popl %edx /* restore mutex address */
+ jmp Lmt_return
+
+Lmt_ilk_fail:
+ popf /* restore interrupt state */
+ pushf /* resave interrupt state on stack */
+
+Lmt_ilk_loop:
+ PAUSE
+ /*
+ * need to do this check outside of the interlock in
+ * case this lock is held as a simple lock which means
+ * we won't be able to take the interlock
+ */
+ movl M_LOCKED,%eax /* get lock owner */
+ testl %eax,%eax /* is the mutex locked? */
+ jne Lmt_fail_no_ilk /* yes, go return failure */
+
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ je Lmt_retry /* yes, go try to grab it */
+ jmp Lmt_ilk_loop /* no - keep spinning */
+
+Lmt_fail:
+ xorl %eax,%eax
+ movl %eax,M_ILK
+
+Lmt_fail_no_ilk:
+ xorl %eax,%eax
+ popf /* restore interrupt state */
+ NONLEAF_RET
+
+
+
+LEAF_ENTRY(mutex_convert_spin)
+ movl L_ARG0,%edx /* fetch lock pointer */
+
+ movl M_LOCKED,%ecx /* is this the spin variant of the mutex */
+ cmpl $(MUTEX_LOCKED_AS_SPIN),%ecx
+ jne Lmcs_exit /* already owned as a mutex, just return */
+
+ movl M_ILK,%ecx /* convert from spin version to mutex */
+ movl %ecx,M_LOCKED /* take control of the mutex */
+
+ cmpw $0,M_WAITERS /* are there any waiters? */
+ jne Lmcs_waiters /* yes, more work to do */
+
+Lmcs_return:
+ xorl %ecx,%ecx
+ movl %ecx,M_ILK /* clear interlock */
+ PREEMPTION_ENABLE
+Lmcs_exit:
+#if CONFIG_DTRACE
+ LOCKSTAT_LABEL(_mutex_convert_spin_lockstat_patch_point)
+ ret
+ /* inherit %edx from above */
+ LOCKSTAT_RECORD(LS_MUTEX_CONVERT_SPIN_ACQUIRE, %edx)
+#endif
+ ret
+
+
+Lmcs_waiters:
+ pushl %edx /* save mutex address */
+ pushl %edx
+ call EXT(lck_mtx_lock_acquire)
+ addl $4,%esp
+ popl %edx /* restore mutex address */
+ jmp Lmcs_return
+
+
+
+NONLEAF_ENTRY(mutex_unlock)
+ movl B_ARG0,%edx /* fetch lock pointer */
+
+ movl M_LOCKED,%ecx /* is this the spin variant of the mutex */
+ cmpl $(MUTEX_LOCKED_AS_SPIN),%ecx
+ jne Lmu_enter /* no, go treat like a real mutex */
+
+ cmpw $0,M_WAITERS /* are there any waiters? */
+ jne Lmus_wakeup /* yes, more work to do */
+
+Lmus_drop_ilk:
+ xorl %ecx,%ecx
+ movl %ecx,M_LOCKED /* yes, clear the spin indicator */
+ movl %ecx,M_ILK /* release the interlock */
+ PREEMPTION_ENABLE /* and re-enable preemption */
+ leave
+#if CONFIG_DTRACE
+ LOCKSTAT_LABEL(_mutex_unlock_lockstat_patch_point)
+ ret
+ /* inherit lock pointer in %edx from above */
+ LOCKSTAT_RECORD(LS_MUTEX_UNLOCK_RELEASE, %edx)
+#endif
+ ret
+
+Lmus_wakeup:
+ pushl %edx /* save mutex address */
+ pushl %edx /* push mutex address */
+ call EXT(lck_mtx_unlockspin_wakeup) /* yes, wake a thread */
+ addl $4,%esp
+ popl %edx /* restore mutex pointer */
+ jmp Lmus_drop_ilk
+
+Lmu_enter:
+ pushf /* save interrupt state */
+
+ CHECK_MUTEX_TYPE()
+ CHECK_THREAD(M_THREAD)
+
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ jne Lmu_ilk_loop /* yes, go try to grab it */
+Lmu_retry:
+ cli /* disable interrupts */
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+
+ /* eax == 0 at this point */
+ lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */
+ jne Lmu_ilk_fail /* branch on failure to spin loop */
+
+ cmpw $0,M_WAITERS /* are there any waiters? */
+ jne Lmu_wakeup /* yes, more work to do */
+
+Lmu_doit:
+#if MACH_LDEBUG
+ movl $0,M_THREAD /* disown thread */
+#endif
+ xorl %ecx,%ecx
+ movl %ecx,M_LOCKED /* unlock the mutex */
+ movl %ecx,M_ILK /* release the interlock */
+ popf /* restore interrupt state */
+ leave
+#if CONFIG_DTRACE
+ LOCKSTAT_LABEL(_mutex_unlock2_lockstat_patch_point)
+ ret
+ /* inherit %edx from above */
+ LOCKSTAT_RECORD(LS_MUTEX_UNLOCK_RELEASE, %edx)
+#endif
+ ret
+
+Lmu_ilk_fail:
+ popf /* restore interrupt state */
+ pushf /* resave interrupt state on stack */
+
+Lmu_ilk_loop:
+ PAUSE
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ je Lmu_retry /* yes, go try to grab it */
+ jmp Lmu_ilk_loop /* no - keep spinning */
+
+Lmu_wakeup:
+ pushl M_LOCKED
+ pushl %edx /* push mutex address */
+ call EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
+ addl $8,%esp
+ movl B_ARG0,%edx /* restore lock pointer */
+ jmp Lmu_doit
+
+/*
+ * void lck_mtx_assert(lck_mtx_t* l, unsigned int)
+ * void _mutex_assert(mutex_t, unsigned int)
+ * Takes the address of a lock, and an assertion type as parameters.
+ * The assertion can take one of two forms determine by the type
+ * parameter: either the lock is held by the current thread, and the
+ * type is LCK_MTX_ASSERT_OWNED, or it isn't and the type is
+ * LCK_MTX_ASSERT_NOT_OWNED. Calls panic on assertion failure.
+ *
+ */
+
+Entry(lck_mtx_assert)
+Entry(_mutex_assert)
+ movl S_ARG0,%edx /* Load lock address */
+ movl %gs:CPU_ACTIVE_THREAD,%ecx /* Load current thread */
+
+ cmpl $(MUTEX_IND),M_ITAG /* Is this an indirect mutex? */
+ cmove M_PTR,%edx /* If so, take indirection */
+
+ movl M_LOCKED,%eax /* Load lock word */
+ cmpl $(MUTEX_LOCKED_AS_SPIN),%eax /* check for spin variant */
+ cmove M_ILK,%eax /* yes, spin lock owner is in the interlock */
+
+ cmpl $(MUTEX_ASSERT_OWNED),S_ARG1 /* Determine assert type */
+ jne 2f /* Assert ownership? */
+ cmpl %eax,%ecx /* Current thread match? */
+ jne 3f /* no, go panic */
+1: /* yes, we own it */
+ ret /* just return */
+2:
+ cmpl %eax,%ecx /* Current thread match? */
+ jne 1b /* No, return */
+ movl %edx,S_ARG1 /* Prep assertion failure */
+ movl $(mutex_assert_owned_str),S_ARG0
+ jmp 4f
+3:
+ movl %edx,S_ARG1 /* Prep assertion failure */
+ movl $(mutex_assert_not_owned_str),S_ARG0
+4:
+ jmp EXT(panic)
+
+.data
+mutex_assert_not_owned_str:
+ .asciz "mutex (%p) not owned\n"
+mutex_assert_owned_str:
+ .asciz "mutex (%p) owned\n"
+.text
+
+/* This preprocessor define controls whether the R-M-W update of the
+ * per-group statistics elements are atomic (LOCK-prefixed)
+ * Enabled by default.
+ */
+#define ATOMIC_STAT_UPDATES 1
+
+#if defined(ATOMIC_STAT_UPDATES)
+#define LOCK_IF_ATOMIC_STAT_UPDATES lock
+#else
+#define LOCK_IF_ATOMIC_STAT_UPDATES
+#endif /* ATOMIC_STAT_UPDATES */
+
+
+/*
+ * lck_mtx_lock()
+ * lck_mtx_try_lock()
+ * lck_mutex_unlock()
+ * lck_mtx_lock_spin()
+ * lck_mtx_convert_spin()
+ *
+ * These are variants of mutex_lock(), mutex_try(), mutex_unlock()
+ * mutex_lock_spin and mutex_convert_spin without
+ * DEBUG checks (which require fields not present in lck_mtx_t's).
+ */
+
+NONLEAF_ENTRY(lck_mtx_lock_spin)
+
+ movl B_ARG0,%edx /* fetch lock pointer */
+ pushf /* save interrupt state */
+
+ CHECK_NO_SIMPLELOCKS()
+ CHECK_PREEMPTION_LEVEL()
+
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ jne Llmls_eval_ilk /* no, go see if indirect */
+Llmls_retry:
+ cli /* disable interrupts */
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+
+ /* eax == 0 at this point */
+ lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */
+ jne Llmls_ilk_fail /* branch on failure to spin loop */
+
+ movl M_LOCKED,%ecx /* get lock owner */
+ testl %ecx,%ecx /* is the mutex locked? */
+ jne Llml_fail /* yes, fall back to a normal mutex */
+
+Llmls_acquire:
+ movl $(MUTEX_LOCKED_AS_SPIN),M_LOCKED /* indicate ownership as a spin lock */
+ PREEMPTION_DISABLE
+ popf /* restore interrupt state */
+ NONLEAF_RET /* return with the interlock held */
+
+Llmls_ilk_fail:
+ popf /* restore interrupt state */
+ pushf /* resave interrupt state on stack */
+
+Llmls_ilk_loop:
+ PAUSE
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ je Llmls_retry /* yes - go try to grab it */
+
+ cmpl $(MUTEX_DESTROYED),%eax /* check to see if its marked destroyed */
+ jne Llmls_ilk_loop /* no - keep spinning */
+
+ pushl %edx
+ call EXT(lck_mtx_interlock_panic)
+ /*
+ * shouldn't return from here, but just in case
+ */
+ popl %edx
+ jmp Llmls_ilk_loop
+
+
+Llmls_eval_ilk:
+ cmpl $(MUTEX_IND),M_ITAG /* Is this an indirect mutex? */
+ cmove M_PTR,%edx /* If so, take indirection */
+ jne Llmls_ilk_loop /* If not, go to spin loop */
+
+Llmls_lck_ext:
+ pushl %esi /* Used to hold the lock group ptr */
+ pushl %edi /* Used for stat update records */
+ movl MUTEX_GRP(%edx),%esi /* Load lock group */
+ xorl %edi,%edi /* Clear stat update records */
+ /* 64-bit increment of acquire attempt statistic (per-group) */
+ LOCK_IF_ATOMIC_STAT_UPDATES
+ addl $1, GRP_MTX_STAT_UTIL(%esi)
+ jnc 1f
+ incl GRP_MTX_STAT_UTIL+4(%esi)
+1:
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ jne Llmls_ext_ilk_loop /* no, go to spin loop */
+Llmls_ext_retry:
+ cli /* disable interrupts */
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+
+ /* eax == 0 at this point */
+ lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */
+ jne Llmls_ext_ilk_fail /* branch on failure to retry */
+
+ movl M_LOCKED,%ecx /* get lock owner */
+ testl %ecx,%ecx /* is the mutex locked? */
+ jne Llml_ext_fail /* yes, we lose */
+
+ popl %edi
+ popl %esi
+ jmp Llmls_acquire
+
+Llmls_ext_ilk_fail:
+ /*
+ * Slow path: call out to do the spinning.
+ */
+ movl 8(%esp),%ecx
+ pushl %ecx
+ popf /* restore interrupt state */
+
+Llmls_ext_ilk_loop:
+ PAUSE
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ je Llmls_ext_retry /* yes - go try to grab it */
+
+ cmpl $(MUTEX_DESTROYED),%eax /* check to see if its marked destroyed */
+ jne Llmls_ext_ilk_loop /* no - keep spinning */
+
+ pushl %edx
+ call EXT(lck_mtx_interlock_panic)
+ /*
+ * shouldn't return from here, but just in case
+ */
+ popl %edx
+ jmp Llmls_ext_ilk_loop /* no - keep spinning */
+
+
+
+NONLEAF_ENTRY(lck_mtx_lock)
+
+ movl B_ARG0,%edx /* fetch lock pointer */
+ pushf /* save interrupt state */
+
+ CHECK_NO_SIMPLELOCKS()
+ CHECK_PREEMPTION_LEVEL()
+
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ jne Llml_eval_ilk /* no, go see if indirect */
+Llml_retry:
+ cli /* disable interrupts */
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+
+ /* eax == 0 at this point */
+ lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */
+ jne Llml_ilk_fail /* branch on failure to spin loop */
+
+ movl M_LOCKED,%ecx /* get lock owner */
+ testl %ecx,%ecx /* is the mutex locked? */
+ jne Llml_fail /* yes, we lose */
+Llml_acquire:
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+ movl %ecx,M_LOCKED
+
+ cmpw $0,M_WAITERS /* are there any waiters? */
+ jne Lml_waiters /* yes, more work to do */
+Llml_return:
+ xorl %eax,%eax
+ movl %eax,M_ILK
+
+ popf /* restore interrupt state */
+ leave
+#if CONFIG_DTRACE
+ LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
+ ret
+ /* inherit lock pointer in %edx above */
+ LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, %edx)
+#endif
+ ret
+
+Llml_waiters:
+ pushl %edx /* save mutex address */
+ pushl %edx
+ call EXT(lck_mtx_lock_acquire)
+ addl $4,%esp
+ popl %edx /* restore mutex address */
+ jmp Llml_return
+
+Llml_restart:
+Llml_ilk_fail:
+ popf /* restore interrupt state */
+ pushf /* resave interrupt state on stack */
+
+Llml_ilk_loop:
+ PAUSE
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ je Llml_retry /* yes - go try to grab it */
+
+ cmpl $(MUTEX_DESTROYED),%eax /* check to see if its marked destroyed */
+ jne Llml_ilk_loop /* no - keep spinning */
+
+ pushl %edx
+ call EXT(lck_mtx_interlock_panic)
+ /*
+ * shouldn't return from here, but just in case
+ */
+ popl %edx
+ jmp Llml_ilk_loop /* no - keep spinning */
+
+Llml_fail:
+ /*
+ * Check if the owner is on another processor and therefore
+ * we should try to spin before blocking.
+ */
+ testl $(OnProc),ACT_SPF(%ecx)
+ jz Llml_block
+
+ /*
+ * Here if owner is on another processor:
+ * - release the interlock
+ * - spin on the holder until release or timeout
+ * - in either case re-acquire the interlock
+ * - if released, acquire it
+ * - otherwise drop thru to block.
+ */
+ xorl %eax,%eax
+ movl %eax,M_ILK /* zero interlock */
+ popf
+ pushf /* restore interrupt state */
+ pushl %edx /* save mutex address */
+ pushl %edx
+ call EXT(lck_mtx_lock_spinwait)
+ addl $4,%esp
+ popl %edx /* restore mutex address */
+
+ /* Re-acquire interlock */
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ jne Llml_ilk_refail /* no, go to spin loop */
+Llml_reget_retry:
+ cli /* disable interrupts */
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+
+ /* eax == 0 at this point */
+ lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */
+ jne Llml_ilk_refail /* branch on failure to retry */
+
+ movl M_LOCKED,%ecx /* get lock owner */
+ testl %ecx,%ecx /* is the mutex free? */
+ je Llml_acquire /* yes, acquire */
+
+Llml_block:
+ CHECK_MYLOCK(M_THREAD)
+ pushl %edx /* save mutex address */
+ pushl M_LOCKED
+ pushl %edx /* push mutex address */
+ /*
+ * N.B.: lck_mtx_lock_wait is called here with interrupts disabled
+ * Consider reworking.
+ */
+ call EXT(lck_mtx_lock_wait) /* wait for the lock */
+ addl $8,%esp
+ popl %edx /* restore mutex address */
+ jmp Llml_restart /* and start over */
+
+Llml_ilk_refail:
+ popf /* restore interrupt state */
+ pushf /* resave interrupt state on stack */
+
+Llml_ilk_reloop:
+ PAUSE
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ je Llml_reget_retry /* yes - go try to grab it */
+
+ cmpl $(MUTEX_DESTROYED),%eax /* check to see if its marked destroyed */
+ jne Llml_ilk_reloop /* no - keep spinning */
+
+ pushl %edx
+ call EXT(lck_mtx_interlock_panic)
+ /*
+ * shouldn't return from here, but just in case
+ */
+ popl %edx
+ jmp Llml_ilk_reloop /* no - keep spinning */
+
+
+Llml_eval_ilk:
+ cmpl $(MUTEX_IND),M_ITAG /* Is this an indirect mutex? */
+ cmove M_PTR,%edx /* If so, take indirection */
+ jne Llml_ilk_loop /* If not, go to spin loop */
+
+/*
+ * Entry into statistics codepath for lck_mtx_lock:
+ * EDX: real lock pointer
+ * first dword on stack contains flags
+ */
+
+/* Enable this preprocessor define to record the first miss alone
+ * By default, we count every miss, hence multiple misses may be
+ * recorded for a single lock acquire attempt via lck_mtx_lock
+ */
+#undef LOG_FIRST_MISS_ALONE
+
+/*
+ * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
+ * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
+ * as a 64-bit quantity (this matches the existing PowerPC implementation,
+ * and the new x86 specific statistics are also maintained as 32-bit
+ * quantities).
+ */
+
+Llml_lck_ext:
+ pushl %esi /* Used to hold the lock group ptr */
+ pushl %edi /* Used for stat update records */
+ movl MUTEX_GRP(%edx),%esi /* Load lock group */
+ xorl %edi,%edi /* Clear stat update records */
+ /* 64-bit increment of acquire attempt statistic (per-group) */
+ LOCK_IF_ATOMIC_STAT_UPDATES
+ addl $1, GRP_MTX_STAT_UTIL(%esi)
+ jnc 1f
+ incl GRP_MTX_STAT_UTIL+4(%esi)
+1:
+ movl M_ILK,%eax /* read interlock */
+ testl %eax,%eax /* unlocked? */
+ jne Llml_ext_ilk_loop /* no, go to spin loop */
+Llml_ext_get_hw:
+ cli
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+
+ /* eax == 0 at this point */
+ lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */
+ jne Llml_ext_ilk_fail /* branch on failure to retry */
+
+ movl M_LOCKED,%ecx /* get lock owner */
+ testl %ecx,%ecx /* is the mutex locked? */
+ jne Llml_ext_fail /* yes, we lose */
+
+Llml_ext_acquire:
+ movl %gs:CPU_ACTIVE_THREAD,%ecx
+ movl %ecx,M_LOCKED
+
+ cmpw $0,M_WAITERS /* are there any waiters? */
+ jne Llml_ext_waiters /* yes, more work to do */
+Llml_ext_return:
+ xorl %eax,%eax
+ movl %eax,M_ILK
+
+ popl %edi
+ popl %esi
+ popf /* restore interrupt state */
+ leave
+#if CONFIG_DTRACE
+ LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
+ ret
+ /* inherit lock pointer in %edx above */
+ LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, %edx)
+#endif
+ ret
+
+Llml_ext_waiters:
+ pushl %edx /* save mutex address */
projects / apple / xnu.git / blobdiff