X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d7e50217d7adf6e52786a38bcaa4cd698cb9a79e..5d5c5d0d5b79ade9a973d55186ffda2638ba2b6e:/osfmk/i386/i386_lock.s diff --git a/osfmk/i386/i386_lock.s b/osfmk/i386/i386_lock.s index ed2710ccc..68598c3cb 100644 --- a/osfmk/i386/i386_lock.s +++ b/osfmk/i386/i386_lock.s @@ -1,26 +1,31 @@ /* * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the + * License may not be used to create, or enable the creation or + * redistribution of, unlawful or unlicensed copies of an Apple operating + * system, or to circumvent, violate, or enable the circumvention or + * violation of, any terms of an Apple operating system software license + * agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ + * + * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -32,71 +37,101 @@ * the terms and conditions for use and redistribution. */ -#include #include #include #include #include -#include #include "assym.s" +#define PAUSE rep; nop + /* * When performance isn't the only concern, it's * nice to build stack frames... */ -#define BUILD_STACK_FRAMES ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB) +#define BUILD_STACK_FRAMES (GPROF || \ + ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB)) #if BUILD_STACK_FRAMES -#define L_PC 4(%ebp) -#define L_ARG0 8(%ebp) -#define L_ARG1 12(%ebp) +/* STack-frame-relative: */ +#define L_PC B_PC +#define L_ARG0 B_ARG0 +#define L_ARG1 B_ARG1 + +#define LEAF_ENTRY(name) \ + Entry(name); \ + FRAME; \ + MCOUNT -#define SWT_HI -4(%ebp) -#define SWT_LO -8(%ebp) -#define MISSED -12(%ebp) +#define LEAF_ENTRY2(n1,n2) \ + Entry(n1); \ + Entry(n2); \ + FRAME; \ + MCOUNT -#else /* BUILD_STACK_FRAMES */ +#define LEAF_RET \ + EMARF; \ + ret + +#else /* BUILD_STACK_FRAMES */ + +/* Stack-pointer-relative: */ +#define L_PC S_PC +#define L_ARG0 S_ARG0 +#define L_ARG1 S_ARG1 + +#define LEAF_ENTRY(name) \ + Entry(name) + +#define LEAF_ENTRY2(n1,n2) \ + Entry(n1); \ + Entry(n2) + +#define LEAF_RET \ + ret -#undef FRAME -#undef EMARF -#define FRAME -#define EMARF -#define L_PC (%esp) -#define L_ARG0 4(%esp) -#define L_ARG1 8(%esp) +#endif /* BUILD_STACK_FRAMES */ -#endif /* BUILD_STACK_FRAMES */ +/* Non-leaf routines always have a stack frame: */ -#define M_ILK (%edx) -#define M_LOCKED 1(%edx) -#define M_WAITERS 2(%edx) -#define M_PROMOTED_PRI 4(%edx) +#define NONLEAF_ENTRY(name) \ + Entry(name); \ + FRAME; \ + MCOUNT + +#define NONLEAF_ENTRY2(n1,n2) \ + Entry(n1); \ + Entry(n2); \ + FRAME; \ + MCOUNT + +#define NONLEAF_RET \ + EMARF; \ + ret + + +#define M_ILK (%edx) +#define M_LOCKED MUTEX_LOCKED(%edx) +#define M_WAITERS MUTEX_WAITERS(%edx) +#define M_PROMOTED_PRI MUTEX_PROMOTED_PRI(%edx) +#define M_ITAG MUTEX_ITAG(%edx) +#define M_PTR MUTEX_PTR(%edx) #if MACH_LDEBUG -#define M_TYPE 6(%edx) -#define M_PC 10(%edx) -#define M_THREAD 14(%edx) +#define M_TYPE MUTEX_TYPE(%edx) +#define M_PC MUTEX_PC(%edx) +#define M_THREAD MUTEX_THREAD(%edx) #endif /* MACH_LDEBUG */ -#include -#if (NCPUS > 1) +#include #define CX(addr,reg) addr(,reg,4) -#else -#define CPU_NUMBER(reg) -#define CX(addr,reg) addr -#endif /* (NCPUS > 1) */ #if MACH_LDEBUG /* * Routines for general lock debugging. */ -#define S_TYPE 4(%edx) -#define S_PC 8(%edx) -#define S_THREAD 12(%edx) -#define S_DURATIONH 16(%edx) -#define S_DURATIONL 20(%edx) /* * Checks for expected lock types and calls "panic" on @@ -114,27 +149,15 @@ .text ; \ 1: -#define CHECK_SIMPLE_LOCK_TYPE() \ - cmpl $ SIMPLE_LOCK_TAG,S_TYPE ; \ - je 1f ; \ - pushl $2f ; \ - call EXT(panic) ; \ - hlt ; \ - .data ; \ -2: String "not a simple lock!" ; \ - .text ; \ -1: - /* * If one or more simplelocks are currently held by a thread, * an attempt to acquire a mutex will cause this check to fail * (since a mutex lock may context switch, holding a simplelock * is not a good thing). */ -#if 0 /*MACH_RT - 11/12/99 - lion@apple.com disable check for now*/ +#if MACH_RT #define CHECK_PREEMPTION_LEVEL() \ - movl $ CPD_PREEMPTION_LEVEL,%eax ; \ - cmpl $0,%gs:(%eax) ; \ + cmpl $0,%gs:CPU_PREEMPTION_LEVEL ; \ je 1f ; \ pushl $2f ; \ call EXT(panic) ; \ @@ -148,8 +171,7 @@ #endif /* MACH_RT */ #define CHECK_NO_SIMPLELOCKS() \ - movl $ CPD_SIMPLE_LOCK_COUNT,%eax ; \ - cmpl $0,%gs:(%eax) ; \ + cmpl $0,%gs:CPU_SIMPLE_LOCK_COUNT ; \ je 1f ; \ pushl $2f ; \ call EXT(panic) ; \ @@ -163,8 +185,7 @@ * Verifies return to the correct thread in "unlock" situations. */ #define CHECK_THREAD(thd) \ - movl $ CPD_ACTIVE_THREAD,%eax ; \ - movl %gs:(%eax),%ecx ; \ + movl %gs:CPU_ACTIVE_THREAD,%ecx ; \ testl %ecx,%ecx ; \ je 1f ; \ cmpl %ecx,thd ; \ @@ -178,8 +199,7 @@ 1: #define CHECK_MYLOCK(thd) \ - movl $ CPD_ACTIVE_THREAD,%eax ; \ - movl %gs:(%eax),%ecx ; \ + movl %gs:CPU_ACTIVE_THREAD,%ecx ; \ testl %ecx,%ecx ; \ je 1f ; \ cmpl %ecx,thd ; \ @@ -219,41 +239,112 @@ * * Initialize a hardware lock. */ -ENTRY(hw_lock_init) - FRAME +LEAF_ENTRY(hw_lock_init) movl L_ARG0,%edx /* fetch lock pointer */ - xorl %eax,%eax - movb %al,0(%edx) /* clear the lock */ - EMARF - ret + movl $0,0(%edx) /* clear the lock */ + LEAF_RET /* * void hw_lock_lock(hw_lock_t) - * unsigned int hw_lock_to(hw_lock_t, unsigned int) * * Acquire lock, spinning until it becomes available. - * XXX: For now, we don't actually implement the timeout. * MACH_RT: also return with preemption disabled. */ -ENTRY2(hw_lock_lock,hw_lock_to) - FRAME +LEAF_ENTRY(hw_lock_lock) movl L_ARG0,%edx /* fetch lock pointer */ -1: DISABLE_PREEMPTION(%eax) - movb $1,%cl - xchgb 0(%edx),%cl /* try to acquire the HW lock */ - testb %cl,%cl /* success? */ + movl %gs:CPU_ACTIVE_THREAD,%ecx + DISABLE_PREEMPTION +1: + movl 0(%edx), %eax + testl %eax,%eax /* lock locked? */ + jne 3f /* branch if so */ + lock; cmpxchgl %ecx,0(%edx) /* try to acquire the HW lock */ jne 3f movl $1,%eax /* In case this was a timeout call */ - EMARF /* if yes, then nothing left to do */ - ret - -3: ENABLE_PREEMPTION(%eax) /* no reason we can't be preemptable now */ + LEAF_RET /* if yes, then nothing left to do */ +3: + PAUSE /* pause for hyper-threading */ + jmp 1b /* try again */ - movb $1,%cl -2: testb %cl,0(%edx) /* spin checking lock value in cache */ - jne 2b /* non-zero means locked, keep spinning */ - jmp 1b /* zero means unlocked, try to grab it */ +/* + * unsigned int hw_lock_to(hw_lock_t, unsigned int) + * + * Acquire lock, spinning until it becomes available or timeout. + * MACH_RT: also return with preemption disabled. + */ +LEAF_ENTRY(hw_lock_to) +1: + movl L_ARG0,%edx /* fetch lock pointer */ + movl %gs:CPU_ACTIVE_THREAD,%ecx + /* + * Attempt to grab the lock immediately + * - fastpath without timeout nonsense. + */ + DISABLE_PREEMPTION + movl 0(%edx), %eax + testl %eax,%eax /* lock locked? */ + jne 2f /* branch if so */ + lock; cmpxchgl %ecx,0(%edx) /* try to acquire the HW lock */ + jne 2f /* branch on failure */ + movl $1,%eax + LEAF_RET + +2: +#define INNER_LOOP_COUNT 1000 + /* + * Failed to get the lock so set the timeout + * and then spin re-checking the lock but pausing + * every so many (INNER_LOOP_COUNT) spins to check for timeout. + */ + movl L_ARG1,%ecx /* fetch timeout */ + push %edi + push %ebx + mov %edx,%edi + + rdtsc /* read cyclecount into %edx:%eax */ + addl %ecx,%eax /* fetch and timeout */ + adcl $0,%edx /* add carry */ + mov %edx,%ecx + mov %eax,%ebx /* %ecx:%ebx is the timeout expiry */ +4: + /* + * The inner-loop spin to look for the lock being freed. + */ + mov $(INNER_LOOP_COUNT),%edx +5: + PAUSE /* pause for hyper-threading */ + movl 0(%edi),%eax /* spin checking lock value in cache */ + testl %eax,%eax + je 6f /* zero => unlocked, try to grab it */ + decl %edx /* decrement inner loop count */ + jnz 5b /* time to check for timeout? */ + + /* + * Here after spinning INNER_LOOP_COUNT times, check for timeout + */ + rdtsc /* cyclecount into %edx:%eax */ + cmpl %ecx,%edx /* compare high-order 32-bits */ + jb 4b /* continue spinning if less, or */ + cmpl %ebx,%eax /* compare low-order 32-bits */ + jb 4b /* continue if less, else bail */ + xor %eax,%eax /* with 0 return value */ + pop %ebx + pop %edi + LEAF_RET + +6: + /* + * Here to try to grab the lock that now appears to be free + * after contention. + */ + movl %gs:CPU_ACTIVE_THREAD,%edx + lock; cmpxchgl %edx,0(%edi) /* try to acquire the HW lock */ + jne 4b /* no - spin again */ + movl $1,%eax /* yes */ + pop %ebx + pop %edi + LEAF_RET /* * void hw_lock_unlock(hw_lock_t) @@ -261,504 +352,575 @@ ENTRY2(hw_lock_lock,hw_lock_to) * Unconditionally release lock. * MACH_RT: release preemption level. */ -ENTRY(hw_lock_unlock) - FRAME +LEAF_ENTRY(hw_lock_unlock) movl L_ARG0,%edx /* fetch lock pointer */ - xorl %eax,%eax - xchgb 0(%edx),%al /* clear the lock... a mov instruction */ - /* ...might be cheaper and less paranoid */ - ENABLE_PREEMPTION(%eax) - EMARF - ret + movl $0,0(%edx) /* clear the lock */ + ENABLE_PREEMPTION + LEAF_RET /* * unsigned int hw_lock_try(hw_lock_t) * MACH_RT: returns with preemption disabled on success. */ -ENTRY(hw_lock_try) - FRAME +LEAF_ENTRY(hw_lock_try) movl L_ARG0,%edx /* fetch lock pointer */ - DISABLE_PREEMPTION(%eax) - movb $1,%cl - xchgb 0(%edx),%cl /* try to acquire the HW lock */ - testb %cl,%cl /* success? */ - jne 1f /* if yes, let the caller know */ + movl %gs:CPU_ACTIVE_THREAD,%ecx + DISABLE_PREEMPTION + movl 0(%edx),%eax + testl %eax,%eax + jne 1f + lock; cmpxchgl %ecx,0(%edx) /* try to acquire the HW lock */ + jne 1f movl $1,%eax /* success */ - EMARF - ret + LEAF_RET -1: ENABLE_PREEMPTION(%eax) /* failure: release preemption... */ +1: + ENABLE_PREEMPTION /* failure: release preemption... */ xorl %eax,%eax /* ...and return failure */ - EMARF - ret + LEAF_RET /* * unsigned int hw_lock_held(hw_lock_t) * MACH_RT: doesn't change preemption state. * N.B. Racy, of course. */ -ENTRY(hw_lock_held) - FRAME +LEAF_ENTRY(hw_lock_held) movl L_ARG0,%edx /* fetch lock pointer */ - movb $1,%cl - testb %cl,0(%edx) /* check lock value */ - jne 1f /* non-zero means locked */ - xorl %eax,%eax /* tell caller: lock wasn't locked */ - EMARF - ret + movl 0(%edx),%eax /* check lock value */ + testl %eax,%eax + movl $1,%ecx + cmovne %ecx,%eax /* 0 => unlocked, 1 => locked */ + LEAF_RET -1: movl $1,%eax /* tell caller: lock was locked */ - EMARF - ret - +LEAF_ENTRY(mutex_init) + movl L_ARG0,%edx /* fetch lock pointer */ + xorl %eax,%eax + movl %eax,M_ILK /* clear interlock */ + movl %eax,M_LOCKED /* clear locked flag */ + movw %ax,M_WAITERS /* init waiter count */ + movw %ax,M_PROMOTED_PRI +#if MACH_LDEBUG + movl $ MUTEX_TAG,M_TYPE /* set lock type */ + movl %eax,M_PC /* init caller pc */ + movl %eax,M_THREAD /* and owning thread */ +#endif -#if 0 + LEAF_RET +NONLEAF_ENTRY2(mutex_lock,_mutex_lock) -ENTRY(_usimple_lock_init) - FRAME - movl L_ARG0,%edx /* fetch lock pointer */ - xorl %eax,%eax - movb %al,USL_INTERLOCK(%edx) /* unlock the HW lock */ - EMARF - ret + movl B_ARG0,%edx /* fetch lock pointer */ -ENTRY(_simple_lock) - FRAME - movl L_ARG0,%edx /* fetch lock pointer */ + CHECK_MUTEX_TYPE() + CHECK_NO_SIMPLELOCKS() + CHECK_PREEMPTION_LEVEL() - CHECK_SIMPLE_LOCK_TYPE() + pushf /* save interrupt state */ + cli /* disable interrupts */ +Lml_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx - DISABLE_PREEMPTION(%eax) +Lml_get_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Lml_ilk_fail /* no - take the slow path */ -sl_get_hw: - movb $1,%cl - xchgb USL_INTERLOCK(%edx),%cl /* try to acquire the HW lock */ - testb %cl,%cl /* did we succeed? */ + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Lml_get_hw /* branch on failure to retry */ -#if MACH_LDEBUG - je 5f - CHECK_MYLOCK(S_THREAD) - jmp sl_get_hw -5: -#else /* MACH_LDEBUG */ - jne sl_get_hw /* no, try again */ -#endif /* MACH_LDEBUG */ + movl M_LOCKED,%ecx /* get lock owner */ + testl %ecx,%ecx /* is the mutex locked? */ + jne Lml_fail /* yes, we lose */ +Lml_acquire: + movl %gs:CPU_ACTIVE_THREAD,%ecx + movl %ecx,M_LOCKED #if MACH_LDEBUG - movl L_PC,%ecx - movl %ecx,S_PC - movl $ CPD_ACTIVE_THREAD,%eax - movl %gs:(%eax),%ecx - movl %ecx,S_THREAD - incl CX(EXT(simple_lock_count),%eax) -#if 0 - METER_SIMPLE_LOCK_LOCK(%edx) + movl %ecx,M_THREAD + movl B_PC,%ecx + movl %ecx,M_PC #endif -#if NCPUS == 1 - pushf + + cmpw $0,M_WAITERS /* are there any waiters? */ + jne Lml_waiters /* yes, more work to do */ +Lml_return: + xorl %eax,%eax + movl %eax,M_ILK + + popf /* restore interrupt state */ + + NONLEAF_RET + +Lml_waiters: + pushl %edx /* save mutex address */ pushl %edx - cli - call EXT(lock_stack_push) - popl %edx - popfl -#endif /* NCPUS == 1 */ -#endif /* MACH_LDEBUG */ + call EXT(lck_mtx_lock_acquire) + addl $4,%esp + popl %edx /* restore mutex address */ + jmp Lml_return + +Lml_ilk_fail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Lml_retry /* try again */ + +Lml_fail: + /* + n Check if the owner is on another processor and therefore + * we should try to spin before blocking. + */ + testl $(OnProc),ACT_SPF(%ecx) + jz Lml_block + + /* + * Here if owner is on another processor: + * - release the interlock + * - spin on the holder until release or timeout + * - in either case re-acquire the interlock + * - if released, acquire it + * - otherwise drop thru to block. + */ + xorl %eax,%eax + movl %eax,M_ILK /* zero interlock */ + popf + pushf /* restore interrupt state */ - EMARF - ret + push %edx /* lock address */ + call EXT(lck_mtx_lock_spin) /* call out to do spinning */ + addl $4,%esp + movl B_ARG0,%edx /* refetch mutex address */ -ENTRY(_simple_lock_try) - FRAME - movl L_ARG0,%edx /* fetch lock pointer */ + /* Re-acquire interlock */ + cli /* disable interrupts */ +Lml_reget_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx - CHECK_SIMPLE_LOCK_TYPE() +Lml_reget_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Lml_ilk_refail /* no - slow path */ - DISABLE_PREEMPTION(%eax) + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Lml_reget_hw /* branch on failure to retry */ - movb $1,%cl - xchgb USL_INTERLOCK(%edx),%cl /* try to acquire the HW lock */ - testb %cl,%cl /* did we succeed? */ - jne 1f /* no, return failure */ + movl M_LOCKED,%ecx /* get lock owner */ + testl %ecx,%ecx /* is the mutex free? */ + je Lml_acquire /* yes, acquire */ + +Lml_block: + CHECK_MYLOCK(M_THREAD) + pushl M_LOCKED + pushl %edx /* push mutex address */ + call EXT(lck_mtx_lock_wait) /* wait for the lock */ + addl $8,%esp + movl B_ARG0,%edx /* refetch mutex address */ + cli /* ensure interrupts disabled */ + jmp Lml_retry /* and try again */ -#if MACH_LDEBUG - movl L_PC,%ecx - movl %ecx,S_PC - movl $ CPD_ACTIVE_THREAD,%eax - movl %gs:(%eax),%ecx - movl %ecx,S_THREAD - incl CX(EXT(simple_lock_count),%eax) -#if 0 - METER_SIMPLE_LOCK_LOCK(%edx) -#endif -#if NCPUS == 1 - pushf - pushl %edx - cli - call EXT(lock_stack_push) - popl %edx - popfl -#endif /* NCPUS == 1 */ -#endif /* MACH_LDEBUG */ +Lml_ilk_refail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Lml_reget_retry /* try again */ - movl $1,%eax /* return success */ +NONLEAF_ENTRY2(mutex_try,_mutex_try) - EMARF - ret + movl B_ARG0,%edx /* fetch lock pointer */ -1: - ENABLE_PREEMPTION(%eax) + CHECK_MUTEX_TYPE() + CHECK_NO_SIMPLELOCKS() - xorl %eax,%eax /* and return failure */ + pushf /* save interrupt state */ + cli /* disable interrupts */ +Lmt_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx - EMARF - ret +Lmt_get_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Lmt_ilk_fail /* no - slow path */ -ENTRY(_simple_unlock) - FRAME - movl L_ARG0,%edx /* fetch lock pointer */ + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Lmt_get_hw /* branch on failure to retry */ - CHECK_SIMPLE_LOCK_TYPE() - CHECK_THREAD(S_THREAD) + movl M_LOCKED,%ecx /* get lock owner */ + testl %ecx,%ecx /* is the mutex locked? */ + jne Lmt_fail /* yes, we lose */ + movl %gs:CPU_ACTIVE_THREAD,%ecx + movl %ecx,M_LOCKED #if MACH_LDEBUG - xorl %eax,%eax - movl %eax,S_THREAD /* disown thread */ - MP_DISABLE_PREEMPTION(%eax) - CPU_NUMBER(%eax) - decl CX(EXT(simple_lock_count),%eax) - MP_ENABLE_PREEMPTION(%eax) -#if 0 - METER_SIMPLE_LOCK_UNLOCK(%edx) + movl %ecx,M_THREAD + movl B_PC,%ecx + movl %ecx,M_PC #endif -#if NCPUS == 1 - pushf - pushl %edx - cli - call EXT(lock_stack_pop) - popl %edx - popfl -#endif /* NCPUS == 1 */ -#endif /* MACH_LDEBUG */ - - xorb %cl,%cl - xchgb USL_INTERLOCK(%edx),%cl /* unlock the HW lock */ - ENABLE_PREEMPTION(%eax) - - EMARF - ret + cmpl $0,M_WAITERS /* are there any waiters? */ + jne Lmt_waiters /* yes, more work to do */ +Lmt_return: + xorl %eax,%eax + movl %eax,M_ILK + popf /* restore interrupt state */ -#endif /* 0 */ + movl $1,%eax + NONLEAF_RET -ENTRY(mutex_init) - FRAME - movl L_ARG0,%edx /* fetch lock pointer */ +Lmt_waiters: + pushl %edx /* save mutex address */ + pushl %edx + call EXT(lck_mtx_lock_acquire) + addl $4,%esp + popl %edx /* restore mutex address */ + jmp Lmt_return + +Lmt_ilk_fail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Lmt_retry /* try again */ + +Lmt_fail: xorl %eax,%eax - movb %al,M_ILK /* clear interlock */ - movb %al,M_LOCKED /* clear locked flag */ - movw %ax,M_WAITERS /* init waiter count */ - movw %ax,M_PROMOTED_PRI + movl %eax,M_ILK -#if MACH_LDEBUG - movl $ MUTEX_TAG,M_TYPE /* set lock type */ - movl %eax,M_PC /* init caller pc */ - movl %eax,M_THREAD /* and owning thread */ -#endif -#if ETAP_LOCK_TRACE - movl L_ARG1,%ecx /* fetch event type */ - pushl %ecx /* push event type */ - pushl %edx /* push mutex address */ - call EXT(etap_mutex_init) /* init ETAP data */ - addl $8,%esp -#endif /* ETAP_LOCK_TRACE */ - - EMARF - ret + popf /* restore interrupt state */ -ENTRY2(mutex_lock,_mutex_lock) - FRAME + xorl %eax,%eax -#if ETAP_LOCK_TRACE - subl $12,%esp /* make room for locals */ - movl $0,SWT_HI /* set wait time to zero (HI) */ - movl $0,SWT_LO /* set wait time to zero (LO) */ - movl $0,MISSED /* clear local miss marker */ -#endif /* ETAP_LOCK_TRACE */ + NONLEAF_RET - movl L_ARG0,%edx /* fetch lock pointer */ +NONLEAF_ENTRY(mutex_unlock) + movl B_ARG0,%edx /* fetch lock pointer */ CHECK_MUTEX_TYPE() - CHECK_NO_SIMPLELOCKS() - CHECK_PREEMPTION_LEVEL() + CHECK_THREAD(M_THREAD) -ml_retry: - DISABLE_PREEMPTION(%eax) + pushf /* save interrupt state */ + cli /* disable interrupts */ +Lmu_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx -ml_get_hw: - movb $1,%cl - xchgb %cl,M_ILK - testb %cl,%cl /* did we succeed? */ - jne ml_get_hw /* no, try again */ +Lmu_get_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Lmu_ilk_fail /* no - slow path */ - movb $1,%cl - xchgb %cl,M_LOCKED /* try to set locked flag */ - testb %cl,%cl /* is the mutex locked? */ - jne ml_fail /* yes, we lose */ + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Lmu_get_hw /* branch on failure to retry */ - pushl %edx - call EXT(mutex_lock_acquire) - addl $4,%esp - movl L_ARG0,%edx + cmpw $0,M_WAITERS /* are there any waiters? */ + jne Lmu_wakeup /* yes, more work to do */ + +Lmu_doit: #if MACH_LDEBUG - movl L_PC,%ecx - movl %ecx,M_PC - movl $ CPD_ACTIVE_THREAD,%eax - movl %gs:(%eax),%ecx - movl %ecx,M_THREAD - testl %ecx,%ecx - je 3f - incl TH_MUTEX_COUNT(%ecx) -3: + movl $0,M_THREAD /* disown thread */ #endif - xorb %cl,%cl - xchgb %cl,M_ILK + xorl %ecx,%ecx + movl %ecx,M_LOCKED /* unlock the mutex */ - ENABLE_PREEMPTION(%eax) + movl %ecx,M_ILK -#if ETAP_LOCK_TRACE - movl L_PC,%eax /* fetch pc */ - pushl SWT_LO /* push wait time (low) */ - pushl SWT_HI /* push wait time (high) */ - pushl %eax /* push pc */ - pushl %edx /* push mutex address */ - call EXT(etap_mutex_hold) /* collect hold timestamp */ - addl $16+12,%esp /* clean up stack, adjusting for locals */ -#endif /* ETAP_LOCK_TRACE */ + popf /* restore interrupt state */ - EMARF - ret + NONLEAF_RET -ml_fail: -#if ETAP_LOCK_TRACE - cmp $0,MISSED /* did we already take a wait timestamp? */ - jne ml_block /* yup. carry-on */ - pushl %edx /* push mutex address */ - call EXT(etap_mutex_miss) /* get wait timestamp */ - movl %eax,SWT_HI /* set wait time (high word) */ - movl %edx,SWT_LO /* set wait time (low word) */ - popl %edx /* clean up stack */ - movl $1,MISSED /* mark wait timestamp as taken */ -#endif /* ETAP_LOCK_TRACE */ - -ml_block: - CHECK_MYLOCK(M_THREAD) - xorl %eax,%eax - pushl %eax /* no promotion here yet */ +Lmu_ilk_fail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Lmu_retry /* try again */ + +Lmu_wakeup: + pushl M_LOCKED pushl %edx /* push mutex address */ - call EXT(mutex_lock_wait) /* wait for the lock */ + call EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */ addl $8,%esp - movl L_ARG0,%edx /* refetch lock pointer */ - jmp ml_retry /* and try again */ + movl B_ARG0,%edx /* restore lock pointer */ + jmp Lmu_doit -ENTRY2(mutex_try,_mutex_try) - FRAME - -#if ETAP_LOCK_TRACE - subl $8,%esp /* make room for locals */ - movl $0,SWT_HI /* set wait time to zero (HI) */ - movl $0,SWT_LO /* set wait time to zero (LO) */ -#endif /* ETAP_LOCK_TRACE */ +/* + * lck_mtx_lock() + * lck_mtx_try_lock() + * lck_mutex_unlock() + * + * These are variants of mutex_lock(), mutex_try() and mutex_unlock() without + * DEBUG checks (which require fields not present in lck_mtx_t's). + */ +NONLEAF_ENTRY(lck_mtx_lock) - movl L_ARG0,%edx /* fetch lock pointer */ + movl B_ARG0,%edx /* fetch lock pointer */ + cmpl $(MUTEX_IND),M_ITAG /* is this indirect? */ + cmove M_PTR,%edx /* yes - take indirection */ - CHECK_MUTEX_TYPE() CHECK_NO_SIMPLELOCKS() + CHECK_PREEMPTION_LEVEL() - DISABLE_PREEMPTION(%eax) + pushf /* save interrupt state */ + cli /* disable interrupts */ +Llml_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx + +Llml_get_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Llml_ilk_fail /* no - slow path */ + + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Llml_get_hw /* branch on failure to retry */ + + movl M_LOCKED,%ecx /* get lock owner */ + testl %ecx,%ecx /* is the mutex locked? */ + jne Llml_fail /* yes, we lose */ +Llml_acquire: + movl %gs:CPU_ACTIVE_THREAD,%ecx + movl %ecx,M_LOCKED + + cmpl $0,M_WAITERS /* are there any waiters? */ + jne Llml_waiters /* yes, more work to do */ +Llml_return: + xorl %eax,%eax + movl %eax,M_ILK -mt_get_hw: - movb $1,%cl - xchgb %cl,M_ILK - testb %cl,%cl - jne mt_get_hw + popf /* restore interrupt state */ - movb $1,%cl - xchgb %cl,M_LOCKED - testb %cl,%cl - jne mt_fail + NONLEAF_RET +Llml_waiters: + pushl %edx /* save mutex address */ pushl %edx - call EXT(mutex_lock_acquire) + call EXT(lck_mtx_lock_acquire) addl $4,%esp - movl L_ARG0,%edx + popl %edx /* restore mutex address */ + jmp Llml_return + +Llml_ilk_fail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Llml_retry /* try again */ + +Llml_fail: + /* + * Check if the owner is on another processor and therefore + * we should try to spin before blocking. + */ + testl $(OnProc),ACT_SPF(%ecx) + jz Llml_block + + /* + * Here if owner is on another processor: + * - release the interlock + * - spin on the holder until release or timeout + * - in either case re-acquire the interlock + * - if released, acquire it + * - otherwise drop thru to block. + */ + xorl %eax,%eax + movl %eax,M_ILK /* zero interlock */ + popf + pushf /* restore interrupt state */ -#if MACH_LDEBUG - movl L_PC,%ecx - movl %ecx,M_PC - movl $ CPD_ACTIVE_THREAD,%ecx - movl %gs:(%ecx),%ecx - movl %ecx,M_THREAD - testl %ecx,%ecx - je 1f - incl TH_MUTEX_COUNT(%ecx) -1: -#endif + pushl %edx /* save mutex address */ + pushl %edx + call EXT(lck_mtx_lock_spin) + addl $4,%esp + popl %edx /* restore mutex address */ - xorb %cl,%cl - xchgb %cl,M_ILK + /* Re-acquire interlock */ + cli /* disable interrupts */ +Llml_reget_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx - ENABLE_PREEMPTION(%eax) +Llml_reget_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Llml_ilk_refail /* no - slow path */ -#if ETAP_LOCK_TRACE - movl L_PC,%eax /* fetch pc */ - pushl SWT_LO /* push wait time (low) */ - pushl SWT_HI /* push wait time (high) */ - pushl %eax /* push pc */ + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Llml_reget_hw /* branch on failure to retry */ + + movl M_LOCKED,%ecx /* get lock owner */ + testl %ecx,%ecx /* is the mutex free? */ + je Llml_acquire /* yes, acquire */ + +Llml_block: + CHECK_MYLOCK(M_THREAD) + pushl %edx /* save mutex address */ + pushl M_LOCKED pushl %edx /* push mutex address */ - call EXT(etap_mutex_hold) /* get start hold timestamp */ - addl $16,%esp /* clean up stack, adjusting for locals */ -#endif /* ETAP_LOCK_TRACE */ + call EXT(lck_mtx_lock_wait) /* wait for the lock */ + addl $8,%esp + popl %edx /* restore mutex address */ + cli /* ensure interrupts disabled */ + jmp Llml_retry /* and try again */ - movl $1,%eax +Llml_ilk_refail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Llml_reget_retry /* try again */ -#if MACH_LDEBUG || ETAP_LOCK_TRACE -#if ETAP_LOCK_TRACE - addl $8,%esp /* pop stack claimed on entry */ -#endif -#endif +NONLEAF_ENTRY(lck_mtx_try_lock) - EMARF - ret + movl B_ARG0,%edx /* fetch lock pointer */ + cmpl $(MUTEX_IND),M_ITAG /* is this indirect? */ + cmove M_PTR,%edx /* yes - take indirection */ -mt_fail: -#if MACH_LDEBUG - movl L_PC,%ecx - movl %ecx,M_PC - movl $ CPD_ACTIVE_THREAD,%ecx - movl %gs:(%ecx),%ecx - movl %ecx,M_THREAD - testl %ecx,%ecx - je 1f - incl TH_MUTEX_COUNT(%ecx) -1: -#endif + CHECK_NO_SIMPLELOCKS() + CHECK_PREEMPTION_LEVEL() - xorb %cl,%cl - xchgb %cl,M_ILK + pushf /* save interrupt state */ + cli /* disable interrupts */ +Llmt_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx - ENABLE_PREEMPTION(%eax) +Llmt_get_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Llmt_ilk_fail /* no - slow path */ -#if ETAP_LOCK_TRACE - movl L_PC,%eax /* fetch pc */ - pushl SWT_LO /* push wait time (low) */ - pushl SWT_HI /* push wait time (high) */ - pushl %eax /* push pc */ - pushl %edx /* push mutex address */ - call EXT(etap_mutex_hold) /* get start hold timestamp */ - addl $16,%esp /* clean up stack, adjusting for locals */ -#endif /* ETAP_LOCK_TRACE */ + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Llmt_get_hw /* branch on failure to retry */ - xorl %eax,%eax + movl M_LOCKED,%ecx /* get lock owner */ + testl %ecx,%ecx /* is the mutex locked? */ + jne Llmt_fail /* yes, we lose */ + movl %gs:CPU_ACTIVE_THREAD,%ecx + movl %ecx,M_LOCKED -#if MACH_LDEBUG || ETAP_LOCK_TRACE -#if ETAP_LOCK_TRACE - addl $8,%esp /* pop stack claimed on entry */ -#endif -#endif + cmpl $0,M_WAITERS /* are there any waiters? */ + jne Llmt_waiters /* yes, more work to do */ +Llmt_return: + xorl %eax,%eax + movl %eax,M_ILK - EMARF - ret + popf /* restore interrupt state */ -ENTRY(mutex_unlock) - FRAME - movl L_ARG0,%edx /* fetch lock pointer */ + movl $1,%eax /* return success */ + NONLEAF_RET -#if ETAP_LOCK_TRACE - pushl %edx /* push mutex address */ - call EXT(etap_mutex_unlock) /* collect ETAP data */ +Llmt_waiters: + pushl %edx /* save mutex address */ + pushl %edx + call EXT(lck_mtx_lock_acquire) + addl $4,%esp popl %edx /* restore mutex address */ -#endif /* ETAP_LOCK_TRACE */ + jmp Llmt_return + +Llmt_ilk_fail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Llmt_retry /* try again */ + +Llmt_fail: + xorl %eax,%eax + movl %eax,M_ILK - CHECK_MUTEX_TYPE() - CHECK_THREAD(M_THREAD) + popf /* restore interrupt state */ + + xorl %eax,%eax /* return failure */ + NONLEAF_RET + +NONLEAF_ENTRY(lck_mtx_unlock) - DISABLE_PREEMPTION(%eax) + movl B_ARG0,%edx /* fetch lock pointer */ + cmpl $(MUTEX_IND),M_ITAG /* is this indirect? */ + cmove M_PTR,%edx /* yes - take indirection */ -mu_get_hw: - movb $1,%cl - xchgb %cl,M_ILK - testb %cl,%cl /* did we succeed? */ - jne mu_get_hw /* no, try again */ + pushf /* save interrupt state */ + cli /* disable interrupts */ +Llmu_retry: + movl %gs:CPU_ACTIVE_THREAD,%ecx + +Llmu_get_hw: + movl M_ILK,%eax /* read interlock */ + testl %eax,%eax /* unlocked? */ + jne Llmu_ilk_fail /* no - slow path */ + + lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ + jne Llmu_get_hw /* branch on failure to retry */ cmpw $0,M_WAITERS /* are there any waiters? */ - jne mu_wakeup /* yes, more work to do */ + jne Llmu_wakeup /* yes, more work to do */ -mu_doit: -#if MACH_LDEBUG - xorl %eax,%eax - movl %eax,M_THREAD /* disown thread */ - movl $ CPD_ACTIVE_THREAD,%eax - movl %gs:(%eax),%ecx - testl %ecx,%ecx - je 0f - decl TH_MUTEX_COUNT(%ecx) -0: -#endif +Llmu_doit: + xorl %ecx,%ecx + movl %ecx,M_LOCKED /* unlock the mutex */ - xorb %cl,%cl - xchgb %cl,M_LOCKED /* unlock the mutex */ + movl %ecx,M_ILK - xorb %cl,%cl - xchgb %cl,M_ILK + popf /* restore interrupt state */ - ENABLE_PREEMPTION(%eax) + NONLEAF_RET - EMARF - ret +Llmu_ilk_fail: + /* + * Slow path: call out to do the spinning. + */ + pushl %edx /* lock address */ + call EXT(lck_mtx_interlock_spin) + popl %edx /* lock pointer */ + jmp Llmu_retry /* try again */ -mu_wakeup: - xorl %eax,%eax - pushl %eax /* no promotion here yet */ +Llmu_wakeup: + pushl %edx /* save mutex address */ + pushl M_LOCKED pushl %edx /* push mutex address */ - call EXT(mutex_unlock_wakeup)/* yes, wake a thread */ + call EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */ addl $8,%esp - movl L_ARG0,%edx /* refetch lock pointer */ - jmp mu_doit - -ENTRY(interlock_unlock) - FRAME - movl L_ARG0,%edx - - xorb %cl,%cl - xchgb %cl,M_ILK + popl %edx /* restore mutex pointer */ + jmp Llmu_doit - ENABLE_PREEMPTION(%eax) +LEAF_ENTRY(lck_mtx_ilk_unlock) + movl L_ARG0,%edx /* no indirection here */ - EMARF - ret + xorl %eax,%eax + movl %eax,M_ILK + LEAF_RET -ENTRY(_disable_preemption) +LEAF_ENTRY(_disable_preemption) #if MACH_RT - _DISABLE_PREEMPTION(%eax) + _DISABLE_PREEMPTION #endif /* MACH_RT */ - ret + LEAF_RET -ENTRY(_enable_preemption) +LEAF_ENTRY(_enable_preemption) #if MACH_RT #if MACH_ASSERT - movl $ CPD_PREEMPTION_LEVEL,%eax - cmpl $0,%gs:(%eax) + cmpl $0,%gs:CPU_PREEMPTION_LEVEL jg 1f - pushl %gs:(%eax) + pushl %gs:CPU_PREEMPTION_LEVEL pushl $2f call EXT(panic) hlt @@ -767,15 +929,14 @@ ENTRY(_enable_preemption) .text 1: #endif /* MACH_ASSERT */ - _ENABLE_PREEMPTION(%eax) + _ENABLE_PREEMPTION #endif /* MACH_RT */ - ret + LEAF_RET -ENTRY(_enable_preemption_no_check) +LEAF_ENTRY(_enable_preemption_no_check) #if MACH_RT #if MACH_ASSERT - movl $ CPD_PREEMPTION_LEVEL,%eax - cmpl $0,%gs:(%eax) + cmpl $0,%gs:CPU_PREEMPTION_LEVEL jg 1f pushl $2f call EXT(panic) @@ -785,24 +946,23 @@ ENTRY(_enable_preemption_no_check) .text 1: #endif /* MACH_ASSERT */ - _ENABLE_PREEMPTION_NO_CHECK(%eax) + _ENABLE_PREEMPTION_NO_CHECK #endif /* MACH_RT */ - ret + LEAF_RET -ENTRY(_mp_disable_preemption) -#if MACH_RT && NCPUS > 1 - _DISABLE_PREEMPTION(%eax) -#endif /* MACH_RT && NCPUS > 1*/ - ret +LEAF_ENTRY(_mp_disable_preemption) +#if MACH_RT + _DISABLE_PREEMPTION +#endif /* MACH_RT */ + LEAF_RET -ENTRY(_mp_enable_preemption) -#if MACH_RT && NCPUS > 1 +LEAF_ENTRY(_mp_enable_preemption) +#if MACH_RT #if MACH_ASSERT - movl $ CPD_PREEMPTION_LEVEL,%eax - cmpl $0,%gs:(%eax) + cmpl $0,%gs:CPU_PREEMPTION_LEVEL jg 1f - pushl %gs:(%eax) + pushl %gs:CPU_PREEMPTION_LEVEL pushl $2f call EXT(panic) hlt @@ -811,15 +971,14 @@ ENTRY(_mp_enable_preemption) .text 1: #endif /* MACH_ASSERT */ - _ENABLE_PREEMPTION(%eax) -#endif /* MACH_RT && NCPUS > 1 */ - ret + _ENABLE_PREEMPTION +#endif /* MACH_RT */ + LEAF_RET -ENTRY(_mp_enable_preemption_no_check) -#if MACH_RT && NCPUS > 1 +LEAF_ENTRY(_mp_enable_preemption_no_check) +#if MACH_RT #if MACH_ASSERT - movl $ CPD_PREEMPTION_LEVEL,%eax - cmpl $0,%gs:(%eax) + cmpl $0,%gs:CPU_PREEMPTION_LEVEL jg 1f pushl $2f call EXT(panic) @@ -829,48 +988,48 @@ ENTRY(_mp_enable_preemption_no_check) .text 1: #endif /* MACH_ASSERT */ - _ENABLE_PREEMPTION_NO_CHECK(%eax) -#endif /* MACH_RT && NCPUS > 1 */ - ret + _ENABLE_PREEMPTION_NO_CHECK +#endif /* MACH_RT */ + LEAF_RET -ENTRY(i_bit_set) - movl S_ARG0,%edx - movl S_ARG1,%eax +LEAF_ENTRY(i_bit_set) + movl L_ARG0,%edx + movl L_ARG1,%eax lock - bts %dl,(%eax) - ret + bts %edx,(%eax) + LEAF_RET -ENTRY(i_bit_clear) - movl S_ARG0,%edx - movl S_ARG1,%eax +LEAF_ENTRY(i_bit_clear) + movl L_ARG0,%edx + movl L_ARG1,%eax lock - btr %dl,(%eax) - ret + btr %edx,(%eax) + LEAF_RET -ENTRY(bit_lock) - movl S_ARG0,%ecx - movl S_ARG1,%eax +LEAF_ENTRY(bit_lock) + movl L_ARG0,%ecx + movl L_ARG1,%eax 1: lock bts %ecx,(%eax) jb 1b - ret + LEAF_RET -ENTRY(bit_lock_try) - movl S_ARG0,%ecx - movl S_ARG1,%eax +LEAF_ENTRY(bit_lock_try) + movl L_ARG0,%ecx + movl L_ARG1,%eax lock bts %ecx,(%eax) jb bit_lock_failed - ret /* %eax better not be null ! */ + LEAF_RET /* %eax better not be null ! */ bit_lock_failed: xorl %eax,%eax - ret + LEAF_RET -ENTRY(bit_unlock) - movl S_ARG0,%ecx - movl S_ARG1,%eax +LEAF_ENTRY(bit_unlock) + movl L_ARG0,%ecx + movl L_ARG1,%eax lock btr %ecx,(%eax) - ret + LEAF_RET