xnu-4570.71.2.tar.gz

[apple/xnu.git] / osfmk / i386 / i386_lock.s

diff --git a/osfmk/i386/i386_lock.s b/osfmk/i386/i386_lock.s
index db7420db5a010f4b0fe30c0b9bea9f2184abacde..d657afaee7aebbf117ab962fef950394345151ce 100644 (file)
--- a/osfmk/i386/i386_lock.s
+++ b/osfmk/i386/i386_lock.s
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
 /*
  * @OSF_COPYRIGHT@


@@ -29,70 +35,69 @@
  * the terms and conditions for use and redistribution.
  */
 
  * the terms and conditions for use and redistribution.
  */
 

-#include <cpus.h>
-#include <mach_rt.h>
-#include <platforms.h>

 #include <mach_ldebug.h>
 #include <i386/asm.h>
 #include <mach_ldebug.h>
 #include <i386/asm.h>

-#include <assym.s>
-#include <kern/etap_options.h>
+#include <i386/eflags.h>
+#include <i386/trap.h>
+#include <config_dtrace.h>
+#include <i386/mp.h>
+       
+#include "assym.s"

 
 

+#define        PAUSE           rep; nop

 
 

-/*
- *     When performance isn't the only concern, it's
- *     nice to build stack frames...
- */
-#define        BUILD_STACK_FRAMES   ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB)
+#include <i386/pal_lock_asm.h>

 
 

-#if    BUILD_STACK_FRAMES
+#define LEAF_ENTRY(name)       \
+       Entry(name)

 
 

-#define        L_PC            4(%ebp)
-#define        L_ARG0          8(%ebp)
-#define        L_ARG1          12(%ebp)
+#define LEAF_ENTRY2(n1,n2)     \
+       Entry(n1);              \
+       Entry(n2)

 
 

-#define SWT_HI          -4(%ebp)
-#define SWT_LO          -8(%ebp)
-#define MISSED          -12(%ebp)
+#define LEAF_RET               \
+       ret

 
 

-#else   /* BUILD_STACK_FRAMES */
+/* Non-leaf routines always have a stack frame: */

 
 

-#undef FRAME
-#undef EMARF
-#define        FRAME
-#define        EMARF
-#define        L_PC            (%esp)
-#define        L_ARG0          4(%esp)
-#define        L_ARG1          8(%esp)
+#define NONLEAF_ENTRY(name)    \
+       Entry(name);            \
+       FRAME

 
 

-#endif   /* BUILD_STACK_FRAMES */
+#define NONLEAF_ENTRY2(n1,n2)  \
+       Entry(n1);              \
+       Entry(n2);              \
+       FRAME

 
 

+#define NONLEAF_RET            \
+       EMARF;                  \
+       ret

 
 

-#define        M_ILK           (%edx)
-#define        M_LOCKED        1(%edx)
-#define        M_WAITERS       2(%edx)
-#if    MACH_LDEBUG
-#define        M_TYPE          4(%edx)
-#define        M_PC            8(%edx)
-#define        M_THREAD        12(%edx)
-#endif /* MACH_LDEBUG */

 
 

-#include <i386/AT386/mp/mp.h>
-#if    (NCPUS > 1)
-#define        CX(addr,reg)    addr(,reg,4)
-#else
-#define        CPU_NUMBER(reg)
-#define        CX(addr,reg)    addr
-#endif /* (NCPUS > 1) */
+/* For x86_64, the varargs ABI requires that %al indicate
+ * how many SSE register contain arguments. In our case, 0 */
+#define ALIGN_STACK()          and  $0xFFFFFFFFFFFFFFF0, %rsp ;
+#define LOAD_STRING_ARG0(label)        leaq label(%rip), %rdi ;
+#define LOAD_ARG1(x)           mov x, %esi ;
+#define LOAD_PTR_ARG1(x)       mov x, %rsi ;
+#define CALL_PANIC()           xorb %al,%al ; call EXT(panic) ;
+
+#define        CHECK_UNLOCK(current, owner)                            \
+       cmp     current, owner                          ;       \
+       je      1f                                      ;       \
+       ALIGN_STACK()                                   ;       \
+       LOAD_STRING_ARG0(2f)                            ;       \
+       CALL_PANIC()                                    ;       \
+       hlt                                             ;       \
+       .data                                           ;       \
+2:     String  "Mutex unlock attempted from non-owner thread"; \
+       .text                                           ;       \
+1:

 
 #if    MACH_LDEBUG
 /*
  *  Routines for general lock debugging.
  */
 
 #if    MACH_LDEBUG
 /*
  *  Routines for general lock debugging.
  */

-#define        S_TYPE          4(%edx)
-#define        S_PC            8(%edx)
-#define        S_THREAD        12(%edx)
-#define        S_DURATIONH     16(%edx)
-#define        S_DURATIONL     20(%edx)

 
 /* 
  * Checks for expected lock types and calls "panic" on
 
 /* 
  * Checks for expected lock types and calls "panic" on


@@ -100,715 +105,992 @@
  * type simplelock and vice versa.
  */
 #define        CHECK_MUTEX_TYPE()                                      \
  * type simplelock and vice versa.
  */
 #define        CHECK_MUTEX_TYPE()                                      \

-       cmpl    $MUTEX_TAG, M_TYPE                      ;       \
+       cmpl    $ MUTEX_TAG,M_TYPE                      ;       \

        je      1f                                      ;       \
        je      1f                                      ;       \

-       pushl   $2f                                     ;       \
-       call    EXT(panic)                              ;       \
+       ALIGN_STACK()                                   ;       \
+       LOAD_STRING_ARG0(2f)                            ;       \
+       CALL_PANIC()                                    ;       \

        hlt                                             ;       \
        .data                                           ;       \
 2:     String  "not a mutex!"                          ;       \
        .text                                           ;       \
 1:
 
        hlt                                             ;       \
        .data                                           ;       \
 2:     String  "not a mutex!"                          ;       \
        .text                                           ;       \
 1:
 

-#define        CHECK_SIMPLE_LOCK_TYPE()                                \
-       cmpl    $SIMPLE_LOCK_TAG,S_TYPE                 ;       \
-       je      1f                                      ;       \
-       pushl   $2f                                     ;       \
-       call    EXT(panic)                              ;       \
+#define        CHECK_MYLOCK(current, owner)                            \
+       cmp     current, owner                          ;       \
+       jne     1f                                      ;       \
+       ALIGN_STACK()                                   ;       \
+       LOAD_STRING_ARG0(2f)                            ;       \
+       CALL_PANIC()                                    ;       \

        hlt                                             ;       \
        .data                                           ;       \
        hlt                                             ;       \
        .data                                           ;       \

-2:     String  "not a simple lock!"                    ;       \
+2:     String  "Attempt to recursively lock a non-recursive lock";     \

        .text                                           ;       \
 1:
 
        .text                                           ;       \
 1:
 

+#else  /* MACH_LDEBUG */
+#define        CHECK_MUTEX_TYPE()
+#define        CHECK_MYLOCK(thd)
+#endif /* MACH_LDEBUG */
+
+#if DEVELOPMENT || DEBUG

 /*
  * If one or more simplelocks are currently held by a thread,
  * an attempt to acquire a mutex will cause this check to fail
  * (since a mutex lock may context switch, holding a simplelock
  * is not a good thing).
  */
 /*
  * If one or more simplelocks are currently held by a thread,
  * an attempt to acquire a mutex will cause this check to fail
  * (since a mutex lock may context switch, holding a simplelock
  * is not a good thing).
  */

-#if    0 /*MACH_RT - 11/12/99 - lion@apple.com disable check for now*/

 #define CHECK_PREEMPTION_LEVEL()                               \
 #define CHECK_PREEMPTION_LEVEL()                               \

-       movl    $CPD_PREEMPTION_LEVEL,%eax              ;       \
-       cmpl    $0,%gs:(%eax)                           ;       \
+       cmpl    $0,%gs:CPU_PREEMPTION_LEVEL             ;       \

        je      1f                                      ;       \
        je      1f                                      ;       \

-       pushl   $2f                                     ;       \
-       call    EXT(panic)                              ;       \
-       hlt                                             ;       \
-       .data                                           ;       \
-2:     String  "preemption_level != 0!"                ;       \
-       .text                                           ;       \
-1:
-#else  /* MACH_RT */
-#define        CHECK_PREEMPTION_LEVEL()
-#endif /* MACH_RT */
-
-#define        CHECK_NO_SIMPLELOCKS()                                  \
-       movl    $CPD_SIMPLE_LOCK_COUNT,%eax             ;       \
-       cmpl    $0,%gs:(%eax)                           ;       \
-       je      1f                                      ;       \
-       pushl   $2f                                     ;       \
-       call    EXT(panic)                              ;       \
+       cmpl    $0,EXT(LckDisablePreemptCheck)(%rip)    ;       \
+       jne     1f                                      ;       \
+       cmpl    $0,%gs:CPU_HIBERNATE                    ;       \
+       jne     1f                                      ;       \
+       ALIGN_STACK()                                   ;       \
+       movl    %gs:CPU_PREEMPTION_LEVEL, %eax          ;       \
+       LOAD_ARG1(%eax)                                 ;       \
+       LOAD_STRING_ARG0(2f)                            ;       \
+       CALL_PANIC()                                    ;       \

        hlt                                             ;       \
        .data                                           ;       \
        hlt                                             ;       \
        .data                                           ;       \

-2:     String  "simple_locks_held!"                    ;       \
+2:     String  "preemption_level(%d) != 0!"            ;       \

        .text                                           ;       \
 1:
        .text                                           ;       \
 1:

+#else /* DEVELOPMENT || DEBUG */
+#define CHECK_PREEMPTION_LEVEL()
+#endif /* DEVELOPMENT || DEBUG */
+
+#define PREEMPTION_DISABLE                             \
+       incl    %gs:CPU_PREEMPTION_LEVEL
+
+#define        PREEMPTION_LEVEL_DEBUG 1        
+#if    PREEMPTION_LEVEL_DEBUG
+#define        PREEMPTION_ENABLE                               \
+       decl    %gs:CPU_PREEMPTION_LEVEL        ;       \
+       js      17f                             ;       \
+       jnz     19f                             ;       \
+       testl   $AST_URGENT,%gs:CPU_PENDING_AST ;       \
+       jz      19f                             ;       \
+       PUSHF                                   ;       \
+       testl   $EFL_IF, S_PC                   ;       \
+       jz      18f                             ;       \
+       POPF                                    ;       \
+       int     $(T_PREEMPT)                    ;       \
+       jmp     19f                             ;       \
+17:                                                    \
+       call    _preemption_underflow_panic     ;       \
+18:                                                    \
+       POPF                                    ;       \
+19:
+#else
+#define        PREEMPTION_ENABLE                               \
+       decl    %gs:CPU_PREEMPTION_LEVEL        ;       \
+       jnz     19f                             ;       \
+       testl   $AST_URGENT,%gs:CPU_PENDING_AST ;       \
+       jz      19f                             ;       \
+       PUSHF                                   ;       \
+       testl   $EFL_IF, S_PC                   ;       \
+       jz      18f                             ;       \
+       POPF                                    ;       \
+       int     $(T_PREEMPT)                    ;       \
+       jmp     19f                             ;       \
+18:                                                    \
+       POPF                                    ;       \
+19:
+#endif

 
 

-/* 
- * Verifies return to the correct thread in "unlock" situations.
- */
-#define        CHECK_THREAD(thd)                                       \
-       movl    $CPD_ACTIVE_THREAD,%eax                 ;       \
-       movl    %gs:(%eax),%ecx                         ;       \
-       testl   %ecx,%ecx                               ;       \
-       je      1f                                      ;       \
-       cmpl    %ecx,thd                                ;       \
-       je      1f                                      ;       \
-       pushl   $2f                                     ;       \
-       call    EXT(panic)                              ;       \
-       hlt                                             ;       \
-       .data                                           ;       \
-2:     String  "wrong thread!"                         ;       \
-       .text                                           ;       \
-1:

 
 

-#define        CHECK_MYLOCK(thd)                                       \
-       movl    $CPD_ACTIVE_THREAD,%eax                 ;       \
-       movl    %gs:(%eax),%ecx                         ;       \
-       testl   %ecx,%ecx                               ;       \
-       je      1f                                      ;       \
-       cmpl    %ecx,thd                                ;       \
-       jne     1f                                      ;       \
-       pushl   $2f                                     ;       \
-       call    EXT(panic)                              ;       \
-       hlt                                             ;       \
-       .data                                           ;       \
-2:     String  "mylock attempt!"                       ;       \
-       .text                                           ;       \
-1:
+#if    CONFIG_DTRACE

 
 

-#define        METER_SIMPLE_LOCK_LOCK(reg)                             \
-       pushl   reg                                     ;       \
-       call    EXT(meter_simple_lock)                  ;       \
-       popl    reg
+       .globl  _lockstat_probe
+       .globl  _lockstat_probemap

 
 

-#define        METER_SIMPLE_LOCK_UNLOCK(reg)                           \
-       pushl   reg                                     ;       \
-       call    EXT(meter_simple_unlock)                ;       \
-       popl    reg
+/*
+ * LOCKSTAT_LABEL creates a dtrace symbol which contains
+ * a pointer into the lock code function body. At that
+ * point is a "ret" instruction that can be patched into
+ * a "nop"
+ */

 
 

-#else  /* MACH_LDEBUG */
-#define        CHECK_MUTEX_TYPE()
-#define        CHECK_SIMPLE_LOCK_TYPE
-#define        CHECK_THREAD(thd)
-#define CHECK_PREEMPTION_LEVEL()
-#define        CHECK_NO_SIMPLELOCKS()
-#define        CHECK_MYLOCK(thd)
-#define        METER_SIMPLE_LOCK_LOCK(reg)
-#define        METER_SIMPLE_LOCK_UNLOCK(reg)
-#endif /* MACH_LDEBUG */
+#define        LOCKSTAT_LABEL(lab) \
+       .data                                       ;\
+       .globl  lab                                 ;\
+       lab:                                        ;\
+       .quad 9f                                    ;\
+       .text                                       ;\
+       9:
+
+#define LOCKSTAT_RECORD(id, lck) \
+       push    %rbp                                ;       \
+       mov     %rsp,%rbp                           ;       \
+       movl    _lockstat_probemap + (id * 4)(%rip),%eax ;  \
+       test    %eax,%eax                           ;       \
+       je              9f                          ;       \
+       mov             lck, %rsi                   ;       \
+       mov             %rax, %rdi                  ;       \
+       mov             $0, %rdx                    ;       \
+       mov             $0, %rcx                    ;       \
+       mov             $0, %r8                     ;       \
+       mov             $0, %r9                     ;       \
+       call    *_lockstat_probe(%rip)              ;       \
+9:     leave
+       /* ret - left to subsequent code, e.g. return values */
+
+#endif /* CONFIG_DTRACE */

 
 

+/*
+ * For most routines, the hw_lock_t pointer is loaded into a
+ * register initially, and then either a byte or register-sized
+ * word is loaded/stored to the pointer
+ */

 
 /*
 
 /*

- *     void hw_lock_init(hw_lock_t)
+ *     void hw_lock_byte_init(volatile uint8_t *)

  *
  *

- *     Initialize a hardware lock.
+ *     Initialize a hardware byte lock.

  */
  */

-ENTRY(hw_lock_init)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
-       xorl    %eax,%eax
-       movb    %al,0(%edx)             /* clear the lock */
-       EMARF
-       ret
+LEAF_ENTRY(hw_lock_byte_init)
+       movb    $0, (%rdi)              /* clear the lock */
+       LEAF_RET

 
 /*
 
 /*

- *     void hw_lock_lock(hw_lock_t)
+ *     void    hw_lock_byte_lock(uint8_t *lock_byte)

  *
  *

- *     Acquire lock, spinning until it becomes available.
- *     MACH_RT:  also return with preemption disabled.
+ *     Acquire byte sized lock operand, spinning until it becomes available.
+ *     return with preemption disabled.

  */
  */

-ENTRY(hw_lock_lock)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */

 
 

-1:     DISABLE_PREEMPTION(%eax)
-       movb    $1,%cl
-       xchgb   0(%edx),%cl             /* try to acquire the HW lock */
-       testb   %cl,%cl                 /* success? */
+LEAF_ENTRY(hw_lock_byte_lock)
+       PREEMPTION_DISABLE
+       movl    $1, %ecx                /* Set lock value */
+1:
+       movb    (%rdi), %al             /* Load byte at address */
+       testb   %al,%al                 /* lock locked? */
+       jne     3f                      /* branch if so */
+       lock; cmpxchg %cl,(%rdi)        /* attempt atomic compare exchange */

        jne     3f
        jne     3f

-
-       EMARF                           /* if yes, then nothing left to do */
-       ret
-
-3:     ENABLE_PREEMPTION(%eax)         /* no reason we can't be preemptable now */
-
-       movb    $1,%cl
-2:     testb   %cl,0(%edx)             /* spin checking lock value in cache */
-       jne     2b                      /* non-zero means locked, keep spinning */
-       jmp     1b                      /* zero means unlocked, try to grab it */
+       LEAF_RET                        /* if yes, then nothing left to do */
+3:
+       PAUSE                           /* pause for hyper-threading */
+       jmp     1b                      /* try again */

 
 /*
 
 /*

- *     void hw_lock_unlock(hw_lock_t)
+ *     void hw_lock_byte_unlock(uint8_t *lock_byte)

  *
  *

- *     Unconditionally release lock.
- *     MACH_RT:  release preemption level.
+ *     Unconditionally release byte sized lock operand,
+ *     release preemption level.

  */
  */

-ENTRY(hw_lock_unlock)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
-       xorl    %eax,%eax
-       xchgb   0(%edx),%al             /* clear the lock... a mov instruction */
-                                       /* ...might be cheaper and less paranoid */
-       ENABLE_PREEMPTION(%eax)
-       EMARF
-       ret

 
 

+LEAF_ENTRY(hw_lock_byte_unlock)
+       movb $0, (%rdi)         /* Clear the lock byte */
+       PREEMPTION_ENABLE
+       LEAF_RET
+       

 /*
 /*

- *     unsigned int hw_lock_try(hw_lock_t)
- *     MACH_RT:  returns with preemption disabled on success.
+ * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
+ * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
+ * as a 64-bit quantity (this matches the existing PowerPC implementation,
+ * and the new x86 specific statistics are also maintained as 32-bit
+ * quantities).
+ *
+ *
+ * Enable this preprocessor define to record the first miss alone
+ * By default, we count every miss, hence multiple misses may be
+ * recorded for a single lock acquire attempt via lck_mtx_lock

  */
  */

-ENTRY(hw_lock_try)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
+#undef LOG_FIRST_MISS_ALONE    

 
 

-       DISABLE_PREEMPTION(%eax)
-       movb    $1,%cl
-       xchgb   0(%edx),%cl             /* try to acquire the HW lock */
-       testb   %cl,%cl                 /* success? */
-       jne     1f                      /* if yes, let the caller know */
+/*
+ * This preprocessor define controls whether the R-M-W update of the
+ * per-group statistics elements are atomic (LOCK-prefixed)
+ * Enabled by default.
+ */
+#define ATOMIC_STAT_UPDATES 1

 
 

-       movl    $1,%eax                 /* success */
-       EMARF
-       ret
+#if defined(ATOMIC_STAT_UPDATES)
+#define LOCK_IF_ATOMIC_STAT_UPDATES lock
+#else
+#define LOCK_IF_ATOMIC_STAT_UPDATES
+#endif /* ATOMIC_STAT_UPDATES */

 
 

-1:     ENABLE_PREEMPTION(%eax)         /* failure:  release preemption... */
-       xorl    %eax,%eax               /* ...and return failure */
-       EMARF
-       ret     

 
 /*
 
 /*

- *     unsigned int hw_lock_held(hw_lock_t)
- *     MACH_RT:  doesn't change preemption state.
- *     N.B.  Racy, of course.
+ * For most routines, the lck_mtx_t pointer is loaded into a
+ * register initially, and the owner field checked for indirection.
+ * Eventually the lock owner is loaded into a register and examined.

  */
  */

-ENTRY(hw_lock_held)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */

 
 

-       movb    $1,%cl
-       testb   %cl,0(%edx)             /* check lock value */
-       jne     1f                      /* non-zero means locked */
-       xorl    %eax,%eax               /* tell caller:  lock wasn't locked */
-       EMARF
-       ret     
+#define M_OWNER                MUTEX_OWNER
+#define M_PTR          MUTEX_PTR
+#define M_STATE                MUTEX_STATE     
+       

 
 

-1:     movl    $1,%eax                 /* tell caller:  lock was locked */
-       EMARF
-       ret
+#define LMTX_ENTER_EXTENDED                                    \
+       mov     M_PTR(%rdx), %rdx                       ;       \
+       xor     %r11, %r11                              ;       \
+       mov     MUTEX_GRP(%rdx), %r10                   ;       \
+       LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
+       incq    GRP_MTX_STAT_UTIL(%r10)
+
+
+#if    LOG_FIRST_MISS_ALONE
+#define LMTX_UPDATE_MISS                                       \
+       test    $1, %r11                                ;       \
+       jnz     11f                                     ;       \
+       LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
+       incl    GRP_MTX_STAT_MISS(%r10)                 ;       \
+       or      $1, %r11                                ;       \
+11:
+#else
+#define LMTX_UPDATE_MISS                                       \
+       LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
+       incl    GRP_MTX_STAT_MISS(%r10)
+#endif

        
 
        
 

+#if    LOG_FIRST_MISS_ALONE
+#define LMTX_UPDATE_WAIT                                       \
+       test    $2, %r11                                ;       \
+       jnz     11f                                     ;       \
+       LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
+       incl    GRP_MTX_STAT_WAIT(%r10)                 ;       \
+       or      $2, %r11                                ;       \
+11:
+#else
+#define LMTX_UPDATE_WAIT                                       \
+       LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
+       incl    GRP_MTX_STAT_WAIT(%r10)
+#endif

 
 

-#if    0

 
 

+/*
+ * Record the "direct wait" statistic, which indicates if a
+ * miss proceeded to block directly without spinning--occurs
+ * if the owner of the mutex isn't running on another processor
+ * at the time of the check.
+ */
+#define LMTX_UPDATE_DIRECT_WAIT                                        \
+       LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
+       incl    GRP_MTX_STAT_DIRECT_WAIT(%r10)

 
 

-ENTRY(_usimple_lock_init)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
-       xorl    %eax,%eax
-       movb    %al,USL_INTERLOCK(%edx) /* unlock the HW lock */
-       EMARF
-       ret
+       
+#define LMTX_CALLEXT1(func_name)               \
+       cmp     %rdx, %rdi              ;       \
+       je      12f                     ;       \
+       push    %r10                    ;       \
+       push    %r11                    ;       \
+12:    push    %rdi                    ;       \
+       push    %rdx                    ;       \
+       mov     %rdx, %rdi              ;       \
+       call    EXT(func_name)          ;       \
+       pop     %rdx                    ;       \
+       pop     %rdi                    ;       \
+       cmp     %rdx, %rdi              ;       \
+       je      12f                     ;       \
+       pop     %r11                    ;       \
+       pop     %r10                    ;       \
+12:
+       
+#define LMTX_CALLEXT2(func_name, reg)          \
+       cmp     %rdx, %rdi              ;       \
+       je      12f                     ;       \
+       push    %r10                    ;       \
+       push    %r11                    ;       \
+12:    push    %rdi                    ;       \
+       push    %rdx                    ;       \
+       mov     reg, %rsi               ;       \
+       mov     %rdx, %rdi              ;       \
+       call    EXT(func_name)          ;       \
+       pop     %rdx                    ;       \
+       pop     %rdi                    ;       \
+       cmp     %rdx, %rdi              ;       \
+       je      12f                     ;       \
+       pop     %r11                    ;       \
+       pop     %r10                    ;       \
+12:
+
+
+#define M_WAITERS_MSK          0x0000ffff
+#define M_PRIORITY_MSK         0x00ff0000
+#define M_ILOCKED_MSK          0x01000000
+#define M_MLOCKED_MSK          0x02000000
+#define M_PROMOTED_MSK         0x04000000
+#define M_SPIN_MSK             0x08000000

 
 

-ENTRY(_simple_lock)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
+/*
+ *     void lck_mtx_assert(lck_mtx_t* l, unsigned int)
+ *     Takes the address of a lock, and an assertion type as parameters.
+ *     The assertion can take one of two forms determine by the type
+ *     parameter: either the lock is held by the current thread, and the
+ *     type is LCK_MTX_ASSERT_OWNED, or it isn't and the type is
+ *     LCK_MTX_ASSERT_NOTOWNED. Calls panic on assertion failure.
+ *     
+ */

 
 

-       CHECK_SIMPLE_LOCK_TYPE()
+NONLEAF_ENTRY(lck_mtx_assert)
+        mov    %rdi, %rdx                      /* Load lock address */
+       mov     %gs:CPU_ACTIVE_THREAD, %rax     /* Load current thread */
+
+       mov     M_STATE(%rdx), %ecx
+       cmp     $(MUTEX_IND), %ecx              /* Is this an indirect mutex? */
+       jne     0f
+       mov     M_PTR(%rdx), %rdx               /* If so, take indirection */
+0:     
+       mov     M_OWNER(%rdx), %rcx             /* Load owner */
+       cmp     $(MUTEX_ASSERT_OWNED), %rsi
+       jne     2f                              /* Assert ownership? */
+       cmp     %rax, %rcx                      /* Current thread match? */
+       jne     3f                              /* no, go panic */
+       testl   $(M_ILOCKED_MSK | M_MLOCKED_MSK), M_STATE(%rdx)
+       je      3f
+1:                                             /* yes, we own it */
+       NONLEAF_RET
+2:
+       cmp     %rax, %rcx                      /* Current thread match? */
+       jne     1b                              /* No, return */
+       ALIGN_STACK()
+       LOAD_PTR_ARG1(%rdx)
+       LOAD_STRING_ARG0(mutex_assert_owned_str)
+       jmp     4f
+3:
+       ALIGN_STACK()
+       LOAD_PTR_ARG1(%rdx)
+       LOAD_STRING_ARG0(mutex_assert_not_owned_str)
+4:
+       CALL_PANIC()
+
+
+lck_mtx_destroyed:
+       ALIGN_STACK()
+       LOAD_PTR_ARG1(%rdx)
+       LOAD_STRING_ARG0(mutex_interlock_destroyed_str)
+       CALL_PANIC()
+       

 
 

-       DISABLE_PREEMPTION(%eax)
+.data
+mutex_assert_not_owned_str:
+       .asciz  "mutex (%p) not owned\n"
+mutex_assert_owned_str:
+       .asciz  "mutex (%p) owned\n"
+mutex_interlock_destroyed_str:
+       .asciz  "trying to interlock destroyed mutex (%p)"
+.text

 
 

-sl_get_hw:
-       movb    $1,%cl
-       xchgb   USL_INTERLOCK(%edx),%cl /* try to acquire the HW lock */
-       testb   %cl,%cl                 /* did we succeed? */

 
 

-#if    MACH_LDEBUG
-       je      5f
-       CHECK_MYLOCK(S_THREAD)
-       jmp     sl_get_hw
-5:
-#else  /* MACH_LDEBUG */
-       jne     sl_get_hw               /* no, try again */
-#endif /* MACH_LDEBUG */

 
 

+/*
+ * lck_mtx_lock()
+ * lck_mtx_try_lock()
+ * lck_mtx_unlock()
+ * lck_mtx_lock_spin()
+ * lck_mtx_lock_spin_always()
+ * lck_mtx_try_lock_spin()
+ * lck_mtx_try_lock_spin_always()
+ * lck_mtx_convert_spin()
+ */
+NONLEAF_ENTRY(lck_mtx_lock_spin_always)
+       mov     %rdi, %rdx              /* fetch lock pointer */
+       jmp     Llmls_avoid_check
+       
+NONLEAF_ENTRY(lck_mtx_lock_spin)
+       mov     %rdi, %rdx              /* fetch lock pointer */
+
+       CHECK_PREEMPTION_LEVEL()
+Llmls_avoid_check:
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx  /* is the interlock or mutex held */
+       jnz     Llmls_slow
+Llmls_try:                             /* no - can't be INDIRECT, DESTROYED or locked */
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK | M_SPIN_MSK), %ecx
+
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     Llmls_busy_disabled
+
+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       mov     %rax, M_OWNER(%rdx)     /* record owner of interlock */

 #if    MACH_LDEBUG
 #if    MACH_LDEBUG

-       movl    L_PC,%ecx
-       movl    %ecx,S_PC
-       movl    $CPD_ACTIVE_THREAD,%eax
-       movl    %gs:(%eax),%ecx
-       movl    %ecx,S_THREAD
-       incl    CX(EXT(simple_lock_count),%eax)
-#if 0
-       METER_SIMPLE_LOCK_LOCK(%edx)
-#endif
-#if    NCPUS == 1
-       pushf
-       pushl   %edx
-       cli
-       call    EXT(lock_stack_push)
-       popl    %edx
-       popfl
-#endif /* NCPUS == 1 */
+       test    %rax, %rax
+       jz      1f
+       incl    TH_MUTEX_COUNT(%rax)    /* lock statistic */
+1:     

 #endif /* MACH_LDEBUG */
 
 #endif /* MACH_LDEBUG */
 

-       EMARF
+       /* return with the interlock held and preemption disabled */
+       leave
+#if    CONFIG_DTRACE
+       LOCKSTAT_LABEL(_lck_mtx_lock_spin_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, %rdx)
+#endif

        ret
 
        ret
 

-ENTRY(_simple_lock_try)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
+Llmls_slow:    
+       test    $M_ILOCKED_MSK, %ecx            /* is the interlock held */
+       jz      Llml_contended                  /* no, must have been the mutex */

 
 

-       CHECK_SIMPLE_LOCK_TYPE()
+       cmp     $(MUTEX_DESTROYED), %ecx        /* check to see if its marked destroyed */
+       je      lck_mtx_destroyed
+       cmp     $(MUTEX_IND), %ecx              /* Is this an indirect mutex */
+       jne     Llmls_loop                      /* no... must be interlocked */

 
 

-       DISABLE_PREEMPTION(%eax)
+       LMTX_ENTER_EXTENDED

 
 

-       movb    $1,%cl
-       xchgb   USL_INTERLOCK(%edx),%cl /* try to acquire the HW lock */
-       testb   %cl,%cl                 /* did we succeed? */
-       jne     1f                      /* no, return failure */
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_SPIN_MSK), %ecx
+       jz      Llmls_loop1

 
 

-#if    MACH_LDEBUG
-       movl    L_PC,%ecx
-       movl    %ecx,S_PC
-       movl    $CPD_ACTIVE_THREAD,%eax
-       movl    %gs:(%eax),%ecx
-       movl    %ecx,S_THREAD
-       incl    CX(EXT(simple_lock_count),%eax)
-#if 0
-       METER_SIMPLE_LOCK_LOCK(%edx)
-#endif
-#if    NCPUS == 1
-       pushf
-       pushl   %edx
-       cli
-       call    EXT(lock_stack_push)
-       popl    %edx
-       popfl
-#endif /* NCPUS == 1 */
-#endif /* MACH_LDEBUG */
+       LMTX_UPDATE_MISS                /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
+Llmls_loop:
+       PAUSE
+       mov     M_STATE(%rdx), %ecx
+Llmls_loop1:
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
+       jz      Llmls_try
+       test    $(M_MLOCKED_MSK), %ecx
+       jnz     Llml_contended                  /* mutex owned by someone else, go contend for it */
+       jmp     Llmls_loop

 
 

-       movl    $1,%eax                 /* return success */
+Llmls_busy_disabled:
+       PREEMPTION_ENABLE
+       jmp     Llmls_loop

 
 

-       EMARF
-       ret

 
 

-1:
-       ENABLE_PREEMPTION(%eax)
-
-       xorl    %eax,%eax               /* and return failure */
+       
+NONLEAF_ENTRY(lck_mtx_lock)
+       mov     %rdi, %rdx              /* fetch lock pointer */

 
 

-       EMARF
-       ret
+       CHECK_PREEMPTION_LEVEL()

 
 

-ENTRY(_simple_unlock)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx  /* is the interlock or mutex held */
+       jnz     Llml_slow
+Llml_try:                              /* no - can't be INDIRECT, DESTROYED or locked */
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx

 
 

-       CHECK_SIMPLE_LOCK_TYPE()
-       CHECK_THREAD(S_THREAD)
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     Llml_busy_disabled

 
 

+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       mov     %rax, M_OWNER(%rdx)     /* record owner of mutex */

 #if    MACH_LDEBUG
 #if    MACH_LDEBUG

-       xorl    %eax,%eax
-       movl    %eax,S_THREAD           /* disown thread */
-       MP_DISABLE_PREEMPTION(%eax)
-       CPU_NUMBER(%eax)
-       decl    CX(EXT(simple_lock_count),%eax)
-       MP_ENABLE_PREEMPTION(%eax)
-#if 0
-       METER_SIMPLE_LOCK_UNLOCK(%edx)
-#endif
-#if    NCPUS == 1
-       pushf
-       pushl   %edx
-       cli
-       call    EXT(lock_stack_pop)
-       popl    %edx
-       popfl
-#endif /* NCPUS == 1 */
+       test    %rax, %rax
+       jz      1f
+       incl    TH_MUTEX_COUNT(%rax)    /* lock statistic */
+1:

 #endif /* MACH_LDEBUG */
 
 #endif /* MACH_LDEBUG */
 

-       xorb    %cl,%cl
-       xchgb   USL_INTERLOCK(%edx),%cl /* unlock the HW lock */
-
-       ENABLE_PREEMPTION(%eax)
-
-       EMARF
-       ret
-
-#endif /* 0 */
+       testl   $(M_WAITERS_MSK), M_STATE(%rdx)
+       jz      Llml_finish

 
 

+       LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)

 
 

-ENTRY(mutex_init)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */
-       xorl    %eax,%eax
-       movb    %al,M_ILK               /* clear interlock */
-       movb    %al,M_LOCKED            /* clear locked flag */
-       movw    %ax,M_WAITERS           /* init waiter count */
+Llml_finish:
+       andl    $(~M_ILOCKED_MSK), M_STATE(%rdx)
+       PREEMPTION_ENABLE
+       
+       cmp     %rdx, %rdi              /* is this an extended mutex */
+       jne     2f

 
 

-#if    MACH_LDEBUG
-       movl    $MUTEX_TAG,M_TYPE       /* set lock type */
-       movl    %eax,M_PC               /* init caller pc */
-       movl    %eax,M_THREAD           /* and owning thread */
+       leave
+#if    CONFIG_DTRACE
+       LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, %rdx)
+#endif
+       ret
+2:     
+       leave
+#if    CONFIG_DTRACE
+       LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, %rdx)

 #endif
 #endif

-#if    ETAP_LOCK_TRACE
-       movl    L_ARG1,%ecx             /* fetch event type */
-       pushl   %ecx                    /* push event type */
-       pushl   %edx                    /* push mutex address */
-       call    EXT(etap_mutex_init)    /* init ETAP data */
-       addl    $8,%esp
-#endif /* ETAP_LOCK_TRACE */
-
-       EMARF

        ret
 
        ret
 

-ENTRY(_mutex_lock)
-       FRAME
+       
+Llml_slow:
+       test    $M_ILOCKED_MSK, %ecx            /* is the interlock held */
+       jz      Llml_contended                  /* no, must have been the mutex */
+       
+       cmp     $(MUTEX_DESTROYED), %ecx        /* check to see if its marked destroyed */
+       je      lck_mtx_destroyed
+       cmp     $(MUTEX_IND), %ecx              /* Is this an indirect mutex? */
+       jne     Llml_loop                       /* no... must be interlocked */
+
+       LMTX_ENTER_EXTENDED
+
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_SPIN_MSK), %ecx
+       jz      Llml_loop1
+
+       LMTX_UPDATE_MISS                /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
+Llml_loop:
+       PAUSE
+       mov     M_STATE(%rdx), %ecx
+Llml_loop1:
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
+       jz      Llml_try
+       test    $(M_MLOCKED_MSK), %ecx
+       jnz     Llml_contended                  /* mutex owned by someone else, go contend for it */
+       jmp     Llml_loop
+
+Llml_busy_disabled:
+       PREEMPTION_ENABLE
+       jmp     Llml_loop

 
 

-#if    ETAP_LOCK_TRACE
-       subl    $12,%esp                /* make room for locals */
-       movl    $0,SWT_HI               /* set wait time to zero (HI) */
-       movl    $0,SWT_LO               /* set wait time to zero (LO) */
-       movl    $0,MISSED               /* clear local miss marker */
-#endif /* ETAP_LOCK_TRACE */
+       
+Llml_contended:
+       cmp     %rdx, %rdi              /* is this an extended mutex */
+       je      0f
+       LMTX_UPDATE_MISS
+0:     
+       LMTX_CALLEXT1(lck_mtx_lock_spinwait_x86)
+
+       test    %rax, %rax
+       jz      Llml_acquired           /* acquired mutex, interlock held and preemption disabled */
+
+       cmp     $1, %rax                /* check for direct wait status */
+       je      2f
+       cmp     %rdx, %rdi              /* is this an extended mutex */
+       je      2f
+       LMTX_UPDATE_DIRECT_WAIT
+2:     
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_ILOCKED_MSK), %ecx
+       jnz     6f
+
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK), %ecx  /* try to take the interlock */
+
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     5f

 
 

-       movl    L_ARG0,%edx             /* fetch lock pointer */
+       test    $(M_MLOCKED_MSK), %ecx  /* we've got the interlock and */
+       jnz     3f
+       or      $(M_MLOCKED_MSK), %ecx  /* the mutex is free... grab it directly */
+       mov     %ecx, M_STATE(%rdx)
+       
+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       mov     %rax, M_OWNER(%rdx)     /* record owner of mutex */
+#if    MACH_LDEBUG
+       test    %rax, %rax
+       jz      1f
+       incl    TH_MUTEX_COUNT(%rax)    /* lock statistic */
+1:
+#endif /* MACH_LDEBUG */

 
 

-       CHECK_MUTEX_TYPE()
-       CHECK_NO_SIMPLELOCKS()
-       CHECK_PREEMPTION_LEVEL()
+Llml_acquired:
+       testl   $(M_WAITERS_MSK), M_STATE(%rdx)
+       jnz     1f
+       mov     M_OWNER(%rdx), %rax
+       mov     TH_WAS_PROMOTED_ON_WAKEUP(%rax), %eax
+       test    %eax, %eax
+       jz      Llml_finish
+1:     
+       LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
+       jmp     Llml_finish
+
+3:                                     /* interlock held, mutex busy */
+       cmp     %rdx, %rdi              /* is this an extended mutex */
+       je      4f
+       LMTX_UPDATE_WAIT
+4:     
+       LMTX_CALLEXT1(lck_mtx_lock_wait_x86)
+       jmp     Llml_contended
+5:     
+       PREEMPTION_ENABLE
+6:
+       PAUSE
+       jmp     2b
+       

 
 

-ml_retry:
-       DISABLE_PREEMPTION(%eax)
+NONLEAF_ENTRY(lck_mtx_try_lock_spin_always)
+       mov     %rdi, %rdx              /* fetch lock pointer */
+       jmp     Llmts_avoid_check

 
 

-ml_get_hw:
-       movb    $1,%cl
-       xchgb   %cl,M_ILK
-       testb   %cl,%cl                 /* did we succeed? */
-       jne     ml_get_hw               /* no, try again */
+NONLEAF_ENTRY(lck_mtx_try_lock_spin)
+       mov     %rdi, %rdx              /* fetch lock pointer */

 
 

-/*
-/ Beware of a race between this code path and the inline ASM fast-path locking
-/ sequence which attempts to lock a mutex by directly setting the locked flag 
-/ 
-*/
+Llmts_avoid_check:
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx  /* is the interlock or mutex held */
+       jnz     Llmts_slow
+Llmts_try:                             /* no - can't be INDIRECT, DESTROYED or locked */
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK | M_SPIN_MSK), %rcx

 
 

-       movb    $1,%cl
-       xchgb   %cl,M_LOCKED            /* try to set locked flag */
-       testb   %cl,%cl                 /* is the mutex locked? */
-       jne     ml_fail                 /* yes, we lose */
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     Llmts_busy_disabled

 
 

+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       mov     %rax, M_OWNER(%rdx)     /* record owner of mutex */

 #if    MACH_LDEBUG
 #if    MACH_LDEBUG

-       movl    L_PC,%ecx
-       movl    %ecx,M_PC
-       movl    $CPD_ACTIVE_THREAD,%eax
-       movl    %gs:(%eax),%ecx
-       movl    %ecx,M_THREAD
-       testl   %ecx,%ecx
-       je      3f
-       incl    TH_MUTEX_COUNT(%ecx)
-3:
+       test    %rax, %rax
+       jz      1f
+       incl    TH_MUTEX_COUNT(%rax)    /* lock statistic */
+1:
+#endif /* MACH_LDEBUG */
+
+       leave
+
+#if    CONFIG_DTRACE
+       mov     $1, %rax                        /* return success */
+       LOCKSTAT_LABEL(_lck_mtx_try_lock_spin_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, %rdx)

 #endif
 #endif

+       mov     $1, %rax                        /* return success */
+       ret

 
 

-       xorb    %cl,%cl
-       xchgb   %cl,M_ILK
+Llmts_slow:
+       test    $(M_ILOCKED_MSK), %ecx  /* is the interlock held */
+       jz      Llmts_fail                      /* no, must be held as a mutex */
+
+       cmp     $(MUTEX_DESTROYED), %ecx        /* check to see if its marked destroyed */
+       je      lck_mtx_destroyed
+       cmp     $(MUTEX_IND), %ecx              /* Is this an indirect mutex? */
+       jne     Llmts_loop1
+
+       LMTX_ENTER_EXTENDED
+Llmts_loop:
+       PAUSE
+       mov     M_STATE(%rdx), %ecx
+Llmts_loop1:
+       test    $(M_MLOCKED_MSK | M_SPIN_MSK), %ecx
+       jnz     Llmts_fail
+       test    $(M_ILOCKED_MSK), %ecx
+       jz      Llmts_try
+       jmp     Llmts_loop
+       
+Llmts_busy_disabled:
+       PREEMPTION_ENABLE
+       jmp     Llmts_loop

 
 

-       ENABLE_PREEMPTION(%eax)

 
 

-#if    ETAP_LOCK_TRACE
-       movl    L_PC,%eax               /* fetch pc */
-       pushl   SWT_LO                  /* push wait time (low) */
-       pushl   SWT_HI                  /* push wait time (high) */
-       pushl   %eax                    /* push pc */
-       pushl   %edx                    /* push mutex address */
-       call    EXT(etap_mutex_hold)    /* collect hold timestamp */
-       addl    $16+12,%esp             /* clean up stack, adjusting for locals */
-#endif /* ETAP_LOCK_TRACE */
+       
+NONLEAF_ENTRY(lck_mtx_try_lock)
+       mov     %rdi, %rdx              /* fetch lock pointer */
+
+       mov     M_STATE(%rdx), %ecx
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx  /* is the interlock or mutex held */
+       jnz     Llmt_slow       
+Llmt_try:                              /* no - can't be INDIRECT, DESTROYED or locked */
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
+       
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     Llmt_busy_disabled

 
 

-       EMARF
+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       mov     %rax, M_OWNER(%rdx)     /* record owner of mutex */
+#if    MACH_LDEBUG
+       test    %rax, %rax
+       jz      1f
+       incl    TH_MUTEX_COUNT(%rax)    /* lock statistic */
+1:
+#endif /* MACH_LDEBUG */
+
+       test    $(M_WAITERS_MSK), %ecx
+       jz      0f
+
+       LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
+0:
+       andl    $(~M_ILOCKED_MSK), M_STATE(%rdx)
+       PREEMPTION_ENABLE
+
+       leave
+#if    CONFIG_DTRACE
+       mov     $1, %rax                        /* return success */
+       /* Dtrace probe: LS_LCK_MTX_TRY_LOCK_ACQUIRE */
+       LOCKSTAT_LABEL(_lck_mtx_try_lock_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx from above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, %rdx)
+#endif 
+       mov     $1, %rax                        /* return success */

        ret
 
        ret
 

-ml_fail:
-#if    ETAP_LOCK_TRACE
-       cmp     $0,MISSED               /* did we already take a wait timestamp? */
-       jne     ml_block                /* yup. carry-on */
-       pushl   %edx                    /* push mutex address */
-       call    EXT(etap_mutex_miss)    /* get wait timestamp */
-       movl    %eax,SWT_HI             /* set wait time (high word) */
-       movl    %edx,SWT_LO             /* set wait time (low word) */
-       popl    %edx                    /* clean up stack */
-       movl    $1,MISSED               /* mark wait timestamp as taken */
-#endif /* ETAP_LOCK_TRACE */
-
-ml_block:
-       CHECK_MYLOCK(M_THREAD)
-       pushl   %edx                    /* push mutex address */
-       call    EXT(mutex_lock_wait)    /* wait for the lock */
-       addl    $4,%esp
-       movl    L_ARG0,%edx             /* refetch lock pointer */
-       jmp     ml_retry                /* and try again */
-
-ENTRY(_mutex_try)
-       FRAME
+Llmt_slow:
+       test    $(M_ILOCKED_MSK), %ecx  /* is the interlock held */
+       jz      Llmt_fail                       /* no, must be held as a mutex */
+
+       cmp     $(MUTEX_DESTROYED), %ecx        /* check to see if its marked destroyed */
+       je      lck_mtx_destroyed
+       cmp     $(MUTEX_IND), %ecx              /* Is this an indirect mutex? */
+       jne     Llmt_loop
+
+       LMTX_ENTER_EXTENDED
+Llmt_loop:
+       PAUSE
+       mov     M_STATE(%rdx), %ecx
+Llmt_loop1:
+       test    $(M_MLOCKED_MSK | M_SPIN_MSK), %ecx
+       jnz     Llmt_fail
+       test    $(M_ILOCKED_MSK), %ecx
+       jz      Llmt_try
+       jmp     Llmt_loop
+
+Llmt_busy_disabled:
+       PREEMPTION_ENABLE
+       jmp     Llmt_loop
+
+
+Llmt_fail:
+Llmts_fail:
+       cmp     %rdx, %rdi                      /* is this an extended mutex */
+       je      0f
+       LMTX_UPDATE_MISS
+0:
+       xor     %rax, %rax
+       NONLEAF_RET
+

 
 

-#if    ETAP_LOCK_TRACE
-       subl    $8,%esp                 /* make room for locals */
-       movl    $0,SWT_HI               /* set wait time to zero (HI) */
-       movl    $0,SWT_LO               /* set wait time to zero (LO) */
-#endif /* ETAP_LOCK_TRACE */

 
 

-       movl    L_ARG0,%edx             /* fetch lock pointer */
+NONLEAF_ENTRY(lck_mtx_convert_spin)
+       mov     %rdi, %rdx                      /* fetch lock pointer */

 
 

-       CHECK_MUTEX_TYPE()
-       CHECK_NO_SIMPLELOCKS()
+       mov     M_STATE(%rdx), %ecx
+       cmp     $(MUTEX_IND), %ecx              /* Is this an indirect mutex? */
+       jne     0f
+       mov     M_PTR(%rdx), %rdx               /* If so, take indirection */
+       mov     M_STATE(%rdx), %ecx
+0:
+       test    $(M_MLOCKED_MSK), %ecx          /* already owned as a mutex, just return */
+       jnz     2f
+       test    $(M_WAITERS_MSK), %ecx          /* are there any waiters? */
+       jz      1f
+
+       LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
+       mov     M_STATE(%rdx), %ecx
+1:     
+       and     $(~(M_ILOCKED_MSK | M_SPIN_MSK)), %ecx  /* convert from spin version to mutex */
+       or      $(M_MLOCKED_MSK), %ecx
+       mov     %ecx, M_STATE(%rdx)             /* since I own the interlock, I don't need an atomic update */
+
+       PREEMPTION_ENABLE
+2:     
+       NONLEAF_RET

 
 

-       xorl    %eax,%eax
-       movb    $1,%al                  /* locked value for mutex */
-       xchgb   %al,M_LOCKED            /* swap locked values */
-       xorb    $1,%al                  /* generate return value */
+       

 
 

-#if    MACH_LDEBUG || ETAP_LOCK_TRACE
-       testl   %eax,%eax               /* did we succeed? */
-       je      2f                      /* no, skip */
-#endif
+NONLEAF_ENTRY(lck_mtx_unlock)
+       mov     %rdi, %rdx              /* fetch lock pointer */
+Llmu_entry:
+       mov     M_STATE(%rdx), %ecx
+Llmu_prim:
+       cmp     $(MUTEX_IND), %ecx      /* Is this an indirect mutex? */
+       je      Llmu_ext
+
+Llmu_chktype:
+       test    $(M_MLOCKED_MSK), %ecx  /* check for full mutex */
+       jz      Llmu_unlock
+Llmu_mutex:
+       test    $(M_ILOCKED_MSK), %rcx  /* have to wait for interlock to clear */
+       jnz     Llmu_busy
+
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       and     $(~M_MLOCKED_MSK), %ecx /* drop mutex */
+       or      $(M_ILOCKED_MSK), %ecx  /* pick up interlock */
+
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     Llmu_busy_disabled      /* branch on failure to spin loop */
+
+Llmu_unlock:
+       xor     %rax, %rax
+       mov     %rax, M_OWNER(%rdx)
+       mov     %rcx, %rax              /* keep original state in %ecx for later evaluation */
+       and     $(~(M_ILOCKED_MSK | M_SPIN_MSK | M_PROMOTED_MSK)), %rax
+
+       test    $(M_WAITERS_MSK), %eax
+       jz      2f
+       dec     %eax                    /* decrement waiter count */
+2:     
+       mov     %eax, M_STATE(%rdx)     /* since I own the interlock, I don't need an atomic update */

 
 #if    MACH_LDEBUG
 
 #if    MACH_LDEBUG

-       movl    L_PC,%ecx
-       movl    %ecx,M_PC
-       movl    $CPD_ACTIVE_THREAD,%ecx
-       movl    %gs:(%ecx),%ecx
-       movl    %ecx,M_THREAD
-       testl   %ecx,%ecx
-       je      1f
-       incl    TH_MUTEX_COUNT(%ecx)
+       /* perform lock statistics after drop to prevent delay */
+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       test    %rax, %rax
+       jz      1f
+       decl    TH_MUTEX_COUNT(%rax)    /* lock statistic */

 1:
 1:

-#endif
+#endif /* MACH_LDEBUG */

 
 

-#if    ETAP_LOCK_TRACE
-       movl    L_PC,%eax               /* fetch pc */
-       pushl   SWT_LO                  /* push wait time (low) */
-       pushl   SWT_HI                  /* push wait time (high) */
-       pushl   %eax                    /* push pc */
-       pushl   %edx                    /* push mutex address */
-       call    EXT(etap_mutex_hold)    /* get start hold timestamp */
-       addl    $16,%esp                /* clean up stack, adjusting for locals */
-       movl    $1,%eax                 /* put back successful return value */
-#endif /* ETAP_LOCK_TRACE */
-
-#if    MACH_LDEBUG || ETAP_LOCK_TRACE
-2:
-#if    ETAP_LOCK_TRACE
-       addl    $8,%esp                 /* pop stack claimed on entry */
+       test    $(M_PROMOTED_MSK | M_WAITERS_MSK), %ecx
+       jz      3f
+
+       LMTX_CALLEXT2(lck_mtx_unlock_wakeup_x86, %rcx)
+3:     
+       PREEMPTION_ENABLE
+
+       cmp     %rdx, %rdi
+       jne     4f
+
+       leave
+#if    CONFIG_DTRACE
+       /* Dtrace: LS_LCK_MTX_UNLOCK_RELEASE */
+       LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx from above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, %rdx)

 #endif
 #endif

+       ret
+4:     
+       leave
+#if    CONFIG_DTRACE
+       /* Dtrace: LS_LCK_MTX_EXT_UNLOCK_RELEASE */
+       LOCKSTAT_LABEL(_lck_mtx_ext_unlock_lockstat_patch_point)
+       ret
+       /* inherit lock pointer in %rdx from above */
+       LOCKSTAT_RECORD(LS_LCK_MTX_EXT_UNLOCK_RELEASE, %rdx)

 #endif
 #endif

-
-       EMARF

        ret
 
        ret
 

-ENTRY(mutex_unlock)
-       FRAME
-       movl    L_ARG0,%edx             /* fetch lock pointer */

 
 

-#if    ETAP_LOCK_TRACE
-       pushl   %edx                    /* push mutex address */
-       call    EXT(etap_mutex_unlock)  /* collect ETAP data */
-       popl    %edx                    /* restore mutex address */
-#endif /* ETAP_LOCK_TRACE */
+Llmu_busy_disabled:
+       PREEMPTION_ENABLE
+Llmu_busy:
+       PAUSE
+       mov     M_STATE(%rdx), %ecx
+       jmp     Llmu_mutex

 
 

-       CHECK_MUTEX_TYPE()
-       CHECK_THREAD(M_THREAD)
+Llmu_ext:
+       mov     M_PTR(%rdx), %rdx
+       mov     M_OWNER(%rdx), %rax
+       mov     %gs:CPU_ACTIVE_THREAD, %rcx
+       CHECK_UNLOCK(%rcx, %rax)
+       mov     M_STATE(%rdx), %ecx
+       jmp     Llmu_chktype

 
 

-       DISABLE_PREEMPTION(%eax)

 
 

-mu_get_hw:
-       movb    $1,%cl
-       xchgb   %cl,M_ILK
-       testb   %cl,%cl                 /* did we succeed? */
-       jne     mu_get_hw               /* no, try again */
+       
+LEAF_ENTRY(lck_mtx_ilk_try_lock)
+       mov     %rdi, %rdx              /* fetch lock pointer - no indirection here */

 
 

-       cmpw    $0,M_WAITERS            /* are there any waiters? */
-       jne     mu_wakeup               /* yes, more work to do */
+       mov     M_STATE(%rdx), %ecx

 
 

-mu_doit:
-#if    MACH_LDEBUG
-       xorl    %eax,%eax
-       movl    %eax,M_THREAD           /* disown thread */
-       movl    $CPD_ACTIVE_THREAD,%eax
-       movl    %gs:(%eax),%ecx
-       testl   %ecx,%ecx
-       je      0f
-       decl    TH_MUTEX_COUNT(%ecx)
-0:
-#endif
+       test    $(M_ILOCKED_MSK), %ecx  /* can't have the interlock yet */
+       jnz     3f

 
 

-       xorb    %cl,%cl
-       xchgb   %cl,M_LOCKED            /* unlock the mutex */
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK), %ecx

 
 

-       xorb    %cl,%cl
-       xchgb   %cl,M_ILK
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     2f                      /* return failure after re-enabling preemption */
+
+       mov     $1, %rax                /* return success with preemption disabled */
+       LEAF_RET
+2:     
+       PREEMPTION_ENABLE               /* need to re-enable preemption */
+3:     
+       xor     %rax, %rax              /* return failure */
+       LEAF_RET
+       

 
 

-       ENABLE_PREEMPTION(%eax)
+LEAF_ENTRY(lck_mtx_ilk_unlock)
+       mov     %rdi, %rdx              /* fetch lock pointer - no indirection here */

 
 

-       EMARF
-       ret
+       andl    $(~M_ILOCKED_MSK), M_STATE(%rdx)

 
 

-mu_wakeup:
-       pushl   %edx                    /* push mutex address */
-       call    EXT(mutex_unlock_wakeup)/* yes, wake a thread */
-       addl    $4,%esp
-       movl    L_ARG0,%edx             /* refetch lock pointer */
-       jmp     mu_doit
+       PREEMPTION_ENABLE               /* need to re-enable preemption */

 
 

-ENTRY(interlock_unlock)
-       FRAME
-       movl    L_ARG0,%edx
+       LEAF_RET

 
 

-       xorb    %cl,%cl
-       xchgb   %cl,M_ILK
+       
+LEAF_ENTRY(lck_mtx_lock_grab_mutex)
+       mov     %rdi, %rdx              /* fetch lock pointer - no indirection here */

 
 

-       ENABLE_PREEMPTION(%eax)
+       mov     M_STATE(%rdx), %ecx

 
 

-       EMARF
-       ret
+       test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx  /* can't have the mutex yet */
+       jnz     3f

 
 

-       
-ENTRY(_disable_preemption)
-#if    MACH_RT
-       _DISABLE_PREEMPTION(%eax)
-#endif /* MACH_RT */
-       ret
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx

 
 

-ENTRY(_enable_preemption)
-#if    MACH_RT
-#if    MACH_ASSERT
-       movl    $CPD_PREEMPTION_LEVEL,%eax
-       cmpl    $0,%gs:(%eax)
-       jg      1f
-       pushl   %gs:(%eax)
-       pushl   $2f
-       call    EXT(panic)
-       hlt
-       .data
-2:     String  "_enable_preemption: preemption_level(%d)  < 0!"
-       .text
-1:
-#endif /* MACH_ASSERT */
-       _ENABLE_PREEMPTION(%eax)
-#endif /* MACH_RT */
-       ret
+       PREEMPTION_DISABLE
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     2f                              /* branch on failure to spin loop */

 
 

-ENTRY(_enable_preemption_no_check)
-#if    MACH_RT
-#if    MACH_ASSERT
-       movl    $CPD_PREEMPTION_LEVEL,%eax
-       cmpl    $0,%gs:(%eax)
-       jg      1f
-       pushl   $2f
-       call    EXT(panic)
-       hlt
-       .data
-2:     String  "_enable_preemption_no_check: preemption_level <= 0!"
-       .text
+       mov     %gs:CPU_ACTIVE_THREAD, %rax
+       mov     %rax, M_OWNER(%rdx)     /* record owner of mutex */
+#if    MACH_LDEBUG
+       test    %rax, %rax
+       jz      1f
+       incl    TH_MUTEX_COUNT(%rax)    /* lock statistic */

 1:
 1:

-#endif /* MACH_ASSERT */
-       _ENABLE_PREEMPTION_NO_CHECK(%eax)
-#endif /* MACH_RT */
-       ret
-       
+#endif /* MACH_LDEBUG */
+
+       mov     $1, %rax                /* return success */
+       LEAF_RET
+2:                                             
+       PREEMPTION_ENABLE
+3:
+       xor     %rax, %rax      /* return failure */
+       LEAF_RET

        
        

-ENTRY(_mp_disable_preemption)
-#if    MACH_RT && NCPUS > 1
-       _DISABLE_PREEMPTION(%eax)
-#endif /* MACH_RT && NCPUS > 1*/
-       ret

 
 

-ENTRY(_mp_enable_preemption)
-#if    MACH_RT && NCPUS > 1
-#if    MACH_ASSERT
-       movl    $CPD_PREEMPTION_LEVEL,%eax
-       cmpl    $0,%gs:(%eax)
-       jg      1f
-       pushl   %gs:(%eax)
-       pushl   $2f
-       call    EXT(panic)
-       hlt
-       .data
-2:     String  "_mp_enable_preemption: preemption_level (%d) <= 0!"
-       .text
+
+LEAF_ENTRY(lck_mtx_lock_mark_destroyed)
+       mov     %rdi, %rdx

 1:
 1:

-#endif /* MACH_ASSERT */
-       _ENABLE_PREEMPTION(%eax)
-#endif /* MACH_RT && NCPUS > 1 */
-       ret
+       mov     M_STATE(%rdx), %ecx
+       cmp     $(MUTEX_IND), %ecx      /* Is this an indirect mutex? */
+       jne     2f
+
+       movl    $(MUTEX_DESTROYED), M_STATE(%rdx)       /* convert to destroyed state */
+       jmp     3f
+2:     
+       test    $(M_ILOCKED_MSK), %rcx  /* have to wait for interlock to clear */
+       jnz     5f
+
+       PREEMPTION_DISABLE
+       mov     %rcx, %rax              /* eax contains snapshot for cmpxchgl */
+       or      $(M_ILOCKED_MSK), %ecx
+       lock
+       cmpxchg %ecx, M_STATE(%rdx)     /* atomic compare and exchange */
+       jne     4f                      /* branch on failure to spin loop */
+       movl    $(MUTEX_DESTROYED), M_STATE(%rdx)       /* convert to destroyed state */
+       PREEMPTION_ENABLE
+3:
+       LEAF_RET                        /* return with M_ILOCKED set */
+4:
+       PREEMPTION_ENABLE
+5:
+       PAUSE
+       jmp     1b

 
 

-ENTRY(_mp_enable_preemption_no_check)
-#if    MACH_RT && NCPUS > 1
-#if    MACH_ASSERT
-       movl    $CPD_PREEMPTION_LEVEL,%eax
-       cmpl    $0,%gs:(%eax)
-       jg      1f
-       pushl   $2f
-       call    EXT(panic)
+LEAF_ENTRY(preemption_underflow_panic)
+       FRAME
+       incl    %gs:CPU_PREEMPTION_LEVEL
+       ALIGN_STACK()
+       LOAD_STRING_ARG0(16f)
+       CALL_PANIC()

        hlt
        .data
        hlt
        .data

-2:     String  "_mp_enable_preemption_no_check: preemption_level <= 0!"
+16:    String  "Preemption level underflow, possible cause unlocking an unlocked mutex or spinlock"

        .text
        .text

-1:
-#endif /* MACH_ASSERT */
-       _ENABLE_PREEMPTION_NO_CHECK(%eax)
-#endif /* MACH_RT && NCPUS > 1 */
-       ret
-       
-       
-ENTRY(i_bit_set)
-       movl    S_ARG0,%edx
-       movl    S_ARG1,%eax
-       lock
-       bts     %dl,(%eax)
-       ret

 
 

-ENTRY(i_bit_clear)
-       movl    S_ARG0,%edx
-       movl    S_ARG1,%eax
-       lock
-       btr     %dl,(%eax)
-       ret

 
 

-ENTRY(bit_lock)
-       movl    S_ARG0,%ecx
-       movl    S_ARG1,%eax
-1:
-       lock
-       bts     %ecx,(%eax)
-       jb      1b
-       ret
-
-ENTRY(bit_lock_try)
-       movl    S_ARG0,%ecx
-       movl    S_ARG1,%eax
-       lock
-       bts     %ecx,(%eax)
-       jb      bit_lock_failed
-       ret                     /* %eax better not be null ! */
-bit_lock_failed:
-       xorl    %eax,%eax
-       ret
-
-ENTRY(bit_unlock)
-       movl    S_ARG0,%ecx
-       movl    S_ARG1,%eax
-       lock
-       btr     %ecx,(%eax)
-       ret