libpthread-330.201.1.tar.gz

[apple/libpthread.git] / src / pthread_mutex.c

diff --git a/src/pthread_mutex.c b/src/pthread_mutex.c
index 566c5e903f4574816b14e8ae88a231340f3235d8..edc97ee38cbcdb10948eed3da81fd43d4f748084 100644 (file)
--- a/src/pthread_mutex.c
+++ b/src/pthread_mutex.c
@@ -2,14 +2,14 @@
  * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved.
  *
  * @APPLE_LICENSE_HEADER_START@
  * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved.
  *
  * @APPLE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * compliance with the License. Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this
  * file.
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * compliance with the License. Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this
  * file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -17,7 +17,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_LICENSE_HEADER_END@
  */
 /*
  * @APPLE_LICENSE_HEADER_END@
  */
 /*


@@ -53,12 +53,21 @@
 #include "resolver.h"
 #include "internal.h"
 #include "kern/kern_trace.h"
 #include "resolver.h"
 #include "internal.h"
 #include "kern/kern_trace.h"

-#include <sys/syscall.h>

 
 

-#include "os/atomic.h"
+#ifndef BUILDING_VARIANT /* [ */

 
 #ifdef PLOCKSTAT
 #include "plockstat.h"
 
 #ifdef PLOCKSTAT
 #include "plockstat.h"

+/* This function is never called and exists to provide never-fired dtrace
+ * probes so that user d scripts don't get errors.
+ */
+PTHREAD_NOEXPORT PTHREAD_USED
+void
+_plockstat_never_fired(void)
+{
+       PLOCKSTAT_MUTEX_SPIN(NULL);
+       PLOCKSTAT_MUTEX_SPUN(NULL, 0, 0);
+}

 #else /* !PLOCKSTAT */
 #define        PLOCKSTAT_MUTEX_SPIN(x)
 #define        PLOCKSTAT_MUTEX_SPUN(x, y, z)
 #else /* !PLOCKSTAT */
 #define        PLOCKSTAT_MUTEX_SPIN(x)
 #define        PLOCKSTAT_MUTEX_SPUN(x, y, z)


@@ -69,38 +78,97 @@
 #define        PLOCKSTAT_MUTEX_RELEASE(x, y)
 #endif /* PLOCKSTAT */
 
 #define        PLOCKSTAT_MUTEX_RELEASE(x, y)
 #endif /* PLOCKSTAT */
 

+#define BLOCK_FAIL_PLOCKSTAT    0
+#define BLOCK_SUCCESS_PLOCKSTAT 1
+

 #define PTHREAD_MUTEX_INIT_UNUSED 1
 
 #define PTHREAD_MUTEX_INIT_UNUSED 1
 

-extern int __unix_conforming;
+PTHREAD_NOEXPORT PTHREAD_WEAK
+int _pthread_mutex_lock_init_slow(_pthread_mutex *mutex, bool trylock);

 
 

-#ifndef BUILDING_VARIANT
+PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
+int _pthread_mutex_fairshare_lock_slow(_pthread_mutex *mutex, bool trylock);

 
 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
 
 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers

-int
-_pthread_mutex_unlock_slow(pthread_mutex_t *omutex);
+int _pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock);

 
 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
 
 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers

-int
-_pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock);
+int _pthread_mutex_fairshare_unlock_slow(_pthread_mutex *mutex);

 
 

-PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into _pthread_mutex_lock
-int
-_pthread_mutex_lock_wait(pthread_mutex_t *omutex, uint64_t newval64, uint64_t oldtid);
+PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
+int _pthread_mutex_firstfit_unlock_slow(_pthread_mutex *mutex);

 
 

-#endif /* BUILDING_VARIANT */
+PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
+int _pthread_mutex_corruption_abort(_pthread_mutex *mutex);
+
+extern int __pthread_mutex_default_opt_policy PTHREAD_NOEXPORT;
+
+
+int __pthread_mutex_default_opt_policy PTHREAD_NOEXPORT =
+               _PTHREAD_MTX_OPT_POLICY_DEFAULT;
+
+static inline bool
+_pthread_mutex_policy_validate(int policy)
+{
+       return (policy >= 0 && policy < _PTHREAD_MUTEX_POLICY_LAST);
+}
+
+static inline int
+_pthread_mutex_policy_to_opt(int policy)
+{
+       switch (policy) {
+       case PTHREAD_MUTEX_POLICY_FAIRSHARE_NP:
+               return _PTHREAD_MTX_OPT_POLICY_FAIRSHARE;
+       case PTHREAD_MUTEX_POLICY_FIRSTFIT_NP:
+               return _PTHREAD_MTX_OPT_POLICY_FIRSTFIT;
+       default:
+               __builtin_unreachable();
+       }
+}
+
+PTHREAD_NOEXPORT
+void
+_pthread_mutex_global_init(const char *envp[],
+               struct _pthread_registration_data *registration_data)
+{
+
+       int opt = _PTHREAD_MTX_OPT_POLICY_DEFAULT;
+       if (registration_data->mutex_default_policy) {
+               int policy = registration_data->mutex_default_policy;
+               if (_pthread_mutex_policy_validate(policy)) {
+                       opt = _pthread_mutex_policy_to_opt(policy);
+               }
+       }
+
+       const char *envvar = _simple_getenv(envp, "PTHREAD_MUTEX_DEFAULT_POLICY");
+       if (envvar) {
+               int policy = envvar[0] - '0';
+               if (_pthread_mutex_policy_validate(policy)) {
+                       opt = _pthread_mutex_policy_to_opt(policy);
+               }
+       }
+
+       if (opt != __pthread_mutex_default_opt_policy) {
+               __pthread_mutex_default_opt_policy = opt;
+       }
+}

 
 

-#define DEBUG_TRACE_POINTS 0

 
 

-#if DEBUG_TRACE_POINTS
-extern int __syscall(int number, ...);
-#define DEBUG_TRACE(x, a, b, c, d) __syscall(SYS_kdebug_trace, TRACE_##x, a, b, c, d)
-#else
-#define DEBUG_TRACE(x, a, b, c, d) do { } while(0)
-#endif

 
 

-#include <machine/cpu_capabilities.h>
+PTHREAD_ALWAYS_INLINE
+static inline int _pthread_mutex_init(_pthread_mutex *mutex,
+               const pthread_mutexattr_t *attr, uint32_t static_type);
+
+typedef union mutex_seq {
+       uint32_t seq[2];
+       struct { uint32_t lgenval; uint32_t ugenval; };
+       struct { uint32_t mgen; uint32_t ugen; };
+       uint64_t seq_LU;
+       uint64_t _Atomic atomic_seq_LU;
+} mutex_seq;

 
 

-static inline int _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, uint32_t static_type);
+_Static_assert(sizeof(mutex_seq) == 2 * sizeof(uint32_t),
+               "Incorrect mutex_seq size");

 
 #if !__LITTLE_ENDIAN__
 #error MUTEX_GETSEQ_ADDR assumes little endian layout of 2 32-bit sequence words
 
 #if !__LITTLE_ENDIAN__
 #error MUTEX_GETSEQ_ADDR assumes little endian layout of 2 32-bit sequence words


@@ -108,47 +176,67 @@ static inline int _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutex
 
 PTHREAD_ALWAYS_INLINE
 static inline void
 
 PTHREAD_ALWAYS_INLINE
 static inline void

-MUTEX_GETSEQ_ADDR(_pthread_mutex *mutex,
-                 volatile uint64_t **seqaddr)
+MUTEX_GETSEQ_ADDR(_pthread_mutex *mutex, mutex_seq **seqaddr)

 {
        // 64-bit aligned address inside m_seq array (&m_seq[0] for aligned mutex)
        // We don't require more than byte alignment on OS X. rdar://22278325
 {
        // 64-bit aligned address inside m_seq array (&m_seq[0] for aligned mutex)
        // We don't require more than byte alignment on OS X. rdar://22278325

-       *seqaddr = (volatile uint64_t*)(((uintptr_t)mutex->m_seq + 0x7ul) & ~0x7ul);
+       *seqaddr = (void *)(((uintptr_t)mutex->m_seq + 0x7ul) & ~0x7ul);

 }
 
 PTHREAD_ALWAYS_INLINE
 static inline void
 }
 
 PTHREAD_ALWAYS_INLINE
 static inline void

-MUTEX_GETTID_ADDR(_pthread_mutex *mutex,
-                                 volatile uint64_t **tidaddr)
+MUTEX_GETTID_ADDR(_pthread_mutex *mutex, uint64_t **tidaddr)

 {
        // 64-bit aligned address inside m_tid array (&m_tid[0] for aligned mutex)
        // We don't require more than byte alignment on OS X. rdar://22278325
 {
        // 64-bit aligned address inside m_tid array (&m_tid[0] for aligned mutex)
        // We don't require more than byte alignment on OS X. rdar://22278325

-       *tidaddr = (volatile uint64_t*)(((uintptr_t)mutex->m_tid + 0x7ul) & ~0x7ul);
+       *tidaddr = (void*)(((uintptr_t)mutex->m_tid + 0x7ul) & ~0x7ul);

 }
 
 }
 

-#ifndef BUILDING_VARIANT /* [ */
-#ifndef OS_UP_VARIANT_ONLY
+PTHREAD_ALWAYS_INLINE
+static inline void
+mutex_seq_load(mutex_seq *seqaddr, mutex_seq *oldseqval)
+{
+       oldseqval->seq_LU = seqaddr->seq_LU;
+}

 
 

-#define BLOCK_FAIL_PLOCKSTAT    0
-#define BLOCK_SUCCESS_PLOCKSTAT 1
+#define mutex_seq_atomic_load(seqaddr, oldseqval, m) \
+               mutex_seq_atomic_load_##m(seqaddr, oldseqval)

 
 

-#ifdef PLOCKSTAT
-/* This function is never called and exists to provide never-fired dtrace
- * probes so that user d scripts don't get errors.
- */
-PTHREAD_NOEXPORT PTHREAD_USED
-void
-_plockstat_never_fired(void) 
+PTHREAD_ALWAYS_INLINE PTHREAD_USED
+static inline bool
+mutex_seq_atomic_cmpxchgv_relaxed(mutex_seq *seqaddr, mutex_seq *oldseqval,
+               mutex_seq *newseqval)

 {
 {

-       PLOCKSTAT_MUTEX_SPIN(NULL);
-       PLOCKSTAT_MUTEX_SPUN(NULL, 0, 0);
+       return os_atomic_cmpxchgv(&seqaddr->atomic_seq_LU, oldseqval->seq_LU,
+                       newseqval->seq_LU, &oldseqval->seq_LU, relaxed);
+}
+
+PTHREAD_ALWAYS_INLINE PTHREAD_USED
+static inline bool
+mutex_seq_atomic_cmpxchgv_acquire(mutex_seq *seqaddr, mutex_seq *oldseqval,
+               mutex_seq *newseqval)
+{
+       return os_atomic_cmpxchgv(&seqaddr->atomic_seq_LU, oldseqval->seq_LU,
+                       newseqval->seq_LU, &oldseqval->seq_LU, acquire);

 }
 }

-#endif // PLOCKSTAT
+
+PTHREAD_ALWAYS_INLINE PTHREAD_USED
+static inline bool
+mutex_seq_atomic_cmpxchgv_release(mutex_seq *seqaddr, mutex_seq *oldseqval,
+               mutex_seq *newseqval)
+{
+       return os_atomic_cmpxchgv(&seqaddr->atomic_seq_LU, oldseqval->seq_LU,
+                       newseqval->seq_LU, &oldseqval->seq_LU, release);
+}
+
+#define mutex_seq_atomic_cmpxchgv(seqaddr, oldseqval, newseqval, m)\
+               mutex_seq_atomic_cmpxchgv_##m(seqaddr, oldseqval, newseqval)

 
 /*
  * Initialize a mutex variable, possibly with additional attributes.
  * Public interface - so don't trust the lock - initialize it first.
  */
 
 /*
  * Initialize a mutex variable, possibly with additional attributes.
  * Public interface - so don't trust the lock - initialize it first.
  */

+PTHREAD_NOEXPORT_VARIANT

 int
 pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr)
 {
 int
 pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr)
 {


@@ -159,43 +247,48 @@ pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr)
                return EBUSY;
 #endif
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
                return EBUSY;
 #endif
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;

-       LOCK_INIT(mutex->lock);
+       _PTHREAD_LOCK_INIT(mutex->lock);

        return (_pthread_mutex_init(mutex, attr, 0x7));
 }
 
        return (_pthread_mutex_init(mutex, attr, 0x7));
 }
 

+PTHREAD_NOEXPORT_VARIANT

 int
 pthread_mutex_getprioceiling(const pthread_mutex_t *omutex, int *prioceiling)
 {
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (_pthread_mutex_check_signature(mutex)) {
 int
 pthread_mutex_getprioceiling(const pthread_mutex_t *omutex, int *prioceiling)
 {
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (_pthread_mutex_check_signature(mutex)) {

-               LOCK(mutex->lock);
+               _PTHREAD_LOCK(mutex->lock);

                *prioceiling = mutex->prioceiling;
                res = 0;
                *prioceiling = mutex->prioceiling;
                res = 0;

-               UNLOCK(mutex->lock);
+               _PTHREAD_UNLOCK(mutex->lock);

        }
        return res;
 }
 
        }
        return res;
 }
 

+PTHREAD_NOEXPORT_VARIANT

 int
 int

-pthread_mutex_setprioceiling(pthread_mutex_t *omutex, int prioceiling, int *old_prioceiling)
+pthread_mutex_setprioceiling(pthread_mutex_t *omutex, int prioceiling,
+               int *old_prioceiling)

 {
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (_pthread_mutex_check_signature(mutex)) {
 {
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (_pthread_mutex_check_signature(mutex)) {

-               LOCK(mutex->lock);
-               if (prioceiling >= -999 || prioceiling <= 999) {
+               _PTHREAD_LOCK(mutex->lock);
+               if (prioceiling >= -999 && prioceiling <= 999) {

                        *old_prioceiling = mutex->prioceiling;
                        *old_prioceiling = mutex->prioceiling;

-                       mutex->prioceiling = prioceiling;
+                       mutex->prioceiling = (int16_t)prioceiling;

                        res = 0;
                }
                        res = 0;
                }

-               UNLOCK(mutex->lock);
+               _PTHREAD_UNLOCK(mutex->lock);

        }
        return res;
 }
 
        }
        return res;
 }
 

+

 int
 int

-pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *prioceiling)
+pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr,
+               int *prioceiling)

 {
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
 {
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {


@@ -216,6 +309,25 @@ pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int *protocol)
        return res;
 }
 
        return res;
 }
 

+int
+pthread_mutexattr_getpolicy_np(const pthread_mutexattr_t *attr, int *policy)
+{
+       int res = EINVAL;
+       if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
+               switch (attr->opt) {
+               case _PTHREAD_MTX_OPT_POLICY_FAIRSHARE:
+                       *policy = PTHREAD_MUTEX_POLICY_FAIRSHARE_NP;
+                       res = 0;
+                       break;
+               case _PTHREAD_MTX_OPT_POLICY_FIRSTFIT:
+                       *policy = PTHREAD_MUTEX_POLICY_FIRSTFIT_NP;
+                       res = 0;
+                       break;
+               }
+       }
+       return res;
+}
+

 int
 pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
 {
 int
 pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
 {


@@ -243,7 +355,7 @@ pthread_mutexattr_init(pthread_mutexattr_t *attr)
 {
        attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
        attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;
 {
        attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
        attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;

-       attr->policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
+       attr->opt = __pthread_mutex_default_opt_policy;

        attr->type = PTHREAD_MUTEX_DEFAULT;
        attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
        attr->pshared = _PTHREAD_DEFAULT_PSHARED;
        attr->type = PTHREAD_MUTEX_DEFAULT;
        attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
        attr->pshared = _PTHREAD_DEFAULT_PSHARED;


@@ -255,7 +367,7 @@ pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int prioceiling)
 {
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
 {
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {

-               if (prioceiling >= -999 || prioceiling <= 999) {
+               if (prioceiling >= -999 && prioceiling <= 999) {

                        attr->prioceiling = prioceiling;
                        res = 0;
                }
                        attr->prioceiling = prioceiling;
                        res = 0;
                }


@@ -285,12 +397,18 @@ pthread_mutexattr_setpolicy_np(pthread_mutexattr_t *attr, int policy)
 {
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
 {
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {

+               // <rdar://problem/35844519> the first-fit implementation was broken
+               // pre-Liberty so this mapping exists to ensure that the old first-fit
+               // define (2) is no longer valid when used on older systems.

                switch (policy) {
                switch (policy) {

-                       case _PTHREAD_MUTEX_POLICY_FAIRSHARE:
-                       case _PTHREAD_MUTEX_POLICY_FIRSTFIT:
-                               attr->policy = policy;
-                               res = 0;
-                               break;
+               case PTHREAD_MUTEX_POLICY_FAIRSHARE_NP:
+                       attr->opt = _PTHREAD_MTX_OPT_POLICY_FAIRSHARE;
+                       res = 0;
+                       break;
+               case PTHREAD_MUTEX_POLICY_FIRSTFIT_NP:
+                       attr->opt = _PTHREAD_MTX_OPT_POLICY_FIRSTFIT;
+                       res = 0;
+                       break;

                }
        }
        return res;
                }
        }
        return res;


@@ -314,23 +432,6 @@ pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
        return res;
 }
 
        return res;
 }
 

-// XXX remove
-void
-cthread_yield(void) 
-{
-       sched_yield();
-}
-
-void
-pthread_yield_np(void) 
-{
-       sched_yield();
-}
-
-
-/*
- * Temp: till pshared is fixed correctly
- */

 int
 pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
 {
 int
 pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
 {


@@ -343,28 +444,135 @@ pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
 
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
 #if __DARWIN_UNIX03
 
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
 #if __DARWIN_UNIX03

-               if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED))
+               if (( pshared == PTHREAD_PROCESS_PRIVATE) ||
+                               (pshared == PTHREAD_PROCESS_SHARED))

 #else /* __DARWIN_UNIX03 */
                if ( pshared == PTHREAD_PROCESS_PRIVATE)
 #endif /* __DARWIN_UNIX03 */
                {
 #else /* __DARWIN_UNIX03 */
                if ( pshared == PTHREAD_PROCESS_PRIVATE)
 #endif /* __DARWIN_UNIX03 */
                {

-                       attr->pshared = pshared; 
+                       attr->pshared = pshared;

                        res = 0;
                }
        }
        return res;
 }
 
                        res = 0;
                }
        }
        return res;
 }
 

-PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
+PTHREAD_NOEXPORT PTHREAD_NOINLINE PTHREAD_NORETURN

 int
 int

-_pthread_mutex_corruption_abort(_pthread_mutex *mutex);
+_pthread_mutex_corruption_abort(_pthread_mutex *mutex)
+{
+       PTHREAD_ABORT("pthread_mutex corruption: mutex owner changed in the "
+                       "middle of lock/unlock");
+}
+

 
 PTHREAD_NOINLINE
 
 PTHREAD_NOINLINE

-int
-_pthread_mutex_corruption_abort(_pthread_mutex *mutex)
+static int
+_pthread_mutex_check_init_slow(_pthread_mutex *mutex)
+{
+       int res = EINVAL;
+
+       if (_pthread_mutex_check_signature_init(mutex)) {
+               _PTHREAD_LOCK(mutex->lock);
+               if (_pthread_mutex_check_signature_init(mutex)) {
+                       // initialize a statically initialized mutex to provide
+                       // compatibility for misbehaving applications.
+                       // (unlock should not be the first operation on a mutex)
+                       res = _pthread_mutex_init(mutex, NULL, (mutex->sig & 0xf));
+               } else if (_pthread_mutex_check_signature(mutex)) {
+                       res = 0;
+               }
+               _PTHREAD_UNLOCK(mutex->lock);
+       } else if (_pthread_mutex_check_signature(mutex)) {
+               res = 0;
+       }
+       if (res != 0) {
+               PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, res);
+       }
+       return res;
+}
+
+PTHREAD_ALWAYS_INLINE
+static inline int
+_pthread_mutex_check_init(_pthread_mutex *mutex)
+{
+       int res = 0;
+       if (!_pthread_mutex_check_signature(mutex)) {
+               return _pthread_mutex_check_init_slow(mutex);
+       }
+       return res;
+}
+
+PTHREAD_ALWAYS_INLINE
+static inline bool
+_pthread_mutex_is_fairshare(_pthread_mutex *mutex)
+{
+       return (mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FAIRSHARE);
+}
+
+PTHREAD_ALWAYS_INLINE
+static inline bool
+_pthread_mutex_is_firstfit(_pthread_mutex *mutex)
+{
+       return (mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FIRSTFIT);
+}
+
+PTHREAD_ALWAYS_INLINE
+static inline bool
+_pthread_mutex_is_recursive(_pthread_mutex *mutex)
+{
+       return (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE);
+}
+
+PTHREAD_ALWAYS_INLINE
+static int
+_pthread_mutex_lock_handle_options(_pthread_mutex *mutex, bool trylock,
+               uint64_t *tidaddr)
+{
+       if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL) {
+               // NORMAL does not do EDEADLK checking
+               return 0;
+       }
+
+       uint64_t selfid = _pthread_selfid_direct();
+       if (os_atomic_load(tidaddr, relaxed) == selfid) {
+               if (_pthread_mutex_is_recursive(mutex)) {
+                       if (mutex->mtxopts.options.lock_count < USHRT_MAX) {
+                               mutex->mtxopts.options.lock_count += 1;
+                               return mutex->mtxopts.options.lock_count;
+                       } else {
+                               return -EAGAIN;
+                       }
+               } else if (trylock) { /* PTHREAD_MUTEX_ERRORCHECK */
+                       // <rdar://problem/16261552> as per OpenGroup, trylock cannot
+                       // return EDEADLK on a deadlock, it should return EBUSY.
+                       return -EBUSY;
+               } else { /* PTHREAD_MUTEX_ERRORCHECK */
+                       return -EDEADLK;
+               }
+       }
+
+       // Not recursive, or recursive but first lock.
+       return 0;
+}
+
+PTHREAD_ALWAYS_INLINE
+static int
+_pthread_mutex_unlock_handle_options(_pthread_mutex *mutex, uint64_t *tidaddr)

 {
 {

-       PTHREAD_ABORT("pthread_mutex corruption: mutex %p owner changed in the middle of lock/unlock");
-       return EINVAL; // NOTREACHED
+       if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL) {
+               // NORMAL does not do EDEADLK checking
+               return 0;
+       }
+
+       uint64_t selfid = _pthread_selfid_direct();
+       if (os_atomic_load(tidaddr, relaxed) != selfid) {
+               return -EPERM;
+       } else if (_pthread_mutex_is_recursive(mutex) &&
+                       --mutex->mtxopts.options.lock_count) {
+               return 1;
+       }
+       return 0;

 }
 
 /*
 }
 
 /*


@@ -376,100 +584,95 @@ _pthread_mutex_corruption_abort(_pthread_mutex *mutex)
  * the unlock path will then transition to D=[L5 U4 TID0] and then finally
  * E=[L5 U5 TID0].
  *
  * the unlock path will then transition to D=[L5 U4 TID0] and then finally
  * E=[L5 U5 TID0].
  *

- * If a contender comes in after B, the mutex will instead transition to E=[L6+KE U4 TID0]
- * and then F=[L6+KE U4 TID940]. If a contender comes in after C, it will transition to
- * F=[L6+KE U4 TID940] directly. In both cases, the contender will enter the kernel with either
- * mutexwait(U4, TID0) or mutexwait(U4, TID940). The first owner will unlock the mutex
- * by first updating the owner to G=[L6+KE U4 TID-1] and then doing the actual unlock to
- * H=[L6+KE U5 TID=-1] before entering the kernel with mutexdrop(U5, -1) to signal the next waiter
- * (potentially as a prepost). When the waiter comes out of the kernel, it will update the owner to
- * I=[L6+KE U5 TID941]. An unlock at this point is simply J=[L6 U5 TID0] and then K=[L6 U6 TID0].
+ * If a contender comes in after B, the mutex will instead transition to
+ * E=[L6+KE U4 TID0] and then F=[L6+KE U4 TID940]. If a contender comes in after
+ * C, it will transition to F=[L6+KE U4 TID940] directly. In both cases, the
+ * contender will enter the kernel with either mutexwait(U4, TID0) or
+ * mutexwait(U4, TID940). The first owner will unlock the mutex by first
+ * updating the owner to G=[L6+KE U4 TID-1] and then doing the actual unlock to
+ * H=[L6+KE U5 TID=-1] before entering the kernel with mutexdrop(U5, -1) to
+ * signal the next waiter (potentially as a prepost). When the waiter comes out
+ * of the kernel, it will update the owner to I=[L6+KE U5 TID941]. An unlock at
+ * this point is simply J=[L6 U5 TID0] and then K=[L6 U6 TID0].

  *
  *

- * At various points along these timelines, since the sequence words and TID are written independently,
- * a thread may get preempted and another thread might see inconsistent data. In the worst case, another
- * thread may see the TID in the SWITCHING (-1) state or unlocked (0) state for longer because the
- * owning thread was preempted.
+ * At various points along these timelines, since the sequence words and TID are
+ * written independently, a thread may get preempted and another thread might
+ * see inconsistent data. In the worst case, another thread may see the TID in
+ * the SWITCHING (-1) state or unlocked (0) state for longer because the owning
+ * thread was preempted.

  */
 
 /*
  */
 
 /*

- * Drop the mutex unlock references from cond_wait. or mutex_unlock.
+ * Drop the mutex unlock references from cond_wait or mutex_unlock.

  */
 PTHREAD_ALWAYS_INLINE
 static inline int
  */
 PTHREAD_ALWAYS_INLINE
 static inline int

-_pthread_mutex_unlock_updatebits(_pthread_mutex *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
+_pthread_mutex_fairshare_unlock_updatebits(_pthread_mutex *mutex,
+               uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)

 {
 {

-       bool firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
-       uint32_t lgenval, ugenval, flags;
-       uint64_t oldtid, newtid;
-       volatile uint64_t *tidaddr;
-       MUTEX_GETTID_ADDR(mutex, &tidaddr);
-
-       flags = mutex->mtxopts.value;
+       uint32_t flags = mutex->mtxopts.value;

        flags &= ~_PTHREAD_MTX_OPT_NOTIFY; // no notification by default
 
        flags &= ~_PTHREAD_MTX_OPT_NOTIFY; // no notification by default
 

-       if (mutex->mtxopts.options.type != PTHREAD_MUTEX_NORMAL) {
-               uint64_t selfid = _pthread_selfid_direct();
-
-               if (*tidaddr != selfid) {
-                       //PTHREAD_ABORT("dropping recur or error mutex not owned by the thread");
-                       PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, EPERM);
-                       return EPERM;
-               } else if (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE &&
-                          --mutex->mtxopts.options.lock_count) {
-                       PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
-                       if (flagsp != NULL) {
-                               *flagsp = flags;
-                       }
-                       return 0;
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t oldtid, newtid;
+
+       int res = _pthread_mutex_unlock_handle_options(mutex, tidaddr);
+       if (res > 0) {
+               // Valid recursive unlock
+               if (flagsp) {
+                       *flagsp = flags;

                }
                }

+               PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
+               return 0;
+       } else if (res < 0) {
+               PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, -res);
+               return -res;

        }
 
        }
 

-       uint64_t oldval64, newval64;
-       volatile uint64_t *seqaddr;
-       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
-
-       bool clearprepost, clearnotify, spurious;
+       bool clearnotify, spurious;

        do {
        do {

-               oldval64 = *seqaddr;
-               oldtid = *tidaddr;
-               lgenval = (uint32_t)oldval64;
-               ugenval = (uint32_t)(oldval64 >> 32);
+               newseq = oldseq;
+               oldtid = os_atomic_load(tidaddr, relaxed);

 
 

-               clearprepost = false;

                clearnotify = false;
                spurious = false;
 
                clearnotify = false;
                spurious = false;
 

-               int numwaiters = diff_genseq(lgenval, ugenval); // pending waiters
-
+               // pending waiters
+               int numwaiters = diff_genseq(oldseq.lgenval, oldseq.ugenval);

                if (numwaiters == 0) {
                if (numwaiters == 0) {

-                       // spurious unlock; do not touch tid
+                       // spurious unlock (unlock of unlocked lock)

                        spurious = true;
                } else {
                        spurious = true;
                } else {

-                       ugenval += PTHRW_INC;
+                       newseq.ugenval += PTHRW_INC;

 
 

-                       if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
-                               // our unlock sequence matches to lock sequence, so if the CAS is successful, the mutex is unlocked
+                       if ((oldseq.lgenval & PTHRW_COUNT_MASK) ==
+                                       (newseq.ugenval & PTHRW_COUNT_MASK)) {
+                               // our unlock sequence matches to lock sequence, so if the
+                               // CAS is successful, the mutex is unlocked

 
                                /* do not reset Ibit, just K&E */
 
                                /* do not reset Ibit, just K&E */

-                               lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
+                               newseq.lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);

                                clearnotify = true;
                                newtid = 0; // clear owner
                        } else {
                                clearnotify = true;
                                newtid = 0; // clear owner
                        } else {

-                               if (firstfit) {
-                                       lgenval &= ~PTH_RWL_EBIT; // reset E bit so another can acquire meanwhile
-                                       newtid = 0;
-                               } else {
-                                       newtid = PTHREAD_MTX_TID_SWITCHING;
-                               }
+                               newtid = PTHREAD_MTX_TID_SWITCHING;

                                // need to signal others waiting for mutex
                                flags |= _PTHREAD_MTX_OPT_NOTIFY;
                        }
                                // need to signal others waiting for mutex
                                flags |= _PTHREAD_MTX_OPT_NOTIFY;
                        }

-                       
+

                        if (newtid != oldtid) {
                        if (newtid != oldtid) {

-                               // We're giving up the mutex one way or the other, so go ahead and update the owner to SWITCHING
-                               // or 0 so that once the CAS below succeeds, there is no stale ownership information.
-                               // If the CAS of the seqaddr fails, we may loop, but it's still valid for the owner
-                               // to be SWITCHING/0
+                               // We're giving up the mutex one way or the other, so go ahead
+                               // and update the owner to 0 so that once the CAS below
+                               // succeeds, there is no stale ownership information. If the
+                               // CAS of the seqaddr fails, we may loop, but it's still valid
+                               // for the owner to be SWITCHING/0

                                if (!os_atomic_cmpxchg(tidaddr, oldtid, newtid, relaxed)) {
                                        // we own this mutex, nobody should be updating it except us
                                        return _pthread_mutex_corruption_abort(mutex);
                                if (!os_atomic_cmpxchg(tidaddr, oldtid, newtid, relaxed)) {
                                        // we own this mutex, nobody should be updating it except us
                                        return _pthread_mutex_corruption_abort(mutex);


@@ -479,26 +682,17 @@ _pthread_mutex_unlock_updatebits(_pthread_mutex *mutex, uint32_t *flagsp, uint32
 
                if (clearnotify || spurious) {
                        flags &= ~_PTHREAD_MTX_OPT_NOTIFY;
 
                if (clearnotify || spurious) {
                        flags &= ~_PTHREAD_MTX_OPT_NOTIFY;

-                       if (firstfit && ((lgenval & PTH_RWL_PBIT) != 0)) {
-                               clearprepost = true;
-                               lgenval &= ~PTH_RWL_PBIT;
-                       }

                }
                }

-               
-               newval64 = (((uint64_t)ugenval) << 32);
-               newval64 |= lgenval;
-
-       } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, release));

 
 

-       if (clearprepost) {
-                __psynch_cvclrprepost(mutex, lgenval, ugenval, 0, 0, lgenval, (flags | _PTHREAD_MTX_OPT_MUTEX));
-       }
+       PTHREAD_TRACE(psynch_mutex_unlock_updatebits, mutex, oldseq.lgenval,
+                       newseq.lgenval, oldtid);

 
        if (mgenp != NULL) {
 
        if (mgenp != NULL) {

-               *mgenp = lgenval;
+               *mgenp = newseq.lgenval;

        }
        if (ugenp != NULL) {
        }
        if (ugenp != NULL) {

-               *ugenp = ugenval;
+               *ugenp = newseq.ugenval;

        }
        if (pmtxp != NULL) {
                *pmtxp = (uint32_t *)mutex;
        }
        if (pmtxp != NULL) {
                *pmtxp = (uint32_t *)mutex;


@@ -510,236 +704,111 @@ _pthread_mutex_unlock_updatebits(_pthread_mutex *mutex, uint32_t *flagsp, uint32
        return 0;
 }
 
        return 0;
 }
 

-PTHREAD_NOEXPORT
-int
-__mtx_droplock(_pthread_mutex *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
-{
-       return _pthread_mutex_unlock_updatebits(mutex, flagsp, pmtxp, mgenp, ugenp);
-}
-

 PTHREAD_ALWAYS_INLINE
 static inline int
 PTHREAD_ALWAYS_INLINE
 static inline int

-_pthread_mutex_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)
+_pthread_mutex_fairshare_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)

 {
 {

-       int res = 0;
-       int firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
-       int isebit = 0;
+       bool firstfit = _pthread_mutex_is_firstfit(mutex);
+       bool gotlock = true;

 
 

-       uint32_t lgenval, ugenval;
-       uint64_t oldval64, newval64;
-       volatile uint64_t *seqaddr;
+       mutex_seq *seqaddr;

        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

-       uint64_t oldtid;
-       volatile uint64_t *tidaddr;
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       uint64_t *tidaddr;

        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 
        do {
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 
        do {

-               do {
-                       oldval64 = *seqaddr;
-                       oldtid = *tidaddr;
-                       lgenval = (uint32_t)oldval64;
-                       ugenval = (uint32_t)(oldval64 >> 32);
-
-                       // E bit was set on first pass through the loop but is no longer
-                       // set. Apparently we spin until it arrives.
-                       // XXX: verify this is desired behavior.
-               } while (isebit && (lgenval & PTH_RWL_EBIT) == 0);
-
-               if (isebit) {
-                       // first fit mutex now has the E bit set. Return 1.
-                       res = 1;
-                       break;
-               }
+               newseq = oldseq;

 
                if (firstfit) {
 
                if (firstfit) {

-                       isebit = (lgenval & PTH_RWL_EBIT) != 0;
-               } else if ((lgenval & (PTH_RWL_KBIT|PTH_RWL_EBIT)) == (PTH_RWL_KBIT|PTH_RWL_EBIT)) {
-                       // fairshare mutex and the bits are already set, just update tid
+                       // firstfit locks can have the lock stolen out from under a locker
+                       // between the unlock from the kernel and this lock path. When this
+                       // happens, we still want to set the K bit before leaving the loop
+                       // (or notice if the lock unlocks while we try to update).
+                       gotlock = !is_rwl_ebit_set(oldseq.lgenval);
+               } else if ((oldseq.lgenval & (PTH_RWL_KBIT | PTH_RWL_EBIT)) == 
+                               (PTH_RWL_KBIT | PTH_RWL_EBIT)) {
+                       // bit are already set, just update the owner tidaddr

                        break;
                }
 
                        break;
                }
 

-               // either first fit or no E bit set
-               // update the bits
-               lgenval |= PTH_RWL_KBIT | PTH_RWL_EBIT;
-
-               newval64 = (((uint64_t)ugenval) << 32);
-               newval64 |= lgenval;
+               newseq.lgenval |= PTH_RWL_KBIT | PTH_RWL_EBIT;
+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
+                       acquire));

 
 

-               // set s and b bit
-               // Retry if CAS fails, or if it succeeds with firstfit and E bit already set
-       } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire) || (firstfit && isebit));
-
-       if (res == 0) {
-               if (!os_atomic_cmpxchg(tidaddr, oldtid, selfid, relaxed)) {
-                       // we own this mutex, nobody should be updating it except us
-                       return _pthread_mutex_corruption_abort(mutex);
-               }
+       if (gotlock) {
+               os_atomic_store(tidaddr, selfid, relaxed);

        }
 
        }
 

-       return res;
+       PTHREAD_TRACE(psynch_mutex_lock_updatebits, mutex, oldseq.lgenval,
+                       newseq.lgenval, 0);
+
+       // failing to take the lock in firstfit returns 1 to force the caller
+       // to wait in the kernel
+       return gotlock ? 0 : 1;

 }
 
 PTHREAD_NOINLINE
 static int
 }
 
 PTHREAD_NOINLINE
 static int

-__mtx_markprepost(_pthread_mutex *mutex, uint32_t updateval, int firstfit)
+_pthread_mutex_fairshare_lock_wait(_pthread_mutex *mutex, mutex_seq newseq,
+               uint64_t oldtid)

 {
 {

-       uint32_t flags;
-       uint32_t lgenval, ugenval;
-       uint64_t oldval64, newval64;
-
-       volatile uint64_t *seqaddr;
-       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
-
-       if (firstfit != 0 && (updateval & PTH_RWL_PBIT) != 0) {
-               int clearprepost;
-               do {                            
-                       clearprepost = 0;
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t selfid = _pthread_selfid_direct();

 
 

-                       flags = mutex->mtxopts.value;
+       PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex);
+       do {
+               uint32_t updateval;
+               do {
+                       updateval = __psynch_mutexwait(mutex, newseq.lgenval,
+                                       newseq.ugenval, oldtid, mutex->mtxopts.value);
+                       oldtid = os_atomic_load(tidaddr, relaxed);
+               } while (updateval == (uint32_t)-1);

 
 

-                       oldval64 = *seqaddr;
-                       lgenval = (uint32_t)oldval64;
-                       ugenval = (uint32_t)(oldval64 >> 32);
+               // returns 0 on succesful update; in firstfit it may fail with 1
+       } while (_pthread_mutex_fairshare_lock_updatebits(mutex, selfid) == 1);
+       PLOCKSTAT_MUTEX_BLOCKED((pthread_mutex_t *)mutex, BLOCK_SUCCESS_PLOCKSTAT);

 
 

-                       /* update the bits */
-                       if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
-                               clearprepost = 1;       
-                               lgenval &= ~PTH_RWL_PBIT;
-                       } else {
-                               lgenval |= PTH_RWL_PBIT;
-                       }
-                       newval64 = (((uint64_t)ugenval) << 32);
-                       newval64 |= lgenval;
-               } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
-               
-               if (clearprepost != 0) {
-                       __psynch_cvclrprepost(mutex, lgenval, ugenval, 0, 0, lgenval, (flags | _PTHREAD_MTX_OPT_MUTEX));
-               }
-       }

        return 0;
 }
 
        return 0;
 }
 

-PTHREAD_NOINLINE
-static int
-_pthread_mutex_check_init_slow(pthread_mutex_t *omutex)
+PTHREAD_NOEXPORT PTHREAD_NOINLINE
+int
+_pthread_mutex_fairshare_lock_slow(_pthread_mutex *omutex, bool trylock)

 {
 {

-       int res = EINVAL;
+       int res, recursive = 0;

        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
 
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
 

-       if (_pthread_mutex_check_signature_init(mutex)) {
-               LOCK(mutex->lock);
-               if (_pthread_mutex_check_signature_init(mutex)) {
-                       // initialize a statically initialized mutex to provide
-                       // compatibility for misbehaving applications.
-                       // (unlock should not be the first operation on a mutex)
-                       res = _pthread_mutex_init(mutex, NULL, (mutex->sig & 0xf));
-               } else if (_pthread_mutex_check_signature(mutex)) {
-                       res = 0;
-               }
-               UNLOCK(mutex->lock);
-       } else if (_pthread_mutex_check_signature(mutex)) {
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t oldtid, selfid = _pthread_selfid_direct();
+
+       res = _pthread_mutex_lock_handle_options(mutex, trylock, tidaddr);
+       if (res > 0) {
+               recursive = 1;

                res = 0;
                res = 0;

+               goto out;
+       } else if (res < 0) {
+               res = -res;
+               goto out;

        }
        }

-       if (res != 0) {
-               PLOCKSTAT_MUTEX_ERROR(omutex, res);
-       }
-       return res;
-}
-
-PTHREAD_ALWAYS_INLINE
-static inline int
-_pthread_mutex_check_init(pthread_mutex_t *omutex)
-{
-       int res = 0;
-       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
-       
-       if (!_pthread_mutex_check_signature(mutex)) {
-               return _pthread_mutex_check_init_slow(omutex);
-       }
-       return res;
-}
-
-PTHREAD_NOINLINE
-int
-_pthread_mutex_lock_wait(pthread_mutex_t *omutex, uint64_t newval64, uint64_t oldtid)
-{
-       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
-       uint32_t lgenval = (uint32_t)newval64;
-       uint32_t ugenval = (uint32_t)(newval64 >> 32);
-
-       volatile uint64_t *tidaddr;
-       MUTEX_GETTID_ADDR(mutex, &tidaddr);
-       uint64_t selfid = _pthread_selfid_direct();
-
-       PLOCKSTAT_MUTEX_BLOCK(omutex);
-       do {
-               uint32_t updateval;
-               do {
-                       updateval = __psynch_mutexwait(omutex, lgenval, ugenval, oldtid, mutex->mtxopts.value);
-                       oldtid = *tidaddr;
-               } while (updateval == (uint32_t)-1);
-
-               // returns 0 on succesful update; in firstfit it may fail with 1
-       } while (_pthread_mutex_lock_updatebits(mutex, selfid) == 1);
-       PLOCKSTAT_MUTEX_BLOCKED(omutex, BLOCK_SUCCESS_PLOCKSTAT);
-
-       return 0;
-}
-
-int
-_pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock)
-{
-       int res;
-       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
-
-       res = _pthread_mutex_check_init(omutex);
-       if (res != 0) {
-               return res;
-       }
-
-       uint64_t oldtid;
-       volatile uint64_t *tidaddr;
-       MUTEX_GETTID_ADDR(mutex, &tidaddr);
-       uint64_t selfid = _pthread_selfid_direct();
-
-       if (mutex->mtxopts.options.type != PTHREAD_MUTEX_NORMAL) {
-               if (*tidaddr == selfid) {
-                       if (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
-                               if (mutex->mtxopts.options.lock_count < USHRT_MAX) {
-                                       mutex->mtxopts.options.lock_count++;
-                                       PLOCKSTAT_MUTEX_ACQUIRE(omutex, 1, 0);
-                                       res = 0;
-                               } else {
-                                       res = EAGAIN;
-                                       PLOCKSTAT_MUTEX_ERROR(omutex, res);
-                               }
-                       } else if (trylock) { /* PTHREAD_MUTEX_ERRORCHECK */
-                               // <rdar://problem/16261552> as per OpenGroup, trylock cannot
-                               // return EDEADLK on a deadlock, it should return EBUSY.
-                               res = EBUSY;
-                               PLOCKSTAT_MUTEX_ERROR(omutex, res);
-                       } else  { /* PTHREAD_MUTEX_ERRORCHECK */
-                               res = EDEADLK;
-                               PLOCKSTAT_MUTEX_ERROR(omutex, res);
-                       }
-                       return res;
-               }
-       }
-
-       uint64_t oldval64, newval64;
-       volatile uint64_t *seqaddr;
-       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
-
-       uint32_t lgenval, ugenval;
-       bool gotlock = false;

 
 

+       bool gotlock;

        do {
        do {

-               oldval64 = *seqaddr;
-               oldtid = *tidaddr;
-               lgenval = (uint32_t)oldval64;
-               ugenval = (uint32_t)(oldval64 >> 32);
+               newseq = oldseq;
+               oldtid = os_atomic_load(tidaddr, relaxed);

 
 

-               gotlock = ((lgenval & PTH_RWL_EBIT) == 0);
+               gotlock = ((oldseq.lgenval & PTH_RWL_EBIT) == 0);

 
                if (trylock && !gotlock) {
                        // A trylock on a held lock will fail immediately. But since
 
                if (trylock && !gotlock) {
                        // A trylock on a held lock will fail immediately. But since


@@ -748,255 +817,682 @@ _pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock)
                } else {
                        // Increment the lock sequence number and force the lock into E+K
                        // mode, whether "gotlock" is true or not.
                } else {
                        // Increment the lock sequence number and force the lock into E+K
                        // mode, whether "gotlock" is true or not.

-                       lgenval += PTHRW_INC;
-                       lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
+                       newseq.lgenval += PTHRW_INC;
+                       newseq.lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;

                }
                }

+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire));

 
 

-               newval64 = (((uint64_t)ugenval) << 32);
-               newval64 |= lgenval;
-               
-               // Set S and B bit
-       } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire));
+       PTHREAD_TRACE(psynch_mutex_lock_updatebits, omutex, oldseq.lgenval,
+                       newseq.lgenval, 0);

 
        if (gotlock) {
                os_atomic_store(tidaddr, selfid, relaxed);
                res = 0;
 
        if (gotlock) {
                os_atomic_store(tidaddr, selfid, relaxed);
                res = 0;

-               DEBUG_TRACE(psynch_mutex_ulock, omutex, lgenval, ugenval, selfid);
-               PLOCKSTAT_MUTEX_ACQUIRE(omutex, 0, 0);
+               PTHREAD_TRACE(psynch_mutex_ulock, omutex, newseq.lgenval,
+                               newseq.ugenval, selfid);

        } else if (trylock) {
                res = EBUSY;
        } else if (trylock) {
                res = EBUSY;

-               DEBUG_TRACE(psynch_mutex_utrylock_failed, omutex, lgenval, ugenval, oldtid);
-               PLOCKSTAT_MUTEX_ERROR(omutex, res);
+               PTHREAD_TRACE(psynch_mutex_utrylock_failed, omutex, newseq.lgenval,
+                               newseq.ugenval, oldtid);

        } else {
        } else {

-               res = _pthread_mutex_lock_wait(omutex, newval64, oldtid);
+               PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_START, omutex,
+                               newseq.lgenval, newseq.ugenval, oldtid);
+               res = _pthread_mutex_fairshare_lock_wait(mutex, newseq, oldtid);
+               PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_END, omutex,
+                               newseq.lgenval, newseq.ugenval, oldtid);

        }
 
        }
 

-       if (res == 0 && mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
+       if (res == 0 && _pthread_mutex_is_recursive(mutex)) {

                mutex->mtxopts.options.lock_count = 1;
        }
 
                mutex->mtxopts.options.lock_count = 1;
        }
 

-       PLOCKSTAT_MUTEX_ACQUIRE(omutex, 0, 0);
+out:
+#if PLOCKSTAT
+       if (res == 0) {
+               PLOCKSTAT_MUTEX_ACQUIRE((pthread_mutex_t *)mutex, recursive, 0);
+       } else {
+               PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, res);
+       }
+#endif

 
        return res;
 }
 
 
        return res;
 }
 

-#endif // OS_UP_VARIANT_ONLY
-
-PTHREAD_ALWAYS_INLINE
+PTHREAD_NOINLINE

 static inline int
 static inline int

-_pthread_mutex_lock(pthread_mutex_t *omutex, bool trylock)
+_pthread_mutex_fairshare_lock(_pthread_mutex *mutex, bool trylock)

 {
 {

-#if PLOCKSTAT || DEBUG_TRACE_POINTS
-       if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
-                       DEBUG_TRACE_POINTS) {
-               return _pthread_mutex_lock_slow(omutex, trylock);
+#if ENABLE_USERSPACE_TRACE
+       return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
+#elif PLOCKSTAT
+       if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
+               return _pthread_mutex_fairshare_lock_slow(mutex, trylock);

        }
 #endif
        }
 #endif

-       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
-       if (!_pthread_mutex_check_signature_fast(mutex)) {
-               return _pthread_mutex_lock_slow(omutex, trylock);
-       }

 
 

-       uint64_t oldtid;
-       volatile uint64_t *tidaddr;
+       uint64_t *tidaddr;

        MUTEX_GETTID_ADDR(mutex, &tidaddr);
        uint64_t selfid = _pthread_selfid_direct();
 
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
        uint64_t selfid = _pthread_selfid_direct();
 

-       uint64_t oldval64, newval64;
-       volatile uint64_t *seqaddr;
+       mutex_seq *seqaddr;

        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
 
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
 

-       uint32_t lgenval, ugenval;
-       bool gotlock = false;
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       if (os_unlikely(oldseq.lgenval & PTH_RWL_EBIT)) {
+               return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
+       }

 
 

+       bool gotlock;

        do {
        do {

-               oldval64 = *seqaddr;
-               oldtid = *tidaddr;
-               lgenval = (uint32_t)oldval64;
-               ugenval = (uint32_t)(oldval64 >> 32);
+               newseq = oldseq;

 
 

-               gotlock = ((lgenval & PTH_RWL_EBIT) == 0);
+               gotlock = ((oldseq.lgenval & PTH_RWL_EBIT) == 0);

 
                if (trylock && !gotlock) {
                        // A trylock on a held lock will fail immediately. But since
                        // we did not load the sequence words atomically, perform a
                        // no-op CAS64 to ensure that nobody has unlocked concurrently.
 
                if (trylock && !gotlock) {
                        // A trylock on a held lock will fail immediately. But since
                        // we did not load the sequence words atomically, perform a
                        // no-op CAS64 to ensure that nobody has unlocked concurrently.

-               } else {
+               } else if (os_likely(gotlock)) {

                        // Increment the lock sequence number and force the lock into E+K
                        // mode, whether "gotlock" is true or not.
                        // Increment the lock sequence number and force the lock into E+K
                        // mode, whether "gotlock" is true or not.

-                       lgenval += PTHRW_INC;
-                       lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
+                       newseq.lgenval += PTHRW_INC;
+                       newseq.lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
+               } else {
+                       return _pthread_mutex_fairshare_lock_slow(mutex, trylock);

                }
                }

+       } while (os_unlikely(!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
+                       acquire)));

 
 

-               newval64 = (((uint64_t)ugenval) << 32);
-               newval64 |= lgenval;
-
-               // Set S and B bit
-       } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire));
-
-       if (os_fastpath(gotlock)) {
+       if (os_likely(gotlock)) {

                os_atomic_store(tidaddr, selfid, relaxed);
                return 0;
        } else if (trylock) {
                return EBUSY;
        } else {
                os_atomic_store(tidaddr, selfid, relaxed);
                return 0;
        } else if (trylock) {
                return EBUSY;
        } else {

-               return _pthread_mutex_lock_wait(omutex, newval64, oldtid);
+               __builtin_trap();

        }
 }
 
        }
 }
 

-PTHREAD_NOEXPORT_VARIANT
-int
-pthread_mutex_lock(pthread_mutex_t *mutex)
+PTHREAD_NOINLINE
+static int
+_pthread_mutex_fairshare_unlock_drop(_pthread_mutex *mutex, mutex_seq newseq,
+               uint32_t flags)

 {
 {

-       return _pthread_mutex_lock(mutex, false);
+       int res;
+       uint32_t updateval;
+
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+
+       PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_START, mutex, newseq.lgenval,
+                       newseq.ugenval, os_atomic_load(tidaddr, relaxed));
+
+       updateval = __psynch_mutexdrop(mutex, newseq.lgenval, newseq.ugenval,
+                       os_atomic_load(tidaddr, relaxed), flags);
+
+       PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_END, mutex, updateval, 0, 0);
+
+       if (updateval == (uint32_t)-1) {
+               res = errno;
+
+               if (res == EINTR) {
+                       res = 0;
+               }
+               if (res != 0) {
+                       PTHREAD_ABORT("__psynch_mutexdrop failed with error %d", res);
+               }
+               return res;
+       }
+
+       return 0;

 }
 
 }
 

-PTHREAD_NOEXPORT_VARIANT
+PTHREAD_NOEXPORT PTHREAD_NOINLINE

 int
 int

-pthread_mutex_trylock(pthread_mutex_t *mutex)
+_pthread_mutex_fairshare_unlock_slow(_pthread_mutex *mutex)

 {
 {

-       return _pthread_mutex_lock(mutex, true);
-}
+       int res;
+       mutex_seq newseq;
+       uint32_t flags;

 
 

-#ifndef OS_UP_VARIANT_ONLY
-/*
- * Unlock a mutex.
- * TODO: Priority inheritance stuff
- */
+       res = _pthread_mutex_fairshare_unlock_updatebits(mutex, &flags, NULL,
+                       &newseq.lgenval, &newseq.ugenval);
+       if (res != 0) return res;
+
+       if ((flags & _PTHREAD_MTX_OPT_NOTIFY) != 0) {
+               return _pthread_mutex_fairshare_unlock_drop(mutex, newseq, flags);
+       } else {
+               uint64_t *tidaddr;
+               MUTEX_GETTID_ADDR(mutex, &tidaddr);
+               PTHREAD_TRACE(psynch_mutex_uunlock, mutex, newseq.lgenval,
+                               newseq.ugenval, os_atomic_load(tidaddr, relaxed));
+       }
+
+       return 0;
+}

 
 PTHREAD_NOINLINE
 static int
 
 PTHREAD_NOINLINE
 static int

-_pthread_mutex_unlock_drop(pthread_mutex_t *omutex, uint64_t newval64, uint32_t flags)
+_pthread_mutex_fairshare_unlock(_pthread_mutex *mutex)

 {
 {

-       int res;
-       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
-       uint32_t lgenval = (uint32_t)newval64;
-       uint32_t ugenval = (uint32_t)(newval64 >> 32);
+#if ENABLE_USERSPACE_TRACE
+       return _pthread_mutex_fairshare_unlock_slow(mutex);
+#elif PLOCKSTAT
+       if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
+               return _pthread_mutex_fairshare_unlock_slow(mutex);
+       }
+#endif

 
 

-       uint32_t updateval;
-       int firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
-       volatile uint64_t *tidaddr;
+       uint64_t *tidaddr;

        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 

-       updateval = __psynch_mutexdrop(omutex, lgenval, ugenval, *tidaddr, flags);
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

 
 

-       if (updateval == (uint32_t)-1) {
-               res = errno;
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       int numwaiters = diff_genseq(oldseq.lgenval, oldseq.ugenval);
+       if (os_unlikely(numwaiters == 0)) {
+               // spurious unlock (unlock of unlocked lock)
+               return 0;
+       }
+
+       // We're giving up the mutex one way or the other, so go ahead and
+       // update the owner to 0 so that once the CAS below succeeds, there
+       // is no stale ownership information. If the CAS of the seqaddr
+       // fails, we may loop, but it's still valid for the owner to be
+       // SWITCHING/0
+       os_atomic_store(tidaddr, 0, relaxed);
+
+       do {
+               newseq = oldseq;
+               newseq.ugenval += PTHRW_INC;
+
+               if (os_likely((oldseq.lgenval & PTHRW_COUNT_MASK) ==
+                               (newseq.ugenval & PTHRW_COUNT_MASK))) {
+                       // if we succeed in performing the CAS we can be sure of a fast
+                       // path (only needing the CAS) unlock, if:
+                       //   a. our lock and unlock sequence are equal
+                       //   b. we don't need to clear an unlock prepost from the kernel
+
+                       // do not reset Ibit, just K&E
+                       newseq.lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
+               } else {
+                       return _pthread_mutex_fairshare_unlock_slow(mutex);
+               }
+       } while (os_unlikely(!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
+                       release)));
+
+       return 0;
+}

 
 

+#pragma mark firstfit
+
+PTHREAD_ALWAYS_INLINE
+static inline int
+_pthread_mutex_firstfit_unlock_updatebits(_pthread_mutex *mutex,
+               uint32_t *flagsp, uint32_t **mutexp, uint32_t *lvalp, uint32_t *uvalp)
+{
+       uint32_t flags = mutex->mtxopts.value & ~_PTHREAD_MTX_OPT_NOTIFY;
+       bool kernel_wake;
+
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t oldtid;
+
+       int res = _pthread_mutex_unlock_handle_options(mutex, tidaddr);
+       if (res > 0) {
+               // Valid recursive unlock
+               if (flagsp) {
+                       *flagsp = flags;
+               }
+               PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
+               return 0;
+       } else if (res < 0) {
+               PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, -res);
+               return -res;
+       }
+
+       do {
+               newseq = oldseq;
+               oldtid = os_atomic_load(tidaddr, relaxed);
+               // More than one kernel waiter means we need to do a wake.
+               kernel_wake = diff_genseq(oldseq.lgenval, oldseq.ugenval) > 0;
+               newseq.lgenval &= ~PTH_RWL_EBIT;
+
+               if (kernel_wake) {
+                       // Going to the kernel post-unlock removes a single waiter unlock
+                       // from the mutex counts.
+                       newseq.ugenval += PTHRW_INC;
+               }
+
+               if (oldtid != 0) {
+                       if (!os_atomic_cmpxchg(tidaddr, oldtid, 0, relaxed)) {
+                               return _pthread_mutex_corruption_abort(mutex);
+                       }
+               }
+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, release));
+
+       PTHREAD_TRACE(psynch_ffmutex_unlock_updatebits, mutex, oldseq.lgenval,
+                       newseq.lgenval, newseq.ugenval);
+
+       if (kernel_wake) {
+               // We choose to return this out via flags because the condition
+               // variable also uses this to determine whether to do a kernel wake
+               // when beginning a cvwait.
+               flags |= _PTHREAD_MTX_OPT_NOTIFY;
+       }
+       if (lvalp) {
+               *lvalp = newseq.lgenval;
+       }
+       if (uvalp) {
+               *uvalp = newseq.ugenval;
+       }
+       if (mutexp) {
+               *mutexp = (uint32_t *)mutex;
+       }
+       if (flagsp) {
+               *flagsp = flags;
+       }
+       return 0;
+}
+
+PTHREAD_NOEXPORT PTHREAD_NOINLINE
+static int
+_pthread_mutex_firstfit_wake(_pthread_mutex *mutex, mutex_seq newseq,
+               uint32_t flags)
+{
+       PTHREAD_TRACE(psynch_ffmutex_wake, mutex, newseq.lgenval, newseq.ugenval,
+                       0);
+       int res = __psynch_mutexdrop(mutex, newseq.lgenval, newseq.ugenval, 0,
+                       flags);
+
+       if (res == -1) {
+               res = errno;

                if (res == EINTR) {
                        res = 0;
                }
                if (res != 0) {
                if (res == EINTR) {
                        res = 0;
                }
                if (res != 0) {

-                       PTHREAD_ABORT("__p_mutexdrop failed with error %d", res);
+                       PTHREAD_ABORT("__psynch_mutexdrop failed with error %d", res);

                }
                return res;
                }
                return res;

-       } else if (firstfit == 1) {
-               if ((updateval & PTH_RWL_PBIT) != 0) {
-                       __mtx_markprepost(mutex, updateval, firstfit);
+       }
+       return 0;
+}
+
+PTHREAD_NOEXPORT PTHREAD_NOINLINE
+int
+_pthread_mutex_firstfit_unlock_slow(_pthread_mutex *mutex)
+{
+       mutex_seq newseq;
+       uint32_t flags;
+       int res;
+
+       res = _pthread_mutex_firstfit_unlock_updatebits(mutex, &flags, NULL,
+                       &newseq.lgenval, &newseq.ugenval);
+       if (res != 0) return res;
+
+       if (flags & _PTHREAD_MTX_OPT_NOTIFY) {
+               return _pthread_mutex_firstfit_wake(mutex, newseq, flags);
+       }
+       return 0;
+}
+
+PTHREAD_ALWAYS_INLINE
+static bool
+_pthread_mutex_firstfit_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid,
+               mutex_seq *newseqp)
+{
+       bool gotlock;
+
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+
+       PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_START, mutex,
+                       oldseq.lgenval, oldseq.ugenval, 0);
+
+       do {
+               newseq = oldseq;
+               gotlock = is_rwl_ebit_clear(oldseq.lgenval);
+
+               if (gotlock) {
+                       // If we see the E-bit cleared, we should just attempt to take it.
+                       newseq.lgenval |= PTH_RWL_EBIT;
+               } else {
+                       // If we failed to get the lock then we need to put ourselves back
+                       // in the queue of waiters. The previous unlocker that woke us out
+                       // of the kernel consumed the S-count for our previous wake. So
+                       // take another ticket on L and go back in the kernel to sleep.
+                       newseq.lgenval += PTHRW_INC;

                }
                }

+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire));
+
+       if (gotlock) {
+               os_atomic_store(tidaddr, selfid, relaxed);
+       }
+
+       PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_END, mutex,
+                       newseq.lgenval, newseq.ugenval, 0);
+
+       if (newseqp) {
+               *newseqp = newseq;

        }
        }

+       return gotlock;
+}
+
+PTHREAD_NOINLINE
+static int
+_pthread_mutex_firstfit_lock_wait(_pthread_mutex *mutex, mutex_seq newseq,
+               uint64_t oldtid)
+{
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t selfid = _pthread_selfid_direct();
+
+       PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex);
+       do {
+               uint32_t uval;
+               do {
+                       PTHREAD_TRACE(psynch_ffmutex_wait | DBG_FUNC_START, mutex,
+                                       newseq.lgenval, newseq.ugenval, mutex->mtxopts.value);
+                       uval = __psynch_mutexwait(mutex, newseq.lgenval, newseq.ugenval,
+                                       oldtid, mutex->mtxopts.value);
+                       PTHREAD_TRACE(psynch_ffmutex_wait | DBG_FUNC_END, mutex,
+                                       uval, 0, 0);
+                       oldtid = os_atomic_load(tidaddr, relaxed);
+               } while (uval == (uint32_t)-1);
+       } while (!_pthread_mutex_firstfit_lock_updatebits(mutex, selfid, &newseq));
+       PLOCKSTAT_MUTEX_BLOCKED((pthread_mutex_t *)mutex, BLOCK_SUCCESS_PLOCKSTAT);

 
        return 0;
 }
 
 
        return 0;
 }
 

+PTHREAD_NOEXPORT PTHREAD_NOINLINE

 int
 int

-_pthread_mutex_unlock_slow(pthread_mutex_t *omutex)
+_pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock)

 {
 {

-       int res;
-       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
-       uint32_t mtxgen, mtxugen, flags;
+       int res, recursive = 0;

 
 

-       // Initialize static mutexes for compatibility with misbehaving
-       // applications (unlock should not be the first operation on a mutex).
-       res = _pthread_mutex_check_init(omutex);
-       if (res != 0) {
-               return res;
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t oldtid, selfid = _pthread_selfid_direct();
+
+       res = _pthread_mutex_lock_handle_options(mutex, trylock, tidaddr);
+       if (res > 0) {
+               recursive = 1;
+               res = 0;
+               goto out;
+       } else if (res < 0) {
+               res = -res;
+               goto out;

        }
 
        }
 

-       res = _pthread_mutex_unlock_updatebits(mutex, &flags, NULL, &mtxgen, &mtxugen);
-       if (res != 0) {
-               return res;
+       PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_START, mutex,
+                       oldseq.lgenval, oldseq.ugenval, 0);
+
+       bool gotlock;
+       do {
+               newseq = oldseq;
+               oldtid = os_atomic_load(tidaddr, relaxed);
+
+               gotlock = is_rwl_ebit_clear(oldseq.lgenval);
+               if (trylock && !gotlock) {
+                       // We still want to perform the CAS here, even though it won't
+                       // do anything so that it fails if someone unlocked while we were
+                       // in the loop
+               } else if (gotlock) {
+                       // In first-fit, getting the lock simply adds the E-bit
+                       newseq.lgenval |= PTH_RWL_EBIT;
+               } else {
+                       // Failed to get the lock, increment the L-val and go to
+                       // the kernel to sleep
+                       newseq.lgenval += PTHRW_INC;
+               }
+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire));
+
+       PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_END, mutex,
+                       newseq.lgenval, newseq.ugenval, 0);
+
+       if (gotlock) {
+               os_atomic_store(tidaddr, selfid, relaxed);
+               res = 0;
+               PTHREAD_TRACE(psynch_mutex_ulock, mutex, newseq.lgenval,
+                               newseq.ugenval, selfid);
+       } else if (trylock) {
+               res = EBUSY;
+               PTHREAD_TRACE(psynch_mutex_utrylock_failed, mutex, newseq.lgenval,
+                               newseq.ugenval, oldtid);
+       } else {
+               PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_START, mutex,
+                               newseq.lgenval, newseq.ugenval, oldtid);
+               res = _pthread_mutex_firstfit_lock_wait(mutex, newseq, oldtid);
+               PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_END, mutex,
+                               newseq.lgenval, newseq.ugenval, oldtid);

        }
 
        }
 

-       if ((flags & _PTHREAD_MTX_OPT_NOTIFY) != 0) {
-               uint64_t newval64;
-               newval64 = (((uint64_t)mtxugen) << 32);
-               newval64 |= mtxgen;
-               return _pthread_mutex_unlock_drop(omutex, newval64, flags);
+       if (res == 0 && _pthread_mutex_is_recursive(mutex)) {
+               mutex->mtxopts.options.lock_count = 1;
+       }
+
+out:
+#if PLOCKSTAT
+       if (res == 0) {
+               PLOCKSTAT_MUTEX_ACQUIRE((pthread_mutex_t *)mutex, recursive, 0);

        } else {
        } else {

-               volatile uint64_t *tidaddr;
-               MUTEX_GETTID_ADDR(mutex, &tidaddr);
-               DEBUG_TRACE(psynch_mutex_uunlock, omutex, mtxgen, mtxugen, *tidaddr);
+               PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, res);

        }
        }

+#endif
+       return res;
+}

 
 

-       return 0;
+#pragma mark fast path
+
+PTHREAD_NOEXPORT PTHREAD_NOINLINE
+int
+_pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t *flagsp,
+               uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
+{
+       if (_pthread_mutex_is_fairshare(mutex)) {
+               return _pthread_mutex_fairshare_unlock_updatebits(mutex, flagsp,
+                               pmtxp, mgenp, ugenp);
+       }
+       return _pthread_mutex_firstfit_unlock_updatebits(mutex, flagsp, pmtxp,
+                       mgenp, ugenp);

 }
 
 }
 

-#endif // OS_UP_VARIANT_ONLY
+PTHREAD_NOEXPORT PTHREAD_NOINLINE
+int
+_pthread_mutex_lock_init_slow(_pthread_mutex *mutex, bool trylock)
+{
+       int res;
+
+       res = _pthread_mutex_check_init(mutex);
+       if (res != 0) return res;
+
+       if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
+               return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
+       }
+       return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
+}
+
+PTHREAD_NOEXPORT PTHREAD_NOINLINE
+static int
+_pthread_mutex_unlock_init_slow(_pthread_mutex *mutex)
+{
+       int res;
+
+       // Initialize static mutexes for compatibility with misbehaving
+       // applications (unlock should not be the first operation on a mutex).
+       res = _pthread_mutex_check_init(mutex);
+       if (res != 0) return res;
+
+       if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
+               return _pthread_mutex_fairshare_unlock_slow(mutex);
+       }
+       return _pthread_mutex_firstfit_unlock_slow(mutex);
+}

 
 PTHREAD_NOEXPORT_VARIANT
 int
 pthread_mutex_unlock(pthread_mutex_t *omutex)
 {
 
 PTHREAD_NOEXPORT_VARIANT
 int
 pthread_mutex_unlock(pthread_mutex_t *omutex)
 {

-#if PLOCKSTAT || DEBUG_TRACE_POINTS
-       if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
-                       DEBUG_TRACE_POINTS) {
-               return _pthread_mutex_unlock_slow(omutex);
-       }
-#endif

        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;

-       if (!_pthread_mutex_check_signature_fast(mutex)) {
-               return _pthread_mutex_unlock_slow(omutex);
+       if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) {
+               return _pthread_mutex_unlock_init_slow(mutex);
+       }
+
+       if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
+               return _pthread_mutex_fairshare_unlock(mutex);

        }
 
        }
 

-       volatile uint64_t *tidaddr;
+#if ENABLE_USERSPACE_TRACE
+       return _pthread_mutex_firstfit_unlock_slow(mutex);
+#elif PLOCKSTAT
+       if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
+               return _pthread_mutex_firstfit_unlock_slow(mutex);
+       }
+#endif
+
+       /*
+        * This is the first-fit fast path. The fairshare fast-ish path is in
+        * _pthread_mutex_firstfit_unlock()
+        */
+       uint64_t *tidaddr;

        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 

-       uint64_t oldval64, newval64;
-       volatile uint64_t *seqaddr;
+       mutex_seq *seqaddr;

        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
 
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
 

-       uint32_t lgenval, ugenval;
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);

 
 

-       do {
-               oldval64 = *seqaddr;
-               lgenval = (uint32_t)oldval64;
-               ugenval = (uint32_t)(oldval64 >> 32);
+       // We're giving up the mutex one way or the other, so go ahead and
+       // update the owner to 0 so that once the CAS below succeeds, there
+       // is no stale ownership information. If the CAS of the seqaddr
+       // fails, we may loop, but it's still valid for the owner to be
+       // SWITCHING/0
+       os_atomic_store(tidaddr, 0, relaxed);

 
 

-               int numwaiters = diff_genseq(lgenval, ugenval); // pending waiters
+       do {
+               newseq = oldseq;

 
 

-               if (numwaiters == 0) {
-                       // spurious unlock; do not touch tid
+               if (diff_genseq(oldseq.lgenval, oldseq.ugenval) == 0) {
+                       // No outstanding waiters in kernel, we can simply drop the E-bit
+                       // and return.
+                       newseq.lgenval &= ~PTH_RWL_EBIT;

                } else {
                } else {

-                       ugenval += PTHRW_INC;
+                       return _pthread_mutex_firstfit_unlock_slow(mutex);
+               }
+       } while (os_unlikely(!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
+                       release)));

 
 

-                       if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
-                               // our unlock sequence matches to lock sequence, so if the CAS is successful, the mutex is unlocked
+       return 0;
+}

 
 

-                               /* do not reset Ibit, just K&E */
-                               lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
-                       } else {
-                               return _pthread_mutex_unlock_slow(omutex);
-                       }
+PTHREAD_ALWAYS_INLINE
+static inline int
+_pthread_mutex_firstfit_lock(pthread_mutex_t *omutex, bool trylock)
+{
+       _pthread_mutex *mutex = (_pthread_mutex *)omutex;
+       if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) {
+               return _pthread_mutex_lock_init_slow(mutex, trylock);
+       }

 
 

-                       // We're giving up the mutex one way or the other, so go ahead and update the owner
-                       // to 0 so that once the CAS below succeeds, there is no stale ownership information.
-                       // If the CAS of the seqaddr fails, we may loop, but it's still valid for the owner
-                       // to be SWITCHING/0
-                       os_atomic_store(tidaddr, 0, relaxed);
-               }
+       if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
+               return _pthread_mutex_fairshare_lock(mutex, trylock);
+       }

 
 

-               newval64 = (((uint64_t)ugenval) << 32);
-               newval64 |= lgenval;
+#if ENABLE_USERSPACE_TRACE
+       return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
+#elif PLOCKSTAT
+       if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
+               return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
+       }
+#endif

 
 

-       } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
+       /*
+        * This is the first-fit fast path. The fairshare fast-ish path is in
+        * _pthread_mutex_firstfit_lock()
+        */
+       uint64_t *tidaddr;
+       MUTEX_GETTID_ADDR(mutex, &tidaddr);
+       uint64_t selfid = _pthread_selfid_direct();

 
 

-       return 0;
+       mutex_seq *seqaddr;
+       MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
+
+       mutex_seq oldseq, newseq;
+       mutex_seq_load(seqaddr, &oldseq);
+
+       if (os_unlikely(oldseq.lgenval & PTH_RWL_EBIT)) {
+               return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
+       }
+
+       bool gotlock;
+       do {
+               newseq = oldseq;
+               gotlock = is_rwl_ebit_clear(oldseq.lgenval);
+
+               if (trylock && !gotlock) {
+                       // A trylock on a held lock will fail immediately. But since
+                       // we did not load the sequence words atomically, perform a
+                       // no-op CAS64 to ensure that nobody has unlocked concurrently.
+               } else if (os_likely(gotlock)) {
+                       // In first-fit, getting the lock simply adds the E-bit
+                       newseq.lgenval |= PTH_RWL_EBIT;
+               } else {
+                       return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
+               }
+       } while (os_unlikely(!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
+                       acquire)));
+
+       if (os_likely(gotlock)) {
+               os_atomic_store(tidaddr, selfid, relaxed);
+               return 0;
+       } else if (trylock) {
+               return EBUSY;
+       } else {
+               __builtin_trap();
+       }

 }
 
 }
 

-#ifndef OS_UP_VARIANT_ONLY
+PTHREAD_NOEXPORT_VARIANT
+int
+pthread_mutex_lock(pthread_mutex_t *mutex)
+{
+       return _pthread_mutex_firstfit_lock(mutex, false);
+}
+
+PTHREAD_NOEXPORT_VARIANT
+int
+pthread_mutex_trylock(pthread_mutex_t *mutex)
+{
+       return _pthread_mutex_firstfit_lock(mutex, true);
+}

 
 
 
 

+PTHREAD_ALWAYS_INLINE

 static inline int
 _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
                uint32_t static_type)
 static inline int
 _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
                uint32_t static_type)


@@ -1007,9 +1503,9 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
                if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
                        return EINVAL;
                }
                if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
                        return EINVAL;
                }

-               mutex->prioceiling = attr->prioceiling;
+               mutex->prioceiling = (int16_t)attr->prioceiling;

                mutex->mtxopts.options.protocol = attr->protocol;
                mutex->mtxopts.options.protocol = attr->protocol;

-               mutex->mtxopts.options.policy = attr->policy;
+               mutex->mtxopts.options.policy = attr->opt;

                mutex->mtxopts.options.type = attr->type;
                mutex->mtxopts.options.pshared = attr->pshared;
        } else {
                mutex->mtxopts.options.type = attr->type;
                mutex->mtxopts.options.pshared = attr->pshared;
        } else {


@@ -1032,18 +1528,20 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
                mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
                mutex->mtxopts.options.protocol = _PTHREAD_DEFAULT_PROTOCOL;
                if (static_type != 3) {
                mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
                mutex->mtxopts.options.protocol = _PTHREAD_DEFAULT_PROTOCOL;
                if (static_type != 3) {

-                       mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
+                       mutex->mtxopts.options.policy = __pthread_mutex_default_opt_policy;

                } else {
                } else {

-                       mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
+                       mutex->mtxopts.options.policy = _PTHREAD_MTX_OPT_POLICY_FIRSTFIT;

                }
                mutex->mtxopts.options.pshared = _PTHREAD_DEFAULT_PSHARED;
        }
        mutex->priority = 0;
 
                }
                mutex->mtxopts.options.pshared = _PTHREAD_DEFAULT_PSHARED;
        }
        mutex->priority = 0;
 

-       volatile uint64_t *seqaddr;
+       mutex_seq *seqaddr;

        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

-       volatile uint64_t *tidaddr;
+
+       uint64_t *tidaddr;

        MUTEX_GETTID_ADDR(mutex, &tidaddr);
        MUTEX_GETTID_ADDR(mutex, &tidaddr);

+

 #if PTHREAD_MUTEX_INIT_UNUSED
        if ((uint32_t*)tidaddr != mutex->m_tid) {
                mutex->mtxopts.options.misalign = 1;
 #if PTHREAD_MUTEX_INIT_UNUSED
        if ((uint32_t*)tidaddr != mutex->m_tid) {
                mutex->mtxopts.options.misalign = 1;


@@ -1052,11 +1550,12 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
        __builtin_memset(mutex->m_mis, 0xff, sizeof(mutex->m_mis));
 #endif // PTHREAD_MUTEX_INIT_UNUSED
        *tidaddr = 0;
        __builtin_memset(mutex->m_mis, 0xff, sizeof(mutex->m_mis));
 #endif // PTHREAD_MUTEX_INIT_UNUSED
        *tidaddr = 0;

-       *seqaddr = 0;
+       *seqaddr = (mutex_seq){ };

 
        long sig = _PTHREAD_MUTEX_SIG;
        if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL &&
 
        long sig = _PTHREAD_MUTEX_SIG;
        if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL &&

-                       mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FAIRSHARE) {
+                       (_pthread_mutex_is_fairshare(mutex) ||
+                        _pthread_mutex_is_firstfit(mutex))) {

                // rdar://18148854 _pthread_mutex_lock & pthread_mutex_unlock fastpath
                sig = _PTHREAD_MUTEX_SIG_fast;
        }
                // rdar://18148854 _pthread_mutex_lock & pthread_mutex_unlock fastpath
                sig = _PTHREAD_MUTEX_SIG_fast;
        }


@@ -1065,7 +1564,7 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
        // For detecting copied mutexes and smashes during debugging
        uint32_t sig32 = (uint32_t)sig;
 #if defined(__LP64__)
        // For detecting copied mutexes and smashes during debugging
        uint32_t sig32 = (uint32_t)sig;
 #if defined(__LP64__)

-       uintptr_t guard =  ~(uintptr_t)mutex; // use ~ to hide from leaks
+       uintptr_t guard = ~(uintptr_t)mutex; // use ~ to hide from leaks

        __builtin_memcpy(mutex->_reserved, &guard, sizeof(guard));
        mutex->_reserved[2] = sig32;
        mutex->_reserved[3] = sig32;
        __builtin_memcpy(mutex->_reserved, &guard, sizeof(guard));
        mutex->_reserved[2] = sig32;
        mutex->_reserved[3] = sig32;


@@ -1081,7 +1580,7 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
        // the 64bit 'sig' long value in the struct. rdar://problem/21610439
        uint32_t *sig32_ptr = (uint32_t*)&mutex->sig;
        uint32_t *sig32_val = (uint32_t*)&sig;
        // the 64bit 'sig' long value in the struct. rdar://problem/21610439
        uint32_t *sig32_ptr = (uint32_t*)&mutex->sig;
        uint32_t *sig32_val = (uint32_t*)&sig;

-       *(sig32_ptr+1) = *(sig32_val+1);
+       *(sig32_ptr + 1) = *(sig32_val + 1);

        os_atomic_store(sig32_ptr, *sig32_val, release);
 #else
        os_atomic_store2o(mutex, sig, sig, release);
        os_atomic_store(sig32_ptr, *sig32_val, release);
 #else
        os_atomic_store2o(mutex, sig, sig, release);


@@ -1090,6 +1589,7 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
        return 0;
 }
 
        return 0;
 }
 

+PTHREAD_NOEXPORT_VARIANT

 int
 pthread_mutex_destroy(pthread_mutex_t *omutex)
 {
 int
 pthread_mutex_destroy(pthread_mutex_t *omutex)
 {


@@ -1097,20 +1597,20 @@ pthread_mutex_destroy(pthread_mutex_t *omutex)
 
        int res = EINVAL;
 
 
        int res = EINVAL;
 

-       LOCK(mutex->lock);
+       _PTHREAD_LOCK(mutex->lock);

        if (_pthread_mutex_check_signature(mutex)) {
        if (_pthread_mutex_check_signature(mutex)) {

-               uint32_t lgenval, ugenval;
-               uint64_t oldval64;
-               volatile uint64_t *seqaddr;
+               mutex_seq *seqaddr;

                MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
                MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

-               volatile uint64_t *tidaddr;
+
+               mutex_seq seq;
+               mutex_seq_load(seqaddr, &seq);
+
+               uint64_t *tidaddr;

                MUTEX_GETTID_ADDR(mutex, &tidaddr);
 
                MUTEX_GETTID_ADDR(mutex, &tidaddr);
 

-               oldval64 = *seqaddr;
-               lgenval = (uint32_t)oldval64;
-               ugenval = (uint32_t)(oldval64 >> 32);
-               if ((*tidaddr == (uint64_t)0) &&
-                   ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK))) {
+               if ((os_atomic_load(tidaddr, relaxed) == 0) &&
+                               (seq.lgenval & PTHRW_COUNT_MASK) ==
+                               (seq.ugenval & PTHRW_COUNT_MASK)) {

                        mutex->sig = _PTHREAD_NO_SIG;
                        res = 0;
                } else {
                        mutex->sig = _PTHREAD_NO_SIG;
                        res = 0;
                } else {


@@ -1120,16 +1620,13 @@ pthread_mutex_destroy(pthread_mutex_t *omutex)
                mutex->sig = _PTHREAD_NO_SIG;
                res = 0;
        }
                mutex->sig = _PTHREAD_NO_SIG;
                res = 0;
        }

-       UNLOCK(mutex->lock);
-       
-       return res;     
-}
+       _PTHREAD_UNLOCK(mutex->lock);

 
 

-#endif // OS_UP_VARIANT_ONLY
+       return res;
+}

 
 #endif /* !BUILDING_VARIANT ] */
 
 
 #endif /* !BUILDING_VARIANT ] */
 

-#ifndef OS_UP_VARIANT_ONLY

 /*
  * Destroy a mutex attribute structure.
  */
 /*
  * Destroy a mutex attribute structure.
  */


@@ -1149,4 +1646,3 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
        return 0;
 }
 
        return 0;
 }
 

-#endif // OS_UP_VARIANT_ONLY