[apple/libpthread.git] / src / pthread_mutex.c

/*
 * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
 *              All Rights Reserved
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted,
 * provided that the above copyright notice appears in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE.
 *
 * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
 * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *
 */
/*
 * MkLinux
 */

/*
 * POSIX Pthread Library
 * -- Mutex variable support
 */

#include "resolver.h"
#include "internal.h"
#include "kern/kern_trace.h"
#include <sys/syscall.h>

#ifdef PLOCKSTAT
#include "plockstat.h"
#else /* !PLOCKSTAT */
#define PLOCKSTAT_MUTEX_SPIN(x)
#define PLOCKSTAT_MUTEX_SPUN(x, y, z)
#define PLOCKSTAT_MUTEX_ERROR(x, y)
#define PLOCKSTAT_MUTEX_BLOCK(x)
#define PLOCKSTAT_MUTEX_BLOCKED(x, y)
#define PLOCKSTAT_MUTEX_ACQUIRE(x, y, z)
#define PLOCKSTAT_MUTEX_RELEASE(x, y)
#endif /* PLOCKSTAT */

#define PTHREAD_MUTEX_INIT_UNUSED 1

extern int __unix_conforming;

#ifndef BUILDING_VARIANT

PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
int
_pthread_mutex_unlock_slow(pthread_mutex_t *omutex);

PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
int
_pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock);

PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into _pthread_mutex_lock
int
_pthread_mutex_lock_wait(pthread_mutex_t *omutex, uint64_t newval64, uint64_t oldtid);

#endif /* BUILDING_VARIANT */

#define DEBUG_TRACE_POINTS 0

#if DEBUG_TRACE_POINTS
extern int __syscall(int number, ...);
#define DEBUG_TRACE(x, a, b, c, d) __syscall(SYS_kdebug_trace, TRACE_##x, a, b, c, d)
#else
#define DEBUG_TRACE(x, a, b, c, d) do { } while(0)
#endif

#include <machine/cpu_capabilities.h>

static inline int _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, uint32_t static_type);

#if !__LITTLE_ENDIAN__
#error MUTEX_GETSEQ_ADDR assumes little endian layout of 2 32-bit sequence words
#endif

PTHREAD_ALWAYS_INLINE
static inline void
MUTEX_GETSEQ_ADDR(_pthread_mutex *mutex,
                  volatile uint64_t **seqaddr)
{
        // 64-bit aligned address inside m_seq array (&m_seq[0] for aligned mutex)
        // We don't require more than byte alignment on OS X. rdar://22278325
        *seqaddr = (volatile uint64_t*)(((uintptr_t)mutex->m_seq + 0x7ul) & ~0x7ul);
}

PTHREAD_ALWAYS_INLINE
static inline void
MUTEX_GETTID_ADDR(_pthread_mutex *mutex,
                                  volatile uint64_t **tidaddr)
{
        // 64-bit aligned address inside m_tid array (&m_tid[0] for aligned mutex)
        // We don't require more than byte alignment on OS X. rdar://22278325
        *tidaddr = (volatile uint64_t*)(((uintptr_t)mutex->m_tid + 0x7ul) & ~0x7ul);
}

#ifndef BUILDING_VARIANT /* [ */
#ifndef OS_UP_VARIANT_ONLY

#define BLOCK_FAIL_PLOCKSTAT    0
#define BLOCK_SUCCESS_PLOCKSTAT 1

#ifdef PLOCKSTAT
/* This function is never called and exists to provide never-fired dtrace
 * probes so that user d scripts don't get errors.
 */
PTHREAD_NOEXPORT PTHREAD_USED
void
_plockstat_never_fired(void) 
{
        PLOCKSTAT_MUTEX_SPIN(NULL);
        PLOCKSTAT_MUTEX_SPUN(NULL, 0, 0);
}
#endif // PLOCKSTAT

/*
 * Initialize a mutex variable, possibly with additional attributes.
 * Public interface - so don't trust the lock - initialize it first.
 */
int
pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr)
{
#if 0
        /* conformance tests depend on not having this behavior */
        /* The test for this behavior is optional */
        if (_pthread_mutex_check_signature(mutex))
                return EBUSY;
#endif
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        _PTHREAD_LOCK_INIT(mutex->lock);
        return (_pthread_mutex_init(mutex, attr, 0x7));
}

int
pthread_mutex_getprioceiling(const pthread_mutex_t *omutex, int *prioceiling)
{
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (_pthread_mutex_check_signature(mutex)) {
                _PTHREAD_LOCK(mutex->lock);
                *prioceiling = mutex->prioceiling;
                res = 0;
                _PTHREAD_UNLOCK(mutex->lock);
        }
        return res;
}

int
pthread_mutex_setprioceiling(pthread_mutex_t *omutex, int prioceiling, int *old_prioceiling)
{
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (_pthread_mutex_check_signature(mutex)) {
                _PTHREAD_LOCK(mutex->lock);
                if (prioceiling >= -999 || prioceiling <= 999) {
                        *old_prioceiling = mutex->prioceiling;
                        mutex->prioceiling = prioceiling;
                        res = 0;
                }
                _PTHREAD_UNLOCK(mutex->lock);
        }
        return res;
}

int
pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *prioceiling)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                *prioceiling = attr->prioceiling;
                res = 0;
        }
        return res;
}

int
pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int *protocol)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                *protocol = attr->protocol;
                res = 0;
        }
        return res;
}

int
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                *type = attr->type;
                res = 0;
        }
        return res;
}

int
pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                *pshared = (int)attr->pshared;
                res = 0;
        }
        return res;
}

int
pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
        attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
        attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;
        attr->policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
        attr->type = PTHREAD_MUTEX_DEFAULT;
        attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
        attr->pshared = _PTHREAD_DEFAULT_PSHARED;
        return 0;
}

int
pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int prioceiling)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                if (prioceiling >= -999 || prioceiling <= 999) {
                        attr->prioceiling = prioceiling;
                        res = 0;
                }
        }
        return res;
}

int
pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                switch (protocol) {
                        case PTHREAD_PRIO_NONE:
                        case PTHREAD_PRIO_INHERIT:
                        case PTHREAD_PRIO_PROTECT:
                                attr->protocol = protocol;
                                res = 0;
                                break;
                }
        }
        return res;
}

int
pthread_mutexattr_setpolicy_np(pthread_mutexattr_t *attr, int policy)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                switch (policy) {
                        case _PTHREAD_MUTEX_POLICY_FAIRSHARE:
                        case _PTHREAD_MUTEX_POLICY_FIRSTFIT:
                                attr->policy = policy;
                                res = 0;
                                break;
                }
        }
        return res;
}

int
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
{
        int res = EINVAL;
        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
                switch (type) {
                        case PTHREAD_MUTEX_NORMAL:
                        case PTHREAD_MUTEX_ERRORCHECK:
                        case PTHREAD_MUTEX_RECURSIVE:
                        //case PTHREAD_MUTEX_DEFAULT:
                                attr->type = type;
                                res = 0;
                                break;
                }
        }
        return res;
}

// XXX remove
void
cthread_yield(void) 
{
        sched_yield();
}

void
pthread_yield_np(void) 
{
        sched_yield();
}


/*
 * Temp: till pshared is fixed correctly
 */
int
pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
{
        int res = EINVAL;
#if __DARWIN_UNIX03
        if (__unix_conforming == 0) {
                __unix_conforming = 1;
        }
#endif /* __DARWIN_UNIX03 */

        if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
#if __DARWIN_UNIX03
                if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED))
#else /* __DARWIN_UNIX03 */
                if ( pshared == PTHREAD_PROCESS_PRIVATE)
#endif /* __DARWIN_UNIX03 */
                {
                        attr->pshared = pshared; 
                        res = 0;
                }
        }
        return res;
}

PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
int
_pthread_mutex_corruption_abort(_pthread_mutex *mutex);

PTHREAD_NOINLINE
int
_pthread_mutex_corruption_abort(_pthread_mutex *mutex)
{
        PTHREAD_ABORT("pthread_mutex corruption: mutex %p owner changed in the middle of lock/unlock");
        return EINVAL; // NOTREACHED
}

/*
 * Sequence numbers and TID:
 *
 * In steady (and uncontended) state, an unlocked mutex will
 * look like A=[L4 U4 TID0]. When it is being locked, it transitions
 * to B=[L5+KE U4 TID0] and then C=[L5+KE U4 TID940]. For an uncontended mutex,
 * the unlock path will then transition to D=[L5 U4 TID0] and then finally
 * E=[L5 U5 TID0].
 *
 * If a contender comes in after B, the mutex will instead transition to E=[L6+KE U4 TID0]
 * and then F=[L6+KE U4 TID940]. If a contender comes in after C, it will transition to
 * F=[L6+KE U4 TID940] directly. In both cases, the contender will enter the kernel with either
 * mutexwait(U4, TID0) or mutexwait(U4, TID940). The first owner will unlock the mutex
 * by first updating the owner to G=[L6+KE U4 TID-1] and then doing the actual unlock to
 * H=[L6+KE U5 TID=-1] before entering the kernel with mutexdrop(U5, -1) to signal the next waiter
 * (potentially as a prepost). When the waiter comes out of the kernel, it will update the owner to
 * I=[L6+KE U5 TID941]. An unlock at this point is simply J=[L6 U5 TID0] and then K=[L6 U6 TID0].
 *
 * At various points along these timelines, since the sequence words and TID are written independently,
 * a thread may get preempted and another thread might see inconsistent data. In the worst case, another
 * thread may see the TID in the SWITCHING (-1) state or unlocked (0) state for longer because the
 * owning thread was preempted.
 */

/*
 * Drop the mutex unlock references from cond_wait. or mutex_unlock.
 */
PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_unlock_updatebits(_pthread_mutex *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
{
        bool firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
        uint32_t lgenval, ugenval, flags;
        uint64_t oldtid, newtid;
        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);

        flags = mutex->mtxopts.value;
        flags &= ~_PTHREAD_MTX_OPT_NOTIFY; // no notification by default

        if (mutex->mtxopts.options.type != PTHREAD_MUTEX_NORMAL) {
                uint64_t selfid = _pthread_selfid_direct();

                if (*tidaddr != selfid) {
                        //PTHREAD_ABORT("dropping recur or error mutex not owned by the thread");
                        PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, EPERM);
                        return EPERM;
                } else if (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE &&
                           --mutex->mtxopts.options.lock_count) {
                        PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
                        if (flagsp != NULL) {
                                *flagsp = flags;
                        }
                        return 0;
                }
        }

        uint64_t oldval64, newval64;
        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

        bool clearprepost, clearnotify, spurious;
        do {
                oldval64 = *seqaddr;
                oldtid = *tidaddr;
                lgenval = (uint32_t)oldval64;
                ugenval = (uint32_t)(oldval64 >> 32);

                clearprepost = false;
                clearnotify = false;
                spurious = false;

                int numwaiters = diff_genseq(lgenval, ugenval); // pending waiters

                if (numwaiters == 0) {
                        // spurious unlock; do not touch tid
                        spurious = true;
                } else {
                        ugenval += PTHRW_INC;

                        if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
                                // our unlock sequence matches to lock sequence, so if the CAS is successful, the mutex is unlocked

                                /* do not reset Ibit, just K&E */
                                lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
                                clearnotify = true;
                                newtid = 0; // clear owner
                        } else {
                                if (firstfit) {
                                        lgenval &= ~PTH_RWL_EBIT; // reset E bit so another can acquire meanwhile
                                        newtid = 0;
                                } else {
                                        newtid = PTHREAD_MTX_TID_SWITCHING;
                                }
                                // need to signal others waiting for mutex
                                flags |= _PTHREAD_MTX_OPT_NOTIFY;
                        }
                        
                        if (newtid != oldtid) {
                                // We're giving up the mutex one way or the other, so go ahead and update the owner to SWITCHING
                                // or 0 so that once the CAS below succeeds, there is no stale ownership information.
                                // If the CAS of the seqaddr fails, we may loop, but it's still valid for the owner
                                // to be SWITCHING/0
                                if (!os_atomic_cmpxchg(tidaddr, oldtid, newtid, relaxed)) {
                                        // we own this mutex, nobody should be updating it except us
                                        return _pthread_mutex_corruption_abort(mutex);
                                }
                        }
                }

                if (clearnotify || spurious) {
                        flags &= ~_PTHREAD_MTX_OPT_NOTIFY;
                        if (firstfit && ((lgenval & PTH_RWL_PBIT) != 0)) {
                                clearprepost = true;
                                lgenval &= ~PTH_RWL_PBIT;
                        }
                }
                
                newval64 = (((uint64_t)ugenval) << 32);
                newval64 |= lgenval;

        } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));

        if (clearprepost) {
                 __psynch_cvclrprepost(mutex, lgenval, ugenval, 0, 0, lgenval, (flags | _PTHREAD_MTX_OPT_MUTEX));
        }

        if (mgenp != NULL) {
                *mgenp = lgenval;
        }
        if (ugenp != NULL) {
                *ugenp = ugenval;
        }
        if (pmtxp != NULL) {
                *pmtxp = (uint32_t *)mutex;
        }
        if (flagsp != NULL) {
                *flagsp = flags;
        }

        return 0;
}

PTHREAD_NOEXPORT
int
__mtx_droplock(_pthread_mutex *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
{
        return _pthread_mutex_unlock_updatebits(mutex, flagsp, pmtxp, mgenp, ugenp);
}

PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)
{
        int res = 0;
        int firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
        int isebit = 0;

        uint32_t lgenval, ugenval;
        uint64_t oldval64, newval64;
        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
        uint64_t oldtid;
        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);

        do {
                do {
                        oldval64 = *seqaddr;
                        oldtid = *tidaddr;
                        lgenval = (uint32_t)oldval64;
                        ugenval = (uint32_t)(oldval64 >> 32);

                        // E bit was set on first pass through the loop but is no longer
                        // set. Apparently we spin until it arrives.
                        // XXX: verify this is desired behavior.
                } while (isebit && (lgenval & PTH_RWL_EBIT) == 0);

                if (isebit) {
                        // first fit mutex now has the E bit set. Return 1.
                        res = 1;
                        break;
                }

                if (firstfit) {
                        isebit = (lgenval & PTH_RWL_EBIT) != 0;
                } else if ((lgenval & (PTH_RWL_KBIT|PTH_RWL_EBIT)) == (PTH_RWL_KBIT|PTH_RWL_EBIT)) {
                        // fairshare mutex and the bits are already set, just update tid
                        break;
                }

                // either first fit or no E bit set
                // update the bits
                lgenval |= PTH_RWL_KBIT | PTH_RWL_EBIT;

                newval64 = (((uint64_t)ugenval) << 32);
                newval64 |= lgenval;

                // set s and b bit
                // Retry if CAS fails, or if it succeeds with firstfit and E bit already set
        } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire) || (firstfit && isebit));

        if (res == 0) {
                if (!os_atomic_cmpxchg(tidaddr, oldtid, selfid, relaxed)) {
                        // we own this mutex, nobody should be updating it except us
                        return _pthread_mutex_corruption_abort(mutex);
                }
        }

        return res;
}

PTHREAD_NOINLINE
static int
__mtx_markprepost(_pthread_mutex *mutex, uint32_t updateval, int firstfit)
{
        uint32_t flags;
        uint32_t lgenval, ugenval;
        uint64_t oldval64, newval64;

        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

        if (firstfit != 0 && (updateval & PTH_RWL_PBIT) != 0) {
                int clearprepost;
                do {                            
                        clearprepost = 0;

                        flags = mutex->mtxopts.value;

                        oldval64 = *seqaddr;
                        lgenval = (uint32_t)oldval64;
                        ugenval = (uint32_t)(oldval64 >> 32);

                        /* update the bits */
                        if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
                                clearprepost = 1;       
                                lgenval &= ~PTH_RWL_PBIT;
                        } else {
                                lgenval |= PTH_RWL_PBIT;
                        }
                        newval64 = (((uint64_t)ugenval) << 32);
                        newval64 |= lgenval;
                } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
                
                if (clearprepost != 0) {
                        __psynch_cvclrprepost(mutex, lgenval, ugenval, 0, 0, lgenval, (flags | _PTHREAD_MTX_OPT_MUTEX));
                }
        }
        return 0;
}

PTHREAD_NOINLINE
static int
_pthread_mutex_check_init_slow(pthread_mutex_t *omutex)
{
        int res = EINVAL;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;

        if (_pthread_mutex_check_signature_init(mutex)) {
                _PTHREAD_LOCK(mutex->lock);
                if (_pthread_mutex_check_signature_init(mutex)) {
                        // initialize a statically initialized mutex to provide
                        // compatibility for misbehaving applications.
                        // (unlock should not be the first operation on a mutex)
                        res = _pthread_mutex_init(mutex, NULL, (mutex->sig & 0xf));
                } else if (_pthread_mutex_check_signature(mutex)) {
                        res = 0;
                }
                _PTHREAD_UNLOCK(mutex->lock);
        } else if (_pthread_mutex_check_signature(mutex)) {
                res = 0;
        }
        if (res != 0) {
                PLOCKSTAT_MUTEX_ERROR(omutex, res);
        }
        return res;
}

PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_check_init(pthread_mutex_t *omutex)
{
        int res = 0;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        
        if (!_pthread_mutex_check_signature(mutex)) {
                return _pthread_mutex_check_init_slow(omutex);
        }
        return res;
}

PTHREAD_NOINLINE
int
_pthread_mutex_lock_wait(pthread_mutex_t *omutex, uint64_t newval64, uint64_t oldtid)
{
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        uint32_t lgenval = (uint32_t)newval64;
        uint32_t ugenval = (uint32_t)(newval64 >> 32);

        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
        uint64_t selfid = _pthread_selfid_direct();

        PLOCKSTAT_MUTEX_BLOCK(omutex);
        do {
                uint32_t updateval;
                do {
                        updateval = __psynch_mutexwait(omutex, lgenval, ugenval, oldtid, mutex->mtxopts.value);
                        oldtid = *tidaddr;
                } while (updateval == (uint32_t)-1);

                // returns 0 on succesful update; in firstfit it may fail with 1
        } while (_pthread_mutex_lock_updatebits(mutex, selfid) == 1);
        PLOCKSTAT_MUTEX_BLOCKED(omutex, BLOCK_SUCCESS_PLOCKSTAT);

        return 0;
}

int
_pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock)
{
        int res;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;

        res = _pthread_mutex_check_init(omutex);
        if (res != 0) {
                return res;
        }

        uint64_t oldtid;
        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
        uint64_t selfid = _pthread_selfid_direct();

        if (mutex->mtxopts.options.type != PTHREAD_MUTEX_NORMAL) {
                if (*tidaddr == selfid) {
                        if (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
                                if (mutex->mtxopts.options.lock_count < USHRT_MAX) {
                                        mutex->mtxopts.options.lock_count++;
                                        PLOCKSTAT_MUTEX_ACQUIRE(omutex, 1, 0);
                                        res = 0;
                                } else {
                                        res = EAGAIN;
                                        PLOCKSTAT_MUTEX_ERROR(omutex, res);
                                }
                        } else if (trylock) { /* PTHREAD_MUTEX_ERRORCHECK */
                                // <rdar://problem/16261552> as per OpenGroup, trylock cannot
                                // return EDEADLK on a deadlock, it should return EBUSY.
                                res = EBUSY;
                                PLOCKSTAT_MUTEX_ERROR(omutex, res);
                        } else  { /* PTHREAD_MUTEX_ERRORCHECK */
                                res = EDEADLK;
                                PLOCKSTAT_MUTEX_ERROR(omutex, res);
                        }
                        return res;
                }
        }

        uint64_t oldval64, newval64;
        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

        uint32_t lgenval, ugenval;
        bool gotlock = false;

        do {
                oldval64 = *seqaddr;
                oldtid = *tidaddr;
                lgenval = (uint32_t)oldval64;
                ugenval = (uint32_t)(oldval64 >> 32);

                gotlock = ((lgenval & PTH_RWL_EBIT) == 0);

                if (trylock && !gotlock) {
                        // A trylock on a held lock will fail immediately. But since
                        // we did not load the sequence words atomically, perform a
                        // no-op CAS64 to ensure that nobody has unlocked concurrently.
                } else {
                        // Increment the lock sequence number and force the lock into E+K
                        // mode, whether "gotlock" is true or not.
                        lgenval += PTHRW_INC;
                        lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
                }

                newval64 = (((uint64_t)ugenval) << 32);
                newval64 |= lgenval;
                
                // Set S and B bit
        } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire));

        if (gotlock) {
                os_atomic_store(tidaddr, selfid, relaxed);
                res = 0;
                DEBUG_TRACE(psynch_mutex_ulock, omutex, lgenval, ugenval, selfid);
                PLOCKSTAT_MUTEX_ACQUIRE(omutex, 0, 0);
        } else if (trylock) {
                res = EBUSY;
                DEBUG_TRACE(psynch_mutex_utrylock_failed, omutex, lgenval, ugenval, oldtid);
                PLOCKSTAT_MUTEX_ERROR(omutex, res);
        } else {
                res = _pthread_mutex_lock_wait(omutex, newval64, oldtid);
        }

        if (res == 0 && mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
                mutex->mtxopts.options.lock_count = 1;
        }

        PLOCKSTAT_MUTEX_ACQUIRE(omutex, 0, 0);

        return res;
}

#endif // OS_UP_VARIANT_ONLY

PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_lock(pthread_mutex_t *omutex, bool trylock)
{
#if PLOCKSTAT || DEBUG_TRACE_POINTS
        if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
                        DEBUG_TRACE_POINTS) {
                return _pthread_mutex_lock_slow(omutex, trylock);
        }
#endif
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (!_pthread_mutex_check_signature_fast(mutex)) {
                return _pthread_mutex_lock_slow(omutex, trylock);
        }

        uint64_t oldtid;
        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
        uint64_t selfid = _pthread_selfid_direct();

        uint64_t oldval64, newval64;
        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

        uint32_t lgenval, ugenval;
        bool gotlock = false;

        do {
                oldval64 = *seqaddr;
                oldtid = *tidaddr;
                lgenval = (uint32_t)oldval64;
                ugenval = (uint32_t)(oldval64 >> 32);

                gotlock = ((lgenval & PTH_RWL_EBIT) == 0);

                if (trylock && !gotlock) {
                        // A trylock on a held lock will fail immediately. But since
                        // we did not load the sequence words atomically, perform a
                        // no-op CAS64 to ensure that nobody has unlocked concurrently.
                } else {
                        // Increment the lock sequence number and force the lock into E+K
                        // mode, whether "gotlock" is true or not.
                        lgenval += PTHRW_INC;
                        lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
                }

                newval64 = (((uint64_t)ugenval) << 32);
                newval64 |= lgenval;

                // Set S and B bit
        } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire));

        if (os_fastpath(gotlock)) {
                os_atomic_store(tidaddr, selfid, relaxed);
                return 0;
        } else if (trylock) {
                return EBUSY;
        } else {
                return _pthread_mutex_lock_wait(omutex, newval64, oldtid);
        }
}

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_lock(pthread_mutex_t *mutex)
{
        return _pthread_mutex_lock(mutex, false);
}

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_trylock(pthread_mutex_t *mutex)
{
        return _pthread_mutex_lock(mutex, true);
}

#ifndef OS_UP_VARIANT_ONLY
/*
 * Unlock a mutex.
 * TODO: Priority inheritance stuff
 */

PTHREAD_NOINLINE
static int
_pthread_mutex_unlock_drop(pthread_mutex_t *omutex, uint64_t newval64, uint32_t flags)
{
        int res;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        uint32_t lgenval = (uint32_t)newval64;
        uint32_t ugenval = (uint32_t)(newval64 >> 32);

        uint32_t updateval;
        int firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);

        updateval = __psynch_mutexdrop(omutex, lgenval, ugenval, *tidaddr, flags);

        if (updateval == (uint32_t)-1) {
                res = errno;

                if (res == EINTR) {
                        res = 0;
                }
                if (res != 0) {
                        PTHREAD_ABORT("__p_mutexdrop failed with error %d", res);
                }
                return res;
        } else if (firstfit == 1) {
                if ((updateval & PTH_RWL_PBIT) != 0) {
                        __mtx_markprepost(mutex, updateval, firstfit);
                }
        }

        return 0;
}

int
_pthread_mutex_unlock_slow(pthread_mutex_t *omutex)
{
        int res;
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        uint32_t mtxgen, mtxugen, flags;

        // Initialize static mutexes for compatibility with misbehaving
        // applications (unlock should not be the first operation on a mutex).
        res = _pthread_mutex_check_init(omutex);
        if (res != 0) {
                return res;
        }

        res = _pthread_mutex_unlock_updatebits(mutex, &flags, NULL, &mtxgen, &mtxugen);
        if (res != 0) {
                return res;
        }

        if ((flags & _PTHREAD_MTX_OPT_NOTIFY) != 0) {
                uint64_t newval64;
                newval64 = (((uint64_t)mtxugen) << 32);
                newval64 |= mtxgen;
                return _pthread_mutex_unlock_drop(omutex, newval64, flags);
        } else {
                volatile uint64_t *tidaddr;
                MUTEX_GETTID_ADDR(mutex, &tidaddr);
                DEBUG_TRACE(psynch_mutex_uunlock, omutex, mtxgen, mtxugen, *tidaddr);
        }

        return 0;
}

#endif // OS_UP_VARIANT_ONLY

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_unlock(pthread_mutex_t *omutex)
{
#if PLOCKSTAT || DEBUG_TRACE_POINTS
        if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
                        DEBUG_TRACE_POINTS) {
                return _pthread_mutex_unlock_slow(omutex);
        }
#endif
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (!_pthread_mutex_check_signature_fast(mutex)) {
                return _pthread_mutex_unlock_slow(omutex);
        }

        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);

        uint64_t oldval64, newval64;
        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

        uint32_t lgenval, ugenval;

        do {
                oldval64 = *seqaddr;
                lgenval = (uint32_t)oldval64;
                ugenval = (uint32_t)(oldval64 >> 32);

                int numwaiters = diff_genseq(lgenval, ugenval); // pending waiters

                if (numwaiters == 0) {
                        // spurious unlock; do not touch tid
                } else {
                        ugenval += PTHRW_INC;

                        if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
                                // our unlock sequence matches to lock sequence, so if the CAS is successful, the mutex is unlocked

                                /* do not reset Ibit, just K&E */
                                lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
                        } else {
                                return _pthread_mutex_unlock_slow(omutex);
                        }

                        // We're giving up the mutex one way or the other, so go ahead and update the owner
                        // to 0 so that once the CAS below succeeds, there is no stale ownership information.
                        // If the CAS of the seqaddr fails, we may loop, but it's still valid for the owner
                        // to be SWITCHING/0
                        os_atomic_store(tidaddr, 0, relaxed);
                }

                newval64 = (((uint64_t)ugenval) << 32);
                newval64 |= lgenval;

        } while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));

        return 0;
}

#ifndef OS_UP_VARIANT_ONLY


static inline int
_pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
                uint32_t static_type)
{
        mutex->mtxopts.value = 0;
        mutex->mtxopts.options.mutex = 1;
        if (attr) {
                if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
                        return EINVAL;
                }
                mutex->prioceiling = attr->prioceiling;
                mutex->mtxopts.options.protocol = attr->protocol;
                mutex->mtxopts.options.policy = attr->policy;
                mutex->mtxopts.options.type = attr->type;
                mutex->mtxopts.options.pshared = attr->pshared;
        } else {
                switch (static_type) {
                        case 1:
                                mutex->mtxopts.options.type = PTHREAD_MUTEX_ERRORCHECK;
                                break;
                        case 2:
                                mutex->mtxopts.options.type = PTHREAD_MUTEX_RECURSIVE;
                                break;
                        case 3:
                                /* firstfit fall thru */
                        case 7:
                                mutex->mtxopts.options.type = PTHREAD_MUTEX_DEFAULT;
                                break;
                        default:
                                return EINVAL;
                }

                mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
                mutex->mtxopts.options.protocol = _PTHREAD_DEFAULT_PROTOCOL;
                if (static_type != 3) {
                        mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
                } else {
                        mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
                }
                mutex->mtxopts.options.pshared = _PTHREAD_DEFAULT_PSHARED;
        }
        mutex->priority = 0;

        volatile uint64_t *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
        volatile uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
#if PTHREAD_MUTEX_INIT_UNUSED
        if ((uint32_t*)tidaddr != mutex->m_tid) {
                mutex->mtxopts.options.misalign = 1;
                __builtin_memset(mutex->m_tid, 0xff, sizeof(mutex->m_tid));
        }
        __builtin_memset(mutex->m_mis, 0xff, sizeof(mutex->m_mis));
#endif // PTHREAD_MUTEX_INIT_UNUSED
        *tidaddr = 0;
        *seqaddr = 0;

        long sig = _PTHREAD_MUTEX_SIG;
        if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL &&
                        mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FAIRSHARE) {
                // rdar://18148854 _pthread_mutex_lock & pthread_mutex_unlock fastpath
                sig = _PTHREAD_MUTEX_SIG_fast;
        }

#if PTHREAD_MUTEX_INIT_UNUSED
        // For detecting copied mutexes and smashes during debugging
        uint32_t sig32 = (uint32_t)sig;
#if defined(__LP64__)
        uintptr_t guard =  ~(uintptr_t)mutex; // use ~ to hide from leaks
        __builtin_memcpy(mutex->_reserved, &guard, sizeof(guard));
        mutex->_reserved[2] = sig32;
        mutex->_reserved[3] = sig32;
        mutex->_pad = sig32;
#else
        mutex->_reserved[0] = sig32;
#endif
#endif // PTHREAD_MUTEX_INIT_UNUSED

        // Ensure all contents are properly set before setting signature.
#if defined(__LP64__)
        // For binary compatibility reasons we cannot require natural alignment of
        // the 64bit 'sig' long value in the struct. rdar://problem/21610439
        uint32_t *sig32_ptr = (uint32_t*)&mutex->sig;
        uint32_t *sig32_val = (uint32_t*)&sig;
        *(sig32_ptr+1) = *(sig32_val+1);
        os_atomic_store(sig32_ptr, *sig32_val, release);
#else
        os_atomic_store2o(mutex, sig, sig, release);
#endif

        return 0;
}

int
pthread_mutex_destroy(pthread_mutex_t *omutex)
{
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;

        int res = EINVAL;

        _PTHREAD_LOCK(mutex->lock);
        if (_pthread_mutex_check_signature(mutex)) {
                uint32_t lgenval, ugenval;
                uint64_t oldval64;
                volatile uint64_t *seqaddr;
                MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
                volatile uint64_t *tidaddr;
                MUTEX_GETTID_ADDR(mutex, &tidaddr);

                oldval64 = *seqaddr;
                lgenval = (uint32_t)oldval64;
                ugenval = (uint32_t)(oldval64 >> 32);
                if ((*tidaddr == (uint64_t)0) &&
                    ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK))) {
                        mutex->sig = _PTHREAD_NO_SIG;
                        res = 0;
                } else {
                        res = EBUSY;
                }
        } else if (_pthread_mutex_check_signature_init(mutex)) {
                mutex->sig = _PTHREAD_NO_SIG;
                res = 0;
        }
        _PTHREAD_UNLOCK(mutex->lock);
        
        return res;     
}

#endif // OS_UP_VARIANT_ONLY

#endif /* !BUILDING_VARIANT ] */

#ifndef OS_UP_VARIANT_ONLY
/*
 * Destroy a mutex attribute structure.
 */
int
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
{
#if __DARWIN_UNIX03
        if (__unix_conforming == 0) {
                __unix_conforming = 1;
        }
        if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
                return EINVAL;
        }
#endif /* __DARWIN_UNIX03 */

        attr->sig = _PTHREAD_NO_SIG;
        return 0;
}

#endif // OS_UP_VARIANT_ONLY