X-Git-Url: https://git.saurik.com/apple/libc.git/blobdiff_plain/5b2abdfbf4211b6592cdd02b9507555a0ecbb04b..7b00c0c43f52e9d27168e67a26aac19065cdb40c:/pthreads/pthread_rwlock.c diff --git a/pthreads/pthread_rwlock.c b/pthreads/pthread_rwlock.c index 978ed80..3920969 100644 --- a/pthreads/pthread_rwlock.c +++ b/pthreads/pthread_rwlock.c @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ /*- * Copyright (c) 1998 Alex Nash * All rights reserved. @@ -34,286 +56,1825 @@ */ #include "pthread_internals.h" +#include /* For printf(). */ + +extern int __unix_conforming; + +#ifdef PLOCKSTAT +#include "plockstat.h" +#else /* !PLOCKSTAT */ +#define PLOCKSTAT_RW_ERROR(x, y, z) +#define PLOCKSTAT_RW_BLOCK(x, y) +#define PLOCKSTAT_RW_BLOCKED(x, y, z) +#define PLOCKSTAT_RW_ACQUIRE(x, y) +#define PLOCKSTAT_RW_RELEASE(x, y) +#endif /* PLOCKSTAT */ + +#define READ_LOCK_PLOCKSTAT 0 +#define WRITE_LOCK_PLOCKSTAT 1 + +#define BLOCK_FAIL_PLOCKSTAT 0 +#define BLOCK_SUCCESS_PLOCKSTAT 1 /* maximum number of times a read lock may be obtained */ -#define MAX_READ_LOCKS (INT_MAX - 1) +#define MAX_READ_LOCKS (INT_MAX - 1) + + +#ifndef BUILDING_VARIANT /* [ */ +__private_extern__ int usenew_impl = 1; +#else /* BUILDING_VARIANT */ +extern int usenew_impl; +#endif /* BUILDING_VARIANT */ + +extern int PR_5243343_flag; + +#if defined(__LP64__) +#define RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr) \ +{ \ + if (rwlock->misalign != 0) { \ + lcntaddr = &rwlock->rw_seq[1]; \ + seqaddr = &rwlock->rw_seq[2]; \ + ucntaddr = &rwlock->rw_seq[3]; \ + } else { \ + lcntaddr = &rwlock->rw_seq[0]; \ + seqaddr = &rwlock->rw_seq[1]; \ + ucntaddr = &rwlock->rw_seq[2]; \ + } \ +} +#else /* __LP64__ */ +#define RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr) \ +{ \ + if (rwlock->misalign != 0) { \ + lcntaddr = &rwlock->rw_seq[1]; \ + seqaddr = &rwlock->rw_seq[2]; \ + ucntaddr = &rwlock->rw_seq[3]; \ + } else { \ + lcntaddr = &rwlock->rw_seq[0]; \ + seqaddr = &rwlock->rw_seq[1]; \ + ucntaddr = &rwlock->rw_seq[2]; \ + } \ +} +#endif /* __LP64__ */ + +__private_extern__ int __pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr); + + +#define _KSYN_TRACE_ 0 + +#if _KSYN_TRACE_ +#include +#ifndef BUILDING_VARIANT /* [ */ +static void set_enable(int); +#endif /* !BUILDING_VARIANT ] */ + +/* The Function qualifiers */ +#define DBG_FUNC_START 1 +#define DBG_FUNC_END 2 +#define DBG_FUNC_NONE 0 + +int __kdebug_trace(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t); + +#define _KSYN_TRACE_RW_RDLOCK 0x9000080 +#define _KSYN_TRACE_RW_WRLOCK 0x9000084 +#define _KSYN_TRACE_RW_UNLOCK 0x9000088 +#define _KSYN_TRACE_RW_UNACT1 0x900808c +#define _KSYN_TRACE_RW_UNACT2 0x9008090 +#define _KSYN_TRACE_RW_UNACTK 0x9008094 +#define _KSYN_TRACE_RW_UNACTE 0x9008098 +#define _KSYN_TRACE_RW_UNACTR 0x900809c +#define _KSYN_TRACE_RW_TOOMANY 0x90080a0 +#define _KSYN_TRACE_RW_TRYWRLOCK 0x90080a4 +#define _KSYN_TRACE_RW_TRYRDLOCK 0x90080a8 +#endif /* _KSYN_TRACE_ */ + +__private_extern__ void rwlock_action_onreturn(pthread_rwlock_t * rwlock, uint32_t updateval); +__private_extern__ int rw_diffgenseq(uint32_t x, uint32_t y); + +#ifndef BUILDING_VARIANT /* [ */ +static uint32_t modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits); + +int +pthread_rwlockattr_init(pthread_rwlockattr_t *attr) +{ + attr->sig = _PTHREAD_RWLOCK_ATTR_SIG; + attr->pshared = _PTHREAD_DEFAULT_PSHARED; + return (0); +} + +int +pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr) +{ + attr->sig = _PTHREAD_NO_SIG; /* Uninitialized */ + attr->pshared = 0; + return (0); +} int -pthread_rwlock_destroy(pthread_rwlock_t *rwlock) +pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, + int *pshared) { + if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) + { + *pshared = (int)attr->pshared; + return (0); + } else + { + return (EINVAL); /* Not an initialized 'attribute' structure */ + } +} - if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { - return(EINVAL); + +int +pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared) +{ + if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) + { +#if __DARWIN_UNIX03 + if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED)) +#else /* __DARWIN_UNIX03 */ + if ( pshared == PTHREAD_PROCESS_PRIVATE) +#endif /* __DARWIN_UNIX03 */ + { + attr->pshared = pshared ; + return (0); + } else + { + return (EINVAL); /* Invalid parameter */ + } + } else + { + return (EINVAL); /* Not an initialized 'attribute' structure */ + } + +} + +__private_extern__ int +__pthread_rwlock_init(pthread_rwlock_t * orwlock, const pthread_rwlockattr_t *attr) +{ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + + if ((attr != NULL) && (attr->pshared == PTHREAD_PROCESS_SHARED)) { + rwlock->pshared = PTHREAD_PROCESS_SHARED; + rwlock->rw_flags = PTHRW_KERN_PROCESS_SHARED; + } else { + rwlock->pshared = _PTHREAD_DEFAULT_PSHARED; + rwlock->rw_flags = PTHRW_KERN_PROCESS_PRIVATE; + } + + if (((uintptr_t)rwlock & 0x07) != 0) { + rwlock->misalign = 1; +#if defined(__LP64__) + rwlock->rw_lcntaddr = &rwlock->rw_seq[1]; + rwlock->rw_seqaddr = &rwlock->rw_seq[2]; + rwlock->rw_ucntaddr = &rwlock->rw_seq[3]; + rwlock->rw_seq[1]= PTHRW_RWLOCK_INIT; + rwlock->rw_seq[2]= PTHRW_RWS_INIT; + rwlock->rw_seq[3]= 0; +#else /* __LP64__ */ + rwlock->rw_lcntaddr = &rwlock->rw_seq[1]; + rwlock->rw_seqaddr = &rwlock->rw_seq[2]; + rwlock->rw_ucntaddr = &rwlock->rw_seq[3]; + rwlock->rw_seq[1]= PTHRW_RWLOCK_INIT; + rwlock->rw_seq[2]= PTHRW_RWS_INIT; + rwlock->rw_seq[3]= 0; +#endif /* __LP64__ */ + } else { - pthread_mutex_destroy(&rwlock->lock); - pthread_cond_destroy(&rwlock->read_signal); - pthread_cond_destroy(&rwlock->write_signal); - rwlock->sig = _PTHREAD_NO_SIG; - return(ESUCCESS); + rwlock->misalign = 0; +#if defined(__LP64__) + rwlock->rw_lcntaddr = &rwlock->rw_seq[0]; + rwlock->rw_seqaddr = &rwlock->rw_seq[1]; + rwlock->rw_ucntaddr = &rwlock->rw_seq[2]; + rwlock->rw_seq[0]= PTHRW_RWLOCK_INIT; + rwlock->rw_seq[1]= PTHRW_RWS_INIT; + rwlock->rw_seq[2]= 0; +#else /* __LP64__ */ + rwlock->rw_lcntaddr = &rwlock->rw_seq[0]; + rwlock->rw_seqaddr = &rwlock->rw_seq[1]; + rwlock->rw_ucntaddr = &rwlock->rw_seq[2]; + rwlock->rw_seq[0]= PTHRW_RWLOCK_INIT; + rwlock->rw_seq[1]= PTHRW_RWS_INIT; + rwlock->rw_seq[2]= 0; +#endif /* __LP64__ */ + } + + rwlock->reserv = 0; + rwlock->rw_owner = NULL; +#if defined(__LP64__) + memset(rwlock->rfu, 0, PTHRW_RFU_64BIT); +#else + memset(rwlock->rfu, 0, PTHRW_RFU_32BIT); +#endif + + rwlock->sig = _PTHREAD_RWLOCK_SIG; + + return(0); } -int -pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) +#if _KSYN_TRACE_ +static void +set_enable(int val) { - int ret; - - /* initialize the lock */ - if ((ret = pthread_mutex_init(&rwlock->lock, NULL)) != 0) - return(ret); - else { - /* initialize the read condition signal */ - ret = pthread_cond_init(&rwlock->read_signal, NULL); - - if (ret != 0) { - pthread_mutex_destroy(&rwlock->lock); - return(ret); - } else { - /* initialize the write condition signal */ - ret = pthread_cond_init(&rwlock->write_signal, NULL); - - if (ret != 0) { - pthread_cond_destroy(&rwlock->read_signal); - pthread_mutex_destroy(&rwlock->lock); - return(ret); - } else { - /* success */ - rwlock->state = 0; - rwlock->blocked_writers = 0; - rwlock->sig = _PTHREAD_RWLOCK_SIG; - return(ESUCCESS); - } + int mib[6]; + size_t needed = 0; + + mib[0] = CTL_KERN; + mib[1] = KERN_KDEBUG; + mib[2] = KERN_KDENABLE; + mib[3] = val; + mib[4] = 0; + mib[5] = 0; + /* best effort to stop the trace */ + (void)sysctl(mib, 4, NULL, &needed, NULL, 0); +} +#endif + +static uint32_t +modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits) +{ + uint32_t lval = lgenval & PTHRW_BIT_MASK; + uint32_t uval = updateval & PTHRW_BIT_MASK; + uint32_t rval, nlval; + + nlval = (lval | uval) & ~(PTH_RWL_MBIT); + + /* reconcile bits on the lock with what kernel needs to set */ + if ((uval & PTH_RWL_LBIT) != 0) + nlval &= ~PTH_RWL_KBIT; + else if (((uval & PTH_RWL_KBIT) == 0) && ((lval & PTH_RWL_WBIT) == 0)) + nlval &= ~PTH_RWL_KBIT; + + if (savebits !=0 ) { + if (((savebits & PTH_RWS_WSVBIT) != 0) && ((nlval & PTH_RWL_WBIT) == 0) && + ((nlval & PTH_RWL_EBIT) == 0)) { + if ((nlval & PTH_RWL_LBIT) == 0) + nlval |= (PTH_RWL_WBIT | PTH_RWL_KBIT); + else + nlval |= PTH_RWL_WBIT; + } + if (((savebits & PTH_RWS_YSVBIT) != 0) && ((nlval & PTH_RWL_YBIT) == 0) && + ((nlval & PTH_RWL_EBIT) == 0)) { + nlval |= PTH_RWL_YBIT; + } + if (((savebits & PTH_RWS_USVBIT) != 0) && ((nlval & PTH_RWL_EBIT) == 0)) { + if ((nlval & PTH_RWL_LBIT) == 0) + nlval |= (PTH_RWL_UBIT | PTH_RWL_KBIT); + else + nlval |= PTH_RWL_UBIT; } } + rval = (lgenval & PTHRW_COUNT_MASK) | nlval; + return(rval); } -int -pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) + +__private_extern__ void +rwlock_action_onreturn(pthread_rwlock_t * orwlock, uint32_t updateval) { - int ret; - if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { - if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) { - return(ret); - } + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint32_t lcntval, rw_seq, newval = 0, newsval, lval, uval; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t oldval64, newval64; + int setbits = 0; + int overlap = 0; + uint32_t savebits = 0; + int isoverlap = 0; + + /* TBD: restore U bit */ + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + seqaddr = rwlock->rw_seqaddr; } - if (rwlock->sig != _PTHREAD_RWLOCK_SIG) - return(EINVAL); - /* grab the monitor lock */ - if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) - return(ret); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_START, updateval, 0, 0, 0, 0); +#endif - /* give writers priority over readers */ - while (rwlock->blocked_writers || rwlock->state < 0) { - ret = pthread_cond_wait(&rwlock->read_signal, &rwlock->lock); + isoverlap = updateval & PTH_RWL_MBIT; - if (ret != 0) { - /* can't do a whole lot if this fails */ - pthread_mutex_unlock(&rwlock->lock); - return(ret); +loop: + setbits = 0; + lcntval = *lcntaddr; + rw_seq = *seqaddr; + savebits = 0; + + if (isoverlap != 0) { + /* overlap return, just increment and inspect bits */ + setbits = 1; + overlap = 1; + /* set s word, increment by specified value */ + newsval = rw_seq + (updateval & PTHRW_COUNT_MASK); + if ((newsval & PTHRW_RWS_SAVEMASK) != 0) { + savebits = newsval & PTHRW_RWS_SAVEMASK; + newsval &= ~PTHRW_RWS_SAVEMASK; + } + } else { + /* normal return */ + if (is_rws_setunlockinit(rw_seq) != 0) { + setbits = 1; + /* set s word to passed in value */ + newsval = (rw_seq & PTHRW_COUNT_MASK) + (updateval & PTHRW_COUNT_MASK); + if ((rw_seq & PTHRW_RWS_SAVEMASK) != 0) { + savebits = rw_seq & PTHRW_RWS_SAVEMASK; + newsval &= ~PTHRW_RWS_SAVEMASK; + } + } else { + newval = lcntval; + newsval = rw_seq; } } + if (setbits != 0) { + newval = modbits(lcntval, updateval, savebits); - /* check lock count */ - if (rwlock->state == MAX_READ_LOCKS) - ret = EAGAIN; - else - ++rwlock->state; /* indicate we are locked for reading */ +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_NONE, rw_seq, newsval, 0xeeeeeeee, updateval, 0); + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_NONE, lcntval, newval, 0xeeeeeeee, updateval, 0); +#endif + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; + /* Check for consistency */ + lval = lcntval & PTHRW_BIT_MASK; + uval = updateval & PTHRW_BIT_MASK; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_END, rw_seq, newsval, 0xffffffff, 0, 0); +#endif + return; +} - /* - * Something is really wrong if this call fails. Returning - * error won't do because we've already obtained the read - * lock. Decrementing 'state' is no good because we probably - * don't have the monitor lock. - */ - pthread_mutex_unlock(&rwlock->lock); +/* returns are not bit shifted */ +__private_extern__ int +rw_diffgenseq(uint32_t x, uint32_t y) +{ + uint32_t lx = (x & PTHRW_COUNT_MASK); + uint32_t ly = (y &PTHRW_COUNT_MASK); + + if (lx > ly) { + return(lx-ly); + } else { + return((PTHRW_MAX_READERS - y) + lx + PTHRW_INC); + } - return(ret); } +#ifdef NOTYET +/********************************************************** */ +static int pthread_rwlock_upgrade_internal(pthread_rwlock_t * orwlock, int trylock); + int -pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) +pthread_rwlock_longrdlock_np(pthread_rwlock_t * orwlock) { - int ret; + pthread_t self; + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval; + int error = 0, retry_count = 0; + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + if (can_rwl_longreadinuser(lcntval)) + goto gotlock; - /* check for static initialization */ - if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { - if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) { - return(ret); +#if __DARWIN_UNIX03 + if (is_rwl_ebit_set(lcntval)) { + self = pthread_self(); + if(rwlock->rw_owner == self) { + error = EDEADLK; + goto out; } } +#endif /* __DARWIN_UNIX03 */ + + /* need to block in kernel */ + newval = (lcntval + PTHRW_INC); - if (rwlock->sig != _PTHREAD_RWLOCK_SIG) - return(EINVAL); - /* grab the monitor lock */ - if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) - return(ret); + newsval = rw_seq; + if (is_rws_setseq(rw_seq)) { + newsval &= PTHRW_SW_Reset_BIT_MASK; + newsval |= (newval & PTHRW_COUNT_MASK); + } - /* give writers priority over readers */ - if (rwlock->blocked_writers || rwlock->state < 0) - ret = EBUSY; - else if (rwlock->state == MAX_READ_LOCKS) - ret = EAGAIN; /* too many read locks acquired */ - else - ++rwlock->state; /* indicate we are locked for reading */ + /* update lock seq and block in kernel */ + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)(newsval)) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; +kblock: + updateval = __psynch_rw_longrdlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags); + if (updateval == (uint32_t)-1) { + error = errno; + } else + error = 0; + + if (error == EINTR) + goto kblock; + + if (error == 0) { + rwlock_action_onreturn(orwlock, updateval); + if ( is_rwl_lbit_clear(updateval)) { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("yieldwrlock from kernel without EBit %x: tid %x\n", updateval, (uint32_t)myid); + /* kernel cannot wakeup without granting E bit */ + } + goto successout; + } else { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("yieldwrlock from kernel with unknown error %x: tid %x\n", updateval, (uint32_t)myid); + goto out; + } + +gotlock: + if (rw_diffgenseq(lcntval, ucntval) >= PTHRW_MAX_READERS) { + /* since ucntval may be newer, just redo */ + retry_count++; + if (retry_count > 1024) { + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TOOMANY | DBG_FUNC_NONE, (uint32_t)rwlock, 0XEEEEEEEE, lcntval, ucntval, 0); +#endif + error = EAGAIN; + goto out; + } else { + sched_yield(); + goto loop; + } + } + + /* Need to update L and S word */ + newval = (lcntval + PTHRW_INC) | PTH_RWL_LBIT; + newsval = (rw_seq + PTHRW_INC); - /* see the comment on this in pthread_rwlock_rdlock */ - pthread_mutex_unlock(&rwlock->lock); + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; - return(ret); +successout: + PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT); + return(0); +out: + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); + return(error); } int -pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) +pthread_rwlock_yieldwrlock_np(pthread_rwlock_t * orwlock) { - int ret; + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval; + int error = 0; +#if __DARWIN_UNIX03 + pthread_t self = pthread_self(); +#endif /* __DARWIN_UNIX03 */ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } - /* check for static initialization */ - if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { - if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) { - return(ret); + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + +#if __DARWIN_UNIX03 + if (is_rwl_ebit_set(lcntval)) { + if (rwlock->rw_owner == self) { + error = EDEADLK; + goto out; } } +#endif /* __DARWIN_UNIX03 */ + + if (lcntval == PTHRW_RWL_INIT) { + /* if we can acquire set L and S word */ + lcntval = PTHRW_RWL_INIT; + newval = PTHRW_RWL_INIT | PTHRW_INC | PTH_RWL_KBIT| PTH_RWL_EBIT; + newsval = rw_seq + PTHRW_INC; + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) == TRUE) { + goto gotit; + } else + goto loop; + } + + newval = (lcntval + PTHRW_INC)| PTH_RWL_YBIT; - if (rwlock->sig != _PTHREAD_RWLOCK_SIG) + newsval = rw_seq; + if (is_rws_setseq(rw_seq)) { + newsval &= PTHRW_SW_Reset_BIT_MASK; + newsval |= (newval & PTHRW_COUNT_MASK); + } + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)(newsval)) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; + + PLOCKSTAT_RW_BLOCK(orwlock, WRITE_LOCK_PLOCKSTAT); +retry: + updateval = __psynch_rw_yieldwrlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags); + if (updateval == (uint32_t)-1) { + error = errno; + } else + error = 0; + + if (error == EINTR) + goto retry; + + + PLOCKSTAT_RW_BLOCKED(orwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT); + if (error != 0) { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("yieldwrlock from kernel with unknown error %x: tid %x\n", updateval, (uint32_t)myid); + } + + +out: + if (error == 0) { +gotit: + rwlock_action_onreturn(orwlock, updateval); + if ( is_rwl_ebit_clear(updateval)) { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("yieldwrlock from kernel without EBit %x: tid %x\n", updateval, (uint32_t)myid); + } +#if __DARWIN_UNIX03 + rwlock->rw_owner = self; +#endif /* __DARWIN_UNIX03 */ + PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT); + return(0); + } else { + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error); + return(error); + } +} +int +pthread_rwlock_downgrade_np(pthread_rwlock_t * orwlock) +{ + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval; + int error = 0, haswbit = 0, hasubit = 0, hasybit = 0; +#if __DARWIN_UNIX03 + pthread_t self = pthread_self(); +#endif /* __DARWIN_UNIX03 */ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + + /* if not holding exclusive lock, return */ + if ((is_rwl_ebit_set(lcntval )== 0) || (rwlock->rw_owner != self)) { return(EINVAL); - /* grab the monitor lock */ - if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) - return(ret); + } + + /* no other waiters and be granted in user space? ? */ + if ((lcntval & PTHRW_COUNT_MASK) == (ucntval + PTHRW_INC)) { +#if 0 + /* should have no write waiters pending */ + if (is_rwl_wbit_set(lcntval) != 0) { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("downgrade in user mode but W bit set %x: tid %x\n", lcntval, (uint32_t)myid); + } +#endif + /* preserve count and remove ke bits */ + newval = lcntval & ~(PTH_RWL_EBIT | PTH_RWL_KBIT); + /* if we can acquire set L and S word */ + newsval = rw_seq; + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) == TRUE) { +#if __DARWIN_UNIX03 + rwlock->rw_owner = (pthread_t)0; +#endif /* __DARWIN_UNIX03 */ + return(0); + } else + goto loop; + } else { + + haswbit = lcntval & PTH_RWL_WBIT; + hasubit = lcntval & PTH_RWL_UBIT; + hasybit = lcntval & PTH_RWL_YBIT; + + /* reset all bits and set k */ + newval = (lcntval & PTHRW_COUNT_MASK) | PTH_RWL_KBIT; + /* set I bit on S word */ + newsval = rw_seq | PTH_RWS_IBIT; + if (haswbit != 0) + newsval |= PTH_RWS_WSVBIT; + if (hasubit != 0) + newsval |= PTH_RWS_USVBIT; + if (hasybit != 0) + newsval |= PTH_RWS_YSVBIT; - if (rwlock->state != 0) - ret = EBUSY; - else - /* indicate we are locked for writing */ - rwlock->state = -1; + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; - /* see the comment on this in pthread_rwlock_rdlock */ - pthread_mutex_unlock(&rwlock->lock); +#if __DARWIN_UNIX03 + rwlock->rw_owner = 0; +#endif /* __DARWIN_UNIX03 */ - return(ret); +retry: + updateval = __psynch_rw_downgrade(orwlock, newval, ucntval, newsval, rwlock->rw_flags); + if (updateval == (uint32_t)-1) { + error = errno; + } else + error = 0; + + /* TBD: what to do with the error, EINTR ?? */ + if (error == EINTR) + goto retry; + + if (error == 0) { + rwlock_action_onreturn(orwlock, updateval); + return(0); + } else { +#if _KSYN_TRACE_ + set_enable(1); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("downgrade from kernel with unknown error %x with tid %x\n", updateval, (uint32_t)myid); + } + /* Not reached */ + } + return(EINVAL); } int -pthread_rwlock_unlock(pthread_rwlock_t *rwlock) +pthread_rwlock_upgrade_np(pthread_rwlock_t * orwlock) { - int ret; + return(pthread_rwlock_upgrade_internal(orwlock, 0)); +} - if (rwlock->sig != _PTHREAD_RWLOCK_SIG) - return(EINVAL); - /* grab the monitor lock */ - if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) - return(ret); - - if (rwlock->state > 0) { - if (--rwlock->state == 0 && rwlock->blocked_writers) - ret = pthread_cond_signal(&rwlock->write_signal); - } else if (rwlock->state < 0) { - rwlock->state = 0; - - if (rwlock->blocked_writers) - ret = pthread_cond_signal(&rwlock->write_signal); - else - ret = pthread_cond_broadcast(&rwlock->read_signal); +int +pthread_rwlock_tryupgrade_np(pthread_rwlock_t *orwlock) +{ + return(pthread_rwlock_upgrade_internal(orwlock, 1)); +} + +static int +pthread_rwlock_upgrade_internal(pthread_rwlock_t * orwlock, int trylock) +{ + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval; + int error = 0, flags ; +#if __DARWIN_UNIX03 + pthread_t self = pthread_self(); +#endif /* __DARWIN_UNIX03 */ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + /* check for static initialization */ + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + if (is_rwl_eubit_set(lcntval) !=0) { + return(EBUSY); + } + + /* set U and K bit and go to kernel */ + newval = (lcntval | (PTH_RWL_UBIT | PTH_RWL_KBIT)); + newsval = rw_seq; +#if 0 + if (is_rws_setseq(rw_seq)) { + newsval &= PTHRW_SW_Reset_BIT_MASK; + newsval |= (newval & PTHRW_COUNT_MASK); + } +#endif + + /* update lock seq and block in kernel */ + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)(newsval)) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; + flags = rwlock->rw_flags; + if (trylock != 0) { + flags |= _PTHREAD_RWLOCK_UPGRADE_TRY; + } +retry: + updateval = __psynch_rw_upgrade(orwlock, newval, ucntval, newsval, rwlock->rw_flags); + if (updateval == (uint32_t)-1) { + error = errno; } else - ret = EINVAL; + error = 0; + + if (error == EINTR) + goto retry; + + + if (error == 0) { + rwlock_action_onreturn(orwlock, updateval); + if ( is_rwl_ebit_clear(updateval)) { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("upgrade from kernel without EBit %x: tid %x\n", updateval, (uint32_t)myid); + } +#if __DARWIN_UNIX03 + rwlock->rw_owner = self; +#endif /* __DARWIN_UNIX03 */ + return(0); + } else { + if (trylock != 0) { + return (EBUSY); + } + } - /* see the comment on this in pthread_rwlock_rdlock */ - pthread_mutex_unlock(&rwlock->lock); + return(error); +} - return(ret); +/* Returns true if the rwlock is held for reading by any thread or held for writing by the current thread */ +int +pthread_rwlock_held_np(pthread_rwlock_t * orwlock) +{ + uint32_t lcntval, ucntval, rw_seq; + int error = 0; + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + return(0); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + return(-1); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + if ((lcntval & PTHRW_COUNT_MASK) == (ucntval & PTHRW_COUNT_MASK)) + return(0); + + return(1); } -int -pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) +/* Returns true if the rwlock is held for reading by any thread */ +int +pthread_rwlock_rdheld_np(pthread_rwlock_t * orwlock) { - int ret; + uint32_t lcntval, ucntval, rw_seq; + int error = 0; + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + return(0); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + return(-1); + } + UNLOCK(rwlock->lock); + } + + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } - /* check for static initialization */ - if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { - if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) { - return(ret); + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + if ((lcntval & PTHRW_COUNT_MASK) == (ucntval & PTHRW_COUNT_MASK)) + return(0); + + if (is_rwl_ebit_set(lcntval) !=0) { + return(0); + } + return(1); +} + +/* Returns true if the rwlock is held for writing by the current thread */ +int +pthread_rwlock_wrheld_np(pthread_rwlock_t * orwlock) +{ + uint32_t lcntval, ucntval, rw_seq; + pthread_t self = pthread_self(); + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + int error = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + return(0); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + return(-1); } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + if ((is_rwl_ebit_set(lcntval)) && (rwlock->rw_owner == self)) { + return(1); } + return(0); +} +/******************************************************/ +#endif /* NOTYET */ - if (rwlock->sig != _PTHREAD_RWLOCK_SIG) + +#endif /* !BUILDING_VARIANT ] */ + +int +pthread_rwlock_destroy(pthread_rwlock_t *orwlock) +{ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; +#if __DARWIN_UNIX03 + uint32_t rw_lcnt, rw_ucnt; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; +#endif /* __DARWIN_UNIX03 */ + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG && rwlock->sig != _PTHREAD_RWLOCK_SIG_init) return(EINVAL); - /* grab the monitor lock */ - if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) - return(ret); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG) { +#if __DARWIN_UNIX03 + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } - while (rwlock->state != 0) { - ++rwlock->blocked_writers; + rw_lcnt = *lcntaddr; + rw_ucnt = *ucntaddr; + + if((rw_lcnt & PTHRW_COUNT_MASK) != rw_ucnt) + return(EBUSY); + +#endif /* __DARWIN_UNIX03 */ + //bzero(rwlock, sizeof(npthread_rwlock_t)); + rwlock->sig = _PTHREAD_NO_SIG; + return(0); + } else if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + rwlock->sig = _PTHREAD_NO_SIG; + return(0); + } else + return(EINVAL); +} - ret = pthread_cond_wait(&rwlock->write_signal, &rwlock->lock); - if (ret != 0) { - --rwlock->blocked_writers; - pthread_mutex_unlock(&rwlock->lock); - return(ret); +int +pthread_rwlock_init(pthread_rwlock_t * orwlock, const pthread_rwlockattr_t *attr) +{ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; +#if __DARWIN_UNIX03 + uint32_t rw_lcnt, rw_ucnt; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 + if (attr && (attr->sig != _PTHREAD_RWLOCK_ATTR_SIG)) { + return(EINVAL); + } + + if (rwlock->sig == _PTHREAD_RWLOCK_SIG) { + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; } + + rw_lcnt = *lcntaddr; + rw_ucnt = *ucntaddr; + + if ((rw_lcnt & PTHRW_COUNT_MASK) != rw_ucnt) + return(EBUSY); + + } +#endif + LOCK_INIT(rwlock->lock); + return(__pthread_rwlock_init(orwlock, attr)); + +} - --rwlock->blocked_writers; +int +pthread_rwlock_rdlock(pthread_rwlock_t * orwlock) +{ +#if __DARWIN_UNIX03 + pthread_t self; +#endif /* __DARWIN_UNIX03 */ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval; + int error = 0, retry_count = 0; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0); +#endif +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lcntval, (ucntval | 0xee), rw_seq, 0); +#endif + + /* if l bit is on or u and k bit is clear, acquire lock in userland */ + if (can_rwl_readinuser(lcntval)) + goto gotlock; + +#if __DARWIN_UNIX03 + if (is_rwl_ebit_set(lcntval)) { + self = pthread_self(); + if(rwlock->rw_owner == self) { + error = EDEADLK; + goto out; + } + } +#endif /* __DARWIN_UNIX03 */ + + + /* Need to block in kernel , remove Rbit */ + newval = (lcntval + PTHRW_INC) & PTH_RWLOCK_RESET_RBIT; + + newsval = rw_seq; + if (is_rws_setseq(rw_seq)) { + newsval &= PTHRW_SW_Reset_BIT_MASK; + newsval |= (newval & PTHRW_COUNT_MASK); } - /* indicate we are locked for writing */ - rwlock->state = -1; + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; - /* see the comment on this in pthread_rwlock_rdlock */ - pthread_mutex_unlock(&rwlock->lock); + /* give writers priority over readers */ + PLOCKSTAT_RW_BLOCK(orwlock, READ_LOCK_PLOCKSTAT); + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lcntval, newval, newsval, 0); +#endif - return(ret); +retry: + updateval = __psynch_rw_rdlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags); + + if (updateval == (uint32_t)-1) { + error = errno; + } else + error = 0; + + if (error == EINTR) + goto retry; + + if (error == 0) { + rwlock_action_onreturn(orwlock, updateval); + PLOCKSTAT_RW_BLOCKED(orwlock, READ_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT); + PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT); + return(0); + } else { + PLOCKSTAT_RW_BLOCKED(orwlock, READ_LOCK_PLOCKSTAT, BLOCK_FAIL_PLOCKSTAT); +#if _KSYN_TRACE_ + set_enable(1); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("rdlock from kernel with unknown error %x with tid %x\n", updateval, (uint32_t)myid); + goto out; + } + /* Not reached */ + +gotlock: + if (rw_diffgenseq(lcntval, ucntval) >= PTHRW_MAX_READERS) { + /* since ucntval may be newer, just redo */ + retry_count++; + if (retry_count > 1024) { + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TOOMANY | DBG_FUNC_NONE, (uint32_t)rwlock, 0XEEEEEEEE, lcntval, ucntval, 0); +#endif + error = EAGAIN; + goto out; + } else { + sched_yield(); + goto loop; + } + } + + /* Need to update L (remove R bit) and S word */ + newval = (lcntval + PTHRW_INC) & PTH_RWLOCK_RESET_RBIT; + newsval = (rw_seq + PTHRW_INC); + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lcntval, newval, 0); +#endif + + PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, 0, 0, 0); +#endif + return(0); +out: + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0); +#endif + return(error); } int -pthread_rwlockattr_init(pthread_rwlockattr_t *attr) +pthread_rwlock_tryrdlock(pthread_rwlock_t * orwlock) { - attr->sig = _PTHREAD_RWLOCK_ATTR_SIG; - attr->pshared = PTHREAD_PROCESS_PRIVATE; - return (ESUCCESS); + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint32_t lcntval, ucntval, rw_seq, newval, newsval; + int error = 0, retry_count = 0; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TRYRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lcntval, ucntval, rw_seq, 0); +#endif + + /* if l bit is on or u and k bit is clear, acquire lock in userland */ + if (can_rwl_readinuser(lcntval)) + goto gotlock; + + error = EBUSY; + goto out; + +gotlock: + if (rw_diffgenseq(lcntval, ucntval) >= PTHRW_MAX_READERS) { + /* since ucntval may be newer, just redo */ + retry_count++; + if (retry_count > 1024) { + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TOOMANY | DBG_FUNC_NONE, (uint32_t)rwlock, 0XEEEEEEEE, lcntval, ucntval, 0); +#endif + error = EAGAIN; + goto out; + } else { + sched_yield(); + goto loop; + } + } + + /* Need to update L(remove Rbit ) and S word */ + newval = (lcntval + PTHRW_INC) & PTH_RWLOCK_RESET_RBIT; + newsval = (rw_seq + PTHRW_INC); + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TRYRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lcntval, newval, 0); +#endif + + PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT); + return(0); + +out: + PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TRYRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, error, 0, 0); +#endif + return(error); } -int -pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr) +int +pthread_rwlock_trywrlock(pthread_rwlock_t * orwlock) { - attr->sig = _PTHREAD_NO_SIG; /* Uninitialized */ - attr->pshared = 0; - return (ESUCCESS); +#if __DARWIN_UNIX03 + pthread_t self = pthread_self(); +#endif /* __DARWIN_UNIX03 */ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint32_t lcntval, rw_seq, newval, newsval; + int error = 0, gotlock = 0; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TRYWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0); +#endif +loop: + lcntval = *lcntaddr; + rw_seq = *seqaddr; + + /* can we acquire in userland? */ + if ((lcntval & PTH_RWL_RBIT) != 0) { + newval = ((lcntval + PTHRW_INC) & PTHRW_COUNT_MASK) | PTH_RWL_IBIT | PTH_RWL_KBIT| PTH_RWL_EBIT; + newsval = rw_seq + PTHRW_INC; + gotlock = 1; + } else + gotlock = 0; + + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + if (gotlock != 0) { + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + } else + newval64 = oldval64; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) { + goto loop; + } + if (gotlock == 1) { +#if __DARWIN_UNIX03 + rwlock->rw_owner = self; +#endif /* __DARWIN_UNIX03 */ + PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TRYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, 0, 0, 0); +#endif + return(0); + } else { +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_TRYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, EBUSY, 0, 0); +#endif + + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EBUSY); + return(EBUSY); + } } int -pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, - int *pshared) +pthread_rwlock_wrlock(pthread_rwlock_t * orwlock) { - if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) - { - /* *pshared = (int)attr->pshared; */ - *pshared = (int)PTHREAD_PROCESS_PRIVATE; - return (ESUCCESS); - } else - { - return (EINVAL); /* Not an initialized 'attribute' structure */ - } +#if __DARWIN_UNIX03 + pthread_t self = pthread_self(); +#endif /* __DARWIN_UNIX03 */ + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval; + int error = 0, gotlock = 0; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0); +#endif +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lcntval, ucntval, rw_seq, 0); +#endif + +#if __DARWIN_UNIX03 + if (is_rwl_ebit_set(lcntval)) { + if(rwlock->rw_owner == self) { + error = EDEADLK; + goto out; + } + } +#endif /* __DARWIN_UNIX03 */ + + + if ((lcntval & PTH_RWL_RBIT) != 0) { + /* lock is restart state, writer can acquire the lock */ + newval = ((lcntval + PTHRW_INC) & PTHRW_COUNT_MASK) | PTH_RWL_IBIT | PTH_RWL_KBIT| PTH_RWL_EBIT; + + newsval = rw_seq + PTHRW_INC; + gotlock = 1; + + } else { + if (is_rwl_lbit_set(lcntval)) + newval = (lcntval + PTHRW_INC)| PTH_RWL_WBIT; + else + newval = (lcntval + PTHRW_INC) | PTH_RWL_KBIT| PTH_RWL_WBIT; + + newsval = rw_seq; + if (is_rws_setseq(rw_seq)) { + newsval &= PTHRW_SW_Reset_BIT_MASK; + newsval |= (newval & PTHRW_COUNT_MASK); + } + gotlock = 0; + } + + /* update lock seq */ + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lcntval, newval, 0); +#endif + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto loop; + + /* lock acquired in userland itself? */ + if (gotlock != 0) + goto gotit; + + /* unable to acquire in userland, transition to kernel */ + + PLOCKSTAT_RW_BLOCK(orwlock, WRITE_LOCK_PLOCKSTAT); +retry: + updateval = __psynch_rw_wrlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags); + if (updateval == (uint32_t)-1) { + error = errno; + } else + error = 0; + + if (error == EINTR) { + goto retry; + } + + if (error != 0) { +#if _KSYN_TRACE_ + set_enable(2); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("wrlock from kernel with unknown error %x: tid %x\n", updateval, (uint32_t)myid); + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x33333333, newval, updateval, 0); +#endif + PLOCKSTAT_RW_BLOCKED(orwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT); + if (error == 0) { + rwlock_action_onreturn(orwlock, updateval); +gotit: +#if __DARWIN_UNIX03 + rwlock->rw_owner = self; +#endif /* __DARWIN_UNIX03 */ + PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0); +#endif + return(0); + } +#if __DARWIN_UNIX03 +out: +#endif /* __DARWIN_UNIX03 */ + PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0); +#endif + return(error); } int -pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared) +pthread_rwlock_unlock(pthread_rwlock_t * orwlock) { - if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) - { - if ( pshared == PTHREAD_PROCESS_PRIVATE) - { - attr->pshared = pshared ; - return (ESUCCESS); - } else - { - return (EINVAL); /* Invalid parameter */ - } - } else - { - return (EINVAL); /* Not an initialized 'attribute' structure */ - } + npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock; + uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval, ulval; + int error = 0, wrlock = 0, haswbit = 0, hasubit = 0, hasybit = 0; + uint64_t oldval64, newval64; + volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr; + uint64_t myid = 0; + + if (rwlock->sig != _PTHREAD_RWLOCK_SIG) { + LOCK(rwlock->lock); + if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) { + if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) { + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, wrlock, error); + return(error); + } + } else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){ + UNLOCK(rwlock->lock); + PLOCKSTAT_RW_ERROR(orwlock, wrlock, EINVAL); + return(EINVAL); + } + UNLOCK(rwlock->lock); + } + + if (rwlock->pshared == PTHREAD_PROCESS_SHARED) { + RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr); + } else { + lcntaddr = rwlock->rw_lcntaddr; + ucntaddr = rwlock->rw_ucntaddr; + seqaddr = rwlock->rw_seqaddr; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0); +#endif +loop: + lcntval = *lcntaddr; + ucntval = *ucntaddr; + rw_seq = *seqaddr; + + + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x51515151, lcntval, ucntval, 0); + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x51515151, rw_seq, 0, 0); +#endif + /* check for spurious unlocks */ + if ((lcntval & PTH_RWL_RBIT) != 0) { + newval = lcntval ; + newsval = rw_seq; + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) == TRUE) { + /* spurious unlock, return */ + error = EINVAL; +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x1a1b1c1d, lcntval, ucntval, 0); +#endif + goto succout; + } else + goto loop; + } + + if (is_rwl_ebit_set(lcntval)) { + wrlock = 1; +#if __DARWIN_UNIX03 + rwlock->rw_owner = (pthread_t)0; +#endif /* __DARWIN_UNIX03 */ + } + + /* update U */ + + ulval = (ucntval + PTHRW_INC); + + if (OSAtomicCompareAndSwap32(ucntval, ulval, (volatile int32_t *)ucntaddr) != TRUE) + goto loop; + +lp11: + /* just validate the l and S values */ + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = oldval64; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) { + lcntval = *lcntaddr; + rw_seq = *seqaddr; + goto lp11; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xd1d2d3d4, lcntval, rw_seq, 0); + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xd1d2d3d4, ulval, 0, 0); +#endif + + /* last unlock, note U is already updated ? */ + if((lcntval & PTHRW_COUNT_MASK) == (ulval & PTHRW_COUNT_MASK)) { + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xbbbbbbbb, lcntval, ucntval, 0); +#endif + /* Set L with R and init bits and set S to L */ + newval = (lcntval & PTHRW_COUNT_MASK)| PTHRW_RWLOCK_INIT; + newsval = (lcntval & PTHRW_COUNT_MASK)| PTHRW_RWS_INIT; + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) { +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xcccccccc, 0, 0, 0); +#endif + lcntval = *lcntaddr; + rw_seq = *seqaddr; + goto lp11; + } +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xdddddddd, lcntval, ucntval, 0); +#endif + goto succout; + } + + /* if it is not exclusive or no Writer/yield pending, skip */ + if ((lcntval & (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_YBIT | PTH_RWL_KBIT)) == 0) { + goto succout; + } + + /* kernel transition needed? */ + /* U+1 == S? */ + if ((ulval + PTHRW_INC) != (rw_seq & PTHRW_COUNT_MASK)) { + if ((lcntval & PTH_RWL_UBIT) != 0) { + /* if U bit is set U + 2 == S ? */ + if ((ulval + PTHRW_INC + PTHRW_INC) != (rw_seq & PTHRW_COUNT_MASK)) + goto succout; + } else + goto succout; + } + + haswbit = lcntval & PTH_RWL_WBIT; + hasubit = lcntval & PTH_RWL_UBIT; + hasybit = lcntval & PTH_RWL_YBIT; + + /* reset all bits and set k */ + newval = (lcntval & PTHRW_COUNT_MASK) | PTH_RWL_KBIT; + /* set I bit on S word */ + newsval = rw_seq | PTH_RWS_IBIT; + if (haswbit != 0) + newsval |= PTH_RWS_WSVBIT; + if (hasubit != 0) + newsval |= PTH_RWS_USVBIT; + if (hasybit != 0) + newsval |= PTH_RWS_YSVBIT; + + oldval64 = (((uint64_t)rw_seq) << 32); + oldval64 |= lcntval; + + newval64 = (((uint64_t)newsval) << 32); + newval64 |= newval; + + if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) + goto lp11; + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555511, 1, ulval, 0); +#endif + updateval = __psynch_rw_unlock(orwlock, lcntval, ulval, newsval, rwlock->rw_flags); + if (updateval == (uint32_t)-1) { + error = errno; + } else + error = 0; + + if(error != 0) { + + /* not sure what is the scenario */ + if(error != EINTR) { +#if _KSYN_TRACE_ + set_enable(4); +#endif /* _KSYN_TRACE_ */ + (void)pthread_threadid_np(pthread_self(), &myid); + LIBC_ABORT("rwunlock from kernel with unknown error %x: tid %x\n", error, (uint32_t)myid); + goto succout; + } + error = 0; + } + +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555522, 3, lcntval, 0); +#endif +succout: + PLOCKSTAT_RW_RELEASE(orwlock, wrlock); +#if _KSYN_TRACE_ + if (__pthread_lock_debug != 0) + (void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0); +#endif + return(0); }