src/pthread_rwlock.c

   1 /*
   2  * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*-
  24  * Copyright (c) 1998 Alex Nash
  25  * All rights reserved.
  26  *
  27  * Redistribution and use in source and binary forms, with or without
  28  * modification, are permitted provided that the following conditions
  29  * are met:
  30  * 1. Redistributions of source code must retain the above copyright
  31  *    notice, this list of conditions and the following disclaimer.
  32  * 2. Redistributions in binary form must reproduce the above copyright
  33  *    notice, this list of conditions and the following disclaimer in the
  34  *    documentation and/or other materials provided with the distribution.
  35  *
  36  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  37  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  38  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  40  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  42  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  43  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  44  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  45  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  46  * SUCH DAMAGE.
  47  *
  48  * $FreeBSD: src/lib/libc_r/uthread/uthread_rwlock.c,v 1.6 2001/04/10 04:19:20 deischen Exp $
  49  */
  50
  51 /*
  52  * POSIX Pthread Library
  53  * -- Read Write Lock support
  54  * 4/24/02: A. Ramesh
  55  *         Ported from FreeBSD
  56  */
  57
  58 #include "resolver.h"
  59 #include "internal.h"
  60 #if DEBUG
  61 #include <platform/compat.h> // for bzero
  62 #endif
  63
  64 #ifdef PLOCKSTAT
  65 #include "plockstat.h"
  66 #else /* !PLOCKSTAT */
  67 #define PLOCKSTAT_RW_ERROR(x, y, z)
  68 #define PLOCKSTAT_RW_BLOCK(x, y)
  69 #define PLOCKSTAT_RW_BLOCKED(x, y, z)
  70 #define PLOCKSTAT_RW_ACQUIRE(x, y)
  71 #define PLOCKSTAT_RW_RELEASE(x, y)
  72 #endif /* PLOCKSTAT */
  73
  74 #define READ_LOCK_PLOCKSTAT  0
  75 #define WRITE_LOCK_PLOCKSTAT 1
  76
  77 #define BLOCK_FAIL_PLOCKSTAT    0
  78 #define BLOCK_SUCCESS_PLOCKSTAT 1
  79
  80 #define PTHREAD_RWLOCK_INIT_UNUSED 1
  81
  82 // maximum number of times a read lock may be obtained
  83 #define MAX_READ_LOCKS          (INT_MAX - 1)
  84
  85 union rwlock_seq; // forward declaration
  86 enum rwlock_seqfields; // forward declaration
  87
  88 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
  89 int _pthread_rwlock_lock_slow(pthread_rwlock_t *orwlock, bool readlock,
  90                 bool trylock);
  91
  92 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
  93 int _pthread_rwlock_unlock_slow(pthread_rwlock_t *orwlock,
  94                 enum rwlock_seqfields updated_seqfields);
  95
  96
  97 #if defined(__LP64__)
  98 #define RWLOCK_USE_INT128 1
  99 #endif
 100
 101 typedef union rwlock_seq {
 102         uint32_t seq[4];
 103         struct { uint32_t lcntval; uint32_t rw_seq; uint32_t ucntval; };
 104         struct { uint32_t lgen; uint32_t rw_wc; uint32_t ugen; };
 105 #if RWLOCK_USE_INT128
 106         unsigned __int128 seq_LSU;
 107         unsigned __int128 _Atomic atomic_seq_LSU;
 108 #endif
 109         struct {
 110                 uint64_t seq_LS;
 111                 uint32_t seq_U;
 112                 uint32_t _pad;
 113         };
 114         struct {
 115                 uint64_t _Atomic atomic_seq_LS;
 116                 uint32_t _Atomic atomic_seq_U;
 117                 uint32_t _Atomic _atomic_pad;
 118         };
 119 } rwlock_seq;
 120
 121 _Static_assert(sizeof(rwlock_seq) == 4 * sizeof(uint32_t),
 122                 "Incorrect rwlock_seq size");
 123
 124 typedef enum rwlock_seqfields {
 125         RWLOCK_SEQ_NONE = 0,
 126         RWLOCK_SEQ_LS = 1,
 127         RWLOCK_SEQ_U = 2,
 128         RWLOCK_SEQ_LSU = RWLOCK_SEQ_LS | RWLOCK_SEQ_U,
 129 } rwlock_seqfields;
 130
 131 #if PTHREAD_DEBUG_LOG
 132 #define RWLOCK_DEBUG_SEQ(op, rwlock, oldseq, newseq, updateval, f) \
 133                 if (_pthread_debuglog >= 0) { \
 134                 _simple_dprintf(_pthread_debuglog, "rw_" #op " %p tck %7llu thr %llx " \
 135                 "L %x -> %x S %x -> %x U %x -> %x updt %x\n", rwlock, \
 136                 mach_absolute_time() - _pthread_debugstart, _pthread_selfid_direct(), \
 137                 (f) & RWLOCK_SEQ_LS ? (oldseq).lcntval : 0, \
 138                 (f) & RWLOCK_SEQ_LS ? (newseq).lcntval : 0, \
 139                 (f) & RWLOCK_SEQ_LS ? (oldseq).rw_seq  : 0, \
 140                 (f) & RWLOCK_SEQ_LS ? (newseq).rw_seq  : 0, \
 141                 (f) & RWLOCK_SEQ_U  ? (oldseq).ucntval : 0, \
 142                 (f) & RWLOCK_SEQ_U  ? (newseq).ucntval : 0, updateval); }
 143 #else
 144 #define RWLOCK_DEBUG_SEQ(m, rwlock, oldseq, newseq, updateval, f)
 145 #endif
 146
 147 #if !__LITTLE_ENDIAN__
 148 #error RWLOCK_GETSEQ_ADDR assumes little endian layout of sequence words
 149 #endif
 150
 151 PTHREAD_ALWAYS_INLINE
 152 static inline void
 153 RWLOCK_GETSEQ_ADDR(_pthread_rwlock *rwlock, rwlock_seq **seqaddr)
 154 {
 155         // 128-bit aligned address inside rw_seq & rw_mis arrays
 156         *seqaddr = (void*)(((uintptr_t)rwlock->rw_seq + 0xful) & ~0xful);
 157 }
 158
 159 PTHREAD_ALWAYS_INLINE
 160 static inline void
 161 RWLOCK_GETTID_ADDR(_pthread_rwlock *rwlock, uint64_t **tidaddr)
 162 {
 163         // 64-bit aligned address inside rw_tid array (&rw_tid[0] for aligned lock)
 164         *tidaddr = (void*)(((uintptr_t)rwlock->rw_tid + 0x7ul) & ~0x7ul);
 165 }
 166
 167 PTHREAD_ALWAYS_INLINE
 168 static inline void
 169 rwlock_seq_load(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 170                 const rwlock_seqfields seqfields)
 171 {
 172         switch (seqfields) {
 173         case RWLOCK_SEQ_LSU:
 174 #if RWLOCK_USE_INT128
 175                 oldseqval->seq_LSU = seqaddr->seq_LSU;
 176 #else
 177                 oldseqval->seq_LS = seqaddr->seq_LS;
 178                 oldseqval->seq_U = seqaddr->seq_U;
 179 #endif
 180                 break;
 181         case RWLOCK_SEQ_LS:
 182                 oldseqval->seq_LS = seqaddr->seq_LS;
 183                 break;
 184 #if DEBUG // unused
 185         case RWLOCK_SEQ_U:
 186                 oldseqval->seq_U = seqaddr->seq_U;
 187                 break;
 188 #endif // unused
 189         default:
 190                 __builtin_trap();
 191         }
 192 }
 193
 194 PTHREAD_ALWAYS_INLINE
 195 static inline void
 196 rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 197                 const rwlock_seqfields seqfields)
 198 {
 199         switch (seqfields) {
 200         case RWLOCK_SEQ_LSU:
 201 #if RWLOCK_USE_INT128
 202                 oldseqval->seq_LSU = os_atomic_load(&seqaddr->atomic_seq_LSU, relaxed);
 203 #else
 204                 oldseqval->seq_LS = os_atomic_load(&seqaddr->atomic_seq_LS, relaxed);
 205                 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 206 #endif
 207                 break;
 208         case RWLOCK_SEQ_LS:
 209                 oldseqval->seq_LS = os_atomic_load(&seqaddr->atomic_seq_LS, relaxed);
 210                 break;
 211 #if DEBUG // unused
 212         case RWLOCK_SEQ_U:
 213                 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 214                 break;
 215 #endif // unused
 216         default:
 217                 __builtin_trap();
 218         }
 219 }
 220
 221 #define rwlock_seq_atomic_load(seqaddr, oldseqval, seqfields, m) \
 222                 rwlock_seq_atomic_load_##m(seqaddr, oldseqval, seqfields)
 223
 224 PTHREAD_ALWAYS_INLINE
 225 static inline rwlock_seqfields
 226 rwlock_seq_atomic_cmpxchgv_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 227                 rwlock_seq *newseqval, const rwlock_seqfields seqfields)
 228 {
 229         bool r;
 230         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 231         switch (seqfields) {
 232 #if DEBUG // unused
 233         case RWLOCK_SEQ_LSU:
 234 #if RWLOCK_USE_INT128
 235                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU,
 236                                 newseqval->seq_LSU, &oldseqval->seq_LSU, relaxed);
 237                 if (r) updated_seqfields = RWLOCK_SEQ_LSU;
 238 #else
 239                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 240                                 newseqval->seq_LS, &oldseqval->seq_LS, relaxed);
 241                 if (r) {
 242                         r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 243                                         newseqval->seq_U, &oldseqval->seq_U, relaxed);
 244                         if (!r) oldseqval->seq_LS = newseqval->seq_LS;
 245                         updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_LS;
 246                 } else {
 247                         oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 248                 }
 249 #endif
 250                 break;
 251         case RWLOCK_SEQ_U:
 252                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 253                                 newseqval->seq_U, &oldseqval->seq_U, relaxed);
 254                 if (r) updated_seqfields = RWLOCK_SEQ_U;
 255                 break;
 256 #endif // unused
 257         case RWLOCK_SEQ_LS:
 258                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 259                                 newseqval->seq_LS, &oldseqval->seq_LS, relaxed);
 260                 if (r) updated_seqfields = RWLOCK_SEQ_LS;
 261                 break;
 262         default:
 263                 __builtin_trap();
 264         }
 265         return updated_seqfields;
 266 }
 267
 268 PTHREAD_ALWAYS_INLINE
 269 static inline rwlock_seqfields
 270 rwlock_seq_atomic_cmpxchgv_acquire(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 271                 rwlock_seq *newseqval, const rwlock_seqfields seqfields)
 272 {
 273         bool r;
 274         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 275         switch (seqfields) {
 276 #if DEBUG // unused
 277         case RWLOCK_SEQ_LSU:
 278 #if RWLOCK_USE_INT128
 279                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU,
 280                                 newseqval->seq_LSU, &oldseqval->seq_LSU, acquire);
 281                 if (r) updated_seqfields = RWLOCK_SEQ_LSU;
 282 #else
 283                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 284                                 newseqval->seq_LS, &oldseqval->seq_LS, acquire);
 285                 if (r) {
 286                         r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 287                                         newseqval->seq_U, &oldseqval->seq_U, relaxed);
 288                         if (!r) oldseqval->seq_LS = newseqval->seq_LS;
 289                         updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_LS;
 290                 } else {
 291                         oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 292                 }
 293 #endif
 294                 break;
 295         case RWLOCK_SEQ_U:
 296                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 297                                 newseqval->seq_U, &oldseqval->seq_U, acquire);
 298                 if (r) updated_seqfields = RWLOCK_SEQ_U;
 299                 break;
 300 #endif // unused
 301         case RWLOCK_SEQ_LS:
 302                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 303                                 newseqval->seq_LS, &oldseqval->seq_LS, acquire);
 304                 if (r) updated_seqfields = RWLOCK_SEQ_LS;
 305                 break;
 306         default:
 307                 __builtin_trap();
 308         }
 309         return updated_seqfields;
 310 }
 311
 312 PTHREAD_ALWAYS_INLINE
 313 static inline rwlock_seqfields
 314 rwlock_seq_atomic_cmpxchgv_release(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 315                 rwlock_seq *newseqval, const rwlock_seqfields seqfields)
 316 {
 317         bool r;
 318         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 319         switch (seqfields) {
 320         case RWLOCK_SEQ_LSU:
 321 #if RWLOCK_USE_INT128
 322                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU,
 323                                 newseqval->seq_LSU, &oldseqval->seq_LSU, release);
 324                 if (r) updated_seqfields = RWLOCK_SEQ_LSU;
 325 #else
 326                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 327                                 newseqval->seq_U, &oldseqval->seq_U, release);
 328                 if (r) {
 329                         r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 330                                         newseqval->seq_LS, &oldseqval->seq_LS, relaxed);
 331                         if (!r) oldseqval->seq_U = newseqval->seq_U;
 332                         updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_U;
 333                 } else {
 334                         oldseqval->seq_LS = os_atomic_load(&seqaddr->atomic_seq_LS,relaxed);
 335                 }
 336 #endif
 337                 break;
 338         case RWLOCK_SEQ_LS:
 339                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 340                                 newseqval->seq_LS, &oldseqval->seq_LS, release);
 341                 if (r) updated_seqfields = RWLOCK_SEQ_LS;
 342                 break;
 343 #if DEBUG // unused
 344         case RWLOCK_SEQ_U:
 345                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 346                                 newseqval->seq_U, &oldseqval->seq_U, release);
 347                 if (r) updated_seqfields = RWLOCK_SEQ_U;
 348                 break;
 349 #endif // unused
 350         default:
 351                 __builtin_trap();
 352         }
 353         return updated_seqfields;
 354 }
 355
 356 #define rwlock_seq_atomic_cmpxchgv(seqaddr, oldseqval, newseqval, seqfields, m)\
 357                 rwlock_seq_atomic_cmpxchgv_##m(seqaddr, oldseqval, newseqval, seqfields)
 358
 359 #ifndef BUILDING_VARIANT /* [ */
 360
 361 int
 362 pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
 363 {
 364         attr->sig = _PTHREAD_RWLOCK_ATTR_SIG;
 365         attr->pshared = _PTHREAD_DEFAULT_PSHARED;
 366         return 0;
 367 }
 368
 369 int
 370 pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
 371 {
 372         attr->sig = _PTHREAD_NO_SIG;
 373         attr->pshared = 0;
 374         return 0;
 375 }
 376
 377 int
 378 pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, int *pshared)
 379 {
 380         int res = EINVAL;
 381         if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) {
 382                 *pshared = (int)attr->pshared;
 383                 res = 0;
 384         }
 385         return res;
 386 }
 387
 388 int
 389 pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared)
 390 {
 391         int res = EINVAL;
 392         if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) {
 393 #if __DARWIN_UNIX03
 394                 if (( pshared == PTHREAD_PROCESS_PRIVATE) ||
 395                                 (pshared == PTHREAD_PROCESS_SHARED))
 396 #else /* __DARWIN_UNIX03 */
 397                 if ( pshared == PTHREAD_PROCESS_PRIVATE)
 398 #endif /* __DARWIN_UNIX03 */
 399                 {
 400                         attr->pshared = pshared ;
 401                         res = 0;
 402                 }
 403         }
 404         return res;
 405 }
 406
 407 #endif /* !BUILDING_VARIANT ] */
 408
 409 PTHREAD_ALWAYS_INLINE
 410 static inline int
 411 _pthread_rwlock_init(_pthread_rwlock *rwlock, const pthread_rwlockattr_t *attr)
 412 {
 413         uint64_t *tidaddr;
 414         RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 415
 416         rwlock_seq *seqaddr;
 417         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 418
 419 #if PTHREAD_RWLOCK_INIT_UNUSED
 420         if ((uint32_t*)tidaddr != rwlock->rw_tid) {
 421                 rwlock->misalign = 1;
 422                 __builtin_memset(rwlock->rw_tid, 0xff, sizeof(rwlock->rw_tid));
 423         }
 424         if ((uint32_t*)seqaddr != rwlock->rw_seq) {
 425                 __builtin_memset(rwlock->rw_seq, 0xff, sizeof(rwlock->rw_seq));
 426         }
 427         __builtin_memset(rwlock->rw_mis, 0xff, sizeof(rwlock->rw_mis));
 428 #endif // PTHREAD_MUTEX_INIT_UNUSED
 429         *tidaddr = 0;
 430         *seqaddr = (rwlock_seq){
 431                 .lcntval = PTHRW_RWLOCK_INIT,
 432                 .rw_seq = PTHRW_RWS_INIT,
 433                 .ucntval = 0,
 434         };
 435
 436         if (attr != NULL && attr->pshared == PTHREAD_PROCESS_SHARED) {
 437                 rwlock->pshared = PTHREAD_PROCESS_SHARED;
 438                 rwlock->rw_flags = PTHRW_KERN_PROCESS_SHARED;
 439         } else {
 440                 rwlock->pshared = _PTHREAD_DEFAULT_PSHARED;
 441                 rwlock->rw_flags = PTHRW_KERN_PROCESS_PRIVATE;
 442         }
 443
 444         long sig = _PTHREAD_RWLOCK_SIG;
 445
 446 #if DEBUG
 447         bzero(rwlock->_reserved, sizeof(rwlock->_reserved));
 448 #endif
 449 #if PTHREAD_RWLOCK_INIT_UNUSED
 450         // For detecting copied rwlocks and smashes during debugging
 451         uint32_t sig32 = (uint32_t)sig;
 452         uintptr_t guard = ~(uintptr_t)rwlock; // use ~ to hide from leaks
 453         __builtin_memcpy(rwlock->_reserved, &guard, sizeof(guard));
 454 #define countof(x) (sizeof(x) / sizeof(x[0]))
 455         rwlock->_reserved[countof(rwlock->_reserved) - 1] = sig32;
 456 #if defined(__LP64__)
 457         rwlock->_pad = sig32;
 458 #endif
 459 #endif // PTHREAD_RWLOCK_INIT_UNUSED
 460
 461         // Ensure all contents are properly set before setting signature.
 462 #if defined(__LP64__)
 463         // For binary compatibility reasons we cannot require natural alignment of
 464         // the 64bit 'sig' long value in the struct. rdar://problem/21610439
 465         uint32_t *sig32_ptr = (uint32_t*)&rwlock->sig;
 466         uint32_t *sig32_val = (uint32_t*)&sig;
 467         *(sig32_ptr + 1) = *(sig32_val + 1);
 468         os_atomic_store(sig32_ptr, *sig32_val, release);
 469 #else
 470         os_atomic_store2o(rwlock, sig, sig, release);
 471 #endif
 472
 473         return 0;
 474 }
 475
 476 static uint32_t
 477 _pthread_rwlock_modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits)
 478 {
 479         uint32_t lval = lgenval & PTHRW_BIT_MASK;
 480         uint32_t uval = updateval & PTHRW_BIT_MASK;
 481         uint32_t rval, nlval;
 482
 483         nlval = (lval | uval) & ~(PTH_RWL_MBIT);
 484
 485         // reconcile bits on the lock with what kernel needs to set
 486         if ((uval & PTH_RWL_KBIT) == 0 && (lval & PTH_RWL_WBIT) == 0) {
 487                 nlval &= ~PTH_RWL_KBIT;
 488         }
 489
 490         if (savebits != 0) {
 491                 if ((savebits & PTH_RWS_WSVBIT) != 0 && (nlval & PTH_RWL_WBIT) == 0 &&
 492                                 (nlval & PTH_RWL_EBIT) == 0) {
 493                         nlval |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
 494                 }
 495         }
 496         rval = (lgenval & PTHRW_COUNT_MASK) | nlval;
 497         return(rval);
 498 }
 499
 500 PTHREAD_ALWAYS_INLINE
 501 static inline void
 502 _pthread_rwlock_updateval(_pthread_rwlock *rwlock, uint32_t updateval)
 503 {
 504         bool isoverlap = (updateval & PTH_RWL_MBIT) != 0;
 505
 506         // TBD: restore U bit
 507         rwlock_seq *seqaddr;
 508         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 509
 510         rwlock_seq oldseq, newseq;
 511         rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LS);
 512         do {
 513                 newseq = oldseq;
 514                 if (isoverlap || is_rws_unlockinit_set(oldseq.rw_seq)) {
 515                         // Set S word to the specified value
 516                         uint32_t savebits = (oldseq.rw_seq & PTHRW_RWS_SAVEMASK);
 517                         newseq.lcntval = _pthread_rwlock_modbits(oldseq.lcntval, updateval,
 518                                         savebits);
 519                         newseq.rw_seq += (updateval & PTHRW_COUNT_MASK);
 520                         if (!isoverlap) {
 521                                 newseq.rw_seq &= PTHRW_COUNT_MASK;
 522                         }
 523                         newseq.rw_seq &= ~PTHRW_RWS_SAVEMASK;
 524                 }
 525         } while (!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
 526                         RWLOCK_SEQ_LS, relaxed));
 527         RWLOCK_DEBUG_SEQ(update, rwlock, oldseq, newseq, updateval, RWLOCK_SEQ_LS);
 528 }
 529
 530 #if __DARWIN_UNIX03
 531 PTHREAD_ALWAYS_INLINE
 532 static inline int
 533 _pthread_rwlock_check_busy(_pthread_rwlock *rwlock)
 534 {
 535         int res = 0;
 536
 537         rwlock_seq *seqaddr;
 538         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 539
 540         rwlock_seq seq;
 541         rwlock_seq_atomic_load(seqaddr, &seq, RWLOCK_SEQ_LSU, relaxed);
 542         if ((seq.lcntval & PTHRW_COUNT_MASK) != seq.ucntval) {
 543                 res = EBUSY;
 544         }
 545
 546         return res;
 547 }
 548 #endif /* __DARWIN_UNIX03 */
 549
 550 PTHREAD_NOEXPORT_VARIANT
 551 int
 552 pthread_rwlock_destroy(pthread_rwlock_t *orwlock)
 553 {
 554         int res = 0;
 555         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 556
 557         _PTHREAD_LOCK(rwlock->lock);
 558         if (_pthread_rwlock_check_signature(rwlock)) {
 559 #if __DARWIN_UNIX03
 560                 res = _pthread_rwlock_check_busy(rwlock);
 561 #endif /* __DARWIN_UNIX03 */
 562         } else if (!_pthread_rwlock_check_signature_init(rwlock)) {
 563                 res = EINVAL;
 564         }
 565         if (res == 0) {
 566                 rwlock->sig = _PTHREAD_NO_SIG;
 567         }
 568         _PTHREAD_UNLOCK(rwlock->lock);
 569         return res;
 570 }
 571
 572 PTHREAD_NOEXPORT_VARIANT
 573 int
 574 pthread_rwlock_init(pthread_rwlock_t *orwlock, const pthread_rwlockattr_t *attr)
 575 {
 576         int res = 0;
 577         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 578
 579 #if __DARWIN_UNIX03
 580         if (attr && attr->sig != _PTHREAD_RWLOCK_ATTR_SIG) {
 581                 res = EINVAL;
 582         }
 583
 584         if (res == 0 && _pthread_rwlock_check_signature(rwlock)) {
 585                 res = _pthread_rwlock_check_busy(rwlock);
 586         }
 587 #endif
 588         if (res == 0) {
 589                 _PTHREAD_LOCK_INIT(rwlock->lock);
 590                 res = _pthread_rwlock_init(rwlock, attr);
 591         }
 592         return res;
 593 }
 594
 595 PTHREAD_NOINLINE
 596 static int
 597 _pthread_rwlock_check_init_slow(pthread_rwlock_t *orwlock)
 598 {
 599         int res = EINVAL;
 600         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 601
 602         if (_pthread_rwlock_check_signature_init(rwlock)) {
 603                 _PTHREAD_LOCK(rwlock->lock);
 604                 if (_pthread_rwlock_check_signature_init(rwlock)) {
 605                         res = _pthread_rwlock_init(rwlock, NULL);
 606                 } else if (_pthread_rwlock_check_signature(rwlock)){
 607                         res = 0;
 608                 }
 609                 _PTHREAD_UNLOCK(rwlock->lock);
 610         } else if (_pthread_rwlock_check_signature(rwlock)){
 611                 res = 0;
 612         }
 613         if (res != 0) {
 614                 PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, res);
 615         }
 616         return res;
 617 }
 618
 619 PTHREAD_ALWAYS_INLINE
 620 static inline int
 621 _pthread_rwlock_check_init(pthread_rwlock_t *orwlock)
 622 {
 623         int res = 0;
 624         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 625
 626         if (!_pthread_rwlock_check_signature(rwlock)) {
 627                 return _pthread_rwlock_check_init_slow(orwlock);
 628         }
 629         return res;
 630 }
 631
 632 PTHREAD_NOINLINE
 633 static int
 634 _pthread_rwlock_lock_wait(pthread_rwlock_t *orwlock, bool readlock,
 635                 rwlock_seq newseq)
 636 {
 637         int res;
 638         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 639
 640 #ifdef PLOCKSTAT
 641         int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT;
 642 #endif
 643
 644         if (readlock) {
 645                 RWLOCK_DEBUG_SEQ(rdlock, rwlock, oldseq, newseq, gotlock,
 646                                 RWLOCK_SEQ_LSU);
 647         } else {
 648                 RWLOCK_DEBUG_SEQ(wrlock, rwlock, oldseq, newseq, gotlock,
 649                                 RWLOCK_SEQ_LSU);
 650         }
 651
 652         uint32_t updateval;
 653
 654         PLOCKSTAT_RW_BLOCK(orwlock, plockstat);
 655
 656         do {
 657                 if (readlock) {
 658                         updateval = __psynch_rw_rdlock(orwlock, newseq.lcntval,
 659                                         newseq.ucntval, newseq.rw_seq, rwlock->rw_flags);
 660                 } else {
 661                         updateval = __psynch_rw_wrlock(orwlock, newseq.lcntval,
 662                                         newseq.ucntval, newseq.rw_seq, rwlock->rw_flags);
 663                 }
 664                 if (updateval == (uint32_t)-1) {
 665                         res = errno;
 666                 } else {
 667                         res = 0;
 668                 }
 669         } while (res == EINTR);
 670
 671         if (res == 0) {
 672                 _pthread_rwlock_updateval(rwlock, updateval);
 673                 PLOCKSTAT_RW_BLOCKED(orwlock, plockstat, BLOCK_SUCCESS_PLOCKSTAT);
 674         } else {
 675                 PLOCKSTAT_RW_BLOCKED(orwlock, plockstat, BLOCK_FAIL_PLOCKSTAT);
 676                 PTHREAD_ABORT("kernel rwlock returned unknown error %x: "
 677                                 "tid %llx\n", res, _pthread_selfid_direct());
 678         }
 679
 680         return res;
 681 }
 682
 683 PTHREAD_NOEXPORT PTHREAD_NOINLINE
 684 int
 685 _pthread_rwlock_lock_slow(pthread_rwlock_t *orwlock, bool readlock,
 686                 bool trylock)
 687 {
 688         int res;
 689         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 690
 691 #ifdef PLOCKSTAT
 692         int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT;
 693 #endif
 694
 695         res = _pthread_rwlock_check_init(orwlock);
 696         if (res != 0) return res;
 697
 698         rwlock_seq *seqaddr;
 699         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 700
 701         rwlock_seq oldseq, newseq;
 702         rwlock_seq_atomic_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU, relaxed);
 703
 704 #if __DARWIN_UNIX03
 705         uint64_t *tidaddr;
 706         RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 707         uint64_t selfid = _pthread_selfid_direct();
 708         if (is_rwl_ebit_set(oldseq.lcntval)) {
 709                 if (os_atomic_load(tidaddr, relaxed) == selfid) return EDEADLK;
 710         }
 711 #endif /* __DARWIN_UNIX03 */
 712
 713         int retry_count;
 714         bool gotlock;
 715         do {
 716                 retry_count = 0;
 717 retry:
 718                 newseq = oldseq;
 719
 720                 // if W and K bit are clear or U bit is on, acquire lock in userland
 721                 if (readlock) {
 722                         gotlock = (oldseq.lcntval & (PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0;
 723                 } else {
 724                         gotlock = (oldseq.lcntval & PTH_RWL_UBIT) != 0;
 725                 }
 726
 727                 if (trylock && !gotlock) {
 728                         // A trylock on a held lock will fail immediately. But since
 729                         // we did not load the sequence words atomically, perform a
 730                         // no-op CAS to ensure that nobody has unlocked concurrently.
 731                 } else if (gotlock) {
 732                         if (readlock) {
 733                                 if (diff_genseq(oldseq.lcntval, oldseq.ucntval) >=
 734                                                 PTHRW_MAX_READERS) {
 735                                         // since ucntval may be newer, just redo
 736                                         retry_count++;
 737                                         if (retry_count > 1024) {
 738                                                 gotlock = false;
 739                                                 res = EAGAIN;
 740                                                 goto out;
 741                                         } else {
 742                                                 sched_yield();
 743                                                 rwlock_seq_atomic_load(seqaddr, &oldseq,
 744                                                                 RWLOCK_SEQ_LSU, relaxed);
 745                                                 goto retry;
 746                                         }
 747                                 }
 748                                 // Need to update L (remove U bit) and S word
 749                                 newseq.lcntval &= ~PTH_RWL_UBIT;
 750                         } else {
 751                                 newseq.lcntval &= PTHRW_COUNT_MASK;
 752                                 newseq.lcntval |= PTH_RWL_IBIT | PTH_RWL_KBIT | PTH_RWL_EBIT;
 753                         }
 754                         newseq.lcntval += PTHRW_INC;
 755                         newseq.rw_seq  += PTHRW_INC;
 756                 } else {
 757                         if (readlock) {
 758                                 // Need to block in kernel. Remove U bit.
 759                                 newseq.lcntval &= ~PTH_RWL_UBIT;
 760                         } else {
 761                                 newseq.lcntval |= PTH_RWL_KBIT | PTH_RWL_WBIT;
 762                         }
 763                         newseq.lcntval += PTHRW_INC;
 764                         if (is_rws_sbit_set(oldseq.rw_seq)) {
 765                                 // Clear the S bit and set S to L
 766                                 newseq.rw_seq &= (PTHRW_BIT_MASK & ~PTH_RWS_SBIT);
 767                                 newseq.rw_seq |= (oldseq.lcntval & PTHRW_COUNT_MASK);
 768                         }
 769                 }
 770         } while (!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
 771                         RWLOCK_SEQ_LS, acquire));
 772
 773         if (gotlock) {
 774 #if __DARWIN_UNIX03
 775                 if (!readlock) os_atomic_store(tidaddr, selfid, relaxed);
 776 #endif /* __DARWIN_UNIX03 */
 777                 res = 0;
 778         } else if (trylock) {
 779                 res = EBUSY;
 780         } else {
 781                 res = _pthread_rwlock_lock_wait(orwlock, readlock, newseq);
 782         }
 783
 784 out:
 785 #ifdef PLOCKSTAT
 786         if (res == 0) {
 787                 PLOCKSTAT_RW_ACQUIRE(orwlock, plockstat);
 788         } else {
 789                 PLOCKSTAT_RW_ERROR(orwlock, plockstat, res);
 790         }
 791 #endif
 792
 793         return res;
 794 }
 795
 796 PTHREAD_ALWAYS_INLINE
 797 static inline int
 798 _pthread_rwlock_lock(pthread_rwlock_t *orwlock, bool readlock, bool trylock)
 799 {
 800         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 801 #if PLOCKSTAT
 802         if (PLOCKSTAT_RW_ACQUIRE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) {
 803                 return _pthread_rwlock_lock_slow(orwlock, readlock, trylock);
 804         }
 805 #endif
 806
 807         if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) {
 808                 return _pthread_rwlock_lock_slow(orwlock, readlock, trylock);
 809         }
 810
 811         rwlock_seq *seqaddr;
 812         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 813
 814         rwlock_seq oldseq, newseq;
 815         // no need to perform a single-copy-atomic 128-bit load in the fastpath,
 816         // if stores to L and U are seen out of order, we will fallback to the
 817         // slowpath below (which has rwlock_seq_atomic_load)
 818         rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU);
 819
 820 #if __DARWIN_UNIX03
 821         if (os_unlikely(is_rwl_ebit_set(oldseq.lcntval))) {
 822                 return _pthread_rwlock_lock_slow(orwlock, readlock, trylock);
 823         }
 824 #endif /* __DARWIN_UNIX03 */
 825
 826         bool gotlock;
 827         do {
 828                 newseq = oldseq;
 829
 830                 // if W and K bit are clear or U bit is on, acquire lock in userland
 831                 if (readlock) {
 832                         gotlock = (oldseq.lcntval & (PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0;
 833                 } else {
 834                         gotlock = (oldseq.lcntval & PTH_RWL_UBIT) != 0;
 835                 }
 836
 837                 if (trylock && !gotlock) {
 838                         // A trylock on a held lock will fail immediately. But since
 839                         // we did not load the sequence words atomically, perform a
 840                         // no-op CAS to ensure that nobody has unlocked concurrently.
 841                 } else if (os_likely(gotlock)) {
 842                         if (readlock) {
 843                                 if (os_unlikely(diff_genseq(oldseq.lcntval, oldseq.ucntval) >=
 844                                                 PTHRW_MAX_READERS)) {
 845                                         return _pthread_rwlock_lock_slow(orwlock, readlock,trylock);
 846                                 }
 847                                 // Need to update L (remove U bit) and S word
 848                                 newseq.lcntval &= ~PTH_RWL_UBIT;
 849                         } else {
 850                                 newseq.lcntval &= PTHRW_COUNT_MASK;
 851                                 newseq.lcntval |= PTH_RWL_IBIT | PTH_RWL_KBIT | PTH_RWL_EBIT;
 852                         }
 853                         newseq.lcntval += PTHRW_INC;
 854                         newseq.rw_seq  += PTHRW_INC;
 855                 } else {
 856                         return _pthread_rwlock_lock_slow(orwlock, readlock, trylock);
 857                 }
 858         } while (os_unlikely(!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
 859                         RWLOCK_SEQ_LS, acquire)));
 860
 861         if (os_likely(gotlock)) {
 862 #if __DARWIN_UNIX03
 863                 if (!readlock) {
 864                         uint64_t *tidaddr;
 865                         RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 866                         uint64_t selfid = _pthread_selfid_direct();
 867                         os_atomic_store(tidaddr, selfid, relaxed);
 868                 }
 869 #endif /* __DARWIN_UNIX03 */
 870                 return 0;
 871         } else if (trylock) {
 872                 return EBUSY;
 873         } else {
 874                 __builtin_trap();
 875         }
 876 }
 877
 878 PTHREAD_NOEXPORT_VARIANT
 879 int
 880 pthread_rwlock_rdlock(pthread_rwlock_t *orwlock)
 881 {
 882         // read lock, no try
 883         return _pthread_rwlock_lock(orwlock, true, false);
 884 }
 885
 886 PTHREAD_NOEXPORT_VARIANT
 887 int
 888 pthread_rwlock_tryrdlock(pthread_rwlock_t *orwlock)
 889 {
 890         // read lock, try lock
 891         return _pthread_rwlock_lock(orwlock, true, true);
 892 }
 893
 894 PTHREAD_NOEXPORT_VARIANT
 895 int
 896 pthread_rwlock_wrlock(pthread_rwlock_t *orwlock)
 897 {
 898         // write lock, no try
 899         return _pthread_rwlock_lock(orwlock, false, false);
 900 }
 901
 902 PTHREAD_NOEXPORT_VARIANT
 903 int
 904 pthread_rwlock_trywrlock(pthread_rwlock_t *orwlock)
 905 {
 906         // write lock, try lock
 907         return _pthread_rwlock_lock(orwlock, false, true);
 908 }
 909
 910 PTHREAD_NOINLINE
 911 static int
 912 _pthread_rwlock_unlock_drop(pthread_rwlock_t *orwlock, rwlock_seq oldseq,
 913                 rwlock_seq newseq)
 914 {
 915         int res;
 916         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 917
 918         RWLOCK_DEBUG_SEQ(unlock, rwlock, oldseq, newseq, !droplock, RWLOCK_SEQ_LSU);
 919         uint32_t updateval;
 920         do {
 921                 updateval = __psynch_rw_unlock(orwlock, oldseq.lcntval,
 922                                 newseq.ucntval, newseq.rw_seq, rwlock->rw_flags);
 923                 if (updateval == (uint32_t)-1) {
 924                         res = errno;
 925                 } else {
 926                         res = 0;
 927                         RWLOCK_DEBUG_SEQ(wakeup, rwlock, oldseq, newseq, updateval,
 928                                         RWLOCK_SEQ_LSU);
 929                 }
 930         } while (res == EINTR);
 931
 932         if (res != 0) {
 933                 PTHREAD_ABORT("kernel rwunlock returned unknown error %x: "
 934                                 "tid %llx\n", res, _pthread_selfid_direct());
 935         }
 936
 937         return res;
 938 }
 939
 940 PTHREAD_NOEXPORT PTHREAD_NOINLINE
 941 int
 942 _pthread_rwlock_unlock_slow(pthread_rwlock_t *orwlock,
 943                 rwlock_seqfields updated_seqfields)
 944 {
 945         int res;
 946         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
 947         rwlock_seqfields seqfields = RWLOCK_SEQ_LSU;
 948 #ifdef PLOCKSTAT
 949         int wrlock = 0;
 950 #endif
 951
 952         res = _pthread_rwlock_check_init(orwlock);
 953         if (res != 0) return res;
 954
 955         rwlock_seq *seqaddr;
 956         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 957
 958         rwlock_seq oldseq, newseq;
 959         rwlock_seq_load(seqaddr, &oldseq, seqfields);
 960
 961         if ((oldseq.lcntval & PTH_RWL_UBIT) != 0) {
 962                 // spurious unlock (unlock of unlocked lock)
 963                 return 0;
 964         }
 965
 966         if (is_rwl_ebit_set(oldseq.lcntval)) {
 967 #ifdef PLOCKSTAT
 968                 wrlock = 1;
 969 #endif
 970 #if __DARWIN_UNIX03
 971                 uint64_t *tidaddr;
 972                 RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 973                 os_atomic_store(tidaddr, 0, relaxed);
 974 #endif /* __DARWIN_UNIX03 */
 975         }
 976
 977         bool droplock;
 978         do {
 979                 // stop loading & updating fields that have successfully been stored
 980                 seqfields &= ~updated_seqfields;
 981
 982                 newseq = oldseq;
 983                 if (seqfields & RWLOCK_SEQ_U) {
 984                         newseq.ucntval += PTHRW_INC;
 985                 }
 986
 987                 droplock = false;
 988                 uint32_t oldlcnt = (oldseq.lcntval & PTHRW_COUNT_MASK);
 989                 if (newseq.ucntval == oldlcnt) {
 990                         // last unlock, set L with U and init bits and set S to L with S bit
 991                         newseq.lcntval = oldlcnt | PTHRW_RWLOCK_INIT;
 992                         newseq.rw_seq =  oldlcnt | PTHRW_RWS_INIT;
 993                 } else {
 994                         // no L/S update if lock is not exclusive or no writer pending
 995                         if ((oldseq.lcntval &
 996                                         (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0) {
 997                                 continue;
 998                         }
 999
1000                         // kernel transition only needed if U == S
1001                         if (newseq.ucntval != (oldseq.rw_seq & PTHRW_COUNT_MASK)) {
1002                                 continue;
1003                         }
1004
1005                         droplock = true;
1006                         // reset all bits and set K
1007                         newseq.lcntval = oldlcnt | PTH_RWL_KBIT;
1008                         // set I bit on S word
1009                         newseq.rw_seq |= PTH_RWS_IBIT;
1010                         if ((oldseq.lcntval & PTH_RWL_WBIT) != 0) {
1011                                 newseq.rw_seq |= PTH_RWS_WSVBIT;
1012                         }
1013                 }
1014         } while (seqfields != (updated_seqfields = rwlock_seq_atomic_cmpxchgv(
1015                         seqaddr, &oldseq, &newseq, seqfields, release)));
1016
1017         if (droplock) {
1018                 res = _pthread_rwlock_unlock_drop(orwlock, oldseq, newseq);
1019         }
1020
1021         PLOCKSTAT_RW_RELEASE(orwlock, wrlock);
1022
1023         return res;
1024 }
1025
1026 PTHREAD_NOEXPORT_VARIANT
1027 int
1028 pthread_rwlock_unlock(pthread_rwlock_t *orwlock)
1029 {
1030         _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock;
1031         rwlock_seqfields seqfields = RWLOCK_SEQ_LSU;
1032         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
1033
1034 #if PLOCKSTAT
1035         if (PLOCKSTAT_RW_RELEASE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) {
1036                 return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields);
1037         }
1038 #endif
1039
1040         if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) {
1041                 return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields);
1042         }
1043
1044         rwlock_seq *seqaddr;
1045         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
1046
1047         rwlock_seq oldseq, newseq;
1048         rwlock_seq_load(seqaddr, &oldseq, seqfields);
1049
1050         if (os_unlikely(oldseq.lcntval & PTH_RWL_UBIT)) {
1051                 // spurious unlock (unlock of unlocked lock)
1052                 return 0;
1053         }
1054
1055         if (is_rwl_ebit_set(oldseq.lcntval)) {
1056 #if __DARWIN_UNIX03
1057                 uint64_t *tidaddr;
1058                 RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
1059                 os_atomic_store(tidaddr, 0, relaxed);
1060 #endif /* __DARWIN_UNIX03 */
1061         }
1062
1063         do {
1064                 if (updated_seqfields) {
1065                         return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields);
1066                 }
1067
1068                 newseq = oldseq;
1069                 if (seqfields & RWLOCK_SEQ_U) {
1070                         newseq.ucntval += PTHRW_INC;
1071                 }
1072
1073                 uint32_t oldlcnt = (oldseq.lcntval & PTHRW_COUNT_MASK);
1074                 if (os_likely(newseq.ucntval == oldlcnt)) {
1075                         // last unlock, set L with U and init bits and set S to L with S bit
1076                         newseq.lcntval = oldlcnt | PTHRW_RWLOCK_INIT;
1077                         newseq.rw_seq =  oldlcnt | PTHRW_RWS_INIT;
1078                 } else {
1079                         if (os_likely((oldseq.lcntval &
1080                                         (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0 ||
1081                                         newseq.ucntval != (oldseq.rw_seq & PTHRW_COUNT_MASK))) {
1082                                 // no L/S update if lock is not exclusive or no writer pending
1083                                 // kernel transition only needed if U == S
1084                         } else {
1085                                 return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields);
1086                         }
1087                 }
1088         } while (os_unlikely(seqfields != (updated_seqfields =
1089                         rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, seqfields,
1090                         release))));
1091
1092         return 0;
1093 }
1094