src/pthread_rwlock.c

   1 /*
   2  * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*-
  24  * Copyright (c) 1998 Alex Nash
  25  * All rights reserved.
  26  *
  27  * Redistribution and use in source and binary forms, with or without
  28  * modification, are permitted provided that the following conditions
  29  * are met:
  30  * 1. Redistributions of source code must retain the above copyright
  31  *    notice, this list of conditions and the following disclaimer.
  32  * 2. Redistributions in binary form must reproduce the above copyright
  33  *    notice, this list of conditions and the following disclaimer in the
  34  *    documentation and/or other materials provided with the distribution.
  35  *
  36  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  37  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  38  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  40  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  42  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  43  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  44  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  45  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  46  * SUCH DAMAGE.
  47  *
  48  * $FreeBSD: src/lib/libc_r/uthread/uthread_rwlock.c,v 1.6 2001/04/10 04:19:20 deischen Exp $
  49  */
  50
  51 /*
  52  * POSIX Pthread Library
  53  * -- Read Write Lock support
  54  * 4/24/02: A. Ramesh
  55  *         Ported from FreeBSD
  56  */
  57
  58 #include "resolver.h"
  59 #include "internal.h"
  60
  61 #ifdef PLOCKSTAT
  62 #include "plockstat.h"
  63 #else /* !PLOCKSTAT */
  64 #define PLOCKSTAT_RW_ERROR(x, y, z)
  65 #define PLOCKSTAT_RW_BLOCK(x, y)
  66 #define PLOCKSTAT_RW_BLOCKED(x, y, z)
  67 #define PLOCKSTAT_RW_ACQUIRE(x, y)
  68 #define PLOCKSTAT_RW_RELEASE(x, y)
  69 #endif /* PLOCKSTAT */
  70
  71 #define READ_LOCK_PLOCKSTAT  0
  72 #define WRITE_LOCK_PLOCKSTAT 1
  73
  74 #define BLOCK_FAIL_PLOCKSTAT    0
  75 #define BLOCK_SUCCESS_PLOCKSTAT 1
  76
  77 #define PTHREAD_RWLOCK_INIT_UNUSED 1
  78
  79 // maximum number of times a read lock may be obtained
  80 #define MAX_READ_LOCKS          (INT_MAX - 1)
  81
  82
  83 #if defined(__LP64__)
  84 #define RWLOCK_USE_INT128 1
  85 #endif
  86
  87 typedef union rwlock_seq {
  88         uint32_t seq[4];
  89         struct { uint32_t lcntval; uint32_t rw_seq; uint32_t ucntval; };
  90         struct { uint32_t lgen; uint32_t rw_wc; uint32_t ugen; };
  91 #if RWLOCK_USE_INT128
  92         unsigned __int128 seq_LSU;
  93         unsigned __int128 _Atomic atomic_seq_LSU;
  94 #endif
  95         struct {
  96                 uint64_t seq_LS;
  97                 uint32_t seq_U;
  98                 uint32_t _pad;
  99         };
 100         struct {
 101                 uint64_t _Atomic atomic_seq_LS;
 102                 uint32_t _Atomic atomic_seq_U;
 103                 uint32_t _Atomic _atomic_pad;
 104         };
 105 } rwlock_seq;
 106
 107 _Static_assert(sizeof(rwlock_seq) == 4 * sizeof(uint32_t),
 108                 "Incorrect rwlock_seq size");
 109
 110 typedef enum rwlock_seqfields {
 111         RWLOCK_SEQ_NONE = 0,
 112         RWLOCK_SEQ_LS = 1,
 113         RWLOCK_SEQ_U = 2,
 114         RWLOCK_SEQ_LSU = RWLOCK_SEQ_LS | RWLOCK_SEQ_U,
 115 } rwlock_seqfields;
 116
 117 #if PTHREAD_DEBUG_LOG
 118 #define RWLOCK_DEBUG_SEQ(op, rwlock, oldseq, newseq, updateval, f) \
 119                 if (_pthread_debuglog >= 0) { \
 120                 _simple_dprintf(_pthread_debuglog, "rw_" #op " %p tck %7llu thr %llx " \
 121                 "L %x -> %x S %x -> %x U %x -> %x updt %x\n", rwlock, \
 122                 mach_absolute_time() - _pthread_debugstart, _pthread_threadid_self_np_direct(), \
 123                 (f) & RWLOCK_SEQ_LS ? (oldseq).lcntval : 0, \
 124                 (f) & RWLOCK_SEQ_LS ? (newseq).lcntval : 0, \
 125                 (f) & RWLOCK_SEQ_LS ? (oldseq).rw_seq  : 0, \
 126                 (f) & RWLOCK_SEQ_LS ? (newseq).rw_seq  : 0, \
 127                 (f) & RWLOCK_SEQ_U  ? (oldseq).ucntval : 0, \
 128                 (f) & RWLOCK_SEQ_U  ? (newseq).ucntval : 0, updateval); }
 129 #else
 130 #define RWLOCK_DEBUG_SEQ(m, rwlock, oldseq, newseq, updateval, f)
 131 #endif
 132
 133 #if !__LITTLE_ENDIAN__
 134 #error RWLOCK_GETSEQ_ADDR assumes little endian layout of sequence words
 135 #endif
 136
 137 OS_ALWAYS_INLINE
 138 static inline void
 139 RWLOCK_GETSEQ_ADDR(pthread_rwlock_t *rwlock, rwlock_seq **seqaddr)
 140 {
 141         // 128-bit aligned address inside rw_seq & rw_mis arrays
 142         *seqaddr = (void*)(((uintptr_t)rwlock->rw_seq + 0xful) & ~0xful);
 143 }
 144
 145 OS_ALWAYS_INLINE
 146 static inline void
 147 RWLOCK_GETTID_ADDR(pthread_rwlock_t *rwlock, uint64_t **tidaddr)
 148 {
 149         // 64-bit aligned address inside rw_tid array (&rw_tid[0] for aligned lock)
 150         *tidaddr = (void*)(((uintptr_t)rwlock->rw_tid + 0x7ul) & ~0x7ul);
 151 }
 152
 153 OS_ALWAYS_INLINE
 154 static inline void
 155 rwlock_seq_load(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 156                 const rwlock_seqfields seqfields)
 157 {
 158         switch (seqfields) {
 159         case RWLOCK_SEQ_LSU:
 160 #if RWLOCK_USE_INT128
 161                 oldseqval->seq_LSU = seqaddr->seq_LSU;
 162 #else
 163                 oldseqval->seq_LS = seqaddr->seq_LS;
 164                 oldseqval->seq_U = seqaddr->seq_U;
 165 #endif
 166                 break;
 167         case RWLOCK_SEQ_LS:
 168                 oldseqval->seq_LS = seqaddr->seq_LS;
 169                 break;
 170 #if DEBUG // unused
 171         case RWLOCK_SEQ_U:
 172                 oldseqval->seq_U = seqaddr->seq_U;
 173                 break;
 174 #endif // unused
 175         default:
 176                 __builtin_trap();
 177         }
 178 }
 179
 180 OS_ALWAYS_INLINE
 181 static inline void
 182 rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 183                 const rwlock_seqfields seqfields)
 184 {
 185         switch (seqfields) {
 186         case RWLOCK_SEQ_LSU:
 187 #if RWLOCK_USE_INT128
 188 #if defined(__arm64__) && defined(__ARM_ARCH_8_2__)
 189                 // Workaround clang armv81 codegen bug for 128bit os_atomic_load
 190                 // rdar://problem/31213932
 191                 oldseqval->seq_LSU = seqaddr->seq_LSU;
 192                 while (!os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU,
 193                                 oldseqval->seq_LSU, oldseqval->seq_LSU, &oldseqval->seq_LSU,
 194                                 relaxed));
 195 #else
 196                 oldseqval->seq_LSU = os_atomic_load_wide(&seqaddr->atomic_seq_LSU, relaxed);
 197 #endif
 198 #else
 199                 oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed);
 200                 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 201 #endif
 202                 break;
 203         case RWLOCK_SEQ_LS:
 204                 oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed);
 205                 break;
 206 #if DEBUG // unused
 207         case RWLOCK_SEQ_U:
 208                 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 209                 break;
 210 #endif // unused
 211         default:
 212                 __builtin_trap();
 213         }
 214 }
 215
 216 #define rwlock_seq_atomic_load(seqaddr, oldseqval, seqfields, m) \
 217                 rwlock_seq_atomic_load_##m(seqaddr, oldseqval, seqfields)
 218
 219 OS_ALWAYS_INLINE
 220 static inline rwlock_seqfields
 221 rwlock_seq_atomic_cmpxchgv_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 222                 rwlock_seq *newseqval, const rwlock_seqfields seqfields)
 223 {
 224         bool r;
 225         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 226         switch (seqfields) {
 227 #if DEBUG // unused
 228         case RWLOCK_SEQ_LSU:
 229 #if RWLOCK_USE_INT128
 230                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU,
 231                                 newseqval->seq_LSU, &oldseqval->seq_LSU, relaxed);
 232                 if (r) updated_seqfields = RWLOCK_SEQ_LSU;
 233 #else
 234                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 235                                 newseqval->seq_LS, &oldseqval->seq_LS, relaxed);
 236                 if (r) {
 237                         r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 238                                         newseqval->seq_U, &oldseqval->seq_U, relaxed);
 239                         if (!r) oldseqval->seq_LS = newseqval->seq_LS;
 240                         updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_LS;
 241                 } else {
 242                         oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 243                 }
 244 #endif
 245                 break;
 246         case RWLOCK_SEQ_U:
 247                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 248                                 newseqval->seq_U, &oldseqval->seq_U, relaxed);
 249                 if (r) updated_seqfields = RWLOCK_SEQ_U;
 250                 break;
 251 #endif // unused
 252         case RWLOCK_SEQ_LS:
 253                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 254                                 newseqval->seq_LS, &oldseqval->seq_LS, relaxed);
 255                 if (r) updated_seqfields = RWLOCK_SEQ_LS;
 256                 break;
 257         default:
 258                 __builtin_trap();
 259         }
 260         return updated_seqfields;
 261 }
 262
 263 OS_ALWAYS_INLINE
 264 static inline rwlock_seqfields
 265 rwlock_seq_atomic_cmpxchgv_acquire(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 266                 rwlock_seq *newseqval, const rwlock_seqfields seqfields)
 267 {
 268         bool r;
 269         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 270         switch (seqfields) {
 271 #if DEBUG // unused
 272         case RWLOCK_SEQ_LSU:
 273 #if RWLOCK_USE_INT128
 274                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU,
 275                                 newseqval->seq_LSU, &oldseqval->seq_LSU, acquire);
 276                 if (r) updated_seqfields = RWLOCK_SEQ_LSU;
 277 #else
 278                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 279                                 newseqval->seq_LS, &oldseqval->seq_LS, acquire);
 280                 if (r) {
 281                         r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 282                                         newseqval->seq_U, &oldseqval->seq_U, relaxed);
 283                         if (!r) oldseqval->seq_LS = newseqval->seq_LS;
 284                         updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_LS;
 285                 } else {
 286                         oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
 287                 }
 288 #endif
 289                 break;
 290         case RWLOCK_SEQ_U:
 291                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 292                                 newseqval->seq_U, &oldseqval->seq_U, acquire);
 293                 if (r) updated_seqfields = RWLOCK_SEQ_U;
 294                 break;
 295 #endif // unused
 296         case RWLOCK_SEQ_LS:
 297                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 298                                 newseqval->seq_LS, &oldseqval->seq_LS, acquire);
 299                 if (r) updated_seqfields = RWLOCK_SEQ_LS;
 300                 break;
 301         default:
 302                 __builtin_trap();
 303         }
 304         return updated_seqfields;
 305 }
 306
 307 OS_ALWAYS_INLINE
 308 static inline rwlock_seqfields
 309 rwlock_seq_atomic_cmpxchgv_release(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
 310                 rwlock_seq *newseqval, const rwlock_seqfields seqfields)
 311 {
 312         bool r;
 313         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 314         switch (seqfields) {
 315         case RWLOCK_SEQ_LSU:
 316 #if RWLOCK_USE_INT128
 317                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU,
 318                                 newseqval->seq_LSU, &oldseqval->seq_LSU, release);
 319                 if (r) updated_seqfields = RWLOCK_SEQ_LSU;
 320 #else
 321                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 322                                 newseqval->seq_U, &oldseqval->seq_U, release);
 323                 if (r) {
 324                         r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 325                                         newseqval->seq_LS, &oldseqval->seq_LS, relaxed);
 326                         if (!r) oldseqval->seq_U = newseqval->seq_U;
 327                         updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_U;
 328                 } else {
 329                         oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS,
 330                                         relaxed);
 331                 }
 332 #endif
 333                 break;
 334         case RWLOCK_SEQ_LS:
 335                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS,
 336                                 newseqval->seq_LS, &oldseqval->seq_LS, release);
 337                 if (r) updated_seqfields = RWLOCK_SEQ_LS;
 338                 break;
 339 #if DEBUG // unused
 340         case RWLOCK_SEQ_U:
 341                 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U,
 342                                 newseqval->seq_U, &oldseqval->seq_U, release);
 343                 if (r) updated_seqfields = RWLOCK_SEQ_U;
 344                 break;
 345 #endif // unused
 346         default:
 347                 __builtin_trap();
 348         }
 349         return updated_seqfields;
 350 }
 351
 352 #define rwlock_seq_atomic_cmpxchgv(seqaddr, oldseqval, newseqval, seqfields, m)\
 353                 rwlock_seq_atomic_cmpxchgv_##m(seqaddr, oldseqval, newseqval, seqfields)
 354
 355 #ifndef BUILDING_VARIANT /* [ */
 356
 357 int
 358 pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
 359 {
 360         attr->sig = _PTHREAD_RWLOCK_ATTR_SIG;
 361         attr->pshared = _PTHREAD_DEFAULT_PSHARED;
 362         return 0;
 363 }
 364
 365 int
 366 pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
 367 {
 368         attr->sig = _PTHREAD_NO_SIG;
 369         attr->pshared = 0;
 370         return 0;
 371 }
 372
 373 int
 374 pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, int *pshared)
 375 {
 376         int res = EINVAL;
 377         if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) {
 378                 *pshared = (int)attr->pshared;
 379                 res = 0;
 380         }
 381         return res;
 382 }
 383
 384 int
 385 pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared)
 386 {
 387         int res = EINVAL;
 388         if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) {
 389                 if (( pshared == PTHREAD_PROCESS_PRIVATE) ||
 390                                 (pshared == PTHREAD_PROCESS_SHARED))
 391                 {
 392                         attr->pshared = pshared ;
 393                         res = 0;
 394                 }
 395         }
 396         return res;
 397 }
 398
 399 #endif /* !BUILDING_VARIANT ] */
 400
 401 OS_ALWAYS_INLINE
 402 static inline int
 403 _pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
 404 {
 405         uint64_t *tidaddr;
 406         RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 407
 408         rwlock_seq *seqaddr;
 409         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 410
 411 #if PTHREAD_RWLOCK_INIT_UNUSED
 412         if ((uint32_t*)tidaddr != rwlock->rw_tid) {
 413                 rwlock->misalign = 1;
 414                 __builtin_memset(rwlock->rw_tid, 0xff, sizeof(rwlock->rw_tid));
 415         }
 416         if ((uint32_t*)seqaddr != rwlock->rw_seq) {
 417                 __builtin_memset(rwlock->rw_seq, 0xff, sizeof(rwlock->rw_seq));
 418         }
 419         __builtin_memset(rwlock->rw_mis, 0xff, sizeof(rwlock->rw_mis));
 420 #endif // PTHREAD_MUTEX_INIT_UNUSED
 421         *tidaddr = 0;
 422         *seqaddr = (rwlock_seq){
 423                 .lcntval = PTHRW_RWLOCK_INIT,
 424                 .rw_seq = PTHRW_RWS_INIT,
 425                 .ucntval = 0,
 426         };
 427
 428         if (attr != NULL && attr->pshared == PTHREAD_PROCESS_SHARED) {
 429                 rwlock->pshared = PTHREAD_PROCESS_SHARED;
 430                 rwlock->rw_flags = PTHRW_KERN_PROCESS_SHARED;
 431         } else {
 432                 rwlock->pshared = _PTHREAD_DEFAULT_PSHARED;
 433                 rwlock->rw_flags = PTHRW_KERN_PROCESS_PRIVATE;
 434         }
 435
 436         long sig = _PTHREAD_RWLOCK_SIG;
 437
 438 #if DEBUG
 439         bzero(rwlock->_reserved, sizeof(rwlock->_reserved));
 440 #endif
 441 #if PTHREAD_RWLOCK_INIT_UNUSED
 442         // For detecting copied rwlocks and smashes during debugging
 443         uint32_t sig32 = (uint32_t)sig;
 444         uintptr_t guard = ~(uintptr_t)rwlock; // use ~ to hide from leaks
 445         __builtin_memcpy(rwlock->_reserved, &guard, sizeof(guard));
 446 #define countof(x) (sizeof(x) / sizeof(x[0]))
 447         rwlock->_reserved[countof(rwlock->_reserved) - 1] = sig32;
 448 #if defined(__LP64__)
 449         rwlock->_pad = sig32;
 450 #endif
 451 #endif // PTHREAD_RWLOCK_INIT_UNUSED
 452
 453         // Ensure all contents are properly set before setting signature.
 454 #if defined(__LP64__)
 455         // For binary compatibility reasons we cannot require natural alignment of
 456         // the 64bit 'sig' long value in the struct. rdar://problem/21610439
 457         uint32_t *sig32_ptr = (uint32_t*)&rwlock->sig;
 458         uint32_t *sig32_val = (uint32_t*)&sig;
 459         *(sig32_ptr + 1) = *(sig32_val + 1);
 460         os_atomic_store(sig32_ptr, *sig32_val, release);
 461 #else
 462         os_atomic_store(&rwlock->sig, sig, release);
 463 #endif
 464
 465         return 0;
 466 }
 467
 468 static uint32_t
 469 _pthread_rwlock_modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits)
 470 {
 471         uint32_t lval = lgenval & PTHRW_BIT_MASK;
 472         uint32_t uval = updateval & PTHRW_BIT_MASK;
 473         uint32_t rval, nlval;
 474
 475         nlval = (lval | uval) & ~(PTH_RWL_MBIT);
 476
 477         // reconcile bits on the lock with what kernel needs to set
 478         if ((uval & PTH_RWL_KBIT) == 0 && (lval & PTH_RWL_WBIT) == 0) {
 479                 nlval &= ~PTH_RWL_KBIT;
 480         }
 481
 482         if (savebits != 0) {
 483                 if ((savebits & PTH_RWS_WSVBIT) != 0 && (nlval & PTH_RWL_WBIT) == 0 &&
 484                                 (nlval & PTH_RWL_EBIT) == 0) {
 485                         nlval |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
 486                 }
 487         }
 488         rval = (lgenval & PTHRW_COUNT_MASK) | nlval;
 489         return(rval);
 490 }
 491
 492 OS_ALWAYS_INLINE
 493 static inline void
 494 _pthread_rwlock_updateval(pthread_rwlock_t *rwlock, uint32_t updateval)
 495 {
 496         bool isoverlap = (updateval & PTH_RWL_MBIT) != 0;
 497
 498         // TBD: restore U bit
 499         rwlock_seq *seqaddr;
 500         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 501
 502         rwlock_seq oldseq, newseq;
 503         rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LS);
 504         do {
 505                 newseq = oldseq;
 506                 if (isoverlap || is_rws_unlockinit_set(oldseq.rw_seq)) {
 507                         // Set S word to the specified value
 508                         uint32_t savebits = (oldseq.rw_seq & PTHRW_RWS_SAVEMASK);
 509                         newseq.lcntval = _pthread_rwlock_modbits(oldseq.lcntval, updateval,
 510                                         savebits);
 511                         newseq.rw_seq += (updateval & PTHRW_COUNT_MASK);
 512                         if (!isoverlap) {
 513                                 newseq.rw_seq &= PTHRW_COUNT_MASK;
 514                         }
 515                         newseq.rw_seq &= ~PTHRW_RWS_SAVEMASK;
 516                 }
 517         } while (!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
 518                         RWLOCK_SEQ_LS, relaxed));
 519         RWLOCK_DEBUG_SEQ(update, rwlock, oldseq, newseq, updateval, RWLOCK_SEQ_LS);
 520 }
 521
 522 OS_ALWAYS_INLINE
 523 static inline int
 524 _pthread_rwlock_check_busy(pthread_rwlock_t *rwlock)
 525 {
 526         int res = 0;
 527
 528         rwlock_seq *seqaddr;
 529         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 530
 531         rwlock_seq seq;
 532         rwlock_seq_atomic_load(seqaddr, &seq, RWLOCK_SEQ_LSU, relaxed);
 533         if ((seq.lcntval & PTHRW_COUNT_MASK) != seq.ucntval) {
 534                 res = EBUSY;
 535         }
 536
 537         return res;
 538 }
 539
 540 PTHREAD_NOEXPORT_VARIANT
 541 int
 542 pthread_rwlock_destroy(pthread_rwlock_t *rwlock)
 543 {
 544         int res = 0;
 545
 546         _pthread_lock_lock(&rwlock->lock);
 547         if (_pthread_rwlock_check_signature(rwlock)) {
 548                 res = _pthread_rwlock_check_busy(rwlock);
 549         } else if (!_pthread_rwlock_check_signature_init(rwlock)) {
 550                 res = EINVAL;
 551         }
 552         if (res == 0) {
 553                 rwlock->sig = _PTHREAD_NO_SIG;
 554         }
 555         _pthread_lock_unlock(&rwlock->lock);
 556         return res;
 557 }
 558
 559 PTHREAD_NOEXPORT_VARIANT
 560 int
 561 pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
 562 {
 563         int res = 0;
 564
 565         if (attr && attr->sig != _PTHREAD_RWLOCK_ATTR_SIG) {
 566                 res = EINVAL;
 567         }
 568
 569         if (res == 0 && _pthread_rwlock_check_signature(rwlock)) {
 570                 res = _pthread_rwlock_check_busy(rwlock);
 571         }
 572         if (res == 0) {
 573                 _pthread_lock_init(&rwlock->lock);
 574                 res = _pthread_rwlock_init(rwlock, attr);
 575         }
 576         return res;
 577 }
 578
 579 OS_NOINLINE
 580 static int
 581 _pthread_rwlock_check_init_slow(pthread_rwlock_t *rwlock)
 582 {
 583         int res = EINVAL;
 584
 585         if (_pthread_rwlock_check_signature_init(rwlock)) {
 586                 _pthread_lock_lock(&rwlock->lock);
 587                 if (_pthread_rwlock_check_signature_init(rwlock)) {
 588                         res = _pthread_rwlock_init(rwlock, NULL);
 589                 } else if (_pthread_rwlock_check_signature(rwlock)){
 590                         res = 0;
 591                 }
 592                 _pthread_lock_unlock(&rwlock->lock);
 593         } else if (_pthread_rwlock_check_signature(rwlock)){
 594                 res = 0;
 595         }
 596         if (res != 0) {
 597                 PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, res);
 598         }
 599         return res;
 600 }
 601
 602 OS_ALWAYS_INLINE
 603 static inline int
 604 _pthread_rwlock_check_init(pthread_rwlock_t *rwlock)
 605 {
 606         int res = 0;
 607
 608         if (!_pthread_rwlock_check_signature(rwlock)) {
 609                 return _pthread_rwlock_check_init_slow(rwlock);
 610         }
 611         return res;
 612 }
 613
 614 OS_NOINLINE
 615 static int
 616 _pthread_rwlock_lock_wait(pthread_rwlock_t *rwlock, bool readlock,
 617                 rwlock_seq newseq)
 618 {
 619         int res;
 620
 621 #ifdef PLOCKSTAT
 622         int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT;
 623 #endif
 624
 625         if (readlock) {
 626                 RWLOCK_DEBUG_SEQ(rdlock, rwlock, oldseq, newseq, gotlock,
 627                                 RWLOCK_SEQ_LSU);
 628         } else {
 629                 RWLOCK_DEBUG_SEQ(wrlock, rwlock, oldseq, newseq, gotlock,
 630                                 RWLOCK_SEQ_LSU);
 631         }
 632
 633         uint32_t updateval;
 634
 635         PLOCKSTAT_RW_BLOCK(rwlock, plockstat);
 636
 637         do {
 638                 if (readlock) {
 639                         updateval = __psynch_rw_rdlock(rwlock, newseq.lcntval,
 640                                         newseq.ucntval, newseq.rw_seq, rwlock->rw_flags);
 641                 } else {
 642                         updateval = __psynch_rw_wrlock(rwlock, newseq.lcntval,
 643                                         newseq.ucntval, newseq.rw_seq, rwlock->rw_flags);
 644                 }
 645                 if (updateval == (uint32_t)-1) {
 646                         res = errno;
 647                 } else {
 648                         res = 0;
 649                 }
 650         } while (res == EINTR);
 651
 652         if (res == 0) {
 653                 _pthread_rwlock_updateval(rwlock, updateval);
 654                 PLOCKSTAT_RW_BLOCKED(rwlock, plockstat, BLOCK_SUCCESS_PLOCKSTAT);
 655         } else {
 656                 PLOCKSTAT_RW_BLOCKED(rwlock, plockstat, BLOCK_FAIL_PLOCKSTAT);
 657                 PTHREAD_INTERNAL_CRASH(res, "kernel rwlock returned unknown error");
 658         }
 659
 660         return res;
 661 }
 662
 663 OS_NOINLINE
 664 int
 665 _pthread_rwlock_lock_slow(pthread_rwlock_t *rwlock, bool readlock,
 666                 bool trylock)
 667 {
 668         int res;
 669
 670 #ifdef PLOCKSTAT
 671         int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT;
 672 #endif
 673
 674         res = _pthread_rwlock_check_init(rwlock);
 675         if (res != 0) return res;
 676
 677         rwlock_seq *seqaddr;
 678         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 679
 680         rwlock_seq oldseq, newseq;
 681         rwlock_seq_atomic_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU, relaxed);
 682
 683         uint64_t *tidaddr;
 684         RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 685         uint64_t selfid = _pthread_threadid_self_np_direct();
 686         if (is_rwl_ebit_set(oldseq.lcntval)) {
 687                 if (os_atomic_load_wide(tidaddr, relaxed) == selfid) return EDEADLK;
 688         }
 689
 690         int retry_count;
 691         bool gotlock;
 692         do {
 693                 retry_count = 0;
 694 retry:
 695                 newseq = oldseq;
 696
 697                 // if W and K bit are clear or U bit is on, acquire lock in userland
 698                 if (readlock) {
 699                         gotlock = (oldseq.lcntval & (PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0;
 700                 } else {
 701                         gotlock = (oldseq.lcntval & PTH_RWL_UBIT) != 0;
 702                 }
 703
 704                 if (trylock && !gotlock) {
 705                         // A trylock on a held lock will fail immediately. But since
 706                         // we did not load the sequence words atomically, perform a
 707                         // no-op CAS to ensure that nobody has unlocked concurrently.
 708                 } else if (gotlock) {
 709                         if (readlock) {
 710                                 if (diff_genseq(oldseq.lcntval, oldseq.ucntval) >=
 711                                                 PTHRW_MAX_READERS) {
 712                                         // since ucntval may be newer, just redo
 713                                         retry_count++;
 714                                         if (retry_count > 1024) {
 715                                                 gotlock = false;
 716                                                 res = EAGAIN;
 717                                                 goto out;
 718                                         } else {
 719                                                 sched_yield();
 720                                                 rwlock_seq_atomic_load(seqaddr, &oldseq,
 721                                                                 RWLOCK_SEQ_LSU, relaxed);
 722                                                 goto retry;
 723                                         }
 724                                 }
 725                                 // Need to update L (remove U bit) and S word
 726                                 newseq.lcntval &= ~PTH_RWL_UBIT;
 727                         } else {
 728                                 newseq.lcntval &= PTHRW_COUNT_MASK;
 729                                 newseq.lcntval |= PTH_RWL_IBIT | PTH_RWL_KBIT | PTH_RWL_EBIT;
 730                         }
 731                         newseq.lcntval += PTHRW_INC;
 732                         newseq.rw_seq  += PTHRW_INC;
 733                 } else {
 734                         if (readlock) {
 735                                 // Need to block in kernel. Remove U bit.
 736                                 newseq.lcntval &= ~PTH_RWL_UBIT;
 737                         } else {
 738                                 newseq.lcntval |= PTH_RWL_KBIT | PTH_RWL_WBIT;
 739                         }
 740                         newseq.lcntval += PTHRW_INC;
 741                         if (is_rws_sbit_set(oldseq.rw_seq)) {
 742                                 // Clear the S bit and set S to L
 743                                 newseq.rw_seq &= (PTHRW_BIT_MASK & ~PTH_RWS_SBIT);
 744                                 newseq.rw_seq |= (oldseq.lcntval & PTHRW_COUNT_MASK);
 745                         }
 746                 }
 747         } while (!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
 748                         RWLOCK_SEQ_LS, acquire));
 749
 750         if (gotlock) {
 751                 if (!readlock) os_atomic_store_wide(tidaddr, selfid, relaxed);
 752                 res = 0;
 753         } else if (trylock) {
 754                 res = EBUSY;
 755         } else {
 756                 res = _pthread_rwlock_lock_wait(rwlock, readlock, newseq);
 757         }
 758
 759 out:
 760 #ifdef PLOCKSTAT
 761         if (res == 0) {
 762                 PLOCKSTAT_RW_ACQUIRE(rwlock, plockstat);
 763         } else {
 764                 PLOCKSTAT_RW_ERROR(rwlock, plockstat, res);
 765         }
 766 #endif
 767
 768         return res;
 769 }
 770
 771 OS_ALWAYS_INLINE
 772 static inline int
 773 _pthread_rwlock_lock(pthread_rwlock_t *rwlock, bool readlock, bool trylock)
 774 {
 775 #if PLOCKSTAT
 776         if (PLOCKSTAT_RW_ACQUIRE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) {
 777                 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock);
 778         }
 779 #endif
 780
 781         if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) {
 782                 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock);
 783         }
 784
 785         rwlock_seq *seqaddr;
 786         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 787
 788         rwlock_seq oldseq, newseq;
 789         // no need to perform a single-copy-atomic 128-bit load in the fastpath,
 790         // if stores to L and U are seen out of order, we will fallback to the
 791         // slowpath below (which has rwlock_seq_atomic_load)
 792         rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU);
 793
 794         if (os_unlikely(is_rwl_ebit_set(oldseq.lcntval))) {
 795                 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock);
 796         }
 797
 798         bool gotlock;
 799         do {
 800                 newseq = oldseq;
 801
 802                 // if W and K bit are clear or U bit is on, acquire lock in userland
 803                 if (readlock) {
 804                         gotlock = (oldseq.lcntval & (PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0;
 805                 } else {
 806                         gotlock = (oldseq.lcntval & PTH_RWL_UBIT) != 0;
 807                 }
 808
 809                 if (trylock && !gotlock) {
 810                         // A trylock on a held lock will fail immediately. But since
 811                         // we did not load the sequence words atomically, perform a
 812                         // no-op CAS to ensure that nobody has unlocked concurrently.
 813                 } else if (os_likely(gotlock)) {
 814                         if (readlock) {
 815                                 if (os_unlikely(diff_genseq(oldseq.lcntval, oldseq.ucntval) >=
 816                                                 PTHRW_MAX_READERS)) {
 817                                         return _pthread_rwlock_lock_slow(rwlock, readlock, trylock);
 818                                 }
 819                                 // Need to update L (remove U bit) and S word
 820                                 newseq.lcntval &= ~PTH_RWL_UBIT;
 821                         } else {
 822                                 newseq.lcntval &= PTHRW_COUNT_MASK;
 823                                 newseq.lcntval |= PTH_RWL_IBIT | PTH_RWL_KBIT | PTH_RWL_EBIT;
 824                         }
 825                         newseq.lcntval += PTHRW_INC;
 826                         newseq.rw_seq  += PTHRW_INC;
 827                 } else {
 828                         return _pthread_rwlock_lock_slow(rwlock, readlock, trylock);
 829                 }
 830         } while (os_unlikely(!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
 831                         RWLOCK_SEQ_LS, acquire)));
 832
 833         if (os_likely(gotlock)) {
 834                 if (!readlock) {
 835                         uint64_t *tidaddr;
 836                         RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 837                         uint64_t selfid = _pthread_threadid_self_np_direct();
 838                         os_atomic_store_wide(tidaddr, selfid, relaxed);
 839                 }
 840                 return 0;
 841         } else if (trylock) {
 842                 return EBUSY;
 843         } else {
 844                 __builtin_trap();
 845         }
 846 }
 847
 848 PTHREAD_NOEXPORT_VARIANT
 849 int
 850 pthread_rwlock_rdlock(pthread_rwlock_t *rwlock)
 851 {
 852         // read lock, no try
 853         return _pthread_rwlock_lock(rwlock, true, false);
 854 }
 855
 856 PTHREAD_NOEXPORT_VARIANT
 857 int
 858 pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
 859 {
 860         // read lock, try lock
 861         return _pthread_rwlock_lock(rwlock, true, true);
 862 }
 863
 864 PTHREAD_NOEXPORT_VARIANT
 865 int
 866 pthread_rwlock_wrlock(pthread_rwlock_t *rwlock)
 867 {
 868         // write lock, no try
 869         return _pthread_rwlock_lock(rwlock, false, false);
 870 }
 871
 872 PTHREAD_NOEXPORT_VARIANT
 873 int
 874 pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
 875 {
 876         // write lock, try lock
 877         return _pthread_rwlock_lock(rwlock, false, true);
 878 }
 879
 880 OS_NOINLINE
 881 static int
 882 _pthread_rwlock_unlock_drop(pthread_rwlock_t *rwlock, rwlock_seq oldseq,
 883                 rwlock_seq newseq)
 884 {
 885         int res;
 886
 887         RWLOCK_DEBUG_SEQ(unlock, rwlock, oldseq, newseq, !droplock, RWLOCK_SEQ_LSU);
 888         uint32_t updateval;
 889         do {
 890                 updateval = __psynch_rw_unlock(rwlock, oldseq.lcntval,
 891                                 newseq.ucntval, newseq.rw_seq, rwlock->rw_flags);
 892                 if (updateval == (uint32_t)-1) {
 893                         res = errno;
 894                 } else {
 895                         res = 0;
 896                         RWLOCK_DEBUG_SEQ(wakeup, rwlock, oldseq, newseq, updateval,
 897                                         RWLOCK_SEQ_LSU);
 898                 }
 899         } while (res == EINTR);
 900
 901         if (res != 0) {
 902                 PTHREAD_INTERNAL_CRASH(res, "kernel rwunlock returned unknown error");
 903         }
 904
 905         return res;
 906 }
 907
 908 OS_NOINLINE
 909 int
 910 _pthread_rwlock_unlock_slow(pthread_rwlock_t *rwlock,
 911                 rwlock_seqfields updated_seqfields)
 912 {
 913         int res;
 914         rwlock_seqfields seqfields = RWLOCK_SEQ_LSU;
 915 #ifdef PLOCKSTAT
 916         int wrlock = 0;
 917 #endif
 918
 919         res = _pthread_rwlock_check_init(rwlock);
 920         if (res != 0) return res;
 921
 922         rwlock_seq *seqaddr;
 923         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
 924
 925         rwlock_seq oldseq, newseq;
 926         rwlock_seq_load(seqaddr, &oldseq, seqfields);
 927
 928         if ((oldseq.lcntval & PTH_RWL_UBIT) != 0) {
 929                 // spurious unlock (unlock of unlocked lock)
 930                 return 0;
 931         }
 932
 933         if (is_rwl_ebit_set(oldseq.lcntval)) {
 934 #ifdef PLOCKSTAT
 935                 wrlock = 1;
 936 #endif
 937                 uint64_t *tidaddr;
 938                 RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
 939                 os_atomic_store_wide(tidaddr, 0, relaxed);
 940         }
 941
 942         bool droplock;
 943         do {
 944                 // stop loading & updating fields that have successfully been stored
 945                 seqfields &= ~updated_seqfields;
 946
 947                 newseq = oldseq;
 948                 if (seqfields & RWLOCK_SEQ_U) {
 949                         newseq.ucntval += PTHRW_INC;
 950                 }
 951
 952                 droplock = false;
 953                 uint32_t oldlcnt = (oldseq.lcntval & PTHRW_COUNT_MASK);
 954                 if (newseq.ucntval == oldlcnt) {
 955                         // last unlock, set L with U and init bits and set S to L with S bit
 956                         newseq.lcntval = oldlcnt | PTHRW_RWLOCK_INIT;
 957                         newseq.rw_seq =  oldlcnt | PTHRW_RWS_INIT;
 958                 } else {
 959                         // no L/S update if lock is not exclusive or no writer pending
 960                         if ((oldseq.lcntval &
 961                                         (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0) {
 962                                 continue;
 963                         }
 964
 965                         // kernel transition only needed if U == S
 966                         if (newseq.ucntval != (oldseq.rw_seq & PTHRW_COUNT_MASK)) {
 967                                 continue;
 968                         }
 969
 970                         droplock = true;
 971                         // reset all bits and set K
 972                         newseq.lcntval = oldlcnt | PTH_RWL_KBIT;
 973                         // set I bit on S word
 974                         newseq.rw_seq |= PTH_RWS_IBIT;
 975                         if ((oldseq.lcntval & PTH_RWL_WBIT) != 0) {
 976                                 newseq.rw_seq |= PTH_RWS_WSVBIT;
 977                         }
 978                 }
 979         } while (seqfields != (updated_seqfields = rwlock_seq_atomic_cmpxchgv(
 980                         seqaddr, &oldseq, &newseq, seqfields, release)));
 981
 982         if (droplock) {
 983                 res = _pthread_rwlock_unlock_drop(rwlock, oldseq, newseq);
 984         }
 985
 986         PLOCKSTAT_RW_RELEASE(rwlock, wrlock);
 987
 988         return res;
 989 }
 990
 991 PTHREAD_NOEXPORT_VARIANT
 992 int
 993 pthread_rwlock_unlock(pthread_rwlock_t *rwlock)
 994 {
 995         rwlock_seqfields seqfields = RWLOCK_SEQ_LSU;
 996         rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE;
 997
 998 #if PLOCKSTAT
 999         if (PLOCKSTAT_RW_RELEASE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) {
1000                 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields);
1001         }
1002 #endif
1003
1004         if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) {
1005                 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields);
1006         }
1007
1008         rwlock_seq *seqaddr;
1009         RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr);
1010
1011         rwlock_seq oldseq, newseq;
1012         rwlock_seq_load(seqaddr, &oldseq, seqfields);
1013
1014         if (os_unlikely(oldseq.lcntval & PTH_RWL_UBIT)) {
1015                 // spurious unlock (unlock of unlocked lock)
1016                 return 0;
1017         }
1018
1019         if (is_rwl_ebit_set(oldseq.lcntval)) {
1020                 uint64_t *tidaddr;
1021                 RWLOCK_GETTID_ADDR(rwlock, &tidaddr);
1022                 os_atomic_store_wide(tidaddr, 0, relaxed);
1023         }
1024
1025         do {
1026                 if (updated_seqfields) {
1027                         return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields);
1028                 }
1029
1030                 newseq = oldseq;
1031                 if (seqfields & RWLOCK_SEQ_U) {
1032                         newseq.ucntval += PTHRW_INC;
1033                 }
1034
1035                 uint32_t oldlcnt = (oldseq.lcntval & PTHRW_COUNT_MASK);
1036                 if (os_likely(newseq.ucntval == oldlcnt)) {
1037                         // last unlock, set L with U and init bits and set S to L with S bit
1038                         newseq.lcntval = oldlcnt | PTHRW_RWLOCK_INIT;
1039                         newseq.rw_seq =  oldlcnt | PTHRW_RWS_INIT;
1040                 } else {
1041                         if (os_likely((oldseq.lcntval &
1042                                         (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0 ||
1043                                         newseq.ucntval != (oldseq.rw_seq & PTHRW_COUNT_MASK))) {
1044                                 // no L/S update if lock is not exclusive or no writer pending
1045                                 // kernel transition only needed if U == S
1046                         } else {
1047                                 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields);
1048                         }
1049                 }
1050         } while (os_unlikely(seqfields != (updated_seqfields =
1051                         rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, seqfields,
1052                         release))));
1053
1054         return 0;
1055 }
1056