src/os/lock.c

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #define OS_UNFAIR_LOCK_INLINE 1
  22 #include "lock_internal.h"
  23 #include "os/internal.h"
  24 #include "resolver.h"
  25 #include "libkern/OSAtomic.h"
  26 #include "os/lock.h"
  27 #include "os/lock_private.h"
  28 #include "os/once_private.h"
  29
  30 #include <mach/mach_init.h>
  31 #include <mach/mach_traps.h>
  32 #include <mach/thread_switch.h>
  33 #include <mach/mach_time.h>
  34 #include <os/tsd.h>
  35
  36 #pragma mark -
  37 #pragma mark _os_lock_base_t
  38
  39 OS_NOINLINE OS_NORETURN OS_COLD
  40 void _os_lock_corruption_abort(void *lock_ptr OS_UNUSED, uintptr_t lock_value);
  41
  42
  43 OS_LOCK_STRUCT_DECL_INTERNAL(base);
  44 OS_USED static OS_LOCK_TYPE_STRUCT_DECL(base);
  45
  46 void
  47 os_lock_lock(os_lock_t l)
  48 {
  49         return l._osl_base->osl_type->osl_lock(l);
  50 }
  51
  52 bool
  53 os_lock_trylock(os_lock_t l)
  54 {
  55         return l._osl_base->osl_type->osl_trylock(l);
  56 }
  57
  58 void
  59 os_lock_unlock(os_lock_t l)
  60 {
  61         return l._osl_base->osl_type->osl_unlock(l);
  62 }
  63
  64 OS_NOINLINE OS_NORETURN OS_COLD
  65 void
  66 _os_lock_corruption_abort(void *lock_ptr OS_UNUSED, uintptr_t lock_value)
  67 {
  68         __LIBPLATFORM_CLIENT_CRASH__(lock_value, "os_lock is corrupt");
  69 }
  70
  71
  72 #pragma mark -
  73 #pragma mark OSSpinLock
  74
  75 OS_NOEXPORT OS_NOINLINE void _OSSpinLockLockSlow(volatile OSSpinLock *l);
  76
  77 OS_ATOMIC_EXPORT void OSSpinLockLock(volatile OSSpinLock *l);
  78 OS_ATOMIC_EXPORT bool OSSpinLockTry(volatile OSSpinLock *l);
  79 OS_ATOMIC_EXPORT int spin_lock_try(volatile OSSpinLock *l);
  80 OS_ATOMIC_EXPORT void OSSpinLockUnlock(volatile OSSpinLock *l);
  81
  82 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
  83 static const OSSpinLock _OSSpinLockLocked = 1;
  84 #else
  85 static const OSSpinLock _OSSpinLockLocked = -1;
  86 #endif
  87
  88
  89
  90 #if OS_ATOMIC_UP
  91 // Don't spin on UP
  92 #else
  93 #define OS_LOCK_SPIN_SPIN_TRIES 1000
  94 #define OS_LOCK_SPIN_PAUSE() os_hardware_pause()
  95 #endif
  96
  97 OS_ALWAYS_INLINE
  98 static uint64_t
  99 _os_lock_yield_deadline(mach_msg_timeout_t timeout)
 100 {
 101         uint64_t abstime = timeout * NSEC_PER_MSEC;
 102 #if !(defined(__i386__) || defined(__x86_64__))
 103         mach_timebase_info_data_t tbi;
 104         kern_return_t kr = mach_timebase_info(&tbi);
 105         if (kr) return UINT64_MAX;
 106         abstime *= tbi.denom;
 107         abstime /= tbi.numer;
 108 #endif
 109         return mach_absolute_time() + abstime;
 110 }
 111
 112 OS_ALWAYS_INLINE
 113 static bool
 114 _os_lock_yield_until(uint64_t deadline)
 115 {
 116         return mach_absolute_time() < deadline;
 117 }
 118
 119 OS_NOINLINE
 120 static void
 121 _OSSpinLockLockYield(volatile OSSpinLock *l)
 122 {
 123         int option = SWITCH_OPTION_DEPRESS;
 124         mach_msg_timeout_t timeout = 1;
 125         uint64_t deadline = _os_lock_yield_deadline(timeout);
 126         OSSpinLock lock;
 127         while (unlikely(lock = *l)) {
 128 _yield:
 129                 if (unlikely(lock != _OSSpinLockLocked)) {
 130                         _os_lock_corruption_abort((void *)l, (uintptr_t)lock);
 131                 }
 132                 thread_switch(MACH_PORT_NULL, option, timeout);
 133                 if (option == SWITCH_OPTION_WAIT) {
 134                         timeout++;
 135                 } else if (!_os_lock_yield_until(deadline)) {
 136                         option = SWITCH_OPTION_WAIT;
 137                 }
 138         }
 139         bool r = os_atomic_cmpxchgv(l, 0, _OSSpinLockLocked, &lock, acquire);
 140         if (likely(r)) return;
 141         goto _yield;
 142 }
 143
 144 #if OS_ATOMIC_UP
 145 void
 146 _OSSpinLockLockSlow(volatile OSSpinLock *l)
 147 {
 148         return _OSSpinLockLockYield(l); // Don't spin on UP
 149 }
 150 #else // !OS_ATOMIC_UP
 151 void
 152 _OSSpinLockLockSlow(volatile OSSpinLock *l)
 153 {
 154         uint32_t tries = OS_LOCK_SPIN_SPIN_TRIES;
 155         OSSpinLock lock;
 156         while (unlikely(lock = *l)) {
 157 _spin:
 158                 if (unlikely(lock != _OSSpinLockLocked)) {
 159                         return _os_lock_corruption_abort((void *)l, (uintptr_t)lock);
 160                 }
 161                 if (unlikely(!tries--)) return _OSSpinLockLockYield(l);
 162                 OS_LOCK_SPIN_PAUSE();
 163         }
 164         bool r = os_atomic_cmpxchgv(l, 0, _OSSpinLockLocked, &lock, acquire);
 165         if (likely(r)) return;
 166         goto _spin;
 167 }
 168 #endif // !OS_ATOMIC_UP
 169
 170
 171
 172 #if OS_LOCK_OSSPINLOCK_IS_NOSPINLOCK && !TARGET_OS_SIMULATOR
 173
 174 typedef struct _os_nospin_lock_s *_os_nospin_lock_t;
 175
 176 OS_ATOMIC_EXPORT void _os_nospin_lock_lock(_os_nospin_lock_t lock);
 177 OS_ATOMIC_EXPORT bool _os_nospin_lock_trylock(_os_nospin_lock_t lock);
 178 OS_ATOMIC_EXPORT void _os_nospin_lock_unlock(_os_nospin_lock_t lock);
 179
 180 void
 181 OSSpinLockLock(volatile OSSpinLock *l)
 182 {
 183         OS_ATOMIC_ALIAS(spin_lock, OSSpinLockLock);
 184         OS_ATOMIC_ALIAS(_spin_lock, OSSpinLockLock);
 185         return _os_nospin_lock_lock((_os_nospin_lock_t)l);
 186 }
 187
 188 bool
 189 OSSpinLockTry(volatile OSSpinLock *l)
 190 {
 191         return _os_nospin_lock_trylock((_os_nospin_lock_t)l);
 192 }
 193
 194 int
 195 spin_lock_try(volatile OSSpinLock *l)
 196 {
 197         OS_ATOMIC_ALIAS(_spin_lock_try, spin_lock_try);
 198         return _os_nospin_lock_trylock((_os_nospin_lock_t)l);
 199 }
 200
 201 void
 202 OSSpinLockUnlock(volatile OSSpinLock *l)
 203 {
 204         OS_ATOMIC_ALIAS(spin_unlock, OSSpinLockUnlock);
 205         OS_ATOMIC_ALIAS(_spin_unlock, OSSpinLockUnlock);
 206         return _os_nospin_lock_unlock((_os_nospin_lock_t)l);
 207 }
 208
 209 #undef OS_ATOMIC_ALIAS
 210 #define OS_ATOMIC_ALIAS(n, o)
 211 static void _OSSpinLockLock(volatile OSSpinLock *l);
 212 #undef OSSpinLockLock
 213 #define OSSpinLockLock _OSSpinLockLock
 214 static bool _OSSpinLockTry(volatile OSSpinLock *l);
 215 #undef OSSpinLockTry
 216 #define OSSpinLockTry _OSSpinLockTry
 217 static __unused int __spin_lock_try(volatile OSSpinLock *l);
 218 #undef spin_lock_try
 219 #define spin_lock_try __spin_lock_try
 220 static void _OSSpinLockUnlock(volatile OSSpinLock *l);
 221 #undef OSSpinLockUnlock
 222 #define OSSpinLockUnlock _OSSpinLockUnlock
 223
 224 #endif // OS_LOCK_OSSPINLOCK_IS_NOSPINLOCK
 225
 226 void
 227 OSSpinLockLock(volatile OSSpinLock *l)
 228 {
 229         OS_ATOMIC_ALIAS(spin_lock, OSSpinLockLock);
 230         OS_ATOMIC_ALIAS(_spin_lock, OSSpinLockLock);
 231         bool r = os_atomic_cmpxchg(l, 0, _OSSpinLockLocked, acquire);
 232         if (likely(r)) return;
 233         return _OSSpinLockLockSlow(l);
 234 }
 235
 236 bool
 237 OSSpinLockTry(volatile OSSpinLock *l)
 238 {
 239         bool r = os_atomic_cmpxchg(l, 0, _OSSpinLockLocked, acquire);
 240         return r;
 241 }
 242
 243 int
 244 spin_lock_try(volatile OSSpinLock *l) // <rdar://problem/13316060>
 245 {
 246         OS_ATOMIC_ALIAS(_spin_lock_try, spin_lock_try);
 247         return OSSpinLockTry(l);
 248 }
 249
 250 void
 251 OSSpinLockUnlock(volatile OSSpinLock *l)
 252 {
 253         OS_ATOMIC_ALIAS(spin_unlock, OSSpinLockUnlock);
 254         OS_ATOMIC_ALIAS(_spin_unlock, OSSpinLockUnlock);
 255         os_atomic_store(l, 0, release);
 256 }
 257
 258
 259 #pragma mark -
 260 #pragma mark os_lock_spin_t
 261
 262 OS_LOCK_STRUCT_DECL_INTERNAL(spin,
 263         OSSpinLock volatile osl_spinlock;
 264 );
 265 OS_LOCK_METHODS_DECL(spin);
 266 OS_LOCK_TYPE_INSTANCE(spin);
 267
 268 void
 269 _os_lock_spin_lock(_os_lock_spin_t l)
 270 {
 271         return OSSpinLockLock(&l->osl_spinlock);
 272 }
 273
 274 bool
 275 _os_lock_spin_trylock(_os_lock_spin_t l)
 276 {
 277         return OSSpinLockTry(&l->osl_spinlock);
 278 }
 279
 280 void
 281 _os_lock_spin_unlock(_os_lock_spin_t l)
 282 {
 283         return OSSpinLockUnlock(&l->osl_spinlock);
 284 }
 285
 286
 287 #pragma mark -
 288 #pragma mark os_lock_owner_t
 289
 290 #ifndef __TSD_MACH_THREAD_SELF
 291 #define __TSD_MACH_THREAD_SELF 3
 292 #endif
 293
 294 typedef mach_port_name_t os_lock_owner_t;
 295 #define OS_LOCK_NO_OWNER MACH_PORT_NULL
 296
 297
 298 OS_ALWAYS_INLINE OS_CONST
 299 static inline os_lock_owner_t
 300 _os_lock_owner_get_self(void)
 301 {
 302         os_lock_owner_t self;
 303         self = (os_lock_owner_t)_os_tsd_get_direct(__TSD_MACH_THREAD_SELF);
 304         return self;
 305 }
 306
 307
 308 OS_NOINLINE OS_NORETURN OS_COLD
 309 static void
 310 _os_lock_recursive_abort(os_lock_owner_t owner)
 311 {
 312         __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an "
 313                         "os_lock");
 314 }
 315
 316
 317 #pragma mark -
 318 #pragma mark os_lock_handoff_t
 319
 320 OS_LOCK_STRUCT_DECL_INTERNAL(handoff,
 321         os_lock_owner_t volatile osl_owner;
 322 );
 323 OS_LOCK_METHODS_DECL(handoff);
 324 OS_LOCK_TYPE_INSTANCE(handoff);
 325
 326 #define OS_LOCK_HANDOFF_YIELD_TRIES 100
 327
 328 OS_NOINLINE
 329 static void
 330 _os_lock_handoff_lock_slow(_os_lock_handoff_t l)
 331 {
 332         int option = SWITCH_OPTION_OSLOCK_DEPRESS;
 333         mach_msg_timeout_t timeout = 1;
 334         uint32_t tries = OS_LOCK_HANDOFF_YIELD_TRIES;
 335         os_lock_owner_t self = _os_lock_owner_get_self(), owner;
 336         while (unlikely(owner = l->osl_owner)) {
 337 _handoff:
 338                 if (unlikely(owner == self)) return _os_lock_recursive_abort(self);
 339                 // Yield until tries first hits zero, then permanently switch to wait
 340                 if (unlikely(!tries--)) option = SWITCH_OPTION_OSLOCK_WAIT;
 341                 thread_switch(owner, option, timeout);
 342                 // Redrive the handoff every 1ms until switching to wait
 343                 if (option == SWITCH_OPTION_OSLOCK_WAIT) timeout++;
 344         }
 345         bool r = os_atomic_cmpxchgv2o(l, osl_owner, MACH_PORT_NULL, self, &owner,
 346                         acquire);
 347         if (likely(r)) return;
 348         goto _handoff;
 349 }
 350
 351 void
 352 _os_lock_handoff_lock(_os_lock_handoff_t l)
 353 {
 354         os_lock_owner_t self = _os_lock_owner_get_self();
 355         bool r = os_atomic_cmpxchg2o(l, osl_owner, MACH_PORT_NULL, self, acquire);
 356         if (likely(r)) return;
 357         return _os_lock_handoff_lock_slow(l);
 358 }
 359
 360 bool
 361 _os_lock_handoff_trylock(_os_lock_handoff_t l)
 362 {
 363         os_lock_owner_t self = _os_lock_owner_get_self();
 364         bool r = os_atomic_cmpxchg2o(l, osl_owner, MACH_PORT_NULL, self, acquire);
 365         return r;
 366 }
 367
 368 void
 369 _os_lock_handoff_unlock(_os_lock_handoff_t l)
 370 {
 371         os_atomic_store2o(l, osl_owner, MACH_PORT_NULL, release);
 372 }
 373
 374
 375 #pragma mark -
 376 #pragma mark os_ulock_value_t
 377
 378 #include <sys/errno.h>
 379 #include <sys/ulock.h>
 380
 381 typedef os_lock_owner_t os_ulock_value_t;
 382
 383 // This assumes that all thread mach port values always have the low bit set!
 384 // Clearing this bit is used to communicate the existence of waiters to unlock.
 385 #define OS_ULOCK_NOWAITERS_BIT ((os_ulock_value_t)1u)
 386 #define OS_ULOCK_OWNER(value) ((value) | OS_ULOCK_NOWAITERS_BIT)
 387
 388 #define OS_ULOCK_ANONYMOUS_OWNER MACH_PORT_DEAD
 389 #define OS_ULOCK_IS_OWNER(value, self, allow_anonymous_owner) ({ \
 390                 os_lock_owner_t _owner = OS_ULOCK_OWNER(value); (_owner == (self)) && \
 391                 (!(allow_anonymous_owner) || _owner != OS_ULOCK_ANONYMOUS_OWNER); })
 392 #define OS_ULOCK_IS_NOT_OWNER(value, self, allow_anonymous_owner) ({ \
 393                 os_lock_owner_t _owner = OS_ULOCK_OWNER(value); (_owner != (self)) && \
 394                 (!(allow_anonymous_owner) || _owner != OS_ULOCK_ANONYMOUS_OWNER); })
 395
 396 #pragma mark -
 397 #pragma mark os_unfair_lock
 398
 399 typedef struct _os_unfair_lock_s {
 400         os_ulock_value_t oul_value;
 401 } *_os_unfair_lock_t;
 402
 403 _Static_assert(sizeof(struct os_unfair_lock_s) ==
 404                 sizeof(struct _os_unfair_lock_s), "os_unfair_lock size mismatch");
 405
 406 OS_ATOMIC_EXPORT void os_unfair_lock_lock(os_unfair_lock_t lock);
 407 OS_ATOMIC_EXPORT void os_unfair_lock_lock_with_options(os_unfair_lock_t lock,
 408                 os_unfair_lock_options_t options);
 409 OS_ATOMIC_EXPORT bool os_unfair_lock_trylock(os_unfair_lock_t lock);
 410 OS_ATOMIC_EXPORT void os_unfair_lock_unlock(os_unfair_lock_t lock);
 411
 412 OS_ATOMIC_EXPORT void os_unfair_lock_lock_no_tsd_4libpthread(
 413                 os_unfair_lock_t lock);
 414 OS_ATOMIC_EXPORT void os_unfair_lock_unlock_no_tsd_4libpthread(
 415                 os_unfair_lock_t lock);
 416 OS_ATOMIC_EXPORT void os_unfair_lock_lock_with_options_4Libc(
 417                 os_unfair_lock_t lock, os_unfair_lock_options_t options);
 418 OS_ATOMIC_EXPORT void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock);
 419
 420 OS_NOINLINE OS_NORETURN OS_COLD
 421 void _os_unfair_lock_recursive_abort(os_lock_owner_t owner);
 422 OS_NOINLINE OS_NORETURN OS_COLD
 423 void _os_unfair_lock_unowned_abort(os_lock_owner_t owner);
 424 OS_NOINLINE OS_NORETURN OS_COLD
 425 void _os_unfair_lock_corruption_abort(os_ulock_value_t current);
 426
 427 _Static_assert(OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION ==
 428                 ULF_WAIT_WORKQ_DATA_CONTENTION,
 429                 "check value for OS_UNFAIR_LOCK_OPTIONS_MASK");
 430 #define OS_UNFAIR_LOCK_OPTIONS_MASK \
 431                 (os_unfair_lock_options_t)(OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 432 #define OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER 0x01000000u
 433
 434
 435 OS_NOINLINE OS_NORETURN OS_COLD
 436 void
 437 _os_unfair_lock_recursive_abort(os_lock_owner_t owner)
 438 {
 439         __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an "
 440                         "os_unfair_lock");
 441 }
 442
 443 OS_NOINLINE OS_NORETURN OS_COLD
 444 void
 445 _os_unfair_lock_unowned_abort(os_lock_owner_t owner)
 446 {
 447         __LIBPLATFORM_CLIENT_CRASH__(owner, "Unlock of an os_unfair_lock not "
 448                         "owned by current thread");
 449 }
 450
 451 OS_NOINLINE OS_NORETURN OS_COLD
 452 void
 453 _os_unfair_lock_corruption_abort(os_ulock_value_t current)
 454 {
 455         __LIBPLATFORM_CLIENT_CRASH__(current, "os_unfair_lock is corrupt");
 456 }
 457
 458
 459 OS_NOINLINE
 460 static void
 461 _os_unfair_lock_lock_slow(_os_unfair_lock_t l, os_lock_owner_t self,
 462                 os_unfair_lock_options_t options)
 463 {
 464         os_unfair_lock_options_t allow_anonymous_owner =
 465                         options & OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER;
 466         options &= ~OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER;
 467         if (unlikely(options & ~OS_UNFAIR_LOCK_OPTIONS_MASK)) {
 468                 __LIBPLATFORM_CLIENT_CRASH__(options, "Invalid options");
 469         }
 470         os_ulock_value_t current, new, waiters_mask = 0;
 471         while (unlikely((current = os_atomic_load2o(l, oul_value, relaxed)) !=
 472                         OS_LOCK_NO_OWNER)) {
 473 _retry:
 474                 if (unlikely(OS_ULOCK_IS_OWNER(current, self, allow_anonymous_owner))) {
 475                         return _os_unfair_lock_recursive_abort(self);
 476                 }
 477                 new = current & ~OS_ULOCK_NOWAITERS_BIT;
 478                 if (current != new) {
 479                         // Clear nowaiters bit in lock value before waiting
 480                         if (!os_atomic_cmpxchgv2o(l, oul_value, current, new, &current,
 481                                         relaxed)){
 482                                 continue;
 483                         }
 484                         current = new;
 485                 }
 486                 int ret = __ulock_wait(UL_UNFAIR_LOCK | ULF_NO_ERRNO | options,
 487                                 l, current, 0);
 488                 if (unlikely(ret < 0)) {
 489                         switch (-ret) {
 490                         case EINTR:
 491                         case EFAULT:
 492                                 continue;
 493                         case EOWNERDEAD:
 494                                 _os_unfair_lock_corruption_abort(current);
 495                                 break;
 496                         default:
 497                                 __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wait failure");
 498                         }
 499                 }
 500                 // If there are more waiters, unset nowaiters bit when acquiring lock
 501                 waiters_mask = (ret > 0) ? OS_ULOCK_NOWAITERS_BIT : 0;
 502         }
 503         new = self & ~waiters_mask;
 504         bool r = os_atomic_cmpxchgv2o(l, oul_value, OS_LOCK_NO_OWNER, new,
 505                         &current, acquire);
 506         if (unlikely(!r)) goto _retry;
 507 }
 508
 509 OS_NOINLINE
 510 static void
 511 _os_unfair_lock_unlock_slow(_os_unfair_lock_t l, os_ulock_value_t current,
 512                 os_lock_owner_t self, os_unfair_lock_options_t options)
 513 {
 514         os_unfair_lock_options_t allow_anonymous_owner =
 515                         options & OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER;
 516         options &= ~OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER;
 517         if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, allow_anonymous_owner))) {
 518                 return _os_unfair_lock_unowned_abort(OS_ULOCK_OWNER(current));
 519         }
 520         if (current & OS_ULOCK_NOWAITERS_BIT) {
 521                 __LIBPLATFORM_INTERNAL_CRASH__(current, "unlock_slow with no waiters");
 522         }
 523         for (;;) {
 524                 int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO, l, 0);
 525                 if (unlikely(ret < 0)) {
 526                         switch (-ret) {
 527                         case EINTR:
 528                                 continue;
 529                         case ENOENT:
 530                                 break;
 531                         default:
 532                                 __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wake failure");
 533                         }
 534                 }
 535                 break;
 536         }
 537 }
 538
 539 void
 540 os_unfair_lock_lock(os_unfair_lock_t lock)
 541 {
 542         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 543         os_lock_owner_t self = _os_lock_owner_get_self();
 544         bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire);
 545         if (likely(r)) return;
 546         return _os_unfair_lock_lock_slow(l, self, OS_UNFAIR_LOCK_NONE);
 547 }
 548
 549 void
 550 os_unfair_lock_lock_with_options(os_unfair_lock_t lock,
 551                 os_unfair_lock_options_t options)
 552 {
 553         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 554         os_lock_owner_t self = _os_lock_owner_get_self();
 555         bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire);
 556         if (likely(r)) return;
 557         return _os_unfair_lock_lock_slow(l, self, options);
 558 }
 559
 560 bool
 561 os_unfair_lock_trylock(os_unfair_lock_t lock)
 562 {
 563         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 564         os_lock_owner_t self = _os_lock_owner_get_self();
 565         bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire);
 566         return r;
 567 }
 568
 569 void
 570 os_unfair_lock_unlock(os_unfair_lock_t lock)
 571 {
 572         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 573         os_lock_owner_t self = _os_lock_owner_get_self();
 574         os_ulock_value_t current;
 575         current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release);
 576         if (likely(current == self)) return;
 577         return _os_unfair_lock_unlock_slow(l, current, self, 0);
 578 }
 579
 580 void
 581 os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock)
 582 {
 583         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 584         os_lock_owner_t self = OS_ULOCK_ANONYMOUS_OWNER;
 585         bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire);
 586         if (likely(r)) return;
 587         return _os_unfair_lock_lock_slow(l, self,
 588                         OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION|
 589                         OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER);
 590 }
 591
 592 void
 593 os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock)
 594 {
 595         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 596         os_lock_owner_t self = OS_ULOCK_ANONYMOUS_OWNER;
 597         os_ulock_value_t current;
 598         current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release);
 599         if (likely(current == self)) return;
 600         return _os_unfair_lock_unlock_slow(l, current, self,
 601                         OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER);
 602 }
 603
 604
 605 void
 606 os_unfair_lock_assert_owner(os_unfair_lock_t lock)
 607 {
 608         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 609         os_lock_owner_t self = _os_lock_owner_get_self();
 610         os_ulock_value_t current = os_atomic_load2o(l, oul_value, relaxed);
 611         if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, 0))) {
 612                 __LIBPLATFORM_CLIENT_CRASH__(current, "Assertion failed: "
 613                                 "Lock unexpectedly not owned by current thread");
 614         }
 615 }
 616
 617 void
 618 os_unfair_lock_assert_not_owner(os_unfair_lock_t lock)
 619 {
 620         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 621         os_lock_owner_t self = _os_lock_owner_get_self();
 622         os_ulock_value_t current = os_atomic_load2o(l, oul_value, relaxed);
 623         if (unlikely(OS_ULOCK_IS_OWNER(current, self, 0))) {
 624                 __LIBPLATFORM_CLIENT_CRASH__(current, "Assertion failed: "
 625                                 "Lock unexpectedly owned by current thread");
 626         }
 627 }
 628
 629
 630 #pragma mark -
 631 #pragma mark os_unfair_recursive_lock
 632
 633 OS_ATOMIC_EXPORT
 634 void os_unfair_recursive_lock_lock_with_options(os_unfair_recursive_lock_t lock,
 635                 os_unfair_lock_options_t options);
 636
 637 OS_ATOMIC_EXPORT
 638 bool os_unfair_recursive_lock_trylock(os_unfair_recursive_lock_t lock);
 639
 640 OS_ATOMIC_EXPORT
 641 void os_unfair_recursive_lock_unlock(os_unfair_recursive_lock_t lock);
 642
 643 OS_ATOMIC_EXPORT
 644 bool os_unfair_recursive_lock_tryunlock4objc(os_unfair_recursive_lock_t lock);
 645
 646
 647 static inline os_lock_owner_t
 648 _os_unfair_lock_owner(os_unfair_lock_t lock)
 649 {
 650         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 651         return OS_ULOCK_OWNER(os_atomic_load(&l->oul_value, relaxed));
 652 }
 653
 654 void
 655 os_unfair_recursive_lock_lock_with_options(os_unfair_recursive_lock_t lock,
 656                 os_unfair_lock_options_t options)
 657 {
 658         os_lock_owner_t cur, self = _os_lock_owner_get_self();
 659         _os_unfair_lock_t l = (_os_unfair_lock_t)&lock->ourl_lock;
 660
 661         if (likely(os_atomic_cmpxchgv2o(l, oul_value,
 662                         OS_LOCK_NO_OWNER, self, &cur, acquire))) {
 663                 return;
 664         }
 665
 666         if (OS_ULOCK_OWNER(cur) == self) {
 667                 lock->ourl_count++;
 668                 return;
 669         }
 670
 671         return _os_unfair_lock_lock_slow(l, self, options);
 672 }
 673
 674 bool
 675 os_unfair_recursive_lock_trylock(os_unfair_recursive_lock_t lock)
 676 {
 677         os_lock_owner_t cur, self = _os_lock_owner_get_self();
 678         _os_unfair_lock_t l = (_os_unfair_lock_t)&lock->ourl_lock;
 679
 680         if (likely(os_atomic_cmpxchgv2o(l, oul_value,
 681                         OS_LOCK_NO_OWNER, self, &cur, acquire))) {
 682                 return true;
 683         }
 684
 685         if (likely(OS_ULOCK_OWNER(cur) == self)) {
 686                 lock->ourl_count++;
 687                 return true;
 688         }
 689
 690         return false;
 691 }
 692
 693
 694 OS_ALWAYS_INLINE
 695 static inline void
 696 _os_unfair_recursive_lock_unlock(os_unfair_recursive_lock_t lock,
 697                 os_lock_owner_t self)
 698 {
 699         if (unlikely(lock->ourl_count)) {
 700                 os_lock_owner_t cur = _os_unfair_lock_owner(&lock->ourl_lock);
 701                 if (unlikely(cur != self)) {
 702                         _os_unfair_lock_unowned_abort(cur);
 703                 }
 704                 lock->ourl_count--;
 705                 return;
 706         }
 707
 708         _os_unfair_lock_t l = (_os_unfair_lock_t)lock;
 709         os_ulock_value_t current;
 710         current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release);
 711         if (likely(current == self)) return;
 712         return _os_unfair_lock_unlock_slow(l, current, self, 0);
 713 }
 714
 715 void
 716 os_unfair_recursive_lock_unlock(os_unfair_recursive_lock_t lock)
 717 {
 718         os_lock_owner_t self = _os_lock_owner_get_self();
 719         _os_unfair_recursive_lock_unlock(lock, self);
 720 }
 721
 722 bool
 723 os_unfair_recursive_lock_tryunlock4objc(os_unfair_recursive_lock_t lock)
 724 {
 725         os_lock_owner_t cur = _os_unfair_lock_owner(&lock->ourl_lock);
 726         os_lock_owner_t self = _os_lock_owner_get_self();
 727         if (likely(cur == self)) {
 728                 _os_unfair_recursive_lock_unlock(lock, self);
 729                 return true;
 730         }
 731         return false;
 732 }
 733
 734
 735 #pragma mark -
 736 #pragma mark _os_lock_unfair_t
 737
 738 OS_LOCK_STRUCT_DECL_INTERNAL(unfair,
 739         os_unfair_lock osl_unfair_lock;
 740 );
 741 OS_LOCK_METHODS_DECL(unfair);
 742 OS_LOCK_TYPE_INSTANCE(unfair);
 743
 744 void
 745 _os_lock_unfair_lock(_os_lock_unfair_t l)
 746 {
 747         return os_unfair_lock_lock(&l->osl_unfair_lock);
 748 }
 749
 750 bool
 751 _os_lock_unfair_trylock(_os_lock_unfair_t l)
 752 {
 753         return os_unfair_lock_trylock(&l->osl_unfair_lock);
 754 }
 755
 756 void
 757 _os_lock_unfair_unlock(_os_lock_unfair_t l)
 758 {
 759         return os_unfair_lock_unlock(&l->osl_unfair_lock);
 760 }
 761
 762
 763 #pragma mark -
 764 #pragma mark _os_nospin_lock
 765
 766 typedef struct _os_nospin_lock_s {
 767         os_ulock_value_t oul_value;
 768 } _os_nospin_lock, *_os_nospin_lock_t;
 769
 770 _Static_assert(sizeof(OSSpinLock) ==
 771                 sizeof(struct _os_nospin_lock_s), "os_nospin_lock size mismatch");
 772
 773 OS_ATOMIC_EXPORT void _os_nospin_lock_lock(_os_nospin_lock_t lock);
 774 OS_ATOMIC_EXPORT bool _os_nospin_lock_trylock(_os_nospin_lock_t lock);
 775 OS_ATOMIC_EXPORT void _os_nospin_lock_unlock(_os_nospin_lock_t lock);
 776
 777
 778 OS_NOINLINE
 779 static void
 780 _os_nospin_lock_lock_slow(_os_nospin_lock_t l)
 781 {
 782         os_lock_owner_t self = _os_lock_owner_get_self();
 783         os_ulock_value_t current, new, waiters_mask = 0;
 784         uint32_t timeout = 1;
 785         while (unlikely((current = os_atomic_load2o(l, oul_value, relaxed)) !=
 786                         OS_LOCK_NO_OWNER)) {
 787 _retry:
 788                 new = current & ~OS_ULOCK_NOWAITERS_BIT;
 789                 // For safer compatibility with OSSpinLock where _OSSpinLockLocked may
 790                 // be 1, check that new didn't become 0 (unlocked) by clearing this bit
 791                 if (current != new && new) {
 792                         // Clear nowaiters bit in lock value before waiting
 793                         if (!os_atomic_cmpxchgv2o(l, oul_value, current, new, &current,
 794                                         relaxed)){
 795                                 continue;
 796                         }
 797                         current = new;
 798                 }
 799                 int ret = __ulock_wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, l, current,
 800                                 timeout * 1000);
 801                 if (unlikely(ret < 0)) {
 802                         switch (-ret) {
 803                         case ETIMEDOUT:
 804                                 timeout++;
 805                                 continue;
 806                         case EINTR:
 807                         case EFAULT:
 808                                 continue;
 809                         default:
 810                                 __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wait failure");
 811                         }
 812                 }
 813                 // If there are more waiters, unset nowaiters bit when acquiring lock
 814                 waiters_mask = (ret > 0) ? OS_ULOCK_NOWAITERS_BIT : 0;
 815         }
 816         new = self & ~waiters_mask;
 817         bool r = os_atomic_cmpxchgv2o(l, oul_value, OS_LOCK_NO_OWNER, new,
 818                         &current, acquire);
 819         if (unlikely(!r)) goto _retry;
 820 }
 821
 822 OS_NOINLINE
 823 static void
 824 _os_nospin_lock_unlock_slow(_os_nospin_lock_t l, os_ulock_value_t current)
 825 {
 826         os_lock_owner_t self = _os_lock_owner_get_self();
 827         if (unlikely(OS_ULOCK_OWNER(current) != self)) {
 828                 return; // no unowned_abort for drop-in compatibility with OSSpinLock
 829         }
 830         if (current & OS_ULOCK_NOWAITERS_BIT) {
 831                 __LIBPLATFORM_INTERNAL_CRASH__(current, "unlock_slow with no waiters");
 832         }
 833         for (;;) {
 834                 int ret = __ulock_wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, l, 0);
 835                 if (unlikely(ret < 0)) {
 836                         switch (-ret) {
 837                         case EINTR:
 838                                 continue;
 839                         case ENOENT:
 840                                 break;
 841                         default:
 842                                 __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wake failure");
 843                         }
 844                 }
 845                 break;
 846         }
 847 }
 848
 849 void
 850 _os_nospin_lock_lock(_os_nospin_lock_t l)
 851 {
 852         os_lock_owner_t self = _os_lock_owner_get_self();
 853         bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire);
 854         if (likely(r)) return;
 855         return _os_nospin_lock_lock_slow(l);
 856 }
 857
 858 bool
 859 _os_nospin_lock_trylock(_os_nospin_lock_t l)
 860 {
 861         os_lock_owner_t self = _os_lock_owner_get_self();
 862         bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire);
 863         return r;
 864 }
 865
 866 void
 867 _os_nospin_lock_unlock(_os_nospin_lock_t l)
 868 {
 869         os_lock_owner_t self = _os_lock_owner_get_self();
 870         os_ulock_value_t current;
 871         current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release);
 872         if (likely(current == self)) return;
 873         return _os_nospin_lock_unlock_slow(l, current);
 874 }
 875
 876
 877 #pragma mark -
 878 #pragma mark _os_lock_nospin_t
 879
 880 OS_LOCK_STRUCT_DECL_INTERNAL(nospin,
 881         _os_nospin_lock osl_nospin_lock;
 882 );
 883 OS_LOCK_METHODS_DECL(nospin);
 884 OS_LOCK_TYPE_INSTANCE(nospin);
 885
 886 void
 887 _os_lock_nospin_lock(_os_lock_nospin_t l)
 888 {
 889         return _os_nospin_lock_lock(&l->osl_nospin_lock);
 890 }
 891
 892 bool
 893 _os_lock_nospin_trylock(_os_lock_nospin_t l)
 894 {
 895         return _os_nospin_lock_trylock(&l->osl_nospin_lock);
 896 }
 897
 898 void
 899 _os_lock_nospin_unlock(_os_lock_nospin_t l)
 900 {
 901         return _os_nospin_lock_unlock(&l->osl_nospin_lock);
 902 }
 903
 904
 905 #pragma mark -
 906 #pragma mark os_once_t
 907
 908 typedef struct os_once_gate_s {
 909         union {
 910                 os_ulock_value_t ogo_lock;
 911                 uintptr_t ogo_once;
 912         };
 913 } os_once_gate_s, *os_once_gate_t;
 914
 915 #define OS_ONCE_INIT ((uintptr_t)0l)
 916 #define OS_ONCE_DONE (~(uintptr_t)0l)
 917
 918 #if defined(__i386__) || defined(__x86_64__)
 919 #define OS_ONCE_USE_QUIESCENT_COUNTER 0
 920 #else
 921 #define OS_ONCE_USE_QUIESCENT_COUNTER 1
 922 #endif
 923
 924 OS_ATOMIC_EXPORT void _os_once(os_once_t *val, void *ctxt, os_function_t func);
 925 OS_ATOMIC_EXPORT void __os_once_reset(os_once_t *val);
 926
 927 OS_NOINLINE OS_NORETURN OS_COLD
 928 void _os_once_gate_recursive_abort(os_lock_owner_t owner);
 929 OS_NOINLINE OS_NORETURN OS_COLD
 930 void _os_once_gate_unowned_abort(os_lock_owner_t owner);
 931 OS_NOINLINE OS_NORETURN OS_COLD
 932 void _os_once_gate_corruption_abort(os_ulock_value_t current);
 933
 934
 935 OS_NOINLINE OS_NORETURN OS_COLD
 936 void
 937 _os_once_gate_recursive_abort(os_lock_owner_t owner)
 938 {
 939         __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an "
 940                         "os_once_t");
 941 }
 942
 943 OS_NOINLINE OS_NORETURN OS_COLD
 944 void
 945 _os_once_gate_unowned_abort(os_lock_owner_t owner)
 946 {
 947         __LIBPLATFORM_CLIENT_CRASH__(owner, "Unlock of an os_once_t not "
 948                         "owned by current thread");
 949 }
 950
 951 OS_NOINLINE OS_NORETURN OS_COLD
 952 void
 953 _os_once_gate_corruption_abort(os_ulock_value_t current)
 954 {
 955         __LIBPLATFORM_CLIENT_CRASH__(current, "os_once_t is corrupt");
 956 }
 957
 958
 959 #if OS_ONCE_USE_QUIESCENT_COUNTER
 960 #define OS_ONCE_MAKE_GEN(gen)  (((gen) << 2) + OS_ULOCK_NOWAITERS_BIT)
 961 #define OS_ONCE_IS_GEN(gen)    (((gen) & 3) == OS_ULOCK_NOWAITERS_BIT)
 962
 963 // the _COMM_PAGE_CPU_QUIESCENT_COUNTER value is incremented every time
 964 // all CPUs have performed a context switch.
 965 //
 966 // To make sure all CPUs context switched at least once since `gen`,
 967 // we need to observe 4 increments, see libdispatch/src/shims/lock.h
 968 #define OS_ONCE_GEN_SAFE_DELTA  (4 << 2)
 969
 970 OS_ALWAYS_INLINE
 971 static inline uintptr_t
 972 _os_once_generation(void)
 973 {
 974         uintptr_t value = *(volatile uintptr_t *)_COMM_PAGE_CPU_QUIESCENT_COUNTER;
 975         return OS_ONCE_MAKE_GEN(value);
 976 }
 977
 978 OS_ALWAYS_INLINE
 979 static inline uintptr_t
 980 _os_once_mark_quiescing(os_once_gate_t og)
 981 {
 982         return os_atomic_xchg(&og->ogo_once, _os_once_generation(), release);
 983 }
 984
 985 OS_ALWAYS_INLINE
 986 static void
 987 _os_once_mark_done_if_quiesced(os_once_gate_t og, uintptr_t gen)
 988 {
 989         if (_os_once_generation() - gen >= OS_ONCE_GEN_SAFE_DELTA) {
 990                 os_atomic_store(&og->ogo_once, OS_ONCE_DONE, relaxed);
 991         }
 992 }
 993 #else
 994 OS_ALWAYS_INLINE
 995 static inline uintptr_t
 996 _os_once_mark_done(os_once_gate_t og)
 997 {
 998         return os_atomic_xchg(&og->ogo_once, OS_ONCE_DONE, release);
 999 }
1000 #endif
1001
1002 OS_NOINLINE
1003 static void
1004 _os_once_gate_broadcast(os_once_gate_t og, os_ulock_value_t current,
1005                 os_lock_owner_t self)
1006 {
1007         if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, 0))) {
1008                 return _os_once_gate_unowned_abort(OS_ULOCK_OWNER(current));
1009         }
1010         if (current & OS_ULOCK_NOWAITERS_BIT) {
1011                 __LIBPLATFORM_INTERNAL_CRASH__(current, "unlock_slow with no waiters");
1012         }
1013         for (;;) {
1014                 int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO | ULF_WAKE_ALL,
1015                                 &og->ogo_lock, 0);
1016                 if (unlikely(ret < 0)) {
1017                         switch (-ret) {
1018                         case EINTR:
1019                                 continue;
1020                         case ENOENT:
1021                                 break;
1022                         default:
1023                                 __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wake failure");
1024                         }
1025                 }
1026                 break;
1027         }
1028 }
1029
1030 OS_NOINLINE
1031 static void
1032 _os_once_callout(os_once_gate_t og, void *ctxt, os_function_t func,
1033                 os_lock_owner_t self)
1034 {
1035         uintptr_t v;
1036
1037         func(ctxt);
1038
1039 #if OS_ONCE_USE_QUIESCENT_COUNTER
1040         v = _os_once_mark_quiescing(og);
1041 #else
1042         v = _os_once_mark_done(og);
1043 #endif
1044         if (likely((os_ulock_value_t)v == self)) return;
1045         _os_once_gate_broadcast(og, (os_ulock_value_t)v, self);
1046 }
1047
1048 OS_NOINLINE
1049 static void
1050 _os_once_gate_wait(os_once_gate_t og, void *ctxt, os_function_t func,
1051                 os_lock_owner_t self)
1052 {
1053         uintptr_t old, new;
1054
1055         for (;;) {
1056                 os_atomic_rmw_loop(&og->ogo_once, old, new, relaxed, {
1057                         if (old == OS_ONCE_DONE) {
1058                                 os_atomic_rmw_loop_give_up(return);
1059 #if OS_ONCE_USE_QUIESCENT_COUNTER
1060                         } else if (OS_ONCE_IS_GEN(old)) {
1061                                 os_atomic_rmw_loop_give_up({
1062                                         os_atomic_thread_fence(acquire);
1063                                         return _os_once_mark_done_if_quiesced(og, old);
1064                                 });
1065 #endif
1066                         } else if (old == OS_ONCE_INIT) {
1067                                 // __os_once_reset was used, try to become the new initializer
1068                                 new = (uintptr_t)self;
1069                         } else {
1070                                 new = old & ~(uintptr_t)OS_ULOCK_NOWAITERS_BIT;
1071                                 if (new == old) os_atomic_rmw_loop_give_up(break);
1072                         }
1073                 });
1074                 if (old == OS_ONCE_INIT) {
1075                         // see comment in _os_once, pairs with the release barrier
1076                         // in __os_once_reset()
1077                         os_atomic_thread_fence(acquire);
1078                         return _os_once_callout(og, ctxt, func, self);
1079                 }
1080                 if (unlikely(OS_ULOCK_IS_OWNER((os_lock_owner_t)old, self, 0))) {
1081                         return _os_once_gate_recursive_abort(self);
1082                 }
1083                 int ret = __ulock_wait(UL_UNFAIR_LOCK | ULF_NO_ERRNO,
1084                                 &og->ogo_lock, (os_ulock_value_t)new, 0);
1085                 if (unlikely(ret < 0)) {
1086                         switch (-ret) {
1087                         case EINTR:
1088                         case EFAULT:
1089                                 continue;
1090                         case EOWNERDEAD:
1091                                 _os_once_gate_corruption_abort((os_lock_owner_t)old);
1092                                 break;
1093                         default:
1094                                 __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wait failure");
1095                         }
1096                 }
1097         }
1098 }
1099
1100 // Atomically resets the once value to zero and then signals all
1101 // pending waiters to return from their __ulock_wait()
1102 void
1103 __os_once_reset(os_once_t *val)
1104 {
1105         os_once_gate_t og = (os_once_gate_t)val;
1106         os_lock_owner_t self = _os_lock_owner_get_self();
1107         uintptr_t v;
1108
1109         v = os_atomic_xchg(&og->ogo_once, OS_ONCE_INIT, release);
1110         if (likely((os_ulock_value_t)v == self)) return;
1111         return _os_once_gate_broadcast(og, (os_ulock_value_t)v, self);
1112 }
1113
1114 void
1115 _os_once(os_once_t *val, void *ctxt, os_function_t func)
1116 {
1117         os_once_gate_t og = (os_once_gate_t)val;
1118         os_lock_owner_t self;
1119         uintptr_t v;
1120
1121 #if OS_ONCE_USE_QUIESCENT_COUNTER
1122         v = os_atomic_load(&og->ogo_once, acquire);
1123         if (likely(OS_ONCE_IS_GEN(v))) {
1124                 return _os_once_mark_done_if_quiesced(og, v);
1125         }
1126 #endif
1127
1128         self = _os_lock_owner_get_self();
1129         v = (uintptr_t)self;
1130
1131         // The acquire barrier pairs with the release in __os_once_reset()
1132         // for cases when a previous initializer failed.
1133         if (likely(os_atomic_cmpxchg(&og->ogo_once, OS_ONCE_INIT, v, acquire))) {
1134                 return _os_once_callout(og, ctxt, func, self);
1135         }
1136         return _os_once_gate_wait(og, ctxt, func, self);
1137 }
1138
1139
1140 #pragma mark -
1141 #pragma mark os_lock_eliding_t
1142
1143 #if !TARGET_OS_IPHONE
1144
1145 #define _os_lock_eliding_t _os_lock_spin_t
1146 #define _os_lock_eliding_lock _os_lock_spin_lock
1147 #define _os_lock_eliding_trylock _os_lock_spin_trylock
1148 #define _os_lock_eliding_unlock _os_lock_spin_unlock
1149 OS_LOCK_METHODS_DECL(eliding);
1150 OS_LOCK_TYPE_INSTANCE(eliding);
1151
1152 #pragma mark -
1153 #pragma mark os_lock_transactional_t
1154
1155 OS_LOCK_STRUCT_DECL_INTERNAL(transactional,
1156         uintptr_t volatile osl_lock;
1157 );
1158
1159 #define _os_lock_transactional_t _os_lock_eliding_t
1160 #define _os_lock_transactional_lock _os_lock_eliding_lock
1161 #define _os_lock_transactional_trylock _os_lock_eliding_trylock
1162 #define _os_lock_transactional_unlock _os_lock_eliding_unlock
1163 OS_LOCK_METHODS_DECL(transactional);
1164 OS_LOCK_TYPE_INSTANCE(transactional);
1165
1166 #endif // !TARGET_OS_IPHONE