runtime/objc-os.h

   1 /*
   2  * Copyright (c) 2007 Apple Inc.  All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /***********************************************************************
  25 * objc-os.h
  26 * OS portability layer.
  27 **********************************************************************/
  28
  29 #ifndef _OBJC_OS_H
  30 #define _OBJC_OS_H
  31
  32 #include <TargetConditionals.h>
  33 #include "objc-config.h"
  34
  35 #ifdef __LP64__
  36 #   define WORD_SHIFT 3UL
  37 #   define WORD_MASK 7UL
  38 #   define WORD_BITS 64
  39 #else
  40 #   define WORD_SHIFT 2UL
  41 #   define WORD_MASK 3UL
  42 #   define WORD_BITS 32
  43 #endif
  44
  45 static inline uint32_t word_align(uint32_t x) {
  46     return (x + WORD_MASK) & ~WORD_MASK;
  47 }
  48 static inline size_t word_align(size_t x) {
  49     return (x + WORD_MASK) & ~WORD_MASK;
  50 }
  51
  52
  53 // Mix-in for classes that must not be copied.
  54 class nocopy_t {
  55   private:
  56     nocopy_t(const nocopy_t&) = delete;
  57     const nocopy_t& operator=(const nocopy_t&) = delete;
  58   protected:
  59     constexpr nocopy_t() = default;
  60     ~nocopy_t() = default;
  61 };
  62
  63
  64 #if TARGET_OS_MAC
  65
  66 #   define OS_UNFAIR_LOCK_INLINE 1
  67
  68 #   ifndef __STDC_LIMIT_MACROS
  69 #       define __STDC_LIMIT_MACROS
  70 #   endif
  71
  72 #   include <stdio.h>
  73 #   include <stdlib.h>
  74 #   include <stdint.h>
  75 #   include <stdarg.h>
  76 #   include <string.h>
  77 #   include <ctype.h>
  78 #   include <errno.h>
  79 #   include <dlfcn.h>
  80 #   include <fcntl.h>
  81 #   include <assert.h>
  82 #   include <limits.h>
  83 #   include <syslog.h>
  84 #   include <unistd.h>
  85 #   include <pthread.h>
  86 #   include <crt_externs.h>
  87 #   undef check
  88 #   include <Availability.h>
  89 #   include <TargetConditionals.h>
  90 #   include <sys/mman.h>
  91 #   include <sys/time.h>
  92 #   include <sys/stat.h>
  93 #   include <sys/param.h>
  94 #   include <sys/reason.h>
  95 #   include <mach/mach.h>
  96 #   include <mach/vm_param.h>
  97 #   include <mach/mach_time.h>
  98 #   include <mach-o/dyld.h>
  99 #   include <mach-o/ldsyms.h>
 100 #   include <mach-o/loader.h>
 101 #   include <mach-o/getsect.h>
 102 #   include <mach-o/dyld_priv.h>
 103 #   include <malloc/malloc.h>
 104 #   include <os/lock_private.h>
 105 #   include <libkern/OSAtomic.h>
 106 #   include <libkern/OSCacheControl.h>
 107 #   include <System/pthread_machdep.h>
 108 #   include "objc-probes.h"  // generated dtrace probe definitions.
 109
 110 // Some libc functions call objc_msgSend()
 111 // so we can't use them without deadlocks.
 112 void syslog(int, const char *, ...) UNAVAILABLE_ATTRIBUTE;
 113 void vsyslog(int, const char *, va_list) UNAVAILABLE_ATTRIBUTE;
 114
 115
 116 #define ALWAYS_INLINE inline __attribute__((always_inline))
 117 #define NEVER_INLINE inline __attribute__((noinline))
 118
 119 #define fastpath(x) (__builtin_expect(bool(x), 1))
 120 #define slowpath(x) (__builtin_expect(bool(x), 0))
 121
 122
 123 static ALWAYS_INLINE uintptr_t
 124 addc(uintptr_t lhs, uintptr_t rhs, uintptr_t carryin, uintptr_t *carryout)
 125 {
 126     return __builtin_addcl(lhs, rhs, carryin, carryout);
 127 }
 128
 129 static ALWAYS_INLINE uintptr_t
 130 subc(uintptr_t lhs, uintptr_t rhs, uintptr_t carryin, uintptr_t *carryout)
 131 {
 132     return __builtin_subcl(lhs, rhs, carryin, carryout);
 133 }
 134
 135 #if __arm64__ && !__arm64e__
 136
 137 static ALWAYS_INLINE
 138 uintptr_t
 139 LoadExclusive(uintptr_t *src)
 140 {
 141     return __builtin_arm_ldrex(src);
 142 }
 143
 144 static ALWAYS_INLINE
 145 bool
 146 StoreExclusive(uintptr_t *dst, uintptr_t oldvalue __unused, uintptr_t value)
 147 {
 148     return !__builtin_arm_strex(value, dst);
 149 }
 150
 151
 152 static ALWAYS_INLINE
 153 bool
 154 StoreReleaseExclusive(uintptr_t *dst, uintptr_t oldvalue __unused, uintptr_t value)
 155 {
 156     return !__builtin_arm_stlex(value, dst);
 157 }
 158
 159 static ALWAYS_INLINE
 160 void
 161 ClearExclusive(uintptr_t *dst __unused)
 162 {
 163     __builtin_arm_clrex();
 164 }
 165
 166 #else
 167
 168 static ALWAYS_INLINE
 169 uintptr_t
 170 LoadExclusive(uintptr_t *src)
 171 {
 172     return __c11_atomic_load((_Atomic(uintptr_t) *)src, __ATOMIC_RELAXED);
 173 }
 174
 175 static ALWAYS_INLINE
 176 bool
 177 StoreExclusive(uintptr_t *dst, uintptr_t oldvalue, uintptr_t value)
 178 {
 179     return __c11_atomic_compare_exchange_weak((_Atomic(uintptr_t) *)dst, &oldvalue, value, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
 180 }
 181
 182
 183 static ALWAYS_INLINE
 184 bool
 185 StoreReleaseExclusive(uintptr_t *dst, uintptr_t oldvalue, uintptr_t value)
 186 {
 187     return __c11_atomic_compare_exchange_weak((_Atomic(uintptr_t) *)dst, &oldvalue, value, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 188 }
 189
 190 static ALWAYS_INLINE
 191 void
 192 ClearExclusive(uintptr_t *dst __unused)
 193 {
 194 }
 195
 196 #endif
 197
 198
 199 #if !TARGET_OS_IPHONE
 200 #   include <CrashReporterClient.h>
 201 #else
 202     // CrashReporterClient not yet available on iOS
 203     __BEGIN_DECLS
 204     extern const char *CRSetCrashLogMessage(const char *msg);
 205     extern const char *CRGetCrashLogMessage(void);
 206     __END_DECLS
 207 #endif
 208
 209 #   if __cplusplus
 210 #       include <vector>
 211 #       include <algorithm>
 212 #       include <functional>
 213         using namespace std;
 214 #   endif
 215
 216 #   define PRIVATE_EXTERN __attribute__((visibility("hidden")))
 217 #   undef __private_extern__
 218 #   define __private_extern__ use_PRIVATE_EXTERN_instead
 219 #   undef private_extern
 220 #   define private_extern use_PRIVATE_EXTERN_instead
 221
 222 /* Use this for functions that are intended to be breakpoint hooks.
 223    If you do not, the compiler may optimize them away.
 224    BREAKPOINT_FUNCTION( void stop_on_error(void) ); */
 225 #   define BREAKPOINT_FUNCTION(prototype)                             \
 226     OBJC_EXTERN __attribute__((noinline, used, visibility("hidden"))) \
 227     prototype { asm(""); }
 228
 229 #elif TARGET_OS_WIN32
 230
 231 #   define WINVER 0x0501                // target Windows XP and later
 232 #   define _WIN32_WINNT 0x0501  // target Windows XP and later
 233 #   define WIN32_LEAN_AND_MEAN
 234     // hack: windef.h typedefs BOOL as int
 235 #   define BOOL WINBOOL
 236 #   include <windows.h>
 237 #   undef BOOL
 238
 239 #   include <stdio.h>
 240 #   include <stdlib.h>
 241 #   include <stdint.h>
 242 #   include <stdarg.h>
 243 #   include <string.h>
 244 #   include <assert.h>
 245 #   include <malloc.h>
 246 #   include <Availability.h>
 247
 248 #   if __cplusplus
 249 #       include <vector>
 250 #       include <algorithm>
 251 #       include <functional>
 252         using namespace std;
 253 #       define __BEGIN_DECLS extern "C" {
 254 #       define __END_DECLS   }
 255 #   else
 256 #       define __BEGIN_DECLS /*empty*/
 257 #       define __END_DECLS   /*empty*/
 258 #   endif
 259
 260 #   define PRIVATE_EXTERN
 261 #   define __attribute__(x)
 262 #   define inline __inline
 263
 264 /* Use this for functions that are intended to be breakpoint hooks.
 265    If you do not, the compiler may optimize them away.
 266    BREAKPOINT_FUNCTION( void MyBreakpointFunction(void) ); */
 267 #   define BREAKPOINT_FUNCTION(prototype) \
 268     __declspec(noinline) prototype { __asm { } }
 269
 270 /* stub out dtrace probes */
 271 #   define OBJC_RUNTIME_OBJC_EXCEPTION_RETHROW() do {} while(0)
 272 #   define OBJC_RUNTIME_OBJC_EXCEPTION_THROW(arg0) do {} while(0)
 273
 274 #else
 275 #   error unknown OS
 276 #endif
 277
 278
 279 #include <objc/objc.h>
 280 #include <objc/objc-api.h>
 281
 282 extern void _objc_fatal(const char *fmt, ...)
 283     __attribute__((noreturn, format (printf, 1, 2)));
 284 extern void _objc_fatal_with_reason(uint64_t reason, uint64_t flags,
 285                                     const char *fmt, ...)
 286     __attribute__((noreturn, format (printf, 3, 4)));
 287
 288 #define INIT_ONCE_PTR(var, create, delete)                              \
 289     do {                                                                \
 290         if (var) break;                                                 \
 291         typeof(var) v = create;                                         \
 292         while (!var) {                                                  \
 293             if (OSAtomicCompareAndSwapPtrBarrier(0, (void*)v, (void**)&var)){ \
 294                 goto done;                                              \
 295             }                                                           \
 296         }                                                               \
 297         delete;                                                         \
 298     done:;                                                              \
 299     } while (0)
 300
 301 #define INIT_ONCE_32(var, create, delete)                               \
 302     do {                                                                \
 303         if (var) break;                                                 \
 304         typeof(var) v = create;                                         \
 305         while (!var) {                                                  \
 306             if (OSAtomicCompareAndSwap32Barrier(0, v, (volatile int32_t *)&var)) { \
 307                 goto done;                                              \
 308             }                                                           \
 309         }                                                               \
 310         delete;                                                         \
 311     done:;                                                              \
 312     } while (0)
 313
 314
 315 // Thread keys reserved by libc for our use.
 316 #if defined(__PTK_FRAMEWORK_OBJC_KEY0)
 317 #   define SUPPORT_DIRECT_THREAD_KEYS 1
 318 #   define TLS_DIRECT_KEY        ((tls_key_t)__PTK_FRAMEWORK_OBJC_KEY0)
 319 #   define SYNC_DATA_DIRECT_KEY  ((tls_key_t)__PTK_FRAMEWORK_OBJC_KEY1)
 320 #   define SYNC_COUNT_DIRECT_KEY ((tls_key_t)__PTK_FRAMEWORK_OBJC_KEY2)
 321 #   define AUTORELEASE_POOL_KEY  ((tls_key_t)__PTK_FRAMEWORK_OBJC_KEY3)
 322 # if SUPPORT_RETURN_AUTORELEASE
 323 #   define RETURN_DISPOSITION_KEY ((tls_key_t)__PTK_FRAMEWORK_OBJC_KEY4)
 324 # endif
 325 #else
 326 #   define SUPPORT_DIRECT_THREAD_KEYS 0
 327 #endif
 328
 329
 330 #if TARGET_OS_WIN32
 331
 332 // Compiler compatibility
 333
 334 // OS compatibility
 335
 336 #define strdup _strdup
 337
 338 #define issetugid() 0
 339
 340 #define MIN(x, y) ((x) < (y) ? (x) : (y))
 341
 342 static __inline void bcopy(const void *src, void *dst, size_t size) { memcpy(dst, src, size); }
 343 static __inline void bzero(void *dst, size_t size) { memset(dst, 0, size); }
 344
 345 int asprintf(char **dstp, const char *format, ...);
 346
 347 typedef void * malloc_zone_t;
 348
 349 static __inline malloc_zone_t malloc_default_zone(void) { return (malloc_zone_t)-1; }
 350 static __inline void *malloc_zone_malloc(malloc_zone_t z, size_t size) { return malloc(size); }
 351 static __inline void *malloc_zone_calloc(malloc_zone_t z, size_t size, size_t count) { return calloc(size, count); }
 352 static __inline void *malloc_zone_realloc(malloc_zone_t z, void *p, size_t size) { return realloc(p, size); }
 353 static __inline void malloc_zone_free(malloc_zone_t z, void *p) { free(p); }
 354 static __inline malloc_zone_t malloc_zone_from_ptr(const void *p) { return (malloc_zone_t)-1; }
 355 static __inline size_t malloc_size(const void *p) { return _msize((void*)p); /* fixme invalid pointer check? */ }
 356
 357
 358 // OSAtomic
 359
 360 static __inline BOOL OSAtomicCompareAndSwapLong(long oldl, long newl, long volatile *dst)
 361 {
 362     // fixme barrier is overkill
 363     long original = InterlockedCompareExchange(dst, newl, oldl);
 364     return (original == oldl);
 365 }
 366
 367 static __inline BOOL OSAtomicCompareAndSwapPtrBarrier(void *oldp, void *newp, void * volatile *dst)
 368 {
 369     void *original = InterlockedCompareExchangePointer(dst, newp, oldp);
 370     return (original == oldp);
 371 }
 372
 373 static __inline BOOL OSAtomicCompareAndSwap32Barrier(int32_t oldl, int32_t newl, int32_t volatile *dst)
 374 {
 375     long original = InterlockedCompareExchange((volatile long *)dst, newl, oldl);
 376     return (original == oldl);
 377 }
 378
 379 static __inline int32_t OSAtomicDecrement32Barrier(volatile int32_t *dst)
 380 {
 381     return InterlockedDecrement((volatile long *)dst);
 382 }
 383
 384 static __inline int32_t OSAtomicIncrement32Barrier(volatile int32_t *dst)
 385 {
 386     return InterlockedIncrement((volatile long *)dst);
 387 }
 388
 389
 390 // Internal data types
 391
 392 typedef DWORD objc_thread_t;  // thread ID
 393 static __inline int thread_equal(objc_thread_t t1, objc_thread_t t2) {
 394     return t1 == t2;
 395 }
 396 static __inline objc_thread_t thread_self(void) {
 397     return GetCurrentThreadId();
 398 }
 399
 400 typedef struct {
 401     DWORD key;
 402     void (*dtor)(void *);
 403 } tls_key_t;
 404 static __inline tls_key_t tls_create(void (*dtor)(void*)) {
 405     // fixme need dtor registry for DllMain to call on thread detach
 406     tls_key_t k;
 407     k.key = TlsAlloc();
 408     k.dtor = dtor;
 409     return k;
 410 }
 411 static __inline void *tls_get(tls_key_t k) {
 412     return TlsGetValue(k.key);
 413 }
 414 static __inline void tls_set(tls_key_t k, void *value) {
 415     TlsSetValue(k.key, value);
 416 }
 417
 418 typedef struct {
 419     CRITICAL_SECTION *lock;
 420 } mutex_t;
 421 #define MUTEX_INITIALIZER {0};
 422 extern void mutex_init(mutex_t *m);
 423 static __inline int _mutex_lock_nodebug(mutex_t *m) {
 424     // fixme error check
 425     if (!m->lock) {
 426         mutex_init(m);
 427     }
 428     EnterCriticalSection(m->lock);
 429     return 0;
 430 }
 431 static __inline bool _mutex_try_lock_nodebug(mutex_t *m) {
 432     // fixme error check
 433     if (!m->lock) {
 434         mutex_init(m);
 435     }
 436     return TryEnterCriticalSection(m->lock);
 437 }
 438 static __inline int _mutex_unlock_nodebug(mutex_t *m) {
 439     // fixme error check
 440     LeaveCriticalSection(m->lock);
 441     return 0;
 442 }
 443
 444
 445 typedef mutex_t spinlock_t;
 446 #define spinlock_lock(l) mutex_lock(l)
 447 #define spinlock_unlock(l) mutex_unlock(l)
 448 #define SPINLOCK_INITIALIZER MUTEX_INITIALIZER
 449
 450
 451 typedef struct {
 452     HANDLE mutex;
 453 } recursive_mutex_t;
 454 #define RECURSIVE_MUTEX_INITIALIZER {0};
 455 #define RECURSIVE_MUTEX_NOT_LOCKED 1
 456 extern void recursive_mutex_init(recursive_mutex_t *m);
 457 static __inline int _recursive_mutex_lock_nodebug(recursive_mutex_t *m) {
 458     assert(m->mutex);
 459     return WaitForSingleObject(m->mutex, INFINITE);
 460 }
 461 static __inline bool _recursive_mutex_try_lock_nodebug(recursive_mutex_t *m) {
 462     assert(m->mutex);
 463     return (WAIT_OBJECT_0 == WaitForSingleObject(m->mutex, 0));
 464 }
 465 static __inline int _recursive_mutex_unlock_nodebug(recursive_mutex_t *m) {
 466     assert(m->mutex);
 467     return ReleaseMutex(m->mutex) ? 0 : RECURSIVE_MUTEX_NOT_LOCKED;
 468 }
 469
 470
 471 /*
 472 typedef HANDLE mutex_t;
 473 static inline void mutex_init(HANDLE *m) { *m = CreateMutex(NULL, FALSE, NULL); }
 474 static inline void _mutex_lock(mutex_t *m) { WaitForSingleObject(*m, INFINITE); }
 475 static inline bool mutex_try_lock(mutex_t *m) { return WaitForSingleObject(*m, 0) == WAIT_OBJECT_0; }
 476 static inline void _mutex_unlock(mutex_t *m) { ReleaseMutex(*m); }
 477 */
 478
 479 // based on http://www.cs.wustl.edu/~schmidt/win32-cv-1.html
 480 // Vista-only CONDITION_VARIABLE would be better
 481 typedef struct {
 482     HANDLE mutex;
 483     HANDLE waiters;      // semaphore for those in cond_wait()
 484     HANDLE waitersDone;  // auto-reset event after everyone gets a broadcast
 485     CRITICAL_SECTION waitCountLock;  // guards waitCount and didBroadcast
 486     unsigned int waitCount;
 487     int didBroadcast;
 488 } monitor_t;
 489 #define MONITOR_INITIALIZER { 0 }
 490 #define MONITOR_NOT_ENTERED 1
 491 extern int monitor_init(monitor_t *c);
 492
 493 static inline int _monitor_enter_nodebug(monitor_t *c) {
 494     if (!c->mutex) {
 495         int err = monitor_init(c);
 496         if (err) return err;
 497     }
 498     return WaitForSingleObject(c->mutex, INFINITE);
 499 }
 500 static inline int _monitor_leave_nodebug(monitor_t *c) {
 501     if (!ReleaseMutex(c->mutex)) return MONITOR_NOT_ENTERED;
 502     else return 0;
 503 }
 504 static inline int _monitor_wait_nodebug(monitor_t *c) {
 505     int last;
 506     EnterCriticalSection(&c->waitCountLock);
 507     c->waitCount++;
 508     LeaveCriticalSection(&c->waitCountLock);
 509
 510     SignalObjectAndWait(c->mutex, c->waiters, INFINITE, FALSE);
 511
 512     EnterCriticalSection(&c->waitCountLock);
 513     c->waitCount--;
 514     last = c->didBroadcast  &&  c->waitCount == 0;
 515     LeaveCriticalSection(&c->waitCountLock);
 516
 517     if (last) {
 518         // tell broadcaster that all waiters have awoken
 519         SignalObjectAndWait(c->waitersDone, c->mutex, INFINITE, FALSE);
 520     } else {
 521         WaitForSingleObject(c->mutex, INFINITE);
 522     }
 523
 524     // fixme error checking
 525     return 0;
 526 }
 527 static inline int monitor_notify(monitor_t *c) {
 528     int haveWaiters;
 529
 530     EnterCriticalSection(&c->waitCountLock);
 531     haveWaiters = c->waitCount > 0;
 532     LeaveCriticalSection(&c->waitCountLock);
 533
 534     if (haveWaiters) {
 535         ReleaseSemaphore(c->waiters, 1, 0);
 536     }
 537
 538     // fixme error checking
 539     return 0;
 540 }
 541 static inline int monitor_notifyAll(monitor_t *c) {
 542     EnterCriticalSection(&c->waitCountLock);
 543     if (c->waitCount == 0) {
 544         LeaveCriticalSection(&c->waitCountLock);
 545         return 0;
 546     }
 547     c->didBroadcast = 1;
 548     ReleaseSemaphore(c->waiters, c->waitCount, 0);
 549     LeaveCriticalSection(&c->waitCountLock);
 550
 551     // fairness: wait for everyone to move from waiters to mutex
 552     WaitForSingleObject(c->waitersDone, INFINITE);
 553     // not under waitCountLock, but still under mutex
 554     c->didBroadcast = 0;
 555
 556     // fixme error checking
 557     return 0;
 558 }
 559
 560
 561 typedef IMAGE_DOS_HEADER headerType;
 562 // fixme YES bundle? NO bundle? sometimes?
 563 #define headerIsBundle(hi) YES
 564 OBJC_EXTERN IMAGE_DOS_HEADER __ImageBase;
 565 #define libobjc_header ((headerType *)&__ImageBase)
 566
 567 // Prototypes
 568
 569
 570 #elif TARGET_OS_MAC
 571
 572
 573 // OS headers
 574 #include <mach-o/loader.h>
 575 #ifndef __LP64__
 576 #   define SEGMENT_CMD LC_SEGMENT
 577 #else
 578 #   define SEGMENT_CMD LC_SEGMENT_64
 579 #endif
 580
 581 #ifndef VM_MEMORY_OBJC_DISPATCHERS
 582 #   define VM_MEMORY_OBJC_DISPATCHERS 0
 583 #endif
 584
 585
 586 // Compiler compatibility
 587
 588 // OS compatibility
 589
 590 static inline uint64_t nanoseconds() {
 591     return clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW);
 592 }
 593
 594 // Internal data types
 595
 596 typedef pthread_t objc_thread_t;
 597
 598 static __inline int thread_equal(objc_thread_t t1, objc_thread_t t2) {
 599     return pthread_equal(t1, t2);
 600 }
 601 static __inline objc_thread_t thread_self(void) {
 602     return pthread_self();
 603 }
 604
 605
 606 typedef pthread_key_t tls_key_t;
 607
 608 static inline tls_key_t tls_create(void (*dtor)(void*)) {
 609     tls_key_t k;
 610     pthread_key_create(&k, dtor);
 611     return k;
 612 }
 613 static inline void *tls_get(tls_key_t k) {
 614     return pthread_getspecific(k);
 615 }
 616 static inline void tls_set(tls_key_t k, void *value) {
 617     pthread_setspecific(k, value);
 618 }
 619
 620 #if SUPPORT_DIRECT_THREAD_KEYS
 621
 622 #if DEBUG
 623 static bool is_valid_direct_key(tls_key_t k) {
 624     return (   k == SYNC_DATA_DIRECT_KEY
 625             || k == SYNC_COUNT_DIRECT_KEY
 626             || k == AUTORELEASE_POOL_KEY
 627 #   if SUPPORT_RETURN_AUTORELEASE
 628             || k == RETURN_DISPOSITION_KEY
 629 #   endif
 630                );
 631 }
 632 #endif
 633
 634 static inline void *tls_get_direct(tls_key_t k)
 635 {
 636     assert(is_valid_direct_key(k));
 637
 638     if (_pthread_has_direct_tsd()) {
 639         return _pthread_getspecific_direct(k);
 640     } else {
 641         return pthread_getspecific(k);
 642     }
 643 }
 644 static inline void tls_set_direct(tls_key_t k, void *value)
 645 {
 646     assert(is_valid_direct_key(k));
 647
 648     if (_pthread_has_direct_tsd()) {
 649         _pthread_setspecific_direct(k, value);
 650     } else {
 651         pthread_setspecific(k, value);
 652     }
 653 }
 654
 655 // SUPPORT_DIRECT_THREAD_KEYS
 656 #endif
 657
 658
 659 static inline pthread_t pthread_self_direct()
 660 {
 661     return (pthread_t)
 662         _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 663 }
 664
 665 static inline mach_port_t mach_thread_self_direct()
 666 {
 667     return (mach_port_t)(uintptr_t)
 668         _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF);
 669 }
 670
 671
 672 template <bool Debug> class mutex_tt;
 673 template <bool Debug> class monitor_tt;
 674 template <bool Debug> class recursive_mutex_tt;
 675
 676 #if DEBUG
 677 #   define LOCKDEBUG 1
 678 #else
 679 #   define LOCKDEBUG 0
 680 #endif
 681
 682 using spinlock_t = mutex_tt<LOCKDEBUG>;
 683 using mutex_t = mutex_tt<LOCKDEBUG>;
 684 using monitor_t = monitor_tt<LOCKDEBUG>;
 685 using recursive_mutex_t = recursive_mutex_tt<LOCKDEBUG>;
 686
 687 // Use fork_unsafe_lock to get a lock that isn't
 688 // acquired and released around fork().
 689 // All fork-safe locks are checked in debug builds.
 690 struct fork_unsafe_lock_t {
 691     constexpr fork_unsafe_lock_t() = default;
 692 };
 693 extern const fork_unsafe_lock_t fork_unsafe_lock;
 694
 695 #include "objc-lockdebug.h"
 696
 697 template <bool Debug>
 698 class mutex_tt : nocopy_t {
 699     os_unfair_lock mLock;
 700  public:
 701     constexpr mutex_tt() : mLock(OS_UNFAIR_LOCK_INIT) {
 702         lockdebug_remember_mutex(this);
 703     }
 704
 705     constexpr mutex_tt(const fork_unsafe_lock_t unsafe) : mLock(OS_UNFAIR_LOCK_INIT) { }
 706
 707     void lock() {
 708         lockdebug_mutex_lock(this);
 709
 710         os_unfair_lock_lock_with_options_inline
 711             (&mLock, OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION);
 712     }
 713
 714     void unlock() {
 715         lockdebug_mutex_unlock(this);
 716
 717         os_unfair_lock_unlock_inline(&mLock);
 718     }
 719
 720     void forceReset() {
 721         lockdebug_mutex_unlock(this);
 722
 723         bzero(&mLock, sizeof(mLock));
 724         mLock = os_unfair_lock OS_UNFAIR_LOCK_INIT;
 725     }
 726
 727     void assertLocked() {
 728         lockdebug_mutex_assert_locked(this);
 729     }
 730
 731     void assertUnlocked() {
 732         lockdebug_mutex_assert_unlocked(this);
 733     }
 734
 735
 736     // Address-ordered lock discipline for a pair of locks.
 737
 738     static void lockTwo(mutex_tt *lock1, mutex_tt *lock2) {
 739         if (lock1 < lock2) {
 740             lock1->lock();
 741             lock2->lock();
 742         } else {
 743             lock2->lock();
 744             if (lock2 != lock1) lock1->lock();
 745         }
 746     }
 747
 748     static void unlockTwo(mutex_tt *lock1, mutex_tt *lock2) {
 749         lock1->unlock();
 750         if (lock2 != lock1) lock2->unlock();
 751     }
 752
 753     // Scoped lock and unlock
 754     class locker : nocopy_t {
 755         mutex_tt& lock;
 756     public:
 757         locker(mutex_tt& newLock)
 758             : lock(newLock) { lock.lock(); }
 759         ~locker() { lock.unlock(); }
 760     };
 761
 762     // Either scoped lock and unlock, or NOP.
 763     class conditional_locker : nocopy_t {
 764         mutex_tt& lock;
 765         bool didLock;
 766     public:
 767         conditional_locker(mutex_tt& newLock, bool shouldLock)
 768             : lock(newLock), didLock(shouldLock)
 769         {
 770             if (shouldLock) lock.lock();
 771         }
 772         ~conditional_locker() { if (didLock) lock.unlock(); }
 773     };
 774 };
 775
 776 using mutex_locker_t = mutex_tt<LOCKDEBUG>::locker;
 777 using conditional_mutex_locker_t = mutex_tt<LOCKDEBUG>::conditional_locker;
 778
 779
 780 template <bool Debug>
 781 class recursive_mutex_tt : nocopy_t {
 782     os_unfair_recursive_lock mLock;
 783
 784   public:
 785     constexpr recursive_mutex_tt() : mLock(OS_UNFAIR_RECURSIVE_LOCK_INIT) {
 786         lockdebug_remember_recursive_mutex(this);
 787     }
 788
 789     constexpr recursive_mutex_tt(const fork_unsafe_lock_t unsafe)
 790         : mLock(OS_UNFAIR_RECURSIVE_LOCK_INIT)
 791     { }
 792
 793     void lock()
 794     {
 795         lockdebug_recursive_mutex_lock(this);
 796         os_unfair_recursive_lock_lock(&mLock);
 797     }
 798
 799     void unlock()
 800     {
 801         lockdebug_recursive_mutex_unlock(this);
 802
 803         os_unfair_recursive_lock_unlock(&mLock);
 804     }
 805
 806     void forceReset()
 807     {
 808         lockdebug_recursive_mutex_unlock(this);
 809
 810         bzero(&mLock, sizeof(mLock));
 811         mLock = os_unfair_recursive_lock OS_UNFAIR_RECURSIVE_LOCK_INIT;
 812     }
 813
 814     bool tryUnlock()
 815     {
 816         if (os_unfair_recursive_lock_tryunlock4objc(&mLock)) {
 817             lockdebug_recursive_mutex_unlock(this);
 818             return true;
 819         }
 820         return false;
 821     }
 822
 823     void assertLocked() {
 824         lockdebug_recursive_mutex_assert_locked(this);
 825     }
 826
 827     void assertUnlocked() {
 828         lockdebug_recursive_mutex_assert_unlocked(this);
 829     }
 830 };
 831
 832
 833 template <bool Debug>
 834 class monitor_tt {
 835     pthread_mutex_t mutex;
 836     pthread_cond_t cond;
 837
 838   public:
 839     constexpr monitor_tt()
 840         : mutex(PTHREAD_MUTEX_INITIALIZER), cond(PTHREAD_COND_INITIALIZER)
 841     {
 842         lockdebug_remember_monitor(this);
 843     }
 844
 845     monitor_tt(const fork_unsafe_lock_t unsafe)
 846         : mutex(PTHREAD_MUTEX_INITIALIZER), cond(PTHREAD_COND_INITIALIZER)
 847     { }
 848
 849     void enter()
 850     {
 851         lockdebug_monitor_enter(this);
 852
 853         int err = pthread_mutex_lock(&mutex);
 854         if (err) _objc_fatal("pthread_mutex_lock failed (%d)", err);
 855     }
 856
 857     void leave()
 858     {
 859         lockdebug_monitor_leave(this);
 860
 861         int err = pthread_mutex_unlock(&mutex);
 862         if (err) _objc_fatal("pthread_mutex_unlock failed (%d)", err);
 863     }
 864
 865     void wait()
 866     {
 867         lockdebug_monitor_wait(this);
 868
 869         int err = pthread_cond_wait(&cond, &mutex);
 870         if (err) _objc_fatal("pthread_cond_wait failed (%d)", err);
 871     }
 872
 873     void notify()
 874     {
 875         int err = pthread_cond_signal(&cond);
 876         if (err) _objc_fatal("pthread_cond_signal failed (%d)", err);
 877     }
 878
 879     void notifyAll()
 880     {
 881         int err = pthread_cond_broadcast(&cond);
 882         if (err) _objc_fatal("pthread_cond_broadcast failed (%d)", err);
 883     }
 884
 885     void forceReset()
 886     {
 887         lockdebug_monitor_leave(this);
 888
 889         bzero(&mutex, sizeof(mutex));
 890         bzero(&cond, sizeof(cond));
 891         mutex = pthread_mutex_t PTHREAD_MUTEX_INITIALIZER;
 892         cond = pthread_cond_t PTHREAD_COND_INITIALIZER;
 893     }
 894
 895     void assertLocked()
 896     {
 897         lockdebug_monitor_assert_locked(this);
 898     }
 899
 900     void assertUnlocked()
 901     {
 902         lockdebug_monitor_assert_unlocked(this);
 903     }
 904 };
 905
 906
 907 // semaphore_create formatted for INIT_ONCE use
 908 static inline semaphore_t create_semaphore(void)
 909 {
 910     semaphore_t sem;
 911     kern_return_t k;
 912     k = semaphore_create(mach_task_self(), &sem, SYNC_POLICY_FIFO, 0);
 913     if (k) _objc_fatal("semaphore_create failed (0x%x)", k);
 914     return sem;
 915 }
 916
 917
 918 #ifndef __LP64__
 919 typedef struct mach_header headerType;
 920 typedef struct segment_command segmentType;
 921 typedef struct section sectionType;
 922 #else
 923 typedef struct mach_header_64 headerType;
 924 typedef struct segment_command_64 segmentType;
 925 typedef struct section_64 sectionType;
 926 #endif
 927 #define headerIsBundle(hi) (hi->mhdr()->filetype == MH_BUNDLE)
 928 #define libobjc_header ((headerType *)&_mh_dylib_header)
 929
 930 // Prototypes
 931
 932 /* Secure /tmp usage */
 933 extern int secure_open(const char *filename, int flags, uid_t euid);
 934
 935
 936 #else
 937
 938
 939 #error unknown OS
 940
 941
 942 #endif
 943
 944
 945 static inline void *
 946 memdup(const void *mem, size_t len)
 947 {
 948     void *dup = malloc(len);
 949     memcpy(dup, mem, len);
 950     return dup;
 951 }
 952
 953 // strdup that doesn't copy read-only memory
 954 static inline char *
 955 strdupIfMutable(const char *str)
 956 {
 957     size_t size = strlen(str) + 1;
 958     if (_dyld_is_memory_immutable(str, size)) {
 959         return (char *)str;
 960     } else {
 961         return (char *)memdup(str, size);
 962     }
 963 }
 964
 965 // free strdupIfMutable() result
 966 static inline void
 967 freeIfMutable(char *str)
 968 {
 969     size_t size = strlen(str) + 1;
 970     if (_dyld_is_memory_immutable(str, size)) {
 971         // nothing
 972     } else {
 973         free(str);
 974     }
 975 }
 976
 977 // nil-checking unsigned strdup
 978 static inline uint8_t *
 979 ustrdupMaybeNil(const uint8_t *str)
 980 {
 981     if (!str) return nil;
 982     return (uint8_t *)strdupIfMutable((char *)str);
 983 }
 984
 985 // OS version checking:
 986 //
 987 // sdkVersion()
 988 // DYLD_OS_VERSION(mac, ios, tv, watch, bridge)
 989 // sdkIsOlderThan(mac, ios, tv, watch, bridge)
 990 // sdkIsAtLeast(mac, ios, tv, watch, bridge)
 991 //
 992 // This version order matches OBJC_AVAILABLE.
 993
 994 #if TARGET_OS_OSX
 995 #   define DYLD_OS_VERSION(x, i, t, w, b) DYLD_MACOSX_VERSION_##x
 996 #   define sdkVersion() dyld_get_program_sdk_version()
 997
 998 #elif TARGET_OS_IOS
 999 #   define DYLD_OS_VERSION(x, i, t, w, b) DYLD_IOS_VERSION_##i
1000 #   define sdkVersion() dyld_get_program_sdk_version()
1001
1002 #elif TARGET_OS_TV
1003     // dyld does not currently have distinct constants for tvOS
1004 #   define DYLD_OS_VERSION(x, i, t, w, b) DYLD_IOS_VERSION_##t
1005 #   define sdkVersion() dyld_get_program_sdk_version()
1006
1007 #elif TARGET_OS_BRIDGE
1008 #   if TARGET_OS_WATCH
1009 #       error bridgeOS 1.0 not supported
1010 #   endif
1011     // fixme don't need bridgeOS versioning yet
1012 #   define DYLD_OS_VERSION(x, i, t, w, b) DYLD_IOS_VERSION_##t
1013 #   define sdkVersion() dyld_get_program_sdk_bridge_os_version()
1014
1015 #elif TARGET_OS_WATCH
1016 #   define DYLD_OS_VERSION(x, i, t, w, b) DYLD_WATCHOS_VERSION_##w
1017     // watchOS has its own API for compatibility reasons
1018 #   define sdkVersion() dyld_get_program_sdk_watch_os_version()
1019
1020 #else
1021 #   error unknown OS
1022 #endif
1023
1024
1025 #define sdkIsOlderThan(x, i, t, w, b)                           \
1026             (sdkVersion() < DYLD_OS_VERSION(x, i, t, w, b))
1027 #define sdkIsAtLeast(x, i, t, w, b)                             \
1028             (sdkVersion() >= DYLD_OS_VERSION(x, i, t, w, b))
1029
1030 // Allow bare 0 to be used in DYLD_OS_VERSION() and sdkIsOlderThan()
1031 #define DYLD_MACOSX_VERSION_0 0
1032 #define DYLD_IOS_VERSION_0 0
1033 #define DYLD_TVOS_VERSION_0 0
1034 #define DYLD_WATCHOS_VERSION_0 0
1035 #define DYLD_BRIDGEOS_VERSION_0 0
1036
1037 // Pretty-print a DYLD_*_VERSION_* constant.
1038 #define SDK_FORMAT "%hu.%hhu.%hhu"
1039 #define FORMAT_SDK(v) \
1040     (unsigned short)(((uint32_t)(v))>>16),  \
1041     (unsigned  char)(((uint32_t)(v))>>8),   \
1042     (unsigned  char)(((uint32_t)(v))>>0)
1043
1044 // fork() safety requires careful tracking of all locks.
1045 // Our custom lock types check this in debug builds.
1046 // Disallow direct use of all other lock types.
1047 typedef __darwin_pthread_mutex_t pthread_mutex_t UNAVAILABLE_ATTRIBUTE;
1048 typedef __darwin_pthread_rwlock_t pthread_rwlock_t UNAVAILABLE_ATTRIBUTE;
1049 typedef int32_t OSSpinLock UNAVAILABLE_ATTRIBUTE;
1050 typedef struct os_unfair_lock_s os_unfair_lock UNAVAILABLE_ATTRIBUTE;
1051
1052
1053 #endif