src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <libkern/OSAtomic.h>
  71 #include <mach/mach.h>
  72 #include <mach/mach_error.h>
  73 #include <os/once_private.h>
  74 #include <sys/queue.h>
  75
  76 #ifndef __POSIX_LIB__
  77 #define __POSIX_LIB__
  78 #endif
  79
  80 #ifndef PTHREAD_LAYOUT_SPI
  81 #define PTHREAD_LAYOUT_SPI 1
  82 #endif
  83
  84 #include "posix_sched.h"
  85 #include "tsd_private.h"
  86 #include "spinlock_private.h"
  87
  88 #if TARGET_IPHONE_SIMULATOR
  89 #error Unsupported target
  90 #endif
  91
  92 // List of all pthreads in the process.
  93 TAILQ_HEAD(__pthread_list, _pthread);
  94 extern struct __pthread_list __pthread_head;
  95
  96 // Lock protects access to above list.
  97 extern pthread_lock_t _pthread_list_lock;
  98
  99 extern int __is_threaded;
 100
 101 /*
 102  * Compiled-in limits
 103  */
 104 #if TARGET_OS_EMBEDDED
 105 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 106 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 107 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 108 #else
 109 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 110 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 111 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 112 #endif
 113
 114 #define MAXTHREADNAMESIZE       64
 115 #define _PTHREAD_T
 116 typedef struct _pthread {
 117         //
 118         // ABI - These fields are externally known as struct _opaque_pthread_t.
 119         //
 120         long sig; // _PTHREAD_SIG
 121         struct __darwin_pthread_handler_rec *__cleanup_stack;
 122
 123         //
 124         // SPI - These fields are private.
 125         //
 126         // these fields are globally protected by _pthread_list_lock:
 127         uint32_t childrun:1,
 128                         parentcheck:1,
 129                         childexit:1,
 130                         pad3:29;
 131
 132         pthread_lock_t lock; // protect access to everything below
 133         uint32_t detached:8,
 134                         inherit:8,
 135                         policy:8,
 136                         kernalloc:1,
 137                         schedset:1,
 138                         wqthread:1,
 139                         wqkillset:1,
 140                         pad:4;
 141
 142 #if __LP64__
 143         uint32_t pad0;
 144 #endif
 145         uint64_t thread_id;     // 64-bit unique thread id
 146
 147         void *(*fun)(void*);    // thread start routine
 148         void *arg;              // thread start routine argument
 149         void *exit_value;       // thread exit value storage
 150
 151         semaphore_t joiner_notify;      // pthread_join notification
 152
 153         int max_tsd_key;
 154         int cancel_state;       // whether the thread can be cancelled
 155         int cancel_error;
 156
 157 #ifdef __i386__
 158         // i386 err_no must be at a 68 byte offset
 159         // See <rdar://problem/13249323>
 160         uint32_t __13249323_pad[3];
 161 #endif
 162         int err_no;             // thread-local errno
 163
 164         struct _pthread *joiner;
 165
 166         struct sched_param param;       // [aligned]
 167
 168         TAILQ_ENTRY(_pthread) plist;    // global thread list [aligned]
 169
 170         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 171
 172         void *stackaddr;        // base of the stack (page aligned)
 173         size_t stacksize;       // size of stack (page multiple and >= PTHREAD_STACK_MIN)
 174
 175         void* freeaddr;         // stack/thread allocation base address
 176         size_t freesize;        // stack/thread allocation size
 177         size_t guardsize;       // guard page size in bytes
 178
 179         // thread specific data
 180         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX] __attribute__ ((aligned (16)));
 181 } *pthread_t;
 182
 183
 184 struct _pthread_attr_t {
 185         long sig;
 186         pthread_lock_t lock;
 187         uint32_t detached:8,
 188                 inherit:8,
 189                 policy:8,
 190                 fastpath:1,
 191                 schedset:1,
 192                 qosset:1,
 193                 unused:5;
 194         struct sched_param param; // [aligned]
 195         void *stackaddr; // stack base; vm_page_size aligned
 196         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 197         size_t guardsize; // size in bytes of stack overflow guard area
 198         unsigned long qosclass;
 199 #if defined(__LP64__)
 200         uint32_t _reserved[2];
 201 #else
 202         uint32_t _reserved[1];
 203 #endif
 204 };
 205
 206 /*
 207  * Mutex attributes
 208  */
 209 #define _PTHREAD_MUTEX_POLICY_NONE              0
 210 #define _PTHREAD_MUTEX_POLICY_FAIRSHARE         1
 211 #define _PTHREAD_MUTEX_POLICY_FIRSTFIT          2
 212 #define _PTHREAD_MUTEX_POLICY_REALTIME          3
 213 #define _PTHREAD_MUTEX_POLICY_ADAPTIVE          4
 214 #define _PTHREAD_MUTEX_POLICY_PRIPROTECT        5
 215 #define _PTHREAD_MUTEX_POLICY_PRIINHERIT        6
 216
 217 #define _PTHREAD_MUTEXATTR_T
 218 typedef struct {
 219         long sig;
 220         int prioceiling;
 221         uint32_t protocol:2,
 222                 type:2,
 223                 pshared:2,
 224                 policy:3,
 225                 unused:23;
 226 } pthread_mutexattr_t;
 227
 228 struct _pthread_mutex_options {
 229         uint32_t protocol:2,
 230                 type:2,
 231                 pshared:2,
 232                 policy:3,
 233                 hold:2,
 234                 misalign:1,
 235                 notify:1,
 236                 mutex:1,
 237                 unused:2,
 238                 lock_count:16;
 239 };
 240
 241 typedef struct {
 242         long sig;
 243         pthread_lock_t lock;
 244         union {
 245                 uint32_t value;
 246                 struct _pthread_mutex_options options;
 247         } mtxopts;
 248         int16_t prioceiling;
 249         int16_t priority;
 250 #if defined(__LP64__)
 251         uint32_t _pad;
 252 #endif
 253         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 254         uint32_t m_seq[2]; // mutex sequence id
 255         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 256 #if defined(__LP64__)
 257         uint32_t _reserved[4];
 258 #else
 259         uint32_t _reserved[1];
 260 #endif
 261 } _pthread_mutex;
 262
 263
 264 #define _PTHREAD_CONDATTR_T
 265 typedef struct {
 266         long sig;
 267         uint32_t pshared:2,
 268                 unsupported:30;
 269 } pthread_condattr_t;
 270
 271
 272 typedef struct {
 273         long sig;
 274         pthread_lock_t lock;
 275         uint32_t unused:29,
 276                 misalign:1,
 277                 pshared:2;
 278         _pthread_mutex *busy;
 279         uint32_t c_seq[3];
 280 #if defined(__LP64__)
 281         uint32_t _reserved[3];
 282 #endif
 283 } _pthread_cond;
 284
 285
 286 #define _PTHREAD_ONCE_T
 287 typedef struct {
 288         long sig;
 289         os_once_t once;
 290 } pthread_once_t;
 291
 292
 293 #define _PTHREAD_RWLOCKATTR_T
 294 typedef struct {
 295         long sig;
 296         int pshared;
 297 #if defined(__LP64__)
 298         uint32_t _reserved[3];
 299 #else
 300         uint32_t _reserved[2];
 301 #endif
 302 } pthread_rwlockattr_t;
 303
 304
 305 typedef struct {
 306         long sig;
 307         pthread_lock_t lock;
 308         uint32_t unused:29,
 309                 misalign:1,
 310                 pshared:2;
 311         uint32_t rw_flags;
 312 #if defined(__LP64__)
 313         uint32_t _pad;
 314 #endif
 315         volatile uint32_t rw_seq[4];
 316         struct _pthread *rw_owner;
 317         volatile uint32_t *rw_lcntaddr;
 318         volatile uint32_t *rw_seqaddr;
 319         volatile uint32_t *rw_ucntaddr;
 320 #if defined(__LP64__)
 321         uint32_t _reserved[31];
 322 #else
 323         uint32_t _reserved[19];
 324 #endif
 325 } _pthread_rwlock;
 326
 327 #include "pthread.h"
 328 #include "pthread_spis.h"
 329
 330 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 331                 "Incorrect _pthread_mutex structure size");
 332
 333 // Internal references to pthread_self() use TSD slot 0 directly.
 334 inline static pthread_t __attribute__((__pure__))
 335 _pthread_self_direct(void)
 336 {
 337         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 338 }
 339 #define pthread_self() _pthread_self_direct()
 340
 341 inline static pthread_t __attribute__((__pure__))
 342 _pthread_selfid_direct(void)
 343 {
 344         return (_pthread_self_direct())->thread_id;
 345 }
 346
 347 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 348 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 349 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 350 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 351 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 352 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 353
 354 #define _PTHREAD_NO_SIG                 0x00000000
 355 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 356 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 357 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 358 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 359 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 360 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 361 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 362 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 363 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 364 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 365 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 366 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 367 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 368 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 369 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 370 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 371 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 372 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 373 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 374 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 375 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 376
 377
 378 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 379 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 380 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 381
 382 #define _PTHREAD_CREATE_PARENT          4
 383 #define _PTHREAD_EXITED                 8
 384 // 4597450: begin
 385 #define _PTHREAD_WASCANCEL              0x10
 386 // 4597450: end
 387
 388 #if defined(DEBUG)
 389 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 390 #else
 391 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 392 #endif
 393 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 394
 395 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 396 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 397 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 398
 399 extern boolean_t swtch_pri(int);
 400
 401 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 402 #define PTHREAD_EXTERN extern
 403 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 404 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 405 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 406 #define PTHREAD_NOINLINE __attribute__((noinline))
 407 #define PTHREAD_WEAK __attribute__((weak))
 408 #define PTHREAD_USED __attribute__((used))
 409
 410 #include "kern/kern_internal.h"
 411
 412 /* Prototypes. */
 413
 414 /* Internal globals. */
 415 PTHREAD_NOEXPORT extern int __pthread_supported_features;
 416
 417 /* Functions defined in machine-dependent files. */
 418 PTHREAD_NOEXPORT void _pthread_setup(pthread_t th, void (*f)(pthread_t), void *sp, int suspended, int needresume);
 419
 420 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 421
 422 PTHREAD_NOEXPORT int __mtx_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 423
 424 /* internally redirected upcalls. */
 425 PTHREAD_NOEXPORT void* malloc(size_t);
 426 PTHREAD_NOEXPORT void free(void*);
 427
 428 /* syscall interfaces */
 429 extern uint32_t __psynch_mutexwait(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 430 extern uint32_t __psynch_mutexdrop(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 431
 432 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 433 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 434 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 435 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 436 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 437 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 438 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 439 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 440 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 441 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 442 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 443 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 444 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 445
 446 PTHREAD_EXTERN
 447 int
 448 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 449
 450 PTHREAD_NOEXPORT int _pthread_lookup_thread(pthread_t thread, mach_port_t * port, int only_joinable);
 451 PTHREAD_NOEXPORT int _pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming);
 452
 453 PTHREAD_NORETURN PTHREAD_NOEXPORT
 454 void
 455 __pthread_abort(void);
 456
 457 PTHREAD_NORETURN PTHREAD_NOEXPORT
 458 void
 459 __pthread_abort_reason(const char *fmt, ...);
 460
 461 PTHREAD_NOEXPORT
 462 void
 463 _pthread_set_main_qos(pthread_priority_t qos);
 464
 465 PTHREAD_NOEXPORT
 466 void
 467 _pthread_key_global_init(const char *envp[]);
 468
 469 PTHREAD_EXPORT
 470 void
 471 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 472
 473 PTHREAD_EXPORT
 474 void
 475 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 476
 477 PTHREAD_NOEXPORT
 478 void
 479 __pthread_fork_child_internal(pthread_t p);
 480
 481 PTHREAD_EXPORT
 482 void
 483 _pthread_clear_qos_tsd(mach_port_t thread_port);
 484
 485 PTHREAD_EXPORT
 486 void
 487 _pthread_testcancel(pthread_t thread, int isconforming);
 488
 489 PTHREAD_EXPORT
 490 void
 491 _pthread_exit_if_canceled(int error);
 492
 493 PTHREAD_ALWAYS_INLINE
 494 static inline mach_port_t
 495 _pthread_kernel_thread(pthread_t t)
 496 {
 497         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 498 }
 499
 500 PTHREAD_ALWAYS_INLINE
 501 static inline void
 502 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 503 {
 504         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 505 }
 506
 507 #define PTHREAD_ABORT(f,...) __pthread_abort_reason("%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
 508
 509 #define PTHREAD_ASSERT(b) do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
 510
 511 #include <os/semaphore_private.h>
 512 #include <os/alloc_once_private.h>
 513
 514 struct pthread_atfork_entry {
 515         void (*prepare)(void);
 516         void (*parent)(void);
 517         void (*child)(void);
 518 };
 519
 520 #define PTHREAD_ATFORK_INLINE_MAX 10
 521 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 522
 523 struct pthread_globals_s {
 524         // atfork.c
 525         pthread_t psaved_self;
 526         OSSpinLock psaved_self_global_lock;
 527         OSSpinLock pthread_atfork_lock;
 528
 529         size_t atfork_count;
 530         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 531         struct pthread_atfork_entry *atfork;
 532 };
 533 typedef struct pthread_globals_s *pthread_globals_t;
 534
 535 __attribute__((__pure__))
 536 static inline pthread_globals_t
 537 _pthread_globals(void)
 538 {
 539         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 540                              sizeof(struct pthread_globals_s),
 541                              NULL);
 542 }
 543
 544 #pragma mark _pthread_mutex_check_signature
 545
 546 PTHREAD_ALWAYS_INLINE
 547 static inline bool
 548 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 549 {
 550         return os_fastpath(mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 551 }
 552
 553 PTHREAD_ALWAYS_INLINE
 554 static inline bool
 555 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 556 {
 557         return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 558 }
 559
 560 PTHREAD_ALWAYS_INLINE
 561 static inline bool
 562 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 563 {
 564         return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) == _PTHREAD_MUTEX_SIG_init_CMP);
 565 }
 566
 567 #endif /* _POSIX_PTHREAD_INTERNALS_H */