src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <libkern/OSAtomic.h>
  71 #include <mach/mach.h>
  72 #include <mach/mach_error.h>
  73 #include <sys/queue.h>
  74
  75 #define __OS_EXPOSE_INTERNALS__ 1
  76 #include <os/internal/internal_shared.h>
  77 #include <os/once_private.h>
  78
  79 #define PTHREAD_INTERNAL_CRASH(c, x) do { \
  80                 _os_set_crash_log_cause_and_message((c), \
  81                                 "BUG IN LIBPTHREAD: " x); \
  82                 __builtin_trap(); \
  83         } while (0)
  84
  85 #define PTHREAD_CLIENT_CRASH(c, x) do { \
  86                 _os_set_crash_log_cause_and_message((c), \
  87                                 "BUG IN CLIENT OF LIBPTHREAD: " x); \
  88                 __builtin_trap(); \
  89         } while (0)
  90
  91 #ifndef __POSIX_LIB__
  92 #define __POSIX_LIB__
  93 #endif
  94
  95 #ifndef PTHREAD_LAYOUT_SPI
  96 #define PTHREAD_LAYOUT_SPI 1
  97 #endif
  98
  99 #include "posix_sched.h"
 100 #include "tsd_private.h"
 101 #include "spinlock_private.h"
 102
 103 #define OS_UNFAIR_LOCK_INLINE 1
 104 #include <os/lock_private.h>
 105 typedef os_unfair_lock _pthread_lock;
 106 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 107 #define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
 108 #define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 109 #define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
 110 #define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
 111 #define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
 112
 113 #if TARGET_IPHONE_SIMULATOR
 114 #error Unsupported target
 115 #endif
 116
 117 // List of all pthreads in the process.
 118 TAILQ_HEAD(__pthread_list, _pthread);
 119 extern struct __pthread_list __pthread_head;
 120
 121 // Lock protects access to above list.
 122 extern _pthread_lock _pthread_list_lock;
 123
 124 extern int __is_threaded;
 125
 126 /*
 127  * Compiled-in limits
 128  */
 129 #if TARGET_OS_EMBEDDED
 130 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 131 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 132 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 133 #else
 134 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 135 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 136 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 137 #endif
 138
 139 #define MAXTHREADNAMESIZE       64
 140 #define _PTHREAD_T
 141 typedef struct _pthread {
 142         //
 143         // ABI - These fields are externally known as struct _opaque_pthread_t.
 144         //
 145         long sig; // _PTHREAD_SIG
 146         struct __darwin_pthread_handler_rec *__cleanup_stack;
 147
 148         //
 149         // SPI - These fields are private.
 150         //
 151         // these fields are globally protected by _pthread_list_lock:
 152         uint32_t childrun:1,
 153                         parentcheck:1,
 154                         childexit:1,
 155                         pad3:29;
 156
 157         _pthread_lock lock; // protect access to everything below
 158         uint32_t detached:8,
 159                         inherit:8,
 160                         policy:8,
 161                         kernalloc:1,
 162                         schedset:1,
 163                         wqthread:1,
 164                         wqkillset:1,
 165                         pad:4;
 166
 167 #if defined(__LP64__)
 168         uint32_t pad0;
 169 #endif
 170
 171         void *(*fun)(void*);    // thread start routine
 172         void *arg;              // thread start routine argument
 173         void *exit_value;       // thread exit value storage
 174
 175         semaphore_t joiner_notify;      // pthread_join notification
 176
 177         int max_tsd_key;
 178         int cancel_state;       // whether the thread can be cancelled
 179         int cancel_error;
 180
 181         int err_no;             // thread-local errno
 182
 183         struct _pthread *joiner;
 184
 185         struct sched_param param;       // [aligned]
 186
 187         TAILQ_ENTRY(_pthread) plist;    // global thread list [aligned]
 188
 189         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 190
 191         void *stackaddr;        // base of the stack (page aligned)
 192         size_t stacksize;       // size of stack (page multiple and >= PTHREAD_STACK_MIN)
 193
 194         void* freeaddr;         // stack/thread allocation base address
 195         size_t freesize;        // stack/thread allocation size
 196         size_t guardsize;       // guard page size in bytes
 197
 198         // tsd-base relative accessed elements
 199         __attribute__((aligned(8)))
 200         uint64_t thread_id;     // 64-bit unique thread id
 201
 202         /* Thread Specific Data slots
 203          *
 204          * The offset of this field from the start of the structure is difficult to
 205          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 206          * the value into their source.  Newer versions of mono will fall back to
 207          * scanning to determine it at runtime, but there's lots of software built
 208          * with older mono that won't.  We will have to break them someday...
 209          */
 210         __attribute__ ((aligned (16)))
 211         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 212 } *pthread_t;
 213
 214
 215 struct _pthread_attr_t {
 216         long sig;
 217         _pthread_lock lock;
 218         uint32_t detached:8,
 219                 inherit:8,
 220                 policy:8,
 221                 fastpath:1,
 222                 schedset:1,
 223                 qosset:1,
 224                 unused:5;
 225         struct sched_param param; // [aligned]
 226         void *stackaddr; // stack base; vm_page_size aligned
 227         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 228         size_t guardsize; // size in bytes of stack overflow guard area
 229         unsigned long qosclass;
 230 #if defined(__LP64__)
 231         uint32_t _reserved[2];
 232 #else
 233         uint32_t _reserved[1];
 234 #endif
 235 };
 236
 237 /*
 238  * Mutex attributes
 239  */
 240 #define _PTHREAD_MUTEX_POLICY_NONE              0
 241 #define _PTHREAD_MUTEX_POLICY_FAIRSHARE         1
 242 #define _PTHREAD_MUTEX_POLICY_FIRSTFIT          2
 243 #define _PTHREAD_MUTEX_POLICY_REALTIME          3
 244 #define _PTHREAD_MUTEX_POLICY_ADAPTIVE          4
 245 #define _PTHREAD_MUTEX_POLICY_PRIPROTECT        5
 246 #define _PTHREAD_MUTEX_POLICY_PRIINHERIT        6
 247
 248 #define _PTHREAD_MUTEXATTR_T
 249 typedef struct {
 250         long sig;
 251         int prioceiling;
 252         uint32_t protocol:2,
 253                 type:2,
 254                 pshared:2,
 255                 policy:3,
 256                 unused:23;
 257 } pthread_mutexattr_t;
 258
 259 struct _pthread_mutex_options {
 260         uint32_t protocol:2,
 261                 type:2,
 262                 pshared:2,
 263                 policy:3,
 264                 hold:2,
 265                 misalign:1,
 266                 notify:1,
 267                 mutex:1,
 268                 unused:2,
 269                 lock_count:16;
 270 };
 271
 272 typedef struct {
 273         long sig;
 274         _pthread_lock lock;
 275         union {
 276                 uint32_t value;
 277                 struct _pthread_mutex_options options;
 278         } mtxopts;
 279         int16_t prioceiling;
 280         int16_t priority;
 281 #if defined(__LP64__)
 282         uint32_t _pad;
 283 #endif
 284         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 285         uint32_t m_seq[2]; // mutex sequence id
 286         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 287 #if defined(__LP64__)
 288         uint32_t _reserved[4];
 289 #else
 290         uint32_t _reserved[1];
 291 #endif
 292 } _pthread_mutex;
 293
 294
 295 #define _PTHREAD_CONDATTR_T
 296 typedef struct {
 297         long sig;
 298         uint32_t pshared:2,
 299                 unsupported:30;
 300 } pthread_condattr_t;
 301
 302
 303 typedef struct {
 304         long sig;
 305         _pthread_lock lock;
 306         uint32_t unused:29,
 307                 misalign:1,
 308                 pshared:2;
 309         _pthread_mutex *busy;
 310         uint32_t c_seq[3];
 311 #if defined(__LP64__)
 312         uint32_t _reserved[3];
 313 #endif
 314 } _pthread_cond;
 315
 316
 317 #define _PTHREAD_ONCE_T
 318 typedef struct {
 319         long sig;
 320         os_once_t once;
 321 } pthread_once_t;
 322
 323
 324 #define _PTHREAD_RWLOCKATTR_T
 325 typedef struct {
 326         long sig;
 327         int pshared;
 328 #if defined(__LP64__)
 329         uint32_t _reserved[3];
 330 #else
 331         uint32_t _reserved[2];
 332 #endif
 333 } pthread_rwlockattr_t;
 334
 335
 336 typedef struct {
 337         long sig;
 338         _pthread_lock lock;
 339         uint32_t unused:29,
 340                 misalign:1,
 341                 pshared:2;
 342         uint32_t rw_flags;
 343 #if defined(__LP64__)
 344         uint32_t _pad;
 345 #endif
 346         volatile uint32_t rw_seq[4];
 347         struct _pthread *rw_owner;
 348         volatile uint32_t *rw_lcntaddr;
 349         volatile uint32_t *rw_seqaddr;
 350         volatile uint32_t *rw_ucntaddr;
 351 #if defined(__LP64__)
 352         uint32_t _reserved[31];
 353 #else
 354         uint32_t _reserved[19];
 355 #endif
 356 } _pthread_rwlock;
 357
 358 #include "pthread.h"
 359 #include "pthread_spis.h"
 360
 361 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 362                 "Incorrect _pthread_mutex structure size");
 363
 364 // Internal references to pthread_self() use TSD slot 0 directly.
 365 inline static pthread_t __attribute__((__pure__))
 366 _pthread_self_direct(void)
 367 {
 368         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 369 }
 370 #define pthread_self() _pthread_self_direct()
 371
 372 inline static pthread_t __attribute__((__pure__))
 373 _pthread_selfid_direct(void)
 374 {
 375         return (_pthread_self_direct())->thread_id;
 376 }
 377
 378 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 379 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 380 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 381 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 382 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 383 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 384
 385 #define _PTHREAD_NO_SIG                 0x00000000
 386 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 387 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 388 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 389 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 390 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 391 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 392 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 393 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 394 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 395 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 396 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 397 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 398 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 399 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 400 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 401 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 402 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 403 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 404 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 405 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 406 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 407
 408
 409 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 410 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 411 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 412
 413 #define _PTHREAD_CREATE_PARENT          4
 414 #define _PTHREAD_EXITED                 8
 415 // 4597450: begin
 416 #define _PTHREAD_WASCANCEL              0x10
 417 // 4597450: end
 418
 419 #if defined(DEBUG)
 420 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 421 #else
 422 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 423 #endif
 424 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 425
 426 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 427 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 428 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 429
 430 extern boolean_t swtch_pri(int);
 431
 432 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 433 #define PTHREAD_EXTERN extern
 434 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 435 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 436 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 437 #define PTHREAD_NOINLINE __attribute__((noinline))
 438 #define PTHREAD_WEAK __attribute__((weak))
 439 #define PTHREAD_USED __attribute__((used))
 440
 441 #include "kern/kern_internal.h"
 442
 443 /* Prototypes. */
 444
 445 /* Internal globals. */
 446 PTHREAD_NOEXPORT extern int __pthread_supported_features;
 447
 448 /* Functions defined in machine-dependent files. */
 449 PTHREAD_NOEXPORT void _pthread_setup(pthread_t th, void (*f)(pthread_t), void *sp, int suspended, int needresume);
 450
 451 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 452
 453 PTHREAD_NOEXPORT int __mtx_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 454
 455 /* internally redirected upcalls. */
 456 PTHREAD_NOEXPORT void* malloc(size_t);
 457 PTHREAD_NOEXPORT void free(void*);
 458
 459 /* syscall interfaces */
 460 extern uint32_t __psynch_mutexwait(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 461 extern uint32_t __psynch_mutexdrop(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 462
 463 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 464 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 465 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 466 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 467 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 468 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 469 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 470 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 471 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 472 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 473 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 474 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 475 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 476
 477 PTHREAD_EXTERN
 478 int
 479 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 480
 481 PTHREAD_NOEXPORT int _pthread_lookup_thread(pthread_t thread, mach_port_t * port, int only_joinable);
 482 PTHREAD_NOEXPORT int _pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming);
 483
 484 PTHREAD_NORETURN PTHREAD_NOEXPORT
 485 void
 486 __pthread_abort(void);
 487
 488 PTHREAD_NORETURN PTHREAD_NOEXPORT
 489 void
 490 __pthread_abort_reason(const char *fmt, ...);
 491
 492 PTHREAD_NOEXPORT
 493 void
 494 _pthread_set_main_qos(pthread_priority_t qos);
 495
 496 PTHREAD_NOEXPORT
 497 void
 498 _pthread_key_global_init(const char *envp[]);
 499
 500 PTHREAD_EXPORT
 501 void
 502 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 503
 504 PTHREAD_EXPORT
 505 void
 506 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 507
 508 PTHREAD_NOEXPORT
 509 void
 510 __pthread_fork_child_internal(pthread_t p);
 511
 512 PTHREAD_EXPORT
 513 void
 514 _pthread_clear_qos_tsd(mach_port_t thread_port);
 515
 516 PTHREAD_EXPORT
 517 void
 518 _pthread_testcancel(pthread_t thread, int isconforming);
 519
 520 PTHREAD_EXPORT
 521 void
 522 _pthread_exit_if_canceled(int error);
 523
 524 PTHREAD_ALWAYS_INLINE
 525 static inline mach_port_t
 526 _pthread_kernel_thread(pthread_t t)
 527 {
 528         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 529 }
 530
 531 PTHREAD_ALWAYS_INLINE
 532 static inline void
 533 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 534 {
 535         if (os_slowpath(!MACH_PORT_VALID(p))) {
 536                 PTHREAD_INTERNAL_CRASH(t, "Invalid thread port");
 537         }
 538         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 539 }
 540
 541 #define PTHREAD_ABORT(f,...) __pthread_abort_reason("%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
 542
 543 #define PTHREAD_ASSERT(b) do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
 544
 545 #include <os/semaphore_private.h>
 546 #include <os/alloc_once_private.h>
 547
 548 struct pthread_atfork_entry {
 549         void (*prepare)(void);
 550         void (*parent)(void);
 551         void (*child)(void);
 552 };
 553
 554 #define PTHREAD_ATFORK_INLINE_MAX 10
 555 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 556
 557 struct pthread_globals_s {
 558         // atfork.c
 559         pthread_t psaved_self;
 560         _pthread_lock psaved_self_global_lock;
 561         _pthread_lock pthread_atfork_lock;
 562
 563         size_t atfork_count;
 564         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 565         struct pthread_atfork_entry *atfork;
 566 };
 567 typedef struct pthread_globals_s *pthread_globals_t;
 568
 569 __attribute__((__pure__))
 570 static inline pthread_globals_t
 571 _pthread_globals(void)
 572 {
 573         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 574                              sizeof(struct pthread_globals_s),
 575                              NULL);
 576 }
 577
 578 #pragma mark _pthread_mutex_check_signature
 579
 580 PTHREAD_ALWAYS_INLINE
 581 static inline bool
 582 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 583 {
 584         return os_fastpath(mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 585 }
 586
 587 PTHREAD_ALWAYS_INLINE
 588 static inline bool
 589 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 590 {
 591         return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 592 }
 593
 594 PTHREAD_ALWAYS_INLINE
 595 static inline bool
 596 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 597 {
 598         return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) == _PTHREAD_MUTEX_SIG_init_CMP);
 599 }
 600
 601 #endif /* _POSIX_PTHREAD_INTERNALS_H */