src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <mach/mach.h>
  71 #include <mach/mach_error.h>
  72 #include <sys/queue.h>
  73 #include <pthread/bsdthread_private.h>
  74 #include <pthread/workqueue_syscalls.h>
  75
  76 #define __OS_EXPOSE_INTERNALS__ 1
  77 #include <os/internal/internal_shared.h>
  78 #include <os/once_private.h>
  79
  80 #if TARGET_IPHONE_SIMULATOR
  81 #error Unsupported target
  82 #endif
  83
  84 #define PTHREAD_INTERNAL_CRASH(c, x) do { \
  85                 _os_set_crash_log_cause_and_message((c), \
  86                                 "BUG IN LIBPTHREAD: " x); \
  87                 __builtin_trap(); \
  88         } while (0)
  89
  90 #define PTHREAD_CLIENT_CRASH(c, x) do { \
  91                 _os_set_crash_log_cause_and_message((c), \
  92                                 "BUG IN CLIENT OF LIBPTHREAD: " x); \
  93                 __builtin_trap(); \
  94         } while (0)
  95
  96 #ifndef __POSIX_LIB__
  97 #define __POSIX_LIB__
  98 #endif
  99
 100 #ifndef PTHREAD_LAYOUT_SPI
 101 #define PTHREAD_LAYOUT_SPI 1
 102 #endif
 103
 104 #include "posix_sched.h"
 105 #include "tsd_private.h"
 106 #include "spinlock_private.h"
 107
 108 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 109 #define PTHREAD_EXTERN extern
 110 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 111 #define PTHREAD_NOEXPORT_VARIANT
 112 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 113 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 114 #define PTHREAD_NOINLINE __attribute__((noinline))
 115 #define PTHREAD_WEAK __attribute__((weak))
 116 #define PTHREAD_USED __attribute__((used))
 117 #define PTHREAD_NOT_TAIL_CALLED __attribute__((__not_tail_called__))
 118
 119
 120 #define OS_UNFAIR_LOCK_INLINE 1
 121 #include <os/lock_private.h>
 122 typedef os_unfair_lock _pthread_lock;
 123 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 124 #define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
 125 #define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 126 #define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
 127 #define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
 128 #define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
 129
 130 #define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO)
 131
 132 extern int __is_threaded;
 133 extern int __unix_conforming;
 134
 135 // List of all pthreads in the process.
 136 TAILQ_HEAD(__pthread_list, _pthread);
 137 PTHREAD_NOEXPORT extern struct __pthread_list __pthread_head;
 138
 139 // Lock protects access to above list.
 140 PTHREAD_NOEXPORT extern _pthread_lock _pthread_list_lock;
 141
 142 PTHREAD_NOEXPORT extern uint32_t _main_qos;
 143
 144 #if PTHREAD_DEBUG_LOG
 145 #include <mach/mach_time.h>
 146 PTHREAD_NOEXPORT extern int _pthread_debuglog;
 147 PTHREAD_NOEXPORT extern uint64_t _pthread_debugstart;
 148 #endif
 149
 150 /*
 151  * Compiled-in limits
 152  */
 153 #if TARGET_OS_EMBEDDED
 154 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 155 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 156 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 157 #else
 158 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 159 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 160 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 161 #endif
 162
 163 #define PTHREAD_T_OFFSET 0
 164
 165 #define MAXTHREADNAMESIZE       64
 166 #define _PTHREAD_T
 167 typedef struct _pthread {
 168         //
 169         // ABI - These fields are externally known as struct _opaque_pthread_t.
 170         //
 171         long sig; // _PTHREAD_SIG
 172         struct __darwin_pthread_handler_rec *__cleanup_stack;
 173
 174         //
 175         // SPI - These fields are private.
 176         //
 177
 178         //
 179         // Fields protected by _pthread_list_lock
 180         //
 181
 182         TAILQ_ENTRY(_pthread) tl_plist; // global thread list [aligned]
 183         struct pthread_join_context_s *tl_join_ctx;
 184         void *tl_exit_value;
 185         uint32_t tl_policy:8,
 186                         tl_joinable:1,
 187                         tl_joiner_cleans_up:1,
 188                         tl_has_custom_stack:1,
 189                         __tl_pad:21;
 190         // MACH_PORT_NULL if no joiner
 191         // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner
 192         // MACH_PORT_DEAD if the thread exited
 193         uint32_t tl_exit_gate;
 194         struct sched_param tl_param;
 195
 196         //
 197         // Fields protected by pthread_t::lock
 198         //
 199
 200         _pthread_lock lock;
 201         uint16_t max_tsd_key;
 202         uint16_t inherit:8,
 203                         kernalloc:1,
 204                         schedset:1,
 205                         wqthread:1,
 206                         wqkillset:1,
 207                         wqoutsideqos:1,
 208                         __flags_pad:3;
 209
 210         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 211
 212         void *(*fun)(void *);   // thread start routine
 213         void *wq_kqid_ptr;              // wqthreads (workloop)
 214         void *arg;                              // thread start routine argument
 215         int   wq_nevents;               // wqthreads (workloop / kevent)
 216         uint16_t wq_retop;              // wqthreads
 217         uint8_t cancel_state;   // whether the thread can be canceled [atomic]
 218         uint8_t canceled;               // 4597450 set if conformant cancelation happened
 219         errno_t cancel_error;
 220         errno_t err_no;                 // thread-local errno
 221
 222         void *stackaddr;                // base of the stack (page aligned)
 223         void *stackbottom;              // stackaddr - stacksize
 224         void *freeaddr;                 // stack/thread allocation base address
 225         size_t freesize;                // stack/thread allocation size
 226         size_t guardsize;               // guard page size in bytes
 227
 228         // tsd-base relative accessed elements
 229         __attribute__((aligned(8)))
 230         uint64_t thread_id;     // 64-bit unique thread id
 231
 232         /* Thread Specific Data slots
 233          *
 234          * The offset of this field from the start of the structure is difficult to
 235          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 236          * the value into their source.  Newer versions of mono will fall back to
 237          * scanning to determine it at runtime, but there's lots of software built
 238          * with older mono that won't.  We will have to break them someday...
 239          */
 240         __attribute__ ((aligned (16)))
 241         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 242 } *pthread_t;
 243
 244 #define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1)
 245 struct _pthread_attr_t {
 246         long   sig;
 247         size_t guardsize; // size in bytes of stack overflow guard area
 248         void  *stackaddr; // stack base; vm_page_size aligned
 249         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 250         union {
 251                 struct sched_param param; // [aligned]
 252                 unsigned long qosclass; // pthread_priority_t
 253         };
 254         uint32_t
 255                 detached:8,
 256                 inherit:8,
 257                 policy:8,
 258                 schedset:1,
 259                 qosset:1,
 260                 policyset:1,
 261                 cpupercentset:1,
 262                 defaultguardpage:1,
 263                 unused:3;
 264         uint32_t
 265                 cpupercent:8,
 266                 refillms:24;
 267 #if defined(__LP64__)
 268         uint32_t _reserved[4];
 269 #else
 270         uint32_t _reserved[2];
 271 #endif
 272 };
 273
 274 /*
 275  * Mutex attributes
 276  */
 277
 278 #define _PTHREAD_MUTEXATTR_T
 279 typedef struct {
 280         long sig;
 281         int prioceiling;
 282         uint32_t protocol:2,
 283                 type:2,
 284                 pshared:2,
 285                 opt:3,
 286                 unused:23;
 287 } pthread_mutexattr_t;
 288
 289 struct _pthread_mutex_options {
 290         uint32_t protocol:2,
 291                 type:2,
 292                 pshared:2,
 293                 policy:3,
 294                 hold:2,
 295                 misalign:1,
 296                 notify:1,
 297                 mutex:1,
 298                 unused:2,
 299                 lock_count:16;
 300 };
 301 //
 302 #define _PTHREAD_MUTEX_POLICY_LAST              (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1)
 303 #define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE 1
 304 #define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT 2
 305 #define _PTHREAD_MTX_OPT_POLICY_DEFAULT _PTHREAD_MTX_OPT_POLICY_FIRSTFIT
 306 // The following _pthread_mutex_options defintions exist in synch_internal.h
 307 // such that the kernel extension can test for flags. They must be kept in
 308 // sync with the bit values in the struct above.
 309 // _PTHREAD_MTX_OPT_PSHARED 0x010
 310 // _PTHREAD_MTX_OPT_NOTIFY 0x1000
 311 // _PTHREAD_MTX_OPT_MUTEX 0x2000
 312
 313 // The fixed mask is used to mask out portions of the mutex options that
 314 // change on a regular basis (notify, lock_count).
 315 #define _PTHREAD_MTX_OPT_FIXED_MASK     0x27ff
 316
 317 typedef struct {
 318         long sig;
 319         _pthread_lock lock;
 320         union {
 321                 uint32_t value;
 322                 struct _pthread_mutex_options options;
 323         } mtxopts;
 324         int16_t prioceiling;
 325         int16_t priority;
 326 #if defined(__LP64__)
 327         uint32_t _pad;
 328 #endif
 329         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 330         uint32_t m_seq[2]; // mutex sequence id
 331         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 332 #if defined(__LP64__)
 333         uint32_t _reserved[4];
 334 #else
 335         uint32_t _reserved[1];
 336 #endif
 337 } _pthread_mutex;
 338
 339
 340 #define _PTHREAD_CONDATTR_T
 341 typedef struct {
 342         long sig;
 343         uint32_t pshared:2,
 344                 unsupported:30;
 345 } pthread_condattr_t;
 346
 347
 348 typedef struct {
 349         long sig;
 350         _pthread_lock lock;
 351         uint32_t unused:29,
 352                 misalign:1,
 353                 pshared:2;
 354         _pthread_mutex *busy;
 355         uint32_t c_seq[3];
 356 #if defined(__LP64__)
 357         uint32_t _reserved[3];
 358 #endif
 359 } _pthread_cond;
 360
 361
 362 #define _PTHREAD_ONCE_T
 363 typedef struct {
 364         long sig;
 365         os_once_t once;
 366 } pthread_once_t;
 367
 368
 369 #define _PTHREAD_RWLOCKATTR_T
 370 typedef struct {
 371         long sig;
 372         int pshared;
 373 #if defined(__LP64__)
 374         uint32_t _reserved[3];
 375 #else
 376         uint32_t _reserved[2];
 377 #endif
 378 } pthread_rwlockattr_t;
 379
 380
 381 typedef struct {
 382         long sig;
 383         _pthread_lock lock;
 384         uint32_t unused:29,
 385                         misalign:1,
 386                         pshared:2;
 387         uint32_t rw_flags;
 388 #if defined(__LP64__)
 389         uint32_t _pad;
 390 #endif
 391         uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock
 392         uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary)
 393         uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here
 394 #if defined(__LP64__)
 395         uint32_t _reserved[34];
 396 #else
 397         uint32_t _reserved[18];
 398 #endif
 399 } _pthread_rwlock;
 400
 401 #include "pthread.h"
 402 #include "pthread_spis.h"
 403
 404 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 405                 "Incorrect _pthread_mutex structure size");
 406
 407 _Static_assert(sizeof(_pthread_rwlock) == sizeof(pthread_rwlock_t),
 408                 "Incorrect _pthread_rwlock structure size");
 409
 410 // Internal references to pthread_self() use TSD slot 0 directly.
 411 inline static pthread_t __attribute__((__pure__))
 412 _pthread_self_direct(void)
 413 {
 414         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 415 }
 416 #define pthread_self() _pthread_self_direct()
 417
 418 PTHREAD_ALWAYS_INLINE
 419 inline static uint64_t __attribute__((__pure__))
 420 _pthread_selfid_direct(void)
 421 {
 422         return (_pthread_self_direct())->thread_id;
 423 }
 424
 425 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 426 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 427 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 428 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 429 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 430 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 431
 432 #define _PTHREAD_NO_SIG                 0x00000000
 433 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 434 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 435 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 436 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 437 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 438 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 439 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 440 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 441 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 442 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 443 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 444 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 445 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 446 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 447 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 448 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 449 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 450 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 451 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 452 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 453 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 454
 455
 456 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 457 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 458 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 459
 460 #if defined(DEBUG)
 461 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 462 #else
 463 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 464 #endif
 465 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 466
 467 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 468 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 469 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 470 #define _PTHREAD_CANCEL_INITIALIZED  0x20  /* the thread in the list is properly initialized */
 471
 472 extern boolean_t swtch_pri(int);
 473
 474 #include "kern/kern_internal.h"
 475
 476 /* Prototypes. */
 477
 478 /* Internal globals. */
 479 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 480
 481 PTHREAD_NOEXPORT int _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 482
 483 /* internally redirected upcalls. */
 484 PTHREAD_NOEXPORT void* malloc(size_t);
 485 PTHREAD_NOEXPORT void free(void*);
 486
 487 /* syscall interfaces */
 488 extern uint32_t __psynch_mutexwait(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 489 extern uint32_t __psynch_mutexdrop(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 490
 491 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 492 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 493 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 494 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 495 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 496 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 497 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 498 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 499 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 500 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 501 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 502 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 503 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 504
 505 PTHREAD_EXTERN
 506 int
 507 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 508
 509 PTHREAD_NOEXPORT
 510 void
 511 _pthread_deallocate(pthread_t t, bool from_mach_thread);
 512
 513 PTHREAD_NORETURN PTHREAD_NOEXPORT
 514 void
 515 __pthread_abort(void);
 516
 517 PTHREAD_NORETURN PTHREAD_NOEXPORT
 518 void
 519 __pthread_abort_reason(const char *fmt, ...) __printflike(1,2);
 520
 521 PTHREAD_NOEXPORT
 522 thread_qos_t
 523 _pthread_qos_class_to_thread_qos(qos_class_t qos);
 524
 525 PTHREAD_NOEXPORT
 526 void
 527 _pthread_set_main_qos(pthread_priority_t qos);
 528
 529 PTHREAD_NOEXPORT
 530 void
 531 _pthread_key_global_init(const char *envp[]);
 532
 533 PTHREAD_NOEXPORT
 534 void
 535 _pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data);
 536
 537 PTHREAD_EXPORT
 538 void
 539 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 540
 541 PTHREAD_NORETURN PTHREAD_EXPORT
 542 void
 543 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 544
 545 PTHREAD_NOEXPORT
 546 void
 547 _pthread_main_thread_init(pthread_t p);
 548
 549 PTHREAD_NOEXPORT
 550 void
 551 _pthread_main_thread_postfork_init(pthread_t p);
 552
 553 PTHREAD_NOEXPORT
 554 void
 555 _pthread_bsdthread_init(struct _pthread_registration_data *data);
 556
 557 PTHREAD_NOEXPORT_VARIANT
 558 void
 559 _pthread_clear_qos_tsd(mach_port_t thread_port);
 560
 561 #define PTHREAD_CONFORM_DARWIN_LEGACY     0
 562 #define PTHREAD_CONFORM_UNIX03_NOCANCEL   1
 563 #define PTHREAD_CONFORM_UNIX03_CANCELABLE 2
 564
 565 PTHREAD_NOEXPORT_VARIANT
 566 void
 567 _pthread_testcancel(int conforming);
 568
 569 PTHREAD_EXPORT
 570 void
 571 _pthread_exit_if_canceled(int error);
 572
 573 PTHREAD_NOEXPORT
 574 void
 575 _pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport);
 576
 577 PTHREAD_NOEXPORT
 578 void
 579 _pthread_setcancelstate_exit(pthread_t self, void *value_ptr);
 580
 581 PTHREAD_NOEXPORT
 582 semaphore_t
 583 _pthread_joiner_prepost_wake(pthread_t thread);
 584
 585 PTHREAD_ALWAYS_INLINE
 586 static inline mach_port_t
 587 _pthread_kernel_thread(pthread_t t)
 588 {
 589         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 590 }
 591
 592 PTHREAD_ALWAYS_INLINE
 593 static inline void
 594 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 595 {
 596         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 597 }
 598
 599 #define PTHREAD_ABORT(f,...) __pthread_abort_reason( \
 600                 "%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
 601
 602 #define PTHREAD_ASSERT(b) \
 603                 do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
 604
 605 #include <os/semaphore_private.h>
 606 #include <os/alloc_once_private.h>
 607
 608 struct pthread_atfork_entry {
 609         void (*prepare)(void);
 610         void (*parent)(void);
 611         void (*child)(void);
 612 };
 613
 614 #define PTHREAD_ATFORK_INLINE_MAX 10
 615 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 616
 617 struct pthread_globals_s {
 618         // atfork.c
 619         pthread_t psaved_self;
 620         _pthread_lock psaved_self_global_lock;
 621         _pthread_lock pthread_atfork_lock;
 622
 623         size_t atfork_count;
 624         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 625         struct pthread_atfork_entry *atfork;
 626         uint16_t qmp_logical[THREAD_QOS_LAST];
 627         uint16_t qmp_physical[THREAD_QOS_LAST];
 628
 629 };
 630 typedef struct pthread_globals_s *pthread_globals_t;
 631
 632 __attribute__((__pure__))
 633 static inline pthread_globals_t
 634 _pthread_globals(void)
 635 {
 636         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 637                              sizeof(struct pthread_globals_s),
 638                              NULL);
 639 }
 640
 641 #pragma mark _pthread_mutex_check_signature
 642
 643 PTHREAD_ALWAYS_INLINE
 644 static inline bool
 645 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 646 {
 647         return (mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 648 }
 649
 650 PTHREAD_ALWAYS_INLINE
 651 static inline bool
 652 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 653 {
 654         return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 655 }
 656
 657 PTHREAD_ALWAYS_INLINE
 658 static inline bool
 659 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 660 {
 661         return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) ==
 662                         _PTHREAD_MUTEX_SIG_init_CMP);
 663 }
 664
 665 #pragma mark _pthread_rwlock_check_signature
 666
 667 PTHREAD_ALWAYS_INLINE
 668 static inline bool
 669 _pthread_rwlock_check_signature(_pthread_rwlock *rwlock)
 670 {
 671         return (rwlock->sig == _PTHREAD_RWLOCK_SIG);
 672 }
 673
 674 PTHREAD_ALWAYS_INLINE
 675 static inline bool
 676 _pthread_rwlock_check_signature_init(_pthread_rwlock *rwlock)
 677 {
 678         return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init);
 679 }
 680
 681 /*
 682  * ALWAYS called without list lock and return with list lock held on success
 683  *
 684  * This weird calling convention exists because this function will sometimes
 685  * drop the lock, and it's best callers don't have to remember this.
 686  */
 687 PTHREAD_ALWAYS_INLINE
 688 static inline bool
 689 _pthread_validate_thread_and_list_lock(pthread_t thread)
 690 {
 691         pthread_t p;
 692         if (thread == NULL) return false;
 693 loop:
 694         _PTHREAD_LOCK(_pthread_list_lock);
 695         TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
 696                 if (p != thread) continue;
 697                 int state = os_atomic_load(&p->cancel_state, relaxed);
 698                 if (os_likely(state & _PTHREAD_CANCEL_INITIALIZED)) {
 699                         if (os_unlikely(p->sig != _PTHREAD_SIG)) {
 700                                 PTHREAD_CLIENT_CRASH(0, "pthread_t was corrupted");
 701                         }
 702                         return true;
 703                 }
 704                 _PTHREAD_UNLOCK(_pthread_list_lock);
 705                 thread_switch(_pthread_kernel_thread(p),
 706                                           SWITCH_OPTION_OSLOCK_DEPRESS, 1);
 707                 goto loop;
 708         }
 709         _PTHREAD_UNLOCK(_pthread_list_lock);
 710
 711         return false;
 712 }
 713
 714 PTHREAD_ALWAYS_INLINE
 715 static inline bool
 716 _pthread_is_valid(pthread_t thread, mach_port_t *portp)
 717 {
 718         mach_port_t kport = MACH_PORT_NULL;
 719         bool valid;
 720
 721         if (thread == pthread_self()) {
 722                 valid = true;
 723                 kport = _pthread_kernel_thread(thread);
 724         } else if (!_pthread_validate_thread_and_list_lock(thread)) {
 725                 valid = false;
 726         } else {
 727                 kport = _pthread_kernel_thread(thread);
 728                 valid = true;
 729                 _PTHREAD_UNLOCK(_pthread_list_lock);
 730         }
 731
 732         if (portp != NULL) {
 733                 *portp = kport;
 734         }
 735         return valid;
 736 }
 737
 738 PTHREAD_ALWAYS_INLINE
 739 static inline void*
 740 _pthread_atomic_xchg_ptr_inline(void **p, void *v)
 741 {
 742         return os_atomic_xchg(p, v, seq_cst);
 743 }
 744
 745 PTHREAD_ALWAYS_INLINE
 746 static inline uint32_t
 747 _pthread_atomic_xchg_uint32_relaxed_inline(uint32_t *p,uint32_t v)
 748 {
 749         return os_atomic_xchg(p, v, relaxed);
 750 }
 751
 752 #define _pthread_atomic_xchg_ptr(p, v) \
 753                 _pthread_atomic_xchg_ptr_inline(p, v)
 754 #define _pthread_atomic_xchg_uint32_relaxed(p, v) \
 755                 _pthread_atomic_xchg_uint32_relaxed_inline(p, v)
 756
 757 #endif /* _POSIX_PTHREAD_INTERNALS_H */