src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <mach/mach.h>
  71 #include <mach/mach_error.h>
  72 #include <sys/queue.h>
  73 #include <pthread/bsdthread_private.h>
  74 #include <pthread/workqueue_syscalls.h>
  75
  76 #define __OS_EXPOSE_INTERNALS__ 1
  77 #include <os/internal/internal_shared.h>
  78 #include <os/once_private.h>
  79
  80 #if TARGET_IPHONE_SIMULATOR
  81 #error Unsupported target
  82 #endif
  83
  84 #define PTHREAD_INTERNAL_CRASH(c, x) do { \
  85                 _os_set_crash_log_cause_and_message((c), \
  86                                 "BUG IN LIBPTHREAD: " x); \
  87                 __builtin_trap(); \
  88         } while (0)
  89
  90 #define PTHREAD_CLIENT_CRASH(c, x) do { \
  91                 _os_set_crash_log_cause_and_message((c), \
  92                                 "BUG IN CLIENT OF LIBPTHREAD: " x); \
  93                 __builtin_trap(); \
  94         } while (0)
  95
  96 #ifndef __POSIX_LIB__
  97 #define __POSIX_LIB__
  98 #endif
  99
 100 #ifndef PTHREAD_LAYOUT_SPI
 101 #define PTHREAD_LAYOUT_SPI 1
 102 #endif
 103
 104 #include "posix_sched.h"
 105 #include "tsd_private.h"
 106 #include "spinlock_private.h"
 107
 108 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 109 #define PTHREAD_EXTERN extern
 110 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 111 #define PTHREAD_NOEXPORT_VARIANT
 112 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 113 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 114 #define PTHREAD_NOINLINE __attribute__((noinline))
 115 #define PTHREAD_WEAK __attribute__((weak))
 116 #define PTHREAD_USED __attribute__((used))
 117 #define PTHREAD_NOT_TAIL_CALLED __attribute__((__not_tail_called__))
 118
 119
 120 #define OS_UNFAIR_LOCK_INLINE 1
 121 #include <os/lock_private.h>
 122 typedef os_unfair_lock _pthread_lock;
 123 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 124 #define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
 125 #define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 126 #define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
 127 #define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
 128 #define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
 129
 130 #define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO)
 131
 132 extern int __is_threaded;
 133 extern int __unix_conforming;
 134
 135 // List of all pthreads in the process.
 136 TAILQ_HEAD(__pthread_list, _pthread);
 137 PTHREAD_NOEXPORT extern struct __pthread_list __pthread_head;
 138
 139 // Lock protects access to above list.
 140 PTHREAD_NOEXPORT extern _pthread_lock _pthread_list_lock;
 141
 142 PTHREAD_NOEXPORT extern uint32_t _main_qos;
 143
 144 #if PTHREAD_DEBUG_LOG
 145 #include <mach/mach_time.h>
 146 PTHREAD_NOEXPORT extern int _pthread_debuglog;
 147 PTHREAD_NOEXPORT extern uint64_t _pthread_debugstart;
 148 #endif
 149
 150 /*
 151  * Compiled-in limits
 152  */
 153 #if TARGET_OS_EMBEDDED
 154 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 155 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 156 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 157 #else
 158 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 159 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 160 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 161 #endif
 162
 163 #define PTHREAD_T_OFFSET 0
 164
 165 #define MAXTHREADNAMESIZE       64
 166 #define _PTHREAD_T
 167 typedef struct _pthread {
 168         //
 169         // ABI - These fields are externally known as struct _opaque_pthread_t.
 170         //
 171         long sig; // _PTHREAD_SIG
 172         struct __darwin_pthread_handler_rec *__cleanup_stack;
 173
 174         //
 175         // SPI - These fields are private.
 176         //
 177
 178         //
 179         // Fields protected by _pthread_list_lock
 180         //
 181
 182         TAILQ_ENTRY(_pthread) tl_plist; // global thread list [aligned]
 183         struct pthread_join_context_s *tl_join_ctx;
 184         void *tl_exit_value;
 185         uint32_t tl_policy:8,
 186                         tl_joinable:1,
 187                         tl_joiner_cleans_up:1,
 188                         tl_has_custom_stack:1,
 189                         __tl_pad:21;
 190         // MACH_PORT_NULL if no joiner
 191         // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner
 192         // MACH_PORT_DEAD if the thread exited
 193         uint32_t tl_exit_gate;
 194         struct sched_param tl_param;
 195
 196         //
 197         // Fields protected by pthread_t::lock
 198         //
 199
 200         _pthread_lock lock;
 201         uint16_t max_tsd_key;
 202         uint16_t inherit:8,
 203                         kernalloc:1,
 204                         schedset:1,
 205                         wqthread:1,
 206                         wqkillset:1,
 207                         wqoutsideqos:1,
 208                         __flags_pad:3;
 209
 210         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 211
 212         void *(*fun)(void *);   // thread start routine
 213         void *wq_kqid_ptr;              // wqthreads (workloop)
 214         void *arg;                              // thread start routine argument
 215         int   wq_nevents;               // wqthreads (workloop / kevent)
 216         uint16_t wq_retop;              // wqthreads
 217         uint8_t cancel_state;   // whether the thread can be canceled [atomic]
 218         uint8_t canceled;               // 4597450 set if conformant cancelation happened
 219         errno_t cancel_error;
 220         errno_t err_no;                 // thread-local errno
 221
 222         void *stackaddr;                // base of the stack (page aligned)
 223         void *stackbottom;              // stackaddr - stacksize
 224         void *freeaddr;                 // stack/thread allocation base address
 225         size_t freesize;                // stack/thread allocation size
 226         size_t guardsize;               // guard page size in bytes
 227
 228         // tsd-base relative accessed elements
 229         __attribute__((aligned(8)))
 230         uint64_t thread_id;     // 64-bit unique thread id
 231
 232         /* Thread Specific Data slots
 233          *
 234          * The offset of this field from the start of the structure is difficult to
 235          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 236          * the value into their source.  Newer versions of mono will fall back to
 237          * scanning to determine it at runtime, but there's lots of software built
 238          * with older mono that won't.  We will have to break them someday...
 239          */
 240         __attribute__ ((aligned (16)))
 241         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 242 } *pthread_t;
 243
 244 #define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1)
 245 struct _pthread_attr_t {
 246         long   sig;
 247         size_t guardsize; // size in bytes of stack overflow guard area
 248         void  *stackaddr; // stack base; vm_page_size aligned
 249         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 250         union {
 251                 struct sched_param param; // [aligned]
 252                 unsigned long qosclass; // pthread_priority_t
 253         };
 254         uint32_t
 255                 detached:8,
 256                 inherit:8,
 257                 policy:8,
 258                 schedset:1,
 259                 qosset:1,
 260                 policyset:1,
 261                 cpupercentset:1,
 262                 defaultguardpage:1,
 263                 unused:3;
 264         uint32_t
 265                 cpupercent:8,
 266                 refillms:24;
 267 #if defined(__LP64__)
 268         uint32_t _reserved[4];
 269 #else
 270         uint32_t _reserved[2];
 271 #endif
 272 };
 273
 274 /*
 275  * Mutex attributes
 276  */
 277
 278 #define _PTHREAD_MUTEXATTR_T
 279 typedef struct {
 280         long sig;
 281         int prioceiling;
 282         uint32_t protocol:2,
 283                 type:2,
 284                 pshared:2,
 285                 opt:3,
 286                 unused:23;
 287 } pthread_mutexattr_t;
 288
 289 struct _pthread_mutex_options {
 290         uint32_t protocol:2,
 291                 type:2,
 292                 pshared:2,
 293                 policy:3,
 294                 hold:2,
 295                 misalign:1,
 296                 notify:1,
 297                 mutex:1,
 298                 unused:2,
 299                 lock_count:16;
 300 };
 301 //
 302 #define _PTHREAD_MUTEX_POLICY_LAST              (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1)
 303 #define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE 1
 304 #define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT 2
 305 #define _PTHREAD_MTX_OPT_POLICY_DEFAULT _PTHREAD_MTX_OPT_POLICY_FIRSTFIT
 306 // The following _pthread_mutex_options defintions exist in synch_internal.h
 307 // such that the kernel extension can test for flags. They must be kept in
 308 // sync with the bit values in the struct above.
 309 // _PTHREAD_MTX_OPT_PSHARED 0x010
 310 // _PTHREAD_MTX_OPT_NOTIFY 0x1000
 311 // _PTHREAD_MTX_OPT_MUTEX 0x2000
 312
 313 // The fixed mask is used to mask out portions of the mutex options that
 314 // change on a regular basis (notify, lock_count).
 315 #define _PTHREAD_MTX_OPT_FIXED_MASK     0x27ff
 316
 317 typedef struct {
 318         long sig;
 319         _pthread_lock lock;
 320         union {
 321                 uint32_t value;
 322                 struct _pthread_mutex_options options;
 323         } mtxopts;
 324         int16_t prioceiling;
 325         int16_t priority;
 326 #if defined(__LP64__)
 327         uint32_t _pad;
 328 #endif
 329         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 330         uint32_t m_seq[2]; // mutex sequence id
 331         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 332 #if defined(__LP64__)
 333         uint32_t _reserved[4];
 334 #else
 335         uint32_t _reserved[1];
 336 #endif
 337 } _pthread_mutex;
 338
 339
 340 #define _PTHREAD_CONDATTR_T
 341 typedef struct {
 342         long sig;
 343         uint32_t pshared:2,
 344                 unsupported:30;
 345 } pthread_condattr_t;
 346
 347
 348 typedef struct {
 349         long sig;
 350         _pthread_lock lock;
 351         uint32_t unused:29,
 352                 misalign:1,
 353                 pshared:2;
 354         _pthread_mutex *busy;
 355         uint32_t c_seq[3];
 356 #if defined(__LP64__)
 357         uint32_t _reserved[3];
 358 #endif
 359 } _pthread_cond;
 360
 361
 362 #define _PTHREAD_ONCE_T
 363 typedef struct {
 364         long sig;
 365         os_once_t once;
 366 } pthread_once_t;
 367
 368
 369 #define _PTHREAD_RWLOCKATTR_T
 370 typedef struct {
 371         long sig;
 372         int pshared;
 373 #if defined(__LP64__)
 374         uint32_t _reserved[3];
 375 #else
 376         uint32_t _reserved[2];
 377 #endif
 378 } pthread_rwlockattr_t;
 379
 380
 381 typedef struct {
 382         long sig;
 383         _pthread_lock lock;
 384         uint32_t unused:29,
 385                         misalign:1,
 386                         pshared:2;
 387         uint32_t rw_flags;
 388 #if defined(__LP64__)
 389         uint32_t _pad;
 390 #endif
 391         uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock
 392         uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary)
 393         uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here
 394 #if defined(__LP64__)
 395         uint32_t _reserved[34];
 396 #else
 397         uint32_t _reserved[18];
 398 #endif
 399 } _pthread_rwlock;
 400
 401 #include "pthread.h"
 402 #include "pthread_spis.h"
 403
 404 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 405                 "Incorrect _pthread_mutex structure size");
 406
 407 _Static_assert(sizeof(_pthread_rwlock) == sizeof(pthread_rwlock_t),
 408                 "Incorrect _pthread_rwlock structure size");
 409
 410 // Internal references to pthread_self() use TSD slot 0 directly.
 411 inline static pthread_t __attribute__((__pure__))
 412 _pthread_self_direct(void)
 413 {
 414         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 415 }
 416 #define pthread_self() _pthread_self_direct()
 417
 418 PTHREAD_ALWAYS_INLINE
 419 inline static uint64_t __attribute__((__pure__))
 420 _pthread_selfid_direct(void)
 421 {
 422         return (_pthread_self_direct())->thread_id;
 423 }
 424
 425 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 426 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 427 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 428 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 429 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 430 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 431
 432 #define _PTHREAD_NO_SIG                 0x00000000
 433 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 434 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 435 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 436 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 437 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 438 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 439 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 440 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 441 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 442 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 443 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 444 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 445 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 446 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 447 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 448 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 449 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 450 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 451 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 452 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 453 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 454
 455
 456 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 457 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 458 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 459
 460 #if defined(DEBUG)
 461 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 462 #else
 463 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 464 #endif
 465 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 466
 467 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 468 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 469 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 470 #define _PTHREAD_CANCEL_INITIALIZED  0x20  /* the thread in the list is properly initialized */
 471
 472 extern boolean_t swtch_pri(int);
 473
 474 #include "kern/kern_internal.h"
 475
 476 /* Prototypes. */
 477
 478 /* Internal globals. */
 479 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 480
 481 PTHREAD_NOEXPORT int _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 482
 483 /* internally redirected upcalls. */
 484 PTHREAD_NOEXPORT void* malloc(size_t);
 485 PTHREAD_NOEXPORT void free(void*);
 486
 487 /* syscall interfaces */
 488 extern uint32_t __psynch_mutexwait(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 489 extern uint32_t __psynch_mutexdrop(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 490
 491 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 492 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 493 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 494 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 495 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 496 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 497 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 498 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 499 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 500 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 501 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 502 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 503 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 504
 505 PTHREAD_EXTERN
 506 int
 507 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 508
 509 PTHREAD_NOEXPORT
 510 void
 511 _pthread_deallocate(pthread_t t, bool from_mach_thread);
 512
 513 PTHREAD_NORETURN PTHREAD_NOEXPORT
 514 void
 515 __pthread_abort(void);
 516
 517 PTHREAD_NORETURN PTHREAD_NOEXPORT
 518 void
 519 __pthread_abort_reason(const char *fmt, ...) __printflike(1,2);
 520
 521 PTHREAD_NOEXPORT
 522 thread_qos_t
 523 _pthread_qos_class_to_thread_qos(qos_class_t qos);
 524
 525 PTHREAD_NOEXPORT
 526 void
 527 _pthread_set_main_qos(pthread_priority_t qos);
 528
 529 PTHREAD_NOEXPORT
 530 void
 531 _pthread_key_global_init(const char *envp[]);
 532
 533 PTHREAD_NOEXPORT
 534 void
 535 _pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data);
 536
 537 PTHREAD_EXPORT
 538 void
 539 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 540
 541 PTHREAD_NORETURN PTHREAD_EXPORT
 542 void
 543 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 544
 545 PTHREAD_NOEXPORT
 546 void
 547 _pthread_main_thread_init(pthread_t p);
 548
 549 PTHREAD_NOEXPORT
 550 void
 551 _pthread_bsdthread_init(struct _pthread_registration_data *data);
 552
 553 PTHREAD_NOEXPORT_VARIANT
 554 void
 555 _pthread_clear_qos_tsd(mach_port_t thread_port);
 556
 557 #define PTHREAD_CONFORM_DARWIN_LEGACY     0
 558 #define PTHREAD_CONFORM_UNIX03_NOCANCEL   1
 559 #define PTHREAD_CONFORM_UNIX03_CANCELABLE 2
 560
 561 PTHREAD_NOEXPORT_VARIANT
 562 void
 563 _pthread_testcancel(int conforming);
 564
 565 PTHREAD_EXPORT
 566 void
 567 _pthread_exit_if_canceled(int error);
 568
 569 PTHREAD_NOEXPORT
 570 void
 571 _pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport);
 572
 573 PTHREAD_NOEXPORT
 574 void
 575 _pthread_setcancelstate_exit(pthread_t self, void *value_ptr);
 576
 577 PTHREAD_NOEXPORT
 578 semaphore_t
 579 _pthread_joiner_prepost_wake(pthread_t thread);
 580
 581 PTHREAD_ALWAYS_INLINE
 582 static inline mach_port_t
 583 _pthread_kernel_thread(pthread_t t)
 584 {
 585         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 586 }
 587
 588 PTHREAD_ALWAYS_INLINE
 589 static inline void
 590 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 591 {
 592         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 593 }
 594
 595 #define PTHREAD_ABORT(f,...) __pthread_abort_reason( \
 596                 "%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
 597
 598 #define PTHREAD_ASSERT(b) \
 599                 do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
 600
 601 #include <os/semaphore_private.h>
 602 #include <os/alloc_once_private.h>
 603
 604 struct pthread_atfork_entry {
 605         void (*prepare)(void);
 606         void (*parent)(void);
 607         void (*child)(void);
 608 };
 609
 610 #define PTHREAD_ATFORK_INLINE_MAX 10
 611 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 612
 613 struct pthread_globals_s {
 614         // atfork.c
 615         pthread_t psaved_self;
 616         _pthread_lock psaved_self_global_lock;
 617         _pthread_lock pthread_atfork_lock;
 618
 619         size_t atfork_count;
 620         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 621         struct pthread_atfork_entry *atfork;
 622         uint16_t qmp_logical[THREAD_QOS_LAST];
 623         uint16_t qmp_physical[THREAD_QOS_LAST];
 624
 625 };
 626 typedef struct pthread_globals_s *pthread_globals_t;
 627
 628 __attribute__((__pure__))
 629 static inline pthread_globals_t
 630 _pthread_globals(void)
 631 {
 632         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 633                              sizeof(struct pthread_globals_s),
 634                              NULL);
 635 }
 636
 637 #pragma mark _pthread_mutex_check_signature
 638
 639 PTHREAD_ALWAYS_INLINE
 640 static inline bool
 641 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 642 {
 643         return (mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 644 }
 645
 646 PTHREAD_ALWAYS_INLINE
 647 static inline bool
 648 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 649 {
 650         return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 651 }
 652
 653 PTHREAD_ALWAYS_INLINE
 654 static inline bool
 655 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 656 {
 657         return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) ==
 658                         _PTHREAD_MUTEX_SIG_init_CMP);
 659 }
 660
 661 #pragma mark _pthread_rwlock_check_signature
 662
 663 PTHREAD_ALWAYS_INLINE
 664 static inline bool
 665 _pthread_rwlock_check_signature(_pthread_rwlock *rwlock)
 666 {
 667         return (rwlock->sig == _PTHREAD_RWLOCK_SIG);
 668 }
 669
 670 PTHREAD_ALWAYS_INLINE
 671 static inline bool
 672 _pthread_rwlock_check_signature_init(_pthread_rwlock *rwlock)
 673 {
 674         return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init);
 675 }
 676
 677 /*
 678  * ALWAYS called without list lock and return with list lock held on success
 679  *
 680  * This weird calling convention exists because this function will sometimes
 681  * drop the lock, and it's best callers don't have to remember this.
 682  */
 683 PTHREAD_ALWAYS_INLINE
 684 static inline bool
 685 _pthread_validate_thread_and_list_lock(pthread_t thread)
 686 {
 687         pthread_t p;
 688         if (thread == NULL) return false;
 689 loop:
 690         _PTHREAD_LOCK(_pthread_list_lock);
 691         TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
 692                 if (p != thread) continue;
 693                 int state = os_atomic_load(&p->cancel_state, relaxed);
 694                 if (os_likely(state & _PTHREAD_CANCEL_INITIALIZED)) {
 695                         if (os_unlikely(p->sig != _PTHREAD_SIG)) {
 696                                 PTHREAD_CLIENT_CRASH(0, "pthread_t was corrupted");
 697                         }
 698                         return true;
 699                 }
 700                 _PTHREAD_UNLOCK(_pthread_list_lock);
 701                 thread_switch(_pthread_kernel_thread(p),
 702                                           SWITCH_OPTION_OSLOCK_DEPRESS, 1);
 703                 goto loop;
 704         }
 705         _PTHREAD_UNLOCK(_pthread_list_lock);
 706
 707         return false;
 708 }
 709
 710 PTHREAD_ALWAYS_INLINE
 711 static inline bool
 712 _pthread_is_valid(pthread_t thread, mach_port_t *portp)
 713 {
 714         mach_port_t kport = MACH_PORT_NULL;
 715         bool valid;
 716
 717         if (thread == pthread_self()) {
 718                 valid = true;
 719                 kport = _pthread_kernel_thread(thread);
 720         } else if (!_pthread_validate_thread_and_list_lock(thread)) {
 721                 valid = false;
 722         } else {
 723                 kport = _pthread_kernel_thread(thread);
 724                 valid = true;
 725                 _PTHREAD_UNLOCK(_pthread_list_lock);
 726         }
 727
 728         if (portp != NULL) {
 729                 *portp = kport;
 730         }
 731         return valid;
 732 }
 733
 734 PTHREAD_ALWAYS_INLINE
 735 static inline void*
 736 _pthread_atomic_xchg_ptr_inline(void **p, void *v)
 737 {
 738         return os_atomic_xchg(p, v, seq_cst);
 739 }
 740
 741 PTHREAD_ALWAYS_INLINE
 742 static inline uint32_t
 743 _pthread_atomic_xchg_uint32_relaxed_inline(uint32_t *p,uint32_t v)
 744 {
 745         return os_atomic_xchg(p, v, relaxed);
 746 }
 747
 748 #define _pthread_atomic_xchg_ptr(p, v) \
 749                 _pthread_atomic_xchg_ptr_inline(p, v)
 750 #define _pthread_atomic_xchg_uint32_relaxed(p, v) \
 751                 _pthread_atomic_xchg_uint32_relaxed_inline(p, v)
 752
 753 #endif /* _POSIX_PTHREAD_INTERNALS_H */