src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <mach/mach.h>
  71 #include <mach/mach_error.h>
  72 #include <sys/queue.h>
  73 #include <pthread/bsdthread_private.h>
  74 #include <pthread/workqueue_syscalls.h>
  75
  76 #define __OS_EXPOSE_INTERNALS__ 1
  77 #include <os/internal/internal_shared.h>
  78 #include <os/once_private.h>
  79
  80 #if TARGET_IPHONE_SIMULATOR
  81 #error Unsupported target
  82 #endif
  83
  84
  85 #define PTHREAD_INTERNAL_CRASH(c, x) do { \
  86                 _os_set_crash_log_cause_and_message((c), \
  87                                 "BUG IN LIBPTHREAD: " x); \
  88                 __builtin_trap(); \
  89         } while (0)
  90
  91 #define PTHREAD_CLIENT_CRASH(c, x) do { \
  92                 _os_set_crash_log_cause_and_message((c), \
  93                                 "BUG IN CLIENT OF LIBPTHREAD: " x); \
  94                 __builtin_trap(); \
  95         } while (0)
  96
  97 #ifndef __POSIX_LIB__
  98 #define __POSIX_LIB__
  99 #endif
 100
 101 #ifndef PTHREAD_LAYOUT_SPI
 102 #define PTHREAD_LAYOUT_SPI 1
 103 #endif
 104
 105 #include "posix_sched.h"
 106 #include "tsd_private.h"
 107 #include "spinlock_private.h"
 108
 109 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 110 #define PTHREAD_EXTERN extern
 111 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 112 #define PTHREAD_NOEXPORT_VARIANT
 113 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 114 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 115 #define PTHREAD_NOINLINE __attribute__((noinline))
 116 #define PTHREAD_WEAK __attribute__((weak))
 117 #define PTHREAD_USED __attribute__((used))
 118 #define PTHREAD_NOT_TAIL_CALLED __attribute__((__not_tail_called__))
 119
 120
 121 #define OS_UNFAIR_LOCK_INLINE 1
 122 #include <os/lock_private.h>
 123 typedef os_unfair_lock _pthread_lock;
 124 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 125 #define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
 126 #define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 127 #define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
 128 #define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
 129 #define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
 130
 131 #define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO)
 132
 133 extern int __is_threaded;
 134 extern int __unix_conforming;
 135
 136 // List of all pthreads in the process.
 137 TAILQ_HEAD(__pthread_list, _pthread);
 138 PTHREAD_NOEXPORT extern struct __pthread_list __pthread_head;
 139
 140 // Lock protects access to above list.
 141 PTHREAD_NOEXPORT extern _pthread_lock _pthread_list_lock;
 142
 143 PTHREAD_NOEXPORT extern uint32_t _main_qos;
 144
 145 #if PTHREAD_DEBUG_LOG
 146 #include <mach/mach_time.h>
 147 PTHREAD_NOEXPORT extern int _pthread_debuglog;
 148 PTHREAD_NOEXPORT extern uint64_t _pthread_debugstart;
 149 #endif
 150
 151 /*
 152  * Compiled-in limits
 153  */
 154 #if TARGET_OS_EMBEDDED
 155 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 156 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 157 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 158 #else
 159 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 160 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 161 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 162 #endif
 163
 164 #if defined(__arm64__)
 165 /* Pull the pthread_t into the same page as the top of the stack so we dirty one less page.
 166  * <rdar://problem/19941744> The _pthread struct at the top of the stack shouldn't be page-aligned
 167  */
 168 #define PTHREAD_T_OFFSET (12*1024)
 169 #else
 170 #define PTHREAD_T_OFFSET 0
 171 #endif
 172
 173 #define MAXTHREADNAMESIZE       64
 174 #define _PTHREAD_T
 175 typedef struct _pthread {
 176         //
 177         // ABI - These fields are externally known as struct _opaque_pthread_t.
 178         //
 179         long sig; // _PTHREAD_SIG
 180         struct __darwin_pthread_handler_rec *__cleanup_stack;
 181
 182         //
 183         // SPI - These fields are private.
 184         //
 185
 186         //
 187         // Fields protected by _pthread_list_lock
 188         //
 189
 190         TAILQ_ENTRY(_pthread) tl_plist; // global thread list [aligned]
 191         struct pthread_join_context_s *tl_join_ctx;
 192         void *tl_exit_value;
 193         uint32_t tl_policy:8,
 194                         tl_joinable:1,
 195                         tl_joiner_cleans_up:1,
 196                         tl_has_custom_stack:1,
 197                         __tl_pad:21;
 198         // MACH_PORT_NULL if no joiner
 199         // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner
 200         // MACH_PORT_DEAD if the thread exited
 201         uint32_t tl_exit_gate;
 202         struct sched_param tl_param;
 203         void *__unused_padding;
 204
 205         //
 206         // Fields protected by pthread_t::lock
 207         //
 208
 209         _pthread_lock lock;
 210         uint16_t max_tsd_key;
 211         uint16_t inherit:8,
 212                         kernalloc:1,
 213                         schedset:1,
 214                         wqthread:1,
 215                         wqkillset:1,
 216                         __flags_pad:4;
 217
 218         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 219
 220         void *(*fun)(void *);   // thread start routine
 221         void *arg;                              // thread start routine argument
 222         int   wq_nevents;               // wqthreads (workloop / kevent)
 223         bool  wq_outsideqos;
 224         uint8_t canceled;               // 4597450 set if conformant cancelation happened
 225         uint16_t cancel_state;  // whether the thread can be canceled [atomic]
 226         errno_t cancel_error;
 227         errno_t err_no;                 // thread-local errno
 228
 229         void *stackaddr;                // base of the stack (page aligned)
 230         void *stackbottom;              // stackaddr - stacksize
 231         void *freeaddr;                 // stack/thread allocation base address
 232         size_t freesize;                // stack/thread allocation size
 233         size_t guardsize;               // guard page size in bytes
 234
 235         // tsd-base relative accessed elements
 236         __attribute__((aligned(8)))
 237         uint64_t thread_id;     // 64-bit unique thread id
 238
 239         /* Thread Specific Data slots
 240          *
 241          * The offset of this field from the start of the structure is difficult to
 242          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 243          * the value into their source.  Newer versions of mono will fall back to
 244          * scanning to determine it at runtime, but there's lots of software built
 245          * with older mono that won't.  We will have to break them someday...
 246          */
 247         __attribute__ ((aligned (16)))
 248         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 249 } *pthread_t;
 250
 251 #define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1)
 252 struct _pthread_attr_t {
 253         long   sig;
 254         size_t guardsize; // size in bytes of stack overflow guard area
 255         void  *stackaddr; // stack base; vm_page_size aligned
 256         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 257         union {
 258                 struct sched_param param; // [aligned]
 259                 unsigned long qosclass; // pthread_priority_t
 260         };
 261         uint32_t
 262                 detached:8,
 263                 inherit:8,
 264                 policy:8,
 265                 schedset:1,
 266                 qosset:1,
 267                 policyset:1,
 268                 cpupercentset:1,
 269                 defaultguardpage:1,
 270                 unused:3;
 271         uint32_t
 272                 cpupercent:8,
 273                 refillms:24;
 274 #if defined(__LP64__)
 275         uint32_t _reserved[4];
 276 #else
 277         uint32_t _reserved[2];
 278 #endif
 279 };
 280
 281 /*
 282  * Mutex attributes
 283  */
 284
 285 #define _PTHREAD_MUTEXATTR_T
 286 typedef struct {
 287         long sig;
 288         int prioceiling;
 289         uint32_t protocol:2,
 290                 type:2,
 291                 pshared:2,
 292                 opt:3,
 293                 unused:23;
 294 } pthread_mutexattr_t;
 295
 296 struct _pthread_mutex_options {
 297         uint32_t protocol:2,
 298                 type:2,
 299                 pshared:2,
 300                 policy:3,
 301                 hold:2,
 302                 misalign:1,
 303                 notify:1,
 304                 mutex:1,
 305                 unused:2,
 306                 lock_count:16;
 307 };
 308 //
 309 #define _PTHREAD_MUTEX_POLICY_LAST              (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1)
 310 #define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE 1
 311 #define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT 2
 312 #define _PTHREAD_MTX_OPT_POLICY_DEFAULT _PTHREAD_MTX_OPT_POLICY_FIRSTFIT
 313 // The following _pthread_mutex_options defintions exist in synch_internal.h
 314 // such that the kernel extension can test for flags. They must be kept in
 315 // sync with the bit values in the struct above.
 316 // _PTHREAD_MTX_OPT_PSHARED 0x010
 317 // _PTHREAD_MTX_OPT_NOTIFY 0x1000
 318 // _PTHREAD_MTX_OPT_MUTEX 0x2000
 319
 320 // The fixed mask is used to mask out portions of the mutex options that
 321 // change on a regular basis (notify, lock_count).
 322 #define _PTHREAD_MTX_OPT_FIXED_MASK     0x27ff
 323
 324 typedef struct {
 325         long sig;
 326         _pthread_lock lock;
 327         union {
 328                 uint32_t value;
 329                 struct _pthread_mutex_options options;
 330         } mtxopts;
 331         int16_t prioceiling;
 332         int16_t priority;
 333 #if defined(__LP64__)
 334         uint32_t _pad;
 335 #endif
 336         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 337         uint32_t m_seq[2]; // mutex sequence id
 338         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 339 #if defined(__LP64__)
 340         uint32_t _reserved[4];
 341 #else
 342         uint32_t _reserved[1];
 343 #endif
 344 } _pthread_mutex;
 345
 346
 347 #define _PTHREAD_CONDATTR_T
 348 typedef struct {
 349         long sig;
 350         uint32_t pshared:2,
 351                 unsupported:30;
 352 } pthread_condattr_t;
 353
 354
 355 typedef struct {
 356         long sig;
 357         _pthread_lock lock;
 358         uint32_t unused:29,
 359                 misalign:1,
 360                 pshared:2;
 361         _pthread_mutex *busy;
 362         uint32_t c_seq[3];
 363 #if defined(__LP64__)
 364         uint32_t _reserved[3];
 365 #endif
 366 } _pthread_cond;
 367
 368
 369 #define _PTHREAD_ONCE_T
 370 typedef struct {
 371         long sig;
 372         os_once_t once;
 373 } pthread_once_t;
 374
 375
 376 #define _PTHREAD_RWLOCKATTR_T
 377 typedef struct {
 378         long sig;
 379         int pshared;
 380 #if defined(__LP64__)
 381         uint32_t _reserved[3];
 382 #else
 383         uint32_t _reserved[2];
 384 #endif
 385 } pthread_rwlockattr_t;
 386
 387
 388 typedef struct {
 389         long sig;
 390         _pthread_lock lock;
 391         uint32_t unused:29,
 392                         misalign:1,
 393                         pshared:2;
 394         uint32_t rw_flags;
 395 #if defined(__LP64__)
 396         uint32_t _pad;
 397 #endif
 398         uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock
 399         uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary)
 400         uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here
 401 #if defined(__LP64__)
 402         uint32_t _reserved[34];
 403 #else
 404         uint32_t _reserved[18];
 405 #endif
 406 } _pthread_rwlock;
 407
 408 #include "pthread.h"
 409 #include "pthread_spis.h"
 410
 411 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 412                 "Incorrect _pthread_mutex structure size");
 413
 414 _Static_assert(sizeof(_pthread_rwlock) == sizeof(pthread_rwlock_t),
 415                 "Incorrect _pthread_rwlock structure size");
 416
 417 // Internal references to pthread_self() use TSD slot 0 directly.
 418 inline static pthread_t __attribute__((__pure__))
 419 _pthread_self_direct(void)
 420 {
 421         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 422 }
 423 #define pthread_self() _pthread_self_direct()
 424
 425 PTHREAD_ALWAYS_INLINE
 426 inline static uint64_t __attribute__((__pure__))
 427 _pthread_selfid_direct(void)
 428 {
 429         return (_pthread_self_direct())->thread_id;
 430 }
 431
 432 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 433 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 434 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 435 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 436 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 437 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 438
 439 #define _PTHREAD_NO_SIG                 0x00000000
 440 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 441 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 442 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 443 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 444 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 445 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 446 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 447 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 448 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 449 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 450 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 451 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 452 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 453 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 454 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 455 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 456 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 457 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 458 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 459 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 460 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 461
 462
 463 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 464 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 465 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 466
 467 #if defined(DEBUG)
 468 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 469 #else
 470 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 471 #endif
 472 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 473
 474 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 475 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 476 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 477
 478 extern boolean_t swtch_pri(int);
 479
 480 #include "kern/kern_internal.h"
 481
 482 /* Prototypes. */
 483
 484 /* Internal globals. */
 485 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 486
 487 PTHREAD_NOEXPORT int _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 488
 489 /* internally redirected upcalls. */
 490 PTHREAD_NOEXPORT void* malloc(size_t);
 491 PTHREAD_NOEXPORT void free(void*);
 492
 493 /* syscall interfaces */
 494 extern uint32_t __psynch_mutexwait(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 495 extern uint32_t __psynch_mutexdrop(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 496
 497 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 498 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 499 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 500 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 501 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 502 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 503 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 504 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 505 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 506 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 507 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 508 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 509 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 510
 511 PTHREAD_EXTERN
 512 int
 513 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 514
 515 PTHREAD_NOEXPORT
 516 void
 517 _pthread_deallocate(pthread_t t, bool from_mach_thread);
 518
 519 PTHREAD_NOEXPORT
 520 thread_qos_t
 521 _pthread_qos_class_to_thread_qos(qos_class_t qos);
 522
 523 PTHREAD_NOEXPORT
 524 void
 525 _pthread_set_main_qos(pthread_priority_t qos);
 526
 527 PTHREAD_NOEXPORT
 528 void
 529 _pthread_key_global_init(const char *envp[]);
 530
 531 PTHREAD_NOEXPORT
 532 void
 533 _pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data);
 534
 535 PTHREAD_EXPORT
 536 void
 537 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 538
 539 PTHREAD_EXPORT
 540 void
 541 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 542
 543 PTHREAD_NOEXPORT
 544 void
 545 _pthread_main_thread_init(pthread_t p);
 546
 547 PTHREAD_NOEXPORT
 548 void
 549 _pthread_main_thread_postfork_init(pthread_t p);
 550
 551 PTHREAD_NOEXPORT
 552 void
 553 _pthread_bsdthread_init(struct _pthread_registration_data *data);
 554
 555 PTHREAD_NOEXPORT_VARIANT
 556 void
 557 _pthread_clear_qos_tsd(mach_port_t thread_port);
 558
 559 #define PTHREAD_CONFORM_DARWIN_LEGACY     0
 560 #define PTHREAD_CONFORM_UNIX03_NOCANCEL   1
 561 #define PTHREAD_CONFORM_UNIX03_CANCELABLE 2
 562
 563 PTHREAD_NOEXPORT_VARIANT
 564 void
 565 _pthread_testcancel(int conforming);
 566
 567 PTHREAD_EXPORT
 568 void
 569 _pthread_exit_if_canceled(int error);
 570
 571 PTHREAD_NOEXPORT
 572 void
 573 _pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport);
 574
 575 PTHREAD_NOEXPORT
 576 void
 577 _pthread_setcancelstate_exit(pthread_t self, void *value_ptr);
 578
 579 PTHREAD_NOEXPORT
 580 semaphore_t
 581 _pthread_joiner_prepost_wake(pthread_t thread);
 582
 583 PTHREAD_ALWAYS_INLINE
 584 static inline mach_port_t
 585 _pthread_kernel_thread(pthread_t t)
 586 {
 587         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 588 }
 589
 590 PTHREAD_ALWAYS_INLINE
 591 static inline void
 592 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 593 {
 594         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 595 }
 596
 597 #ifdef DEBUG
 598 #define PTHREAD_DEBUG_ASSERT(b) \
 599                 do { \
 600                         if (os_unlikely(!(b))) { \
 601                                 PTHREAD_INTERNAL_CRASH(0, "Assertion failed: " #b); \
 602                         } \
 603                 } while (0)
 604 #else
 605 #define PTHREAD_DEBUG_ASSERT(b) ((void)0)
 606 #endif
 607
 608 #include <os/semaphore_private.h>
 609 #include <os/alloc_once_private.h>
 610
 611 struct pthread_atfork_entry {
 612         void (*prepare)(void);
 613         void (*parent)(void);
 614         void (*child)(void);
 615 };
 616
 617 #define PTHREAD_ATFORK_INLINE_MAX 10
 618 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 619
 620 struct pthread_globals_s {
 621         // atfork.c
 622         pthread_t psaved_self;
 623         _pthread_lock psaved_self_global_lock;
 624         _pthread_lock pthread_atfork_lock;
 625
 626         size_t atfork_count;
 627         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 628         struct pthread_atfork_entry *atfork;
 629         uint16_t qmp_logical[THREAD_QOS_LAST];
 630         uint16_t qmp_physical[THREAD_QOS_LAST];
 631
 632 };
 633 typedef struct pthread_globals_s *pthread_globals_t;
 634
 635 __attribute__((__pure__))
 636 static inline pthread_globals_t
 637 _pthread_globals(void)
 638 {
 639         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 640                              sizeof(struct pthread_globals_s),
 641                              NULL);
 642 }
 643
 644 #pragma mark _pthread_mutex_check_signature
 645
 646 PTHREAD_ALWAYS_INLINE
 647 static inline bool
 648 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 649 {
 650         return (mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 651 }
 652
 653 PTHREAD_ALWAYS_INLINE
 654 static inline bool
 655 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 656 {
 657         return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 658 }
 659
 660 PTHREAD_ALWAYS_INLINE
 661 static inline bool
 662 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 663 {
 664         return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) ==
 665                         _PTHREAD_MUTEX_SIG_init_CMP);
 666 }
 667
 668 #pragma mark _pthread_rwlock_check_signature
 669
 670 PTHREAD_ALWAYS_INLINE
 671 static inline bool
 672 _pthread_rwlock_check_signature(_pthread_rwlock *rwlock)
 673 {
 674         return (rwlock->sig == _PTHREAD_RWLOCK_SIG);
 675 }
 676
 677 PTHREAD_ALWAYS_INLINE
 678 static inline bool
 679 _pthread_rwlock_check_signature_init(_pthread_rwlock *rwlock)
 680 {
 681         return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init);
 682 }
 683
 684 /*
 685  * ALWAYS called without list lock and return with list lock held on success
 686  *
 687  * This weird calling convention exists because this function will sometimes
 688  * drop the lock, and it's best callers don't have to remember this.
 689  */
 690 PTHREAD_ALWAYS_INLINE
 691 static inline bool
 692 _pthread_validate_thread_and_list_lock(pthread_t thread)
 693 {
 694         pthread_t p;
 695         if (thread == NULL) return false;
 696         _PTHREAD_LOCK(_pthread_list_lock);
 697         TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
 698                 if (p != thread) continue;
 699                 if (os_unlikely(p->sig != _PTHREAD_SIG)) {
 700                         PTHREAD_CLIENT_CRASH(0, "pthread_t was corrupted");
 701                 }
 702                 return true;
 703         }
 704         _PTHREAD_UNLOCK(_pthread_list_lock);
 705
 706         return false;
 707 }
 708
 709 PTHREAD_ALWAYS_INLINE
 710 static inline bool
 711 _pthread_is_valid(pthread_t thread, mach_port_t *portp)
 712 {
 713         mach_port_t kport = MACH_PORT_NULL;
 714         bool valid;
 715
 716         if (thread == pthread_self()) {
 717                 valid = true;
 718                 kport = _pthread_kernel_thread(thread);
 719         } else if (!_pthread_validate_thread_and_list_lock(thread)) {
 720                 valid = false;
 721         } else {
 722                 kport = _pthread_kernel_thread(thread);
 723                 valid = true;
 724                 _PTHREAD_UNLOCK(_pthread_list_lock);
 725         }
 726
 727         if (portp != NULL) {
 728                 *portp = kport;
 729         }
 730         return valid;
 731 }
 732
 733 PTHREAD_ALWAYS_INLINE
 734 static inline void*
 735 _pthread_atomic_xchg_ptr_inline(void **p, void *v)
 736 {
 737         return os_atomic_xchg(p, v, seq_cst);
 738 }
 739
 740 PTHREAD_ALWAYS_INLINE
 741 static inline uint32_t
 742 _pthread_atomic_xchg_uint32_relaxed_inline(uint32_t *p,uint32_t v)
 743 {
 744         return os_atomic_xchg(p, v, relaxed);
 745 }
 746
 747 #define _pthread_atomic_xchg_ptr(p, v) \
 748                 _pthread_atomic_xchg_ptr_inline(p, v)
 749 #define _pthread_atomic_xchg_uint32_relaxed(p, v) \
 750                 _pthread_atomic_xchg_uint32_relaxed_inline(p, v)
 751
 752 #endif /* _POSIX_PTHREAD_INTERNALS_H */