src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <mach/mach.h>
  71 #include <mach/mach_error.h>
  72 #include <sys/queue.h>
  73 #include <sys/reason.h>
  74 #include <pthread/bsdthread_private.h>
  75 #include <pthread/workqueue_syscalls.h>
  76
  77 #define __OS_EXPOSE_INTERNALS__ 1
  78 #include <os/internal/internal_shared.h>
  79 #include <os/once_private.h>
  80 #include <os/reason_private.h>
  81
  82 #if TARGET_IPHONE_SIMULATOR
  83 #error Unsupported target
  84 #endif
  85
  86
  87 #define PTHREAD_INTERNAL_CRASH(c, x) do { \
  88                 _os_set_crash_log_cause_and_message((c), \
  89                                 "BUG IN LIBPTHREAD: " x); \
  90                 __builtin_trap(); \
  91         } while (0)
  92
  93 #define PTHREAD_CLIENT_CRASH(c, x) do { \
  94                 _os_set_crash_log_cause_and_message((c), \
  95                                 "BUG IN CLIENT OF LIBPTHREAD: " x); \
  96                 __builtin_trap(); \
  97         } while (0)
  98
  99 #ifndef __POSIX_LIB__
 100 #define __POSIX_LIB__
 101 #endif
 102
 103 #ifndef PTHREAD_LAYOUT_SPI
 104 #define PTHREAD_LAYOUT_SPI 1
 105 #endif
 106
 107 #include "posix_sched.h"
 108 #include "tsd_private.h"
 109 #include "spinlock_private.h"
 110
 111 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 112 #define PTHREAD_EXTERN extern
 113 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 114 #define PTHREAD_NOEXPORT_VARIANT
 115 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 116 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 117 #define PTHREAD_NOINLINE __attribute__((noinline))
 118 #define PTHREAD_WEAK __attribute__((weak))
 119 #define PTHREAD_USED __attribute__((used))
 120 #define PTHREAD_NOT_TAIL_CALLED __attribute__((__not_tail_called__))
 121
 122
 123 #define OS_UNFAIR_LOCK_INLINE 1
 124 #include <os/lock_private.h>
 125 typedef os_unfair_lock _pthread_lock;
 126 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 127 #define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
 128 #define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 129 #define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
 130 #define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
 131 #define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
 132
 133 #define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO)
 134
 135 extern int __is_threaded;
 136 extern int __unix_conforming;
 137 PTHREAD_NOEXPORT
 138 extern uintptr_t _pthread_ptr_munge_token;
 139
 140 // List of all pthreads in the process.
 141 TAILQ_HEAD(__pthread_list, _pthread);
 142 PTHREAD_NOEXPORT extern struct __pthread_list __pthread_head;
 143
 144 // Lock protects access to above list.
 145 PTHREAD_NOEXPORT extern _pthread_lock _pthread_list_lock;
 146
 147 PTHREAD_NOEXPORT extern uint32_t _main_qos;
 148
 149 #if PTHREAD_DEBUG_LOG
 150 #include <mach/mach_time.h>
 151 PTHREAD_NOEXPORT extern int _pthread_debuglog;
 152 PTHREAD_NOEXPORT extern uint64_t _pthread_debugstart;
 153 #endif
 154
 155 /*
 156  * Compiled-in limits
 157  */
 158 #if TARGET_OS_EMBEDDED
 159 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 160 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 161 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 162 #else
 163 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 164 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 165 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 166 #endif
 167
 168 #if defined(__arm64__)
 169 /* Pull the pthread_t into the same page as the top of the stack so we dirty one less page.
 170  * <rdar://problem/19941744> The _pthread struct at the top of the stack shouldn't be page-aligned
 171  */
 172 #define PTHREAD_T_OFFSET (12*1024)
 173 #else
 174 #define PTHREAD_T_OFFSET 0
 175 #endif
 176
 177 #define MAXTHREADNAMESIZE       64
 178 #define _PTHREAD_T
 179 typedef struct _pthread {
 180         //
 181         // ABI - These fields are externally known as struct _opaque_pthread_t.
 182         //
 183         long sig;
 184         struct __darwin_pthread_handler_rec *__cleanup_stack;
 185
 186         //
 187         // SPI - These fields are private.
 188         //
 189
 190         //
 191         // Fields protected by _pthread_list_lock
 192         //
 193
 194         TAILQ_ENTRY(_pthread) tl_plist; // global thread list [aligned]
 195         struct pthread_join_context_s *tl_join_ctx;
 196         void *tl_exit_value;
 197         uint32_t tl_policy:8,
 198                         tl_joinable:1,
 199                         tl_joiner_cleans_up:1,
 200                         tl_has_custom_stack:1,
 201                         __tl_pad:21;
 202         // MACH_PORT_NULL if no joiner
 203         // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner
 204         // MACH_PORT_DEAD if the thread exited
 205         uint32_t tl_exit_gate;
 206         struct sched_param tl_param;
 207         void *__unused_padding;
 208
 209         //
 210         // Fields protected by pthread_t::lock
 211         //
 212
 213         _pthread_lock lock;
 214         uint16_t max_tsd_key;
 215         uint16_t inherit:8,
 216                         kernalloc:1,
 217                         schedset:1,
 218                         wqthread:1,
 219                         wqkillset:1,
 220                         __flags_pad:4;
 221
 222         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 223
 224         void *(*fun)(void *);   // thread start routine
 225         void *arg;                              // thread start routine argument
 226         int   wq_nevents;               // wqthreads (workloop / kevent)
 227         bool  wq_outsideqos;
 228         uint8_t canceled;               // 4597450 set if conformant cancelation happened
 229         uint16_t cancel_state;  // whether the thread can be canceled [atomic]
 230         errno_t cancel_error;
 231         errno_t err_no;                 // thread-local errno
 232
 233         void *stackaddr;                // base of the stack (page aligned)
 234         void *stackbottom;              // stackaddr - stacksize
 235         void *freeaddr;                 // stack/thread allocation base address
 236         size_t freesize;                // stack/thread allocation size
 237         size_t guardsize;               // guard page size in bytes
 238
 239         // tsd-base relative accessed elements
 240         __attribute__((aligned(8)))
 241         uint64_t thread_id;     // 64-bit unique thread id
 242
 243         /* Thread Specific Data slots
 244          *
 245          * The offset of this field from the start of the structure is difficult to
 246          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 247          * the value into their source.  Newer versions of mono will fall back to
 248          * scanning to determine it at runtime, but there's lots of software built
 249          * with older mono that won't.  We will have to break them someday...
 250          */
 251         __attribute__ ((aligned (16)))
 252         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 253 } *pthread_t;
 254
 255 #define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1)
 256 struct _pthread_attr_t {
 257         long   sig;
 258         size_t guardsize; // size in bytes of stack overflow guard area
 259         void  *stackaddr; // stack base; vm_page_size aligned
 260         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 261         union {
 262                 struct sched_param param; // [aligned]
 263                 unsigned long qosclass; // pthread_priority_t
 264         };
 265         uint32_t
 266                 detached:8,
 267                 inherit:8,
 268                 policy:8,
 269                 schedset:1,
 270                 qosset:1,
 271                 policyset:1,
 272                 cpupercentset:1,
 273                 defaultguardpage:1,
 274                 unused:3;
 275         uint32_t
 276                 cpupercent:8,
 277                 refillms:24;
 278 #if defined(__LP64__)
 279         uint32_t _reserved[4];
 280 #else
 281         uint32_t _reserved[2];
 282 #endif
 283 };
 284
 285 /*
 286  * Mutex attributes
 287  */
 288
 289 #define _PTHREAD_MUTEXATTR_T
 290 typedef struct {
 291         long sig;
 292         int prioceiling;
 293         uint32_t protocol:2,
 294                 type:2,
 295                 pshared:2,
 296                 opt:3,
 297                 unused:23;
 298 } pthread_mutexattr_t;
 299
 300 struct _pthread_mutex_options {
 301         uint32_t protocol:2,
 302                 type:2,
 303                 pshared:2,
 304                 policy:3,
 305                 hold:2,
 306                 misalign:1,
 307                 notify:1,
 308                 mutex:1,
 309                 unused:2,
 310                 lock_count:16;
 311 };
 312 //
 313 #define _PTHREAD_MUTEX_POLICY_LAST              (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1)
 314 #define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE 1
 315 #define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT 2
 316 #define _PTHREAD_MTX_OPT_POLICY_DEFAULT _PTHREAD_MTX_OPT_POLICY_FIRSTFIT
 317 // The following _pthread_mutex_options defintions exist in synch_internal.h
 318 // such that the kernel extension can test for flags. They must be kept in
 319 // sync with the bit values in the struct above.
 320 // _PTHREAD_MTX_OPT_PSHARED 0x010
 321 // _PTHREAD_MTX_OPT_NOTIFY 0x1000
 322 // _PTHREAD_MTX_OPT_MUTEX 0x2000
 323
 324 // The fixed mask is used to mask out portions of the mutex options that
 325 // change on a regular basis (notify, lock_count).
 326 #define _PTHREAD_MTX_OPT_FIXED_MASK     0x27ff
 327
 328 typedef struct {
 329         long sig;
 330         _pthread_lock lock;
 331         union {
 332                 uint32_t value;
 333                 struct _pthread_mutex_options options;
 334         } mtxopts;
 335         int16_t prioceiling;
 336         int16_t priority;
 337 #if defined(__LP64__)
 338         uint32_t _pad;
 339 #endif
 340         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 341         uint32_t m_seq[2]; // mutex sequence id
 342         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 343 #if defined(__LP64__)
 344         uint32_t _reserved[4];
 345 #else
 346         uint32_t _reserved[1];
 347 #endif
 348 } _pthread_mutex;
 349
 350
 351 #define _PTHREAD_CONDATTR_T
 352 typedef struct {
 353         long sig;
 354         uint32_t pshared:2,
 355                 unsupported:30;
 356 } pthread_condattr_t;
 357
 358
 359 typedef struct {
 360         long sig;
 361         _pthread_lock lock;
 362         uint32_t unused:29,
 363                 misalign:1,
 364                 pshared:2;
 365         _pthread_mutex *busy;
 366         uint32_t c_seq[3];
 367 #if defined(__LP64__)
 368         uint32_t _reserved[3];
 369 #endif
 370 } _pthread_cond;
 371
 372
 373 #define _PTHREAD_ONCE_T
 374 typedef struct {
 375         long sig;
 376         os_once_t once;
 377 } pthread_once_t;
 378
 379
 380 #define _PTHREAD_RWLOCKATTR_T
 381 typedef struct {
 382         long sig;
 383         int pshared;
 384 #if defined(__LP64__)
 385         uint32_t _reserved[3];
 386 #else
 387         uint32_t _reserved[2];
 388 #endif
 389 } pthread_rwlockattr_t;
 390
 391
 392 typedef struct {
 393         long sig;
 394         _pthread_lock lock;
 395         uint32_t unused:29,
 396                         misalign:1,
 397                         pshared:2;
 398         uint32_t rw_flags;
 399 #if defined(__LP64__)
 400         uint32_t _pad;
 401 #endif
 402         uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock
 403         uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary)
 404         uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here
 405 #if defined(__LP64__)
 406         uint32_t _reserved[34];
 407 #else
 408         uint32_t _reserved[18];
 409 #endif
 410 } _pthread_rwlock;
 411
 412 #include "pthread.h"
 413 #include "pthread_spis.h"
 414
 415 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 416                 "Incorrect _pthread_mutex structure size");
 417
 418 _Static_assert(sizeof(_pthread_rwlock) == sizeof(pthread_rwlock_t),
 419                 "Incorrect _pthread_rwlock structure size");
 420
 421 // Internal references to pthread_self() use TSD slot 0 directly.
 422 __header_always_inline __pure2 pthread_t
 423 _pthread_self_direct(void)
 424 {
 425 #if TARGET_OS_SIMULATOR || defined(__i386__) || defined(__x86_64__)
 426         return (pthread_t)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 427 #elif defined(__arm__) || defined(__arm64__)
 428         uintptr_t tsd_base = (uintptr_t)_os_tsd_get_base();
 429         return (pthread_t)(tsd_base - offsetof(struct _pthread, tsd));
 430 #else
 431 #error unsupported architecture
 432 #endif
 433 }
 434 #define pthread_self() _pthread_self_direct()
 435
 436 PTHREAD_ALWAYS_INLINE
 437 inline static uint64_t __attribute__((__pure__))
 438 _pthread_selfid_direct(void)
 439 {
 440         return (_pthread_self_direct())->thread_id;
 441 }
 442
 443 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 444 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 445 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 446 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 447 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 448 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 449
 450 #define _PTHREAD_NO_SIG                 0x00000000
 451 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 452 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 453 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 454 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 455 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 456 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 457 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 458 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 459 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 460 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 461 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 462 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 463 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 464 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 465 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 466 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 467 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 468 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 469 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 470 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 471 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 472
 473
 474 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 475 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 476 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 477
 478 #if defined(DEBUG)
 479 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 480 #else
 481 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 482 #endif
 483 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 484
 485 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 486 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 487 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 488
 489 extern boolean_t swtch_pri(int);
 490
 491 #include "kern/kern_internal.h"
 492
 493 /* Prototypes. */
 494
 495 /* Internal globals. */
 496 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 497
 498 PTHREAD_NOEXPORT int _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 499
 500 /* internally redirected upcalls. */
 501 PTHREAD_NOEXPORT void* malloc(size_t);
 502 PTHREAD_NOEXPORT void free(void*);
 503
 504 /* syscall interfaces */
 505 extern uint32_t __psynch_mutexwait(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 506 extern uint32_t __psynch_mutexdrop(_pthread_mutex * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 507
 508 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 509 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 510 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 511 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 512 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 513 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 514 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 515 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 516 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 517 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 518 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 519 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 520 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 521
 522 PTHREAD_EXTERN
 523 int
 524 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 525
 526 PTHREAD_NOEXPORT
 527 void
 528 _pthread_deallocate(pthread_t t, bool from_mach_thread);
 529
 530 PTHREAD_NOEXPORT
 531 thread_qos_t
 532 _pthread_qos_class_to_thread_qos(qos_class_t qos);
 533
 534 PTHREAD_NOEXPORT
 535 void
 536 _pthread_set_main_qos(pthread_priority_t qos);
 537
 538 PTHREAD_NOEXPORT
 539 void
 540 _pthread_key_global_init(const char *envp[]);
 541
 542 PTHREAD_NOEXPORT
 543 void
 544 _pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data);
 545
 546 PTHREAD_EXPORT
 547 void
 548 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 549
 550 PTHREAD_EXPORT
 551 void
 552 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 553
 554 PTHREAD_NOEXPORT
 555 void
 556 _pthread_main_thread_init(pthread_t p);
 557
 558 PTHREAD_NOEXPORT
 559 void
 560 _pthread_main_thread_postfork_init(pthread_t p);
 561
 562 PTHREAD_NOEXPORT
 563 void
 564 _pthread_bsdthread_init(struct _pthread_registration_data *data);
 565
 566 PTHREAD_NOEXPORT_VARIANT
 567 void
 568 _pthread_clear_qos_tsd(mach_port_t thread_port);
 569
 570 #define PTHREAD_CONFORM_DARWIN_LEGACY     0
 571 #define PTHREAD_CONFORM_UNIX03_NOCANCEL   1
 572 #define PTHREAD_CONFORM_UNIX03_CANCELABLE 2
 573
 574 PTHREAD_NOEXPORT_VARIANT
 575 void
 576 _pthread_testcancel(int conforming);
 577
 578 PTHREAD_EXPORT
 579 void
 580 _pthread_exit_if_canceled(int error);
 581
 582 PTHREAD_NOEXPORT
 583 void
 584 _pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport);
 585
 586 PTHREAD_NOEXPORT
 587 void
 588 _pthread_setcancelstate_exit(pthread_t self, void *value_ptr);
 589
 590 PTHREAD_NOEXPORT
 591 semaphore_t
 592 _pthread_joiner_prepost_wake(pthread_t thread);
 593
 594 PTHREAD_ALWAYS_INLINE
 595 static inline mach_port_t
 596 _pthread_kernel_thread(pthread_t t)
 597 {
 598         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 599 }
 600
 601 PTHREAD_ALWAYS_INLINE
 602 static inline void
 603 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 604 {
 605         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 606 }
 607
 608 #ifdef DEBUG
 609 #define PTHREAD_DEBUG_ASSERT(b) \
 610                 do { \
 611                         if (os_unlikely(!(b))) { \
 612                                 PTHREAD_INTERNAL_CRASH(0, "Assertion failed: " #b); \
 613                         } \
 614                 } while (0)
 615 #else
 616 #define PTHREAD_DEBUG_ASSERT(b) ((void)0)
 617 #endif
 618
 619 #include <os/semaphore_private.h>
 620 #include <os/alloc_once_private.h>
 621
 622 struct pthread_atfork_entry {
 623         void (*prepare)(void);
 624         void (*parent)(void);
 625         void (*child)(void);
 626 };
 627
 628 #define PTHREAD_ATFORK_INLINE_MAX 10
 629 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 630
 631 struct pthread_globals_s {
 632         // atfork.c
 633         pthread_t psaved_self;
 634         _pthread_lock psaved_self_global_lock;
 635         _pthread_lock pthread_atfork_lock;
 636
 637         size_t atfork_count;
 638         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 639         struct pthread_atfork_entry *atfork;
 640         uint16_t qmp_logical[THREAD_QOS_LAST];
 641         uint16_t qmp_physical[THREAD_QOS_LAST];
 642
 643 };
 644 typedef struct pthread_globals_s *pthread_globals_t;
 645
 646 __attribute__((__pure__))
 647 static inline pthread_globals_t
 648 _pthread_globals(void)
 649 {
 650         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 651                              sizeof(struct pthread_globals_s),
 652                              NULL);
 653 }
 654
 655 #pragma mark _pthread_mutex_check_signature
 656
 657 PTHREAD_ALWAYS_INLINE
 658 static inline bool
 659 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 660 {
 661         return (mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 662 }
 663
 664 PTHREAD_ALWAYS_INLINE
 665 static inline bool
 666 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 667 {
 668         return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 669 }
 670
 671 PTHREAD_ALWAYS_INLINE
 672 static inline bool
 673 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 674 {
 675         return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) ==
 676                         _PTHREAD_MUTEX_SIG_init_CMP);
 677 }
 678
 679 #pragma mark _pthread_rwlock_check_signature
 680
 681 PTHREAD_ALWAYS_INLINE
 682 static inline bool
 683 _pthread_rwlock_check_signature(_pthread_rwlock *rwlock)
 684 {
 685         return (rwlock->sig == _PTHREAD_RWLOCK_SIG);
 686 }
 687
 688 PTHREAD_ALWAYS_INLINE
 689 static inline bool
 690 _pthread_rwlock_check_signature_init(_pthread_rwlock *rwlock)
 691 {
 692         return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init);
 693 }
 694
 695 PTHREAD_ALWAYS_INLINE
 696 static inline void
 697 _pthread_validate_signature(pthread_t thread)
 698 {
 699         pthread_t th  = (pthread_t)(thread->sig ^ _pthread_ptr_munge_token);
 700 #if __has_feature(ptrauth_calls)
 701         th = ptrauth_auth_data(th, ptrauth_key_process_dependent_data,
 702                         ptrauth_string_discriminator("pthread.signature"));
 703 #endif
 704         if (os_unlikely(th != thread)) {
 705                 /* OS_REASON_LIBSYSTEM_CODE_PTHREAD_CORRUPTION == 4 */
 706                 abort_with_reason(OS_REASON_LIBSYSTEM, 4, "pthread_t was corrupted", 0);
 707         }
 708 }
 709
 710 PTHREAD_ALWAYS_INLINE
 711 static inline void
 712 _pthread_init_signature(pthread_t thread)
 713 {
 714         pthread_t th = thread;
 715 #if __has_feature(ptrauth_calls)
 716         th = ptrauth_sign_unauthenticated(th, ptrauth_key_process_dependent_data,
 717                         ptrauth_string_discriminator("pthread.signature"));
 718 #endif
 719         thread->sig = (uintptr_t)th ^ _pthread_ptr_munge_token;
 720 }
 721
 722 /*
 723  * ALWAYS called without list lock and return with list lock held on success
 724  *
 725  * This weird calling convention exists because this function will sometimes
 726  * drop the lock, and it's best callers don't have to remember this.
 727  */
 728 PTHREAD_ALWAYS_INLINE
 729 static inline bool
 730 _pthread_validate_thread_and_list_lock(pthread_t thread)
 731 {
 732         pthread_t p;
 733         if (thread == NULL) return false;
 734         _PTHREAD_LOCK(_pthread_list_lock);
 735         TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
 736                 if (p != thread) continue;
 737                 _pthread_validate_signature(p);
 738                 return true;
 739         }
 740         _PTHREAD_UNLOCK(_pthread_list_lock);
 741
 742         return false;
 743 }
 744
 745 PTHREAD_ALWAYS_INLINE
 746 static inline bool
 747 _pthread_is_valid(pthread_t thread, mach_port_t *portp)
 748 {
 749         mach_port_t kport = MACH_PORT_NULL;
 750         bool valid;
 751
 752         if (thread == pthread_self()) {
 753                 _pthread_validate_signature(thread);
 754                 valid = true;
 755                 kport = _pthread_kernel_thread(thread);
 756         } else if (!_pthread_validate_thread_and_list_lock(thread)) {
 757                 valid = false;
 758         } else {
 759                 kport = _pthread_kernel_thread(thread);
 760                 valid = true;
 761                 _PTHREAD_UNLOCK(_pthread_list_lock);
 762         }
 763
 764         if (portp != NULL) {
 765                 *portp = kport;
 766         }
 767         return valid;
 768 }
 769
 770 PTHREAD_ALWAYS_INLINE
 771 static inline void*
 772 _pthread_atomic_xchg_ptr_inline(void **p, void *v)
 773 {
 774         return os_atomic_xchg(p, v, seq_cst);
 775 }
 776
 777 PTHREAD_ALWAYS_INLINE
 778 static inline uint32_t
 779 _pthread_atomic_xchg_uint32_relaxed_inline(uint32_t *p,uint32_t v)
 780 {
 781         return os_atomic_xchg(p, v, relaxed);
 782 }
 783
 784 #define _pthread_atomic_xchg_ptr(p, v) \
 785                 _pthread_atomic_xchg_ptr_inline(p, v)
 786 #define _pthread_atomic_xchg_uint32_relaxed(p, v) \
 787                 _pthread_atomic_xchg_uint32_relaxed_inline(p, v)
 788
 789 #endif /* _POSIX_PTHREAD_INTERNALS_H */