src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <_simple.h>
  63 #include <assert.h>
  64 #include <stddef.h>
  65 #include <stdint.h>
  66 #include <stdlib.h>
  67 #include <limits.h>
  68 #include <errno.h>
  69 #include <TargetConditionals.h>
  70 #include <mach/mach.h>
  71 #include <mach/mach_error.h>
  72 #include <sys/queue.h>
  73
  74 #define __OS_EXPOSE_INTERNALS__ 1
  75 #include <os/internal/internal_shared.h>
  76 #include <os/once_private.h>
  77
  78 #if TARGET_IPHONE_SIMULATOR
  79 #error Unsupported target
  80 #endif
  81
  82 #define PTHREAD_INTERNAL_CRASH(c, x) do { \
  83                 _os_set_crash_log_cause_and_message((c), \
  84                                 "BUG IN LIBPTHREAD: " x); \
  85                 __builtin_trap(); \
  86         } while (0)
  87
  88 #define PTHREAD_CLIENT_CRASH(c, x) do { \
  89                 _os_set_crash_log_cause_and_message((c), \
  90                                 "BUG IN CLIENT OF LIBPTHREAD: " x); \
  91                 __builtin_trap(); \
  92         } while (0)
  93
  94 #ifndef __POSIX_LIB__
  95 #define __POSIX_LIB__
  96 #endif
  97
  98 #ifndef PTHREAD_LAYOUT_SPI
  99 #define PTHREAD_LAYOUT_SPI 1
 100 #endif
 101
 102 #include "posix_sched.h"
 103 #include "tsd_private.h"
 104 #include "spinlock_private.h"
 105
 106 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 107 #define PTHREAD_EXTERN extern
 108 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 109 #define PTHREAD_NOEXPORT_VARIANT
 110 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 111 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 112 #define PTHREAD_NOINLINE __attribute__((noinline))
 113 #define PTHREAD_WEAK __attribute__((weak))
 114 #define PTHREAD_USED __attribute__((used))
 115 #define PTHREAD_NOT_TAIL_CALLED __attribute__((__not_tail_called__))
 116
 117
 118 #define OS_UNFAIR_LOCK_INLINE 1
 119 #include <os/lock_private.h>
 120 typedef os_unfair_lock _pthread_lock;
 121 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 122 #define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
 123 #define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
 124 #define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
 125 #define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
 126 #define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
 127
 128 // List of all pthreads in the process.
 129 TAILQ_HEAD(__pthread_list, _pthread);
 130 extern struct __pthread_list __pthread_head;
 131
 132 // Lock protects access to above list.
 133 extern _pthread_lock _pthread_list_lock;
 134
 135 extern int __is_threaded;
 136
 137 #if PTHREAD_DEBUG_LOG
 138 #include <mach/mach_time.h>
 139 extern int _pthread_debuglog;
 140 extern uint64_t _pthread_debugstart;
 141 #endif
 142
 143 /*
 144  * Compiled-in limits
 145  */
 146 #if TARGET_OS_EMBEDDED
 147 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 148 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 149 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 150 #else
 151 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 152 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 153 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 154 #endif
 155
 156 #define MAXTHREADNAMESIZE       64
 157 #define _PTHREAD_T
 158 typedef struct _pthread {
 159         //
 160         // ABI - These fields are externally known as struct _opaque_pthread_t.
 161         //
 162         long sig; // _PTHREAD_SIG
 163         struct __darwin_pthread_handler_rec *__cleanup_stack;
 164
 165         //
 166         // SPI - These fields are private.
 167         //
 168         // these fields are globally protected by _pthread_list_lock:
 169         uint32_t childrun:1,
 170                         parentcheck:1,
 171                         childexit:1,
 172                         pad3:29;
 173
 174         _pthread_lock lock; // protect access to everything below
 175         uint32_t detached:8,
 176                         inherit:8,
 177                         policy:8,
 178                         kernalloc:1,
 179                         schedset:1,
 180                         wqthread:1,
 181                         wqkillset:1,
 182                         pad:4;
 183
 184 #if defined(__LP64__)
 185         uint32_t pad0;
 186 #endif
 187
 188         void *(*fun)(void*);    // thread start routine
 189         void *arg;              // thread start routine argument
 190         void *exit_value;       // thread exit value storage
 191
 192         semaphore_t joiner_notify;      // pthread_join notification
 193
 194         int max_tsd_key;
 195         int cancel_state;       // whether the thread can be cancelled
 196         int cancel_error;
 197
 198         int err_no;             // thread-local errno
 199
 200         struct _pthread *joiner;
 201
 202         struct sched_param param;       // [aligned]
 203
 204         TAILQ_ENTRY(_pthread) plist;    // global thread list [aligned]
 205
 206         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 207
 208         void *stackaddr;        // base of the stack (page aligned)
 209         size_t stacksize;       // size of stack (page multiple and >= PTHREAD_STACK_MIN)
 210
 211         void* freeaddr;         // stack/thread allocation base address
 212         size_t freesize;        // stack/thread allocation size
 213         size_t guardsize;       // guard page size in bytes
 214
 215         // tsd-base relative accessed elements
 216         __attribute__((aligned(8)))
 217         uint64_t thread_id;     // 64-bit unique thread id
 218
 219         /* Thread Specific Data slots
 220          *
 221          * The offset of this field from the start of the structure is difficult to
 222          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 223          * the value into their source.  Newer versions of mono will fall back to
 224          * scanning to determine it at runtime, but there's lots of software built
 225          * with older mono that won't.  We will have to break them someday...
 226          */
 227         __attribute__ ((aligned (16)))
 228         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 229 } *pthread_t;
 230
 231
 232 struct _pthread_attr_t {
 233         long sig;
 234         _pthread_lock lock;
 235         uint32_t detached:8,
 236                 inherit:8,
 237                 policy:8,
 238                 fastpath:1,
 239                 schedset:1,
 240                 qosset:1,
 241                 unused:5;
 242         struct sched_param param; // [aligned]
 243         void *stackaddr; // stack base; vm_page_size aligned
 244         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 245         size_t guardsize; // size in bytes of stack overflow guard area
 246         unsigned long qosclass;
 247 #if defined(__LP64__)
 248         uint32_t _reserved[2];
 249 #else
 250         uint32_t _reserved[1];
 251 #endif
 252 };
 253
 254 /*
 255  * Mutex attributes
 256  */
 257 #define _PTHREAD_MUTEX_POLICY_NONE              0
 258 #define _PTHREAD_MUTEX_POLICY_FAIRSHARE         1
 259 #define _PTHREAD_MUTEX_POLICY_FIRSTFIT          2
 260 #define _PTHREAD_MUTEX_POLICY_REALTIME          3
 261 #define _PTHREAD_MUTEX_POLICY_ADAPTIVE          4
 262 #define _PTHREAD_MUTEX_POLICY_PRIPROTECT        5
 263 #define _PTHREAD_MUTEX_POLICY_PRIINHERIT        6
 264
 265 #define _PTHREAD_MUTEXATTR_T
 266 typedef struct {
 267         long sig;
 268         int prioceiling;
 269         uint32_t protocol:2,
 270                 type:2,
 271                 pshared:2,
 272                 policy:3,
 273                 unused:23;
 274 } pthread_mutexattr_t;
 275
 276 struct _pthread_mutex_options {
 277         uint32_t protocol:2,
 278                 type:2,
 279                 pshared:2,
 280                 policy:3,
 281                 hold:2,
 282                 misalign:1,
 283                 notify:1,
 284                 mutex:1,
 285                 unused:2,
 286                 lock_count:16;
 287 };
 288
 289 typedef struct {
 290         long sig;
 291         _pthread_lock lock;
 292         union {
 293                 uint32_t value;
 294                 struct _pthread_mutex_options options;
 295         } mtxopts;
 296         int16_t prioceiling;
 297         int16_t priority;
 298 #if defined(__LP64__)
 299         uint32_t _pad;
 300 #endif
 301         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 302         uint32_t m_seq[2]; // mutex sequence id
 303         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 304 #if defined(__LP64__)
 305         uint32_t _reserved[4];
 306 #else
 307         uint32_t _reserved[1];
 308 #endif
 309 } _pthread_mutex;
 310
 311
 312 #define _PTHREAD_CONDATTR_T
 313 typedef struct {
 314         long sig;
 315         uint32_t pshared:2,
 316                 unsupported:30;
 317 } pthread_condattr_t;
 318
 319
 320 typedef struct {
 321         long sig;
 322         _pthread_lock lock;
 323         uint32_t unused:29,
 324                 misalign:1,
 325                 pshared:2;
 326         _pthread_mutex *busy;
 327         uint32_t c_seq[3];
 328 #if defined(__LP64__)
 329         uint32_t _reserved[3];
 330 #endif
 331 } _pthread_cond;
 332
 333
 334 #define _PTHREAD_ONCE_T
 335 typedef struct {
 336         long sig;
 337         os_once_t once;
 338 } pthread_once_t;
 339
 340
 341 #define _PTHREAD_RWLOCKATTR_T
 342 typedef struct {
 343         long sig;
 344         int pshared;
 345 #if defined(__LP64__)
 346         uint32_t _reserved[3];
 347 #else
 348         uint32_t _reserved[2];
 349 #endif
 350 } pthread_rwlockattr_t;
 351
 352
 353 typedef struct {
 354         long sig;
 355         _pthread_lock lock;
 356         uint32_t unused:29,
 357                         misalign:1,
 358                         pshared:2;
 359         uint32_t rw_flags;
 360 #if defined(__LP64__)
 361         uint32_t _pad;
 362 #endif
 363         uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock
 364         uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary)
 365         uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here
 366 #if defined(__LP64__)
 367         uint32_t _reserved[34];
 368 #else
 369         uint32_t _reserved[18];
 370 #endif
 371 } _pthread_rwlock;
 372
 373 #include "pthread.h"
 374 #include "pthread_spis.h"
 375
 376 _Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
 377                 "Incorrect _pthread_mutex structure size");
 378
 379 _Static_assert(sizeof(_pthread_rwlock) == sizeof(pthread_rwlock_t),
 380                 "Incorrect _pthread_rwlock structure size");
 381
 382 // Internal references to pthread_self() use TSD slot 0 directly.
 383 inline static pthread_t __attribute__((__pure__))
 384 _pthread_self_direct(void)
 385 {
 386         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 387 }
 388 #define pthread_self() _pthread_self_direct()
 389
 390 PTHREAD_ALWAYS_INLINE
 391 inline static uint64_t __attribute__((__pure__))
 392 _pthread_selfid_direct(void)
 393 {
 394         return (_pthread_self_direct())->thread_id;
 395 }
 396
 397 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 398 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 399 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 400 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 401 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 402 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 403
 404 #define _PTHREAD_NO_SIG                 0x00000000
 405 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 406 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 407 #define _PTHREAD_MUTEX_SIG_fast         0x4D55545A  /* 'MUTZ' */
 408 #define _PTHREAD_MUTEX_SIG_MASK         0xfffffffd
 409 #define _PTHREAD_MUTEX_SIG_CMP          0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
 410 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 411 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 412 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 413 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 414 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 415 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 416 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 417 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 418 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 419 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 420 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 421 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 422 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 423 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 424 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 425 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 426
 427
 428 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 429 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 430 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 431
 432 #define _PTHREAD_CREATE_PARENT          4
 433 #define _PTHREAD_EXITED                 8
 434 // 4597450: begin
 435 #define _PTHREAD_WASCANCEL              0x10
 436 // 4597450: end
 437
 438 #if defined(DEBUG)
 439 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 440 #else
 441 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 442 #endif
 443 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 444
 445 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 446 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 447 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 448 #define _PTHREAD_CANCEL_INITIALIZED  0x20  /* the thread in the list is properly initialized */
 449
 450 extern boolean_t swtch_pri(int);
 451
 452 #include "kern/kern_internal.h"
 453
 454 /* Prototypes. */
 455
 456 /* Internal globals. */
 457 PTHREAD_NOEXPORT extern int __pthread_supported_features;
 458
 459 /* Functions defined in machine-dependent files. */
 460 PTHREAD_NOEXPORT void _pthread_setup(pthread_t th, void (*f)(pthread_t), void *sp, int suspended, int needresume);
 461
 462 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 463
 464 PTHREAD_NOEXPORT int _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 465
 466 /* internally redirected upcalls. */
 467 PTHREAD_NOEXPORT void* malloc(size_t);
 468 PTHREAD_NOEXPORT void free(void*);
 469
 470 /* syscall interfaces */
 471 extern uint32_t __psynch_mutexwait(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 472 extern uint32_t __psynch_mutexdrop(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 473
 474 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 475 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 476 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 477 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 478 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 479 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 480 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 481 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 482 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 483 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 484 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 485 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 486 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 487
 488 PTHREAD_EXTERN
 489 int
 490 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 491
 492 PTHREAD_NOEXPORT int _pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming);
 493
 494 PTHREAD_NORETURN PTHREAD_NOEXPORT
 495 void
 496 __pthread_abort(void);
 497
 498 PTHREAD_NORETURN PTHREAD_NOEXPORT
 499 void
 500 __pthread_abort_reason(const char *fmt, ...) __printflike(1,2);
 501
 502 PTHREAD_NOEXPORT
 503 void
 504 _pthread_set_main_qos(pthread_priority_t qos);
 505
 506 PTHREAD_NOEXPORT
 507 void
 508 _pthread_key_global_init(const char *envp[]);
 509
 510 PTHREAD_EXPORT
 511 void
 512 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 513
 514 PTHREAD_EXPORT
 515 void
 516 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);
 517
 518 PTHREAD_NOEXPORT
 519 void
 520 _pthread_main_thread_init(pthread_t p);
 521
 522 PTHREAD_NOEXPORT
 523 void
 524 _pthread_bsdthread_init(void);
 525
 526 PTHREAD_NOEXPORT_VARIANT
 527 void
 528 _pthread_clear_qos_tsd(mach_port_t thread_port);
 529
 530 PTHREAD_NOEXPORT_VARIANT
 531 void
 532 _pthread_testcancel(pthread_t thread, int isconforming);
 533
 534 PTHREAD_EXPORT
 535 void
 536 _pthread_exit_if_canceled(int error);
 537
 538 PTHREAD_NOEXPORT
 539 void
 540 _pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport);
 541
 542 PTHREAD_NOEXPORT
 543 void
 544 _pthread_setcancelstate_exit(pthread_t self, void *value_ptr, int conforming);
 545
 546 PTHREAD_NOEXPORT
 547 void *
 548 _pthread_get_exit_value(pthread_t t, int conforming);
 549
 550 PTHREAD_ALWAYS_INLINE
 551 static inline mach_port_t
 552 _pthread_kernel_thread(pthread_t t)
 553 {
 554         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 555 }
 556
 557 PTHREAD_ALWAYS_INLINE
 558 static inline void
 559 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 560 {
 561         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 562 }
 563
 564 #define PTHREAD_ABORT(f,...) __pthread_abort_reason( \
 565                 "%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
 566
 567 #define PTHREAD_ASSERT(b) \
 568                 do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
 569
 570 #include <os/semaphore_private.h>
 571 #include <os/alloc_once_private.h>
 572
 573 struct pthread_atfork_entry {
 574         void (*prepare)(void);
 575         void (*parent)(void);
 576         void (*child)(void);
 577 };
 578
 579 #define PTHREAD_ATFORK_INLINE_MAX 10
 580 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 581
 582 struct pthread_globals_s {
 583         // atfork.c
 584         pthread_t psaved_self;
 585         _pthread_lock psaved_self_global_lock;
 586         _pthread_lock pthread_atfork_lock;
 587
 588         size_t atfork_count;
 589         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 590         struct pthread_atfork_entry *atfork;
 591         uint16_t qmp_logical[THREAD_QOS_LAST];
 592         uint16_t qmp_physical[THREAD_QOS_LAST];
 593
 594 };
 595 typedef struct pthread_globals_s *pthread_globals_t;
 596
 597 __attribute__((__pure__))
 598 static inline pthread_globals_t
 599 _pthread_globals(void)
 600 {
 601         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 602                              sizeof(struct pthread_globals_s),
 603                              NULL);
 604 }
 605
 606 #pragma mark _pthread_mutex_check_signature
 607
 608 PTHREAD_ALWAYS_INLINE
 609 static inline bool
 610 _pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
 611 {
 612         return (mutex->sig == _PTHREAD_MUTEX_SIG_fast);
 613 }
 614
 615 PTHREAD_ALWAYS_INLINE
 616 static inline bool
 617 _pthread_mutex_check_signature(_pthread_mutex *mutex)
 618 {
 619         return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
 620 }
 621
 622 PTHREAD_ALWAYS_INLINE
 623 static inline bool
 624 _pthread_mutex_check_signature_init(_pthread_mutex *mutex)
 625 {
 626         return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) ==
 627                         _PTHREAD_MUTEX_SIG_init_CMP);
 628 }
 629
 630 #pragma mark _pthread_rwlock_check_signature
 631
 632 PTHREAD_ALWAYS_INLINE
 633 static inline bool
 634 _pthread_rwlock_check_signature(_pthread_rwlock *rwlock)
 635 {
 636         return (rwlock->sig == _PTHREAD_RWLOCK_SIG);
 637 }
 638
 639 PTHREAD_ALWAYS_INLINE
 640 static inline bool
 641 _pthread_rwlock_check_signature_init(_pthread_rwlock *rwlock)
 642 {
 643         return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init);
 644 }
 645
 646 /* ALWAYS called with list lock and return with list lock */
 647 PTHREAD_ALWAYS_INLINE
 648 static inline bool
 649 _pthread_is_valid_locked(pthread_t thread)
 650 {
 651         pthread_t p;
 652 loop:
 653         TAILQ_FOREACH(p, &__pthread_head, plist) {
 654                 if (p == thread) {
 655                         int state = os_atomic_load(&p->cancel_state, relaxed);
 656                         if (state & _PTHREAD_CANCEL_INITIALIZED) {
 657                                 return true;
 658                         }
 659                         _PTHREAD_UNLOCK(_pthread_list_lock);
 660                         thread_switch(_pthread_kernel_thread(p),
 661                                         SWITCH_OPTION_OSLOCK_DEPRESS, 1);
 662                         _PTHREAD_LOCK(_pthread_list_lock);
 663                         goto loop;
 664                 }
 665         }
 666
 667         return false;
 668 }
 669
 670 #define PTHREAD_IS_VALID_LOCK_THREAD 0x1
 671
 672 PTHREAD_ALWAYS_INLINE
 673 static inline bool
 674 _pthread_is_valid(pthread_t thread, int flags, mach_port_t *portp)
 675 {
 676         mach_port_t kport = MACH_PORT_NULL;
 677         bool valid;
 678
 679         if (thread == NULL) {
 680                 return false;
 681         }
 682
 683         if (thread == pthread_self()) {
 684                 valid = true;
 685                 kport = _pthread_kernel_thread(thread);
 686                 if (flags & PTHREAD_IS_VALID_LOCK_THREAD) {
 687                         _PTHREAD_LOCK(thread->lock);
 688                 }
 689         } else {
 690                 _PTHREAD_LOCK(_pthread_list_lock);
 691                 if (_pthread_is_valid_locked(thread)) {
 692                         kport = _pthread_kernel_thread(thread);
 693                         valid = true;
 694                         if (flags & PTHREAD_IS_VALID_LOCK_THREAD) {
 695                                 _PTHREAD_LOCK(thread->lock);
 696                         }
 697                 } else {
 698                         valid = false;
 699                 }
 700                 _PTHREAD_UNLOCK(_pthread_list_lock);
 701         }
 702
 703         if (portp != NULL) {
 704                 *portp = kport;
 705         }
 706         return valid;
 707 }
 708
 709 PTHREAD_ALWAYS_INLINE
 710 static inline void*
 711 _pthread_atomic_xchg_ptr_inline(void **p, void *v)
 712 {
 713         return os_atomic_xchg(p, v, seq_cst);
 714 }
 715
 716 PTHREAD_ALWAYS_INLINE
 717 static inline uint32_t
 718 _pthread_atomic_xchg_uint32_relaxed_inline(uint32_t *p,uint32_t v)
 719 {
 720         return os_atomic_xchg(p, v, relaxed);
 721 }
 722
 723 #define _pthread_atomic_xchg_ptr(p, v) \
 724                 _pthread_atomic_xchg_ptr_inline(p, v)
 725 #define _pthread_atomic_xchg_uint32_relaxed(p, v) \
 726                 _pthread_atomic_xchg_uint32_relaxed_inline(p, v)
 727
 728 #endif /* _POSIX_PTHREAD_INTERNALS_H */