src/internal.h

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 /*
  24  * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
  25  *              All Rights Reserved
  26  *
  27  * Permission to use, copy, modify, and distribute this software and
  28  * its documentation for any purpose and without fee is hereby granted,
  29  * provided that the above copyright notice appears in all copies and
  30  * that both the copyright notice and this permission notice appear in
  31  * supporting documentation.
  32  *
  33  * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
  34  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35  * FOR A PARTICULAR PURPOSE.
  36  *
  37  * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
  38  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  39  * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
  40  * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  41  * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  42  *
  43  */
  44 /*
  45  * MkLinux
  46  */
  47
  48 /*
  49  * POSIX Threads - IEEE 1003.1c
  50  */
  51
  52 #ifndef _POSIX_PTHREAD_INTERNALS_H
  53 #define _POSIX_PTHREAD_INTERNALS_H
  54
  55 #define _PTHREAD_BUILDING_PTHREAD_
  56
  57 // suppress pthread_attr_t typedef in sys/signal.h
  58 #define _PTHREAD_ATTR_T
  59 struct _pthread_attr_t; /* forward reference */
  60 typedef struct _pthread_attr_t pthread_attr_t;
  61
  62 #include <assert.h>
  63 #include <stddef.h>
  64 #include <stdint.h>
  65 #include <stdlib.h>
  66 #include <limits.h>
  67 #include <errno.h>
  68 #include <TargetConditionals.h>
  69 #include <libkern/OSAtomic.h>
  70 #include <mach/mach.h>
  71 #include <mach/mach_error.h>
  72 #include <os/once_private.h>
  73 #include <sys/queue.h>
  74
  75 #ifndef __POSIX_LIB__
  76 #define __POSIX_LIB__
  77 #endif
  78
  79 #ifndef PTHREAD_LAYOUT_SPI
  80 #define PTHREAD_LAYOUT_SPI 1
  81 #endif
  82
  83 #include "posix_sched.h"
  84 #include "tsd_private.h"
  85 #include "spinlock_private.h"
  86
  87 #if TARGET_IPHONE_SIMULATOR
  88 #error Unsupported target
  89 #endif
  90
  91 // List of all pthreads in the process.
  92 TAILQ_HEAD(__pthread_list, _pthread);
  93 extern struct __pthread_list __pthread_head;
  94
  95 // Lock protects access to above list.
  96 extern pthread_lock_t _pthread_list_lock;
  97
  98 extern int __is_threaded;
  99
 100 /*
 101  * Compiled-in limits
 102  */
 103 #if TARGET_OS_EMBEDDED
 104 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 105 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 106 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 107 #else
 108 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 109 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 110 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 111 #endif
 112
 113 #define MAXTHREADNAMESIZE       64
 114 #define _PTHREAD_T
 115 typedef struct _pthread {
 116         //
 117         // ABI - These fields are externally known as struct _opaque_pthread_t.
 118         //
 119         long sig; // _PTHREAD_SIG
 120         struct __darwin_pthread_handler_rec *__cleanup_stack;
 121
 122         //
 123         // SPI - These fields are private.
 124         //
 125         // these fields are globally protected by _pthread_list_lock:
 126         uint32_t childrun:1,
 127                         parentcheck:1,
 128                         childexit:1,
 129                         pad3:29;
 130
 131         pthread_lock_t lock; // protect access to everything below
 132         uint32_t detached:8,
 133                         inherit:8,
 134                         policy:8,
 135                         kernalloc:1,
 136                         schedset:1,
 137                         wqthread:1,
 138                         wqkillset:1,
 139                         pad:4;
 140
 141 #if __LP64__
 142         uint32_t pad0;
 143 #endif
 144         uint64_t thread_id;     // 64-bit unique thread id
 145
 146         void *(*fun)(void*);    // thread start routine
 147         void *arg;              // thread start routine argument
 148         void *exit_value;       // thread exit value storage
 149
 150         semaphore_t joiner_notify;      // pthread_join notification
 151
 152         int max_tsd_key;
 153         int cancel_state;       // whether the thread can be cancelled
 154         int cancel_error;
 155
 156 #ifdef __i386__
 157         // i386 err_no must be at a 68 byte offset
 158         // See <rdar://problem/13249323>
 159         uint32_t __13249323_pad[3];
 160 #endif
 161         int err_no;             // thread-local errno
 162
 163         struct _pthread *joiner;
 164
 165         struct sched_param param;       // [aligned]
 166
 167         TAILQ_ENTRY(_pthread) plist;    // global thread list [aligned]
 168
 169         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 170
 171         void *stackaddr;        // base of the stack (page aligned)
 172         size_t stacksize;       // size of stack (page multiple and >= PTHREAD_STACK_MIN)
 173
 174         void* freeaddr;         // stack/thread allocation base address
 175         size_t freesize;        // stack/thread allocation size
 176         size_t guardsize;       // guard page size in bytes
 177
 178         // thread specific data
 179         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 180 } *pthread_t;
 181
 182
 183 struct _pthread_attr_t {
 184         long sig;
 185         pthread_lock_t lock;
 186         uint32_t detached:8,
 187                 inherit:8,
 188                 policy:8,
 189                 fastpath:1,
 190                 schedset:1,
 191                 qosset:1,
 192                 unused:5;
 193         struct sched_param param; // [aligned]
 194         void *stackaddr; // stack base; vm_page_size aligned
 195         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 196         size_t guardsize; // size in bytes of stack overflow guard area
 197         unsigned long qosclass;
 198 #if defined(__LP64__)
 199         uint32_t _reserved[2];
 200 #else
 201         uint32_t _reserved[1];
 202 #endif
 203 };
 204
 205 /*
 206  * Mutex attributes
 207  */
 208 #define _PTHREAD_MUTEX_POLICY_NONE              0
 209 #define _PTHREAD_MUTEX_POLICY_FAIRSHARE         1
 210 #define _PTHREAD_MUTEX_POLICY_FIRSTFIT          2
 211 #define _PTHREAD_MUTEX_POLICY_REALTIME          3
 212 #define _PTHREAD_MUTEX_POLICY_ADAPTIVE          4
 213 #define _PTHREAD_MUTEX_POLICY_PRIPROTECT        5
 214 #define _PTHREAD_MUTEX_POLICY_PRIINHERIT        6
 215
 216 #define _PTHREAD_MUTEXATTR_T
 217 typedef struct {
 218         long sig;
 219         int prioceiling;
 220         uint32_t protocol:2,
 221                 type:2,
 222                 pshared:2,
 223                 policy:3,
 224                 unused:23;
 225 } pthread_mutexattr_t;
 226
 227 struct _pthread_mutex_options {
 228         uint32_t protocol:2,
 229                 type:2,
 230                 pshared:2,
 231                 policy:3,
 232                 hold:2,
 233                 misalign:1,
 234                 notify:1,
 235                 mutex:1,
 236                 unused:2,
 237                 lock_count:16;
 238 };
 239
 240 typedef struct {
 241         long sig;
 242         pthread_lock_t lock;
 243         union {
 244                 uint32_t value;
 245                 struct _pthread_mutex_options options;
 246         } mtxopts;
 247         int16_t prioceiling;
 248         int16_t priority;
 249 #if defined(__LP64__)
 250         uint32_t _pad;
 251 #endif
 252         uint32_t m_tid[2]; // thread id of thread that has mutex locked, misaligned locks may span to first field of m_seq
 253         uint32_t m_seq[3];
 254 #if defined(__LP64__)
 255         uint32_t _reserved;
 256 #endif
 257         void *reserved2[2];
 258 } _pthread_mutex;
 259
 260
 261 #define _PTHREAD_CONDATTR_T
 262 typedef struct {
 263         long sig;
 264         uint32_t pshared:2,
 265                 unsupported:30;
 266 } pthread_condattr_t;
 267
 268
 269 typedef struct {
 270         long sig;
 271         pthread_lock_t lock;
 272         uint32_t unused:29,
 273                 misalign:1,
 274                 pshared:2;
 275         _pthread_mutex *busy;
 276         uint32_t c_seq[3];
 277 #if defined(__LP64__)
 278         uint32_t _reserved[3];
 279 #endif
 280 } _pthread_cond;
 281
 282
 283 #define _PTHREAD_ONCE_T
 284 typedef struct {
 285         long sig;
 286         os_once_t once;
 287 } pthread_once_t;
 288
 289
 290 #define _PTHREAD_RWLOCKATTR_T
 291 typedef struct {
 292         long sig;
 293         int pshared;
 294 #if defined(__LP64__)
 295         uint32_t _reserved[3];
 296 #else
 297         uint32_t _reserved[2];
 298 #endif
 299 } pthread_rwlockattr_t;
 300
 301
 302 typedef struct {
 303         long sig;
 304         pthread_lock_t lock;
 305         uint32_t unused:29,
 306                 misalign:1,
 307                 pshared:2;
 308         uint32_t rw_flags;
 309 #if defined(__LP64__)
 310         uint32_t _pad;
 311 #endif
 312         volatile uint32_t rw_seq[4];
 313         struct _pthread *rw_owner;
 314         volatile uint32_t *rw_lcntaddr;
 315         volatile uint32_t *rw_seqaddr;
 316         volatile uint32_t *rw_ucntaddr;
 317 #if defined(__LP64__)
 318         uint32_t _reserved[31];
 319 #else
 320         uint32_t _reserved[19];
 321 #endif
 322 } _pthread_rwlock;
 323
 324 #include "pthread_spis.h"
 325
 326 // Internal references to pthread_self() use TSD slot 0 directly.
 327 inline static pthread_t __attribute__((__pure__))
 328 _pthread_self_direct(void)
 329 {
 330         return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
 331 }
 332 #define pthread_self() _pthread_self_direct()
 333
 334 inline static pthread_t __attribute__((__pure__))
 335 _pthread_selfid_direct(void)
 336 {
 337         return (_pthread_self_direct())->thread_id;
 338 }
 339
 340 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 341 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 342 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 343 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 344 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000   /* 512K */
 345 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 346
 347 #define _PTHREAD_NO_SIG                 0x00000000
 348 #define _PTHREAD_MUTEX_ATTR_SIG         0x4D545841  /* 'MTXA' */
 349 #define _PTHREAD_MUTEX_SIG              0x4D555458  /* 'MUTX' */
 350 #define _PTHREAD_MUTEX_SIG_init         0x32AAABA7  /* [almost] ~'MUTX' */
 351 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
 352 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
 353 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init       0x32AAABA3
 354 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 355 #define _PTHREAD_MUTEX_SIG_CMP                  0x32AAABA0
 356 #define _PTHREAD_COND_ATTR_SIG          0x434E4441  /* 'CNDA' */
 357 #define _PTHREAD_COND_SIG               0x434F4E44  /* 'COND' */
 358 #define _PTHREAD_COND_SIG_init          0x3CB0B1BB  /* [almost] ~'COND' */
 359 #define _PTHREAD_ATTR_SIG               0x54484441  /* 'THDA' */
 360 #define _PTHREAD_ONCE_SIG               0x4F4E4345  /* 'ONCE' */
 361 #define _PTHREAD_ONCE_SIG_init          0x30B1BCBA  /* [almost] ~'ONCE' */
 362 #define _PTHREAD_SIG                    0x54485244  /* 'THRD' */
 363 #define _PTHREAD_RWLOCK_ATTR_SIG        0x52574C41  /* 'RWLA' */
 364 #define _PTHREAD_RWLOCK_SIG             0x52574C4B  /* 'RWLK' */
 365 #define _PTHREAD_RWLOCK_SIG_init        0x2DA8B3B4  /* [almost] ~'RWLK' */
 366
 367
 368 #define _PTHREAD_KERN_COND_SIG          0x12345678  /*  */
 369 #define _PTHREAD_KERN_MUTEX_SIG         0x34567812  /*  */
 370 #define _PTHREAD_KERN_RWLOCK_SIG        0x56781234  /*  */
 371
 372 #define _PTHREAD_CREATE_PARENT          4
 373 #define _PTHREAD_EXITED                 8
 374 // 4597450: begin
 375 #define _PTHREAD_WASCANCEL              0x10
 376 // 4597450: end
 377
 378 #if defined(DEBUG)
 379 #define _PTHREAD_MUTEX_OWNER_SELF       pthread_self()
 380 #else
 381 #define _PTHREAD_MUTEX_OWNER_SELF       (pthread_t)0x12141968
 382 #endif
 383 #define _PTHREAD_MUTEX_OWNER_SWITCHING  (pthread_t)(~0)
 384
 385 #define _PTHREAD_CANCEL_STATE_MASK   0x01
 386 #define _PTHREAD_CANCEL_TYPE_MASK    0x02
 387 #define _PTHREAD_CANCEL_PENDING      0x10  /* pthread_cancel() has been called for this thread */
 388
 389 extern boolean_t swtch_pri(int);
 390
 391 #define PTHREAD_EXPORT extern __attribute__((visibility("default")))
 392 #define PTHREAD_EXTERN extern
 393 #define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
 394 #define PTHREAD_NORETURN __attribute__((__noreturn__))
 395 #define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
 396 #define PTHREAD_NOINLINE __attribute__((noinline))
 397
 398 #include "kern/kern_internal.h"
 399
 400 /* Prototypes. */
 401
 402 /* Internal globals. */
 403 PTHREAD_NOEXPORT extern int __pthread_supported_features;
 404
 405 /* Functions defined in machine-dependent files. */
 406 PTHREAD_NOEXPORT void _pthread_setup(pthread_t th, void (*f)(pthread_t), void *sp, int suspended, int needresume);
 407
 408 PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
 409
 410 PTHREAD_NOEXPORT int __mtx_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
 411
 412 /* internally redirected upcalls. */
 413 PTHREAD_NOEXPORT void* malloc(size_t);
 414 PTHREAD_NOEXPORT void free(void*);
 415
 416 /* syscall interfaces */
 417 extern uint32_t __psynch_mutexwait(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 418 extern uint32_t __psynch_mutexdrop(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
 419
 420 extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
 421 extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
 422 extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
 423 extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
 424 extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 425 extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 426 extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 427 extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 428 extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 429 extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 430 extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 431 extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
 432 extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 433
 434 PTHREAD_EXTERN
 435 int
 436 __proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
 437
 438 PTHREAD_NOEXPORT int _pthread_lookup_thread(pthread_t thread, mach_port_t * port, int only_joinable);
 439 PTHREAD_NOEXPORT int _pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming);
 440
 441 PTHREAD_NORETURN PTHREAD_NOEXPORT
 442 void
 443 __pthread_abort(void);
 444
 445 PTHREAD_NORETURN PTHREAD_NOEXPORT
 446 void
 447 __pthread_abort_reason(const char *fmt, ...);
 448
 449 PTHREAD_NOEXPORT
 450 void
 451 _pthread_set_main_qos(pthread_priority_t qos);
 452
 453 PTHREAD_EXPORT
 454 void
 455 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 456
 457 PTHREAD_EXPORT
 458 void
 459 _pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *unused, int reuse);
 460
 461 PTHREAD_NOEXPORT
 462 void
 463 __pthread_fork_child_internal(pthread_t p);
 464
 465 PTHREAD_EXPORT
 466 void
 467 _pthread_clear_qos_tsd(mach_port_t thread_port);
 468
 469 PTHREAD_EXPORT
 470 void
 471 _pthread_testcancel(pthread_t thread, int isconforming);
 472
 473 PTHREAD_EXPORT
 474 void
 475 _pthread_exit_if_canceled(int error);
 476
 477 PTHREAD_ALWAYS_INLINE
 478 static inline mach_port_t
 479 _pthread_kernel_thread(pthread_t t)
 480 {
 481         return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
 482 }
 483
 484 PTHREAD_ALWAYS_INLINE
 485 static inline void
 486 _pthread_set_kernel_thread(pthread_t t, mach_port_t p)
 487 {
 488         t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
 489 }
 490
 491 #define PTHREAD_ABORT(f,...) __pthread_abort_reason("%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
 492
 493 #define PTHREAD_ASSERT(b) do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
 494
 495 #include <os/semaphore_private.h>
 496 #include <os/alloc_once_private.h>
 497
 498 struct pthread_atfork_entry {
 499         void (*prepare)(void);
 500         void (*parent)(void);
 501         void (*child)(void);
 502 };
 503
 504 #define PTHREAD_ATFORK_INLINE_MAX 10
 505 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 506
 507 struct pthread_globals_s {
 508         // atfork.c
 509         pthread_t psaved_self;
 510         OSSpinLock psaved_self_global_lock;
 511         OSSpinLock pthread_atfork_lock;
 512
 513         size_t atfork_count;
 514         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 515         struct pthread_atfork_entry *atfork;
 516 };
 517 typedef struct pthread_globals_s *pthread_globals_t;
 518
 519 __attribute__((__pure__))
 520 static inline pthread_globals_t
 521 _pthread_globals(void)
 522 {
 523         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 524                              sizeof(struct pthread_globals_s),
 525                              NULL);
 526 }
 527
 528 #endif /* _POSIX_PTHREAD_INTERNALS_H */