src/types_internal.h

   1 /*
   2  * Copyright (c) 2003-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef __LIBPTHREAD_TYPES_INTERNAL_H__
  30 #define __LIBPTHREAD_TYPES_INTERNAL_H__
  31
  32 /*!
  33  * @file types_internal.h
  34  *
  35  * @brief
  36  * This file exposes all the internal pthread types used by the library.
  37  *
  38  * @discussion
  39  * This header must be included first, as it masks the opaque definitions
  40  * exposed to libpthread clients in the SDK.
  41  */
  42
  43 #define _PTHREAD_ONCE_T
  44 typedef struct pthread_once_s pthread_once_t;
  45
  46 #define _PTHREAD_MUTEX_T
  47 #define _PTHREAD_MUTEXATTR_T
  48 typedef struct pthread_mutex_s pthread_mutex_t;
  49 typedef struct pthread_mutexattr_s pthread_mutexattr_t;
  50
  51 #define _PTHREAD_COND_T
  52 #define _PTHREAD_CONDATTR_T
  53 typedef struct pthread_cond_s pthread_cond_t;
  54 typedef struct pthread_condattr_s pthread_condattr_t;
  55
  56 #define _PTHREAD_RWLOCK_T
  57 #define _PTHREAD_RWLOCKATTR_T
  58 typedef struct pthread_rwlock_s pthread_rwlock_t;
  59 typedef struct pthread_rwlockattr_s pthread_rwlockattr_t;
  60
  61 #define _PTHREAD_T
  62 #define _PTHREAD_ATTR_T
  63 typedef struct pthread_s *pthread_t;
  64 typedef struct pthread_attr_s pthread_attr_t;
  65
  66 #include <assert.h>
  67 #include <errno.h>
  68 #include <inttypes.h>
  69 #include <limits.h>
  70 #include <stdbool.h>
  71 #include <stddef.h>
  72 #include <stdint.h>
  73 #include <stdlib.h>
  74 #include <sys/queue.h>
  75 #include <sys/param.h>
  76
  77 #include <mach/mach.h>
  78 #include <mach/mach_error.h>
  79
  80 #include <os/base_private.h>
  81 #include <os/once_private.h>
  82 #include <os/lock.h>
  83
  84 #include "pthread/posix_sched.h"
  85 #include "pthread/workqueue_private.h"
  86 #include "sys/_pthread/_pthread_types.h"
  87
  88 #pragma mark - constants
  89
  90 #define _PTHREAD_NO_SIG                         0x00000000
  91 #define _PTHREAD_MUTEX_ATTR_SIG                 0x4D545841  /* 'MTXA' */
  92 #define _PTHREAD_MUTEX_SIG                      0x4D555458  /* 'MUTX' */
  93 #define _PTHREAD_MUTEX_SIG_fast                 0x4D55545A  /* 'MUTZ' */
  94 #define _PTHREAD_MUTEX_SIG_MASK                 0xfffffffd
  95 #define _PTHREAD_MUTEX_SIG_CMP                  0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
  96 #define _PTHREAD_MUTEX_SIG_init                 0x32AAABA7  /* [almost] ~'MUTX' */
  97 #define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
  98 #define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
  99 #define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
 100 #define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
 101 #define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
 102 #define _PTHREAD_COND_ATTR_SIG                  0x434E4441  /* 'CNDA' */
 103 #define _PTHREAD_COND_SIG_init                  0x3CB0B1BB  /* [almost] ~'COND' */
 104 #define _PTHREAD_COND_SIG_pristine              0x434F4E44  /* 'COND' */
 105 #define _PTHREAD_COND_SIG_psynch                0x434F4E45  /* 'COND' + 0b01: 'CONE' */
 106 #define _PTHREAD_COND_SIG_ulock                 0x434F4E46  /* 'COND' + 0b10: 'CONF' */
 107 #define _PTHREAD_ATTR_SIG                       0x54484441  /* 'THDA' */
 108 #define _PTHREAD_ONCE_SIG                       0x4F4E4345  /* 'ONCE' */
 109 #define _PTHREAD_ONCE_SIG_init                  0x30B1BCBA  /* [almost] ~'ONCE' */
 110 #define _PTHREAD_SIG                            0x54485244  /* 'THRD' */
 111 #define _PTHREAD_RWLOCK_ATTR_SIG                0x52574C41  /* 'RWLA' */
 112 #define _PTHREAD_RWLOCK_SIG                     0x52574C4B  /* 'RWLK' */
 113 #define _PTHREAD_RWLOCK_SIG_init                0x2DA8B3B4  /* [almost] ~'RWLK' */
 114
 115 __enum_closed_decl(pthread_conformance_t, unsigned, {
 116         PTHREAD_CONFORM_UNIX03_NOCANCEL    = 1,
 117         PTHREAD_CONFORM_UNIX03_CANCELABLE  = 2,
 118 });
 119
 120 /* Pull the pthread_t into the same page as the top of the stack so we dirty one less page.
 121  * <rdar://problem/19941744> The pthread_s struct at the top of the stack shouldn't be page-aligned
 122  */
 123 #if defined(__arm64__)
 124 #define PTHREAD_T_OFFSET (12*1024)
 125 #else
 126 #define PTHREAD_T_OFFSET 0
 127 #endif
 128
 129 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 130 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
 131 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 132 #define _INTERNAL_POSIX_THREAD_KEYS_END 512
 133 #else
 134 #define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
 135 #define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
 136 #define _INTERNAL_POSIX_THREAD_KEYS_END 768
 137 #endif
 138
 139 #define PTHREAD_ATFORK_INLINE_MAX       10
 140
 141 #define MAXTHREADNAMESIZE               64
 142
 143 #define _PTHREAD_DEFAULT_INHERITSCHED   PTHREAD_INHERIT_SCHED
 144 #define _PTHREAD_DEFAULT_PROTOCOL       PTHREAD_PRIO_NONE
 145 #define _PTHREAD_DEFAULT_PRIOCEILING    0
 146 #define _PTHREAD_DEFAULT_POLICY         SCHED_OTHER
 147 #define _PTHREAD_DEFAULT_STACKSIZE      0x80000    /* 512K */
 148 #define _PTHREAD_DEFAULT_PSHARED        PTHREAD_PROCESS_PRIVATE
 149
 150 #define _PTHREAD_CANCEL_STATE_MASK      0x01
 151 #define _PTHREAD_CANCEL_TYPE_MASK       0x02
 152 #define _PTHREAD_CANCEL_PENDING         0x10  /* pthread_cancel() has been called for this thread */
 153 #define _PTHREAD_CANCEL_EXITING         0x20
 154
 155 #define pthread_assert_type_size(type) \
 156         static_assert(sizeof(struct type##_s) == sizeof(struct _opaque_##type##_t), "")
 157 #define pthread_assert_type_alias(type, f1, f2) \
 158         static_assert(offsetof(struct type##_s, f1) == offsetof(struct _opaque_##type##_t, f2), "")
 159
 160 typedef os_unfair_lock _pthread_lock;
 161 struct _pthread_registration_data;
 162
 163
 164 #pragma mark - pthread_once_t
 165
 166 struct pthread_once_s {
 167         long sig;
 168         os_once_t once;
 169 };
 170
 171 pthread_assert_type_size(pthread_once);
 172 pthread_assert_type_alias(pthread_once, sig, __sig);
 173
 174 #pragma mark - pthread_mutex_t, pthread_mutexattr_t
 175
 176 #define _PTHREAD_MUTEX_POLICY_LAST          (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1)
 177 #define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE   1
 178 #define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT    2
 179 #define _PTHREAD_MTX_OPT_POLICY_DEFAULT     _PTHREAD_MTX_OPT_POLICY_FIRSTFIT
 180 // The following pthread_mutex_options_s defintions exist in synch_internal.h
 181 // such that the kernel extension can test for flags. They must be kept in
 182 // sync with the bit values in the struct above.
 183 // _PTHREAD_MTX_OPT_PSHARED 0x010
 184 // _PTHREAD_MTX_OPT_NOTIFY 0x1000
 185 // _PTHREAD_MTX_OPT_MUTEX 0x2000
 186
 187 #define _PTHREAD_MTX_OPT_ULOCK_DEFAULT false
 188 #define _PTHREAD_MTX_OPT_ADAPTIVE_DEFAULT false
 189
 190 // The fixed mask is used to mask out portions of the mutex options that
 191 // change on a regular basis (notify, lock_count).
 192 #define _PTHREAD_MTX_OPT_FIXED_MASK         0x27ff
 193
 194 struct pthread_mutex_options_s {
 195         uint32_t
 196                 protocol:2,
 197                 type:2,
 198                 pshared:2,
 199                 policy:3,
 200                 hold:2,
 201                 misalign:1,
 202                 notify:1,
 203                 mutex:1,
 204                 ulock:1,
 205                 unused:1,
 206                 lock_count:16;
 207 };
 208
 209 #define _PTHREAD_MUTEX_ULOCK_OWNER_MASK 0xfffffffcu
 210 #define _PTHREAD_MUTEX_ULOCK_WAITERS_BIT 0x00000001u
 211 #define _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE 0x0u
 212 #define _PTHREAD_MUTEX_ULOCK_UNLOCKED \
 213                 ((struct _pthread_mutex_ulock_s){0})
 214
 215 typedef struct _pthread_mutex_ulock_s {
 216         uint32_t uval;
 217 } *_pthread_mutex_ulock_t;
 218
 219 struct pthread_mutex_s {
 220         long sig;
 221         _pthread_lock lock;
 222         union {
 223                 uint32_t value;
 224                 struct pthread_mutex_options_s options;
 225         } mtxopts;
 226         int16_t prioceiling;
 227         int16_t priority;
 228 #if defined(__LP64__)
 229         uint32_t _pad;
 230 #endif
 231         union {
 232                 struct {
 233                         uint32_t m_tid[2]; // thread id of thread that has mutex locked
 234                         uint32_t m_seq[2]; // mutex sequence id
 235                         uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
 236                 } psynch;
 237                 struct _pthread_mutex_ulock_s ulock;
 238         };
 239 #if defined(__LP64__)
 240         uint32_t _reserved[4];
 241 #else
 242         uint32_t _reserved[1];
 243 #endif
 244 };
 245
 246 pthread_assert_type_size(pthread_mutex);
 247 pthread_assert_type_alias(pthread_mutex, sig, __sig);
 248
 249 struct pthread_mutexattr_s {
 250         long sig;
 251         int prioceiling;
 252         uint32_t
 253                 protocol:2,
 254                 type:2,
 255                 pshared:2,
 256                 opt:3,
 257                 unused:23;
 258 };
 259
 260 pthread_assert_type_size(pthread_mutexattr);
 261 pthread_assert_type_alias(pthread_mutexattr, sig, __sig);
 262
 263 #pragma mark - pthread_rwlock_t, pthread_rwlockattr_t
 264
 265 struct pthread_rwlock_s {
 266         long sig;
 267         _pthread_lock lock;
 268         uint32_t
 269                 unused:29,
 270                 misalign:1,
 271                 pshared:2;
 272         uint32_t rw_flags;
 273 #if defined(__LP64__)
 274         uint32_t _pad;
 275 #endif
 276         uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock
 277         uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary)
 278         uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here
 279 #if defined(__LP64__)
 280         uint32_t _reserved[34];
 281 #else
 282         uint32_t _reserved[18];
 283 #endif
 284 };
 285
 286 pthread_assert_type_size(pthread_rwlock);
 287 pthread_assert_type_alias(pthread_rwlock, sig, __sig);
 288
 289 struct pthread_rwlockattr_s {
 290         long sig;
 291         int pshared;
 292 #if defined(__LP64__)
 293         uint32_t _reserved[3];
 294 #else
 295         uint32_t _reserved[2];
 296 #endif
 297 };
 298
 299 pthread_assert_type_size(pthread_rwlockattr);
 300 pthread_assert_type_alias(pthread_rwlockattr, sig, __sig);
 301
 302 #pragma mark - pthread_cond_t, pthread_condattr_t
 303
 304 struct pthread_cond_s {
 305         struct {
 306                 uint32_t val;
 307 #if defined(__LP64__)
 308                 uint32_t _pad;
 309 #endif
 310         } sig;
 311         _pthread_lock lock;
 312         uint32_t
 313                 unused:29,
 314                 misalign:1,
 315                 pshared:2;
 316         pthread_mutex_t *busy;
 317         uint32_t c_seq[3];
 318 #if defined(__LP64__)
 319         uint32_t _reserved[3];
 320 #endif
 321 };
 322
 323 pthread_assert_type_size(pthread_cond);
 324 pthread_assert_type_alias(pthread_cond, sig, __sig);
 325
 326 struct pthread_condattr_s {
 327         long sig;
 328         uint32_t
 329                 pshared:2,
 330                 unsupported:30;
 331 };
 332
 333 pthread_assert_type_size(pthread_condattr);
 334 pthread_assert_type_alias(pthread_condattr, sig, __sig);
 335
 336 #pragma mark - pthread_t, pthread_attr_t
 337
 338 typedef struct pthread_join_context_s {
 339         pthread_t   waiter;
 340         void      **value_ptr;
 341         mach_port_t kport;
 342         semaphore_t custom_stack_sema;
 343         bool        detached;
 344 } pthread_join_context_s, *pthread_join_context_t;
 345
 346 #define MAXTHREADNAMESIZE               64
 347
 348 struct pthread_s {
 349         long sig;
 350         struct __darwin_pthread_handler_rec *__cleanup_stack;
 351
 352         //
 353         // Fields protected by _pthread_list_lock
 354         //
 355
 356         TAILQ_ENTRY(pthread_s) tl_plist;              // global thread list [aligned]
 357         struct pthread_join_context_s *tl_join_ctx;
 358         void *tl_exit_value;
 359         uint8_t tl_policy;
 360         // pthread knows that tl_joinable bit comes immediately after tl_policy
 361         uint8_t
 362                 tl_joinable:1,
 363                 tl_joiner_cleans_up:1,
 364                 tl_has_custom_stack:1,
 365                 __tl_pad:5;
 366         uint16_t introspection;
 367         // MACH_PORT_NULL if no joiner
 368         // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner
 369         // MACH_PORT_DEAD if the thread exited
 370         uint32_t tl_exit_gate;
 371         struct sched_param tl_param;
 372         void *__unused_padding;
 373
 374         //
 375         // Fields protected by pthread_t::lock
 376         //
 377
 378         _pthread_lock lock;
 379         uint16_t max_tsd_key;
 380         uint16_t
 381                 inherit:8,
 382                 kernalloc:1,
 383                 schedset:1,
 384                 wqthread:1,
 385                 wqkillset:1,
 386                 __flags_pad:4;
 387
 388         char pthread_name[MAXTHREADNAMESIZE];   // includes NUL [aligned]
 389
 390         void  *(*fun)(void *);  // thread start routine
 391         void    *arg;           // thread start routine argument
 392         int      wq_nevents;    // wqthreads (workloop / kevent)
 393         bool     wq_outsideqos;
 394         uint8_t  canceled;      // 4597450 set if conformant cancelation happened
 395         uint16_t cancel_state;  // whether the thread can be canceled [atomic]
 396         errno_t  cancel_error;
 397         errno_t  err_no;        // thread-local errno
 398
 399         void    *stackaddr;     // base of the stack (page aligned)
 400         void    *stackbottom;   // stackaddr - stacksize
 401         void    *freeaddr;      // stack/thread allocation base address
 402         size_t   freesize;      // stack/thread allocation size
 403         size_t   guardsize;     // guard page size in bytes
 404
 405         // tsd-base relative accessed elements
 406         __attribute__((aligned(8)))
 407         uint64_t thread_id;     // 64-bit unique thread id
 408
 409         /* Thread Specific Data slots
 410          *
 411          * The offset of this field from the start of the structure is difficult to
 412          * change on OS X because of a thorny bitcompat issue: mono has hard coded
 413          * the value into their source.  Newer versions of mono will fall back to
 414          * scanning to determine it at runtime, but there's lots of software built
 415          * with older mono that won't.  We will have to break them someday...
 416          */
 417         __attribute__ ((aligned (16)))
 418         void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
 419 };
 420
 421 TAILQ_HEAD(__pthread_list, pthread_s);
 422
 423 #if 0 // pthread_t is never stack-allocated, so it doesn't matter
 424 pthread_assert_type_size(pthread);
 425 #endif
 426 pthread_assert_type_alias(pthread, sig, __sig);
 427 pthread_assert_type_alias(pthread, __cleanup_stack, __cleanup_stack);
 428 #if __LP64__
 429 static_assert(offsetof(struct pthread_s, tsd) == 224, "TSD LP64 offset");
 430 #else
 431 static_assert(offsetof(struct pthread_s, tsd) == 176, "TSD ILP32 offset");
 432 #endif
 433
 434 #define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1)
 435
 436 struct pthread_attr_s {
 437         long   sig;
 438         size_t guardsize; // size in bytes of stack overflow guard area
 439         void  *stackaddr; // stack base; vm_page_size aligned
 440         size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
 441         union {
 442                 struct sched_param param; // [aligned]
 443                 unsigned long qosclass; // pthread_priority_t
 444         };
 445         uint32_t
 446                 detached:8,
 447                 inherit:8,
 448                 policy:8,
 449                 schedset:1,
 450                 qosset:1,
 451                 policyset:1,
 452                 cpupercentset:1,
 453                 defaultguardpage:1,
 454                 unused:3;
 455         uint32_t
 456                 cpupercent:8,
 457                 refillms:24;
 458 #if defined(__LP64__)
 459         uint32_t _reserved[4];
 460 #else
 461         uint32_t _reserved[2];
 462 #endif
 463 };
 464
 465 pthread_assert_type_size(pthread_attr);
 466 pthread_assert_type_alias(pthread_attr, sig, __sig);
 467
 468 #pragma mark - atfork / qos
 469
 470 struct pthread_atfork_entry {
 471         void (*prepare)(void);
 472         void (*parent)(void);
 473         void (*child)(void);
 474 };
 475
 476 #define PTHREAD_ATFORK_INLINE_MAX 10
 477 #if defined(__arm__)
 478 // Hack. We don't want to depend on libcompiler_rt. armv7 implements integer
 479 // division by calling into compiler_rt. vm_page_size isn't a constant and
 480 // pthread_atfork_entry is 12 bytes so the compiler can't strength-reduce the
 481 // division, so it generates a call into compiler_rt.
 482 // So let's just use PAGE_MAX_SIZE on armv7, which is a constant. At worst
 483 // this wastes a maybe dozen K if we are actaully running on a smaller page
 484 // size than the max.
 485 // At the time of this writing we don't have any supported iOS armv7 hardware
 486 // that has different vm_page_size and PAGE_MAX_SIZE.
 487 #define PTHREAD_ATFORK_MAX (PAGE_MAX_SIZE/sizeof(struct pthread_atfork_entry))
 488 #else // defined(__arm__)
 489 #define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
 490 #endif // defined(__arm__)
 491
 492 struct pthread_globals_s {
 493         // atfork.c
 494         pthread_t psaved_self;
 495         _pthread_lock psaved_self_global_lock;
 496         _pthread_lock pthread_atfork_lock;
 497
 498         size_t atfork_count;
 499         struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
 500         struct pthread_atfork_entry *atfork;
 501         uint16_t qmp_logical[THREAD_QOS_LAST];
 502         uint16_t qmp_physical[THREAD_QOS_LAST];
 503
 504 };
 505 typedef struct pthread_globals_s *pthread_globals_t;
 506
 507 #endif // __LIBPTHREAD_TYPES_INTERNAL_H__