src/inline_internal.h

   1 /*
   2  * Copyright (c) 2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #ifndef __LIBPTHREAD_INLINE_INTERNAL_H__
  25 #define __LIBPTHREAD_INLINE_INTERNAL_H__
  26
  27 /*!
  28  * @file inline_internal.h
  29  *
  30  * @brief
  31  * This file exposes inline helpers that are generally useful in libpthread.
  32  */
  33
  34 #define PTHREAD_INTERNAL_CRASH(c, x) OS_BUG_INTERNAL(c, "LIBPTHREAD", x)
  35 #define PTHREAD_CLIENT_CRASH(c, x)   OS_BUG_CLIENT(c, "LIBPTHREAD", x)
  36 #ifdef DEBUG
  37 #define PTHREAD_DEBUG_ASSERT(b) \
  38         do { \
  39                 if (os_unlikely(!(b))) { \
  40                         PTHREAD_INTERNAL_CRASH(0, "Assertion failed: " #b); \
  41                 } \
  42         } while (0)
  43 #else
  44 #define PTHREAD_DEBUG_ASSERT(b) ((void)0)
  45 #endif
  46
  47 #pragma mark _pthread_mutex_check_signature
  48
  49 OS_ALWAYS_INLINE
  50 static inline bool
  51 _pthread_mutex_check_signature_fast(pthread_mutex_t *mutex)
  52 {
  53         return (mutex->sig == _PTHREAD_MUTEX_SIG_fast);
  54 }
  55
  56 OS_ALWAYS_INLINE
  57 static inline bool
  58 _pthread_mutex_check_signature(const pthread_mutex_t *mutex)
  59 {
  60         // TODO: PTHREAD_STRICT candidate
  61         return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
  62 }
  63
  64 OS_ALWAYS_INLINE
  65 static inline bool
  66 _pthread_mutex_check_signature_init(const pthread_mutex_t *mutex)
  67 {
  68         return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) ==
  69                         _PTHREAD_MUTEX_SIG_init_CMP);
  70 }
  71
  72 #pragma mark pthread mutex accessors
  73
  74 OS_ALWAYS_INLINE
  75 static inline bool
  76 _pthread_mutex_uses_ulock(pthread_mutex_t *mutex)
  77 {
  78         return mutex->mtxopts.options.ulock;
  79 }
  80
  81 #pragma mark _pthread_rwlock_check_signature
  82
  83 OS_ALWAYS_INLINE
  84 static inline bool
  85 _pthread_rwlock_check_signature(const pthread_rwlock_t *rwlock)
  86 {
  87         return (rwlock->sig == _PTHREAD_RWLOCK_SIG);
  88 }
  89
  90 OS_ALWAYS_INLINE
  91 static inline bool
  92 _pthread_rwlock_check_signature_init(const pthread_rwlock_t *rwlock)
  93 {
  94         return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init);
  95 }
  96
  97 #pragma mark unfair lock wrappers
  98
  99 #define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
 100 #define _PTHREAD_LOCK_OPTIONS \
 101                 (OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION | OS_UNFAIR_LOCK_ADAPTIVE_SPIN)
 102
 103 OS_ALWAYS_INLINE
 104 static inline void
 105 _pthread_lock_init(os_unfair_lock_t lock)
 106 {
 107         *lock = _PTHREAD_LOCK_INITIALIZER;
 108 }
 109
 110 OS_OVERLOADABLE OS_ALWAYS_INLINE
 111 static inline void
 112 _pthread_lock_lock(os_unfair_lock_t lock)
 113 {
 114 #if OS_UNFAIR_LOCK_INLINE
 115         os_unfair_lock_lock_with_options_inline(lock, _PTHREAD_LOCK_OPTIONS);
 116 #else
 117         os_unfair_lock_lock_with_options(lock, _PTHREAD_LOCK_OPTIONS);
 118 #endif
 119 }
 120
 121 OS_OVERLOADABLE OS_ALWAYS_INLINE
 122 static inline void
 123 _pthread_lock_lock(os_unfair_lock_t lock, mach_port_t mts)
 124 {
 125 #if OS_UNFAIR_LOCK_INLINE
 126         os_unfair_lock_lock_no_tsd_inline(lock, _PTHREAD_LOCK_OPTIONS, mts);
 127 #else
 128         os_unfair_lock_lock_no_tsd(lock, _PTHREAD_LOCK_OPTIONS, mts);
 129 #endif
 130 }
 131
 132 OS_OVERLOADABLE OS_ALWAYS_INLINE
 133 static inline void
 134 _pthread_lock_unlock(os_unfair_lock_t lock)
 135 {
 136 #if OS_UNFAIR_LOCK_INLINE
 137         os_unfair_lock_unlock_inline(lock);
 138 #else
 139         os_unfair_lock_unlock(lock);
 140 #endif
 141 }
 142
 143 OS_OVERLOADABLE OS_ALWAYS_INLINE
 144 static inline void
 145 _pthread_lock_unlock(os_unfair_lock_t lock, mach_port_t mts)
 146 {
 147 #if OS_UNFAIR_LOCK_INLINE
 148         os_unfair_lock_unlock_no_tsd_inline(lock, mts);
 149 #else
 150         os_unfair_lock_unlock_no_tsd(lock, mts);
 151 #endif
 152 }
 153
 154 #pragma mark pthread accessors
 155
 156 // Internal references to pthread_self() use TSD slot 0 directly.
 157 #define pthread_self() _pthread_self_direct()
 158
 159 // Internal references to errno use TSD slot 1 directly.
 160 #undef  errno
 161 #define errno          (*_pthread_errno_address_direct())
 162
 163 #define _pthread_tsd_slot(th, name) \
 164                 (*(_PTHREAD_TSD_SLOT_##name##_TYPE *)(uintptr_t *)&(th)->tsd[_PTHREAD_TSD_SLOT_##name])
 165
 166 OS_ALWAYS_INLINE
 167 static inline void
 168 _pthread_validate_signature(pthread_t thread)
 169 {
 170         pthread_t th  = (pthread_t)(thread->sig ^ _pthread_ptr_munge_token);
 171 #if __has_feature(ptrauth_calls)
 172         th = ptrauth_auth_data(th, ptrauth_key_process_dependent_data,
 173                         ptrauth_string_discriminator("pthread.signature"));
 174 #endif
 175         if (os_unlikely(th != thread)) {
 176                 /* OS_REASON_LIBSYSTEM_CODE_PTHREAD_CORRUPTION == 4 */
 177                 abort_with_reason(OS_REASON_LIBSYSTEM, 4, "pthread_t was corrupted", 0);
 178         }
 179 }
 180
 181 OS_ALWAYS_INLINE
 182 static inline void
 183 _pthread_init_signature(pthread_t thread)
 184 {
 185         pthread_t th = thread;
 186 #if __has_feature(ptrauth_calls)
 187         th = ptrauth_sign_unauthenticated(th, ptrauth_key_process_dependent_data,
 188                         ptrauth_string_discriminator("pthread.signature"));
 189 #endif
 190         thread->sig = (uintptr_t)th ^ _pthread_ptr_munge_token;
 191 }
 192
 193 /*
 194  * ALWAYS called without list lock and return with list lock held on success
 195  *
 196  * This weird calling convention exists because this function will sometimes
 197  * drop the lock, and it's best callers don't have to remember this.
 198  */
 199 OS_ALWAYS_INLINE
 200 static inline bool
 201 _pthread_validate_thread_and_list_lock(pthread_t thread)
 202 {
 203         pthread_t p;
 204         if (thread == NULL) return false;
 205         _pthread_lock_lock(&_pthread_list_lock);
 206         TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
 207                 if (p != thread) continue;
 208                 _pthread_validate_signature(p);
 209                 return true;
 210         }
 211         _pthread_lock_unlock(&_pthread_list_lock);
 212
 213         return false;
 214 }
 215
 216 OS_ALWAYS_INLINE
 217 static inline bool
 218 _pthread_is_valid(pthread_t thread, mach_port_t *portp)
 219 {
 220         mach_port_t kport = MACH_PORT_NULL;
 221         bool valid;
 222
 223         if (thread == pthread_self()) {
 224                 _pthread_validate_signature(thread);
 225                 valid = true;
 226                 kport = _pthread_tsd_slot(thread, MACH_THREAD_SELF);
 227         } else if (!_pthread_validate_thread_and_list_lock(thread)) {
 228                 valid = false;
 229         } else {
 230                 kport = _pthread_tsd_slot(thread, MACH_THREAD_SELF);
 231                 valid = true;
 232                 _pthread_lock_unlock(&_pthread_list_lock);
 233         }
 234
 235         if (portp != NULL) {
 236                 *portp = kport;
 237         }
 238         return valid;
 239 }
 240
 241 OS_ALWAYS_INLINE OS_CONST
 242 static inline pthread_globals_t
 243 _pthread_globals(void)
 244 {
 245         return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
 246                         sizeof(struct pthread_globals_s), NULL);
 247 }
 248
 249 #endif // __LIBPTHREAD_INLINE_INTERNAL_H__