src/shims/tsd.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_SHIMS_TSD__
  28 #define __DISPATCH_SHIMS_TSD__
  29
  30 #if HAVE_PTHREAD_MACHDEP_H
  31 #include <pthread_machdep.h>
  32 #endif
  33
  34 #define DISPATCH_TSD_INLINE DISPATCH_ALWAYS_INLINE_NDEBUG
  35
  36 #if USE_APPLE_TSD_OPTIMIZATIONS && HAVE_PTHREAD_KEY_INIT_NP && \
  37         !defined(DISPATCH_USE_DIRECT_TSD)
  38 #define DISPATCH_USE_DIRECT_TSD 1
  39 #if __has_include(<os/tsd.h>)
  40 #include <os/tsd.h>
  41 #endif
  42
  43 #if !defined(OS_GS_RELATIVE) && (defined(__i386__) || defined(__x86_64__))
  44 #define OS_GS_RELATIVE __attribute__((address_space(256)))
  45 #endif
  46
  47 #ifdef _os_tsd_get_base
  48 #ifdef OS_GS_RELATIVE
  49 typedef long dispatch_tsd_pair_t \
  50                 __attribute__((vector_size(sizeof(long) * 2), aligned(sizeof(long))));
  51 #define _os_tsd_get_pair_address(k) \
  52         (dispatch_tsd_pair_t OS_GS_RELATIVE *)((k) * sizeof(long))
  53 #else
  54 typedef struct { void *a; void *b; } dispatch_tsd_pair_t;
  55 #define _os_tsd_get_pair_address(k) \
  56         (dispatch_tsd_pair_t *)(_os_tsd_get_base() + (k))
  57 #endif
  58 #endif // _os_tsd_get_base
  59 #endif
  60
  61 #if DISPATCH_USE_DIRECT_TSD
  62 #ifndef __TSD_THREAD_QOS_CLASS
  63 #define __TSD_THREAD_QOS_CLASS 4
  64 #endif
  65 #ifndef __TSD_RETURN_TO_KERNEL
  66 #define __TSD_RETURN_TO_KERNEL 5
  67 #endif
  68 #ifndef __TSD_MACH_SPECIAL_REPLY
  69 #define __TSD_MACH_SPECIAL_REPLY 8
  70 #endif
  71
  72 static const unsigned long dispatch_priority_key        = __TSD_THREAD_QOS_CLASS;
  73 static const unsigned long dispatch_r2k_key                     = __TSD_RETURN_TO_KERNEL;
  74
  75 // dispatch_queue_key & dispatch_frame_key need to be contiguous
  76 // in that order, and queue_key to be an even number
  77 static const unsigned long dispatch_queue_key           = __PTK_LIBDISPATCH_KEY0;
  78 static const unsigned long dispatch_frame_key           = __PTK_LIBDISPATCH_KEY1;
  79 static const unsigned long dispatch_cache_key           = __PTK_LIBDISPATCH_KEY2;
  80 static const unsigned long dispatch_context_key         = __PTK_LIBDISPATCH_KEY3;
  81 static const unsigned long dispatch_pthread_root_queue_observer_hooks_key =
  82                 __PTK_LIBDISPATCH_KEY4;
  83 static const unsigned long dispatch_basepri_key     = __PTK_LIBDISPATCH_KEY5;
  84 #if DISPATCH_INTROSPECTION
  85 static const unsigned long dispatch_introspection_key = __PTK_LIBDISPATCH_KEY6;
  86 #elif DISPATCH_PERF_MON
  87 static const unsigned long dispatch_bcounter_key        = __PTK_LIBDISPATCH_KEY6;
  88 #endif
  89 static const unsigned long dispatch_wlh_key                     = __PTK_LIBDISPATCH_KEY7;
  90 static const unsigned long dispatch_voucher_key         = __PTK_LIBDISPATCH_KEY8;
  91 static const unsigned long dispatch_deferred_items_key = __PTK_LIBDISPATCH_KEY9;
  92
  93 DISPATCH_TSD_INLINE
  94 static inline void
  95 _dispatch_thread_key_create(const unsigned long *k, void (*d)(void *))
  96 {
  97         if (!*k || !d) return;
  98         dispatch_assert_zero(pthread_key_init_np((int)*k, d));
  99 }
 100 #elif DISPATCH_USE_THREAD_LOCAL_STORAGE
 101
 102 DISPATCH_TSD_INLINE
 103 static inline void
 104 _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *))
 105 {
 106         dispatch_assert_zero(pthread_key_create(k, d));
 107 }
 108
 109 struct dispatch_tsd {
 110         pid_t tid;
 111         void *dispatch_queue_key;
 112         void *dispatch_frame_key;
 113         void *dispatch_cache_key;
 114         void *dispatch_context_key;
 115         void *dispatch_pthread_root_queue_observer_hooks_key;
 116         void *dispatch_basepri_key;
 117 #if DISPATCH_INTROSPECTION
 118         void *dispatch_introspection_key;
 119 #elif DISPATCH_PERF_MON
 120         void *dispatch_bcounter_key;
 121 #endif
 122         void *dispatch_priority_key;
 123         void *dispatch_r2k_key;
 124         void *dispatch_wlh_key;
 125         void *dispatch_voucher_key;
 126         void *dispatch_deferred_items_key;
 127 };
 128
 129 extern __thread struct dispatch_tsd __dispatch_tsd;
 130 extern pthread_key_t __dispatch_tsd_key;
 131 extern void libdispatch_tsd_init(void);
 132 extern void _libdispatch_tsd_cleanup(void *ctx);
 133
 134 DISPATCH_ALWAYS_INLINE
 135 static inline struct dispatch_tsd *
 136 _dispatch_get_tsd_base(void)
 137 {
 138         if (unlikely(__dispatch_tsd.tid == 0)) {
 139                 libdispatch_tsd_init();
 140         }
 141         OS_COMPILER_CAN_ASSUME(__dispatch_tsd.tid != 0);
 142         return &__dispatch_tsd;
 143 }
 144
 145 #define _dispatch_thread_getspecific(key) \
 146         (_dispatch_get_tsd_base()->key)
 147 #define _dispatch_thread_setspecific(key, value) \
 148         (void)(_dispatch_get_tsd_base()->key = (value))
 149
 150 #define _dispatch_thread_getspecific_pair(k1, p1, k2, p2) \
 151         ( *(p1) = _dispatch_thread_getspecific(k1), \
 152           *(p2) = _dispatch_thread_getspecific(k2) )
 153
 154 #define _dispatch_thread_getspecific_packed_pair(k1, k2, p) \
 155         ( (p)[0] = _dispatch_thread_getspecific(k1), \
 156           (p)[1] = _dispatch_thread_getspecific(k2) )
 157
 158 #define _dispatch_thread_setspecific_pair(k1, p1, k2, p2) \
 159         ( _dispatch_thread_setspecific(k1,p1), \
 160           _dispatch_thread_setspecific(k2,p2) )
 161
 162 #define _dispatch_thread_setspecific_packed_pair(k1, k2, p) \
 163         ( _dispatch_thread_setspecific(k1,(p)[0]), \
 164           _dispatch_thread_setspecific(k2,(p)[1]) )
 165
 166 #else
 167 extern pthread_key_t dispatch_priority_key;
 168 extern pthread_key_t dispatch_r2k_key;
 169 extern pthread_key_t dispatch_queue_key;
 170 extern pthread_key_t dispatch_frame_key;
 171 extern pthread_key_t dispatch_cache_key;
 172 extern pthread_key_t dispatch_context_key;
 173 extern pthread_key_t dispatch_pthread_root_queue_observer_hooks_key;
 174 extern pthread_key_t dispatch_basepri_key;
 175 #if DISPATCH_INTROSPECTION
 176 extern pthread_key_t dispatch_introspection_key;
 177 #elif DISPATCH_PERF_MON
 178 extern pthread_key_t dispatch_bcounter_key;
 179 #endif
 180 extern pthread_key_t dispatch_wlh_key;
 181 extern pthread_key_t dispatch_voucher_key;
 182 extern pthread_key_t dispatch_deferred_items_key;
 183
 184 DISPATCH_TSD_INLINE
 185 static inline void
 186 _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *))
 187 {
 188         dispatch_assert_zero(pthread_key_create(k, d));
 189 }
 190 #endif
 191
 192 #ifndef DISPATCH_USE_THREAD_LOCAL_STORAGE
 193 DISPATCH_TSD_INLINE
 194 static inline void
 195 _dispatch_thread_setspecific(pthread_key_t k, void *v)
 196 {
 197 #if DISPATCH_USE_DIRECT_TSD
 198         if (_pthread_has_direct_tsd()) {
 199                 (void)_pthread_setspecific_direct(k, v);
 200         } else {
 201 #if TARGET_IPHONE_SIMULATOR
 202                 (void)_pthread_setspecific_static(k, v); // rdar://26058142
 203 #else
 204                 __builtin_trap(); // unreachable
 205 #endif
 206         }
 207         return;
 208 #endif
 209         dispatch_assert_zero(pthread_setspecific(k, v));
 210 }
 211
 212 DISPATCH_TSD_INLINE
 213 static inline void *
 214 _dispatch_thread_getspecific(pthread_key_t k)
 215 {
 216 #if DISPATCH_USE_DIRECT_TSD
 217         if (_pthread_has_direct_tsd()) {
 218                 return _pthread_getspecific_direct(k);
 219         }
 220 #endif
 221         return pthread_getspecific(k);
 222 }
 223
 224 // this is used when loading a pair at once and the caller will want to
 225 // look at each component individually.
 226 // some platforms can load a pair of pointers efficiently that way (like arm)
 227 // intel doesn't, hence this degrades to two loads on intel
 228 DISPATCH_TSD_INLINE
 229 static inline void
 230 _dispatch_thread_getspecific_pair(pthread_key_t k1, void **p1,
 231                 pthread_key_t k2, void **p2)
 232 {
 233         *p1 = _dispatch_thread_getspecific(k1);
 234         *p2 = _dispatch_thread_getspecific(k2);
 235 }
 236
 237 // this is used for save/restore purposes
 238 // and the caller doesn't need to look at a specific component
 239 // this does SSE on intel, and SSE is bad at breaking/assembling components
 240 DISPATCH_TSD_INLINE
 241 static inline void
 242 _dispatch_thread_getspecific_packed_pair(pthread_key_t k1, pthread_key_t k2,
 243                 void **p)
 244 {
 245 #if DISPATCH_USE_DIRECT_TSD && defined(_os_tsd_get_pair_address)
 246         dispatch_assert(k2 == k1 + 1);
 247         if (_pthread_has_direct_tsd()) {
 248                 *(dispatch_tsd_pair_t *)p = *_os_tsd_get_pair_address(k1);
 249                 return;
 250         }
 251 #endif
 252         p[0] = _dispatch_thread_getspecific(k1);
 253         p[1] = _dispatch_thread_getspecific(k2);
 254 }
 255
 256 // this is used when storing a pair at once from separated components
 257 // some platforms can store a pair of pointers efficiently that way (like arm)
 258 // intel doesn't, hence this degrades to two stores on intel
 259 DISPATCH_TSD_INLINE
 260 static inline void
 261 _dispatch_thread_setspecific_pair(pthread_key_t k1, void *p1,
 262                 pthread_key_t k2, void *p2)
 263 {
 264         _dispatch_thread_setspecific(k1, p1);
 265         _dispatch_thread_setspecific(k2, p2);
 266 }
 267
 268 // this is used for save/restore purposes
 269 // and the caller doesn't need to look at a specific component
 270 // this does SSE on intel, and SSE is bad at breaking/assembling components
 271 DISPATCH_TSD_INLINE
 272 static inline void
 273 _dispatch_thread_setspecific_packed_pair(pthread_key_t k1, pthread_key_t k2,
 274                 void **p)
 275 {
 276 #if DISPATCH_USE_DIRECT_TSD && defined(_os_tsd_get_pair_address)
 277         dispatch_assert(k2 == k1 + 1);
 278         if (_pthread_has_direct_tsd()) {
 279                 *_os_tsd_get_pair_address(k1) = *(dispatch_tsd_pair_t *)p;
 280                 return;
 281         }
 282 #endif
 283         _dispatch_thread_setspecific(k1, p[0]);
 284         _dispatch_thread_setspecific(k2, p[1]);
 285 }
 286 #endif
 287
 288 #if TARGET_OS_WIN32
 289 #define _dispatch_thread_self() ((uintptr_t)GetCurrentThreadId())
 290 #else
 291 #if DISPATCH_USE_DIRECT_TSD
 292 #define _dispatch_thread_self() ((uintptr_t)_dispatch_thread_getspecific( \
 293                 _PTHREAD_TSD_SLOT_PTHREAD_SELF))
 294 #else
 295 #define _dispatch_thread_self() ((uintptr_t)pthread_self())
 296 #endif
 297 #endif
 298
 299 #if TARGET_OS_WIN32
 300 #define _dispatch_thread_port() ((mach_port_t)0)
 301 #elif !DISPATCH_USE_THREAD_LOCAL_STORAGE
 302 #if DISPATCH_USE_DIRECT_TSD
 303 #define _dispatch_thread_port() ((mach_port_t)(uintptr_t)\
 304                 _dispatch_thread_getspecific(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF))
 305 #else
 306 #define _dispatch_thread_port() pthread_mach_thread_np(_dispatch_thread_self())
 307 #endif
 308 #endif
 309
 310 #if HAVE_MACH
 311 #define _dispatch_get_thread_mig_reply_port() ((mach_port_t)(uintptr_t) \
 312                 _dispatch_thread_getspecific(_PTHREAD_TSD_SLOT_MIG_REPLY))
 313 #define _dispatch_set_thread_mig_reply_port(p) ( \
 314                 _dispatch_thread_setspecific(_PTHREAD_TSD_SLOT_MIG_REPLY, \
 315                 (void*)(uintptr_t)(p)))
 316 #define _dispatch_get_thread_special_reply_port() ((mach_port_t)(uintptr_t) \
 317                 _dispatch_thread_getspecific(__TSD_MACH_SPECIAL_REPLY))
 318 #define _dispatch_set_thread_special_reply_port(p) ( \
 319                 _dispatch_thread_setspecific(__TSD_MACH_SPECIAL_REPLY, \
 320                 (void*)(uintptr_t)(p)))
 321 #endif
 322
 323 DISPATCH_TSD_INLINE DISPATCH_CONST
 324 static inline unsigned int
 325 _dispatch_cpu_number(void)
 326 {
 327 #if __has_include(<os/tsd.h>)
 328         return _os_cpu_number();
 329 #elif defined(__x86_64__) || defined(__i386__)
 330         struct { uintptr_t p1, p2; } p;
 331         __asm__("sidt %[p]" : [p] "=&m" (p));
 332         return (unsigned int)(p.p1 & 0xfff);
 333 #else
 334         // Not yet implemented.
 335         return 0;
 336 #endif
 337 }
 338
 339 #undef DISPATCH_TSD_INLINE
 340
 341 #endif