2 * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
4 * @APPLE_APACHE_LICENSE_HEADER_START@
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 * @APPLE_APACHE_LICENSE_HEADER_END@
22 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
23 * which are subject to change in future releases of Mac OS X. Any applications
24 * relying on these interfaces WILL break.
27 #ifndef __DISPATCH_SHIMS_TSD__
28 #define __DISPATCH_SHIMS_TSD__
30 #if HAVE_PTHREAD_MACHDEP_H
31 #include <pthread_machdep.h>
34 #define DISPATCH_TSD_INLINE DISPATCH_ALWAYS_INLINE_NDEBUG
36 #if USE_APPLE_TSD_OPTIMIZATIONS && HAVE_PTHREAD_KEY_INIT_NP && \
37 !defined(DISPATCH_USE_DIRECT_TSD)
38 #define DISPATCH_USE_DIRECT_TSD 1
39 #if __has_include(<os/tsd.h>)
43 #if !defined(OS_GS_RELATIVE) && (defined(__i386__) || defined(__x86_64__))
44 #define OS_GS_RELATIVE __attribute__((address_space(256)))
47 #ifdef _os_tsd_get_base
49 typedef long dispatch_tsd_pair_t \
50 __attribute__((vector_size(sizeof(long) * 2), aligned(sizeof(long))));
51 #define _os_tsd_get_pair_address(k) \
52 (dispatch_tsd_pair_t OS_GS_RELATIVE *)((k) * sizeof(long))
54 typedef struct { void *a
; void *b
; } dispatch_tsd_pair_t
;
55 #define _os_tsd_get_pair_address(k) \
56 (dispatch_tsd_pair_t *)(_os_tsd_get_base() + (k))
58 #endif // _os_tsd_get_base
61 #if DISPATCH_USE_DIRECT_TSD
62 #ifndef __TSD_THREAD_QOS_CLASS
63 #define __TSD_THREAD_QOS_CLASS 4
65 #ifndef __TSD_RETURN_TO_KERNEL
66 #define __TSD_RETURN_TO_KERNEL 5
68 #ifndef __TSD_MACH_SPECIAL_REPLY
69 #define __TSD_MACH_SPECIAL_REPLY 8
72 static const unsigned long dispatch_priority_key
= __TSD_THREAD_QOS_CLASS
;
73 static const unsigned long dispatch_r2k_key
= __TSD_RETURN_TO_KERNEL
;
75 // dispatch_queue_key & dispatch_frame_key need to be contiguous
76 // in that order, and queue_key to be an even number
77 static const unsigned long dispatch_queue_key
= __PTK_LIBDISPATCH_KEY0
;
78 static const unsigned long dispatch_frame_key
= __PTK_LIBDISPATCH_KEY1
;
79 static const unsigned long dispatch_cache_key
= __PTK_LIBDISPATCH_KEY2
;
80 static const unsigned long dispatch_context_key
= __PTK_LIBDISPATCH_KEY3
;
81 static const unsigned long dispatch_pthread_root_queue_observer_hooks_key
=
82 __PTK_LIBDISPATCH_KEY4
;
83 static const unsigned long dispatch_basepri_key
= __PTK_LIBDISPATCH_KEY5
;
84 #if DISPATCH_INTROSPECTION
85 static const unsigned long dispatch_introspection_key
= __PTK_LIBDISPATCH_KEY6
;
86 #elif DISPATCH_PERF_MON
87 static const unsigned long dispatch_bcounter_key
= __PTK_LIBDISPATCH_KEY6
;
89 static const unsigned long dispatch_wlh_key
= __PTK_LIBDISPATCH_KEY7
;
90 static const unsigned long dispatch_voucher_key
= __PTK_LIBDISPATCH_KEY8
;
91 static const unsigned long dispatch_deferred_items_key
= __PTK_LIBDISPATCH_KEY9
;
95 _dispatch_thread_key_create(const unsigned long *k
, void (*d
)(void *))
97 if (!*k
|| !d
) return;
98 dispatch_assert_zero(pthread_key_init_np((int)*k
, d
));
100 #elif DISPATCH_USE_THREAD_LOCAL_STORAGE
104 _dispatch_thread_key_create(pthread_key_t
*k
, void (*d
)(void *))
106 dispatch_assert_zero(pthread_key_create(k
, d
));
109 struct dispatch_tsd
{
111 void *dispatch_queue_key
;
112 void *dispatch_frame_key
;
113 void *dispatch_cache_key
;
114 void *dispatch_context_key
;
115 void *dispatch_pthread_root_queue_observer_hooks_key
;
116 void *dispatch_basepri_key
;
117 #if DISPATCH_INTROSPECTION
118 void *dispatch_introspection_key
;
119 #elif DISPATCH_PERF_MON
120 void *dispatch_bcounter_key
;
122 void *dispatch_priority_key
;
123 void *dispatch_r2k_key
;
124 void *dispatch_wlh_key
;
125 void *dispatch_voucher_key
;
126 void *dispatch_deferred_items_key
;
129 extern __thread
struct dispatch_tsd __dispatch_tsd
;
130 extern pthread_key_t __dispatch_tsd_key
;
131 extern void libdispatch_tsd_init(void);
132 extern void _libdispatch_tsd_cleanup(void *ctx
);
134 DISPATCH_ALWAYS_INLINE
135 static inline struct dispatch_tsd
*
136 _dispatch_get_tsd_base(void)
138 if (unlikely(__dispatch_tsd
.tid
== 0)) {
139 libdispatch_tsd_init();
141 OS_COMPILER_CAN_ASSUME(__dispatch_tsd
.tid
!= 0);
142 return &__dispatch_tsd
;
145 #define _dispatch_thread_getspecific(key) \
146 (_dispatch_get_tsd_base()->key)
147 #define _dispatch_thread_setspecific(key, value) \
148 (void)(_dispatch_get_tsd_base()->key = (value))
150 #define _dispatch_thread_getspecific_pair(k1, p1, k2, p2) \
151 ( *(p1) = _dispatch_thread_getspecific(k1), \
152 *(p2) = _dispatch_thread_getspecific(k2) )
154 #define _dispatch_thread_getspecific_packed_pair(k1, k2, p) \
155 ( (p)[0] = _dispatch_thread_getspecific(k1), \
156 (p)[1] = _dispatch_thread_getspecific(k2) )
158 #define _dispatch_thread_setspecific_pair(k1, p1, k2, p2) \
159 ( _dispatch_thread_setspecific(k1,p1), \
160 _dispatch_thread_setspecific(k2,p2) )
162 #define _dispatch_thread_setspecific_packed_pair(k1, k2, p) \
163 ( _dispatch_thread_setspecific(k1,(p)[0]), \
164 _dispatch_thread_setspecific(k2,(p)[1]) )
167 extern pthread_key_t dispatch_priority_key
;
168 extern pthread_key_t dispatch_r2k_key
;
169 extern pthread_key_t dispatch_queue_key
;
170 extern pthread_key_t dispatch_frame_key
;
171 extern pthread_key_t dispatch_cache_key
;
172 extern pthread_key_t dispatch_context_key
;
173 extern pthread_key_t dispatch_pthread_root_queue_observer_hooks_key
;
174 extern pthread_key_t dispatch_basepri_key
;
175 #if DISPATCH_INTROSPECTION
176 extern pthread_key_t dispatch_introspection_key
;
177 #elif DISPATCH_PERF_MON
178 extern pthread_key_t dispatch_bcounter_key
;
180 extern pthread_key_t dispatch_wlh_key
;
181 extern pthread_key_t dispatch_voucher_key
;
182 extern pthread_key_t dispatch_deferred_items_key
;
186 _dispatch_thread_key_create(pthread_key_t
*k
, void (*d
)(void *))
188 dispatch_assert_zero(pthread_key_create(k
, d
));
192 #ifndef DISPATCH_USE_THREAD_LOCAL_STORAGE
195 _dispatch_thread_setspecific(pthread_key_t k
, void *v
)
197 #if DISPATCH_USE_DIRECT_TSD
198 if (_pthread_has_direct_tsd()) {
199 (void)_pthread_setspecific_direct(k
, v
);
201 #if TARGET_IPHONE_SIMULATOR
202 (void)_pthread_setspecific_static(k
, v
); // rdar://26058142
204 __builtin_trap(); // unreachable
209 dispatch_assert_zero(pthread_setspecific(k
, v
));
214 _dispatch_thread_getspecific(pthread_key_t k
)
216 #if DISPATCH_USE_DIRECT_TSD
217 if (_pthread_has_direct_tsd()) {
218 return _pthread_getspecific_direct(k
);
221 return pthread_getspecific(k
);
224 // this is used when loading a pair at once and the caller will want to
225 // look at each component individually.
226 // some platforms can load a pair of pointers efficiently that way (like arm)
227 // intel doesn't, hence this degrades to two loads on intel
230 _dispatch_thread_getspecific_pair(pthread_key_t k1
, void **p1
,
231 pthread_key_t k2
, void **p2
)
233 *p1
= _dispatch_thread_getspecific(k1
);
234 *p2
= _dispatch_thread_getspecific(k2
);
237 // this is used for save/restore purposes
238 // and the caller doesn't need to look at a specific component
239 // this does SSE on intel, and SSE is bad at breaking/assembling components
242 _dispatch_thread_getspecific_packed_pair(pthread_key_t k1
, pthread_key_t k2
,
245 #if DISPATCH_USE_DIRECT_TSD && defined(_os_tsd_get_pair_address)
246 dispatch_assert(k2
== k1
+ 1);
247 if (_pthread_has_direct_tsd()) {
248 *(dispatch_tsd_pair_t
*)p
= *_os_tsd_get_pair_address(k1
);
252 p
[0] = _dispatch_thread_getspecific(k1
);
253 p
[1] = _dispatch_thread_getspecific(k2
);
256 // this is used when storing a pair at once from separated components
257 // some platforms can store a pair of pointers efficiently that way (like arm)
258 // intel doesn't, hence this degrades to two stores on intel
261 _dispatch_thread_setspecific_pair(pthread_key_t k1
, void *p1
,
262 pthread_key_t k2
, void *p2
)
264 _dispatch_thread_setspecific(k1
, p1
);
265 _dispatch_thread_setspecific(k2
, p2
);
268 // this is used for save/restore purposes
269 // and the caller doesn't need to look at a specific component
270 // this does SSE on intel, and SSE is bad at breaking/assembling components
273 _dispatch_thread_setspecific_packed_pair(pthread_key_t k1
, pthread_key_t k2
,
276 #if DISPATCH_USE_DIRECT_TSD && defined(_os_tsd_get_pair_address)
277 dispatch_assert(k2
== k1
+ 1);
278 if (_pthread_has_direct_tsd()) {
279 *_os_tsd_get_pair_address(k1
) = *(dispatch_tsd_pair_t
*)p
;
283 _dispatch_thread_setspecific(k1
, p
[0]);
284 _dispatch_thread_setspecific(k2
, p
[1]);
289 #define _dispatch_thread_self() ((uintptr_t)GetCurrentThreadId())
291 #if DISPATCH_USE_DIRECT_TSD
292 #define _dispatch_thread_self() ((uintptr_t)_dispatch_thread_getspecific( \
293 _PTHREAD_TSD_SLOT_PTHREAD_SELF))
295 #define _dispatch_thread_self() ((uintptr_t)pthread_self())
300 #define _dispatch_thread_port() ((mach_port_t)0)
301 #elif !DISPATCH_USE_THREAD_LOCAL_STORAGE
302 #if DISPATCH_USE_DIRECT_TSD
303 #define _dispatch_thread_port() ((mach_port_t)(uintptr_t)\
304 _dispatch_thread_getspecific(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF))
306 #define _dispatch_thread_port() pthread_mach_thread_np(_dispatch_thread_self())
311 #define _dispatch_get_thread_mig_reply_port() ((mach_port_t)(uintptr_t) \
312 _dispatch_thread_getspecific(_PTHREAD_TSD_SLOT_MIG_REPLY))
313 #define _dispatch_set_thread_mig_reply_port(p) ( \
314 _dispatch_thread_setspecific(_PTHREAD_TSD_SLOT_MIG_REPLY, \
315 (void*)(uintptr_t)(p)))
316 #define _dispatch_get_thread_special_reply_port() ((mach_port_t)(uintptr_t) \
317 _dispatch_thread_getspecific(__TSD_MACH_SPECIAL_REPLY))
318 #define _dispatch_set_thread_special_reply_port(p) ( \
319 _dispatch_thread_setspecific(__TSD_MACH_SPECIAL_REPLY, \
320 (void*)(uintptr_t)(p)))
323 DISPATCH_TSD_INLINE DISPATCH_CONST
324 static inline unsigned int
325 _dispatch_cpu_number(void)
327 #if __has_include(<os/tsd.h>)
328 return _os_cpu_number();
329 #elif defined(__x86_64__) || defined(__i386__)
330 struct { uintptr_t p1
, p2
; } p
;
331 __asm__("sidt %[p]" : [p
] "=&m" (p
));
332 return (unsigned int)(p
.p1
& 0xfff);
334 // Not yet implemented.
339 #undef DISPATCH_TSD_INLINE