src/shims/atomic.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_SHIMS_ATOMIC__
  28 #define __DISPATCH_SHIMS_ATOMIC__
  29
  30 // generate error during codegen
  31 #define _dispatch_atomic_unimplemented() \
  32                 ({ __asm__(".err unimplemented"); })
  33
  34 #pragma mark -
  35 #pragma mark memory_order
  36
  37 typedef enum _dispatch_atomic_memory_order
  38 {
  39     _dispatch_atomic_memory_order_relaxed,
  40         _dispatch_atomic_memory_order_consume,
  41         _dispatch_atomic_memory_order_acquire,
  42         _dispatch_atomic_memory_order_release,
  43         _dispatch_atomic_memory_order_acq_rel,
  44         _dispatch_atomic_memory_order_seq_cst,
  45 } _dispatch_atomic_memory_order;
  46
  47 #if !DISPATCH_ATOMIC_UP
  48
  49 #define dispatch_atomic_memory_order_relaxed \
  50                 _dispatch_atomic_memory_order_relaxed
  51 #define dispatch_atomic_memory_order_acquire \
  52                 _dispatch_atomic_memory_order_acquire
  53 #define dispatch_atomic_memory_order_release \
  54                 _dispatch_atomic_memory_order_release
  55 #define dispatch_atomic_memory_order_acq_rel \
  56                 _dispatch_atomic_memory_order_acq_rel
  57 #define dispatch_atomic_memory_order_seq_cst \
  58                 _dispatch_atomic_memory_order_seq_cst
  59
  60 #else // DISPATCH_ATOMIC_UP
  61
  62 #define dispatch_atomic_memory_order_relaxed \
  63                 _dispatch_atomic_memory_order_relaxed
  64 #define dispatch_atomic_memory_order_acquire \
  65                 _dispatch_atomic_memory_order_relaxed
  66 #define dispatch_atomic_memory_order_release \
  67                 _dispatch_atomic_memory_order_relaxed
  68 #define dispatch_atomic_memory_order_acq_rel \
  69                 _dispatch_atomic_memory_order_relaxed
  70 #define dispatch_atomic_memory_order_seq_cst \
  71                 _dispatch_atomic_memory_order_relaxed
  72
  73 #endif // DISPATCH_ATOMIC_UP
  74
  75 #if __has_extension(c_generic_selections)
  76 #define _dispatch_atomic_basetypeof(p) \
  77                 typeof(*_Generic((p), \
  78                 int*: (int*)(p), \
  79                 volatile int*: (int*)(p), \
  80                 unsigned int*: (unsigned int*)(p), \
  81                 volatile unsigned int*: (unsigned int*)(p), \
  82                 long*: (long*)(p), \
  83                 volatile long*: (long*)(p), \
  84                 unsigned long*: (unsigned long*)(p), \
  85                 volatile unsigned long*: (unsigned long*)(p), \
  86                 long long*: (long long*)(p), \
  87                 volatile long long*: (long long*)(p), \
  88                 unsigned long long*: (unsigned long long*)(p), \
  89                 volatile unsigned long long*: (unsigned long long*)(p), \
  90                 default: (void**)(p)))
  91 #endif
  92
  93 #if __has_extension(c_atomic) && __has_extension(c_generic_selections)
  94 #pragma mark -
  95 #pragma mark c11
  96
  97 #define _dispatch_atomic_c11_atomic(p) \
  98                 _Generic((p), \
  99                 int*: (_Atomic(int)*)(p), \
 100                 volatile int*: (volatile _Atomic(int)*)(p), \
 101                 unsigned int*: (_Atomic(unsigned int)*)(p), \
 102                 volatile unsigned int*: (volatile _Atomic(unsigned int)*)(p), \
 103                 long*: (_Atomic(long)*)(p), \
 104                 volatile long*: (volatile _Atomic(long)*)(p), \
 105                 unsigned long*: (_Atomic(unsigned long)*)(p), \
 106                 volatile unsigned long*: (volatile _Atomic(unsigned long)*)(p), \
 107                 long long*: (_Atomic(long long)*)(p), \
 108                 volatile long long*: (volatile _Atomic(long long)*)(p), \
 109                 unsigned long long*: (_Atomic(unsigned long long)*)(p), \
 110                 volatile unsigned long long*: \
 111                                 (volatile _Atomic(unsigned long long)*)(p), \
 112                 default: (volatile _Atomic(void*)*)(p))
 113
 114 #define _dispatch_atomic_barrier(m) \
 115                 ({ __c11_atomic_thread_fence(dispatch_atomic_memory_order_##m); })
 116 #define dispatch_atomic_load(p, m) \
 117                 ({ _dispatch_atomic_basetypeof(p) _r = \
 118                 __c11_atomic_load(_dispatch_atomic_c11_atomic(p), \
 119                 dispatch_atomic_memory_order_##m); (typeof(*(p)))_r; })
 120 #define dispatch_atomic_store(p, v, m) \
 121                 ({ _dispatch_atomic_basetypeof(p) _v = (v); \
 122                 __c11_atomic_store(_dispatch_atomic_c11_atomic(p), _v, \
 123                 dispatch_atomic_memory_order_##m); })
 124 #define dispatch_atomic_xchg(p, v, m) \
 125                 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = \
 126                 __c11_atomic_exchange(_dispatch_atomic_c11_atomic(p), _v, \
 127                 dispatch_atomic_memory_order_##m); (typeof(*(p)))_r; })
 128 #define dispatch_atomic_cmpxchg(p, e, v, m) \
 129                 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = (e); \
 130                 __c11_atomic_compare_exchange_strong(_dispatch_atomic_c11_atomic(p), \
 131                 &_r, _v, dispatch_atomic_memory_order_##m, \
 132                 dispatch_atomic_memory_order_relaxed); })
 133 #define dispatch_atomic_cmpxchgv(p, e, v, g, m) \
 134                 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = (e); _Bool _b = \
 135                 __c11_atomic_compare_exchange_strong(_dispatch_atomic_c11_atomic(p), \
 136                 &_r, _v, dispatch_atomic_memory_order_##m, \
 137                 dispatch_atomic_memory_order_relaxed); *(g) = (typeof(*(p)))_r; _b; })
 138 #define dispatch_atomic_cmpxchgvw(p, e, v, g, m) \
 139                 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = (e); _Bool _b = \
 140                 __c11_atomic_compare_exchange_weak(_dispatch_atomic_c11_atomic(p), \
 141                 &_r, _v, dispatch_atomic_memory_order_##m, \
 142                 dispatch_atomic_memory_order_relaxed); *(g) = (typeof(*(p)))_r;  _b; })
 143 #define _dispatch_atomic_c11_op(p, v, m, o, op) \
 144                 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = \
 145                 __c11_atomic_fetch_##o(_dispatch_atomic_c11_atomic(p), _v, \
 146                 dispatch_atomic_memory_order_##m); (typeof(*(p)))(_r op _v); })
 147 #define _dispatch_atomic_c11_op_orig(p, v, m, o, op) \
 148                 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = \
 149                 __c11_atomic_fetch_##o(_dispatch_atomic_c11_atomic(p), _v, \
 150                 dispatch_atomic_memory_order_##m); (typeof(*(p)))_r; })
 151
 152 #define dispatch_atomic_add(p, v, m) \
 153                 _dispatch_atomic_c11_op((p), (v), m, add, +)
 154 #define dispatch_atomic_add_orig(p, v, m) \
 155                 _dispatch_atomic_c11_op_orig((p), (v), m, add, +)
 156 #define dispatch_atomic_sub(p, v, m) \
 157                 _dispatch_atomic_c11_op((p), (v), m, sub, -)
 158 #define dispatch_atomic_sub_orig(p, v, m) \
 159                 _dispatch_atomic_c11_op_orig((p), (v), m, sub, -)
 160 #define dispatch_atomic_and(p, v, m) \
 161                 _dispatch_atomic_c11_op((p), (v), m, and, &)
 162 #define dispatch_atomic_and_orig(p, v, m) \
 163                 _dispatch_atomic_c11_op_orig((p), (v), m, and, &)
 164 #define dispatch_atomic_or(p, v, m) \
 165                 _dispatch_atomic_c11_op((p), (v), m, or, |)
 166 #define dispatch_atomic_or_orig(p, v, m) \
 167                 _dispatch_atomic_c11_op_orig((p), (v), m, or, |)
 168 #define dispatch_atomic_xor(p, v, m) \
 169                 _dispatch_atomic_c11_op((p), (v), m, xor, ^)
 170 #define dispatch_atomic_xor_orig(p, v, m) \
 171                 _dispatch_atomic_c11_op_orig((p), (v), m, xor, ^)
 172
 173 #elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)
 174 #pragma mark -
 175 #pragma mark gnu99
 176
 177 #define _dispatch_atomic_full_barrier() \
 178                 __sync_synchronize()
 179 #define _dispatch_atomic_barrier(m)     \
 180                 ({ switch(dispatch_atomic_memory_order_##m) { \
 181                 case _dispatch_atomic_memory_order_relaxed: \
 182                         break; \
 183                 default: \
 184                         _dispatch_atomic_full_barrier(); break; \
 185                 } })
 186 // seq_cst: only emulate explicit store(seq_cst) -> load(seq_cst)
 187 #define dispatch_atomic_load(p, m) \
 188                 ({ typeof(*(p)) _r = *(p); \
 189                 switch(dispatch_atomic_memory_order_##m) { \
 190                 case _dispatch_atomic_memory_order_seq_cst: \
 191                         _dispatch_atomic_barrier(m); /* fallthrough */ \
 192                 case _dispatch_atomic_memory_order_relaxed: \
 193                         break; \
 194                 default: \
 195                         _dispatch_atomic_unimplemented(); break; \
 196                 } _r; })
 197 #define dispatch_atomic_store(p, v, m) \
 198                 ({ switch(dispatch_atomic_memory_order_##m) { \
 199                 case _dispatch_atomic_memory_order_release: \
 200                 case _dispatch_atomic_memory_order_seq_cst: \
 201                         _dispatch_atomic_barrier(m); /* fallthrough */ \
 202                 case _dispatch_atomic_memory_order_relaxed: \
 203                         *(p) = (v); break; \
 204                 default: \
 205                         _dispatch_atomic_unimplemented(); break; \
 206                 } switch(dispatch_atomic_memory_order_##m) { \
 207                 case _dispatch_atomic_memory_order_seq_cst: \
 208                         _dispatch_atomic_barrier(m); break; \
 209                 default: \
 210                         break; \
 211                 } })
 212 #if __has_builtin(__sync_swap)
 213 #define dispatch_atomic_xchg(p, v, m) \
 214                 ((typeof(*(p)))__sync_swap((p), (v)))
 215 #else
 216 #define dispatch_atomic_xchg(p, v, m) \
 217                 ((typeof(*(p)))__sync_lock_test_and_set((p), (v)))
 218 #endif
 219 #define dispatch_atomic_cmpxchg(p, e, v, m) \
 220                 __sync_bool_compare_and_swap((p), (e), (v))
 221 #define dispatch_atomic_cmpxchgv(p, e, v, g, m) \
 222                 ({ typeof(*(g)) _e = (e), _r = \
 223                 __sync_val_compare_and_swap((p), _e, (v)); \
 224                 bool _b = (_e == _r); *(g) = _r; _b; })
 225 #define dispatch_atomic_cmpxchgvw(p, e, v, g, m) \
 226                 dispatch_atomic_cmpxchgv((p), (e), (v), (g), m)
 227
 228 #define dispatch_atomic_add(p, v, m) \
 229                 __sync_add_and_fetch((p), (v))
 230 #define dispatch_atomic_add_orig(p, v, m) \
 231                 __sync_fetch_and_add((p), (v))
 232 #define dispatch_atomic_sub(p, v, m) \
 233                 __sync_sub_and_fetch((p), (v))
 234 #define dispatch_atomic_sub_orig(p, v, m) \
 235                 __sync_fetch_and_sub((p), (v))
 236 #define dispatch_atomic_and(p, v, m) \
 237                 __sync_and_and_fetch((p), (v))
 238 #define dispatch_atomic_and_orig(p, v, m) \
 239                 __sync_fetch_and_and((p), (v))
 240 #define dispatch_atomic_or(p, v, m) \
 241                 __sync_or_and_fetch((p), (v))
 242 #define dispatch_atomic_or_orig(p, v, m) \
 243                 __sync_fetch_and_or((p), (v))
 244 #define dispatch_atomic_xor(p, v, m) \
 245                 __sync_xor_and_fetch((p), (v))
 246 #define dispatch_atomic_xor_orig(p, v, m) \
 247                 __sync_fetch_and_xor((p), (v))
 248
 249 #if defined(__x86_64__) || defined(__i386__)
 250 // GCC emits nothing for __sync_synchronize() on x86_64 & i386
 251 #undef _dispatch_atomic_full_barrier
 252 #define _dispatch_atomic_full_barrier() \
 253                 ({ __asm__ __volatile__( \
 254                 "mfence" \
 255                 : : : "memory"); })
 256 #undef dispatch_atomic_load
 257 #define dispatch_atomic_load(p, m) \
 258                 ({ switch(dispatch_atomic_memory_order_##m) { \
 259                 case _dispatch_atomic_memory_order_seq_cst: \
 260                 case _dispatch_atomic_memory_order_relaxed: \
 261                         break; \
 262                 default: \
 263                         _dispatch_atomic_unimplemented(); break; \
 264                 } *(p); })
 265 // xchg is faster than store + mfence
 266 #undef dispatch_atomic_store
 267 #define dispatch_atomic_store(p, v, m) \
 268                 ({ switch(dispatch_atomic_memory_order_##m) { \
 269                 case _dispatch_atomic_memory_order_relaxed: \
 270                 case _dispatch_atomic_memory_order_release: \
 271                         *(p) = (v); break; \
 272                 case _dispatch_atomic_memory_order_seq_cst: \
 273                         (void)dispatch_atomic_xchg((p), (v), m); break; \
 274                 default:\
 275                         _dispatch_atomic_unimplemented(); break; \
 276                 } })
 277 #endif
 278
 279 #else
 280 #error "Please upgrade to GCC 4.2 or newer."
 281 #endif
 282
 283 #pragma mark -
 284 #pragma mark generic
 285
 286 // assume atomic builtins provide barriers
 287 #define dispatch_atomic_barrier(m)
 288 // see comment in dispatch_once.c
 289 #define dispatch_atomic_maximally_synchronizing_barrier() \
 290                 _dispatch_atomic_barrier(seq_cst)
 291
 292 #define dispatch_atomic_load2o(p, f, m) \
 293                 dispatch_atomic_load(&(p)->f, m)
 294 #define dispatch_atomic_store2o(p, f, v, m) \
 295                 dispatch_atomic_store(&(p)->f, (v), m)
 296 #define dispatch_atomic_xchg2o(p, f, v, m) \
 297                 dispatch_atomic_xchg(&(p)->f, (v), m)
 298 #define dispatch_atomic_cmpxchg2o(p, f, e, v, m) \
 299                 dispatch_atomic_cmpxchg(&(p)->f, (e), (v), m)
 300 #define dispatch_atomic_cmpxchgv2o(p, f, e, v, g, m) \
 301                 dispatch_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m)
 302 #define dispatch_atomic_cmpxchgvw2o(p, f, e, v, g, m) \
 303                 dispatch_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m)
 304 #define dispatch_atomic_add2o(p, f, v, m) \
 305                 dispatch_atomic_add(&(p)->f, (v), m)
 306 #define dispatch_atomic_add_orig2o(p, f, v, m) \
 307                 dispatch_atomic_add_orig(&(p)->f, (v), m)
 308 #define dispatch_atomic_sub2o(p, f, v, m) \
 309                 dispatch_atomic_sub(&(p)->f, (v), m)
 310 #define dispatch_atomic_sub_orig2o(p, f, v, m) \
 311                 dispatch_atomic_sub_orig(&(p)->f, (v), m)
 312 #define dispatch_atomic_and2o(p, f, v, m) \
 313                 dispatch_atomic_and(&(p)->f, (v), m)
 314 #define dispatch_atomic_and_orig2o(p, f, v, m) \
 315                 dispatch_atomic_and_orig(&(p)->f, (v), m)
 316 #define dispatch_atomic_or2o(p, f, v, m) \
 317                 dispatch_atomic_or(&(p)->f, (v), m)
 318 #define dispatch_atomic_or_orig2o(p, f, v, m) \
 319                 dispatch_atomic_or_orig(&(p)->f, (v), m)
 320 #define dispatch_atomic_xor2o(p, f, v, m) \
 321                 dispatch_atomic_xor(&(p)->f, (v), m)
 322 #define dispatch_atomic_xor_orig2o(p, f, v, m) \
 323                 dispatch_atomic_xor_orig(&(p)->f, (v), m)
 324
 325 #define dispatch_atomic_inc(p, m) \
 326                 dispatch_atomic_add((p), 1, m)
 327 #define dispatch_atomic_inc_orig(p, m) \
 328                 dispatch_atomic_add_orig((p), 1, m)
 329 #define dispatch_atomic_inc2o(p, f, m) \
 330                 dispatch_atomic_add2o(p, f, 1, m)
 331 #define dispatch_atomic_inc_orig2o(p, f, m) \
 332                 dispatch_atomic_add_orig2o(p, f, 1, m)
 333 #define dispatch_atomic_dec(p, m) \
 334                 dispatch_atomic_sub((p), 1, m)
 335 #define dispatch_atomic_dec_orig(p, m) \
 336                 dispatch_atomic_sub_orig((p), 1, m)
 337 #define dispatch_atomic_dec2o(p, f, m) \
 338                 dispatch_atomic_sub2o(p, f, 1, m)
 339 #define dispatch_atomic_dec_orig2o(p, f, m) \
 340                 dispatch_atomic_sub_orig2o(p, f, 1, m)
 341
 342 #define dispatch_atomic_tsx_xacq_cmpxchgv(p, e, v, g) \
 343                 dispatch_atomic_cmpxchgv((p), (e), (v), (g), acquire)
 344 #define dispatch_atomic_tsx_xrel_store(p, v) \
 345                 dispatch_atomic_store(p, v, release)
 346 #define dispatch_atomic_tsx_xacq_cmpxchgv2o(p, f, e, v, g) \
 347                 dispatch_atomic_tsx_xacq_cmpxchgv(&(p)->f, (e), (v), (g))
 348 #define dispatch_atomic_tsx_xrel_store2o(p, f, v) \
 349                 dispatch_atomic_tsx_xrel_store(&(p)->f, (v))
 350
 351 #if defined(__x86_64__) || defined(__i386__)
 352 #pragma mark -
 353 #pragma mark x86
 354
 355 #undef dispatch_atomic_maximally_synchronizing_barrier
 356 #ifdef __LP64__
 357 #define dispatch_atomic_maximally_synchronizing_barrier() \
 358                 ({ unsigned long _clbr; __asm__ __volatile__( \
 359                 "cpuid" \
 360                 : "=a" (_clbr) : "0" (0) : "rbx", "rcx", "rdx", "cc", "memory"); })
 361 #else
 362 #ifdef __llvm__
 363 #define dispatch_atomic_maximally_synchronizing_barrier() \
 364                 ({ unsigned long _clbr; __asm__ __volatile__( \
 365                 "cpuid" \
 366                 : "=a" (_clbr) : "0" (0) : "ebx", "ecx", "edx", "cc", "memory"); })
 367 #else // gcc does not allow inline i386 asm to clobber ebx
 368 #define dispatch_atomic_maximally_synchronizing_barrier() \
 369                 ({ unsigned long _clbr; __asm__ __volatile__( \
 370                 "pushl  %%ebx\n\t" \
 371                 "cpuid\n\t" \
 372                 "popl   %%ebx" \
 373                 : "=a" (_clbr) : "0" (0) : "ecx", "edx", "cc", "memory"); })
 374 #endif
 375 #endif
 376
 377
 378 #endif
 379
 380
 381 #endif // __DISPATCH_SHIMS_ATOMIC__