include/os/internal/atomic.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #ifndef __OS_INTERNAL_ATOMIC__
  22 #define __OS_INTERNAL_ATOMIC__
  23
  24 #ifndef __OS_EXPOSE_INTERNALS_INDIRECT__
  25 /*
  26  * Use c11 <stdatomic.h> or c++11 std::atomic from <atomic> instead
  27  *
  28  * XXX                           /!\ WARNING /!\                           XXX
  29  *
  30  * This header file describes INTERNAL interfaces to libplatform used by other
  31  * libsystem targets, which are subject to change in future releases of OS X
  32  * and iOS. Any applications relying on these interfaces WILL break.
  33  *
  34  * If you are not a libsystem target, you should NOT EVER use these headers.
  35  * Not even a little.
  36  *
  37  * XXX                           /!\ WARNING /!\                           XXX
  38  */
  39 #error "Please #include <os/internal/internal_shared.h> instead of this file directly."
  40 #else
  41
  42 // generate error during codegen
  43 #define _os_atomic_unimplemented() \
  44                 ({ __asm__(".err unimplemented"); })
  45
  46 #pragma mark -
  47 #pragma mark memory_order
  48
  49 typedef enum _os_atomic_memory_order
  50 {
  51         _os_atomic_memory_order_relaxed,
  52         _os_atomic_memory_order_consume,
  53         _os_atomic_memory_order_acquire,
  54         _os_atomic_memory_order_release,
  55         _os_atomic_memory_order_acq_rel,
  56         _os_atomic_memory_order_seq_cst,
  57         _os_atomic_memory_order_ordered,
  58 } _os_atomic_memory_order;
  59
  60 #if !OS_ATOMIC_UP
  61
  62 #define os_atomic_memory_order_relaxed \
  63                 _os_atomic_memory_order_relaxed
  64 #define os_atomic_memory_order_acquire \
  65                 _os_atomic_memory_order_acquire
  66 #define os_atomic_memory_order_release \
  67                 _os_atomic_memory_order_release
  68 #define os_atomic_memory_order_acq_rel \
  69                 _os_atomic_memory_order_acq_rel
  70 #define os_atomic_memory_order_seq_cst \
  71                 _os_atomic_memory_order_seq_cst
  72 #define os_atomic_memory_order_ordered \
  73                 _os_atomic_memory_order_seq_cst
  74
  75 #else // OS_ATOMIC_UP
  76
  77 #define os_atomic_memory_order_relaxed \
  78                 _os_atomic_memory_order_relaxed
  79 #define os_atomic_memory_order_acquire \
  80                 _os_atomic_memory_order_relaxed
  81 #define os_atomic_memory_order_release \
  82                 _os_atomic_memory_order_relaxed
  83 #define os_atomic_memory_order_acq_rel \
  84                 _os_atomic_memory_order_relaxed
  85 #define os_atomic_memory_order_seq_cst \
  86                 _os_atomic_memory_order_relaxed
  87 #define os_atomic_memory_order_ordered \
  88                 _os_atomic_memory_order_relaxed
  89
  90 #endif // OS_ATOMIC_UP
  91
  92 #if __has_extension(c_generic_selections)
  93 #define _os_atomic_basetypeof(p) \
  94                 typeof(*_Generic((p), \
  95                 char*: (char*)(p), \
  96                 volatile char*: (char*)(p), \
  97                 signed char*: (signed char*)(p), \
  98                 volatile signed char*: (signed char*)(p), \
  99                 unsigned char*: (unsigned char*)(p), \
 100                 volatile unsigned char*: (unsigned char*)(p), \
 101                 short*: (short*)(p), \
 102                 volatile short*: (short*)(p), \
 103                 unsigned short*: (unsigned short*)(p), \
 104                 volatile unsigned short*: (unsigned short*)(p), \
 105                 int*: (int*)(p), \
 106                 volatile int*: (int*)(p), \
 107                 unsigned int*: (unsigned int*)(p), \
 108                 volatile unsigned int*: (unsigned int*)(p), \
 109                 long*: (long*)(p), \
 110                 volatile long*: (long*)(p), \
 111                 unsigned long*: (unsigned long*)(p), \
 112                 volatile unsigned long*: (unsigned long*)(p), \
 113                 long long*: (long long*)(p), \
 114                 volatile long long*: (long long*)(p), \
 115                 unsigned long long*: (unsigned long long*)(p), \
 116                 volatile unsigned long long*: (unsigned long long*)(p), \
 117                 const void**: (const void**)(p), \
 118                 const void*volatile*: (const void**)(p), \
 119                 default: (void**)(p)))
 120 #endif
 121
 122 #if __has_extension(c_atomic) && __has_extension(c_generic_selections)
 123 #pragma mark -
 124 #pragma mark c11
 125
 126 #define _os_atomic_c11_atomic(p) \
 127                 _Generic((p), \
 128                 char*: (_Atomic(char)*)(p), \
 129                 volatile char*: (volatile _Atomic(char)*)(p), \
 130                 signed char*: (_Atomic(signed char)*)(p), \
 131                 volatile signed char*: (volatile _Atomic(signed char)*)(p), \
 132                 unsigned char*: (_Atomic(unsigned char)*)(p), \
 133                 volatile unsigned char*: (volatile _Atomic(unsigned char)*)(p), \
 134                 short*: (_Atomic(short)*)(p), \
 135                 volatile short*: (volatile _Atomic(short)*)(p), \
 136                 unsigned short*: (_Atomic(unsigned short)*)(p), \
 137                 volatile unsigned short*: (volatile _Atomic(unsigned short)*)(p), \
 138                 int*: (_Atomic(int)*)(p), \
 139                 volatile int*: (volatile _Atomic(int)*)(p), \
 140                 unsigned int*: (_Atomic(unsigned int)*)(p), \
 141                 volatile unsigned int*: (volatile _Atomic(unsigned int)*)(p), \
 142                 long*: (_Atomic(long)*)(p), \
 143                 volatile long*: (volatile _Atomic(long)*)(p), \
 144                 unsigned long*: (_Atomic(unsigned long)*)(p), \
 145                 volatile unsigned long*: (volatile _Atomic(unsigned long)*)(p), \
 146                 long long*: (_Atomic(long long)*)(p), \
 147                 volatile long long*: (volatile _Atomic(long long)*)(p), \
 148                 unsigned long long*: (_Atomic(unsigned long long)*)(p), \
 149                 volatile unsigned long long*: \
 150                                 (volatile _Atomic(unsigned long long)*)(p), \
 151                 const void**: (_Atomic(const void*)*)(p), \
 152                 const void*volatile*: (volatile _Atomic(const void*)*)(p), \
 153                 default: (volatile _Atomic(void*)*)(p))
 154
 155 #define _os_atomic_barrier(m) \
 156                 ({ __c11_atomic_thread_fence(os_atomic_memory_order_##m); })
 157 #define os_atomic_load(p, m) \
 158                 ({ _os_atomic_basetypeof(p) _r = \
 159                 __c11_atomic_load(_os_atomic_c11_atomic(p), \
 160                 os_atomic_memory_order_##m); (typeof(*(p)))_r; })
 161 #define os_atomic_store(p, v, m) \
 162                 ({ _os_atomic_basetypeof(p) _v = (v); \
 163                 __c11_atomic_store(_os_atomic_c11_atomic(p), _v, \
 164                 os_atomic_memory_order_##m); })
 165 #define os_atomic_xchg(p, v, m) \
 166                 ({ _os_atomic_basetypeof(p) _v = (v), _r = \
 167                 __c11_atomic_exchange(_os_atomic_c11_atomic(p), _v, \
 168                 os_atomic_memory_order_##m); (typeof(*(p)))_r; })
 169 #define os_atomic_cmpxchg(p, e, v, m) \
 170                 ({ _os_atomic_basetypeof(p) _v = (v), _r = (e); \
 171                 __c11_atomic_compare_exchange_strong(_os_atomic_c11_atomic(p), \
 172                 &_r, _v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); })
 173 #define os_atomic_cmpxchgv(p, e, v, g, m) \
 174                 ({ _os_atomic_basetypeof(p) _v = (v), _r = (e); _Bool _b = \
 175                 __c11_atomic_compare_exchange_strong(_os_atomic_c11_atomic(p), \
 176                 &_r, _v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); \
 177                 *(g) = (typeof(*(p)))_r; _b; })
 178 #define os_atomic_cmpxchgvw(p, e, v, g, m) \
 179                 ({ _os_atomic_basetypeof(p) _v = (v), _r = (e); _Bool _b = \
 180                 __c11_atomic_compare_exchange_weak(_os_atomic_c11_atomic(p), \
 181                 &_r, _v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); \
 182                 *(g) = (typeof(*(p)))_r;  _b; })
 183 #define _os_atomic_c11_op(p, v, m, o, op) \
 184                 ({ _os_atomic_basetypeof(p) _v = (v), _r = \
 185                 __c11_atomic_fetch_##o(_os_atomic_c11_atomic(p), _v, \
 186                 os_atomic_memory_order_##m); (typeof(*(p)))(_r op _v); })
 187 #define _os_atomic_c11_op_orig(p, v, m, o, op) \
 188                 ({ _os_atomic_basetypeof(p) _v = (v), _r = \
 189                 __c11_atomic_fetch_##o(_os_atomic_c11_atomic(p), _v, \
 190                 os_atomic_memory_order_##m); (typeof(*(p)))_r; })
 191
 192 #define os_atomic_add(p, v, m) \
 193                 _os_atomic_c11_op((p), (v), m, add, +)
 194 #define os_atomic_add_orig(p, v, m) \
 195                 _os_atomic_c11_op_orig((p), (v), m, add, +)
 196 #define os_atomic_sub(p, v, m) \
 197                 _os_atomic_c11_op((p), (v), m, sub, -)
 198 #define os_atomic_sub_orig(p, v, m) \
 199                 _os_atomic_c11_op_orig((p), (v), m, sub, -)
 200 #define os_atomic_and(p, v, m) \
 201                 _os_atomic_c11_op((p), (v), m, and, &)
 202 #define os_atomic_and_orig(p, v, m) \
 203                 _os_atomic_c11_op_orig((p), (v), m, and, &)
 204 #define os_atomic_or(p, v, m) \
 205                 _os_atomic_c11_op((p), (v), m, or, |)
 206 #define os_atomic_or_orig(p, v, m) \
 207                 _os_atomic_c11_op_orig((p), (v), m, or, |)
 208 #define os_atomic_xor(p, v, m) \
 209                 _os_atomic_c11_op((p), (v), m, xor, ^)
 210 #define os_atomic_xor_orig(p, v, m) \
 211                 _os_atomic_c11_op_orig((p), (v), m, xor, ^)
 212
 213 #elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)
 214 #pragma mark -
 215 #pragma mark gnu99
 216
 217 #define _os_atomic_full_barrier()       \
 218                 __sync_synchronize()
 219 #define _os_atomic_barrier(m)   \
 220                 ({ switch(os_atomic_memory_order_##m) { \
 221                 case _os_atomic_memory_order_relaxed: \
 222                         break; \
 223                 default: \
 224                         _os_atomic_full_barrier(); break; \
 225                 } })
 226 #define os_atomic_load(p, m) \
 227                 ({ typeof(*(p)) _r = *(p); \
 228                 switch(os_atomic_memory_order_##m) { \
 229                 case _os_atomic_memory_order_relaxed: \
 230                 case _os_atomic_memory_order_acquire: \
 231                 case _os_atomic_memory_order_seq_cst: \
 232                         _os_atomic_barrier(m); \
 233                         break; \
 234                 default: \
 235                         _os_atomic_unimplemented(); break; \
 236                 } _r; })
 237 #define os_atomic_store(p, v, m) \
 238                 ({ switch(os_atomic_memory_order_##m) { \
 239                 case _os_atomic_memory_order_relaxed: \
 240                 case _os_atomic_memory_order_release: \
 241                 case _os_atomic_memory_order_seq_cst: \
 242                         _os_atomic_barrier(m); \
 243                         *(p) = (v); break; \
 244                 default: \
 245                         _os_atomic_unimplemented(); break; \
 246                 } switch(os_atomic_memory_order_##m) { \
 247                 case _os_atomic_memory_order_seq_cst: \
 248                         _os_atomic_barrier(m); break; \
 249                 default: \
 250                         break; \
 251                 } })
 252 #if __has_builtin(__sync_swap)
 253 #define os_atomic_xchg(p, v, m) \
 254                 ((typeof(*(p)))__sync_swap((p), (v)))
 255 #else
 256 #define os_atomic_xchg(p, v, m) \
 257                 ((typeof(*(p)))__sync_lock_test_and_set((p), (v)))
 258 #endif
 259 #define os_atomic_cmpxchg(p, e, v, m) \
 260                 __sync_bool_compare_and_swap((p), (e), (v))
 261 #define os_atomic_cmpxchgv(p, e, v, g, m) \
 262                 ({ typeof(*(g)) _e = (e), _r = \
 263                 __sync_val_compare_and_swap((p), _e, (v)); \
 264                 bool _b = (_e == _r); *(g) = _r; _b; })
 265 #define os_atomic_cmpxchgvw(p, e, v, g, m) \
 266                 os_atomic_cmpxchgv((p), (e), (v), (g), m)
 267
 268 #define os_atomic_add(p, v, m) \
 269                 __sync_add_and_fetch((p), (v))
 270 #define os_atomic_add_orig(p, v, m) \
 271                 __sync_fetch_and_add((p), (v))
 272 #define os_atomic_sub(p, v, m) \
 273                 __sync_sub_and_fetch((p), (v))
 274 #define os_atomic_sub_orig(p, v, m) \
 275                 __sync_fetch_and_sub((p), (v))
 276 #define os_atomic_and(p, v, m) \
 277                 __sync_and_and_fetch((p), (v))
 278 #define os_atomic_and_orig(p, v, m) \
 279                 __sync_fetch_and_and((p), (v))
 280 #define os_atomic_or(p, v, m) \
 281                 __sync_or_and_fetch((p), (v))
 282 #define os_atomic_or_orig(p, v, m) \
 283                 __sync_fetch_and_or((p), (v))
 284 #define os_atomic_xor(p, v, m) \
 285                 __sync_xor_and_fetch((p), (v))
 286 #define os_atomic_xor_orig(p, v, m) \
 287                 __sync_fetch_and_xor((p), (v))
 288
 289 #if defined(__x86_64__) || defined(__i386__)
 290 // GCC emits nothing for __sync_synchronize() on x86_64 & i386
 291 #undef _os_atomic_full_barrier
 292 #define _os_atomic_full_barrier() \
 293                 ({ __asm__ __volatile__( \
 294                 "mfence" \
 295                 : : : "memory"); })
 296 #undef os_atomic_load
 297 #define os_atomic_load(p, m) \
 298                 ({ switch(os_atomic_memory_order_##m) { \
 299                 case _os_atomic_memory_order_relaxed: \
 300                 case _os_atomic_memory_order_acquire: \
 301                 case _os_atomic_memory_order_seq_cst: \
 302                         break; \
 303                 default: \
 304                         _os_atomic_unimplemented(); break; \
 305                 } *(p); })
 306 // xchg is faster than store + mfence
 307 #undef os_atomic_store
 308 #define os_atomic_store(p, v, m) \
 309                 ({ switch(os_atomic_memory_order_##m) { \
 310                 case _os_atomic_memory_order_relaxed: \
 311                 case _os_atomic_memory_order_release: \
 312                         *(p) = (v); break; \
 313                 case _os_atomic_memory_order_seq_cst: \
 314                         (void)os_atomic_xchg((p), (v), m); break; \
 315                 default:\
 316                         _os_atomic_unimplemented(); break; \
 317                 } })
 318 #endif
 319
 320 #else
 321 #error "Please upgrade to GCC 4.2 or newer."
 322 #endif
 323
 324 #pragma mark -
 325 #pragma mark generic
 326
 327 #define os_atomic_thread_fence(m) _os_atomic_barrier(m)
 328 // see comment in os_once.c
 329 #define os_atomic_maximally_synchronizing_barrier() \
 330                 _os_atomic_barrier(seq_cst)
 331
 332 #define os_atomic_load2o(p, f, m) \
 333                 os_atomic_load(&(p)->f, m)
 334 #define os_atomic_store2o(p, f, v, m) \
 335                 os_atomic_store(&(p)->f, (v), m)
 336 #define os_atomic_xchg2o(p, f, v, m) \
 337                 os_atomic_xchg(&(p)->f, (v), m)
 338 #define os_atomic_cmpxchg2o(p, f, e, v, m) \
 339                 os_atomic_cmpxchg(&(p)->f, (e), (v), m)
 340 #define os_atomic_cmpxchgv2o(p, f, e, v, g, m) \
 341                 os_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m)
 342 #define os_atomic_cmpxchgvw2o(p, f, e, v, g, m) \
 343                 os_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m)
 344 #define os_atomic_add2o(p, f, v, m) \
 345                 os_atomic_add(&(p)->f, (v), m)
 346 #define os_atomic_add_orig2o(p, f, v, m) \
 347                 os_atomic_add_orig(&(p)->f, (v), m)
 348 #define os_atomic_sub2o(p, f, v, m) \
 349                 os_atomic_sub(&(p)->f, (v), m)
 350 #define os_atomic_sub_orig2o(p, f, v, m) \
 351                 os_atomic_sub_orig(&(p)->f, (v), m)
 352 #define os_atomic_and2o(p, f, v, m) \
 353                 os_atomic_and(&(p)->f, (v), m)
 354 #define os_atomic_and_orig2o(p, f, v, m) \
 355                 os_atomic_and_orig(&(p)->f, (v), m)
 356 #define os_atomic_or2o(p, f, v, m) \
 357                 os_atomic_or(&(p)->f, (v), m)
 358 #define os_atomic_or_orig2o(p, f, v, m) \
 359                 os_atomic_or_orig(&(p)->f, (v), m)
 360 #define os_atomic_xor2o(p, f, v, m) \
 361                 os_atomic_xor(&(p)->f, (v), m)
 362 #define os_atomic_xor_orig2o(p, f, v, m) \
 363                 os_atomic_xor_orig(&(p)->f, (v), m)
 364
 365 #define os_atomic_inc(p, m) \
 366                 os_atomic_add((p), 1, m)
 367 #define os_atomic_inc_orig(p, m) \
 368                 os_atomic_add_orig((p), 1, m)
 369 #define os_atomic_inc2o(p, f, m) \
 370                 os_atomic_add2o(p, f, 1, m)
 371 #define os_atomic_inc_orig2o(p, f, m) \
 372                 os_atomic_add_orig2o(p, f, 1, m)
 373 #define os_atomic_dec(p, m) \
 374                 os_atomic_sub((p), 1, m)
 375 #define os_atomic_dec_orig(p, m) \
 376                 os_atomic_sub_orig((p), 1, m)
 377 #define os_atomic_dec2o(p, f, m) \
 378                 os_atomic_sub2o(p, f, 1, m)
 379 #define os_atomic_dec_orig2o(p, f, m) \
 380                 os_atomic_sub_orig2o(p, f, 1, m)
 381
 382 #define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
 383                 bool _result = false; \
 384                 typeof(p) _p = (p); \
 385                 ov = os_atomic_load(_p, relaxed); \
 386                 do { \
 387                         __VA_ARGS__; \
 388                         _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \
 389                 } while (os_unlikely(!_result)); \
 390                 _result; \
 391         })
 392 #define os_atomic_rmw_loop2o(p, f, ov, nv, m, ...) \
 393                 os_atomic_rmw_loop(&(p)->f, ov, nv, m, __VA_ARGS__)
 394 #define os_atomic_rmw_loop_give_up_with_fence(m, expr) \
 395                 ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); })
 396 #define os_atomic_rmw_loop_give_up(expr) \
 397                 os_atomic_rmw_loop_give_up_with_fence(relaxed, expr)
 398
 399 #define os_atomic_tsx_xacq_cmpxchgv(p, e, v, g) \
 400                 os_atomic_cmpxchgv((p), (e), (v), (g), acquire)
 401 #define os_atomic_tsx_xrel_store(p, v) \
 402                 os_atomic_store(p, v, release)
 403 #define os_atomic_tsx_xacq_cmpxchgv2o(p, f, e, v, g) \
 404                 os_atomic_tsx_xacq_cmpxchgv(&(p)->f, (e), (v), (g))
 405 #define os_atomic_tsx_xrel_store2o(p, f, v) \
 406                 os_atomic_tsx_xrel_store(&(p)->f, (v))
 407
 408 #if defined(__x86_64__) || defined(__i386__)
 409 #pragma mark -
 410 #pragma mark x86
 411
 412 #undef os_atomic_maximally_synchronizing_barrier
 413 #ifdef __LP64__
 414 #define os_atomic_maximally_synchronizing_barrier() \
 415                 ({ unsigned long _clbr; __asm__ __volatile__( \
 416                 "cpuid" \
 417                 : "=a" (_clbr) : "0" (0) : "rbx", "rcx", "rdx", "cc", "memory"); })
 418 #else
 419 #ifdef __llvm__
 420 #define os_atomic_maximally_synchronizing_barrier() \
 421                 ({ unsigned long _clbr; __asm__ __volatile__( \
 422                 "cpuid" \
 423                 : "=a" (_clbr) : "0" (0) : "ebx", "ecx", "edx", "cc", "memory"); })
 424 #else // gcc does not allow inline i386 asm to clobber ebx
 425 #define os_atomic_maximally_synchronizing_barrier() \
 426                 ({ unsigned long _clbr; __asm__ __volatile__( \
 427                 "pushl  %%ebx\n\t" \
 428                 "cpuid\n\t" \
 429                 "popl   %%ebx" \
 430                 : "=a" (_clbr) : "0" (0) : "ecx", "edx", "cc", "memory"); })
 431 #endif
 432 #endif
 433
 434
 435 #endif
 436
 437
 438 #endif // __OS_EXPOSE_INTERNALS_INDIRECT__
 439
 440 #endif // __OS_ATOMIC__