src/shims/atomic.h

   1 /*
   2  * Copyright (c) 2008-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_SHIMS_ATOMIC__
  28 #define __DISPATCH_SHIMS_ATOMIC__
  29
  30 #if !__has_extension(c_atomic) || !__has_include(<stdatomic.h>)
  31 #error libdispatch requires C11 with <stdatomic.h>
  32 #endif
  33
  34 #include <stdatomic.h>
  35
  36 #define memory_order_ordered    memory_order_seq_cst
  37 #define memory_order_dependency memory_order_acquire
  38
  39 #define os_atomic(type) type _Atomic
  40
  41 #define _os_atomic_c11_atomic(p) \
  42                 ((typeof(*(p)) _Atomic *)(p))
  43
  44 // This removes the _Atomic and volatile qualifiers on the type of *p
  45 #define _os_atomic_basetypeof(p) \
  46                 typeof(atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_relaxed))
  47
  48 #define os_atomic_load(p, m) \
  49                 atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_##m)
  50 #define os_atomic_store(p, v, m) \
  51                 atomic_store_explicit(_os_atomic_c11_atomic(p), v, memory_order_##m)
  52 #define os_atomic_xchg(p, v, m) \
  53                 atomic_exchange_explicit(_os_atomic_c11_atomic(p), v, memory_order_##m)
  54 #define os_atomic_cmpxchg(p, e, v, m) \
  55                 ({ _os_atomic_basetypeof(p) _r = (e); \
  56                 atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
  57                 &_r, v, memory_order_##m, memory_order_relaxed); })
  58 #define os_atomic_cmpxchgv(p, e, v, g, m) \
  59                 ({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \
  60                 atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
  61                 &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
  62 #define os_atomic_cmpxchgvw(p, e, v, g, m) \
  63                 ({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \
  64                 atomic_compare_exchange_weak_explicit(_os_atomic_c11_atomic(p), \
  65                 &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r;  _b; })
  66
  67 #define _os_atomic_c11_op(p, v, m, o, op) \
  68                 ({ _os_atomic_basetypeof(p) _v = (v), _r = \
  69                 atomic_fetch_##o##_explicit(_os_atomic_c11_atomic(p), _v, \
  70                 memory_order_##m); (typeof(_r))(_r op _v); })
  71 #define _os_atomic_c11_op_orig(p, v, m, o, op) \
  72                 atomic_fetch_##o##_explicit(_os_atomic_c11_atomic(p), v, \
  73                 memory_order_##m)
  74 #define os_atomic_add(p, v, m) \
  75                 _os_atomic_c11_op((p), (v), m, add, +)
  76 #define os_atomic_add_orig(p, v, m) \
  77                 _os_atomic_c11_op_orig((p), (v), m, add, +)
  78 #define os_atomic_sub(p, v, m) \
  79                 _os_atomic_c11_op((p), (v), m, sub, -)
  80 #define os_atomic_sub_orig(p, v, m) \
  81                 _os_atomic_c11_op_orig((p), (v), m, sub, -)
  82 #define os_atomic_and(p, v, m) \
  83                 _os_atomic_c11_op((p), (v), m, and, &)
  84 #define os_atomic_and_orig(p, v, m) \
  85                 _os_atomic_c11_op_orig((p), (v), m, and, &)
  86 #define os_atomic_or(p, v, m) \
  87                 _os_atomic_c11_op((p), (v), m, or, |)
  88 #define os_atomic_or_orig(p, v, m) \
  89                 _os_atomic_c11_op_orig((p), (v), m, or, |)
  90 #define os_atomic_xor(p, v, m) \
  91                 _os_atomic_c11_op((p), (v), m, xor, ^)
  92 #define os_atomic_xor_orig(p, v, m) \
  93                 _os_atomic_c11_op_orig((p), (v), m, xor, ^)
  94
  95 #define os_atomic_force_dependency_on(p, e) (p)
  96 #define os_atomic_load_with_dependency_on(p, e) \
  97                 os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed)
  98 #define os_atomic_load_with_dependency_on2o(p, f, e) \
  99                 os_atomic_load_with_dependency_on(&(p)->f, e)
 100
 101 #define os_atomic_thread_fence(m)  atomic_thread_fence(memory_order_##m)
 102
 103 #define os_atomic_load2o(p, f, m) \
 104                 os_atomic_load(&(p)->f, m)
 105 #define os_atomic_store2o(p, f, v, m) \
 106                 os_atomic_store(&(p)->f, (v), m)
 107 #define os_atomic_xchg2o(p, f, v, m) \
 108                 os_atomic_xchg(&(p)->f, (v), m)
 109 #define os_atomic_cmpxchg2o(p, f, e, v, m) \
 110                 os_atomic_cmpxchg(&(p)->f, (e), (v), m)
 111 #define os_atomic_cmpxchgv2o(p, f, e, v, g, m) \
 112                 os_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m)
 113 #define os_atomic_cmpxchgvw2o(p, f, e, v, g, m) \
 114                 os_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m)
 115 #define os_atomic_add2o(p, f, v, m) \
 116                 os_atomic_add(&(p)->f, (v), m)
 117 #define os_atomic_add_orig2o(p, f, v, m) \
 118                 os_atomic_add_orig(&(p)->f, (v), m)
 119 #define os_atomic_sub2o(p, f, v, m) \
 120                 os_atomic_sub(&(p)->f, (v), m)
 121 #define os_atomic_sub_orig2o(p, f, v, m) \
 122                 os_atomic_sub_orig(&(p)->f, (v), m)
 123 #define os_atomic_and2o(p, f, v, m) \
 124                 os_atomic_and(&(p)->f, (v), m)
 125 #define os_atomic_and_orig2o(p, f, v, m) \
 126                 os_atomic_and_orig(&(p)->f, (v), m)
 127 #define os_atomic_or2o(p, f, v, m) \
 128                 os_atomic_or(&(p)->f, (v), m)
 129 #define os_atomic_or_orig2o(p, f, v, m) \
 130                 os_atomic_or_orig(&(p)->f, (v), m)
 131 #define os_atomic_xor2o(p, f, v, m) \
 132                 os_atomic_xor(&(p)->f, (v), m)
 133 #define os_atomic_xor_orig2o(p, f, v, m) \
 134                 os_atomic_xor_orig(&(p)->f, (v), m)
 135
 136 #define os_atomic_inc(p, m) \
 137                 os_atomic_add((p), 1, m)
 138 #define os_atomic_inc_orig(p, m) \
 139                 os_atomic_add_orig((p), 1, m)
 140 #define os_atomic_inc2o(p, f, m) \
 141                 os_atomic_add2o(p, f, 1, m)
 142 #define os_atomic_inc_orig2o(p, f, m) \
 143                 os_atomic_add_orig2o(p, f, 1, m)
 144 #define os_atomic_dec(p, m) \
 145                 os_atomic_sub((p), 1, m)
 146 #define os_atomic_dec_orig(p, m) \
 147                 os_atomic_sub_orig((p), 1, m)
 148 #define os_atomic_dec2o(p, f, m) \
 149                 os_atomic_sub2o(p, f, 1, m)
 150 #define os_atomic_dec_orig2o(p, f, m) \
 151                 os_atomic_sub_orig2o(p, f, 1, m)
 152
 153 #define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
 154                 bool _result = false; \
 155                 typeof(p) _p = (p); \
 156                 ov = os_atomic_load(_p, relaxed); \
 157                 do { \
 158                         __VA_ARGS__; \
 159                         _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \
 160                 } while (os_unlikely(!_result)); \
 161                 _result; \
 162         })
 163 #define os_atomic_rmw_loop2o(p, f, ov, nv, m, ...) \
 164                 os_atomic_rmw_loop(&(p)->f, ov, nv, m, __VA_ARGS__)
 165 #define os_atomic_rmw_loop_give_up_with_fence(m, expr) \
 166                 ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); })
 167 #define os_atomic_rmw_loop_give_up(expr) \
 168                 os_atomic_rmw_loop_give_up_with_fence(relaxed, expr)
 169
 170 #endif // __DISPATCH_SHIMS_ATOMIC__