libkern/os/atomic_private_impl.h

   1 /*
   2  * Copyright (c) 2018 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /*
  30  * This header provides some gory details to implement the <os/atomic_private.h>
  31  * interfaces. Nothing in this header should be called directly, no promise is
  32  * made to keep this interface stable.
  33  */
  34
  35 #ifndef __OS_ATOMIC_PRIVATE_H__
  36 #error "Do not include <os/atomic_private_impl.h> directly, use <os/atomic_private.h>"
  37 #endif
  38
  39 #ifndef __OS_ATOMIC_PRIVATE_IMPL_H__
  40 #define __OS_ATOMIC_PRIVATE_IMPL_H__
  41
  42 #pragma mark - implementation details
  43
  44 static inline int
  45 _os_atomic_mo_has_acquire(OS_ATOMIC_STD memory_order ord)
  46 {
  47         switch (ord) {
  48         case os_atomic_std(memory_order_consume):
  49         case os_atomic_std(memory_order_acquire):
  50         case os_atomic_std(memory_order_acq_rel):
  51         case os_atomic_std(memory_order_seq_cst):
  52                 return 1;
  53         default:
  54                 return 0;
  55         }
  56 }
  57
  58 static inline int
  59 _os_atomic_mo_has_release(OS_ATOMIC_STD memory_order ord)
  60 {
  61         switch (ord) {
  62         case os_atomic_std(memory_order_release):
  63         case os_atomic_std(memory_order_acq_rel):
  64         case os_atomic_std(memory_order_seq_cst):
  65                 return 1;
  66         default:
  67                 return 0;
  68         }
  69 }
  70
  71 #define _os_atomic_mo_relaxed               os_atomic_std(memory_order_relaxed)
  72 #define _os_atomic_mo_compiler_acquire      os_atomic_std(memory_order_relaxed)
  73 #define _os_atomic_mo_compiler_release      os_atomic_std(memory_order_relaxed)
  74 #define _os_atomic_mo_compiler_acq_rel      os_atomic_std(memory_order_relaxed)
  75 #define _os_atomic_mo_consume               os_atomic_std(memory_order_consume)
  76 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
  77 #define _os_atomic_mo_dependency            os_atomic_std(memory_order_acquire)
  78 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
  79 #define _os_atomic_mo_acquire               os_atomic_std(memory_order_acquire)
  80 #define _os_atomic_mo_release               os_atomic_std(memory_order_release)
  81 #define _os_atomic_mo_acq_rel               os_atomic_std(memory_order_acq_rel)
  82 #define _os_atomic_mo_seq_cst               os_atomic_std(memory_order_seq_cst)
  83
  84 /*
  85  * Mapping between symbolic memory orderings and actual ones
  86  * to take SMP into account.
  87  */
  88 #if OS_ATOMIC_CONFIG_SMP
  89 #define _os_atomic_mo_relaxed_smp           _os_atomic_mo_relaxed
  90 #define _os_atomic_mo_compiler_acquire_smp  _os_atomic_mo_relaxed
  91 #define _os_atomic_mo_compiler_release_smp  _os_atomic_mo_relaxed
  92 #define _os_atomic_mo_compiler_acq_rel_smp  _os_atomic_mo_relaxed
  93 #define _os_atomic_mo_consume_smp           _os_atomic_mo_consume
  94 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
  95 #define _os_atomic_mo_dependency_smp        _os_atomic_mo_dependency
  96 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
  97 #define _os_atomic_mo_acquire_smp           _os_atomic_mo_acquire
  98 #define _os_atomic_mo_release_smp           _os_atomic_mo_release
  99 #define _os_atomic_mo_acq_rel_smp           _os_atomic_mo_acq_rel
 100 #define _os_atomic_mo_seq_cst_smp           _os_atomic_mo_seq_cst
 101 #else
 102 #define _os_atomic_mo_relaxed_smp           _os_atomic_mo_relaxed
 103 #define _os_atomic_mo_compiler_acquire_smp  _os_atomic_mo_relaxed
 104 #define _os_atomic_mo_compiler_release_smp  _os_atomic_mo_relaxed
 105 #define _os_atomic_mo_compiler_acq_rel_smp  _os_atomic_mo_relaxed
 106 #define _os_atomic_mo_consume_smp           _os_atomic_mo_relaxed
 107 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 108 #define _os_atomic_mo_dependency_smp        _os_atomic_mo_relaxed
 109 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 110 #define _os_atomic_mo_acquire_smp           _os_atomic_mo_relaxed
 111 #define _os_atomic_mo_release_smp           _os_atomic_mo_relaxed
 112 #define _os_atomic_mo_acq_rel_smp           _os_atomic_mo_relaxed
 113 #define _os_atomic_mo_seq_cst_smp           _os_atomic_mo_relaxed
 114 #endif
 115
 116 #if KERNEL_PRIVATE
 117 #define memory_order_relaxed_smp            _os_atomic_mo_relaxed_smp
 118 #define memory_order_compiler_acquire_smp   _os_atomic_mo_compiler_acquire_smp
 119 #define memory_order_compiler_release_smp   _os_atomic_mo_compiler_release_smp
 120 #define memory_order_compiler_acq_rel_smp   _os_atomic_mo_compiler_acq_rel_smp
 121 #define memory_order_consume_smp            _os_atomic_mo_consume_smp
 122 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 123 #define memory_order_dependency             _os_atomic_mo_dependency
 124 #define memory_order_dependency_smp         _os_atomic_mo_dependency_smp
 125 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 126 #define memory_order_acquire_smp            _os_atomic_mo_acquire_smp
 127 #define memory_order_release_smp            _os_atomic_mo_release_smp
 128 #define memory_order_acq_rel_smp            _os_atomic_mo_acq_rel_smp
 129 #define memory_order_seq_cst_smp            _os_atomic_mo_seq_cst_smp
 130 #endif
 131
 132 /*
 133  * Hack needed for os_compiler_barrier() to work (including with empty argument)
 134  */
 135 #define _os_compiler_barrier_relaxed        _os_atomic_mo_relaxed
 136 #define _os_compiler_barrier_acquire        _os_atomic_mo_acquire
 137 #define _os_compiler_barrier_release        _os_atomic_mo_release
 138 #define _os_compiler_barrier_acq_rel        _os_atomic_mo_acq_rel
 139 #define _os_compiler_barrier_               _os_atomic_mo_acq_rel
 140
 141 /*
 142  * Mapping between compiler barrier/memory orders and:
 143  * - compiler barriers before atomics ("rel_barrier")
 144  * - compiler barriers after atomics ("acq_barrier")
 145  */
 146 #define _os_rel_barrier_relaxed             _os_atomic_mo_relaxed
 147 #define _os_rel_barrier_compiler_acquire    _os_atomic_mo_relaxed
 148 #define _os_rel_barrier_compiler_release    _os_atomic_mo_release
 149 #define _os_rel_barrier_compiler_acq_rel    _os_atomic_mo_release
 150 #define _os_rel_barrier_consume             _os_atomic_mo_relaxed
 151 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 152 #define _os_rel_barrier_dependency          _os_atomic_mo_relaxed
 153 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 154 #define _os_rel_barrier_acquire             _os_atomic_mo_relaxed
 155 #define _os_rel_barrier_release             _os_atomic_mo_release
 156 #define _os_rel_barrier_acq_rel             _os_atomic_mo_release
 157 #define _os_rel_barrier_seq_cst             _os_atomic_mo_release
 158
 159 #define _os_acq_barrier_relaxed             _os_atomic_mo_relaxed
 160 #define _os_acq_barrier_compiler_acquire    _os_atomic_mo_acquire
 161 #define _os_acq_barrier_compiler_release    _os_atomic_mo_relaxed
 162 #define _os_acq_barrier_compiler_acq_rel    _os_atomic_mo_acquire
 163 #define _os_acq_barrier_consume             _os_atomic_mo_acquire
 164 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 165 #define _os_acq_barrier_dependency          _os_atomic_mo_acquire
 166 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 167 #define _os_acq_barrier_acquire             _os_atomic_mo_acquire
 168 #define _os_acq_barrier_release             _os_atomic_mo_relaxed
 169 #define _os_acq_barrier_acq_rel             _os_atomic_mo_acquire
 170 #define _os_acq_barrier_seq_cst             _os_atomic_mo_acquire
 171
 172 #define _os_compiler_barrier_before_atomic(m) \
 173         os_atomic_std(atomic_signal_fence)(_os_rel_barrier_##m)
 174 #define _os_compiler_barrier_after_atomic(m) \
 175         os_atomic_std(atomic_signal_fence)(_os_acq_barrier_##m)
 176
 177 /*
 178  * Mapping between compiler barrier/memmory orders and:
 179  * - memory fences before atomics ("rel_fence")
 180  * - memory fences after atomics ("acq_fence")
 181  */
 182 #define _os_rel_fence_relaxed               _os_atomic_mo_relaxed
 183 #define _os_rel_fence_compiler_acquire      _os_atomic_mo_relaxed
 184 #define _os_rel_fence_compiler_release      _os_atomic_mo_release
 185 #define _os_rel_fence_compiler_acq_rel      _os_atomic_mo_release
 186 #define _os_rel_fence_consume               _os_atomic_mo_relaxed_smp
 187 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 188 #define _os_rel_fence_dependency            _os_atomic_mo_relaxed_smp
 189 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 190 #define _os_rel_fence_acquire               _os_atomic_mo_relaxed_smp
 191 #define _os_rel_fence_release               _os_atomic_mo_release_smp
 192 #define _os_rel_fence_acq_rel               _os_atomic_mo_release_smp
 193 #define _os_rel_fence_seq_cst               _os_atomic_mo_release_smp
 194
 195 #define _os_acq_fence_relaxed               _os_atomic_mo_relaxed
 196 #define _os_acq_fence_compiler_acquire      _os_atomic_mo_relaxed
 197 #define _os_acq_fence_compiler_release      _os_atomic_mo_relaxed
 198 #define _os_acq_fence_compiler_acq_rel      _os_atomic_mo_relaxed
 199 #define _os_acq_fence_consume               _os_atomic_mo_acquire_smp
 200 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 201 #define _os_acq_fence_dependency            _os_atomic_mo_dependency_smp
 202 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 203 #define _os_acq_fence_acquire               _os_atomic_mo_acquire_smp
 204 #define _os_acq_fence_release               _os_atomic_mo_relaxed_smp
 205 #define _os_acq_fence_acq_rel               _os_atomic_mo_acquire_smp
 206 #define _os_acq_fence_seq_cst               _os_atomic_mo_acquire_smp
 207
 208 #define _os_memory_fence_before_atomic(m) \
 209         os_atomic_std(atomic_thread_fence)(_os_rel_fence_##m)
 210 #define _os_memory_fence_after_atomic(m) \
 211         os_atomic_std(atomic_thread_fence)(_os_acq_fence_##m)
 212
 213 /*
 214  * Misc. helpers
 215  */
 216
 217 #define _os_atomic_value_cast(p, v) \
 218         ({ typeof(*os_cast_to_nonatomic_pointer(p)) ___v = (v); ___v; })
 219
 220 #define _os_atomic_c11_op_orig(p, v, m, o)  ({ \
 221         _os_compiler_barrier_before_atomic(m); \
 222         __auto_type _r = os_atomic_std(atomic_##o##_explicit)(\
 223             os_cast_to_atomic_pointer(p), \
 224             _os_atomic_value_cast(p, v), \
 225             _os_atomic_mo_##m##_smp); \
 226         _os_compiler_barrier_after_atomic(m); \
 227         _r; \
 228 })
 229
 230 #define _os_atomic_c11_op(p, v, m, o, op) ({ \
 231         __auto_type _v = _os_atomic_value_cast(p, v); \
 232         _os_atomic_c11_op_orig(p, _v, m, o) op _v; \
 233 })
 234
 235 #define _os_atomic_clang_op_orig(p, v, m, o)  ({ \
 236         _os_compiler_barrier_before_atomic(m); \
 237         __auto_type _r = __atomic_##o(os_cast_to_nonatomic_pointer(p), \
 238             _os_atomic_value_cast(p, v), \
 239             _os_atomic_mo_##m##_smp); \
 240         _os_compiler_barrier_after_atomic(m); \
 241         _r; \
 242 })
 243
 244 #define _os_atomic_clang_op(p, v, m, o, op) ({ \
 245         __auto_type _v = _os_atomic_value_cast(p, v); \
 246         __auto_type _s = _os_atomic_clang_op_orig(p, _v, m, o); \
 247         op(_s, _v); \
 248 })
 249
 250 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 251 #define _os_atomic_auto_dependency(e) \
 252         _Generic(e, \
 253             os_atomic_dependency_t: (e), \
 254             default: os_atomic_make_dependency(e))
 255 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
 256
 257 #endif /* __OS_ATOMIC_PRIVATE_IMPL_H__ */