osfmk/machine/atomic.h

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _MACHINE_ATOMIC_H
  30 #define _MACHINE_ATOMIC_H
  31
  32 #include <stdatomic.h>
  33
  34 #define _os_atomic_c11_atomic(p) \
  35                 ((typeof(*(p)) _Atomic *)(p))
  36
  37 #define _os_atomic_basetypeof(p) \
  38                 typeof(atomic_load(((typeof(*(p)) _Atomic *)(p))))
  39
  40 #define _os_atomic_c11_op_orig(p, v, m, o) \
  41                 atomic_##o##_explicit(_os_atomic_c11_atomic(p), v, \
  42                 memory_order_##m)
  43
  44 #define _os_atomic_c11_op(p, v, m, o, op) \
  45                 ({ typeof(v) _v = (v); _os_atomic_c11_op_orig(p, v, m, o) op _v; })
  46
  47 #define os_atomic_thread_fence(m)  atomic_thread_fence(memory_order_##m)
  48
  49 #define os_atomic_load(p, m) \
  50                 atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_##m)
  51 #define os_atomic_store(p, v, m)    _os_atomic_c11_op_orig(p, v, m, store)
  52
  53 #define os_atomic_add_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_add)
  54 #define os_atomic_add(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_add, +)
  55
  56 #define os_atomic_inc_orig(p, m)    _os_atomic_c11_op_orig(p, 1, m, fetch_add)
  57 #define os_atomic_inc(p, m)         _os_atomic_c11_op(p, 1, m, fetch_add, +)
  58
  59 #define os_atomic_sub_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_sub)
  60 #define os_atomic_sub(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_sub, -)
  61
  62 #define os_atomic_dec_orig(p, m)    _os_atomic_c11_op_orig(p, 1, m, fetch_sub)
  63 #define os_atomic_dec(p, m)         _os_atomic_c11_op(p, 1, m, fetch_sub, -)
  64
  65 #define os_atomic_and_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_and)
  66 #define os_atomic_and(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_and, &)
  67
  68 #define os_atomic_or_orig(p, v, m)  _os_atomic_c11_op_orig(p, v, m, fetch_or)
  69 #define os_atomic_or(p, v, m)       _os_atomic_c11_op(p, v, m, fetch_or, |)
  70
  71 #define os_atomic_xor_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_xor)
  72 #define os_atomic_xor(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_xor, ^)
  73
  74 #define os_atomic_xchg(p, v, m)     _os_atomic_c11_op_orig(p, v, m, exchange)
  75
  76 #define os_atomic_cmpxchg(p, e, v, m) \
  77                 ({ _os_atomic_basetypeof(p) _r = (e); \
  78                 atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
  79                 &_r, v, memory_order_##m, memory_order_relaxed); })
  80 #define os_atomic_cmpxchgv(p, e, v, g, m) \
  81                 ({ _os_atomic_basetypeof(p) _r = (e); int _b = \
  82                 atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
  83                 &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
  84 #define os_atomic_cmpxchgvw(p, e, v, g, m) \
  85                 ({ _os_atomic_basetypeof(p) _r = (e); int _b = \
  86                 atomic_compare_exchange_weak_explicit(_os_atomic_c11_atomic(p), \
  87                 &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r;  _b; })
  88
  89 #define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
  90                 bool _result = false; \
  91                 typeof(p) _p = (p); \
  92                 ov = os_atomic_load(_p, relaxed); \
  93                 do { \
  94                         __VA_ARGS__; \
  95                         _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \
  96                 } while (!_result); \
  97                 _result; \
  98         })
  99
 100 #define os_atomic_rmw_loop_give_up_with_fence(m, expr) \
 101                 ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); })
 102 #define os_atomic_rmw_loop_give_up(expr) \
 103                 os_atomic_rmw_loop_give_up_with_fence(relaxed, expr)
 104
 105 #define os_atomic_force_dependency_on(p, e) (p)
 106 #define os_atomic_load_with_dependency_on(p, e) \
 107                 os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed)
 108
 109 #if defined (__x86_64__)
 110 #include "i386/atomic.h"
 111 #elif defined (__arm__) || defined (__arm64__)
 112 #include "arm/atomic.h"
 113 #else
 114 #error architecture not supported
 115 #endif
 116
 117 #endif /* _MACHINE_ATOMIC_H */