]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/arm/atomic.h
xnu-7195.60.75.tar.gz
[apple/xnu.git] / osfmk / arm / atomic.h
index 380286cded9dddb098292074dc69b3b3623700ff..2b679dbb381d1b60b4b77414a3130c4ea4e56125 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015 Apple Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#ifndef _MACHINE_ATOMIC_H
+#error "Do not include <arm/atomic.h> directly, use <machine/atomic.h>"
+#endif
+
 #ifndef _ARM_ATOMIC_H_
 #define _ARM_ATOMIC_H_
 
 #include <mach/boolean.h>
-#include <arm/smp.h>
 
 // Parameter for __builtin_arm_dmb
+#define DMB_OSHLD       0x1
+#define DMB_OSHST       0x2
+#define DMB_OSH         0x3
+#define DMB_NSHLD       0x5
+#define DMB_NSHST       0x6
 #define DMB_NSH         0x7
 #define DMB_ISHLD       0x9
 #define DMB_ISHST       0xa
 #define DMB_ISH         0xb
+#define DMB_LD          0xd
+#define DMB_ST          0xe
 #define DMB_SY          0xf
 
 // Parameter for __builtin_arm_dsb
+#define DSB_OSHLD       0x1
+#define DSB_OSHST       0x2
+#define DSB_OSH         0x3
+#define DSB_NSHLD       0x5
+#define DSB_NSHST       0x6
 #define DSB_NSH         0x7
 #define DSB_ISHLD       0x9
 #define DSB_ISHST       0xa
 #define DSB_ISH         0xb
+#define DSB_LD          0xd
+#define DSB_ST          0xe
 #define DSB_SY          0xf
 
 // Parameter for __builtin_arm_isb
 #define ISB_SY          0xf
 
-#if     __SMP__
-
-#define memory_order_consume_smp memory_order_consume
-#define memory_order_acquire_smp memory_order_acquire
-#define memory_order_release_smp memory_order_release
-#define memory_order_acq_rel_smp memory_order_acq_rel
-#define memory_order_seq_cst_smp memory_order_seq_cst
-
-#else
-
-#define memory_order_consume_smp memory_order_relaxed
-#define memory_order_acquire_smp memory_order_relaxed
-#define memory_order_release_smp memory_order_relaxed
-#define memory_order_acq_rel_smp memory_order_relaxed
-#define memory_order_seq_cst_smp memory_order_relaxed
-
-#endif
-
-/*
- * Atomic operations functions
- *
- * These static functions are designed for inlining
- * It is expected that the memory_order arguments are
- * known at compile time.  This collapses these
- * functions into a simple atomic operation
- */
-
-static inline boolean_t
-memory_order_has_acquire(enum memory_order ord)
-{
-       switch (ord) {
-       case memory_order_consume:
-       case memory_order_acquire:
-       case memory_order_acq_rel:
-       case memory_order_seq_cst:
-               return TRUE;
-       default:
-               return FALSE;
-       }
-}
-
-static inline boolean_t
-memory_order_has_release(enum memory_order ord)
-{
-       switch (ord) {
-       case memory_order_release:
-       case memory_order_acq_rel:
-       case memory_order_seq_cst:
-               return TRUE;
-       default:
-               return FALSE;
-       }
-}
-
-#ifdef ATOMIC_PRIVATE
-
-#define clear_exclusive()       __builtin_arm_clrex()
-
-__unused static uint32_t
-load_exclusive32(uint32_t *target, enum memory_order ord)
-{
-       uint32_t        value;
-
-#if __arm__
-       if (memory_order_has_release(ord)) {
-               // Pre-load release barrier
-               atomic_thread_fence(memory_order_release);
-       }
-       value = __builtin_arm_ldrex(target);
-#else
-       if (memory_order_has_acquire(ord)) {
-               value = __builtin_arm_ldaex(target);    // ldaxr
-       } else {
-               value = __builtin_arm_ldrex(target);    // ldxr
-       }
-#endif  // __arm__
-       return value;
-}
-
-__unused static boolean_t
-store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord)
-{
-       boolean_t err;
-
-#if __arm__
-       err = __builtin_arm_strex(value, target);
-       if (memory_order_has_acquire(ord)) {
-               // Post-store acquire barrier
-               atomic_thread_fence(memory_order_acquire);
-       }
-#else
-       if (memory_order_has_release(ord)) {
-               err = __builtin_arm_stlex(value, target);       // stlxr
-       } else {
-               err = __builtin_arm_strex(value, target);       // stxr
-       }
-#endif  // __arm__
-       return !err;
-}
-
-__unused static uintptr_t
-load_exclusive(uintptr_t *target, enum memory_order ord)
-{
-#if !__LP64__
-       return load_exclusive32((uint32_t *)target, ord);
-#else
-       uintptr_t       value;
-
-       if (memory_order_has_acquire(ord)) {
-               value = __builtin_arm_ldaex(target);    // ldaxr
-       } else {
-               value = __builtin_arm_ldrex(target);    // ldxr
-       }
-       return value;
-#endif  // __arm__
-}
-
-__unused static uint8_t
-load_exclusive_acquire8(uint8_t *target)
-{
-       uint8_t value;
-#if __arm__
-       value = __builtin_arm_ldrex(target);
-       __c11_atomic_thread_fence(__ATOMIC_ACQUIRE);
-#else
-       value = __builtin_arm_ldaex(target);    // ldaxr
-       /* "Compiler barrier", no barrier instructions are emitted */
-       atomic_signal_fence(memory_order_acquire);
-#endif
-       return value;
-}
-
-__unused static boolean_t
-store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord)
-{
-#if !__LP64__
-       return store_exclusive32((uint32_t *)target, value, ord);
-#else
-       boolean_t err;
-
-       if (memory_order_has_release(ord)) {
-               err = __builtin_arm_stlex(value, target);       // stlxr
-       } else {
-               err = __builtin_arm_strex(value, target);       // stxr
-       }
-       return !err;
-#endif
-}
-
-__unused static boolean_t
-atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval,
-    enum memory_order orig_ord, boolean_t wait)
-{
-       enum memory_order       ord = orig_ord;
-       uintptr_t                       value;
-
-
-#if __arm__
-       ord = memory_order_relaxed;
-       if (memory_order_has_release(orig_ord)) {
-               atomic_thread_fence(memory_order_release);
-       }
-#endif
-       do {
-               value = load_exclusive(target, ord);
-               if (value != oldval) {
-                       if (wait) {
-                               wait_for_event();       // Wait with monitor held
-                       } else {
-                               clear_exclusive();      // Clear exclusive monitor
-                       }
-                       return FALSE;
-               }
-       } while (!store_exclusive(target, newval, ord));
-#if __arm__
-       if (memory_order_has_acquire(orig_ord)) {
-               atomic_thread_fence(memory_order_acquire);
-       }
-#endif
-       return TRUE;
-}
-
-#endif // ATOMIC_PRIVATE
-
-#if __arm__
-#undef os_atomic_rmw_loop
-#define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
-               boolean_t _result = FALSE; uint32_t _err = 0; \
-               typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \
-               for (;;) { \
-                       ov = __builtin_arm_ldrex(_p); \
-                       __VA_ARGS__; \
-                       if (!_err && memory_order_has_release(memory_order_##m)) { \
-       /* only done for the first loop iteration */ \
-                               atomic_thread_fence(memory_order_release); \
-                       } \
-                       _err = __builtin_arm_strex(nv, _p); \
-                       if (__builtin_expect(!_err, 1)) { \
-                               if (memory_order_has_acquire(memory_order_##m)) { \
-                                       atomic_thread_fence(memory_order_acquire); \
-                               } \
-                               _result = TRUE; \
-                               break; \
-                       } \
-               } \
-               _result; \
-       })
-
-#undef os_atomic_rmw_loop_give_up
-#define os_atomic_rmw_loop_give_up(expr) \
-               ({ __builtin_arm_clrex(); expr; __builtin_trap(); })
-
-#else
-
-#undef os_atomic_rmw_loop
-#define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
-               boolean_t _result = FALSE; \
-               typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \
-               do { \
-                       if (memory_order_has_acquire(memory_order_##m)) { \
-                               ov = __builtin_arm_ldaex(_p); \
-                       } else { \
-                               ov = __builtin_arm_ldrex(_p); \
-                       } \
-                       __VA_ARGS__; \
-                       if (memory_order_has_release(memory_order_##m)) { \
-                               _result = !__builtin_arm_stlex(nv, _p); \
-                       } else { \
-                               _result = !__builtin_arm_strex(nv, _p); \
-                       } \
-               } while (__builtin_expect(!_result, 0)); \
-               _result; \
-       })
-
-#undef os_atomic_rmw_loop_give_up
-#define os_atomic_rmw_loop_give_up(expr) \
-               ({ __builtin_arm_clrex(); expr; __builtin_trap(); })
-#endif
-
-#undef os_atomic_force_dependency_on
-#if defined(__arm64__)
-#define os_atomic_force_dependency_on(p, e) ({ \
-               unsigned long _v; \
-               __asm__("and %x[_v], %x[_e], xzr" : [_v] "=r" (_v) : [_e] "r" (e)); \
-               (typeof(*(p)) *)((char *)(p) + _v); \
-       })
-#else
-#define os_atomic_force_dependency_on(p, e) ({ \
-               unsigned long _v; \
-               __asm__("and %[_v], %[_e], #0" : [_v] "=r" (_v) : [_e] "r" (e)); \
-               (typeof(*(p)) *)((char *)(p) + _v); \
-       })
-#endif // defined(__arm64__)
-
 #endif // _ARM_ATOMIC_H_