-#if __SMP__
-
-#define memory_order_consume_smp memory_order_consume
-#define memory_order_acquire_smp memory_order_acquire
-#define memory_order_release_smp memory_order_release
-#define memory_order_acq_rel_smp memory_order_acq_rel
-#define memory_order_seq_cst_smp memory_order_seq_cst
-
-#else
-
-#define memory_order_consume_smp memory_order_relaxed
-#define memory_order_acquire_smp memory_order_relaxed
-#define memory_order_release_smp memory_order_relaxed
-#define memory_order_acq_rel_smp memory_order_relaxed
-#define memory_order_seq_cst_smp memory_order_relaxed
-
-#endif
-
-/*
- * Atomic operations functions
- *
- * These static functions are designed for inlining
- * It is expected that the memory_order arguments are
- * known at compile time. This collapses these
- * functions into a simple atomic operation
- */
-
-static inline boolean_t
-memory_order_has_acquire(enum memory_order ord)
-{
- switch (ord) {
- case memory_order_consume:
- case memory_order_acquire:
- case memory_order_acq_rel:
- case memory_order_seq_cst:
- return TRUE;
- default:
- return FALSE;
- }
-}
-
-static inline boolean_t
-memory_order_has_release(enum memory_order ord)
-{
- switch (ord) {
- case memory_order_release:
- case memory_order_acq_rel:
- case memory_order_seq_cst:
- return TRUE;
- default:
- return FALSE;
- }
-}
-
-#ifdef ATOMIC_PRIVATE
-
-#define clear_exclusive() __builtin_arm_clrex()
-
-__unused static uint32_t
-load_exclusive32(uint32_t *target, enum memory_order ord)
-{
- uint32_t value;
-
-#if __arm__
- if (memory_order_has_release(ord)) {
- // Pre-load release barrier
- atomic_thread_fence(memory_order_release);
- }
- value = __builtin_arm_ldrex(target);
-#else
- if (memory_order_has_acquire(ord)) {
- value = __builtin_arm_ldaex(target); // ldaxr
- } else {
- value = __builtin_arm_ldrex(target); // ldxr
- }
-#endif // __arm__
- return value;
-}
-
-__unused static boolean_t
-store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord)
-{
- boolean_t err;
-
-#if __arm__
- err = __builtin_arm_strex(value, target);
- if (memory_order_has_acquire(ord)) {
- // Post-store acquire barrier
- atomic_thread_fence(memory_order_acquire);
- }
-#else
- if (memory_order_has_release(ord)) {
- err = __builtin_arm_stlex(value, target); // stlxr
- } else {
- err = __builtin_arm_strex(value, target); // stxr
- }
-#endif // __arm__
- return !err;
-}
-
-__unused static uintptr_t
-load_exclusive(uintptr_t *target, enum memory_order ord)
-{
-#if !__LP64__
- return load_exclusive32((uint32_t *)target, ord);
-#else
- uintptr_t value;
-
- if (memory_order_has_acquire(ord)) {
- value = __builtin_arm_ldaex(target); // ldaxr
- } else {
- value = __builtin_arm_ldrex(target); // ldxr
- }
- return value;
-#endif // __arm__
-}
-
-__unused static uint8_t
-load_exclusive_acquire8(uint8_t *target)
-{
- uint8_t value;
-#if __arm__
- value = __builtin_arm_ldrex(target);
- __c11_atomic_thread_fence(__ATOMIC_ACQUIRE);
-#else
- value = __builtin_arm_ldaex(target); // ldaxr
- /* "Compiler barrier", no barrier instructions are emitted */
- atomic_signal_fence(memory_order_acquire);
-#endif
- return value;
-}
-
-__unused static boolean_t
-store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord)
-{
-#if !__LP64__
- return store_exclusive32((uint32_t *)target, value, ord);
-#else
- boolean_t err;
-
- if (memory_order_has_release(ord)) {
- err = __builtin_arm_stlex(value, target); // stlxr
- } else {
- err = __builtin_arm_strex(value, target); // stxr
- }
- return !err;
-#endif
-}
-
-__unused static boolean_t
-atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval,
- enum memory_order orig_ord, boolean_t wait)
-{
- enum memory_order ord = orig_ord;
- uintptr_t value;
-
-
-#if __arm__
- ord = memory_order_relaxed;
- if (memory_order_has_release(orig_ord)) {
- atomic_thread_fence(memory_order_release);
- }
-#endif
- do {
- value = load_exclusive(target, ord);
- if (value != oldval) {
- if (wait) {
- wait_for_event(); // Wait with monitor held
- } else {
- clear_exclusive(); // Clear exclusive monitor
- }
- return FALSE;
- }
- } while (!store_exclusive(target, newval, ord));
-#if __arm__
- if (memory_order_has_acquire(orig_ord)) {
- atomic_thread_fence(memory_order_acquire);
- }
-#endif
- return TRUE;
-}
-
-#endif // ATOMIC_PRIVATE
-
-#if __arm__
-#undef os_atomic_rmw_loop
-#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
- boolean_t _result = FALSE; uint32_t _err = 0; \
- typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \
- for (;;) { \
- ov = __builtin_arm_ldrex(_p); \
- __VA_ARGS__; \
- if (!_err && memory_order_has_release(memory_order_##m)) { \
- /* only done for the first loop iteration */ \
- atomic_thread_fence(memory_order_release); \
- } \
- _err = __builtin_arm_strex(nv, _p); \
- if (__builtin_expect(!_err, 1)) { \
- if (memory_order_has_acquire(memory_order_##m)) { \
- atomic_thread_fence(memory_order_acquire); \
- } \
- _result = TRUE; \
- break; \
- } \
- } \
- _result; \
- })
-
-#undef os_atomic_rmw_loop_give_up
-#define os_atomic_rmw_loop_give_up(expr) \
- ({ __builtin_arm_clrex(); expr; __builtin_trap(); })
-
-#else
-
-#undef os_atomic_rmw_loop
-#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
- boolean_t _result = FALSE; \
- typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \
- do { \
- if (memory_order_has_acquire(memory_order_##m)) { \
- ov = __builtin_arm_ldaex(_p); \
- } else { \
- ov = __builtin_arm_ldrex(_p); \
- } \
- __VA_ARGS__; \
- if (memory_order_has_release(memory_order_##m)) { \
- _result = !__builtin_arm_stlex(nv, _p); \
- } else { \
- _result = !__builtin_arm_strex(nv, _p); \
- } \
- } while (__builtin_expect(!_result, 0)); \
- _result; \
- })
-
-#undef os_atomic_rmw_loop_give_up
-#define os_atomic_rmw_loop_give_up(expr) \
- ({ __builtin_arm_clrex(); expr; __builtin_trap(); })
-#endif
-
-#undef os_atomic_force_dependency_on
-#if defined(__arm64__)
-#define os_atomic_force_dependency_on(p, e) ({ \
- unsigned long _v; \
- __asm__("and %x[_v], %x[_e], xzr" : [_v] "=r" (_v) : [_e] "r" (e)); \
- (typeof(*(p)) *)((char *)(p) + _v); \
- })
-#else
-#define os_atomic_force_dependency_on(p, e) ({ \
- unsigned long _v; \
- __asm__("and %[_v], %[_e], #0" : [_v] "=r" (_v) : [_e] "r" (e)); \
- (typeof(*(p)) *)((char *)(p) + _v); \
- })
-#endif // defined(__arm64__)
-
projects / apple / xnu.git / blobdiff