X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0a7de7458d150b5d4dffc935ba399be265ef0a1a..2a1bd2d3eef5c7a7bb14f4bb9fdbca9a96ee4752:/osfmk/arm/atomic.h diff --git a/osfmk/arm/atomic.h b/osfmk/arm/atomic.h index 380286cde..2b679dbb3 100644 --- a/osfmk/arm/atomic.h +++ b/osfmk/arm/atomic.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Apple Inc. All rights reserved. + * Copyright (c) 2015-2018 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,281 +26,44 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#ifndef _MACHINE_ATOMIC_H +#error "Do not include directly, use " +#endif + #ifndef _ARM_ATOMIC_H_ #define _ARM_ATOMIC_H_ #include -#include // Parameter for __builtin_arm_dmb +#define DMB_OSHLD 0x1 +#define DMB_OSHST 0x2 +#define DMB_OSH 0x3 +#define DMB_NSHLD 0x5 +#define DMB_NSHST 0x6 #define DMB_NSH 0x7 #define DMB_ISHLD 0x9 #define DMB_ISHST 0xa #define DMB_ISH 0xb +#define DMB_LD 0xd +#define DMB_ST 0xe #define DMB_SY 0xf // Parameter for __builtin_arm_dsb +#define DSB_OSHLD 0x1 +#define DSB_OSHST 0x2 +#define DSB_OSH 0x3 +#define DSB_NSHLD 0x5 +#define DSB_NSHST 0x6 #define DSB_NSH 0x7 #define DSB_ISHLD 0x9 #define DSB_ISHST 0xa #define DSB_ISH 0xb +#define DSB_LD 0xd +#define DSB_ST 0xe #define DSB_SY 0xf // Parameter for __builtin_arm_isb #define ISB_SY 0xf -#if __SMP__ - -#define memory_order_consume_smp memory_order_consume -#define memory_order_acquire_smp memory_order_acquire -#define memory_order_release_smp memory_order_release -#define memory_order_acq_rel_smp memory_order_acq_rel -#define memory_order_seq_cst_smp memory_order_seq_cst - -#else - -#define memory_order_consume_smp memory_order_relaxed -#define memory_order_acquire_smp memory_order_relaxed -#define memory_order_release_smp memory_order_relaxed -#define memory_order_acq_rel_smp memory_order_relaxed -#define memory_order_seq_cst_smp memory_order_relaxed - -#endif - -/* - * Atomic operations functions - * - * These static functions are designed for inlining - * It is expected that the memory_order arguments are - * known at compile time. This collapses these - * functions into a simple atomic operation - */ - -static inline boolean_t -memory_order_has_acquire(enum memory_order ord) -{ - switch (ord) { - case memory_order_consume: - case memory_order_acquire: - case memory_order_acq_rel: - case memory_order_seq_cst: - return TRUE; - default: - return FALSE; - } -} - -static inline boolean_t -memory_order_has_release(enum memory_order ord) -{ - switch (ord) { - case memory_order_release: - case memory_order_acq_rel: - case memory_order_seq_cst: - return TRUE; - default: - return FALSE; - } -} - -#ifdef ATOMIC_PRIVATE - -#define clear_exclusive() __builtin_arm_clrex() - -__unused static uint32_t -load_exclusive32(uint32_t *target, enum memory_order ord) -{ - uint32_t value; - -#if __arm__ - if (memory_order_has_release(ord)) { - // Pre-load release barrier - atomic_thread_fence(memory_order_release); - } - value = __builtin_arm_ldrex(target); -#else - if (memory_order_has_acquire(ord)) { - value = __builtin_arm_ldaex(target); // ldaxr - } else { - value = __builtin_arm_ldrex(target); // ldxr - } -#endif // __arm__ - return value; -} - -__unused static boolean_t -store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord) -{ - boolean_t err; - -#if __arm__ - err = __builtin_arm_strex(value, target); - if (memory_order_has_acquire(ord)) { - // Post-store acquire barrier - atomic_thread_fence(memory_order_acquire); - } -#else - if (memory_order_has_release(ord)) { - err = __builtin_arm_stlex(value, target); // stlxr - } else { - err = __builtin_arm_strex(value, target); // stxr - } -#endif // __arm__ - return !err; -} - -__unused static uintptr_t -load_exclusive(uintptr_t *target, enum memory_order ord) -{ -#if !__LP64__ - return load_exclusive32((uint32_t *)target, ord); -#else - uintptr_t value; - - if (memory_order_has_acquire(ord)) { - value = __builtin_arm_ldaex(target); // ldaxr - } else { - value = __builtin_arm_ldrex(target); // ldxr - } - return value; -#endif // __arm__ -} - -__unused static uint8_t -load_exclusive_acquire8(uint8_t *target) -{ - uint8_t value; -#if __arm__ - value = __builtin_arm_ldrex(target); - __c11_atomic_thread_fence(__ATOMIC_ACQUIRE); -#else - value = __builtin_arm_ldaex(target); // ldaxr - /* "Compiler barrier", no barrier instructions are emitted */ - atomic_signal_fence(memory_order_acquire); -#endif - return value; -} - -__unused static boolean_t -store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord) -{ -#if !__LP64__ - return store_exclusive32((uint32_t *)target, value, ord); -#else - boolean_t err; - - if (memory_order_has_release(ord)) { - err = __builtin_arm_stlex(value, target); // stlxr - } else { - err = __builtin_arm_strex(value, target); // stxr - } - return !err; -#endif -} - -__unused static boolean_t -atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval, - enum memory_order orig_ord, boolean_t wait) -{ - enum memory_order ord = orig_ord; - uintptr_t value; - - -#if __arm__ - ord = memory_order_relaxed; - if (memory_order_has_release(orig_ord)) { - atomic_thread_fence(memory_order_release); - } -#endif - do { - value = load_exclusive(target, ord); - if (value != oldval) { - if (wait) { - wait_for_event(); // Wait with monitor held - } else { - clear_exclusive(); // Clear exclusive monitor - } - return FALSE; - } - } while (!store_exclusive(target, newval, ord)); -#if __arm__ - if (memory_order_has_acquire(orig_ord)) { - atomic_thread_fence(memory_order_acquire); - } -#endif - return TRUE; -} - -#endif // ATOMIC_PRIVATE - -#if __arm__ -#undef os_atomic_rmw_loop -#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ - boolean_t _result = FALSE; uint32_t _err = 0; \ - typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ - for (;;) { \ - ov = __builtin_arm_ldrex(_p); \ - __VA_ARGS__; \ - if (!_err && memory_order_has_release(memory_order_##m)) { \ - /* only done for the first loop iteration */ \ - atomic_thread_fence(memory_order_release); \ - } \ - _err = __builtin_arm_strex(nv, _p); \ - if (__builtin_expect(!_err, 1)) { \ - if (memory_order_has_acquire(memory_order_##m)) { \ - atomic_thread_fence(memory_order_acquire); \ - } \ - _result = TRUE; \ - break; \ - } \ - } \ - _result; \ - }) - -#undef os_atomic_rmw_loop_give_up -#define os_atomic_rmw_loop_give_up(expr) \ - ({ __builtin_arm_clrex(); expr; __builtin_trap(); }) - -#else - -#undef os_atomic_rmw_loop -#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ - boolean_t _result = FALSE; \ - typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ - do { \ - if (memory_order_has_acquire(memory_order_##m)) { \ - ov = __builtin_arm_ldaex(_p); \ - } else { \ - ov = __builtin_arm_ldrex(_p); \ - } \ - __VA_ARGS__; \ - if (memory_order_has_release(memory_order_##m)) { \ - _result = !__builtin_arm_stlex(nv, _p); \ - } else { \ - _result = !__builtin_arm_strex(nv, _p); \ - } \ - } while (__builtin_expect(!_result, 0)); \ - _result; \ - }) - -#undef os_atomic_rmw_loop_give_up -#define os_atomic_rmw_loop_give_up(expr) \ - ({ __builtin_arm_clrex(); expr; __builtin_trap(); }) -#endif - -#undef os_atomic_force_dependency_on -#if defined(__arm64__) -#define os_atomic_force_dependency_on(p, e) ({ \ - unsigned long _v; \ - __asm__("and %x[_v], %x[_e], xzr" : [_v] "=r" (_v) : [_e] "r" (e)); \ - (typeof(*(p)) *)((char *)(p) + _v); \ - }) -#else -#define os_atomic_force_dependency_on(p, e) ({ \ - unsigned long _v; \ - __asm__("and %[_v], %[_e], #0" : [_v] "=r" (_v) : [_e] "r" (e)); \ - (typeof(*(p)) *)((char *)(p) + _v); \ - }) -#endif // defined(__arm64__) - #endif // _ARM_ATOMIC_H_