deps/jemalloc.orig/include/jemalloc/internal/atomic.h

   1 /******************************************************************************/
   2 #ifdef JEMALLOC_H_TYPES
   3
   4 #endif /* JEMALLOC_H_TYPES */
   5 /******************************************************************************/
   6 #ifdef JEMALLOC_H_STRUCTS
   7
   8 #endif /* JEMALLOC_H_STRUCTS */
   9 /******************************************************************************/
  10 #ifdef JEMALLOC_H_EXTERNS
  11
  12 #define atomic_read_uint64(p)   atomic_add_uint64(p, 0)
  13 #define atomic_read_uint32(p)   atomic_add_uint32(p, 0)
  14
  15 #if (LG_SIZEOF_PTR == 3)
  16 #  define atomic_read_z(p)                                              \
  17     (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0)
  18 #  define atomic_add_z(p, x)                                            \
  19     (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)
  20 #  define atomic_sub_z(p, x)                                            \
  21     (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x)
  22 #elif (LG_SIZEOF_PTR == 2)
  23 #  define atomic_read_z(p)                                              \
  24     (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0)
  25 #  define atomic_add_z(p, x)                                            \
  26     (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)
  27 #  define atomic_sub_z(p, x)                                            \
  28     (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x)
  29 #endif
  30
  31 #endif /* JEMALLOC_H_EXTERNS */
  32 /******************************************************************************/
  33 #ifdef JEMALLOC_H_INLINES
  34
  35 #ifndef JEMALLOC_ENABLE_INLINE
  36 uint64_t        atomic_add_uint64(uint64_t *p, uint64_t x);
  37 uint64_t        atomic_sub_uint64(uint64_t *p, uint64_t x);
  38 uint32_t        atomic_add_uint32(uint32_t *p, uint32_t x);
  39 uint32_t        atomic_sub_uint32(uint32_t *p, uint32_t x);
  40 #endif
  41
  42 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
  43 /******************************************************************************/
  44 /* 64-bit operations. */
  45 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
  46 JEMALLOC_INLINE uint64_t
  47 atomic_add_uint64(uint64_t *p, uint64_t x)
  48 {
  49
  50         return (__sync_add_and_fetch(p, x));
  51 }
  52
  53 JEMALLOC_INLINE uint64_t
  54 atomic_sub_uint64(uint64_t *p, uint64_t x)
  55 {
  56
  57         return (__sync_sub_and_fetch(p, x));
  58 }
  59 #elif (defined(JEMALLOC_OSATOMIC))
  60 JEMALLOC_INLINE uint64_t
  61 atomic_add_uint64(uint64_t *p, uint64_t x)
  62 {
  63
  64         return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
  65 }
  66
  67 JEMALLOC_INLINE uint64_t
  68 atomic_sub_uint64(uint64_t *p, uint64_t x)
  69 {
  70
  71         return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
  72 }
  73 #elif (defined(__amd64_) || defined(__x86_64__))
  74 JEMALLOC_INLINE uint64_t
  75 atomic_add_uint64(uint64_t *p, uint64_t x)
  76 {
  77
  78         asm volatile (
  79             "lock; xaddq %0, %1;"
  80             : "+r" (x), "=m" (*p) /* Outputs. */
  81             : "m" (*p) /* Inputs. */
  82             );
  83
  84         return (x);
  85 }
  86
  87 JEMALLOC_INLINE uint64_t
  88 atomic_sub_uint64(uint64_t *p, uint64_t x)
  89 {
  90
  91         x = (uint64_t)(-(int64_t)x);
  92         asm volatile (
  93             "lock; xaddq %0, %1;"
  94             : "+r" (x), "=m" (*p) /* Outputs. */
  95             : "m" (*p) /* Inputs. */
  96             );
  97
  98         return (x);
  99 }
 100 #else
 101 #  if (LG_SIZEOF_PTR == 3)
 102 #    error "Missing implementation for 64-bit atomic operations"
 103 #  endif
 104 #endif
 105
 106 /******************************************************************************/
 107 /* 32-bit operations. */
 108 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
 109 JEMALLOC_INLINE uint32_t
 110 atomic_add_uint32(uint32_t *p, uint32_t x)
 111 {
 112
 113         return (__sync_add_and_fetch(p, x));
 114 }
 115
 116 JEMALLOC_INLINE uint32_t
 117 atomic_sub_uint32(uint32_t *p, uint32_t x)
 118 {
 119
 120         return (__sync_sub_and_fetch(p, x));
 121 }
 122 #elif (defined(JEMALLOC_OSATOMIC))
 123 JEMALLOC_INLINE uint32_t
 124 atomic_add_uint32(uint32_t *p, uint32_t x)
 125 {
 126
 127         return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
 128 }
 129
 130 JEMALLOC_INLINE uint32_t
 131 atomic_sub_uint32(uint32_t *p, uint32_t x)
 132 {
 133
 134         return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
 135 }
 136 #elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__))
 137 JEMALLOC_INLINE uint32_t
 138 atomic_add_uint32(uint32_t *p, uint32_t x)
 139 {
 140
 141         asm volatile (
 142             "lock; xaddl %0, %1;"
 143             : "+r" (x), "=m" (*p) /* Outputs. */
 144             : "m" (*p) /* Inputs. */
 145             );
 146
 147         return (x);
 148 }
 149
 150 JEMALLOC_INLINE uint32_t
 151 atomic_sub_uint32(uint32_t *p, uint32_t x)
 152 {
 153
 154         x = (uint32_t)(-(int32_t)x);
 155         asm volatile (
 156             "lock; xaddl %0, %1;"
 157             : "+r" (x), "=m" (*p) /* Outputs. */
 158             : "m" (*p) /* Inputs. */
 159             );
 160
 161         return (x);
 162 }
 163 #else
 164 #  error "Missing implementation for 32-bit atomic operations"
 165 #endif
 166 #endif
 167
 168 #endif /* JEMALLOC_H_INLINES */
 169 /******************************************************************************/