]> git.saurik.com Git - redis.git/blame - deps/jemalloc/include/jemalloc/internal/atomic.h
projects / redis.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Jemalloc updated to 3.0.0.
[redis.git] / deps / jemalloc / include / jemalloc / internal / atomic.h
CommitLineData
a78e148b 1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4#endif /* JEMALLOC_H_TYPES */
5/******************************************************************************/
6#ifdef JEMALLOC_H_STRUCTS
7
8#endif /* JEMALLOC_H_STRUCTS */
9/******************************************************************************/
10#ifdef JEMALLOC_H_EXTERNS
11
12#define atomic_read_uint64(p) atomic_add_uint64(p, 0)
13#define atomic_read_uint32(p) atomic_add_uint32(p, 0)
4934f93d 14#define atomic_read_z(p) atomic_add_z(p, 0)
15#define atomic_read_u(p) atomic_add_u(p, 0)
a78e148b 16
17#endif /* JEMALLOC_H_EXTERNS */
18/******************************************************************************/
19#ifdef JEMALLOC_H_INLINES
20
21#ifndef JEMALLOC_ENABLE_INLINE
22uint64_t atomic_add_uint64(uint64_t *p, uint64_t x);
23uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x);
24uint32_t atomic_add_uint32(uint32_t *p, uint32_t x);
25uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x);
4934f93d 26size_t atomic_add_z(size_t *p, size_t x);
27size_t atomic_sub_z(size_t *p, size_t x);
28unsigned atomic_add_u(unsigned *p, unsigned x);
29unsigned atomic_sub_u(unsigned *p, unsigned x);
a78e148b 30#endif
31
32#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
33/******************************************************************************/
34/* 64-bit operations. */
4934f93d 35#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
36# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
a78e148b 37JEMALLOC_INLINE uint64_t
38atomic_add_uint64(uint64_t *p, uint64_t x)
39{
40
41 return (__sync_add_and_fetch(p, x));
42}
43
44JEMALLOC_INLINE uint64_t
45atomic_sub_uint64(uint64_t *p, uint64_t x)
46{
47
48 return (__sync_sub_and_fetch(p, x));
49}
4934f93d 50#elif (defined(_MSC_VER))
51JEMALLOC_INLINE uint64_t
52atomic_add_uint64(uint64_t *p, uint64_t x)
53{
54
55 return (InterlockedExchangeAdd64(p, x));
56}
57
58JEMALLOC_INLINE uint64_t
59atomic_sub_uint64(uint64_t *p, uint64_t x)
60{
61
62 return (InterlockedExchangeAdd64(p, -((int64_t)x)));
63}
a78e148b 64#elif (defined(JEMALLOC_OSATOMIC))
65JEMALLOC_INLINE uint64_t
66atomic_add_uint64(uint64_t *p, uint64_t x)
67{
68
69 return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
70}
71
72JEMALLOC_INLINE uint64_t
73atomic_sub_uint64(uint64_t *p, uint64_t x)
74{
75
76 return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
77}
4934f93d 78# elif (defined(__amd64__) || defined(__x86_64__))
a78e148b 79JEMALLOC_INLINE uint64_t
80atomic_add_uint64(uint64_t *p, uint64_t x)
81{
82
83 asm volatile (
84 "lock; xaddq %0, %1;"
85 : "+r" (x), "=m" (*p) /* Outputs. */
86 : "m" (*p) /* Inputs. */
87 );
88
89 return (x);
90}
91
92JEMALLOC_INLINE uint64_t
93atomic_sub_uint64(uint64_t *p, uint64_t x)
94{
95
96 x = (uint64_t)(-(int64_t)x);
97 asm volatile (
98 "lock; xaddq %0, %1;"
99 : "+r" (x), "=m" (*p) /* Outputs. */
100 : "m" (*p) /* Inputs. */
101 );
102
103 return (x);
104}
4934f93d 105# elif (defined(JEMALLOC_ATOMIC9))
106JEMALLOC_INLINE uint64_t
107atomic_add_uint64(uint64_t *p, uint64_t x)
108{
109
110 /*
111 * atomic_fetchadd_64() doesn't exist, but we only ever use this
112 * function on LP64 systems, so atomic_fetchadd_long() will do.
113 */
114 assert(sizeof(uint64_t) == sizeof(unsigned long));
115
116 return (atomic_fetchadd_long(p, (unsigned long)x) + x);
117}
118
119JEMALLOC_INLINE uint64_t
120atomic_sub_uint64(uint64_t *p, uint64_t x)
121{
122
123 assert(sizeof(uint64_t) == sizeof(unsigned long));
124
125 return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
126}
127# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
128JEMALLOC_INLINE uint64_t
129atomic_add_uint64(uint64_t *p, uint64_t x)
130{
131
132 return (__sync_add_and_fetch(p, x));
133}
134
135JEMALLOC_INLINE uint64_t
136atomic_sub_uint64(uint64_t *p, uint64_t x)
137{
138
139 return (__sync_sub_and_fetch(p, x));
140}
141# else
a78e148b 142# error "Missing implementation for 64-bit atomic operations"
143# endif
144#endif
145
146/******************************************************************************/
147/* 32-bit operations. */
148#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
149JEMALLOC_INLINE uint32_t
150atomic_add_uint32(uint32_t *p, uint32_t x)
151{
152
153 return (__sync_add_and_fetch(p, x));
154}
155
156JEMALLOC_INLINE uint32_t
157atomic_sub_uint32(uint32_t *p, uint32_t x)
158{
159
160 return (__sync_sub_and_fetch(p, x));
161}
4934f93d 162#elif (defined(_MSC_VER))
163JEMALLOC_INLINE uint32_t
164atomic_add_uint32(uint32_t *p, uint32_t x)
165{
166
167 return (InterlockedExchangeAdd(p, x));
168}
169
170JEMALLOC_INLINE uint32_t
171atomic_sub_uint32(uint32_t *p, uint32_t x)
172{
173
174 return (InterlockedExchangeAdd(p, -((int32_t)x)));
175}
a78e148b 176#elif (defined(JEMALLOC_OSATOMIC))
177JEMALLOC_INLINE uint32_t
178atomic_add_uint32(uint32_t *p, uint32_t x)
179{
180
181 return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
182}
183
184JEMALLOC_INLINE uint32_t
185atomic_sub_uint32(uint32_t *p, uint32_t x)
186{
187
188 return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
189}
4934f93d 190#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
a78e148b 191JEMALLOC_INLINE uint32_t
192atomic_add_uint32(uint32_t *p, uint32_t x)
193{
194
195 asm volatile (
196 "lock; xaddl %0, %1;"
197 : "+r" (x), "=m" (*p) /* Outputs. */
198 : "m" (*p) /* Inputs. */
199 );
200
201 return (x);
202}
203
204JEMALLOC_INLINE uint32_t
205atomic_sub_uint32(uint32_t *p, uint32_t x)
206{
207
208 x = (uint32_t)(-(int32_t)x);
209 asm volatile (
210 "lock; xaddl %0, %1;"
211 : "+r" (x), "=m" (*p) /* Outputs. */
212 : "m" (*p) /* Inputs. */
213 );
214
215 return (x);
216}
4934f93d 217#elif (defined(JEMALLOC_ATOMIC9))
218JEMALLOC_INLINE uint32_t
219atomic_add_uint32(uint32_t *p, uint32_t x)
220{
221
222 return (atomic_fetchadd_32(p, x) + x);
223}
224
225JEMALLOC_INLINE uint32_t
226atomic_sub_uint32(uint32_t *p, uint32_t x)
227{
228
229 return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
230}
231#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
232JEMALLOC_INLINE uint32_t
233atomic_add_uint32(uint32_t *p, uint32_t x)
234{
235
236 return (__sync_add_and_fetch(p, x));
237}
238
239JEMALLOC_INLINE uint32_t
240atomic_sub_uint32(uint32_t *p, uint32_t x)
241{
242
243 return (__sync_sub_and_fetch(p, x));
244}
a78e148b 245#else
246# error "Missing implementation for 32-bit atomic operations"
247#endif
4934f93d 248
249/******************************************************************************/
250/* size_t operations. */
251JEMALLOC_INLINE size_t
252atomic_add_z(size_t *p, size_t x)
253{
254
255#if (LG_SIZEOF_PTR == 3)
256 return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
257#elif (LG_SIZEOF_PTR == 2)
258 return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
259#endif
260}
261
262JEMALLOC_INLINE size_t
263atomic_sub_z(size_t *p, size_t x)
264{
265
266#if (LG_SIZEOF_PTR == 3)
267 return ((size_t)atomic_add_uint64((uint64_t *)p,
268 (uint64_t)-((int64_t)x)));
269#elif (LG_SIZEOF_PTR == 2)
270 return ((size_t)atomic_add_uint32((uint32_t *)p,
271 (uint32_t)-((int32_t)x)));
272#endif
273}
274
275/******************************************************************************/
276/* unsigned operations. */
277JEMALLOC_INLINE unsigned
278atomic_add_u(unsigned *p, unsigned x)
279{
280
281#if (LG_SIZEOF_INT == 3)
282 return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
283#elif (LG_SIZEOF_INT == 2)
284 return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
285#endif
286}
287
288JEMALLOC_INLINE unsigned
289atomic_sub_u(unsigned *p, unsigned x)
290{
291
292#if (LG_SIZEOF_INT == 3)
293 return ((unsigned)atomic_add_uint64((uint64_t *)p,
294 (uint64_t)-((int64_t)x)));
295#elif (LG_SIZEOF_INT == 2)
296 return ((unsigned)atomic_add_uint32((uint32_t *)p,
297 (uint32_t)-((int32_t)x)));
298#endif
299}
300/******************************************************************************/
a78e148b 301#endif
302
303#endif /* JEMALLOC_H_INLINES */
304/******************************************************************************/
Redis += OpenLDAP MDB