[apple/libc.git] / darwin / OSAtomicLoadStoreEx.c

/*
 * Copyright (c) 2011 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 */

// OSAtomic.h is included by files that include this C file.
/* #include "OSAtomic.h" */

int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAdd32);
int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicAdd32);
	int32_t r;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		r += v;
		_osatomic_store_exclusive(p, r, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p) _OSATOMIC_VARIANT_B(OSAtomicAdd64);
int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicAdd64);

	int64_t r;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive64(p, r);
		r += v;
		_osatomic_store_exclusive64(p, r, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int32_t OSAtomicOr32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicOr32);
int32_t OSAtomicOr32(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicOr32);
	int32_t r;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		r |= v;
		_osatomic_store_exclusive(p, r, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int32_t OSAtomicOr32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicOr32Orig);
int32_t OSAtomicOr32Orig(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicOr32Orig);
	int32_t r, n;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		n = r | v;
		_osatomic_store_exclusive(p, n, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int32_t OSAtomicAnd32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAnd32);
int32_t OSAtomicAnd32(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicAnd32);
	int32_t r;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		r &= v;
		_osatomic_store_exclusive(p, r, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int32_t OSAtomicAnd32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAnd32Orig);
int32_t OSAtomicAnd32Orig(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicAnd32Orig);
	int32_t r, n;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		n = r & v;
		_osatomic_store_exclusive(p, n, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int32_t OSAtomicXor32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicXor32);
int32_t OSAtomicXor32(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicXor32);
	int32_t r;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		r ^= v;
		_osatomic_store_exclusive(p, r, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

int32_t OSAtomicXor32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicXor32Orig);
int32_t OSAtomicXor32Orig(int32_t v, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicXor32Orig);
	int32_t r, n;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(p, r);
		n = r ^ v;
		_osatomic_store_exclusive(p, n, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicCompareAndSwap32);
bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p)
{
	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapInt, OSAtomicCompareAndSwap32);

	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwap32);
	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapInt);
#ifndef __LP64__
	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapLong, OSAtomicCompareAndSwap32);
	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapPtr, OSAtomicCompareAndSwap32);
	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapLong);
	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapPtr);
#endif

	int32_t r;
	uint32_t t;

	do {
		_osatomic_load_exclusive(p, r);
		if (r != o) return false;
		_osatomic_store_barrier();
		_osatomic_store_exclusive(p, n, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return true;
}

bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p) _OSATOMIC_VARIANT_B(OSAtomicCompareAndSwap64);
bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwap64);
#ifdef __LP64__
	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapLong, OSAtomicCompareAndSwap64);
	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapPtr, OSAtomicCompareAndSwap64);
	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapLong);
	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapPtr);
#endif

	int64_t r;
	uint32_t t;

	do {
		_osatomic_load_exclusive64(p, r);
		if (r != o) return false;
		_osatomic_store_barrier();
		_osatomic_store_exclusive64(p, n, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return true;
}

#if defined(_OSATOMIC_EXTRAS)

void OSMemoryBarrier(void) _OSATOMIC_VARIANT(OSMemoryBarrier);
void OSMemoryBarrier(void)
{
	_osatomic_barrier();
}

typedef volatile struct {
	void	*item;
	long	 unused;
} OSQueueHead;

void OSAtomicEnqueue(OSQueueHead *l, void *n, size_t o) _OSATOMIC_VARIANT(OSAtomicEnqueue);
void OSAtomicEnqueue(OSQueueHead *l, void *n, size_t o)
{
	void ** r = (void **)((char *)n + o);
	void * volatile * i = (void **)&(l->item);
	void * q;
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(i, q);
		*r = q;
		_osatomic_store_exclusive(i, n, t);
	} while (slowpath(t));
	_osatomic_barrier();
}

void* OSAtomicDequeue(OSQueueHead *l, size_t o) _OSATOMIC_VARIANT(OSAtomicDequeue);
void* OSAtomicDequeue(OSQueueHead *l, size_t o)
{
	void * h;
	void ** r;
	void * volatile * i = (void **)&(l->item);
	uint32_t t;

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(i, r);
		if (!r) return NULL;
		h = *(void **)((char *)r + o);
		_osatomic_store_exclusive(i, h, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return r;
}

#endif

bool OSAtomicTestAndSet(uint32_t n, volatile void * p) _OSATOMIC_VARIANT_B(OSAtomicTestAndSet);
bool OSAtomicTestAndSet(uint32_t n, volatile void * p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicTestAndSet);
	uint32_t * ptr = (uint32_t *)((char *)p + (4 * (n / 32)));
	uint32_t i, r, t;

	n = (0x80 >> (n & 7)) << (n & ~7 & 31);

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(ptr, r);
		i = r | n;
		if (i == r) break;
		_osatomic_store_exclusive(ptr, i, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return ((r & n) != 0);
}

bool OSAtomicTestAndClear(uint32_t n, volatile void * p) _OSATOMIC_VARIANT_B(OSAtomicTestAndClear);
bool OSAtomicTestAndClear(uint32_t n, volatile void * p)
{
	_OSATOMIC_ALIAS_NB(OSAtomicTestAndClear);
	uint32_t * ptr = (uint32_t *)((char *)p + (4 * (n / 32)));
	uint32_t i, r, t;

	n = (0x80 >> (n & 7)) << (n & ~7 & 31);

	_osatomic_store_barrier();
	do {
		_osatomic_load_exclusive(ptr, r);
		i = r & ~n;
		if (i == r) break;
		_osatomic_store_exclusive(ptr, i, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return ((r & n) != 0);
}
Commit	Line	Data
ad3c9f2a A	1	/*
	2	* Copyright (c) 2011 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23
	24	// OSAtomic.h is included by files that include this C file.
	25	/* #include "OSAtomic.h" */
	26
	27	int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAdd32);
	28	int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p)
	29	{
	30	_OSATOMIC_ALIAS_NB(OSAtomicAdd32);
	31	int32_t r;
	32	uint32_t t;
	33
	34	_osatomic_store_barrier();
	35	do {
	36	_osatomic_load_exclusive(p, r);
	37	r += v;
	38	_osatomic_store_exclusive(p, r, t);
	39	} while (slowpath(t));
	40
	41	_osatomic_barrier();
	42	return r;
	43	}
	44
	45	int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p) _OSATOMIC_VARIANT_B(OSAtomicAdd64);
	46	int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p)
	47	{
	48	_OSATOMIC_ALIAS_NB(OSAtomicAdd64);
	49
	50	int64_t r;
	51	uint32_t t;
	52
	53	_osatomic_store_barrier();
	54	do {
	55	_osatomic_load_exclusive64(p, r);
	56	r += v;
	57	_osatomic_store_exclusive64(p, r, t);
	58	} while (slowpath(t));
	59
	60	_osatomic_barrier();
	61	return r;
	62	}
	63
	64	int32_t OSAtomicOr32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicOr32);
65	int32_t OSAtomicOr32(int32_t v, volatile int32_t *p)
66	{
67	_OSATOMIC_ALIAS_NB(OSAtomicOr32);
68	int32_t r;
69	uint32_t t;
70
71	_osatomic_store_barrier();
72	do {
73	_osatomic_load_exclusive(p, r);
74	r \|= v;
75	_osatomic_store_exclusive(p, r, t);
76	} while (slowpath(t));
77
78	_osatomic_barrier();
79	return r;
80	}
81
82	int32_t OSAtomicOr32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicOr32Orig);
83	int32_t OSAtomicOr32Orig(int32_t v, volatile int32_t *p)
84	{
85	_OSATOMIC_ALIAS_NB(OSAtomicOr32Orig);
86	int32_t r, n;
87	uint32_t t;
88
89	_osatomic_store_barrier();
90	do {
91	_osatomic_load_exclusive(p, r);
92	n = r \| v;
93	_osatomic_store_exclusive(p, n, t);
94	} while (slowpath(t));
95
96	_osatomic_barrier();
97	return r;
98	}
99
100	int32_t OSAtomicAnd32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAnd32);
101	int32_t OSAtomicAnd32(int32_t v, volatile int32_t *p)
102	{
103	_OSATOMIC_ALIAS_NB(OSAtomicAnd32);
104	int32_t r;
105	uint32_t t;
106
107	_osatomic_store_barrier();
108	do {
109	_osatomic_load_exclusive(p, r);
110	r &= v;
111	_osatomic_store_exclusive(p, r, t);
112	} while (slowpath(t));
113
114	_osatomic_barrier();
115	return r;
116	}
117
118	int32_t OSAtomicAnd32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAnd32Orig);
119	int32_t OSAtomicAnd32Orig(int32_t v, volatile int32_t *p)
120	{
121	_OSATOMIC_ALIAS_NB(OSAtomicAnd32Orig);
122	int32_t r, n;
123	uint32_t t;
124
125	_osatomic_store_barrier();
126	do {
127	_osatomic_load_exclusive(p, r);
128	n = r & v;
129	_osatomic_store_exclusive(p, n, t);
130	} while (slowpath(t));
131
132	_osatomic_barrier();
133	return r;
134	}
135
136	int32_t OSAtomicXor32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicXor32);
137	int32_t OSAtomicXor32(int32_t v, volatile int32_t *p)
138	{
139	_OSATOMIC_ALIAS_NB(OSAtomicXor32);
140	int32_t r;
141	uint32_t t;
142
143	_osatomic_store_barrier();
144	do {
145	_osatomic_load_exclusive(p, r);
146	r ^= v;
147	_osatomic_store_exclusive(p, r, t);
148	} while (slowpath(t));
149
150	_osatomic_barrier();
151	return r;
152	}
153
154	int32_t OSAtomicXor32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicXor32Orig);
155	int32_t OSAtomicXor32Orig(int32_t v, volatile int32_t *p)
156	{
157	_OSATOMIC_ALIAS_NB(OSAtomicXor32Orig);
158	int32_t r, n;
159	uint32_t t;
160
161	_osatomic_store_barrier();
162	do {
163	_osatomic_load_exclusive(p, r);
164	n = r ^ v;
165	_osatomic_store_exclusive(p, n, t);
166	} while (slowpath(t));
167
168	_osatomic_barrier();
169	return r;
170	}
171
172	bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicCompareAndSwap32);
173	bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p)
174	{
175	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapInt, OSAtomicCompareAndSwap32);
176
177	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwap32);
178	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapInt);
179	#ifndef __LP64__
180	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapLong, OSAtomicCompareAndSwap32);
181	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapPtr, OSAtomicCompareAndSwap32);
182	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapLong);
183	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapPtr);
184	#endif
185
186	int32_t r;
187	uint32_t t;
188
189	do {
190	_osatomic_load_exclusive(p, r);
191	if (r != o) return false;
192	_osatomic_store_barrier();
193	_osatomic_store_exclusive(p, n, t);
194	} while (slowpath(t));
195
196	_osatomic_barrier();
197	return true;
198	}
199
200	bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p) _OSATOMIC_VARIANT_B(OSAtomicCompareAndSwap64);
201	bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p)
202	{
203	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwap64);
204	#ifdef __LP64__
205	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapLong, OSAtomicCompareAndSwap64);
206	_OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapPtr, OSAtomicCompareAndSwap64);
207	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapLong);
208	_OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapPtr);
209	#endif
210
211	int64_t r;
212	uint32_t t;
213
214	do {
215	_osatomic_load_exclusive64(p, r);
216	if (r != o) return false;
217	_osatomic_store_barrier();
218	_osatomic_store_exclusive64(p, n, t);
219	} while (slowpath(t));
220
221	_osatomic_barrier();
222	return true;
223	}
224
225	#if defined(_OSATOMIC_EXTRAS)
226
227	void OSMemoryBarrier(void) _OSATOMIC_VARIANT(OSMemoryBarrier);
228	void OSMemoryBarrier(void)
229	{
230	_osatomic_barrier();
231	}
232
233	typedef volatile struct {
234	void *item;
235	long unused;
236	} OSQueueHead;
237
238	void OSAtomicEnqueue(OSQueueHead l, void n, size_t o) _OSATOMIC_VARIANT(OSAtomicEnqueue);
239	void OSAtomicEnqueue(OSQueueHead l, void n, size_t o)
240	{
241	void r = (void )((char *)n + o);
242	void * volatile * i = (void **)&(l->item);
243	void * q;
244	uint32_t t;
245
246	_osatomic_store_barrier();
247	do {
248	_osatomic_load_exclusive(i, q);
249	*r = q;
250	_osatomic_store_exclusive(i, n, t);
251	} while (slowpath(t));
252	_osatomic_barrier();
253	}
254
255	void* OSAtomicDequeue(OSQueueHead *l, size_t o) _OSATOMIC_VARIANT(OSAtomicDequeue);
256	void* OSAtomicDequeue(OSQueueHead *l, size_t o)
257	{
258	void * h;
259	void ** r;
260	void * volatile * i = (void **)&(l->item);
261	uint32_t t;
262
263	_osatomic_store_barrier();
264	do {
265	_osatomic_load_exclusive(i, r);
266	if (!r) return NULL;
267	h = (void )((char )r + o);
268	_osatomic_store_exclusive(i, h, t);
269	} while (slowpath(t));
270
271	_osatomic_barrier();
272	return r;
273	}
274
275	#endif
276
277	bool OSAtomicTestAndSet(uint32_t n, volatile void * p) _OSATOMIC_VARIANT_B(OSAtomicTestAndSet);
278	bool OSAtomicTestAndSet(uint32_t n, volatile void * p)
279	{
280	_OSATOMIC_ALIAS_NB(OSAtomicTestAndSet);
281	uint32_t * ptr = (uint32_t )((char )p + (4 * (n / 32)));
282	uint32_t i, r, t;
283
284	n = (0x80 >> (n & 7)) << (n & ~7 & 31);
285
286	_osatomic_store_barrier();
287	do {
288	_osatomic_load_exclusive(ptr, r);
289	i = r \| n;
290	if (i == r) break;
291	_osatomic_store_exclusive(ptr, i, t);
292	} while (slowpath(t));
293
294	_osatomic_barrier();
295	return ((r & n) != 0);
296	}
297
298	bool OSAtomicTestAndClear(uint32_t n, volatile void * p) _OSATOMIC_VARIANT_B(OSAtomicTestAndClear);
299	bool OSAtomicTestAndClear(uint32_t n, volatile void * p)
300	{
301	_OSATOMIC_ALIAS_NB(OSAtomicTestAndClear);
302	uint32_t * ptr = (uint32_t )((char )p + (4 * (n / 32)));
303	uint32_t i, r, t;
304
305	n = (0x80 >> (n & 7)) << (n & ~7 & 31);
306
307	_osatomic_store_barrier();
308	do {
309	_osatomic_load_exclusive(ptr, r);
310	i = r & ~n;
311	if (i == r) break;
312	_osatomic_store_exclusive(ptr, i, t);
313	} while (slowpath(t));
314
315	_osatomic_barrier();
316	return ((r & n) != 0);
317	}
318