[apple/libdispatch.git] / src / shims / atomic.h

/*
 * Copyright (c) 2008-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

/*
 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
 * which are subject to change in future releases of Mac OS X. Any applications
 * relying on these interfaces WILL break.
 */

#ifndef __DISPATCH_SHIMS_ATOMIC__
#define __DISPATCH_SHIMS_ATOMIC__

#if !__has_extension(c_atomic) || !__has_include(<stdatomic.h>)
#error libdispatch requires C11 with <stdatomic.h>
#endif

#include <stdatomic.h>

#define memory_order_ordered    memory_order_seq_cst
#define memory_order_dependency memory_order_acquire

#define os_atomic(type) type _Atomic

#define _os_atomic_c11_atomic(p) \
		((typeof(*(p)) _Atomic *)(p))

// This removes the _Atomic and volatile qualifiers on the type of *p
#define _os_atomic_basetypeof(p) \
		typeof(atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_relaxed))

#define os_atomic_load(p, m) \
		atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_##m)
#define os_atomic_store(p, v, m) \
		atomic_store_explicit(_os_atomic_c11_atomic(p), v, memory_order_##m)
#define os_atomic_xchg(p, v, m) \
		atomic_exchange_explicit(_os_atomic_c11_atomic(p), v, memory_order_##m)
#define os_atomic_cmpxchg(p, e, v, m) \
		({ _os_atomic_basetypeof(p) _r = (e); \
		atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
		&_r, v, memory_order_##m, memory_order_relaxed); })
#define os_atomic_cmpxchgv(p, e, v, g, m) \
		({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \
		atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
		&_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
#define os_atomic_cmpxchgvw(p, e, v, g, m) \
		({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \
		atomic_compare_exchange_weak_explicit(_os_atomic_c11_atomic(p), \
		&_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r;  _b; })

#define _os_atomic_c11_op(p, v, m, o, op) \
		({ _os_atomic_basetypeof(p) _v = (v), _r = \
		atomic_fetch_##o##_explicit(_os_atomic_c11_atomic(p), _v, \
		memory_order_##m); (typeof(_r))(_r op _v); })
#define _os_atomic_c11_op_orig(p, v, m, o, op) \
		atomic_fetch_##o##_explicit(_os_atomic_c11_atomic(p), v, \
		memory_order_##m)
#define os_atomic_add(p, v, m) \
		_os_atomic_c11_op((p), (v), m, add, +)
#define os_atomic_add_orig(p, v, m) \
		_os_atomic_c11_op_orig((p), (v), m, add, +)
#define os_atomic_sub(p, v, m) \
		_os_atomic_c11_op((p), (v), m, sub, -)
#define os_atomic_sub_orig(p, v, m) \
		_os_atomic_c11_op_orig((p), (v), m, sub, -)
#define os_atomic_and(p, v, m) \
		_os_atomic_c11_op((p), (v), m, and, &)
#define os_atomic_and_orig(p, v, m) \
		_os_atomic_c11_op_orig((p), (v), m, and, &)
#define os_atomic_or(p, v, m) \
		_os_atomic_c11_op((p), (v), m, or, |)
#define os_atomic_or_orig(p, v, m) \
		_os_atomic_c11_op_orig((p), (v), m, or, |)
#define os_atomic_xor(p, v, m) \
		_os_atomic_c11_op((p), (v), m, xor, ^)
#define os_atomic_xor_orig(p, v, m) \
		_os_atomic_c11_op_orig((p), (v), m, xor, ^)

#define os_atomic_force_dependency_on(p, e) (p)
#define os_atomic_load_with_dependency_on(p, e) \
		os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed)
#define os_atomic_load_with_dependency_on2o(p, f, e) \
		os_atomic_load_with_dependency_on(&(p)->f, e)

#define os_atomic_thread_fence(m)  atomic_thread_fence(memory_order_##m)

#define os_atomic_load2o(p, f, m) \
		os_atomic_load(&(p)->f, m)
#define os_atomic_store2o(p, f, v, m) \
		os_atomic_store(&(p)->f, (v), m)
#define os_atomic_xchg2o(p, f, v, m) \
		os_atomic_xchg(&(p)->f, (v), m)
#define os_atomic_cmpxchg2o(p, f, e, v, m) \
		os_atomic_cmpxchg(&(p)->f, (e), (v), m)
#define os_atomic_cmpxchgv2o(p, f, e, v, g, m) \
		os_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m)
#define os_atomic_cmpxchgvw2o(p, f, e, v, g, m) \
		os_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m)
#define os_atomic_add2o(p, f, v, m) \
		os_atomic_add(&(p)->f, (v), m)
#define os_atomic_add_orig2o(p, f, v, m) \
		os_atomic_add_orig(&(p)->f, (v), m)
#define os_atomic_sub2o(p, f, v, m) \
		os_atomic_sub(&(p)->f, (v), m)
#define os_atomic_sub_orig2o(p, f, v, m) \
		os_atomic_sub_orig(&(p)->f, (v), m)
#define os_atomic_and2o(p, f, v, m) \
		os_atomic_and(&(p)->f, (v), m)
#define os_atomic_and_orig2o(p, f, v, m) \
		os_atomic_and_orig(&(p)->f, (v), m)
#define os_atomic_or2o(p, f, v, m) \
		os_atomic_or(&(p)->f, (v), m)
#define os_atomic_or_orig2o(p, f, v, m) \
		os_atomic_or_orig(&(p)->f, (v), m)
#define os_atomic_xor2o(p, f, v, m) \
		os_atomic_xor(&(p)->f, (v), m)
#define os_atomic_xor_orig2o(p, f, v, m) \
		os_atomic_xor_orig(&(p)->f, (v), m)

#define os_atomic_inc(p, m) \
		os_atomic_add((p), 1, m)
#define os_atomic_inc_orig(p, m) \
		os_atomic_add_orig((p), 1, m)
#define os_atomic_inc2o(p, f, m) \
		os_atomic_add2o(p, f, 1, m)
#define os_atomic_inc_orig2o(p, f, m) \
		os_atomic_add_orig2o(p, f, 1, m)
#define os_atomic_dec(p, m) \
		os_atomic_sub((p), 1, m)
#define os_atomic_dec_orig(p, m) \
		os_atomic_sub_orig((p), 1, m)
#define os_atomic_dec2o(p, f, m) \
		os_atomic_sub2o(p, f, 1, m)
#define os_atomic_dec_orig2o(p, f, m) \
		os_atomic_sub_orig2o(p, f, 1, m)

#define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
		bool _result = false; \
		typeof(p) _p = (p); \
		ov = os_atomic_load(_p, relaxed); \
		do { \
			__VA_ARGS__; \
			_result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \
		} while (os_unlikely(!_result)); \
		_result; \
	})
#define os_atomic_rmw_loop2o(p, f, ov, nv, m, ...) \
		os_atomic_rmw_loop(&(p)->f, ov, nv, m, __VA_ARGS__)
#define os_atomic_rmw_loop_give_up_with_fence(m, expr) \
		({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); })
#define os_atomic_rmw_loop_give_up(expr) \
		os_atomic_rmw_loop_give_up_with_fence(relaxed, expr)

#endif // __DISPATCH_SHIMS_ATOMIC__
Commit	Line	Data
e85f4437	1	/*
beb15981	2	* Copyright (c) 2008-2016 Apple Inc. All rights reserved.
e85f4437 A	3	*
	4	* @APPLE_APACHE_LICENSE_HEADER_START@
	5	*
	6	* Licensed under the Apache License, Version 2.0 (the "License");
	7	* you may not use this file except in compliance with the License.
	8	* You may obtain a copy of the License at
	9	*
	10	* http://www.apache.org/licenses/LICENSE-2.0
	11	*
	12	* Unless required by applicable law or agreed to in writing, software
	13	* distributed under the License is distributed on an "AS IS" BASIS,
	14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	15	* See the License for the specific language governing permissions and
	16	* limitations under the License.
	17	*
	18	* @APPLE_APACHE_LICENSE_HEADER_END@
	19	*/
	20
	21	/*
	22	* IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
	23	* which are subject to change in future releases of Mac OS X. Any applications
	24	* relying on these interfaces WILL break.
	25	*/
	26
	27	#ifndef __DISPATCH_SHIMS_ATOMIC__
	28	#define __DISPATCH_SHIMS_ATOMIC__
	29
6b746eb4 A	30	#if !__has_extension(c_atomic) \|\| !__has_include(<stdatomic.h>)
6b746eb4 A	31	#error libdispatch requires C11 with <stdatomic.h>
beb15981	32	#endif
517da941	33
beb15981	34	#include <stdatomic.h>
517da941	35
fa22f35b A	36	#define memory_order_ordered memory_order_seq_cst
	37	#define memory_order_dependency memory_order_acquire
	38
6b746eb4	39	#define os_atomic(type) type _Atomic
517da941	40
beb15981	41	#define _os_atomic_c11_atomic(p) \
6b746eb4	42	((typeof((p)) _Atomic )(p))
517da941	43
6b746eb4 A	44	// This removes the _Atomic and volatile qualifiers on the type of *p
	45	#define _os_atomic_basetypeof(p) \
	46	typeof(atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_relaxed))
beb15981 A	47
beb15981 A	48	#define os_atomic_load(p, m) \
6b746eb4	49	atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_##m)
beb15981	50	#define os_atomic_store(p, v, m) \
6b746eb4	51	atomic_store_explicit(_os_atomic_c11_atomic(p), v, memory_order_##m)
beb15981	52	#define os_atomic_xchg(p, v, m) \
6b746eb4	53	atomic_exchange_explicit(_os_atomic_c11_atomic(p), v, memory_order_##m)
beb15981	54	#define os_atomic_cmpxchg(p, e, v, m) \
6b746eb4	55	({ _os_atomic_basetypeof(p) _r = (e); \
beb15981	56	atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
6b746eb4	57	&_r, v, memory_order_##m, memory_order_relaxed); })
beb15981	58	#define os_atomic_cmpxchgv(p, e, v, g, m) \
6b746eb4	59	({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \
beb15981	60	atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
6b746eb4	61	&_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
beb15981	62	#define os_atomic_cmpxchgvw(p, e, v, g, m) \
6b746eb4	63	({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \
beb15981	64	atomic_compare_exchange_weak_explicit(_os_atomic_c11_atomic(p), \
6b746eb4	65	&_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
beb15981 A	66
	67	#define _os_atomic_c11_op(p, v, m, o, op) \
	68	({ _os_atomic_basetypeof(p) _v = (v), _r = \
	69	atomic_fetch_##o##_explicit(_os_atomic_c11_atomic(p), _v, \
9d9a7e41	70	memory_order_##m); (typeof(_r))(_r op _v); })
beb15981	71	#define _os_atomic_c11_op_orig(p, v, m, o, op) \
6b746eb4 A	72	atomic_fetch_##o##_explicit(_os_atomic_c11_atomic(p), v, \
6b746eb4 A	73	memory_order_##m)
beb15981 A	74	#define os_atomic_add(p, v, m) \
	75	_os_atomic_c11_op((p), (v), m, add, +)
	76	#define os_atomic_add_orig(p, v, m) \
	77	_os_atomic_c11_op_orig((p), (v), m, add, +)
	78	#define os_atomic_sub(p, v, m) \
	79	_os_atomic_c11_op((p), (v), m, sub, -)
	80	#define os_atomic_sub_orig(p, v, m) \
	81	_os_atomic_c11_op_orig((p), (v), m, sub, -)
	82	#define os_atomic_and(p, v, m) \
	83	_os_atomic_c11_op((p), (v), m, and, &)
	84	#define os_atomic_and_orig(p, v, m) \
	85	_os_atomic_c11_op_orig((p), (v), m, and, &)
	86	#define os_atomic_or(p, v, m) \
	87	_os_atomic_c11_op((p), (v), m, or, \|)
	88	#define os_atomic_or_orig(p, v, m) \
	89	_os_atomic_c11_op_orig((p), (v), m, or, \|)
	90	#define os_atomic_xor(p, v, m) \
	91	_os_atomic_c11_op((p), (v), m, xor, ^)
	92	#define os_atomic_xor_orig(p, v, m) \
	93	_os_atomic_c11_op_orig((p), (v), m, xor, ^)
	94
fa22f35b A	95	#define os_atomic_force_dependency_on(p, e) (p)
	96	#define os_atomic_load_with_dependency_on(p, e) \
	97	os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed)
	98	#define os_atomic_load_with_dependency_on2o(p, f, e) \
	99	os_atomic_load_with_dependency_on(&(p)->f, e)
	100
6b746eb4	101	#define os_atomic_thread_fence(m) atomic_thread_fence(memory_order_##m)
beb15981 A	102
	103	#define os_atomic_load2o(p, f, m) \
	104	os_atomic_load(&(p)->f, m)
	105	#define os_atomic_store2o(p, f, v, m) \
	106	os_atomic_store(&(p)->f, (v), m)
	107	#define os_atomic_xchg2o(p, f, v, m) \
	108	os_atomic_xchg(&(p)->f, (v), m)
	109	#define os_atomic_cmpxchg2o(p, f, e, v, m) \
	110	os_atomic_cmpxchg(&(p)->f, (e), (v), m)
	111	#define os_atomic_cmpxchgv2o(p, f, e, v, g, m) \
	112	os_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m)
	113	#define os_atomic_cmpxchgvw2o(p, f, e, v, g, m) \
	114	os_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m)
	115	#define os_atomic_add2o(p, f, v, m) \
	116	os_atomic_add(&(p)->f, (v), m)
	117	#define os_atomic_add_orig2o(p, f, v, m) \
	118	os_atomic_add_orig(&(p)->f, (v), m)
	119	#define os_atomic_sub2o(p, f, v, m) \
	120	os_atomic_sub(&(p)->f, (v), m)
	121	#define os_atomic_sub_orig2o(p, f, v, m) \
	122	os_atomic_sub_orig(&(p)->f, (v), m)
	123	#define os_atomic_and2o(p, f, v, m) \
	124	os_atomic_and(&(p)->f, (v), m)
	125	#define os_atomic_and_orig2o(p, f, v, m) \
	126	os_atomic_and_orig(&(p)->f, (v), m)
	127	#define os_atomic_or2o(p, f, v, m) \
	128	os_atomic_or(&(p)->f, (v), m)
	129	#define os_atomic_or_orig2o(p, f, v, m) \
	130	os_atomic_or_orig(&(p)->f, (v), m)
	131	#define os_atomic_xor2o(p, f, v, m) \
	132	os_atomic_xor(&(p)->f, (v), m)
	133	#define os_atomic_xor_orig2o(p, f, v, m) \
	134	os_atomic_xor_orig(&(p)->f, (v), m)
	135
	136	#define os_atomic_inc(p, m) \
	137	os_atomic_add((p), 1, m)
	138	#define os_atomic_inc_orig(p, m) \
	139	os_atomic_add_orig((p), 1, m)
	140	#define os_atomic_inc2o(p, f, m) \
	141	os_atomic_add2o(p, f, 1, m)
	142	#define os_atomic_inc_orig2o(p, f, m) \
	143	os_atomic_add_orig2o(p, f, 1, m)
	144	#define os_atomic_dec(p, m) \
	145	os_atomic_sub((p), 1, m)
	146	#define os_atomic_dec_orig(p, m) \
	147	os_atomic_sub_orig((p), 1, m)
	148	#define os_atomic_dec2o(p, f, m) \
	149	os_atomic_sub2o(p, f, 1, m)
	150	#define os_atomic_dec_orig2o(p, f, m) \
	151	os_atomic_sub_orig2o(p, f, 1, m)
517da941	152
6b746eb4 A	153	#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
	154	bool _result = false; \
	155	typeof(p) _p = (p); \
	156	ov = os_atomic_load(_p, relaxed); \
	157	do { \
	158	__VA_ARGS__; \
	159	_result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \
	160	} while (os_unlikely(!_result)); \
	161	_result; \
	162	})
	163	#define os_atomic_rmw_loop2o(p, f, ov, nv, m, ...) \
	164	os_atomic_rmw_loop(&(p)->f, ov, nv, m, __VA_ARGS__)
	165	#define os_atomic_rmw_loop_give_up_with_fence(m, expr) \
	166	({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); })
	167	#define os_atomic_rmw_loop_give_up(expr) \
	168	os_atomic_rmw_loop_give_up_with_fence(relaxed, expr)
e85f4437 A	169
e85f4437 A	170	#endif // __DISPATCH_SHIMS_ATOMIC__