[apple/libc.git] / darwin / SpinlocksLoadStoreEx.c

/*
 * Copyright (c) 2011 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 */

// OSAtomic.h is included by files that include this C file.
/* #include "OSAtomic.h" */

typedef int32_t OSSpinLock;

/*
 * Bear with me, there's method to this madness. clang actually produces more optimal
 * fastpath code if the lock code is split in half like this.
 *
 * Once we're spinning in userspace/kernel like this then we can be more lenient about
 * how much effort is spent doing the spin. Where the lock is uncontended, this split
 * arrangement produces a very good fastpath at the cost of some code duplication.
 *
 * Similarly, the gotos produce less wasteful code during spins then the equivalent code
 * defined with loops.
 */

#import <mach/mach_traps.h>

static inline void _OSSpinLockSlow(volatile OSSpinLock * lock) _OSATOMIC_VARIANT(_OSSpinLockSlow);
static inline void _OSSpinLockSlow(volatile OSSpinLock * lock)
{
	int32_t r;
	uint32_t t;

_spin: ;
#if (defined(_ARM_ARCH_7) && !defined(_OSATOMIC_NO_BARRIERS))
	uint32_t tries = MP_SPIN_TRIES;
	do {
		if (*lock == 0) goto _try_store;
		_osatomic_pause();
	} while (--tries);
#endif

	__asm__ ("mov	r0, %[_a] ;"
			 "mov	r1, %[_b] ;"
			 "mov	r2, %[_c] ;"
			 "bl	_syscall_thread_switch ;"
			 : : [_a] "i" (0), [_b] "i" (1), [_c] "i" (1)
			 : "r0", "r1", "r2", "r9", "r12", "lr" );

_try_store:
	do {
		_osatomic_load_exclusive(lock, r);
		if (slowpath(r)) goto _spin;
		_osatomic_store_exclusive(lock, 1, t);
	} while (slowpath(t));

	_osatomic_barrier();
}

void OSSpinLockLock(volatile OSSpinLock * lock) _OSATOMIC_VARIANT(OSSpinLockLock);
void OSSpinLockLock(volatile OSSpinLock * lock)
{
	_OSATOMIC_ALIAS(spin_lock, OSSpinLockLock);
	_OSATOMIC_ALIAS(_spin_lock, OSSpinLockLock);

	int32_t r;
	uint32_t t;

	do {
		_osatomic_load_exclusive(lock, r);
		if (slowpath(r)) return _OSSpinLockSlow(lock);
		_osatomic_store_exclusive(lock, 1, t);
	} while (slowpath(t));

	_osatomic_barrier();
}

#ifndef _OSATOMIC_WFE

bool OSSpinLockTry(OSSpinLock * lock) _OSATOMIC_VARIANT(OSSpinLockTry);
bool OSSpinLockTry(OSSpinLock * lock)
{
	_OSATOMIC_ALIAS(spin_lock_try, OSSpinLockTry);
	_OSATOMIC_ALIAS(_spin_lock_try, OSSpinLockTry);

	int32_t r;
	uint32_t t;

	do {
		_osatomic_load_exclusive(lock, r);
		if (slowpath(r)) return false;
		_osatomic_store_exclusive(lock, 1, t);
	} while (slowpath(t));

	_osatomic_barrier();
	return (r == 0);
}

void OSSpinLockUnlock(OSSpinLock * lock) _OSATOMIC_VARIANT(OSSpinLockUnlock);
void OSSpinLockUnlock(OSSpinLock * lock)
{
	_OSATOMIC_ALIAS(spin_unlock, OSSpinLockUnlock);
	_OSATOMIC_ALIAS(_spin_unlock, OSSpinLockUnlock);

	_osatomic_barrier();
	*lock = 0;
}

#endif // _OSATOMIC_WFE
Commit	Line	Data
ad3c9f2a A	1	/*
	2	* Copyright (c) 2011 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23
	24	// OSAtomic.h is included by files that include this C file.
	25	/* #include "OSAtomic.h" */
	26
	27	typedef int32_t OSSpinLock;
	28
	29	/*
	30	* Bear with me, there's method to this madness. clang actually produces more optimal
	31	* fastpath code if the lock code is split in half like this.
	32	*
	33	* Once we're spinning in userspace/kernel like this then we can be more lenient about
	34	* how much effort is spent doing the spin. Where the lock is uncontended, this split
	35	* arrangement produces a very good fastpath at the cost of some code duplication.
	36	*
	37	* Similarly, the gotos produce less wasteful code during spins then the equivalent code
	38	* defined with loops.
	39	*/
	40
	41	#import <mach/mach_traps.h>
	42
	43	static inline void _OSSpinLockSlow(volatile OSSpinLock * lock) _OSATOMIC_VARIANT(_OSSpinLockSlow);
	44	static inline void _OSSpinLockSlow(volatile OSSpinLock * lock)
	45	{
	46	int32_t r;
	47	uint32_t t;
	48
	49	_spin: ;
	50	#if (defined(_ARM_ARCH_7) && !defined(_OSATOMIC_NO_BARRIERS))
	51	uint32_t tries = MP_SPIN_TRIES;
	52	do {
	53	if (*lock == 0) goto _try_store;
	54	_osatomic_pause();
	55	} while (--tries);
	56	#endif
	57
	58	__asm__ ("mov r0, %[_a] ;"
	59	"mov r1, %[_b] ;"
	60	"mov r2, %[_c] ;"
	61	"bl _syscall_thread_switch ;"
	62	: : [_a] "i" (0), [_b] "i" (1), [_c] "i" (1)
	63	: "r0", "r1", "r2", "r9", "r12", "lr" );
	64
65	_try_store:
66	do {
67	_osatomic_load_exclusive(lock, r);
68	if (slowpath(r)) goto _spin;
69	_osatomic_store_exclusive(lock, 1, t);
70	} while (slowpath(t));
71
72	_osatomic_barrier();
73	}
74
75	void OSSpinLockLock(volatile OSSpinLock * lock) _OSATOMIC_VARIANT(OSSpinLockLock);
76	void OSSpinLockLock(volatile OSSpinLock * lock)
77	{
78	_OSATOMIC_ALIAS(spin_lock, OSSpinLockLock);
79	_OSATOMIC_ALIAS(_spin_lock, OSSpinLockLock);
80
81	int32_t r;
82	uint32_t t;
83
84	do {
85	_osatomic_load_exclusive(lock, r);
86	if (slowpath(r)) return _OSSpinLockSlow(lock);
87	_osatomic_store_exclusive(lock, 1, t);
88	} while (slowpath(t));
89
90	_osatomic_barrier();
91	}
92
93	#ifndef _OSATOMIC_WFE
94
95	bool OSSpinLockTry(OSSpinLock * lock) _OSATOMIC_VARIANT(OSSpinLockTry);
96	bool OSSpinLockTry(OSSpinLock * lock)
97	{
98	_OSATOMIC_ALIAS(spin_lock_try, OSSpinLockTry);
99	_OSATOMIC_ALIAS(_spin_lock_try, OSSpinLockTry);
100
101	int32_t r;
102	uint32_t t;
103
104	do {
105	_osatomic_load_exclusive(lock, r);
106	if (slowpath(r)) return false;
107	_osatomic_store_exclusive(lock, 1, t);
108	} while (slowpath(t));
109
110	_osatomic_barrier();
111	return (r == 0);
112	}
113
114	void OSSpinLockUnlock(OSSpinLock * lock) _OSATOMIC_VARIANT(OSSpinLockUnlock);
115	void OSSpinLockUnlock(OSSpinLock * lock)
116	{
117	_OSATOMIC_ALIAS(spin_unlock, OSSpinLockUnlock);
118	_OSATOMIC_ALIAS(_spin_unlock, OSSpinLockUnlock);
119
120	_osatomic_barrier();
121	*lock = 0;
122	}
123
124	#endif // _OSATOMIC_WFE