[apple/libc.git] / x86_64 / sys / OSAtomic.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <machine/cpu_capabilities.h>

#define DECLARE(x)   \
.align 2, 0x90      ; \
.globl x            ; \
.globl x ## Barrier ; \
x:                  ; \
x ## Barrier:

.text


// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
DECLARE(_OSAtomicAnd32)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	movl	%edi, %r11d	// save mask
	movl	(%rsi), %eax	// get value
	movq	%rsi, %rdx	// put ptr where compare-and-swap expects it
1:
	movl	%r11d, %esi	// original mask
	movl	%eax, %edi	// old value
	andl	%eax, %esi	// new value
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	jnz	1b
	movl	%esi, %eax
	ret


// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
DECLARE(_OSAtomicOr32)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	movl	%edi, %r11d	// save mask
	movl	(%rsi), %eax	// get value
	movq	%rsi, %rdx	// put ptr where compare-and-swap expects it
1:
	movl	%r11d, %esi	// original mask
	movl	%eax, %edi	// old value
	orl	%eax, %esi	// new value
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	jnz	1b
	movl	%esi, %eax
	ret
	

// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
DECLARE(_OSAtomicXor32)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	movl	%edi, %r11d	// save mask
	movl	(%rsi), %eax	// get value
	movq	%rsi, %rdx	// put ptr where compare-and-swap expects it
1:
	movl	%r11d, %esi	// original mask
	movl	%eax, %edi	// old value
	xorl	%eax, %esi	// new value
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	jnz	1b
	movl	%esi, %eax
	ret


// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
DECLARE(_OSAtomicAnd32Orig)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	movl	%edi, %r11d	// save mask
	movl	(%rsi), %eax	// get value
	movq	%rsi, %rdx	// put ptr where compare-and-swap expects it
1:
	movl	%r11d, %esi	// original mask
	movl	%eax, %edi	// old value
	andl	%eax, %esi	// new value
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	jnz	1b
	movl	%edi, %eax
	ret


// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
DECLARE(_OSAtomicOr32Orig)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	movl	%edi, %r11d	// save mask
	movl	(%rsi), %eax	// get value
	movq	%rsi, %rdx	// put ptr where compare-and-swap expects it
1:
	movl	%r11d, %esi	// original mask
	movl	%eax, %edi	// old value
	orl	%eax, %esi	// new value
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	jnz	1b
	movl	%edi, %eax
	ret
	

// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
DECLARE(_OSAtomicXor32Orig)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	movl	%edi, %r11d	// save mask
	movl	(%rsi), %eax	// get value
	movq	%rsi, %rdx	// put ptr where compare-and-swap expects it
1:
	movl	%r11d, %esi	// original mask
	movl	%eax, %edi	// old value
	xorl	%eax, %esi	// new value
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	jnz	1b
	movl	%edi, %eax
	ret


// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
DECLARE(_OSAtomicCompareAndSwapInt)
DECLARE(_OSAtomicCompareAndSwap32)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	call	*%rcx		// %edi=old value,  %esi=new value. %rdx=ptr
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret


// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
DECLARE(_OSAtomicCompareAndSwapPtr)
DECLARE(_OSAtomicCompareAndSwapLong)
DECLARE(_OSAtomicCompareAndSwap64)
	movq	$(_COMM_PAGE_COMPARE_AND_SWAP64), %rcx
	call	*%rcx		// %rdi=old value,  %rsi=new value. %rdx=ptr
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret


// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
DECLARE(_OSAtomicAdd32)
	movq	$(_COMM_PAGE_ATOMIC_ADD32), %rcx
	movl	%edi, %eax	// save amt to add
	call	*%rcx
	addl	%edi,%eax	// new value
	ret


// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
DECLARE(_OSAtomicAdd64)
	movq	$(_COMM_PAGE_ATOMIC_ADD64), %rcx
	movq	%rdi, %rax	// save amt to add
	call	*%rcx
	addq	%rdi, %rax	// new value
	ret


// bool OSAtomicTestAndSet( uint32_t n, void *value );
DECLARE(_OSAtomicTestAndSet)
	movq	$(_COMM_PAGE_BTS), %rax
	xorl	$7, %edi	// bit position is numbered big endian
	call	*%rax
	setc	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret


// bool OSAtomicTestAndClear( uint32_t n, void *value );
DECLARE(_OSAtomicTestAndClear)
	movq	$(_COMM_PAGE_BTC), %rax
	xorl	$7, %edi	// bit position is numbered big endian
	call	*%rax
	setc	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// bool OSSpinLockTry( OSSpinLock *lock );
	.align	2, 0x90
	.globl	_OSSpinLockTry
	.globl	__spin_lock_try
_OSSpinLockTry:
__spin_lock_try:
	movq	$(_COMM_PAGE_SPINLOCK_TRY), %rax
	jmp	*%rax


// void OSSpinLockLock( OSSpinLock *lock );
	.align	2, 0x90
	.globl	_OSSpinLockLock
	.globl	_spin_lock
	.globl	__spin_lock
_OSSpinLockLock:
_spin_lock:
__spin_lock:
	movq	$(_COMM_PAGE_SPINLOCK_LOCK), %rax
	jmp	*%rax


// void OSSpinLockUnlock( OSSpinLock *lock );
	.align	2, 0x90
	.globl	_OSSpinLockUnlock
	.globl	_spin_unlock
	.globl	__spin_unlock
_OSSpinLockUnlock:
_spin_unlock:
__spin_unlock:
	movl	$0, (%rdi)
	ret


// void OSMemoryBarrier( void );
	.align	2, 0x90
	.globl	_OSMemoryBarrier
_OSMemoryBarrier:
	movq	$(_COMM_PAGE_MEMORY_BARRIER), %rax
	jmp	*%rax


/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to 1st queue element or null
 *		long	 opaque2;  <-- generation count
 *	} OSQueueHead;
 *
 * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicEnqueue
_OSAtomicEnqueue:		// %rdi == list head, %rsi == new, %rdx == offset
	pushq	%rbx
	movq	%rsi,%rbx	// %rbx == new
	movq	%rdx,%rsi	// %rsi == offset
	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
	movq	8(%rdi),%rdx	// %rdx == current generation count
1:
	movq	%rax,(%rbx,%rsi)// link to old list head from new element
	movq	%rdx,%rcx
	incq	%rcx		// increment generation count
	lock			// always lock for now...
	cmpxchg16b (%rdi)	// ...push on new element
	jnz	1b
	popq	%rbx
	ret
	
	
/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicDequeue
_OSAtomicDequeue:		// %rdi == list head, %rsi == offset
	pushq	%rbx
	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
	movq	8(%rdi),%rdx	// %rdx == current generation count
1:
	testq	%rax,%rax	// list empty?
	jz	2f		// yes
	movq	(%rax,%rsi),%rbx // point to 2nd in Q
	movq	%rdx,%rcx
	incq	%rcx		// increment generation count
	lock			// always lock for now...
	cmpxchg16b (%rdi)	// ...pop off 1st element
	jnz	1b
2:
	popq	%rbx
	ret			// ptr to 1st element in Q still in %rax
Commit	Line	Data
8e029c65	1	/*
224c7076	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
8e029c65 A	3	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
	4	*
	5	* @APPLE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. Please obtain a copy of the License at
	11	* http://www.opensource.apple.com/apsl/ and read it before using this
	12	* file.
	13	*
	14	* The Original Code and all software distributed under the License are
	15	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	16	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	17	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	19	* Please see the License for the specific language governing rights and
	20	* limitations under the License.
	21	*
	22	* @APPLE_LICENSE_HEADER_END@
	23	*/
	24
	25	#include <machine/cpu_capabilities.h>
	26
	27	#define DECLARE(x) \
	28	.align 2, 0x90 ; \
	29	.globl x ; \
	30	.globl x ## Barrier ; \
	31	x: ; \
	32	x ## Barrier:
	33
	34	.text
	35
	36
	37	// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
	38	DECLARE(_OSAtomicAnd32)
	39	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	40	movl %edi, %r11d // save mask
	41	movl (%rsi), %eax // get value
	42	movq %rsi, %rdx // put ptr where compare-and-swap expects it
	43	1:
	44	movl %r11d, %esi // original mask
	45	movl %eax, %edi // old value
	46	andl %eax, %esi // new value
	47	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
	48	jnz 1b
	49	movl %esi, %eax
	50	ret
	51
	52
	53	// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
	54	DECLARE(_OSAtomicOr32)
	55	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	56	movl %edi, %r11d // save mask
	57	movl (%rsi), %eax // get value
	58	movq %rsi, %rdx // put ptr where compare-and-swap expects it
	59	1:
	60	movl %r11d, %esi // original mask
	61	movl %eax, %edi // old value
	62	orl %eax, %esi // new value
	63	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
	64	jnz 1b
	65	movl %esi, %eax
	66	ret
67
68
69	// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
70	DECLARE(_OSAtomicXor32)
71	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
72	movl %edi, %r11d // save mask
73	movl (%rsi), %eax // get value
74	movq %rsi, %rdx // put ptr where compare-and-swap expects it
75	1:
76	movl %r11d, %esi // original mask
77	movl %eax, %edi // old value
78	xorl %eax, %esi // new value
79	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
80	jnz 1b
81	movl %esi, %eax
82	ret
83
84
224c7076 A	85	// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
	86	DECLARE(_OSAtomicAnd32Orig)
	87	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	88	movl %edi, %r11d // save mask
	89	movl (%rsi), %eax // get value
	90	movq %rsi, %rdx // put ptr where compare-and-swap expects it
	91	1:
	92	movl %r11d, %esi // original mask
	93	movl %eax, %edi // old value
	94	andl %eax, %esi // new value
	95	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
	96	jnz 1b
	97	movl %edi, %eax
	98	ret
	99
	100
	101	// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
	102	DECLARE(_OSAtomicOr32Orig)
	103	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	104	movl %edi, %r11d // save mask
	105	movl (%rsi), %eax // get value
	106	movq %rsi, %rdx // put ptr where compare-and-swap expects it
	107	1:
	108	movl %r11d, %esi // original mask
	109	movl %eax, %edi // old value
	110	orl %eax, %esi // new value
	111	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
	112	jnz 1b
	113	movl %edi, %eax
	114	ret
	115
	116
	117	// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
	118	DECLARE(_OSAtomicXor32Orig)
	119	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	120	movl %edi, %r11d // save mask
	121	movl (%rsi), %eax // get value
	122	movq %rsi, %rdx // put ptr where compare-and-swap expects it
	123	1:
	124	movl %r11d, %esi // original mask
	125	movl %eax, %edi // old value
	126	xorl %eax, %esi // new value
	127	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
	128	jnz 1b
	129	movl %edi, %eax
	130	ret
	131
	132
8e029c65	133	// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
224c7076	134	DECLARE(_OSAtomicCompareAndSwapInt)
8e029c65 A	135	DECLARE(_OSAtomicCompareAndSwap32)
	136	movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
	137	call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
	138	sete %al
224c7076	139	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65 A	140	ret
	141
	142
	143	// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
224c7076 A	144	DECLARE(_OSAtomicCompareAndSwapPtr)
224c7076 A	145	DECLARE(_OSAtomicCompareAndSwapLong)
8e029c65 A	146	DECLARE(_OSAtomicCompareAndSwap64)
	147	movq $(_COMM_PAGE_COMPARE_AND_SWAP64), %rcx
	148	call *%rcx // %rdi=old value, %rsi=new value. %rdx=ptr
	149	sete %al
224c7076	150	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65 A	151	ret
	152
	153
	154	// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
	155	DECLARE(_OSAtomicAdd32)
	156	movq $(_COMM_PAGE_ATOMIC_ADD32), %rcx
	157	movl %edi, %eax // save amt to add
	158	call *%rcx
	159	addl %edi,%eax // new value
	160	ret
	161
	162
	163	// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
	164	DECLARE(_OSAtomicAdd64)
	165	movq $(_COMM_PAGE_ATOMIC_ADD64), %rcx
	166	movq %rdi, %rax // save amt to add
	167	call *%rcx
	168	addq %rdi, %rax // new value
	169	ret
	170
	171
	172	// bool OSAtomicTestAndSet( uint32_t n, void *value );
	173	DECLARE(_OSAtomicTestAndSet)
	174	movq $(_COMM_PAGE_BTS), %rax
	175	xorl $7, %edi // bit position is numbered big endian
	176	call *%rax
	177	setc %al
224c7076	178	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65 A	179	ret
	180
	181
	182	// bool OSAtomicTestAndClear( uint32_t n, void *value );
	183	DECLARE(_OSAtomicTestAndClear)
	184	movq $(_COMM_PAGE_BTC), %rax
	185	xorl $7, %edi // bit position is numbered big endian
	186	call *%rax
	187	setc %al
224c7076	188	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65 A	189	ret
	190
	191	// bool OSSpinLockTry( OSSpinLock *lock );
	192	.align 2, 0x90
	193	.globl _OSSpinLockTry
224c7076	194	.globl __spin_lock_try
8e029c65	195	_OSSpinLockTry:
224c7076	196	__spin_lock_try:
8e029c65 A	197	movq $(_COMM_PAGE_SPINLOCK_TRY), %rax
	198	jmp *%rax
	199
	200
	201	// void OSSpinLockLock( OSSpinLock *lock );
	202	.align 2, 0x90
	203	.globl _OSSpinLockLock
224c7076 A	204	.globl _spin_lock
224c7076 A	205	.globl __spin_lock
8e029c65	206	_OSSpinLockLock:
224c7076 A	207	_spin_lock:
224c7076 A	208	__spin_lock:
8e029c65 A	209	movq $(_COMM_PAGE_SPINLOCK_LOCK), %rax
	210	jmp *%rax
	211
	212
	213	// void OSSpinLockUnlock( OSSpinLock *lock );
	214	.align 2, 0x90
	215	.globl _OSSpinLockUnlock
224c7076 A	216	.globl _spin_unlock
224c7076 A	217	.globl __spin_unlock
8e029c65	218	_OSSpinLockUnlock:
224c7076 A	219	_spin_unlock:
224c7076 A	220	__spin_unlock:
8e029c65 A	221	movl $0, (%rdi)
	222	ret
	223
	224
	225	// void OSMemoryBarrier( void );
	226	.align 2, 0x90
	227	.globl _OSMemoryBarrier
	228	_OSMemoryBarrier:
224c7076 A	229	movq $(_COMM_PAGE_MEMORY_BARRIER), %rax
	230	jmp *%rax
	231
	232
	233	/*
	234	* typedef volatile struct {
	235	* void *opaque1; <-- ptr to 1st queue element or null
	236	* long opaque2; <-- generation count
	237	* } OSQueueHead;
	238	*
	239	* void OSAtomicEnqueue( OSQueueHead list, void new, size_t offset);
	240	*/
	241	.align 2
	242	.globl _OSAtomicEnqueue
	243	_OSAtomicEnqueue: // %rdi == list head, %rsi == new, %rdx == offset
	244	pushq %rbx
	245	movq %rsi,%rbx // %rbx == new
	246	movq %rdx,%rsi // %rsi == offset
	247	movq (%rdi),%rax // %rax == ptr to 1st element in Q
	248	movq 8(%rdi),%rdx // %rdx == current generation count
	249	1:
	250	movq %rax,(%rbx,%rsi)// link to old list head from new element
	251	movq %rdx,%rcx
	252	incq %rcx // increment generation count
	253	lock // always lock for now...
	254	cmpxchg16b (%rdi) // ...push on new element
	255	jnz 1b
	256	popq %rbx
8e029c65	257	ret
224c7076 A	258
	259
	260	/* void* OSAtomicDequeue( OSQueueHead list, size_t offset); /
	261	.align 2
	262	.globl _OSAtomicDequeue
	263	_OSAtomicDequeue: // %rdi == list head, %rsi == offset
	264	pushq %rbx
	265	movq (%rdi),%rax // %rax == ptr to 1st element in Q
	266	movq 8(%rdi),%rdx // %rdx == current generation count
	267	1:
	268	testq %rax,%rax // list empty?
	269	jz 2f // yes
	270	movq (%rax,%rsi),%rbx // point to 2nd in Q
	271	movq %rdx,%rcx
	272	incq %rcx // increment generation count
	273	lock // always lock for now...
	274	cmpxchg16b (%rdi) // ...pop off 1st element
	275	jnz 1b
	276	2:
	277	popq %rbx
	278	ret // ptr to 1st element in Q still in %rax