[apple/libc.git] / x86_64 / sys / OSAtomic.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <machine/cpu_capabilities.h>
#include <platfunc.h>

#define DECLARE(x)   \
	.align 2, 0x90      ; \
	.globl x            ; \
	.globl x ## Barrier ; \
x:                  ; \
x ## Barrier:

.text

#define ATOMIC_UP	0
#define ATOMIC_MP	1
#define ATOMIC_RET_ORIG	0
#define ATOMIC_RET_NEW	1

// compare and exchange 32-bit
// xchg32 <new> <dst> <mp>
.macro xchg32
	.if $2 == ATOMIC_MP
	lock
	.endif
	cmpxchgl	$0, ($1)
.endm

// xchg64 <new> <dst> <mp>
.macro xchg64
	.if $2 == ATOMIC_MP
	lock
	.endif
	cmpxchg		$0, ($1)
.endm

#define	ATOMIC_ARITHMETIC(instr, orig, mp)	\
	movl	(%rsi), %eax	/* get 2nd arg -> eax */ ;\
1:	movl	%eax, %edx	/* copy value to new reg */ ;\
	instr	%edi, %edx	/* apply instr to %edx with arg2 */ ;\
	xchg32	%edx, %rsi, mp	/* do the compare swap (see macro above) */ ;\
	jnz	1b		/* jump if failed */ ;\
	.if orig == 1		/* to return the new value, overwrite eax */ ;\
	movl	%edx, %eax	/* return the new value */ ;\
	.endif

// Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
// Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
#define	ATOMIC_BIT_OP(instr, mp)	\
	xorl	$7, %edi	/* bit position is numbered big endian so convert to little endian */ ;\
	shlq	$3, %rsi 	;\
	addq	%rdi, %rsi	/* generate bit address */ ;\
	movq	%rsi, %rdi	;\
	andq	$31, %rdi	/* keep bit offset in range 0..31 */ ;\
	xorq	%rdi, %rsi	/* 4-byte align address */ ;\
	shrq	$3, %rsi	/* get 4-byte aligned address */ ;\
	.if mp == ATOMIC_MP 	/* don't plant the lock in UP code */ ;\
	lock			/* lock the bit test */ ;\
	.endif			;\
	instr	%edi, (%rsi)	/* do the bit test, supplied into the macro */ ;\
	setc	%al		;\
	movzbl	%al,%eax	/* widen in case caller assumes we return an int */

// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret
	
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret
	
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret
	
// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
	movl	%edi, %eax
	xchg32	%esi, %rdx, ATOMIC_UP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
	movl	%edi, %eax
	xchg32	%esi, %rdx, ATOMIC_MP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
	mov	%rdi, %rax
	xchg64	%rsi, %rdx, ATOMIC_UP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret
	
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
	mov	%rdi, %rax
	xchg64	%rsi, %rdx, ATOMIC_MP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
	movl	%edi, %eax	// save amt to add
	xaddl	%edi, (%rsi)	// swap and add value, returns old value in %edi
	addl	%edi, %eax	// add old value to amt as return value
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
	movl	%edi, %eax	// save amt to add
	lock			// lock prefix breaks tabs ;)
	xaddl	%edi, (%rsi)	// swap and add value, returns old value in %edi
	addl	%edi, %eax	// add old value to amt as return value
	ret

// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
	movq	%rdi, %rax	// save amt to add
	xadd	%rdi, (%rsi)	// swap and add value, returns old value in %rsi
	addq	%rdi, %rax	// add old value to amt as return value
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
	movq	%rdi, %rax	// save amt to add
	lock
	xadd	%rdi, (%rsi)	// swap and add value, returns old value in %rsi
	addq	%rdi, %rax	// add old value to amt as return value
	ret

// bool OSAtomicTestAndSet( uint32_t n, void *value );
PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	ret

// bool OSAtomicTestAndClear( uint32_t n, void *value );
PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
	ret

// void OSMemoryBarrier( void );
	.align	2, 0x90
	.globl	_OSMemoryBarrier
_OSMemoryBarrier:
	mfence
	ret

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to 1st queue element or null
 *		long	 opaque2;  <-- generation count
 *	} OSQueueHead;
 *
 * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicEnqueue
_OSAtomicEnqueue:		// %rdi == list head, %rsi == new, %rdx == offset
	pushq	%rbx
	movq	%rsi,%rbx	// %rbx == new
	movq	%rdx,%rsi	// %rsi == offset
	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
	movq	8(%rdi),%rdx	// %rdx == current generation count
1:
	movq	%rax,(%rbx,%rsi)// link to old list head from new element
	movq	%rdx,%rcx
	incq	%rcx		// increment generation count
	lock			// always lock for now...
	cmpxchg16b (%rdi)	// ...push on new element
	jnz	1b
	popq	%rbx
	ret


	/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicDequeue
_OSAtomicDequeue:		// %rdi == list head, %rsi == offset
	pushq	%rbx
	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
	movq	8(%rdi),%rdx	// %rdx == current generation count
1:
	testq	%rax,%rax	// list empty?
	jz	2f		// yes
	movq	(%rax,%rsi),%rbx // point to 2nd in Q
	movq	%rdx,%rcx
	incq	%rcx		// increment generation count
	lock			// always lock for now...
	cmpxchg16b (%rdi)	// ...pop off 1st element
	jnz	1b
2:
	popq	%rbx
	ret			// ptr to 1st element in Q still in %rax

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to first queue element or null
 *		void	*opaque2;  <-- ptr to last queue element or null
 *		int	 opaque3;  <-- spinlock
 *	} OSFifoQueueHead;
 *
 * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicFifoEnqueue
_OSAtomicFifoEnqueue:
	pushq	%rbx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movq	$(_COMM_PAGE_PFZ_ENQUEUE), %rcx
	call	*%rcx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt	// call into the kernel to pfz_exit
1:	
	popq	%rbx
	ret
	
	
/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicFifoDequeue
_OSAtomicFifoDequeue:
	pushq	%rbx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movq	%rsi,%rdx	// move offset to %rdx to be like the Enqueue case
	movq	$(_COMM_PAGE_PFZ_DEQUEUE), %rcx
	call	*%rcx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt	// call into the kernel to pfz_exit
1:	
	popq	%rbx
	ret			// ptr to 1st element in Q in %rax

// Local Variables:
// tab-width: 8
// End:
Commit	Line	Data
8e029c65	1	/*
224c7076	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
8e029c65 A	3	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
	4	*
	5	* @APPLE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. Please obtain a copy of the License at
	11	* http://www.opensource.apple.com/apsl/ and read it before using this
	12	* file.
	13	*
	14	* The Original Code and all software distributed under the License are
	15	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	16	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	17	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	19	* Please see the License for the specific language governing rights and
	20	* limitations under the License.
	21	*
	22	* @APPLE_LICENSE_HEADER_END@
	23	*/
	24
	25	#include <machine/cpu_capabilities.h>
1f2f436a	26	#include <platfunc.h>
8e029c65 A	27
8e029c65 A	28	#define DECLARE(x) \
1f2f436a A	29	.align 2, 0x90 ; \
	30	.globl x ; \
	31	.globl x ## Barrier ; \
8e029c65 A	32	x: ; \
	33	x ## Barrier:
	34
	35	.text
	36
1f2f436a A	37	#define ATOMIC_UP 0
	38	#define ATOMIC_MP 1
	39	#define ATOMIC_RET_ORIG 0
	40	#define ATOMIC_RET_NEW 1
	41
	42	// compare and exchange 32-bit
	43	// xchg32 <new> <dst> <mp>
	44	.macro xchg32
	45	.if $2 == ATOMIC_MP
	46	lock
	47	.endif
	48	cmpxchgl $0, ($1)
	49	.endm
	50
	51	// xchg64 <new> <dst> <mp>
	52	.macro xchg64
	53	.if $2 == ATOMIC_MP
	54	lock
	55	.endif
	56	cmpxchg $0, ($1)
	57	.endm
	58
	59	#define ATOMIC_ARITHMETIC(instr, orig, mp) \
	60	movl (%rsi), %eax /* get 2nd arg -> eax */ ;\
	61	1: movl %eax, %edx /* copy value to new reg */ ;\
	62	instr %edi, %edx /* apply instr to %edx with arg2 */ ;\
	63	xchg32 %edx, %rsi, mp /* do the compare swap (see macro above) */ ;\
	64	jnz 1b /* jump if failed */ ;\
	65	.if orig == 1 /* to return the new value, overwrite eax */ ;\
	66	movl %edx, %eax /* return the new value */ ;\
	67	.endif
	68
	69	// Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
	70	// Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
	71	#define ATOMIC_BIT_OP(instr, mp) \
	72	xorl $7, %edi /* bit position is numbered big endian so convert to little endian */ ;\
	73	shlq $3, %rsi ;\
	74	addq %rdi, %rsi /* generate bit address */ ;\
	75	movq %rsi, %rdi ;\
	76	andq $31, %rdi /* keep bit offset in range 0..31 */ ;\
	77	xorq %rdi, %rsi /* 4-byte align address */ ;\
	78	shrq $3, %rsi /* get 4-byte aligned address */ ;\
	79	.if mp == ATOMIC_MP /* don't plant the lock in UP code */ ;\
	80	lock /* lock the bit test */ ;\
	81	.endif ;\
	82	instr %edi, (%rsi) /* do the bit test, supplied into the macro */ ;\
	83	setc %al ;\
	84	movzbl %al,%eax /* widen in case caller assumes we return an int */
8e029c65 A	85
8e029c65 A	86	// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
1f2f436a A	87	PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 64, 2)
	88	PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
	89	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
	90	ret
	91
	92	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
	93	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
	94	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
8e029c65	95	ret
8e029c65 A	96
8e029c65 A	97	// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
1f2f436a A	98	PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 64, 2)
	99	PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
	100	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
8e029c65 A	101	ret
8e029c65 A	102
1f2f436a A	103	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
	104	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
	105	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
	106	ret
8e029c65 A	107
8e029c65 A	108	// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
1f2f436a A	109	PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 64, 2)
	110	PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
	111	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
8e029c65 A	112	ret
8e029c65 A	113
1f2f436a A	114	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
	115	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
	116	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
	117	ret
8e029c65	118
224c7076	119	// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
1f2f436a A	120	PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 64, 2)
	121	PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
	122	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	123	ret
224c7076 A	124
1f2f436a A	125	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
	126	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
	127	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
224c7076 A	128	ret
224c7076 A	129
1f2f436a A	130	// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
	131	PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 64, 2)
	132	PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
	133	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
	134	ret
	135
	136	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
	137	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
	138	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	139	ret
224c7076 A	140
224c7076 A	141	// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
1f2f436a A	142	PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 64, 2)
	143	PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
	144	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	145	ret
224c7076 A	146
1f2f436a A	147	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
	148	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
	149	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	150	ret
224c7076	151
8e029c65	152	// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
1f2f436a A	153	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 64, 2)
	154	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
	155	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
	156	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
	157	movl %edi, %eax
	158	xchg32 %esi, %rdx, ATOMIC_UP
8e029c65	159	sete %al
224c7076	160	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65 A	161	ret
8e029c65 A	162
1f2f436a A	163	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
	164	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
	165	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
	166	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
	167	movl %edi, %eax
	168	xchg32 %esi, %rdx, ATOMIC_MP
	169	sete %al
	170	movzbl %al,%eax // widen in case caller assumes we return an int
	171	ret
8e029c65 A	172
8e029c65 A	173	// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
1f2f436a A	174	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 64, 2)
	175	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
	176	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
	177	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
	178	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
	179	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
	180	mov %rdi, %rax
	181	xchg64 %rsi, %rdx, ATOMIC_UP
	182	sete %al
	183	movzbl %al,%eax // widen in case caller assumes we return an int
	184	ret
	185
	186	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
	187	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
	188	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
	189	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
	190	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
	191	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
	192	mov %rdi, %rax
	193	xchg64 %rsi, %rdx, ATOMIC_MP
8e029c65	194	sete %al
224c7076	195	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65	196	ret
8e029c65 A	197
8e029c65 A	198	// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
1f2f436a A	199	PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 64, 2)
1f2f436a A	200	PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
8e029c65	201	movl %edi, %eax // save amt to add
1f2f436a A	202	xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
1f2f436a A	203	addl %edi, %eax // add old value to amt as return value
8e029c65 A	204	ret
8e029c65 A	205
1f2f436a A	206	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
	207	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
	208	movl %edi, %eax // save amt to add
	209	lock // lock prefix breaks tabs ;)
	210	xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
	211	addl %edi, %eax // add old value to amt as return value
	212	ret
8e029c65 A	213
8e029c65 A	214	// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
1f2f436a A	215	PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 64, 2)
1f2f436a A	216	PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
8e029c65	217	movq %rdi, %rax // save amt to add
1f2f436a A	218	xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
1f2f436a A	219	addq %rdi, %rax // add old value to amt as return value
8e029c65 A	220	ret
8e029c65 A	221
1f2f436a A	222	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
	223	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
	224	movq %rdi, %rax // save amt to add
	225	lock
	226	xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
	227	addq %rdi, %rax // add old value to amt as return value
	228	ret
8e029c65 A	229
8e029c65 A	230	// bool OSAtomicTestAndSet( uint32_t n, void *value );
1f2f436a A	231	PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 64, 2)
	232	PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
	233	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
8e029c65 A	234	ret
8e029c65 A	235
1f2f436a A	236	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
	237	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
	238	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	239	ret
8e029c65 A	240
8e029c65 A	241	// bool OSAtomicTestAndClear( uint32_t n, void *value );
1f2f436a A	242	PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 64, 2)
	243	PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
	244	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
8e029c65 A	245	ret
8e029c65 A	246
1f2f436a A	247	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
	248	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
	249	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
8e029c65 A	250	ret
8e029c65 A	251
8e029c65 A	252	// void OSMemoryBarrier( void );
	253	.align 2, 0x90
	254	.globl _OSMemoryBarrier
	255	_OSMemoryBarrier:
1f2f436a A	256	mfence
1f2f436a A	257	ret
224c7076 A	258
	259	/*
	260	* typedef volatile struct {
	261	* void *opaque1; <-- ptr to 1st queue element or null
	262	* long opaque2; <-- generation count
	263	* } OSQueueHead;
	264	*
	265	* void OSAtomicEnqueue( OSQueueHead list, void new, size_t offset);
	266	*/
	267	.align 2
	268	.globl _OSAtomicEnqueue
	269	_OSAtomicEnqueue: // %rdi == list head, %rsi == new, %rdx == offset
	270	pushq %rbx
	271	movq %rsi,%rbx // %rbx == new
	272	movq %rdx,%rsi // %rsi == offset
	273	movq (%rdi),%rax // %rax == ptr to 1st element in Q
	274	movq 8(%rdi),%rdx // %rdx == current generation count
	275	1:
	276	movq %rax,(%rbx,%rsi)// link to old list head from new element
	277	movq %rdx,%rcx
	278	incq %rcx // increment generation count
	279	lock // always lock for now...
	280	cmpxchg16b (%rdi) // ...push on new element
	281	jnz 1b
	282	popq %rbx
8e029c65	283	ret
1f2f436a A	284
	285
	286	/* void* OSAtomicDequeue( OSQueueHead list, size_t offset); /
224c7076 A	287	.align 2
	288	.globl _OSAtomicDequeue
	289	_OSAtomicDequeue: // %rdi == list head, %rsi == offset
	290	pushq %rbx
	291	movq (%rdi),%rax // %rax == ptr to 1st element in Q
	292	movq 8(%rdi),%rdx // %rdx == current generation count
	293	1:
	294	testq %rax,%rax // list empty?
	295	jz 2f // yes
	296	movq (%rax,%rsi),%rbx // point to 2nd in Q
	297	movq %rdx,%rcx
	298	incq %rcx // increment generation count
	299	lock // always lock for now...
	300	cmpxchg16b (%rdi) // ...pop off 1st element
	301	jnz 1b
	302	2:
	303	popq %rbx
	304	ret // ptr to 1st element in Q still in %rax
1f2f436a A	305
	306	/*
	307	* typedef volatile struct {
	308	* void *opaque1; <-- ptr to first queue element or null
	309	* void *opaque2; <-- ptr to last queue element or null
	310	* int opaque3; <-- spinlock
	311	* } OSFifoQueueHead;
	312	*
	313	* void OSAtomicFifoEnqueue( OSFifoQueueHead list, void new, size_t offset);
	314	*/
	315	.align 2
	316	.globl _OSAtomicFifoEnqueue
	317	_OSAtomicFifoEnqueue:
	318	pushq %rbx
	319	xorl %ebx,%ebx // clear "preemption pending" flag
	320	movq $(_COMM_PAGE_PFZ_ENQUEUE), %rcx
	321	call *%rcx
	322	testl %ebx,%ebx // pending preemption?
	323	jz 1f
	324	call _preempt // call into the kernel to pfz_exit
	325	1:
	326	popq %rbx
	327	ret
	328
	329
	330	/* void* OSAtomicFifoDequeue( OSFifoQueueHead list, size_t offset); /
	331	.align 2
	332	.globl _OSAtomicFifoDequeue
	333	_OSAtomicFifoDequeue:
	334	pushq %rbx
	335	xorl %ebx,%ebx // clear "preemption pending" flag
	336	movq %rsi,%rdx // move offset to %rdx to be like the Enqueue case
	337	movq $(_COMM_PAGE_PFZ_DEQUEUE), %rcx
	338	call *%rcx
	339	testl %ebx,%ebx // pending preemption?
	340	jz 1f
	341	call _preempt // call into the kernel to pfz_exit
	342	1:
	343	popq %rbx
	344	ret // ptr to 1st element in Q in %rax
	345
	346	// Local Variables:
	347	// tab-width: 8
	348	// End: