[apple/libc.git] / x86_64 / sys / OSAtomic.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <machine/cpu_capabilities.h>
#include "platfunc.h"
#include <architecture/i386/asm_help.h>

#define DECLARE(x)   \
	.align 2, 0x90      ; \
	.globl x            ; \
	.globl x ## Barrier ; \
x:                  ; \
x ## Barrier:

.text

#define ATOMIC_UP	0
#define ATOMIC_MP	1
#define ATOMIC_RET_ORIG	0
#define ATOMIC_RET_NEW	1

// compare and exchange 32-bit
// xchg32 <new> <dst> <mp>
.macro xchg32
	.if $2 == ATOMIC_MP
	lock
	.endif
	cmpxchgl	$0, ($1)
.endm

// xchg64 <new> <dst> <mp>
.macro xchg64
	.if $2 == ATOMIC_MP
	lock
	.endif
	cmpxchg		$0, ($1)
.endm

#define	ATOMIC_ARITHMETIC(instr, orig, mp)	\
	movl	(%rsi), %eax	/* get 2nd arg -> eax */ ;\
1:	movl	%eax, %edx	/* copy value to new reg */ ;\
	instr	%edi, %edx	/* apply instr to %edx with arg2 */ ;\
	xchg32	%edx, %rsi, mp	/* do the compare swap (see macro above) */ ;\
	jnz	1b		/* jump if failed */ ;\
	.if orig == 1		/* to return the new value, overwrite eax */ ;\
	movl	%edx, %eax	/* return the new value */ ;\
	.endif

// Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
// Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
#define	ATOMIC_BIT_OP(instr, mp)	\
	xorl	$7, %edi	/* bit position is numbered big endian so convert to little endian */ ;\
	shlq	$3, %rsi 	;\
	addq	%rdi, %rsi	/* generate bit address */ ;\
	movq	%rsi, %rdi	;\
	andq	$31, %rdi	/* keep bit offset in range 0..31 */ ;\
	xorq	%rdi, %rsi	/* 4-byte align address */ ;\
	shrq	$3, %rsi	/* get 4-byte aligned address */ ;\
	.if mp == ATOMIC_MP 	/* don't plant the lock in UP code */ ;\
	lock			/* lock the bit test */ ;\
	.endif			;\
	instr	%edi, (%rsi)	/* do the bit test, supplied into the macro */ ;\
	setc	%al		;\
	movzbl	%al,%eax	/* widen in case caller assumes we return an int */

// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret
	
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret
	
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret
	
// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
	movl	%edi, %eax
	xchg32	%esi, %rdx, ATOMIC_UP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
	movl	%edi, %eax
	xchg32	%esi, %rdx, ATOMIC_MP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
	mov	%rdi, %rax
	xchg64	%rsi, %rdx, ATOMIC_UP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret
	
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
	mov	%rdi, %rax
	xchg64	%rsi, %rdx, ATOMIC_MP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
	movl	%edi, %eax	// save amt to add
	xaddl	%edi, (%rsi)	// swap and add value, returns old value in %edi
	addl	%edi, %eax	// add old value to amt as return value
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
	movl	%edi, %eax	// save amt to add
	lock			// lock prefix breaks tabs ;)
	xaddl	%edi, (%rsi)	// swap and add value, returns old value in %edi
	addl	%edi, %eax	// add old value to amt as return value
	ret

// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
	movq	%rdi, %rax	// save amt to add
	xadd	%rdi, (%rsi)	// swap and add value, returns old value in %rsi
	addq	%rdi, %rax	// add old value to amt as return value
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
	movq	%rdi, %rax	// save amt to add
	lock
	xadd	%rdi, (%rsi)	// swap and add value, returns old value in %rsi
	addq	%rdi, %rax	// add old value to amt as return value
	ret

// bool OSAtomicTestAndSet( uint32_t n, void *value );
PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	ret

// bool OSAtomicTestAndClear( uint32_t n, void *value );
PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
	ret

// void OSMemoryBarrier( void );
	.align	2, 0x90
	.globl	_OSMemoryBarrier
_OSMemoryBarrier:
	mfence
	ret

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to 1st queue element or null
 *		long	 opaque2;  <-- generation count
 *	} OSQueueHead;
 *
 * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicEnqueue
_OSAtomicEnqueue:		// %rdi == list head, %rsi == new, %rdx == offset
	pushq	%rbx
	movq	%rsi,%rbx	// %rbx == new
	movq	%rdx,%rsi	// %rsi == offset
	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
	movq	8(%rdi),%rdx	// %rdx == current generation count
1:
	movq	%rax,(%rbx,%rsi)// link to old list head from new element
	movq	%rdx,%rcx
	incq	%rcx		// increment generation count
	lock			// always lock for now...
	cmpxchg16b (%rdi)	// ...push on new element
	jnz	1b
	popq	%rbx
	ret


	/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicDequeue
_OSAtomicDequeue:		// %rdi == list head, %rsi == offset
	pushq	%rbx
	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
	movq	8(%rdi),%rdx	// %rdx == current generation count
1:
	testq	%rax,%rax	// list empty?
	jz	2f		// yes
	movq	(%rax,%rsi),%rbx // point to 2nd in Q
	movq	%rdx,%rcx
	incq	%rcx		// increment generation count
	lock			// always lock for now...
	cmpxchg16b (%rdi)	// ...pop off 1st element
	jnz	1b
2:
	popq	%rbx
	ret			// ptr to 1st element in Q still in %rax

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to first queue element or null
 *		void	*opaque2;  <-- ptr to last queue element or null
 *		int	 opaque3;  <-- spinlock
 *	} OSFifoQueueHead;
 *
 * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicFifoEnqueue
_OSAtomicFifoEnqueue:
	pushq	%rbx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movq 	_commpage_pfz_base(%rip),%rcx
	addq	$(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %rcx
	call	*%rcx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt	// call into the kernel to pfz_exit
1:	
	popq	%rbx
	ret
	
	
/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicFifoDequeue
_OSAtomicFifoDequeue:
	pushq	%rbx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movq	_commpage_pfz_base(%rip), %rcx
	movq	%rsi,%rdx	// move offset to %rdx to be like the Enqueue case
	addq	$(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %rcx
	call	*%rcx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt	// call into the kernel to pfz_exit
1:	
	popq	%rbx
	ret			// ptr to 1st element in Q in %rax

// Local Variables:
// tab-width: 8
// End:
Commit	Line	Data
8e029c65	1	/*
224c7076	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
8e029c65 A	3	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
	4	*
	5	* @APPLE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. Please obtain a copy of the License at
	11	* http://www.opensource.apple.com/apsl/ and read it before using this
	12	* file.
	13	*
	14	* The Original Code and all software distributed under the License are
	15	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	16	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	17	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	19	* Please see the License for the specific language governing rights and
	20	* limitations under the License.
	21	*
	22	* @APPLE_LICENSE_HEADER_END@
	23	*/
	24
	25	#include <machine/cpu_capabilities.h>
ad3c9f2a A	26	#include "platfunc.h"
ad3c9f2a A	27	#include <architecture/i386/asm_help.h>
8e029c65 A	28
8e029c65 A	29	#define DECLARE(x) \
1f2f436a A	30	.align 2, 0x90 ; \
	31	.globl x ; \
	32	.globl x ## Barrier ; \
8e029c65 A	33	x: ; \
	34	x ## Barrier:
	35
	36	.text
	37
1f2f436a A	38	#define ATOMIC_UP 0
	39	#define ATOMIC_MP 1
	40	#define ATOMIC_RET_ORIG 0
	41	#define ATOMIC_RET_NEW 1
	42
	43	// compare and exchange 32-bit
	44	// xchg32 <new> <dst> <mp>
	45	.macro xchg32
	46	.if $2 == ATOMIC_MP
	47	lock
	48	.endif
	49	cmpxchgl $0, ($1)
	50	.endm
	51
	52	// xchg64 <new> <dst> <mp>
	53	.macro xchg64
	54	.if $2 == ATOMIC_MP
	55	lock
	56	.endif
	57	cmpxchg $0, ($1)
	58	.endm
	59
	60	#define ATOMIC_ARITHMETIC(instr, orig, mp) \
	61	movl (%rsi), %eax /* get 2nd arg -> eax */ ;\
	62	1: movl %eax, %edx /* copy value to new reg */ ;\
	63	instr %edi, %edx /* apply instr to %edx with arg2 */ ;\
	64	xchg32 %edx, %rsi, mp /* do the compare swap (see macro above) */ ;\
	65	jnz 1b /* jump if failed */ ;\
	66	.if orig == 1 /* to return the new value, overwrite eax */ ;\
	67	movl %edx, %eax /* return the new value */ ;\
	68	.endif
	69
	70	// Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
	71	// Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
	72	#define ATOMIC_BIT_OP(instr, mp) \
	73	xorl $7, %edi /* bit position is numbered big endian so convert to little endian */ ;\
	74	shlq $3, %rsi ;\
	75	addq %rdi, %rsi /* generate bit address */ ;\
	76	movq %rsi, %rdi ;\
	77	andq $31, %rdi /* keep bit offset in range 0..31 */ ;\
	78	xorq %rdi, %rsi /* 4-byte align address */ ;\
	79	shrq $3, %rsi /* get 4-byte aligned address */ ;\
	80	.if mp == ATOMIC_MP /* don't plant the lock in UP code */ ;\
	81	lock /* lock the bit test */ ;\
	82	.endif ;\
	83	instr %edi, (%rsi) /* do the bit test, supplied into the macro */ ;\
	84	setc %al ;\
	85	movzbl %al,%eax /* widen in case caller assumes we return an int */
8e029c65 A	86
8e029c65 A	87	// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
1f2f436a A	88	PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 64, 2)
	89	PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
	90	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
	91	ret
	92
	93	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
	94	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
	95	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
8e029c65	96	ret
8e029c65 A	97
8e029c65 A	98	// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
1f2f436a A	99	PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 64, 2)
	100	PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
	101	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
8e029c65 A	102	ret
8e029c65 A	103
1f2f436a A	104	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
	105	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
	106	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
	107	ret
8e029c65 A	108
8e029c65 A	109	// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
1f2f436a A	110	PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 64, 2)
	111	PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
	112	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
8e029c65 A	113	ret
8e029c65 A	114
1f2f436a A	115	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
	116	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
	117	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
	118	ret
8e029c65	119
224c7076	120	// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
1f2f436a A	121	PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 64, 2)
	122	PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
	123	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	124	ret
224c7076 A	125
1f2f436a A	126	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
	127	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
	128	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
224c7076 A	129	ret
224c7076 A	130
1f2f436a A	131	// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
	132	PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 64, 2)
	133	PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
	134	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
	135	ret
	136
	137	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
	138	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
	139	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	140	ret
224c7076 A	141
224c7076 A	142	// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
1f2f436a A	143	PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 64, 2)
	144	PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
	145	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	146	ret
224c7076 A	147
1f2f436a A	148	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
	149	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
	150	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	151	ret
224c7076	152
8e029c65	153	// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
1f2f436a A	154	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 64, 2)
	155	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
	156	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
	157	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
	158	movl %edi, %eax
	159	xchg32 %esi, %rdx, ATOMIC_UP
8e029c65	160	sete %al
224c7076	161	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65 A	162	ret
8e029c65 A	163
1f2f436a A	164	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
	165	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
	166	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
	167	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
	168	movl %edi, %eax
	169	xchg32 %esi, %rdx, ATOMIC_MP
	170	sete %al
	171	movzbl %al,%eax // widen in case caller assumes we return an int
	172	ret
8e029c65 A	173
8e029c65 A	174	// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
1f2f436a A	175	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 64, 2)
	176	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
	177	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
	178	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
	179	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
	180	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
	181	mov %rdi, %rax
	182	xchg64 %rsi, %rdx, ATOMIC_UP
	183	sete %al
	184	movzbl %al,%eax // widen in case caller assumes we return an int
	185	ret
	186
	187	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
	188	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
	189	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
	190	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
	191	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
	192	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
	193	mov %rdi, %rax
	194	xchg64 %rsi, %rdx, ATOMIC_MP
8e029c65	195	sete %al
224c7076	196	movzbl %al,%eax // widen in case caller assumes we return an int
8e029c65	197	ret
8e029c65 A	198
8e029c65 A	199	// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
1f2f436a A	200	PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 64, 2)
1f2f436a A	201	PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
8e029c65	202	movl %edi, %eax // save amt to add
1f2f436a A	203	xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
1f2f436a A	204	addl %edi, %eax // add old value to amt as return value
8e029c65 A	205	ret
8e029c65 A	206
1f2f436a A	207	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
	208	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
	209	movl %edi, %eax // save amt to add
	210	lock // lock prefix breaks tabs ;)
	211	xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
	212	addl %edi, %eax // add old value to amt as return value
	213	ret
8e029c65 A	214
8e029c65 A	215	// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
1f2f436a A	216	PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 64, 2)
1f2f436a A	217	PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
8e029c65	218	movq %rdi, %rax // save amt to add
1f2f436a A	219	xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
1f2f436a A	220	addq %rdi, %rax // add old value to amt as return value
8e029c65 A	221	ret
8e029c65 A	222
1f2f436a A	223	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
	224	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
	225	movq %rdi, %rax // save amt to add
	226	lock
	227	xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
	228	addq %rdi, %rax // add old value to amt as return value
	229	ret
8e029c65 A	230
8e029c65 A	231	// bool OSAtomicTestAndSet( uint32_t n, void *value );
1f2f436a A	232	PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 64, 2)
	233	PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
	234	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
8e029c65 A	235	ret
8e029c65 A	236
1f2f436a A	237	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
	238	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
	239	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	240	ret
8e029c65 A	241
8e029c65 A	242	// bool OSAtomicTestAndClear( uint32_t n, void *value );
1f2f436a A	243	PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 64, 2)
	244	PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
	245	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
8e029c65 A	246	ret
8e029c65 A	247
1f2f436a A	248	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
	249	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
	250	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
8e029c65 A	251	ret
8e029c65 A	252
8e029c65 A	253	// void OSMemoryBarrier( void );
	254	.align 2, 0x90
	255	.globl _OSMemoryBarrier
	256	_OSMemoryBarrier:
1f2f436a A	257	mfence
1f2f436a A	258	ret
224c7076 A	259
	260	/*
	261	* typedef volatile struct {
	262	* void *opaque1; <-- ptr to 1st queue element or null
	263	* long opaque2; <-- generation count
	264	* } OSQueueHead;
	265	*
	266	* void OSAtomicEnqueue( OSQueueHead list, void new, size_t offset);
	267	*/
	268	.align 2
	269	.globl _OSAtomicEnqueue
	270	_OSAtomicEnqueue: // %rdi == list head, %rsi == new, %rdx == offset
	271	pushq %rbx
	272	movq %rsi,%rbx // %rbx == new
	273	movq %rdx,%rsi // %rsi == offset
	274	movq (%rdi),%rax // %rax == ptr to 1st element in Q
	275	movq 8(%rdi),%rdx // %rdx == current generation count
	276	1:
	277	movq %rax,(%rbx,%rsi)// link to old list head from new element
	278	movq %rdx,%rcx
	279	incq %rcx // increment generation count
	280	lock // always lock for now...
	281	cmpxchg16b (%rdi) // ...push on new element
	282	jnz 1b
	283	popq %rbx
8e029c65	284	ret
1f2f436a A	285
	286
	287	/* void* OSAtomicDequeue( OSQueueHead list, size_t offset); /
224c7076 A	288	.align 2
	289	.globl _OSAtomicDequeue
	290	_OSAtomicDequeue: // %rdi == list head, %rsi == offset
	291	pushq %rbx
	292	movq (%rdi),%rax // %rax == ptr to 1st element in Q
	293	movq 8(%rdi),%rdx // %rdx == current generation count
	294	1:
	295	testq %rax,%rax // list empty?
	296	jz 2f // yes
	297	movq (%rax,%rsi),%rbx // point to 2nd in Q
	298	movq %rdx,%rcx
	299	incq %rcx // increment generation count
	300	lock // always lock for now...
	301	cmpxchg16b (%rdi) // ...pop off 1st element
	302	jnz 1b
	303	2:
	304	popq %rbx
	305	ret // ptr to 1st element in Q still in %rax
1f2f436a A	306
	307	/*
	308	* typedef volatile struct {
	309	* void *opaque1; <-- ptr to first queue element or null
	310	* void *opaque2; <-- ptr to last queue element or null
	311	* int opaque3; <-- spinlock
	312	* } OSFifoQueueHead;
	313	*
	314	* void OSAtomicFifoEnqueue( OSFifoQueueHead list, void new, size_t offset);
	315	*/
	316	.align 2
	317	.globl _OSAtomicFifoEnqueue
	318	_OSAtomicFifoEnqueue:
	319	pushq %rbx
	320	xorl %ebx,%ebx // clear "preemption pending" flag
ad3c9f2a A	321	movq _commpage_pfz_base(%rip),%rcx
ad3c9f2a A	322	addq $(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %rcx
1f2f436a A	323	call *%rcx
	324	testl %ebx,%ebx // pending preemption?
	325	jz 1f
	326	call _preempt // call into the kernel to pfz_exit
	327	1:
	328	popq %rbx
	329	ret
	330
	331
	332	/* void* OSAtomicFifoDequeue( OSFifoQueueHead list, size_t offset); /
	333	.align 2
	334	.globl _OSAtomicFifoDequeue
	335	_OSAtomicFifoDequeue:
	336	pushq %rbx
	337	xorl %ebx,%ebx // clear "preemption pending" flag
ad3c9f2a	338	movq _commpage_pfz_base(%rip), %rcx
1f2f436a	339	movq %rsi,%rdx // move offset to %rdx to be like the Enqueue case
ad3c9f2a	340	addq $(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %rcx
1f2f436a A	341	call *%rcx
	342	testl %ebx,%ebx // pending preemption?
	343	jz 1f
	344	call _preempt // call into the kernel to pfz_exit
	345	1:
	346	popq %rbx
	347	ret // ptr to 1st element in Q in %rax
	348
	349	// Local Variables:
	350	// tab-width: 8
	351	// End: