[apple/libplatform.git] / src / atomics / i386 / OSAtomic.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <architecture/i386/asm_help.h>
#include "os/internal_asm.h"

#define ATOMIC_RET_ORIG	0
#define ATOMIC_RET_NEW	1

// compare and exchange 32-bit
// xchg32 <new> <dst>
.macro xchg32
	lock
	cmpxchgl	$0, ($1)
.endm

// compare and exchange 64-bit
// xchg64 <dst>
.macro xchg64
	lock
	cmpxchg8b	($0)
.endm


// int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
#define ATOMIC_ARITHMETIC(instr, orig) \
	movl	8(%esp), %ecx	/* load 2nd arg ptr into ecx */ ;\
	movl	(%ecx), %eax	/* load contents of ecx into eax */ ;\
1:	movl	4(%esp), %edx	/* load 1st arg into edx */ ;\
	instr	%eax, %edx	/* do the operation */ ;\
	xchg32	%edx, %ecx	/* old in %eax, new in %edx, exchange into %ecx */ ;\
	jnz	1b		/* go back if we failed to exchange */ ;\
	.if orig == ATOMIC_RET_NEW ;\
	movl	%edx, %eax	/* return new value */ ;\
	.endif

// bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
#define ATOMIC_BIT_OP(instr) \
	movl	4(%esp), %eax	;\
	movl	8(%esp), %edx	;\
	shldl	$3,%edx,%ecx    /* save top 3 bits of address in %ecx */ ;\
	shll	$3,%edx		;\
	xorl	$7,%eax         /* bit position is numbered big endian so convert to little endian */ ;\
	addl	%eax,%edx       /* generate bit address */ ;\
	adcl	$0,%ecx         /* handle carry out of lower half of address */ ;\
	movl	%edx,%eax       /* copy lower half of bit address */ ;\
	andl	$31,%eax        /* keep bit offset in range 0..31 */ ;\
	xorl	%eax,%edx       /* 4-byte align address */ ;\
	shrdl	$3,%ecx,%edx    /* restore 32-bit byte address in %edx */ ;\
	lock			;\
	instr	%eax, (%edx)	;\
	setc	%al		;\
	movzbl	%al,%eax	// widen in case caller assumes we return an int

// int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
#define ATOMIC_ADD64() \
	pushl	%ebx		;\
	pushl	%esi		;\
	movl	20(%esp), %esi	;\
	movl	0(%esi), %eax	;\
	movl	4(%esi), %edx	;\
1:	movl	12(%esp), %ebx	;\
	movl	16(%esp), %ecx	;\
	addl	%eax, %ebx	;\
	adcl	%edx, %ecx	;\
	xchg64	%esi	;\
	jnz	1b		;\
	movl	%ebx, %eax	;\
	movl	%ecx, %edx	;\
	popl	%esi		;\
	popl	%ebx

// int64_t OSAtomicIncrement64(volatile int64_t *theValue);
#define ATOMIC_INC64() \
	pushl	%ebx		;\
	pushl	%esi		;\
	movl	12(%esp), %esi	;\
	movl	0(%esi), %eax	;\
	movl	4(%esi), %edx	;\
1:	movl	$1, %ebx	;\
	xorl	%ecx, %ecx	;\
	addl	%eax, %ebx	;\
	adcl	%edx, %ecx	;\
	xchg64	%esi	;\
	jnz	1b		;\
	movl	%ebx, %eax	;\
	movl	%ecx, %edx	;\
	popl	%esi		;\
	popl	%ebx

// int64_t OSAtomicDecrement64(volatile int64_t *theValue);
#define ATOMIC_DEC64() \
	pushl	%ebx		;\
	pushl	%esi		;\
	movl	12(%esp), %esi	;\
	movl	0(%esi), %eax	;\
	movl	4(%esi), %edx	;\
1:	movl	$-1, %ebx	;\
	movl	$-1, %ecx	;\
	addl	%eax, %ebx	;\
	adcl	%edx, %ecx	;\
	xchg64	%esi	;\
	jnz	1b		;\
	movl	%ebx, %eax	;\
	movl	%ecx, %edx	;\
	popl	%esi		;\
	popl	%ebx

	.text

OS_ATOMIC_FUNCTION_START(OSAtomicAnd32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicAnd32Barrier, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW)
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicOr32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicOr32Barrier, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW)
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicXor32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicXor32Barrier, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW)
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicAnd32Orig, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicAnd32OrigBarrier, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG)
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicOr32Orig, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicOr32OrigBarrier, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG)
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicXor32Orig, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicXor32OrigBarrier, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG)
	ret

// bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapPtr, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapInt, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapLong, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	12(%esp), %ecx
	xchg32	%edx, %ecx
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap64, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, 2)
	pushl	%ebx		// push out spare stuff for space
	pushl	%esi
	movl    12(%esp), %eax	// load in 1st 64-bit parameter
	movl    16(%esp), %edx
	movl    20(%esp), %ebx	// load in 2nd 64-bit parameter
	movl    24(%esp), %ecx
	movl	28(%esp), %esi	// laod in destination address
	xchg64	%esi	// compare and swap 64-bit
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	popl	%esi
	popl	%ebx
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicAdd32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicAdd32Barrier, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	%eax, %ecx
	lock
	xaddl	%eax, (%edx)
	addl	%ecx, %eax
	ret

OS_VARIANT_FUNCTION_START(OSAtomicIncrement32, up, 2)
OS_VARIANT_FUNCTION_START(OSAtomicIncrement32Barrier, up, 2)
	movl	4(%esp), %ecx
	movl	$1, %eax
	xaddl	%eax, (%ecx)
	incl	%eax
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicIncrement32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicIncrement32Barrier, 2)
	movl	4(%esp), %ecx
	movl	$1, %eax
	lock
	xaddl	%eax, (%ecx)
	incl	%eax
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicDecrement32, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicDecrement32Barrier, 2)
	movl	4(%esp), %ecx
	movl	$-1, %eax
	lock
	xaddl	%eax, (%ecx)
	decl	%eax
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicAdd64, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicAdd64Barrier, 2)
	ATOMIC_ADD64()
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicIncrement64, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicIncrement64Barrier, 2)
	ATOMIC_INC64()
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicDecrement64, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicDecrement64Barrier, 2)
	ATOMIC_DEC64()
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicTestAndSet, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicTestAndSetBarrier, 2)
	ATOMIC_BIT_OP(btsl)
	ret

OS_ATOMIC_FUNCTION_START(OSAtomicTestAndClear, 2)
OS_ATOMIC_FUNCTION_START(OSAtomicTestAndClearBarrier, 2)
	ATOMIC_BIT_OP(btrl)
	ret
	
// OSMemoryBarrier()
// These are used both in 32 and 64-bit mode.  We use a fence even on UP
// machines, so this function can be used with nontemporal stores.

OS_ATOMIC_FUNCTION_START(OSMemoryBarrier, 4)
	mfence
	ret

	/*
	 *	typedef	volatile struct {
	 *		void	*opaque1;  <-- ptr to 1st queue element or null
	 *		long	 opaque2;  <-- generation count
	 *	} OSQueueHead;
	 *
	 * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
	 */
OS_ATOMIC_FUNCTION_START(OSAtomicEnqueue, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%ebx	// %ebx == new
	movl	24(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	movl	%eax,(%ebx,%esi)// link to old list head from new element
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi	// ...push on new element
	jnz	1b
	popl	%ebx
	popl	%esi
	popl	%edi
	ret

/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
OS_ATOMIC_FUNCTION_START(OSAtomicDequeue, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	testl	%eax,%eax	// list empty?
	jz	2f		// yes
	movl	(%eax,%esi),%ebx // point to 2nd in Q
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi	// ...pop off 1st element
	jnz	1b
2:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to first queue element or null
 *		void	*opaque2;  <-- ptr to last queue element or null
 *		int	 opaque3;  <-- spinlock
 *	} OSFifoQueueHead;
 *
 * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 */
OS_ATOMIC_FUNCTION_START(OSAtomicFifoEnqueue, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == new
	EXTERN_TO_REG(_commpage_pfz_base,%ecx)
	movl	(%ecx), %ecx
	addl	$(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %ecx
	movl	24(%esp),%edx	// %edx == offset
	call	*%ecx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt
1:	popl	%ebx
	popl	%esi
	popl	%edi
	ret
	
/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
OS_ATOMIC_FUNCTION_START(OSAtomicFifoDequeue, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movl	16(%esp),%edi	// %edi == ptr to list head
	PICIFY(_commpage_pfz_base)
	movl	(%edx),%ecx
	movl	20(%esp),%edx	// %edx == offset
	addl	$(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %ecx
	call	*%ecx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	pushl	%eax		// save return value across sysenter
	call	_preempt
	popl	%eax
1:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

// Local Variables:
// tab-width: 8
// End:
Commit	Line	Data
ada7c492 A	1	/*
	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
	3	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
	4	*
	5	* @APPLE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. Please obtain a copy of the License at
	11	* http://www.opensource.apple.com/apsl/ and read it before using this
	12	* file.
	13	*
	14	* The Original Code and all software distributed under the License are
	15	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	16	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	17	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	19	* Please see the License for the specific language governing rights and
	20	* limitations under the License.
	21	*
	22	* @APPLE_LICENSE_HEADER_END@
	23	*/
	24
	25	#include <architecture/i386/asm_help.h>
	26	#include "os/internal_asm.h"
	27
	28	#define ATOMIC_RET_ORIG 0
	29	#define ATOMIC_RET_NEW 1
	30
	31	// compare and exchange 32-bit
	32	// xchg32 <new> <dst>
	33	.macro xchg32
	34	lock
	35	cmpxchgl $0, ($1)
	36	.endm
	37
	38	// compare and exchange 64-bit
	39	// xchg64 <dst>
	40	.macro xchg64
	41	lock
	42	cmpxchg8b ($0)
	43	.endm
	44
	45
	46	// int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
	47	#define ATOMIC_ARITHMETIC(instr, orig) \
	48	movl 8(%esp), %ecx /* load 2nd arg ptr into ecx */ ;\
	49	movl (%ecx), %eax /* load contents of ecx into eax */ ;\
	50	1: movl 4(%esp), %edx /* load 1st arg into edx */ ;\
	51	instr %eax, %edx /* do the operation */ ;\
	52	xchg32 %edx, %ecx /* old in %eax, new in %edx, exchange into %ecx */ ;\
	53	jnz 1b /* go back if we failed to exchange */ ;\
	54	.if orig == ATOMIC_RET_NEW ;\
	55	movl %edx, %eax /* return new value */ ;\
	56	.endif
	57
	58	// bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
	59	#define ATOMIC_BIT_OP(instr) \
	60	movl 4(%esp), %eax ;\
	61	movl 8(%esp), %edx ;\
	62	shldl $3,%edx,%ecx /* save top 3 bits of address in %ecx */ ;\
	63	shll $3,%edx ;\
	64	xorl $7,%eax /* bit position is numbered big endian so convert to little endian */ ;\
65	addl %eax,%edx /* generate bit address */ ;\
66	adcl $0,%ecx /* handle carry out of lower half of address */ ;\
67	movl %edx,%eax /* copy lower half of bit address */ ;\
68	andl $31,%eax /* keep bit offset in range 0..31 */ ;\
69	xorl %eax,%edx /* 4-byte align address */ ;\
70	shrdl $3,%ecx,%edx /* restore 32-bit byte address in %edx */ ;\
71	lock ;\
72	instr %eax, (%edx) ;\
73	setc %al ;\
74	movzbl %al,%eax // widen in case caller assumes we return an int
75
76	// int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
77	#define ATOMIC_ADD64() \
78	pushl %ebx ;\
79	pushl %esi ;\
80	movl 20(%esp), %esi ;\
81	movl 0(%esi), %eax ;\
82	movl 4(%esi), %edx ;\
83	1: movl 12(%esp), %ebx ;\
84	movl 16(%esp), %ecx ;\
85	addl %eax, %ebx ;\
86	adcl %edx, %ecx ;\
87	xchg64 %esi ;\
88	jnz 1b ;\
89	movl %ebx, %eax ;\
90	movl %ecx, %edx ;\
91	popl %esi ;\
92	popl %ebx
93
94	// int64_t OSAtomicIncrement64(volatile int64_t *theValue);
95	#define ATOMIC_INC64() \
96	pushl %ebx ;\
97	pushl %esi ;\
98	movl 12(%esp), %esi ;\
99	movl 0(%esi), %eax ;\
100	movl 4(%esi), %edx ;\
101	1: movl $1, %ebx ;\
102	xorl %ecx, %ecx ;\
103	addl %eax, %ebx ;\
104	adcl %edx, %ecx ;\
105	xchg64 %esi ;\
106	jnz 1b ;\
107	movl %ebx, %eax ;\
108	movl %ecx, %edx ;\
109	popl %esi ;\
110	popl %ebx
111
112	// int64_t OSAtomicDecrement64(volatile int64_t *theValue);
113	#define ATOMIC_DEC64() \
114	pushl %ebx ;\
115	pushl %esi ;\
116	movl 12(%esp), %esi ;\
117	movl 0(%esi), %eax ;\
118	movl 4(%esi), %edx ;\
119	1: movl $-1, %ebx ;\
120	movl $-1, %ecx ;\
121	addl %eax, %ebx ;\
122	adcl %edx, %ecx ;\
123	xchg64 %esi ;\
124	jnz 1b ;\
125	movl %ebx, %eax ;\
126	movl %ecx, %edx ;\
127	popl %esi ;\
128	popl %ebx
129
130	.text
131
132	OS_ATOMIC_FUNCTION_START(OSAtomicAnd32, 2)
133	OS_ATOMIC_FUNCTION_START(OSAtomicAnd32Barrier, 2)
134	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW)
135	ret
136
137	OS_ATOMIC_FUNCTION_START(OSAtomicOr32, 2)
138	OS_ATOMIC_FUNCTION_START(OSAtomicOr32Barrier, 2)
139	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW)
140	ret
141
142	OS_ATOMIC_FUNCTION_START(OSAtomicXor32, 2)
143	OS_ATOMIC_FUNCTION_START(OSAtomicXor32Barrier, 2)
144	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW)
145	ret
146
147	OS_ATOMIC_FUNCTION_START(OSAtomicAnd32Orig, 2)
148	OS_ATOMIC_FUNCTION_START(OSAtomicAnd32OrigBarrier, 2)
149	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG)
150	ret
151
152	OS_ATOMIC_FUNCTION_START(OSAtomicOr32Orig, 2)
153	OS_ATOMIC_FUNCTION_START(OSAtomicOr32OrigBarrier, 2)
154	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG)
155	ret
156
157	OS_ATOMIC_FUNCTION_START(OSAtomicXor32Orig, 2)
158	OS_ATOMIC_FUNCTION_START(OSAtomicXor32OrigBarrier, 2)
159	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG)
160	ret
161
162	// bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
163	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapPtr, 2)
164	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, 2)
165	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapInt, 2)
166	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, 2)
167	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapLong, 2)
168	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, 2)
169	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap32, 2)
170	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, 2)
171	movl 4(%esp), %eax
172	movl 8(%esp), %edx
173	movl 12(%esp), %ecx
174	xchg32 %edx, %ecx
175	sete %al
176	movzbl %al,%eax // widen in case caller assumes we return an int
177	ret
178
179	// bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
180	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap64, 2)
181	OS_ATOMIC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, 2)
182	pushl %ebx // push out spare stuff for space
183	pushl %esi
184	movl 12(%esp), %eax // load in 1st 64-bit parameter
185	movl 16(%esp), %edx
186	movl 20(%esp), %ebx // load in 2nd 64-bit parameter
187	movl 24(%esp), %ecx
188	movl 28(%esp), %esi // laod in destination address
189	xchg64 %esi // compare and swap 64-bit
190	sete %al
191	movzbl %al,%eax // widen in case caller assumes we return an int
192	popl %esi
193	popl %ebx
194	ret
195
196	OS_ATOMIC_FUNCTION_START(OSAtomicAdd32, 2)
197	OS_ATOMIC_FUNCTION_START(OSAtomicAdd32Barrier, 2)
198	movl 4(%esp), %eax
199	movl 8(%esp), %edx
200	movl %eax, %ecx
201	lock
202	xaddl %eax, (%edx)
203	addl %ecx, %eax
204	ret
205
206	OS_VARIANT_FUNCTION_START(OSAtomicIncrement32, up, 2)
207	OS_VARIANT_FUNCTION_START(OSAtomicIncrement32Barrier, up, 2)
208	movl 4(%esp), %ecx
209	movl $1, %eax
210	xaddl %eax, (%ecx)
211	incl %eax
212	ret
213
214	OS_ATOMIC_FUNCTION_START(OSAtomicIncrement32, 2)
215	OS_ATOMIC_FUNCTION_START(OSAtomicIncrement32Barrier, 2)
216	movl 4(%esp), %ecx
217	movl $1, %eax
218	lock
219	xaddl %eax, (%ecx)
220	incl %eax
221	ret
222
223	OS_ATOMIC_FUNCTION_START(OSAtomicDecrement32, 2)
224	OS_ATOMIC_FUNCTION_START(OSAtomicDecrement32Barrier, 2)
225	movl 4(%esp), %ecx
226	movl $-1, %eax
227	lock
228	xaddl %eax, (%ecx)
229	decl %eax
230	ret
231
232	OS_ATOMIC_FUNCTION_START(OSAtomicAdd64, 2)
233	OS_ATOMIC_FUNCTION_START(OSAtomicAdd64Barrier, 2)
234	ATOMIC_ADD64()
235	ret
236
237	OS_ATOMIC_FUNCTION_START(OSAtomicIncrement64, 2)
238	OS_ATOMIC_FUNCTION_START(OSAtomicIncrement64Barrier, 2)
239	ATOMIC_INC64()
240	ret
241
242	OS_ATOMIC_FUNCTION_START(OSAtomicDecrement64, 2)
243	OS_ATOMIC_FUNCTION_START(OSAtomicDecrement64Barrier, 2)
244	ATOMIC_DEC64()
245	ret
246
247	OS_ATOMIC_FUNCTION_START(OSAtomicTestAndSet, 2)
248	OS_ATOMIC_FUNCTION_START(OSAtomicTestAndSetBarrier, 2)
249	ATOMIC_BIT_OP(btsl)
250	ret
251
252	OS_ATOMIC_FUNCTION_START(OSAtomicTestAndClear, 2)
253	OS_ATOMIC_FUNCTION_START(OSAtomicTestAndClearBarrier, 2)
254	ATOMIC_BIT_OP(btrl)
255	ret
256
257	// OSMemoryBarrier()
258	// These are used both in 32 and 64-bit mode. We use a fence even on UP
259	// machines, so this function can be used with nontemporal stores.
260
261	OS_ATOMIC_FUNCTION_START(OSMemoryBarrier, 4)
262	mfence
263	ret
264
265	/*
266	* typedef volatile struct {
267	* void *opaque1; <-- ptr to 1st queue element or null
268	* long opaque2; <-- generation count
269	* } OSQueueHead;
270	*
271	* void OSAtomicEnqueue( OSQueueHead list, void new, size_t offset);
272	*/
273	OS_ATOMIC_FUNCTION_START(OSAtomicEnqueue, 2)
274	pushl %edi
275	pushl %esi
276	pushl %ebx
277	movl 16(%esp),%edi // %edi == ptr to list head
278	movl 20(%esp),%ebx // %ebx == new
279	movl 24(%esp),%esi // %esi == offset
280	movl (%edi),%eax // %eax == ptr to 1st element in Q
281	movl 4(%edi),%edx // %edx == current generation count
282	1: movl %eax,(%ebx,%esi)// link to old list head from new element
283	movl %edx,%ecx
284	incl %ecx // increment generation count
285	xchg64 %edi // ...push on new element
286	jnz 1b
287	popl %ebx
288	popl %esi
289	popl %edi
290	ret
291
292	/* void* OSAtomicDequeue( OSQueueHead list, size_t offset); /
293	OS_ATOMIC_FUNCTION_START(OSAtomicDequeue, 2)
294	pushl %edi
295	pushl %esi
296	pushl %ebx
297	movl 16(%esp),%edi // %edi == ptr to list head
298	movl 20(%esp),%esi // %esi == offset
299	movl (%edi),%eax // %eax == ptr to 1st element in Q
300	movl 4(%edi),%edx // %edx == current generation count
301	1: testl %eax,%eax // list empty?
302	jz 2f // yes
303	movl (%eax,%esi),%ebx // point to 2nd in Q
304	movl %edx,%ecx
305	incl %ecx // increment generation count
306	xchg64 %edi // ...pop off 1st element
307	jnz 1b
308	2: popl %ebx
309	popl %esi
310	popl %edi
311	ret // ptr to 1st element in Q still in %eax
312
313	/*
314	* typedef volatile struct {
315	* void *opaque1; <-- ptr to first queue element or null
316	* void *opaque2; <-- ptr to last queue element or null
317	* int opaque3; <-- spinlock
318	* } OSFifoQueueHead;
319	*
320	* void OSAtomicFifoEnqueue( OSFifoQueueHead list, void new, size_t offset);
321	*/
322	OS_ATOMIC_FUNCTION_START(OSAtomicFifoEnqueue, 2)
323	pushl %edi
324	pushl %esi
325	pushl %ebx
326	xorl %ebx,%ebx // clear "preemption pending" flag
327	movl 16(%esp),%edi // %edi == ptr to list head
328	movl 20(%esp),%esi // %esi == new
329	EXTERN_TO_REG(_commpage_pfz_base,%ecx)
330	movl (%ecx), %ecx
331	addl $(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %ecx
332	movl 24(%esp),%edx // %edx == offset
333	call *%ecx
334	testl %ebx,%ebx // pending preemption?
335	jz 1f
336	call _preempt
337	1: popl %ebx
338	popl %esi
339	popl %edi
340	ret
341
342	/* void* OSAtomicFifoDequeue( OSFifoQueueHead list, size_t offset); /
343	OS_ATOMIC_FUNCTION_START(OSAtomicFifoDequeue, 2)
344	pushl %edi
345	pushl %esi
346	pushl %ebx
347	xorl %ebx,%ebx // clear "preemption pending" flag
348	movl 16(%esp),%edi // %edi == ptr to list head
349	PICIFY(_commpage_pfz_base)
350	movl (%edx),%ecx
351	movl 20(%esp),%edx // %edx == offset
352	addl $(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %ecx
353	call *%ecx
354	testl %ebx,%ebx // pending preemption?
355	jz 1f
356	pushl %eax // save return value across sysenter
357	call _preempt
358	popl %eax
359	1: popl %ebx
360	popl %esi
361	popl %edi
362	ret // ptr to 1st element in Q still in %eax
363
364	// Local Variables:
365	// tab-width: 8
366	// End: