[apple/libc.git] / i386 / sys / OSAtomic.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <machine/cpu_capabilities.h>
#include <platfunc.h>

#define ATOMIC_UP	0
#define ATOMIC_MP	1
#define ATOMIC_RET_ORIG	0
#define ATOMIC_RET_NEW	1

// compare and exchange 32-bit
// xchg32 <new> <dst> <mp>
.macro xchg32
	.if $2 == ATOMIC_MP
	lock
	.endif
	cmpxchgl	$0, ($1)
.endm

// compare and exchange 64-bit
// xchg64 <dst> <mp>
.macro xchg64
	.if $1 == ATOMIC_MP
	lock
	.endif
	cmpxchg8b	($0)
.endm


// int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
#define ATOMIC_ARITHMETIC(instr, orig, mp) \
	movl	8(%esp), %ecx	/* load 2nd arg ptr into ecx */ ;\
	movl	(%ecx), %eax	/* load contents of ecx into eax */ ;\
1:	movl	4(%esp), %edx	/* load 1st arg into edx */ ;\
	instr	%eax, %edx	/* do the operation */ ;\
	xchg32	%edx, %ecx, mp	/* old in %eax, new in %edx, exchange into %ecx */ ;\
	jnz	1b		/* go back if we failed to exchange */ ;\
	.if orig == ATOMIC_RET_NEW ;\
	movl	%edx, %eax	/* return new value */ ;\
	.endif

// bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
#define ATOMIC_BIT_OP(instr, mp) \
	movl	4(%esp), %eax	;\
	movl	8(%esp), %edx	;\
	shldl	$3,%edx,%ecx    /* save top 3 bits of address in %ecx */ ;\
	shll	$3,%edx		;\
	xorl	$7,%eax         /* bit position is numbered big endian so convert to little endian */ ;\
	addl	%eax,%edx       /* generate bit address */ ;\
	adcl	$0,%ecx         /* handle carry out of lower half of address */ ;\
	movl	%edx,%eax       /* copy lower half of bit address */ ;\
	andl	$31,%eax        /* keep bit offset in range 0..31 */ ;\
	xorl	%eax,%edx       /* 4-byte align address */ ;\
	shrdl	$3,%ecx,%edx    /* restore 32-bit byte address in %edx */ ;\
	.if mp == ATOMIC_MP	;\
	lock			;\
	.endif			;\
	instr	%eax, (%edx)	;\
	setc	%al		;\
	movzbl	%al,%eax	// widen in case caller assumes we return an int

// int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
#define ATOMIC_ADD64(mp) \
	pushl	%ebx		;\
	pushl	%esi		;\
	movl	20(%esp), %esi	;\
	movl	0(%esi), %eax	;\
	movl	4(%esi), %edx	;\
1:	movl	12(%esp), %ebx	;\
	movl	16(%esp), %ecx	;\
	addl	%eax, %ebx	;\
	adcl	%edx, %ecx	;\
	xchg64	%esi, mp	;\
	jnz	1b		;\
	movl	%ebx, %eax	;\
	movl	%ecx, %edx	;\
	popl	%esi		;\
	popl	%ebx

	.text

PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

// bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	12(%esp), %ecx
	xchg32	%edx, %ecx, ATOMIC_UP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	12(%esp), %ecx
	xchg32	%edx, %ecx, ATOMIC_MP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 32, 2)
	pushl	%ebx		// push out spare stuff for space
	pushl	%esi
	movl    12(%esp), %eax	// load in 1st 64-bit parameter
	movl    16(%esp), %edx
	movl    20(%esp), %ebx	// load in 2nd 64-bit parameter
	movl    24(%esp), %ecx
	movl	28(%esp), %esi	// laod in destination address
	xchg64	%esi, ATOMIC_UP	// compare and swap 64-bit
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	popl	%esi
	popl	%ebx
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 32, 2)
	pushl	%ebx		// push out spare stuff for space
	pushl	%esi
	movl    12(%esp), %eax	// load in 1st 64-bit parameter
	movl    16(%esp), %edx
	movl    20(%esp), %ebx	// load in 2nd 64-bit parameter
	movl    24(%esp), %ecx
	movl	28(%esp), %esi	// laod in destination address
	xchg64	%esi, ATOMIC_MP	// compare and swap 64-bit
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	popl	%esi
	popl	%ebx
	ret

PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	%eax, %ecx
	xaddl	%eax, (%edx)
	addl	%ecx, %eax
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	%eax, %ecx
	lock
	xaddl	%eax, (%edx)
	addl	%ecx, %eax
	ret

PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 32, 2)
	ATOMIC_ADD64(ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 32, 2)
	ATOMIC_ADD64(ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 32, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 32, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 32, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 32, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
	ret
	
// OSMemoryBarrier()
// These are used both in 32 and 64-bit mode.  We use a fence even on UP
// machines, so this function can be used with nontemporal stores.

PLATFUNC_FUNCTION_START_GENERIC(OSMemoryBarrier, all, 32, 4)
	lock
	addl	$0,(%esp)
	ret
PLATFUNC_DESCRIPTOR(OSMemoryBarrier,all,0,kHasSSE2);

PLATFUNC_FUNCTION_START(OSMemoryBarrier, sse2, 32, 4)
	mfence
	ret
PLATFUNC_DESCRIPTOR(OSMemoryBarrier,sse2,kHasSSE2,0);

	/*
	 *	typedef	volatile struct {
	 *		void	*opaque1;  <-- ptr to 1st queue element or null
	 *		long	 opaque2;  <-- generation count
	 *	} OSQueueHead;
	 *
	 * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
	 */
PLATFUNC_FUNCTION_START(OSAtomicEnqueue, up, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%ebx	// %ebx == new
	movl	24(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	movl	%eax,(%ebx,%esi)// link to old list head from new element
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_UP	// ...push on new element
	jnz	1b
	popl	%ebx
	popl	%esi
	popl	%edi
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicEnqueue, mp, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%ebx	// %ebx == new
	movl	24(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	movl	%eax,(%ebx,%esi)// link to old list head from new element
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_MP	// ...push on new element
	jnz	1b
	popl	%ebx
	popl	%esi
	popl	%edi
	ret

/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
PLATFUNC_FUNCTION_START(OSAtomicDequeue, up, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	testl	%eax,%eax	// list empty?
	jz	2f		// yes
	movl	(%eax,%esi),%ebx // point to 2nd in Q
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_UP	// ...pop off 1st element
	jnz	1b
2:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicDequeue, mp, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	testl	%eax,%eax	// list empty?
	jz	2f		// yes
	movl	(%eax,%esi),%ebx // point to 2nd in Q
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_MP	// ...pop off 1st element
	jnz	1b
2:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to first queue element or null
 *		void	*opaque2;  <-- ptr to last queue element or null
 *		int	 opaque3;  <-- spinlock
 *	} OSFifoQueueHead;
 *
 * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicFifoEnqueue
_OSAtomicFifoEnqueue:
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == new
	movl	24(%esp),%edx	// %edx == offset
	movl	$(_COMM_PAGE_PFZ_ENQUEUE), %ecx
	call	*%ecx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt
1:	popl	%ebx
	popl	%esi
	popl	%edi
	ret
	
/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicFifoDequeue
_OSAtomicFifoDequeue:
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%edx	// %edx == offset
	movl	$(_COMM_PAGE_PFZ_DEQUEUE), %ecx
	call	*%ecx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	pushl	%eax		// save return value across sysenter
	call	_preempt
	popl	%eax
1:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

// Local Variables:
// tab-width: 8
// End:
Commit	Line	Data
3d9156a7	1	/*
224c7076 A	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
224c7076 A	3	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3d9156a7 A	4	*
	5	* @APPLE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. Please obtain a copy of the License at
	11	* http://www.opensource.apple.com/apsl/ and read it before using this
	12	* file.
	13	*
	14	* The Original Code and all software distributed under the License are
	15	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	16	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	17	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	19	* Please see the License for the specific language governing rights and
	20	* limitations under the License.
	21	*
	22	* @APPLE_LICENSE_HEADER_END@
	23	*/
	24
	25	#include <machine/cpu_capabilities.h>
1f2f436a	26	#include <platfunc.h>
3d9156a7	27
1f2f436a A	28	#define ATOMIC_UP 0
	29	#define ATOMIC_MP 1
	30	#define ATOMIC_RET_ORIG 0
	31	#define ATOMIC_RET_NEW 1
	32
	33	// compare and exchange 32-bit
	34	// xchg32 <new> <dst> <mp>
	35	.macro xchg32
	36	.if $2 == ATOMIC_MP
	37	lock
	38	.endif
	39	cmpxchgl $0, ($1)
	40	.endm
	41
	42	// compare and exchange 64-bit
	43	// xchg64 <dst> <mp>
	44	.macro xchg64
	45	.if $1 == ATOMIC_MP
	46	lock
	47	.endif
	48	cmpxchg8b ($0)
	49	.endm
	50
	51
	52	// int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
	53	#define ATOMIC_ARITHMETIC(instr, orig, mp) \
	54	movl 8(%esp), %ecx /* load 2nd arg ptr into ecx */ ;\
	55	movl (%ecx), %eax /* load contents of ecx into eax */ ;\
	56	1: movl 4(%esp), %edx /* load 1st arg into edx */ ;\
	57	instr %eax, %edx /* do the operation */ ;\
	58	xchg32 %edx, %ecx, mp /* old in %eax, new in %edx, exchange into %ecx */ ;\
	59	jnz 1b /* go back if we failed to exchange */ ;\
	60	.if orig == ATOMIC_RET_NEW ;\
	61	movl %edx, %eax /* return new value */ ;\
	62	.endif
	63
	64	// bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
	65	#define ATOMIC_BIT_OP(instr, mp) \
	66	movl 4(%esp), %eax ;\
	67	movl 8(%esp), %edx ;\
	68	shldl $3,%edx,%ecx /* save top 3 bits of address in %ecx */ ;\
	69	shll $3,%edx ;\
	70	xorl $7,%eax /* bit position is numbered big endian so convert to little endian */ ;\
	71	addl %eax,%edx /* generate bit address */ ;\
	72	adcl $0,%ecx /* handle carry out of lower half of address */ ;\
	73	movl %edx,%eax /* copy lower half of bit address */ ;\
	74	andl $31,%eax /* keep bit offset in range 0..31 */ ;\
	75	xorl %eax,%edx /* 4-byte align address */ ;\
	76	shrdl $3,%ecx,%edx /* restore 32-bit byte address in %edx */ ;\
	77	.if mp == ATOMIC_MP ;\
	78	lock ;\
	79	.endif ;\
	80	instr %eax, (%edx) ;\
	81	setc %al ;\
	82	movzbl %al,%eax // widen in case caller assumes we return an int
	83
	84	// int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
	85	#define ATOMIC_ADD64(mp) \
	86	pushl %ebx ;\
	87	pushl %esi ;\
	88	movl 20(%esp), %esi ;\
	89	movl 0(%esi), %eax ;\
	90	movl 4(%esi), %edx ;\
	91	1: movl 12(%esp), %ebx ;\
92	movl 16(%esp), %ecx ;\
93	addl %eax, %ebx ;\
94	adcl %edx, %ecx ;\
95	xchg64 %esi, mp ;\
96	jnz 1b ;\
97	movl %ebx, %eax ;\
98	movl %ecx, %edx ;\
99	popl %esi ;\
100	popl %ebx
101
102	.text
103
104	PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 32, 2)
105	PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 32, 2)
106	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
107	ret
108
109	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 32, 2)
110	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 32, 2)
111	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
112	ret
113
114	PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 32, 2)
115	PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 32, 2)
116	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
117	ret
118
119	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 32, 2)
120	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 32, 2)
121	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
3d9156a7 A	122	ret
3d9156a7 A	123
1f2f436a A	124	PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 32, 2)
	125	PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 32, 2)
	126	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
3d9156a7 A	127	ret
3d9156a7 A	128
1f2f436a A	129	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 32, 2)
	130	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 32, 2)
	131	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
3d9156a7 A	132	ret
3d9156a7 A	133
1f2f436a A	134	PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 32, 2)
	135	PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 32, 2)
	136	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	137	ret
224c7076 A	138
1f2f436a A	139	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 32, 2)
	140	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 32, 2)
	141	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
224c7076 A	142	ret
224c7076 A	143
1f2f436a A	144	PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 32, 2)
	145	PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 32, 2)
	146	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	147	ret
224c7076 A	148
1f2f436a A	149	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 32, 2)
	150	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 32, 2)
	151	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	152	ret
	153
	154	PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 32, 2)
	155	PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 32, 2)
	156	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
	157	ret
	158
	159	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 32, 2)
	160	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 32, 2)
	161	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	162	ret
	163
	164	// bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
	165	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 32, 2)
	166	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 32, 2)
	167	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 32, 2)
	168	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 32, 2)
	169	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 32, 2)
	170	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 32, 2)
	171	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 32, 2)
	172	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 32, 2)
	173	movl 4(%esp), %eax
	174	movl 8(%esp), %edx
	175	movl 12(%esp), %ecx
	176	xchg32 %edx, %ecx, ATOMIC_UP
3d9156a7	177	sete %al
224c7076	178	movzbl %al,%eax // widen in case caller assumes we return an int
3d9156a7 A	179	ret
3d9156a7 A	180
1f2f436a A	181	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 32, 2)
	182	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 32, 2)
	183	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 32, 2)
	184	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 32, 2)
	185	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 32, 2)
	186	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 32, 2)
	187	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 32, 2)
	188	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 32, 2)
	189	movl 4(%esp), %eax
	190	movl 8(%esp), %edx
	191	movl 12(%esp), %ecx
	192	xchg32 %edx, %ecx, ATOMIC_MP
	193	sete %al
	194	movzbl %al,%eax // widen in case caller assumes we return an int
	195	ret
	196
	197	// bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
	198	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 32, 2)
	199	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 32, 2)
	200	pushl %ebx // push out spare stuff for space
3d9156a7	201	pushl %esi
1f2f436a	202	movl 12(%esp), %eax // load in 1st 64-bit parameter
3d9156a7	203	movl 16(%esp), %edx
1f2f436a	204	movl 20(%esp), %ebx // load in 2nd 64-bit parameter
3d9156a7	205	movl 24(%esp), %ecx
1f2f436a A	206	movl 28(%esp), %esi // laod in destination address
1f2f436a A	207	xchg64 %esi, ATOMIC_UP // compare and swap 64-bit
3d9156a7	208	sete %al
224c7076	209	movzbl %al,%eax // widen in case caller assumes we return an int
3d9156a7 A	210	popl %esi
	211	popl %ebx
	212	ret
	213
1f2f436a A	214	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 32, 2)
	215	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 32, 2)
	216	pushl %ebx // push out spare stuff for space
3d9156a7	217	pushl %esi
1f2f436a A	218	movl 12(%esp), %eax // load in 1st 64-bit parameter
	219	movl 16(%esp), %edx
	220	movl 20(%esp), %ebx // load in 2nd 64-bit parameter
	221	movl 24(%esp), %ecx
	222	movl 28(%esp), %esi // laod in destination address
	223	xchg64 %esi, ATOMIC_MP // compare and swap 64-bit
	224	sete %al
	225	movzbl %al,%eax // widen in case caller assumes we return an int
3d9156a7	226	popl %esi
1f2f436a	227	popl %ebx
3d9156a7 A	228	ret
3d9156a7 A	229
1f2f436a A	230	PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 32, 2)
1f2f436a A	231	PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 32, 2)
3d9156a7 A	232	movl 4(%esp), %eax
3d9156a7 A	233	movl 8(%esp), %edx
eb1cde05	234	movl %eax, %ecx
1f2f436a A	235	xaddl %eax, (%edx)
1f2f436a A	236	addl %ecx, %eax
3d9156a7 A	237	ret
3d9156a7 A	238
1f2f436a A	239	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 32, 2)
1f2f436a A	240	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 32, 2)
3d9156a7 A	241	movl 4(%esp), %eax
3d9156a7 A	242	movl 8(%esp), %edx
eb1cde05	243	movl %eax, %ecx
1f2f436a A	244	lock
	245	xaddl %eax, (%edx)
	246	addl %ecx, %eax
3d9156a7 A	247	ret
3d9156a7 A	248
1f2f436a A	249	PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 32, 2)
	250	PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 32, 2)
	251	ATOMIC_ADD64(ATOMIC_UP)
3d9156a7 A	252	ret
3d9156a7 A	253
1f2f436a A	254	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 32, 2)
	255	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 32, 2)
	256	ATOMIC_ADD64(ATOMIC_MP)
	257	ret
224c7076	258
1f2f436a A	259	PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 32, 2)
	260	PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 32, 2)
	261	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
	262	ret
	263
	264	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 32, 2)
	265	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 32, 2)
	266	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	267	ret
	268
	269	PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 32, 2)
	270	PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 32, 2)
	271	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
	272	ret
	273
	274	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 32, 2)
	275	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 32, 2)
	276	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
	277	ret
	278
	279	// OSMemoryBarrier()
	280	// These are used both in 32 and 64-bit mode. We use a fence even on UP
	281	// machines, so this function can be used with nontemporal stores.
	282
	283	PLATFUNC_FUNCTION_START_GENERIC(OSMemoryBarrier, all, 32, 4)
	284	lock
	285	addl $0,(%esp)
	286	ret
	287	PLATFUNC_DESCRIPTOR(OSMemoryBarrier,all,0,kHasSSE2);
	288
	289	PLATFUNC_FUNCTION_START(OSMemoryBarrier, sse2, 32, 4)
	290	mfence
	291	ret
	292	PLATFUNC_DESCRIPTOR(OSMemoryBarrier,sse2,kHasSSE2,0);
	293
	294	/*
	295	* typedef volatile struct {
	296	* void *opaque1; <-- ptr to 1st queue element or null
	297	* long opaque2; <-- generation count
	298	* } OSQueueHead;
	299	*
	300	* void OSAtomicEnqueue( OSQueueHead list, void new, size_t offset);
	301	*/
	302	PLATFUNC_FUNCTION_START(OSAtomicEnqueue, up, 32, 2)
224c7076 A	303	pushl %edi
	304	pushl %esi
	305	pushl %ebx
	306	movl 16(%esp),%edi // %edi == ptr to list head
	307	movl 20(%esp),%ebx // %ebx == new
	308	movl 24(%esp),%esi // %esi == offset
	309	movl (%edi),%eax // %eax == ptr to 1st element in Q
	310	movl 4(%edi),%edx // %edx == current generation count
1f2f436a	311	1: movl %eax,(%ebx,%esi)// link to old list head from new element
224c7076 A	312	movl %edx,%ecx
224c7076 A	313	incl %ecx // increment generation count
1f2f436a	314	xchg64 %edi, ATOMIC_UP // ...push on new element
224c7076 A	315	jnz 1b
	316	popl %ebx
	317	popl %esi
	318	popl %edi
3d9156a7	319	ret
1f2f436a A	320
	321	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicEnqueue, mp, 32, 2)
	322	pushl %edi
	323	pushl %esi
	324	pushl %ebx
	325	movl 16(%esp),%edi // %edi == ptr to list head
	326	movl 20(%esp),%ebx // %ebx == new
	327	movl 24(%esp),%esi // %esi == offset
	328	movl (%edi),%eax // %eax == ptr to 1st element in Q
	329	movl 4(%edi),%edx // %edx == current generation count
	330	1: movl %eax,(%ebx,%esi)// link to old list head from new element
	331	movl %edx,%ecx
	332	incl %ecx // increment generation count
	333	xchg64 %edi, ATOMIC_MP // ...push on new element
	334	jnz 1b
	335	popl %ebx
	336	popl %esi
	337	popl %edi
	338	ret
	339
224c7076	340	/* void* OSAtomicDequeue( OSQueueHead list, size_t offset); /
1f2f436a	341	PLATFUNC_FUNCTION_START(OSAtomicDequeue, up, 32, 2)
224c7076 A	342	pushl %edi
	343	pushl %esi
	344	pushl %ebx
	345	movl 16(%esp),%edi // %edi == ptr to list head
	346	movl 20(%esp),%esi // %esi == offset
	347	movl (%edi),%eax // %eax == ptr to 1st element in Q
	348	movl 4(%edi),%edx // %edx == current generation count
1f2f436a	349	1: testl %eax,%eax // list empty?
224c7076 A	350	jz 2f // yes
	351	movl (%eax,%esi),%ebx // point to 2nd in Q
	352	movl %edx,%ecx
	353	incl %ecx // increment generation count
1f2f436a	354	xchg64 %edi, ATOMIC_UP // ...pop off 1st element
224c7076	355	jnz 1b
1f2f436a A	356	2: popl %ebx
	357	popl %esi
	358	popl %edi
	359	ret // ptr to 1st element in Q still in %eax
	360
	361	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicDequeue, mp, 32, 2)
	362	pushl %edi
	363	pushl %esi
	364	pushl %ebx
	365	movl 16(%esp),%edi // %edi == ptr to list head
	366	movl 20(%esp),%esi // %esi == offset
	367	movl (%edi),%eax // %eax == ptr to 1st element in Q
	368	movl 4(%edi),%edx // %edx == current generation count
	369	1: testl %eax,%eax // list empty?
	370	jz 2f // yes
	371	movl (%eax,%esi),%ebx // point to 2nd in Q
	372	movl %edx,%ecx
	373	incl %ecx // increment generation count
	374	xchg64 %edi, ATOMIC_MP // ...pop off 1st element
	375	jnz 1b
	376	2: popl %ebx
	377	popl %esi
	378	popl %edi
	379	ret // ptr to 1st element in Q still in %eax
	380
	381	/*
	382	* typedef volatile struct {
	383	* void *opaque1; <-- ptr to first queue element or null
	384	* void *opaque2; <-- ptr to last queue element or null
	385	* int opaque3; <-- spinlock
	386	* } OSFifoQueueHead;
	387	*
	388	* void OSAtomicFifoEnqueue( OSFifoQueueHead list, void new, size_t offset);
	389	*/
	390	.align 2
	391	.globl _OSAtomicFifoEnqueue
	392	_OSAtomicFifoEnqueue:
	393	pushl %edi
	394	pushl %esi
	395	pushl %ebx
	396	xorl %ebx,%ebx // clear "preemption pending" flag
	397	movl 16(%esp),%edi // %edi == ptr to list head
	398	movl 20(%esp),%esi // %esi == new
	399	movl 24(%esp),%edx // %edx == offset
	400	movl $(_COMM_PAGE_PFZ_ENQUEUE), %ecx
	401	call *%ecx
	402	testl %ebx,%ebx // pending preemption?
	403	jz 1f
	404	call _preempt
	405	1: popl %ebx
	406	popl %esi
	407	popl %edi
	408	ret
	409
	410	/* void* OSAtomicFifoDequeue( OSFifoQueueHead list, size_t offset); /
	411	.align 2
	412	.globl _OSAtomicFifoDequeue
	413	_OSAtomicFifoDequeue:
	414	pushl %edi
	415	pushl %esi
	416	pushl %ebx
	417	xorl %ebx,%ebx // clear "preemption pending" flag
	418	movl 16(%esp),%edi // %edi == ptr to list head
	419	movl 20(%esp),%edx // %edx == offset
420	movl $(_COMM_PAGE_PFZ_DEQUEUE), %ecx
421	call *%ecx
422	testl %ebx,%ebx // pending preemption?
423	jz 1f
424	pushl %eax // save return value across sysenter
425	call _preempt
426	popl %eax
427	1: popl %ebx
224c7076 A	428	popl %esi
	429	popl %edi
	430	ret // ptr to 1st element in Q still in %eax
	431
1f2f436a A	432	// Local Variables:
	433	// tab-width: 8
	434	// End: