[apple/libc.git] / i386 / sys / OSAtomic.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <machine/cpu_capabilities.h>
#include <platfunc.h>
#include <architecture/i386/asm_help.h>

#define ATOMIC_UP	0
#define ATOMIC_MP	1
#define ATOMIC_RET_ORIG	0
#define ATOMIC_RET_NEW	1

// compare and exchange 32-bit
// xchg32 <new> <dst> <mp>
.macro xchg32
	.if $2 == ATOMIC_MP
	lock
	.endif
	cmpxchgl	$0, ($1)
.endm

// compare and exchange 64-bit
// xchg64 <dst> <mp>
.macro xchg64
	.if $1 == ATOMIC_MP
	lock
	.endif
	cmpxchg8b	($0)
.endm


// int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
#define ATOMIC_ARITHMETIC(instr, orig, mp) \
	movl	8(%esp), %ecx	/* load 2nd arg ptr into ecx */ ;\
	movl	(%ecx), %eax	/* load contents of ecx into eax */ ;\
1:	movl	4(%esp), %edx	/* load 1st arg into edx */ ;\
	instr	%eax, %edx	/* do the operation */ ;\
	xchg32	%edx, %ecx, mp	/* old in %eax, new in %edx, exchange into %ecx */ ;\
	jnz	1b		/* go back if we failed to exchange */ ;\
	.if orig == ATOMIC_RET_NEW ;\
	movl	%edx, %eax	/* return new value */ ;\
	.endif

// bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
#define ATOMIC_BIT_OP(instr, mp) \
	movl	4(%esp), %eax	;\
	movl	8(%esp), %edx	;\
	shldl	$3,%edx,%ecx    /* save top 3 bits of address in %ecx */ ;\
	shll	$3,%edx		;\
	xorl	$7,%eax         /* bit position is numbered big endian so convert to little endian */ ;\
	addl	%eax,%edx       /* generate bit address */ ;\
	adcl	$0,%ecx         /* handle carry out of lower half of address */ ;\
	movl	%edx,%eax       /* copy lower half of bit address */ ;\
	andl	$31,%eax        /* keep bit offset in range 0..31 */ ;\
	xorl	%eax,%edx       /* 4-byte align address */ ;\
	shrdl	$3,%ecx,%edx    /* restore 32-bit byte address in %edx */ ;\
	.if mp == ATOMIC_MP	;\
	lock			;\
	.endif			;\
	instr	%eax, (%edx)	;\
	setc	%al		;\
	movzbl	%al,%eax	// widen in case caller assumes we return an int

// int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
#define ATOMIC_ADD64(mp) \
	pushl	%ebx		;\
	pushl	%esi		;\
	movl	20(%esp), %esi	;\
	movl	0(%esi), %eax	;\
	movl	4(%esi), %edx	;\
1:	movl	12(%esp), %ebx	;\
	movl	16(%esp), %ecx	;\
	addl	%eax, %ebx	;\
	adcl	%edx, %ecx	;\
	xchg64	%esi, mp	;\
	jnz	1b		;\
	movl	%ebx, %eax	;\
	movl	%ecx, %edx	;\
	popl	%esi		;\
	popl	%ebx

	.text

PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 32, 2)
	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	ret

// bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	12(%esp), %ecx
	xchg32	%edx, %ecx, ATOMIC_UP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	12(%esp), %ecx
	xchg32	%edx, %ecx, ATOMIC_MP
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	ret

// bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 32, 2)
	pushl	%ebx		// push out spare stuff for space
	pushl	%esi
	movl    12(%esp), %eax	// load in 1st 64-bit parameter
	movl    16(%esp), %edx
	movl    20(%esp), %ebx	// load in 2nd 64-bit parameter
	movl    24(%esp), %ecx
	movl	28(%esp), %esi	// laod in destination address
	xchg64	%esi, ATOMIC_UP	// compare and swap 64-bit
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	popl	%esi
	popl	%ebx
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 32, 2)
	pushl	%ebx		// push out spare stuff for space
	pushl	%esi
	movl    12(%esp), %eax	// load in 1st 64-bit parameter
	movl    16(%esp), %edx
	movl    20(%esp), %ebx	// load in 2nd 64-bit parameter
	movl    24(%esp), %ecx
	movl	28(%esp), %esi	// laod in destination address
	xchg64	%esi, ATOMIC_MP	// compare and swap 64-bit
	sete	%al
	movzbl	%al,%eax	// widen in case caller assumes we return an int
	popl	%esi
	popl	%ebx
	ret

PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	%eax, %ecx
	xaddl	%eax, (%edx)
	addl	%ecx, %eax
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 32, 2)
	movl	4(%esp), %eax
	movl	8(%esp), %edx
	movl	%eax, %ecx
	lock
	xaddl	%eax, (%edx)
	addl	%ecx, %eax
	ret

PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 32, 2)
	ATOMIC_ADD64(ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 32, 2)
	ATOMIC_ADD64(ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 32, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 32, 2)
	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	ret

PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 32, 2)
PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 32, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 32, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 32, 2)
	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
	ret
	
// OSMemoryBarrier()
// These are used both in 32 and 64-bit mode.  We use a fence even on UP
// machines, so this function can be used with nontemporal stores.

PLATFUNC_FUNCTION_START_GENERIC(OSMemoryBarrier, all, 32, 4)
	lock
	addl	$0,(%esp)
	ret
PLATFUNC_DESCRIPTOR(OSMemoryBarrier,all,0,kHasSSE2);

PLATFUNC_FUNCTION_START(OSMemoryBarrier, sse2, 32, 4)
	mfence
	ret
PLATFUNC_DESCRIPTOR(OSMemoryBarrier,sse2,kHasSSE2,0);

	/*
	 *	typedef	volatile struct {
	 *		void	*opaque1;  <-- ptr to 1st queue element or null
	 *		long	 opaque2;  <-- generation count
	 *	} OSQueueHead;
	 *
	 * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
	 */
PLATFUNC_FUNCTION_START(OSAtomicEnqueue, up, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%ebx	// %ebx == new
	movl	24(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	movl	%eax,(%ebx,%esi)// link to old list head from new element
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_UP	// ...push on new element
	jnz	1b
	popl	%ebx
	popl	%esi
	popl	%edi
	ret

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicEnqueue, mp, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%ebx	// %ebx == new
	movl	24(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	movl	%eax,(%ebx,%esi)// link to old list head from new element
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_MP	// ...push on new element
	jnz	1b
	popl	%ebx
	popl	%esi
	popl	%edi
	ret

/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
PLATFUNC_FUNCTION_START(OSAtomicDequeue, up, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	testl	%eax,%eax	// list empty?
	jz	2f		// yes
	movl	(%eax,%esi),%ebx // point to 2nd in Q
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_UP	// ...pop off 1st element
	jnz	1b
2:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

PLATFUNC_FUNCTION_START_GENERIC(OSAtomicDequeue, mp, 32, 2)
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == offset
	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
	movl	4(%edi),%edx	// %edx == current generation count
1:	testl	%eax,%eax	// list empty?
	jz	2f		// yes
	movl	(%eax,%esi),%ebx // point to 2nd in Q
	movl	%edx,%ecx
	incl	%ecx		// increment generation count
	xchg64	%edi, ATOMIC_MP	// ...pop off 1st element
	jnz	1b
2:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

/*
 *	typedef	volatile struct {
 *		void	*opaque1;  <-- ptr to first queue element or null
 *		void	*opaque2;  <-- ptr to last queue element or null
 *		int	 opaque3;  <-- spinlock
 *	} OSFifoQueueHead;
 *
 * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 */
	.align	2
	.globl	_OSAtomicFifoEnqueue
_OSAtomicFifoEnqueue:
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movl	16(%esp),%edi	// %edi == ptr to list head
	movl	20(%esp),%esi	// %esi == new
	EXTERN_TO_REG(_commpage_pfz_base,%ecx)
	movl	(%ecx), %ecx
	addl	$(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %ecx
	movl	24(%esp),%edx	// %edx == offset
	call	*%ecx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	call	_preempt
1:	popl	%ebx
	popl	%esi
	popl	%edi
	ret
	
/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
	.align	2
	.globl	_OSAtomicFifoDequeue
_OSAtomicFifoDequeue:
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	xorl	%ebx,%ebx	// clear "preemption pending" flag
	movl	16(%esp),%edi	// %edi == ptr to list head
	PICIFY(_commpage_pfz_base)
	movl	(%edx),%ecx
	movl	20(%esp),%edx	// %edx == offset
	addl	$(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %ecx
	call	*%ecx
	testl	%ebx,%ebx	// pending preemption?
	jz	1f
	pushl	%eax		// save return value across sysenter
	call	_preempt
	popl	%eax
1:	popl	%ebx
	popl	%esi
	popl	%edi
	ret			// ptr to 1st element in Q still in %eax

// Local Variables:
// tab-width: 8
// End:
Commit	Line	Data
3d9156a7	1	/*
224c7076 A	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
224c7076 A	3	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3d9156a7 A	4	*
	5	* @APPLE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. Please obtain a copy of the License at
	11	* http://www.opensource.apple.com/apsl/ and read it before using this
	12	* file.
	13	*
	14	* The Original Code and all software distributed under the License are
	15	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	16	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	17	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	19	* Please see the License for the specific language governing rights and
	20	* limitations under the License.
	21	*
	22	* @APPLE_LICENSE_HEADER_END@
	23	*/
	24
	25	#include <machine/cpu_capabilities.h>
1f2f436a	26	#include <platfunc.h>
ad3c9f2a	27	#include <architecture/i386/asm_help.h>
3d9156a7	28
1f2f436a A	29	#define ATOMIC_UP 0
	30	#define ATOMIC_MP 1
	31	#define ATOMIC_RET_ORIG 0
	32	#define ATOMIC_RET_NEW 1
	33
	34	// compare and exchange 32-bit
	35	// xchg32 <new> <dst> <mp>
	36	.macro xchg32
	37	.if $2 == ATOMIC_MP
	38	lock
	39	.endif
	40	cmpxchgl $0, ($1)
	41	.endm
	42
	43	// compare and exchange 64-bit
	44	// xchg64 <dst> <mp>
	45	.macro xchg64
	46	.if $1 == ATOMIC_MP
	47	lock
	48	.endif
	49	cmpxchg8b ($0)
	50	.endm
	51
	52
	53	// int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
	54	#define ATOMIC_ARITHMETIC(instr, orig, mp) \
	55	movl 8(%esp), %ecx /* load 2nd arg ptr into ecx */ ;\
	56	movl (%ecx), %eax /* load contents of ecx into eax */ ;\
	57	1: movl 4(%esp), %edx /* load 1st arg into edx */ ;\
	58	instr %eax, %edx /* do the operation */ ;\
	59	xchg32 %edx, %ecx, mp /* old in %eax, new in %edx, exchange into %ecx */ ;\
	60	jnz 1b /* go back if we failed to exchange */ ;\
	61	.if orig == ATOMIC_RET_NEW ;\
	62	movl %edx, %eax /* return new value */ ;\
	63	.endif
	64
	65	// bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
	66	#define ATOMIC_BIT_OP(instr, mp) \
	67	movl 4(%esp), %eax ;\
	68	movl 8(%esp), %edx ;\
	69	shldl $3,%edx,%ecx /* save top 3 bits of address in %ecx */ ;\
	70	shll $3,%edx ;\
	71	xorl $7,%eax /* bit position is numbered big endian so convert to little endian */ ;\
	72	addl %eax,%edx /* generate bit address */ ;\
	73	adcl $0,%ecx /* handle carry out of lower half of address */ ;\
	74	movl %edx,%eax /* copy lower half of bit address */ ;\
	75	andl $31,%eax /* keep bit offset in range 0..31 */ ;\
	76	xorl %eax,%edx /* 4-byte align address */ ;\
	77	shrdl $3,%ecx,%edx /* restore 32-bit byte address in %edx */ ;\
	78	.if mp == ATOMIC_MP ;\
	79	lock ;\
	80	.endif ;\
	81	instr %eax, (%edx) ;\
	82	setc %al ;\
	83	movzbl %al,%eax // widen in case caller assumes we return an int
	84
	85	// int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
	86	#define ATOMIC_ADD64(mp) \
	87	pushl %ebx ;\
	88	pushl %esi ;\
	89	movl 20(%esp), %esi ;\
	90	movl 0(%esi), %eax ;\
	91	movl 4(%esi), %edx ;\
	92	1: movl 12(%esp), %ebx ;\
93	movl 16(%esp), %ecx ;\
94	addl %eax, %ebx ;\
95	adcl %edx, %ecx ;\
96	xchg64 %esi, mp ;\
97	jnz 1b ;\
98	movl %ebx, %eax ;\
99	movl %ecx, %edx ;\
100	popl %esi ;\
101	popl %ebx
102
103	.text
104
105	PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 32, 2)
106	PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 32, 2)
107	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
108	ret
109
110	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 32, 2)
111	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 32, 2)
112	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
113	ret
114
115	PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 32, 2)
116	PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 32, 2)
117	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
118	ret
119
120	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 32, 2)
121	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 32, 2)
122	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
3d9156a7 A	123	ret
3d9156a7 A	124
1f2f436a A	125	PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 32, 2)
	126	PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 32, 2)
	127	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
3d9156a7 A	128	ret
3d9156a7 A	129
1f2f436a A	130	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 32, 2)
	131	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 32, 2)
	132	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
3d9156a7 A	133	ret
3d9156a7 A	134
1f2f436a A	135	PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 32, 2)
	136	PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 32, 2)
	137	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	138	ret
224c7076 A	139
1f2f436a A	140	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 32, 2)
	141	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 32, 2)
	142	ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
224c7076 A	143	ret
224c7076 A	144
1f2f436a A	145	PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 32, 2)
	146	PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 32, 2)
	147	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
224c7076 A	148	ret
224c7076 A	149
1f2f436a A	150	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 32, 2)
	151	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 32, 2)
	152	ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
	153	ret
	154
	155	PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 32, 2)
	156	PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 32, 2)
	157	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
	158	ret
	159
	160	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 32, 2)
	161	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 32, 2)
	162	ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
	163	ret
	164
	165	// bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
	166	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 32, 2)
	167	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 32, 2)
	168	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 32, 2)
	169	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 32, 2)
	170	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 32, 2)
	171	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 32, 2)
	172	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 32, 2)
	173	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 32, 2)
	174	movl 4(%esp), %eax
	175	movl 8(%esp), %edx
	176	movl 12(%esp), %ecx
	177	xchg32 %edx, %ecx, ATOMIC_UP
3d9156a7	178	sete %al
224c7076	179	movzbl %al,%eax // widen in case caller assumes we return an int
3d9156a7 A	180	ret
3d9156a7 A	181
1f2f436a A	182	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 32, 2)
	183	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 32, 2)
	184	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 32, 2)
	185	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 32, 2)
	186	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 32, 2)
	187	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 32, 2)
	188	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 32, 2)
	189	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 32, 2)
	190	movl 4(%esp), %eax
	191	movl 8(%esp), %edx
	192	movl 12(%esp), %ecx
	193	xchg32 %edx, %ecx, ATOMIC_MP
	194	sete %al
	195	movzbl %al,%eax // widen in case caller assumes we return an int
	196	ret
	197
	198	// bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
	199	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 32, 2)
	200	PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 32, 2)
	201	pushl %ebx // push out spare stuff for space
3d9156a7	202	pushl %esi
1f2f436a	203	movl 12(%esp), %eax // load in 1st 64-bit parameter
3d9156a7	204	movl 16(%esp), %edx
1f2f436a	205	movl 20(%esp), %ebx // load in 2nd 64-bit parameter
3d9156a7	206	movl 24(%esp), %ecx
1f2f436a A	207	movl 28(%esp), %esi // laod in destination address
1f2f436a A	208	xchg64 %esi, ATOMIC_UP // compare and swap 64-bit
3d9156a7	209	sete %al
224c7076	210	movzbl %al,%eax // widen in case caller assumes we return an int
3d9156a7 A	211	popl %esi
	212	popl %ebx
	213	ret
	214
1f2f436a A	215	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 32, 2)
	216	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 32, 2)
	217	pushl %ebx // push out spare stuff for space
3d9156a7	218	pushl %esi
1f2f436a A	219	movl 12(%esp), %eax // load in 1st 64-bit parameter
	220	movl 16(%esp), %edx
	221	movl 20(%esp), %ebx // load in 2nd 64-bit parameter
	222	movl 24(%esp), %ecx
	223	movl 28(%esp), %esi // laod in destination address
	224	xchg64 %esi, ATOMIC_MP // compare and swap 64-bit
	225	sete %al
	226	movzbl %al,%eax // widen in case caller assumes we return an int
3d9156a7	227	popl %esi
1f2f436a	228	popl %ebx
3d9156a7 A	229	ret
3d9156a7 A	230
1f2f436a A	231	PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 32, 2)
1f2f436a A	232	PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 32, 2)
3d9156a7 A	233	movl 4(%esp), %eax
3d9156a7 A	234	movl 8(%esp), %edx
eb1cde05	235	movl %eax, %ecx
1f2f436a A	236	xaddl %eax, (%edx)
1f2f436a A	237	addl %ecx, %eax
3d9156a7 A	238	ret
3d9156a7 A	239
1f2f436a A	240	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 32, 2)
1f2f436a A	241	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 32, 2)
3d9156a7 A	242	movl 4(%esp), %eax
3d9156a7 A	243	movl 8(%esp), %edx
eb1cde05	244	movl %eax, %ecx
1f2f436a A	245	lock
	246	xaddl %eax, (%edx)
	247	addl %ecx, %eax
3d9156a7 A	248	ret
3d9156a7 A	249
1f2f436a A	250	PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 32, 2)
	251	PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 32, 2)
	252	ATOMIC_ADD64(ATOMIC_UP)
3d9156a7 A	253	ret
3d9156a7 A	254
1f2f436a A	255	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 32, 2)
	256	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 32, 2)
	257	ATOMIC_ADD64(ATOMIC_MP)
	258	ret
224c7076	259
1f2f436a A	260	PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 32, 2)
	261	PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 32, 2)
	262	ATOMIC_BIT_OP(btsl, ATOMIC_UP)
	263	ret
	264
	265	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 32, 2)
	266	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 32, 2)
	267	ATOMIC_BIT_OP(btsl, ATOMIC_MP)
	268	ret
	269
	270	PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 32, 2)
	271	PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 32, 2)
	272	ATOMIC_BIT_OP(btrl, ATOMIC_UP)
	273	ret
	274
	275	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 32, 2)
	276	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 32, 2)
	277	ATOMIC_BIT_OP(btrl, ATOMIC_MP)
	278	ret
	279
	280	// OSMemoryBarrier()
	281	// These are used both in 32 and 64-bit mode. We use a fence even on UP
	282	// machines, so this function can be used with nontemporal stores.
	283
	284	PLATFUNC_FUNCTION_START_GENERIC(OSMemoryBarrier, all, 32, 4)
	285	lock
	286	addl $0,(%esp)
	287	ret
	288	PLATFUNC_DESCRIPTOR(OSMemoryBarrier,all,0,kHasSSE2);
	289
	290	PLATFUNC_FUNCTION_START(OSMemoryBarrier, sse2, 32, 4)
	291	mfence
	292	ret
	293	PLATFUNC_DESCRIPTOR(OSMemoryBarrier,sse2,kHasSSE2,0);
	294
	295	/*
	296	* typedef volatile struct {
	297	* void *opaque1; <-- ptr to 1st queue element or null
	298	* long opaque2; <-- generation count
	299	* } OSQueueHead;
	300	*
	301	* void OSAtomicEnqueue( OSQueueHead list, void new, size_t offset);
	302	*/
	303	PLATFUNC_FUNCTION_START(OSAtomicEnqueue, up, 32, 2)
224c7076 A	304	pushl %edi
	305	pushl %esi
	306	pushl %ebx
	307	movl 16(%esp),%edi // %edi == ptr to list head
	308	movl 20(%esp),%ebx // %ebx == new
	309	movl 24(%esp),%esi // %esi == offset
	310	movl (%edi),%eax // %eax == ptr to 1st element in Q
	311	movl 4(%edi),%edx // %edx == current generation count
1f2f436a	312	1: movl %eax,(%ebx,%esi)// link to old list head from new element
224c7076 A	313	movl %edx,%ecx
224c7076 A	314	incl %ecx // increment generation count
1f2f436a	315	xchg64 %edi, ATOMIC_UP // ...push on new element
224c7076 A	316	jnz 1b
	317	popl %ebx
	318	popl %esi
	319	popl %edi
3d9156a7	320	ret
1f2f436a A	321
	322	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicEnqueue, mp, 32, 2)
	323	pushl %edi
	324	pushl %esi
	325	pushl %ebx
	326	movl 16(%esp),%edi // %edi == ptr to list head
	327	movl 20(%esp),%ebx // %ebx == new
	328	movl 24(%esp),%esi // %esi == offset
	329	movl (%edi),%eax // %eax == ptr to 1st element in Q
	330	movl 4(%edi),%edx // %edx == current generation count
	331	1: movl %eax,(%ebx,%esi)// link to old list head from new element
	332	movl %edx,%ecx
	333	incl %ecx // increment generation count
	334	xchg64 %edi, ATOMIC_MP // ...push on new element
	335	jnz 1b
	336	popl %ebx
	337	popl %esi
	338	popl %edi
	339	ret
	340
224c7076	341	/* void* OSAtomicDequeue( OSQueueHead list, size_t offset); /
1f2f436a	342	PLATFUNC_FUNCTION_START(OSAtomicDequeue, up, 32, 2)
224c7076 A	343	pushl %edi
	344	pushl %esi
	345	pushl %ebx
	346	movl 16(%esp),%edi // %edi == ptr to list head
	347	movl 20(%esp),%esi // %esi == offset
	348	movl (%edi),%eax // %eax == ptr to 1st element in Q
	349	movl 4(%edi),%edx // %edx == current generation count
1f2f436a	350	1: testl %eax,%eax // list empty?
224c7076 A	351	jz 2f // yes
	352	movl (%eax,%esi),%ebx // point to 2nd in Q
	353	movl %edx,%ecx
	354	incl %ecx // increment generation count
1f2f436a	355	xchg64 %edi, ATOMIC_UP // ...pop off 1st element
224c7076	356	jnz 1b
1f2f436a A	357	2: popl %ebx
	358	popl %esi
	359	popl %edi
	360	ret // ptr to 1st element in Q still in %eax
	361
	362	PLATFUNC_FUNCTION_START_GENERIC(OSAtomicDequeue, mp, 32, 2)
	363	pushl %edi
	364	pushl %esi
	365	pushl %ebx
	366	movl 16(%esp),%edi // %edi == ptr to list head
	367	movl 20(%esp),%esi // %esi == offset
	368	movl (%edi),%eax // %eax == ptr to 1st element in Q
	369	movl 4(%edi),%edx // %edx == current generation count
	370	1: testl %eax,%eax // list empty?
	371	jz 2f // yes
	372	movl (%eax,%esi),%ebx // point to 2nd in Q
	373	movl %edx,%ecx
	374	incl %ecx // increment generation count
	375	xchg64 %edi, ATOMIC_MP // ...pop off 1st element
	376	jnz 1b
	377	2: popl %ebx
	378	popl %esi
	379	popl %edi
	380	ret // ptr to 1st element in Q still in %eax
	381
	382	/*
	383	* typedef volatile struct {
	384	* void *opaque1; <-- ptr to first queue element or null
	385	* void *opaque2; <-- ptr to last queue element or null
	386	* int opaque3; <-- spinlock
	387	* } OSFifoQueueHead;
	388	*
	389	* void OSAtomicFifoEnqueue( OSFifoQueueHead list, void new, size_t offset);
	390	*/
	391	.align 2
	392	.globl _OSAtomicFifoEnqueue
	393	_OSAtomicFifoEnqueue:
	394	pushl %edi
	395	pushl %esi
	396	pushl %ebx
	397	xorl %ebx,%ebx // clear "preemption pending" flag
	398	movl 16(%esp),%edi // %edi == ptr to list head
	399	movl 20(%esp),%esi // %esi == new
ad3c9f2a A	400	EXTERN_TO_REG(_commpage_pfz_base,%ecx)
	401	movl (%ecx), %ecx
	402	addl $(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %ecx
1f2f436a	403	movl 24(%esp),%edx // %edx == offset
1f2f436a A	404	call *%ecx
	405	testl %ebx,%ebx // pending preemption?
	406	jz 1f
	407	call _preempt
	408	1: popl %ebx
	409	popl %esi
	410	popl %edi
	411	ret
	412
	413	/* void* OSAtomicFifoDequeue( OSFifoQueueHead list, size_t offset); /
	414	.align 2
	415	.globl _OSAtomicFifoDequeue
	416	_OSAtomicFifoDequeue:
	417	pushl %edi
	418	pushl %esi
	419	pushl %ebx
	420	xorl %ebx,%ebx // clear "preemption pending" flag
	421	movl 16(%esp),%edi // %edi == ptr to list head
ad3c9f2a A	422	PICIFY(_commpage_pfz_base)
ad3c9f2a A	423	movl (%edx),%ecx
1f2f436a	424	movl 20(%esp),%edx // %edx == offset
ad3c9f2a	425	addl $(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %ecx
1f2f436a A	426	call *%ecx
	427	testl %ebx,%ebx // pending preemption?
	428	jz 1f
	429	pushl %eax // save return value across sysenter
	430	call _preempt
	431	popl %eax
	432	1: popl %ebx
224c7076 A	433	popl %esi
	434	popl %edi
	435	ret // ptr to 1st element in Q still in %eax
	436
1f2f436a A	437	// Local Variables:
	438	// tab-width: 8
	439	// End: