*/
#include <machine/cpu_capabilities.h>
+#include <platfunc.h>
#define DECLARE(x) \
-.align 2, 0x90 ; \
-.globl x ; \
-.globl x ## Barrier ; \
+ .align 2, 0x90 ; \
+ .globl x ; \
+ .globl x ## Barrier ; \
x: ; \
x ## Barrier:
.text
+#define ATOMIC_UP 0
+#define ATOMIC_MP 1
+#define ATOMIC_RET_ORIG 0
+#define ATOMIC_RET_NEW 1
+
+// compare and exchange 32-bit
+// xchg32 <new> <dst> <mp>
+.macro xchg32
+ .if $2 == ATOMIC_MP
+ lock
+ .endif
+ cmpxchgl $0, ($1)
+.endm
+
+// xchg64 <new> <dst> <mp>
+.macro xchg64
+ .if $2 == ATOMIC_MP
+ lock
+ .endif
+ cmpxchg $0, ($1)
+.endm
+
+#define ATOMIC_ARITHMETIC(instr, orig, mp) \
+ movl (%rsi), %eax /* get 2nd arg -> eax */ ;\
+1: movl %eax, %edx /* copy value to new reg */ ;\
+ instr %edi, %edx /* apply instr to %edx with arg2 */ ;\
+ xchg32 %edx, %rsi, mp /* do the compare swap (see macro above) */ ;\
+ jnz 1b /* jump if failed */ ;\
+ .if orig == 1 /* to return the new value, overwrite eax */ ;\
+ movl %edx, %eax /* return the new value */ ;\
+ .endif
+
+// Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
+// Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
+#define ATOMIC_BIT_OP(instr, mp) \
+ xorl $7, %edi /* bit position is numbered big endian so convert to little endian */ ;\
+ shlq $3, %rsi ;\
+ addq %rdi, %rsi /* generate bit address */ ;\
+ movq %rsi, %rdi ;\
+ andq $31, %rdi /* keep bit offset in range 0..31 */ ;\
+ xorq %rdi, %rsi /* 4-byte align address */ ;\
+ shrq $3, %rsi /* get 4-byte aligned address */ ;\
+ .if mp == ATOMIC_MP /* don't plant the lock in UP code */ ;\
+ lock /* lock the bit test */ ;\
+ .endif ;\
+ instr %edi, (%rsi) /* do the bit test, supplied into the macro */ ;\
+ setc %al ;\
+ movzbl %al,%eax /* widen in case caller assumes we return an int */
// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
-DECLARE(_OSAtomicAnd32)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- movl %edi, %r11d // save mask
- movl (%rsi), %eax // get value
- movq %rsi, %rdx // put ptr where compare-and-swap expects it
-1:
- movl %r11d, %esi // original mask
- movl %eax, %edi // old value
- andl %eax, %esi // new value
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
- jnz 1b
- movl %esi, %eax
+PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
+ ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
+ ret
+
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
+ ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
ret
-
// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
-DECLARE(_OSAtomicOr32)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- movl %edi, %r11d // save mask
- movl (%rsi), %eax // get value
- movq %rsi, %rdx // put ptr where compare-and-swap expects it
-1:
- movl %r11d, %esi // original mask
- movl %eax, %edi // old value
- orl %eax, %esi // new value
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
- jnz 1b
- movl %esi, %eax
+PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
+ ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
+ ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
+ ret
// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
-DECLARE(_OSAtomicXor32)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- movl %edi, %r11d // save mask
- movl (%rsi), %eax // get value
- movq %rsi, %rdx // put ptr where compare-and-swap expects it
-1:
- movl %r11d, %esi // original mask
- movl %eax, %edi // old value
- xorl %eax, %esi // new value
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
- jnz 1b
- movl %esi, %eax
+PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
+ ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
+ ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
+ ret
// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
-DECLARE(_OSAtomicAnd32Orig)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- movl %edi, %r11d // save mask
- movl (%rsi), %eax // get value
- movq %rsi, %rdx // put ptr where compare-and-swap expects it
-1:
- movl %r11d, %esi // original mask
- movl %eax, %edi // old value
- andl %eax, %esi // new value
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
- jnz 1b
- movl %edi, %eax
+PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
+ ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
ret
-
-// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
-DECLARE(_OSAtomicOr32Orig)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- movl %edi, %r11d // save mask
- movl (%rsi), %eax // get value
- movq %rsi, %rdx // put ptr where compare-and-swap expects it
-1:
- movl %r11d, %esi // original mask
- movl %eax, %edi // old value
- orl %eax, %esi // new value
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
- jnz 1b
- movl %edi, %eax
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
+ ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
ret
+// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
+PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
+ ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
+ ret
+
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
+ ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
+ ret
// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
-DECLARE(_OSAtomicXor32Orig)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- movl %edi, %r11d // save mask
- movl (%rsi), %eax // get value
- movq %rsi, %rdx // put ptr where compare-and-swap expects it
-1:
- movl %r11d, %esi // original mask
- movl %eax, %edi // old value
- xorl %eax, %esi // new value
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
- jnz 1b
- movl %edi, %eax
+PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
+ ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
+ ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
+ ret
// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
-DECLARE(_OSAtomicCompareAndSwapInt)
-DECLARE(_OSAtomicCompareAndSwap32)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
- call *%rcx // %edi=old value, %esi=new value. %rdx=ptr
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
+ movl %edi, %eax
+ xchg32 %esi, %rdx, ATOMIC_UP
sete %al
movzbl %al,%eax // widen in case caller assumes we return an int
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
+ movl %edi, %eax
+ xchg32 %esi, %rdx, ATOMIC_MP
+ sete %al
+ movzbl %al,%eax // widen in case caller assumes we return an int
+ ret
// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
-DECLARE(_OSAtomicCompareAndSwapPtr)
-DECLARE(_OSAtomicCompareAndSwapLong)
-DECLARE(_OSAtomicCompareAndSwap64)
- movq $(_COMM_PAGE_COMPARE_AND_SWAP64), %rcx
- call *%rcx // %rdi=old value, %rsi=new value. %rdx=ptr
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
+ mov %rdi, %rax
+ xchg64 %rsi, %rdx, ATOMIC_UP
+ sete %al
+ movzbl %al,%eax // widen in case caller assumes we return an int
+ ret
+
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
+ mov %rdi, %rax
+ xchg64 %rsi, %rdx, ATOMIC_MP
sete %al
movzbl %al,%eax // widen in case caller assumes we return an int
ret
-
// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
-DECLARE(_OSAtomicAdd32)
- movq $(_COMM_PAGE_ATOMIC_ADD32), %rcx
+PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
movl %edi, %eax // save amt to add
- call *%rcx
- addl %edi,%eax // new value
+ xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
+ addl %edi, %eax // add old value to amt as return value
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
+ movl %edi, %eax // save amt to add
+ lock // lock prefix breaks tabs ;)
+ xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
+ addl %edi, %eax // add old value to amt as return value
+ ret
// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
-DECLARE(_OSAtomicAdd64)
- movq $(_COMM_PAGE_ATOMIC_ADD64), %rcx
+PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
movq %rdi, %rax // save amt to add
- call *%rcx
- addq %rdi, %rax // new value
+ xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
+ addq %rdi, %rax // add old value to amt as return value
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
+ movq %rdi, %rax // save amt to add
+ lock
+ xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
+ addq %rdi, %rax // add old value to amt as return value
+ ret
// bool OSAtomicTestAndSet( uint32_t n, void *value );
-DECLARE(_OSAtomicTestAndSet)
- movq $(_COMM_PAGE_BTS), %rax
- xorl $7, %edi // bit position is numbered big endian
- call *%rax
- setc %al
- movzbl %al,%eax // widen in case caller assumes we return an int
+PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
+ ATOMIC_BIT_OP(btsl, ATOMIC_UP)
ret
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
+ ATOMIC_BIT_OP(btsl, ATOMIC_MP)
+ ret
// bool OSAtomicTestAndClear( uint32_t n, void *value );
-DECLARE(_OSAtomicTestAndClear)
- movq $(_COMM_PAGE_BTC), %rax
- xorl $7, %edi // bit position is numbered big endian
- call *%rax
- setc %al
- movzbl %al,%eax // widen in case caller assumes we return an int
+PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 64, 2)
+PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
+ ATOMIC_BIT_OP(btrl, ATOMIC_UP)
ret
-// bool OSSpinLockTry( OSSpinLock *lock );
- .align 2, 0x90
- .globl _OSSpinLockTry
- .globl __spin_lock_try
-_OSSpinLockTry:
-__spin_lock_try:
- movq $(_COMM_PAGE_SPINLOCK_TRY), %rax
- jmp *%rax
-
-
-// void OSSpinLockLock( OSSpinLock *lock );
- .align 2, 0x90
- .globl _OSSpinLockLock
- .globl _spin_lock
- .globl __spin_lock
-_OSSpinLockLock:
-_spin_lock:
-__spin_lock:
- movq $(_COMM_PAGE_SPINLOCK_LOCK), %rax
- jmp *%rax
-
-
-// void OSSpinLockUnlock( OSSpinLock *lock );
- .align 2, 0x90
- .globl _OSSpinLockUnlock
- .globl _spin_unlock
- .globl __spin_unlock
-_OSSpinLockUnlock:
-_spin_unlock:
-__spin_unlock:
- movl $0, (%rdi)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
+PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
+ ATOMIC_BIT_OP(btrl, ATOMIC_MP)
ret
-
// void OSMemoryBarrier( void );
.align 2, 0x90
.globl _OSMemoryBarrier
_OSMemoryBarrier:
- movq $(_COMM_PAGE_MEMORY_BARRIER), %rax
- jmp *%rax
-
+ mfence
+ ret
/*
* typedef volatile struct {
jnz 1b
popq %rbx
ret
-
-
-/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
+
+
+ /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
.align 2
.globl _OSAtomicDequeue
_OSAtomicDequeue: // %rdi == list head, %rsi == offset
2:
popq %rbx
ret // ptr to 1st element in Q still in %rax
+
+/*
+ * typedef volatile struct {
+ * void *opaque1; <-- ptr to first queue element or null
+ * void *opaque2; <-- ptr to last queue element or null
+ * int opaque3; <-- spinlock
+ * } OSFifoQueueHead;
+ *
+ * void OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
+ */
+ .align 2
+ .globl _OSAtomicFifoEnqueue
+_OSAtomicFifoEnqueue:
+ pushq %rbx
+ xorl %ebx,%ebx // clear "preemption pending" flag
+ movq $(_COMM_PAGE_PFZ_ENQUEUE), %rcx
+ call *%rcx
+ testl %ebx,%ebx // pending preemption?
+ jz 1f
+ call _preempt // call into the kernel to pfz_exit
+1:
+ popq %rbx
+ ret
+
+
+/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
+ .align 2
+ .globl _OSAtomicFifoDequeue
+_OSAtomicFifoDequeue:
+ pushq %rbx
+ xorl %ebx,%ebx // clear "preemption pending" flag
+ movq %rsi,%rdx // move offset to %rdx to be like the Enqueue case
+ movq $(_COMM_PAGE_PFZ_DEQUEUE), %rcx
+ call *%rcx
+ testl %ebx,%ebx // pending preemption?
+ jz 1f
+ call _preempt // call into the kernel to pfz_exit
+1:
+ popq %rbx
+ ret // ptr to 1st element in Q in %rax
+
+// Local Variables:
+// tab-width: 8
+// End:
projects / apple / libc.git / blobdiff