X-Git-Url: https://git.saurik.com/apple/libc.git/blobdiff_plain/3d9156a7a519a5e3aa1b92e9d9d4b991f1aed7ff..fbd86d4cc20b02a10edcca92fb7ae0a143e63cc4:/include/libkern/OSAtomic.h diff --git a/include/libkern/OSAtomic.h b/include/libkern/OSAtomic.h index 83be1e3..9f2dba4 100644 --- a/include/libkern/OSAtomic.h +++ b/include/libkern/OSAtomic.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -42,14 +42,15 @@ * architecture such as PPC. All loads and stores executed in sequential program * order before the barrier will complete before any load or store executed after * the barrier. On a uniprocessor, the barrier operation is typically a nop. - * On a multiprocessor, the barrier can be quite expensive. + * On a multiprocessor, the barrier can be quite expensive on some platforms, + * eg PPC. * * Most code will want to use the barrier functions to insure that memory shared * between threads is properly synchronized. For example, if you want to initialize * a shared data structure and then atomically increment a variable to indicate - * that the initialization is complete, then you MUST use OSAtomicIncrement32Barrier() + * that the initialization is complete, then you must use OSAtomicIncrement32Barrier() * to ensure that the stores to your data structure complete before the atomic add. - * Likewise, the consumer of that data structure MUST use OSAtomicDecrement32Barrier(), + * Likewise, the consumer of that data structure must use OSAtomicDecrement32Barrier(), * in order to ensure that their loads of the structure are not executed before * the atomic decrement. On the other hand, if you are simply incrementing a global * counter, then it is safe and potentially faster to use OSAtomicIncrement32(). @@ -62,79 +63,98 @@ __BEGIN_DECLS -/* Arithmetic functions. They return the new value. All the "or", "and", and "xor" - * operations, and the barrier forms of add, are layered on top of compare-and-swap. +/* Arithmetic functions. They return the new value. */ -int32_t OSAtomicAdd32( int32_t theAmount, int32_t *theValue ); -int32_t OSAtomicAdd32Barrier( int32_t theAmount, int32_t *theValue ); - -inline static -int32_t OSAtomicIncrement32( int32_t *theValue ) - { return OSAtomicAdd32( 1, theValue); } -inline static -int32_t OSAtomicIncrement32Barrier( int32_t *theValue ) - { return OSAtomicAdd32Barrier( 1, theValue); } - -inline static -int32_t OSAtomicDecrement32( int32_t *theValue ) - { return OSAtomicAdd32( -1, theValue); } -inline static -int32_t OSAtomicDecrement32Barrier( int32_t *theValue ) - { return OSAtomicAdd32Barrier( -1, theValue); } - -int32_t OSAtomicOr32( uint32_t theMask, uint32_t *theValue ); -int32_t OSAtomicOr32Barrier( uint32_t theMask, uint32_t *theValue ); - -int32_t OSAtomicAnd32( uint32_t theMask, uint32_t *theValue ); -int32_t OSAtomicAnd32Barrier( uint32_t theMask, uint32_t *theValue ); - -int32_t OSAtomicXor32( uint32_t theMask, uint32_t *theValue ); -int32_t OSAtomicXor32Barrier( uint32_t theMask, uint32_t *theValue ); - -#if defined(__ppc64__) || defined(__i386__) - -int64_t OSAtomicAdd64( int64_t theAmount, int64_t *theValue ); -int64_t OSAtomicAdd64Barrier( int64_t theAmount, int64_t *theValue ); - -inline static -int64_t OSAtomicIncrement64( int64_t *theValue ) - { return OSAtomicAdd64( 1, theValue); } -inline static -int64_t OSAtomicIncrement64Barrier( int64_t *theValue ) - { return OSAtomicAdd64Barrier( 1, theValue); } - -inline static -int64_t OSAtomicDecrement64( int64_t *theValue ) - { return OSAtomicAdd64( -1, theValue); } -inline static -int64_t OSAtomicDecrement64Barrier( int64_t *theValue ) - { return OSAtomicAdd64Barrier( -1, theValue); } - -#endif /* defined(__ppc64__) || defined(__i386__) */ - +int32_t OSAtomicAdd32( int32_t __theAmount, volatile int32_t *__theValue ); +int32_t OSAtomicAdd32Barrier( int32_t __theAmount, volatile int32_t *__theValue ); + +__inline static +int32_t OSAtomicIncrement32( volatile int32_t *__theValue ) + { return OSAtomicAdd32( 1, __theValue); } +__inline static +int32_t OSAtomicIncrement32Barrier( volatile int32_t *__theValue ) + { return OSAtomicAdd32Barrier( 1, __theValue); } + +__inline static +int32_t OSAtomicDecrement32( volatile int32_t *__theValue ) + { return OSAtomicAdd32( -1, __theValue); } +__inline static +int32_t OSAtomicDecrement32Barrier( volatile int32_t *__theValue ) + { return OSAtomicAdd32Barrier( -1, __theValue); } + +#if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) + +int64_t OSAtomicAdd64( int64_t __theAmount, volatile int64_t *__theValue ); +int64_t OSAtomicAdd64Barrier( int64_t __theAmount, volatile int64_t *__theValue ); + +__inline static +int64_t OSAtomicIncrement64( volatile int64_t *__theValue ) + { return OSAtomicAdd64( 1, __theValue); } +__inline static +int64_t OSAtomicIncrement64Barrier( volatile int64_t *__theValue ) + { return OSAtomicAdd64Barrier( 1, __theValue); } + +__inline static +int64_t OSAtomicDecrement64( volatile int64_t *__theValue ) + { return OSAtomicAdd64( -1, __theValue); } +__inline static +int64_t OSAtomicDecrement64Barrier( volatile int64_t *__theValue ) + { return OSAtomicAdd64Barrier( -1, __theValue); } + +#endif /* defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) */ + + +/* Boolean functions (and, or, xor.) These come in four versions for each operation: + * with and without barriers, and returning the old or new value of the operation. + * The "Orig" versions return the original value, ie before the operation, the non-Orig + * versions return the value after the operation. All are layered on top of + * compare-and-swap. + */ +int32_t OSAtomicOr32( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicOr32Barrier( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicOr32Orig( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicOr32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue ); + +int32_t OSAtomicAnd32( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicAnd32Barrier( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicAnd32Orig( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicAnd32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue ); + +int32_t OSAtomicXor32( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicXor32Barrier( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicXor32Orig( uint32_t __theMask, volatile uint32_t *__theValue ); +int32_t OSAtomicXor32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue ); + -/* Compare and swap. They return true if the swap occured. +/* Compare and swap. They return true if the swap occured. There are several versions, + * depending on data type and whether or not a barrier is used. */ -bool OSAtomicCompareAndSwap32( int32_t oldValue, int32_t newValue, int32_t *theValue ); -bool OSAtomicCompareAndSwap32Barrier( int32_t oldValue, int32_t newValue, int32_t *theValue ); +bool OSAtomicCompareAndSwap32( int32_t __oldValue, int32_t __newValue, volatile int32_t *__theValue ); +bool OSAtomicCompareAndSwap32Barrier( int32_t __oldValue, int32_t __newValue, volatile int32_t *__theValue ); +bool OSAtomicCompareAndSwapPtr( void *__oldValue, void *__newValue, void * volatile *__theValue ); +bool OSAtomicCompareAndSwapPtrBarrier( void *__oldValue, void *__newValue, void * volatile *__theValue ); +bool OSAtomicCompareAndSwapInt( int __oldValue, int __newValue, volatile int *__theValue ); +bool OSAtomicCompareAndSwapIntBarrier( int __oldValue, int __newValue, volatile int *__theValue ); +bool OSAtomicCompareAndSwapLong( long __oldValue, long __newValue, volatile long *__theValue ); +bool OSAtomicCompareAndSwapLongBarrier( long __oldValue, long __newValue, volatile long *__theValue ); -#if defined(__ppc64__) || defined(__i386__) +#if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) -bool OSAtomicCompareAndSwap64( int64_t oldValue, int64_t newValue, int64_t *theValue ); -bool OSAtomicCompareAndSwap64Barrier( int64_t oldValue, int64_t newValue, int64_t *theValue ); +bool OSAtomicCompareAndSwap64( int64_t __oldValue, int64_t __newValue, volatile int64_t *__theValue ); +bool OSAtomicCompareAndSwap64Barrier( int64_t __oldValue, int64_t __newValue, volatile int64_t *__theValue ); -#endif /* defined(__ppc64__) || defined(__i386__) */ +#endif /* defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) */ /* Test and set. They return the original value of the bit, and operate on bit (0x80>>(n&7)) - * in byte ((char*)theAddress + (n>>3)). They are layered on top of the compare-and-swap - * operation. + * in byte ((char*)theAddress + (n>>3)). */ -bool OSAtomicTestAndSet( uint32_t n, void *theAddress ); -bool OSAtomicTestAndSetBarrier( uint32_t n, void *theAddress ); -bool OSAtomicTestAndClear( uint32_t n, void *theAddress ); -bool OSAtomicTestAndClearBarrier( uint32_t n, void *theAddress ); +bool OSAtomicTestAndSet( uint32_t __n, volatile void *__theAddress ); +bool OSAtomicTestAndSetBarrier( uint32_t __n, volatile void *__theAddress ); +bool OSAtomicTestAndClear( uint32_t __n, volatile void *__theAddress ); +bool OSAtomicTestAndClearBarrier( uint32_t __n, volatile void *__theAddress ); + /* Spinlocks. These use memory barriers as required to synchronize access to shared * memory protected by the lock. The lock operation spins, but employs various strategies * to back off if the lock is held, making it immune to most priority-inversion livelocks. @@ -143,17 +163,47 @@ bool OSAtomicTestAndClearBarrier( uint32_t n, void *theAddress ); */ #define OS_SPINLOCK_INIT 0 -typedef int32_t OSSpinLock; +typedef int32_t OSSpinLock; + +bool OSSpinLockTry( volatile OSSpinLock *__lock ); +void OSSpinLockLock( volatile OSSpinLock *__lock ); +void OSSpinLockUnlock( volatile OSSpinLock *__lock ); + + +/* Lockless atomic enqueue and dequeue. These routines manipulate singly + * linked LIFO lists. Ie, a dequeue will return the most recently enqueued + * element, or NULL if the list is empty. The "offset" parameter is the offset + * in bytes of the link field within the data structure being queued. The + * link field should be a pointer type. Memory barriers are incorporated as + * needed to permit thread-safe access to the queue element. + */ +#if defined(__x86_64__) + +typedef volatile struct { + void *opaque1; + long opaque2; +} OSQueueHead __attribute__ ((aligned (16))); + +#else + +typedef volatile struct { + void *opaque1; + long opaque2; +} OSQueueHead; -bool OSSpinLockTry( OSSpinLock *lock ); -void OSSpinLockLock( OSSpinLock *lock ); -void OSSpinLockUnlock( OSSpinLock *lock ); +#endif + +#define OS_ATOMIC_QUEUE_INIT { NULL, 0 } + +void OSAtomicEnqueue( OSQueueHead *__list, void *__new, size_t __offset); +void* OSAtomicDequeue( OSQueueHead *__list, size_t __offset); /* Memory barrier. It is both a read and write barrier. */ void OSMemoryBarrier( void ); + __END_DECLS #endif /* _OSATOMIC_H_ */