/*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
*/
#include <libkern/OSAtomic.h>
+#include <kern/debug.h>
+#include <machine/atomic.h>
enum {
false = 0,
#define NULL ((void *)0)
#endif
+#define ATOMIC_DEBUG DEBUG
+
+#if ATOMIC_DEBUG
+#define ALIGN_TEST(p,t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
+#else
+#define ALIGN_TEST(p,t) do{}while(0)
+#endif
+
+// 19831745 - start of big hammer!
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+
/*
* atomic operations
- * these are _the_ atomic operations, currently cast atop CompareAndSwap,
- * which is implemented in assembler. if we are worried about the cost of
- * this layering (we shouldn't be), then all this stuff could be
- * implemented in assembler, as it is in MacOS8/9
- * (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
- * which I wrote for NuKernel in a previous life with a different last name...)
- *
- * native Boolean CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
- *
- * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
- * OSEnqueueAtomic etc -- in assembler, either for speed or correctness. See also the
- * commpage atomic operations, and the platform specific versions.
- * Like standards, there are a lot of atomic ops to choose from!
+ * These are _the_ atomic operations, now implemented via compiler built-ins.
+ * It is expected that this C implementation is a candidate for Link-Time-
+ * Optimization inlining, whereas the assembler implementations they replace
+ * were not.
*/
-#ifndef __ppc__
+#undef OSCompareAndSwap8
+Boolean OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
+{
+ return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue,
+ memory_order_acq_rel_smp, memory_order_relaxed);
+}
+#undef OSCompareAndSwap16
+Boolean OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
+{
+ return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue,
+ memory_order_acq_rel_smp, memory_order_relaxed);
+}
-SInt32 OSIncrementAtomic(volatile SInt32 * value)
+#undef OSCompareAndSwap
+Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
{
- return OSAddAtomic(1, value);
+ ALIGN_TEST(address, UInt32);
+ return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue,
+ memory_order_acq_rel_smp, memory_order_relaxed);
}
-SInt32 OSDecrementAtomic(volatile SInt32 * value)
+#undef OSCompareAndSwap64
+Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
{
- return OSAddAtomic(-1, value);
+ /*
+ * _Atomic uint64 requires 8-byte alignment on all architectures.
+ * This silences the compiler cast warning. ALIGN_TEST() verifies
+ * that the cast was legal, if defined.
+ */
+ _Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address;
+
+ ALIGN_TEST(address, UInt64);
+ return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue,
+ memory_order_acq_rel_smp, memory_order_relaxed);
}
-#ifdef CMPXCHG8B
-void * OSDequeueAtomic(void * volatile * inList, SInt32 inOffset)
+#undef OSCompareAndSwapPtr
+Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
{
- /* The _pointer_ is volatile, not the listhead itself */
- void * volatile oldListHead;
- void * volatile newListHead;
-
- do {
- oldListHead = *inList;
- if (oldListHead == NULL) {
- break;
- }
-
- newListHead = *(void * volatile *) (((char *) oldListHead) + inOffset);
- } while (! OSCompareAndSwap((UInt32)oldListHead,
- (UInt32)newListHead, (volatile UInt32 *)inList));
- return oldListHead;
+#if __LP64__
+ return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address);
+#else
+ return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address);
+#endif
}
-void OSEnqueueAtomic(void * volatile * inList, void * inNewLink, SInt32 inOffset)
+SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
{
- /* The _pointer_ is volatile, not the listhead itself */
- void * volatile oldListHead;
- void * volatile newListHead = inNewLink;
- void * volatile * newLinkNextPtr = (void * volatile *) (((char *) inNewLink) + inOffset);
-
- do {
- oldListHead = *inList;
- *newLinkNextPtr = oldListHead;
- } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
- (volatile UInt32 *)inList));
+ return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed);
}
-#endif /* CMPXCHG8B */
-#endif /* !__ppc__ */
-static UInt32 OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt32 * value)
+SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
{
- UInt32 oldValue;
- UInt32 newValue;
-
- do {
- oldValue = *value;
- newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
- } while (! OSCompareAndSwap(oldValue, newValue, value));
-
- return oldValue;
+ return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed);
}
-UInt32 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
+#undef OSAddAtomic
+SInt32 OSAddAtomic(SInt32 amount, volatile SInt32 *address)
{
- return OSBitwiseAtomic(mask, 0, 0, value);
+ ALIGN_TEST(address, UInt32);
+ return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed);
}
-UInt32 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
+#undef OSAddAtomic64
+SInt64 OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
{
- return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
+ _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
+
+ ALIGN_TEST(address, SInt64);
+ return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed);
}
-UInt32 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
+#undef OSAddAtomicLong
+long
+OSAddAtomicLong(long theAmount, volatile long *address)
{
- return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
+#ifdef __LP64__
+ return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address);
+#else
+ return (long)OSAddAtomic((SInt32)theAmount, address);
+#endif
}
-static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, volatile UInt8 * value8)
+#undef OSIncrementAtomic
+SInt32 OSIncrementAtomic(volatile SInt32 * value)
{
- UInt32 mask = 0x000000ff;
- UInt32 alignment = ((UInt32) value8) & (sizeof(UInt32) - 1);
- UInt32 shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
- int shift = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
- volatile UInt32 * value32 = (volatile UInt32 *) (value8 - alignment);
- UInt32 oldValue;
- UInt32 newValue;
+ return OSAddAtomic(1, value);
+}
- mask <<= shift;
+#undef OSDecrementAtomic
+SInt32 OSDecrementAtomic(volatile SInt32 * value)
+{
+ return OSAddAtomic(-1, value);
+}
+
+#undef OSBitAndAtomic
+UInt32 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
+{
+ return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed);
+}
- oldValue = *value32;
- oldValue = (oldValue & ~mask) | (oldValue8 << shift);
- newValue = (oldValue & ~mask) | (newValue8 << shift);
+#undef OSBitOrAtomic
+UInt32 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
+{
+ return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed);
+}
- return OSCompareAndSwap(oldValue, newValue, value32);
+#undef OSBitXorAtomic
+UInt32 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
+{
+ return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed);
}
static Boolean OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
if ((oldValue & mask) == wantValue) {
break;
}
- } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
+ } while (! __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress,
+ &oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed));
return (oldValue & mask) == wantValue;
}
return OSAddAtomic8(-1, value);
}
-SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 * value)
-{
- SInt8 oldValue;
- SInt8 newValue;
-
- do {
- oldValue = *value;
- newValue = oldValue + amount;
- } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (volatile UInt8 *) value));
-
- return oldValue;
-}
-
-static UInt8 OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt8 * value)
-{
- UInt8 oldValue;
- UInt8 newValue;
-
- do {
- oldValue = *value;
- newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
- } while (! OSCompareAndSwap8(oldValue, newValue, value));
-
- return oldValue;
-}
-
UInt8 OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
{
- return OSBitwiseAtomic8(mask, 0, 0, value);
+ return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed);
}
UInt8 OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
{
- return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
+ return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed);
}
UInt8 OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
{
- return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
-}
-
-static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, volatile UInt16 * value16)
-{
- UInt32 mask = 0x0000ffff;
- UInt32 alignment = ((UInt32) value16) & (sizeof(UInt32) - 1);
- UInt32 shiftValues = (16 << 24) | (16 << 16);
- UInt32 shift = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
- volatile UInt32 * value32 = (volatile UInt32 *) (((UInt32) value16) - alignment);
- UInt32 oldValue;
- UInt32 newValue;
-
- mask <<= shift;
-
- oldValue = *value32;
- oldValue = (oldValue & ~mask) | (oldValue16 << shift);
- newValue = (oldValue & ~mask) | (newValue16 << shift);
-
- return OSCompareAndSwap(oldValue, newValue, value32);
+ return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed);
}
SInt16 OSIncrementAtomic16(volatile SInt16 * value)
return OSAddAtomic16(-1, value);
}
-SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 * value)
-{
- SInt16 oldValue;
- SInt16 newValue;
-
- do {
- oldValue = *value;
- newValue = oldValue + amount;
- } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (volatile UInt16 *) value));
-
- return oldValue;
-}
-
-static UInt16 OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt16 * value)
-{
- UInt16 oldValue;
- UInt16 newValue;
-
- do {
- oldValue = *value;
- newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
- } while (! OSCompareAndSwap16(oldValue, newValue, value));
-
- return oldValue;
-}
-
UInt16 OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
{
- return OSBitwiseAtomic16(mask, 0, 0, value);
+ return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed);
}
UInt16 OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
{
- return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
+ return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed);
}
UInt16 OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
{
- return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
+ return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed);
}
+// 19831745 - end of big hammer!
+#pragma clang diagnostic pop
+
projects / apple / xnu.git / blobdiff