* Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
+ *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
- *
+ *
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
+ *
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#include <machine/atomic.h>
enum {
- false = 0,
- true = 1
+ false = 0,
+ true = 1
};
#ifndef NULL
#define ATOMIC_DEBUG DEBUG
#if ATOMIC_DEBUG
-#define ALIGN_TEST(p,t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
+#define ALIGN_TEST(p, t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
#else
-#define ALIGN_TEST(p,t) do{}while(0)
+#define ALIGN_TEST(p, t) do{}while(0)
#endif
-// 19831745 - start of big hammer!
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wcast-qual"
-
/*
* atomic operations
* These are _the_ atomic operations, now implemented via compiler built-ins.
*/
#undef OSCompareAndSwap8
-Boolean OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
+Boolean
+OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
{
- return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue,
- memory_order_acq_rel_smp, memory_order_relaxed);
+ return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel);
}
#undef OSCompareAndSwap16
-Boolean OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
+Boolean
+OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
{
- return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue,
- memory_order_acq_rel_smp, memory_order_relaxed);
+ return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel);
}
#undef OSCompareAndSwap
-Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
+Boolean
+OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
{
ALIGN_TEST(address, UInt32);
- return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue,
- memory_order_acq_rel_smp, memory_order_relaxed);
+ return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel);
}
#undef OSCompareAndSwap64
-Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
+Boolean
+OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
{
/*
* _Atomic uint64 requires 8-byte alignment on all architectures.
_Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address;
ALIGN_TEST(address, UInt64);
- return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue,
- memory_order_acq_rel_smp, memory_order_relaxed);
+ return os_atomic_cmpxchg(aligned_addr, oldValue, newValue, acq_rel);
}
#undef OSCompareAndSwapPtr
-Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
+Boolean
+OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
{
-#if __LP64__
- return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address);
-#else
- return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address);
-#endif
+ return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel);
}
-SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
+SInt8
+OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
{
- return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed);
+ return os_atomic_add_orig(address, amount, relaxed);
}
-SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
+SInt16
+OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
{
- return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed);
+ return os_atomic_add_orig(address, amount, relaxed);
}
#undef OSAddAtomic
-SInt32 OSAddAtomic(SInt32 amount, volatile SInt32 *address)
+SInt32
+OSAddAtomic(SInt32 amount, volatile SInt32 *address)
{
ALIGN_TEST(address, UInt32);
- return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed);
+ return os_atomic_add_orig(address, amount, relaxed);
}
#undef OSAddAtomic64
-SInt64 OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
+SInt64
+OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
{
- _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
+ _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
ALIGN_TEST(address, SInt64);
- return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed);
+ return os_atomic_add_orig(aligned_address, amount, relaxed);
}
#undef OSAddAtomicLong
long
OSAddAtomicLong(long theAmount, volatile long *address)
{
-#ifdef __LP64__
- return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address);
-#else
- return (long)OSAddAtomic((SInt32)theAmount, address);
-#endif
+ return os_atomic_add_orig(address, theAmount, relaxed);
}
#undef OSIncrementAtomic
-SInt32 OSIncrementAtomic(volatile SInt32 * value)
+SInt32
+OSIncrementAtomic(volatile SInt32 * value)
{
- return OSAddAtomic(1, value);
+ return os_atomic_inc_orig(value, relaxed);
}
#undef OSDecrementAtomic
-SInt32 OSDecrementAtomic(volatile SInt32 * value)
+SInt32
+OSDecrementAtomic(volatile SInt32 * value)
{
- return OSAddAtomic(-1, value);
+ return os_atomic_dec_orig(value, relaxed);
}
#undef OSBitAndAtomic
-UInt32 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
+UInt32
+OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
{
- return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed);
+ return os_atomic_and_orig(value, mask, relaxed);
}
#undef OSBitOrAtomic
-UInt32 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
+UInt32
+OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
{
- return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed);
+ return os_atomic_or_orig(value, mask, relaxed);
}
#undef OSBitXorAtomic
-UInt32 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
+UInt32
+OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
{
- return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed);
+ return os_atomic_xor_orig(value, mask, relaxed);
}
-static Boolean OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
+static Boolean
+OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
{
- UInt8 mask = 1;
- UInt8 oldValue;
- UInt8 wantValue;
-
- startAddress += (bit / 8);
+ UInt8 mask = 1;
+ UInt8 oldValue, newValue;
+ UInt8 wantValue;
+ UInt8 *address;
+
+ address = (UInt8 *)(uintptr_t)(startAddress + (bit / 8));
mask <<= (7 - (bit % 8));
wantValue = wantSet ? mask : 0;
-
- do {
- oldValue = *startAddress;
+
+ return !os_atomic_rmw_loop(address, oldValue, newValue, relaxed, {
if ((oldValue & mask) == wantValue) {
- break;
+ os_atomic_rmw_loop_give_up(break);
}
- } while (! __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress,
- &oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed));
-
- return (oldValue & mask) == wantValue;
+ newValue = (oldValue & ~mask) | wantValue;
+ });
}
-Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
+Boolean
+OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
{
return OSTestAndSetClear(bit, true, startAddress);
}
-Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
+Boolean
+OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
{
return OSTestAndSetClear(bit, false, startAddress);
}
* silly unaligned versions
*/
-SInt8 OSIncrementAtomic8(volatile SInt8 * value)
+SInt8
+OSIncrementAtomic8(volatile SInt8 * value)
{
- return OSAddAtomic8(1, value);
+ return os_atomic_inc_orig(value, relaxed);
}
-SInt8 OSDecrementAtomic8(volatile SInt8 * value)
+SInt8
+OSDecrementAtomic8(volatile SInt8 * value)
{
- return OSAddAtomic8(-1, value);
+ return os_atomic_dec_orig(value, relaxed);
}
-UInt8 OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
+UInt8
+OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
{
- return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed);
+ return os_atomic_and_orig(value, mask, relaxed);
}
-UInt8 OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
+UInt8
+OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
{
- return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed);
+ return os_atomic_or_orig(value, mask, relaxed);
}
-UInt8 OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
+UInt8
+OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
{
- return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed);
+ return os_atomic_xor_orig(value, mask, relaxed);
}
-SInt16 OSIncrementAtomic16(volatile SInt16 * value)
+SInt16
+OSIncrementAtomic16(volatile SInt16 * value)
{
return OSAddAtomic16(1, value);
}
-SInt16 OSDecrementAtomic16(volatile SInt16 * value)
+SInt16
+OSDecrementAtomic16(volatile SInt16 * value)
{
return OSAddAtomic16(-1, value);
}
-UInt16 OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
+UInt16
+OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
{
- return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed);
+ return os_atomic_and_orig(value, mask, relaxed);
}
-UInt16 OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
+UInt16
+OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
{
- return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed);
+ return os_atomic_or_orig(value, mask, relaxed);
}
-UInt16 OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
+UInt16
+OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
{
- return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed);
+ return os_atomic_xor_orig(value, mask, relaxed);
}
-
-// 19831745 - end of big hammer!
-#pragma clang diagnostic pop
-
projects / apple / xnu.git / blobdiff