X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..94ff46dc2849db4d43eaaf144872decc522aafb4:/libkern/gen/OSAtomicOperations.c?ds=inline diff --git a/libkern/gen/OSAtomicOperations.c b/libkern/gen/OSAtomicOperations.c index 5affc1eef..8408c83a4 100644 --- a/libkern/gen/OSAtomicOperations.c +++ b/libkern/gen/OSAtomicOperations.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -31,8 +31,8 @@ #include enum { - false = 0, - true = 1 + false = 0, + true = 1 }; #ifndef NULL @@ -42,15 +42,11 @@ enum { #define ATOMIC_DEBUG DEBUG #if ATOMIC_DEBUG -#define ALIGN_TEST(p,t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0) +#define ALIGN_TEST(p, t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0) #else -#define ALIGN_TEST(p,t) do{}while(0) +#define ALIGN_TEST(p, t) do{}while(0) #endif -// 19831745 - start of big hammer! -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wcast-qual" - /* * atomic operations * These are _the_ atomic operations, now implemented via compiler built-ins. @@ -60,29 +56,30 @@ enum { */ #undef OSCompareAndSwap8 -Boolean OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address) +Boolean +OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address) { - return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue, - memory_order_acq_rel_smp, memory_order_relaxed); + return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); } #undef OSCompareAndSwap16 -Boolean OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address) +Boolean +OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address) { - return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue, - memory_order_acq_rel_smp, memory_order_relaxed); + return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); } #undef OSCompareAndSwap -Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address) +Boolean +OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address) { ALIGN_TEST(address, UInt32); - return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue, - memory_order_acq_rel_smp, memory_order_relaxed); + return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); } #undef OSCompareAndSwap64 -Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address) +Boolean +OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address) { /* * _Atomic uint64 requires 8-byte alignment on all architectures. @@ -92,114 +89,116 @@ Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *ad _Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address; ALIGN_TEST(address, UInt64); - return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue, - memory_order_acq_rel_smp, memory_order_relaxed); + return os_atomic_cmpxchg(aligned_addr, oldValue, newValue, acq_rel); } #undef OSCompareAndSwapPtr -Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address) +Boolean +OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address) { -#if __LP64__ - return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address); -#else - return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address); -#endif + return os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); } -SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 *address) +SInt8 +OSAddAtomic8(SInt32 amount, volatile SInt8 *address) { - return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed); + return os_atomic_add_orig(address, amount, relaxed); } -SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 *address) +SInt16 +OSAddAtomic16(SInt32 amount, volatile SInt16 *address) { - return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed); + return os_atomic_add_orig(address, amount, relaxed); } #undef OSAddAtomic -SInt32 OSAddAtomic(SInt32 amount, volatile SInt32 *address) +SInt32 +OSAddAtomic(SInt32 amount, volatile SInt32 *address) { ALIGN_TEST(address, UInt32); - return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed); + return os_atomic_add_orig(address, amount, relaxed); } #undef OSAddAtomic64 -SInt64 OSAddAtomic64(SInt64 amount, volatile SInt64 *address) +SInt64 +OSAddAtomic64(SInt64 amount, volatile SInt64 *address) { - _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address; + _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address; ALIGN_TEST(address, SInt64); - return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed); + return os_atomic_add_orig(aligned_address, amount, relaxed); } #undef OSAddAtomicLong long OSAddAtomicLong(long theAmount, volatile long *address) { -#ifdef __LP64__ - return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address); -#else - return (long)OSAddAtomic((SInt32)theAmount, address); -#endif + return os_atomic_add_orig(address, theAmount, relaxed); } #undef OSIncrementAtomic -SInt32 OSIncrementAtomic(volatile SInt32 * value) +SInt32 +OSIncrementAtomic(volatile SInt32 * value) { - return OSAddAtomic(1, value); + return os_atomic_inc_orig(value, relaxed); } #undef OSDecrementAtomic -SInt32 OSDecrementAtomic(volatile SInt32 * value) +SInt32 +OSDecrementAtomic(volatile SInt32 * value) { - return OSAddAtomic(-1, value); + return os_atomic_dec_orig(value, relaxed); } #undef OSBitAndAtomic -UInt32 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value) +UInt32 +OSBitAndAtomic(UInt32 mask, volatile UInt32 * value) { - return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed); + return os_atomic_and_orig(value, mask, relaxed); } #undef OSBitOrAtomic -UInt32 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value) +UInt32 +OSBitOrAtomic(UInt32 mask, volatile UInt32 * value) { - return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed); + return os_atomic_or_orig(value, mask, relaxed); } #undef OSBitXorAtomic -UInt32 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value) +UInt32 +OSBitXorAtomic(UInt32 mask, volatile UInt32 * value) { - return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed); + return os_atomic_xor_orig(value, mask, relaxed); } -static Boolean OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress) +static Boolean +OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress) { - UInt8 mask = 1; - UInt8 oldValue; - UInt8 wantValue; - - startAddress += (bit / 8); + UInt8 mask = 1; + UInt8 oldValue, newValue; + UInt8 wantValue; + UInt8 *address; + + address = (UInt8 *)(uintptr_t)(startAddress + (bit / 8)); mask <<= (7 - (bit % 8)); wantValue = wantSet ? mask : 0; - - do { - oldValue = *startAddress; + + return !os_atomic_rmw_loop(address, oldValue, newValue, relaxed, { if ((oldValue & mask) == wantValue) { - break; + os_atomic_rmw_loop_give_up(break); } - } while (! __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress, - &oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed)); - - return (oldValue & mask) == wantValue; + newValue = (oldValue & ~mask) | wantValue; + }); } -Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress) +Boolean +OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress) { return OSTestAndSetClear(bit, true, startAddress); } -Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress) +Boolean +OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress) { return OSTestAndSetClear(bit, false, startAddress); } @@ -208,56 +207,62 @@ Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress) * silly unaligned versions */ -SInt8 OSIncrementAtomic8(volatile SInt8 * value) +SInt8 +OSIncrementAtomic8(volatile SInt8 * value) { - return OSAddAtomic8(1, value); + return os_atomic_inc_orig(value, relaxed); } -SInt8 OSDecrementAtomic8(volatile SInt8 * value) +SInt8 +OSDecrementAtomic8(volatile SInt8 * value) { - return OSAddAtomic8(-1, value); + return os_atomic_dec_orig(value, relaxed); } -UInt8 OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value) +UInt8 +OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value) { - return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed); + return os_atomic_and_orig(value, mask, relaxed); } -UInt8 OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value) +UInt8 +OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value) { - return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed); + return os_atomic_or_orig(value, mask, relaxed); } -UInt8 OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value) +UInt8 +OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value) { - return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed); + return os_atomic_xor_orig(value, mask, relaxed); } -SInt16 OSIncrementAtomic16(volatile SInt16 * value) +SInt16 +OSIncrementAtomic16(volatile SInt16 * value) { return OSAddAtomic16(1, value); } -SInt16 OSDecrementAtomic16(volatile SInt16 * value) +SInt16 +OSDecrementAtomic16(volatile SInt16 * value) { return OSAddAtomic16(-1, value); } -UInt16 OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value) +UInt16 +OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value) { - return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed); + return os_atomic_and_orig(value, mask, relaxed); } -UInt16 OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value) +UInt16 +OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value) { - return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed); + return os_atomic_or_orig(value, mask, relaxed); } -UInt16 OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value) +UInt16 +OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value) { - return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed); + return os_atomic_xor_orig(value, mask, relaxed); } - -// 19831745 - end of big hammer! -#pragma clang diagnostic pop -