X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5d5c5d0d5b79ade9a973d55186ffda2638ba2b6e..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/libkern/gen/OSAtomicOperations.c diff --git a/libkern/gen/OSAtomicOperations.c b/libkern/gen/OSAtomicOperations.c index d3e572868..ba66fc09b 100644 --- a/libkern/gen/OSAtomicOperations.c +++ b/libkern/gen/OSAtomicOperations.c @@ -1,189 +1,204 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include +#include +#include enum { - false = 0, - true = 1 + false = 0, + true = 1 }; -#define NULL 0L + +#ifndef NULL +#define NULL ((void *)0) +#endif + +#define ATOMIC_DEBUG DEBUG + +#if ATOMIC_DEBUG +#define ALIGN_TEST(p, t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0) +#else +#define ALIGN_TEST(p, t) do{}while(0) +#endif /* * atomic operations - * these are _the_ atomic operations, currently cast atop CompareAndSwap, - * which is implemented in assembler. if we are worried about the cost of - * this layering (we shouldn't be), then all this stuff could be - * implemented in assembler, as it is in MacOS8/9 - * (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s, - * which I wrote for NuKernel in a previous life with a different last name...) - * - * native Boolean CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr); - * - * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic, - * OSEnqueueAtomic etc -- in assembler, either for speed or correctness. See also the - * commpage atomic operations, and the platform specific versions. - * Like standards, there are a lot of atomic ops to choose from! + * These are _the_ atomic operations, now implemented via compiler built-ins. + * It is expected that this C implementation is a candidate for Link-Time- + * Optimization inlining, whereas the assembler implementations they replace + * were not. */ -#ifndef __ppc__ +#undef OSCompareAndSwap8 +Boolean +OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address) +{ + return (Boolean)os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); +} -SInt32 OSAddAtomic(SInt32 amount, SInt32 * value) +#undef OSCompareAndSwap16 +Boolean +OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address) { - SInt32 oldValue; - SInt32 newValue; - - do { - oldValue = *value; - newValue = oldValue + amount; - } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value)); - - return oldValue; + return (Boolean)os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); } -SInt32 OSIncrementAtomic(SInt32 * value) +#undef OSCompareAndSwap +Boolean +OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address) { - return OSAddAtomic(1, value); + ALIGN_TEST(address, UInt32); + return (Boolean)os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); } -SInt32 OSDecrementAtomic(SInt32 * value) +#undef OSCompareAndSwap64 +Boolean +OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address) { - return OSAddAtomic(-1, value); + /* + * _Atomic uint64 requires 8-byte alignment on all architectures. + * This silences the compiler cast warning. ALIGN_TEST() verifies + * that the cast was legal, if defined. + */ + _Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address; + + ALIGN_TEST(address, UInt64); + return (Boolean)os_atomic_cmpxchg(aligned_addr, oldValue, newValue, acq_rel); } -#ifdef CMPXCHG8B -void * OSDequeueAtomic(void ** inList, SInt32 inOffset) +#undef OSCompareAndSwapPtr +Boolean +OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address) { - void * oldListHead; - void * newListHead; - - do { - oldListHead = *inList; - if (oldListHead == NULL) { - break; - } - - newListHead = *(void **) (((char *) oldListHead) + inOffset); - } while (! OSCompareAndSwap((UInt32)oldListHead, - (UInt32)newListHead, (UInt32 *)inList)); - return oldListHead; + return (Boolean)os_atomic_cmpxchg(address, oldValue, newValue, acq_rel); +} + +SInt8 +OSAddAtomic8(SInt32 amount, volatile SInt8 *address) +{ + return os_atomic_add_orig(address, (SInt8)amount, relaxed); } -void OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset) +SInt16 +OSAddAtomic16(SInt32 amount, volatile SInt16 *address) { - void * oldListHead; - void * newListHead = inNewLink; - void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset); - - do { - oldListHead = *inList; - *newLinkNextPtr = oldListHead; - } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead, - (UInt32 *)inList)); + return os_atomic_add_orig(address, (SInt16)amount, relaxed); } -#endif /* CMPXCHG8B */ -#endif /* !__ppc__ */ -static UInt32 OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value) +#undef OSAddAtomic +SInt32 +OSAddAtomic(SInt32 amount, volatile SInt32 *address) { - UInt32 oldValue; - UInt32 newValue; - - do { - oldValue = *value; - newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask; - } while (! OSCompareAndSwap(oldValue, newValue, value)); - - return oldValue; + ALIGN_TEST(address, UInt32); + return os_atomic_add_orig(address, amount, relaxed); } -UInt32 OSBitAndAtomic(UInt32 mask, UInt32 * value) +#undef OSAddAtomic64 +SInt64 +OSAddAtomic64(SInt64 amount, volatile SInt64 *address) { - return OSBitwiseAtomic(mask, 0, 0, value); + _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address; + + ALIGN_TEST(address, SInt64); + return os_atomic_add_orig(aligned_address, amount, relaxed); } -UInt32 OSBitOrAtomic(UInt32 mask, UInt32 * value) +#undef OSAddAtomicLong +long +OSAddAtomicLong(long theAmount, volatile long *address) { - return OSBitwiseAtomic((UInt32) -1, mask, 0, value); + return os_atomic_add_orig(address, theAmount, relaxed); } -UInt32 OSBitXorAtomic(UInt32 mask, UInt32 * value) +#undef OSIncrementAtomic +SInt32 +OSIncrementAtomic(volatile SInt32 * value) { - return OSBitwiseAtomic((UInt32) -1, 0, mask, value); + return os_atomic_inc_orig(value, relaxed); } -static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8) +#undef OSDecrementAtomic +SInt32 +OSDecrementAtomic(volatile SInt32 * value) { - UInt32 mask = 0x000000ff; - UInt32 alignment = ((UInt32) value8) & (sizeof(UInt32) - 1); - UInt32 shiftValues = (24 << 24) | (16 << 16) | (8 << 8); - int shift = (UInt32) *(((UInt8 *) &shiftValues) + alignment); - UInt32 * value32 = (UInt32 *) (value8 - alignment); - UInt32 oldValue; - UInt32 newValue; + return os_atomic_dec_orig(value, relaxed); +} - mask <<= shift; +#undef OSBitAndAtomic +UInt32 +OSBitAndAtomic(UInt32 mask, volatile UInt32 * value) +{ + return os_atomic_and_orig(value, mask, relaxed); +} - oldValue = *value32; - oldValue = (oldValue & ~mask) | (oldValue8 << shift); - newValue = (oldValue & ~mask) | (newValue8 << shift); +#undef OSBitOrAtomic +UInt32 +OSBitOrAtomic(UInt32 mask, volatile UInt32 * value) +{ + return os_atomic_or_orig(value, mask, relaxed); +} - return OSCompareAndSwap(oldValue, newValue, value32); +#undef OSBitXorAtomic +UInt32 +OSBitXorAtomic(UInt32 mask, volatile UInt32 * value) +{ + return os_atomic_xor_orig(value, mask, relaxed); } -static Boolean OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress) +static Boolean +OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress) { - UInt8 mask = 1; - UInt8 oldValue; - UInt8 wantValue; - - startAddress += (bit / 8); + UInt8 mask = 1; + UInt8 oldValue, newValue; + UInt8 wantValue; + UInt8 *address; + + address = (UInt8 *)(uintptr_t)(startAddress + (bit / 8)); mask <<= (7 - (bit % 8)); wantValue = wantSet ? mask : 0; - - do { - oldValue = *startAddress; + + return !os_atomic_rmw_loop(address, oldValue, newValue, relaxed, { if ((oldValue & mask) == wantValue) { - break; + os_atomic_rmw_loop_give_up(break); } - } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress)); - - return (oldValue & mask) == wantValue; + newValue = (oldValue & ~mask) | wantValue; + }); } -Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress) +Boolean +OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress) { return OSTestAndSetClear(bit, true, startAddress); } -Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress) +Boolean +OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress) { return OSTestAndSetClear(bit, false, startAddress); } @@ -192,124 +207,62 @@ Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress) * silly unaligned versions */ -SInt8 OSIncrementAtomic8(SInt8 * value) -{ - return OSAddAtomic8(1, value); -} - -SInt8 OSDecrementAtomic8(SInt8 * value) -{ - return OSAddAtomic8(-1, value); -} - -SInt8 OSAddAtomic8(SInt32 amount, SInt8 * value) -{ - SInt8 oldValue; - SInt8 newValue; - - do { - oldValue = *value; - newValue = oldValue + amount; - } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value)); - - return oldValue; -} - -static UInt8 OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value) +SInt8 +OSIncrementAtomic8(volatile SInt8 * value) { - UInt8 oldValue; - UInt8 newValue; - - do { - oldValue = *value; - newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask; - } while (! OSCompareAndSwap8(oldValue, newValue, value)); - - return oldValue; + return os_atomic_inc_orig(value, relaxed); } -UInt8 OSBitAndAtomic8(UInt32 mask, UInt8 * value) +SInt8 +OSDecrementAtomic8(volatile SInt8 * value) { - return OSBitwiseAtomic8(mask, 0, 0, value); + return os_atomic_dec_orig(value, relaxed); } -UInt8 OSBitOrAtomic8(UInt32 mask, UInt8 * value) +UInt8 +OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value) { - return OSBitwiseAtomic8((UInt32) -1, mask, 0, value); + return os_atomic_and_orig(value, (UInt8)mask, relaxed); } -UInt8 OSBitXorAtomic8(UInt32 mask, UInt8 * value) +UInt8 +OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value) { - return OSBitwiseAtomic8((UInt32) -1, 0, mask, value); + return os_atomic_or_orig(value, (UInt8)mask, relaxed); } -static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16) +UInt8 +OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value) { - UInt32 mask = 0x0000ffff; - UInt32 alignment = ((UInt32) value16) & (sizeof(UInt32) - 1); - UInt32 shiftValues = (16 << 24) | (16 << 16); - UInt32 shift = (UInt32) *(((UInt8 *) &shiftValues) + alignment); - UInt32 * value32 = (UInt32 *) (((UInt32) value16) - alignment); - UInt32 oldValue; - UInt32 newValue; - - mask <<= shift; - - oldValue = *value32; - oldValue = (oldValue & ~mask) | (oldValue16 << shift); - newValue = (oldValue & ~mask) | (newValue16 << shift); - - return OSCompareAndSwap(oldValue, newValue, value32); + return os_atomic_xor_orig(value, (UInt8)mask, relaxed); } -SInt16 OSIncrementAtomic16(SInt16 * value) +SInt16 +OSIncrementAtomic16(volatile SInt16 * value) { return OSAddAtomic16(1, value); } -SInt16 OSDecrementAtomic16(SInt16 * value) +SInt16 +OSDecrementAtomic16(volatile SInt16 * value) { return OSAddAtomic16(-1, value); } -SInt16 OSAddAtomic16(SInt32 amount, SInt16 * value) -{ - SInt16 oldValue; - SInt16 newValue; - - do { - oldValue = *value; - newValue = oldValue + amount; - } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value)); - - return oldValue; -} - -static UInt16 OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value) +UInt16 +OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value) { - UInt16 oldValue; - UInt16 newValue; - - do { - oldValue = *value; - newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask; - } while (! OSCompareAndSwap16(oldValue, newValue, value)); - - return oldValue; + return os_atomic_and_orig(value, (UInt16)mask, relaxed); } -UInt16 OSBitAndAtomic16(UInt32 mask, UInt16 * value) +UInt16 +OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value) { - return OSBitwiseAtomic16(mask, 0, 0, value); + return os_atomic_or_orig(value, (UInt16)mask, relaxed); } -UInt16 OSBitOrAtomic16(UInt32 mask, UInt16 * value) +UInt16 +OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value) { - return OSBitwiseAtomic16((UInt32) -1, mask, 0, value); + return os_atomic_xor_orig(value, (UInt16)mask, relaxed); } - -UInt16 OSBitXorAtomic16(UInt32 mask, UInt16 * value) -{ - return OSBitwiseAtomic16((UInt32) -1, 0, mask, value); -} -