libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <libkern/OSAtomic.h>
  30 #include <kern/debug.h>
  31 #include <machine/atomic.h>
  32
  33 enum {
  34         false   = 0,
  35         true    = 1
  36 };
  37
  38 #ifndef NULL
  39 #define NULL ((void *)0)
  40 #endif
  41
  42 #define ATOMIC_DEBUG DEBUG
  43
  44 #if ATOMIC_DEBUG
  45 #define ALIGN_TEST(p,t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
  46 #else
  47 #define ALIGN_TEST(p,t) do{}while(0)
  48 #endif
  49
  50 // 19831745 - start of big hammer!
  51 #pragma clang diagnostic push
  52 #pragma clang diagnostic ignored "-Wcast-qual"
  53
  54 /*
  55  * atomic operations
  56  *      These are _the_ atomic operations, now implemented via compiler built-ins.
  57  *      It is expected that this C implementation is a candidate for Link-Time-
  58  *      Optimization inlining, whereas the assembler implementations they replace
  59  *      were not.
  60  */
  61
  62 #undef OSCompareAndSwap8
  63 Boolean OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
  64 {
  65         return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue,
  66                         memory_order_acq_rel_smp, memory_order_relaxed);
  67 }
  68
  69 #undef OSCompareAndSwap16
  70 Boolean OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
  71 {
  72         return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue,
  73                         memory_order_acq_rel_smp, memory_order_relaxed);
  74 }
  75
  76 #undef OSCompareAndSwap
  77 Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
  78 {
  79         ALIGN_TEST(address, UInt32);
  80         return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue,
  81                         memory_order_acq_rel_smp, memory_order_relaxed);
  82 }
  83
  84 #undef OSCompareAndSwap64
  85 Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
  86 {
  87         /*
  88          * _Atomic uint64 requires 8-byte alignment on all architectures.
  89          * This silences the compiler cast warning.  ALIGN_TEST() verifies
  90          * that the cast was legal, if defined.
  91          */
  92         _Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address;
  93
  94         ALIGN_TEST(address, UInt64);
  95         return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue,
  96                         memory_order_acq_rel_smp, memory_order_relaxed);
  97 }
  98
  99 #undef OSCompareAndSwapPtr
 100 Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
 101 {
 102 #if __LP64__
 103   return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address);
 104 #else
 105   return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address);
 106 #endif
 107 }
 108
 109 SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
 110 {
 111         return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed);
 112 }
 113
 114 SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
 115 {
 116         return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed);
 117 }
 118
 119 #undef OSAddAtomic
 120 SInt32 OSAddAtomic(SInt32 amount, volatile SInt32 *address)
 121 {
 122         ALIGN_TEST(address, UInt32);
 123         return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed);
 124 }
 125
 126 #undef OSAddAtomic64
 127 SInt64 OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
 128 {
 129         _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
 130
 131         ALIGN_TEST(address, SInt64);
 132         return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed);
 133 }
 134
 135 #undef OSAddAtomicLong
 136 long
 137 OSAddAtomicLong(long theAmount, volatile long *address)
 138 {
 139 #ifdef __LP64__
 140         return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address);
 141 #else
 142         return (long)OSAddAtomic((SInt32)theAmount, address);
 143 #endif
 144 }
 145
 146 #undef OSIncrementAtomic
 147 SInt32  OSIncrementAtomic(volatile SInt32 * value)
 148 {
 149         return OSAddAtomic(1, value);
 150 }
 151
 152 #undef OSDecrementAtomic
 153 SInt32  OSDecrementAtomic(volatile SInt32 * value)
 154 {
 155         return OSAddAtomic(-1, value);
 156 }
 157
 158 #undef OSBitAndAtomic
 159 UInt32  OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
 160 {
 161         return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed);
 162 }
 163
 164 #undef OSBitOrAtomic
 165 UInt32  OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
 166 {
 167         return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed);
 168 }
 169
 170 #undef OSBitXorAtomic
 171 UInt32  OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
 172 {
 173         return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed);
 174 }
 175
 176 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
 177 {
 178         UInt8           mask = 1;
 179         UInt8           oldValue;
 180         UInt8           wantValue;
 181
 182         startAddress += (bit / 8);
 183         mask <<= (7 - (bit % 8));
 184         wantValue = wantSet ? mask : 0;
 185
 186         do {
 187                 oldValue = *startAddress;
 188                 if ((oldValue & mask) == wantValue) {
 189                         break;
 190                 }
 191         } while (! __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress,
 192                 &oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed));
 193
 194         return (oldValue & mask) == wantValue;
 195 }
 196
 197 Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
 198 {
 199         return OSTestAndSetClear(bit, true, startAddress);
 200 }
 201
 202 Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
 203 {
 204         return OSTestAndSetClear(bit, false, startAddress);
 205 }
 206
 207 /*
 208  * silly unaligned versions
 209  */
 210
 211 SInt8   OSIncrementAtomic8(volatile SInt8 * value)
 212 {
 213         return OSAddAtomic8(1, value);
 214 }
 215
 216 SInt8   OSDecrementAtomic8(volatile SInt8 * value)
 217 {
 218         return OSAddAtomic8(-1, value);
 219 }
 220
 221 UInt8   OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
 222 {
 223         return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed);
 224 }
 225
 226 UInt8   OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
 227 {
 228         return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed);
 229 }
 230
 231 UInt8   OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
 232 {
 233         return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed);
 234 }
 235
 236 SInt16  OSIncrementAtomic16(volatile SInt16 * value)
 237 {
 238         return OSAddAtomic16(1, value);
 239 }
 240
 241 SInt16  OSDecrementAtomic16(volatile SInt16 * value)
 242 {
 243         return OSAddAtomic16(-1, value);
 244 }
 245
 246 UInt16  OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
 247 {
 248         return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed);
 249 }
 250
 251 UInt16  OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
 252 {
 253         return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed);
 254 }
 255
 256 UInt16  OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
 257 {
 258         return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed);
 259 }
 260
 261 // 19831745 - end of big hammer!
 262 #pragma clang diagnostic pop
 263