libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <libkern/OSAtomic.h>
  30 #include <kern/debug.h>
  31 #include <machine/atomic.h>
  32
  33 enum {
  34         false   = 0,
  35         true    = 1
  36 };
  37
  38 #ifndef NULL
  39 #define NULL ((void *)0)
  40 #endif
  41
  42 #define ATOMIC_DEBUG DEBUG
  43
  44 #if ATOMIC_DEBUG
  45 #define ALIGN_TEST(p, t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
  46 #else
  47 #define ALIGN_TEST(p, t) do{}while(0)
  48 #endif
  49
  50 // 19831745 - start of big hammer!
  51 #pragma clang diagnostic push
  52 #pragma clang diagnostic ignored "-Wcast-qual"
  53
  54 /*
  55  * atomic operations
  56  *      These are _the_ atomic operations, now implemented via compiler built-ins.
  57  *      It is expected that this C implementation is a candidate for Link-Time-
  58  *      Optimization inlining, whereas the assembler implementations they replace
  59  *      were not.
  60  */
  61
  62 #undef OSCompareAndSwap8
  63 Boolean
  64 OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
  65 {
  66         return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue,
  67                    memory_order_acq_rel_smp, memory_order_relaxed);
  68 }
  69
  70 #undef OSCompareAndSwap16
  71 Boolean
  72 OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
  73 {
  74         return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue,
  75                    memory_order_acq_rel_smp, memory_order_relaxed);
  76 }
  77
  78 #undef OSCompareAndSwap
  79 Boolean
  80 OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
  81 {
  82         ALIGN_TEST(address, UInt32);
  83         return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue,
  84                    memory_order_acq_rel_smp, memory_order_relaxed);
  85 }
  86
  87 #undef OSCompareAndSwap64
  88 Boolean
  89 OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
  90 {
  91         /*
  92          * _Atomic uint64 requires 8-byte alignment on all architectures.
  93          * This silences the compiler cast warning.  ALIGN_TEST() verifies
  94          * that the cast was legal, if defined.
  95          */
  96         _Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address;
  97
  98         ALIGN_TEST(address, UInt64);
  99         return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue,
 100                    memory_order_acq_rel_smp, memory_order_relaxed);
 101 }
 102
 103 #undef OSCompareAndSwapPtr
 104 Boolean
 105 OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
 106 {
 107 #if __LP64__
 108         return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address);
 109 #else
 110         return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address);
 111 #endif
 112 }
 113
 114 SInt8
 115 OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
 116 {
 117         return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed);
 118 }
 119
 120 SInt16
 121 OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
 122 {
 123         return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed);
 124 }
 125
 126 #undef OSAddAtomic
 127 SInt32
 128 OSAddAtomic(SInt32 amount, volatile SInt32 *address)
 129 {
 130         ALIGN_TEST(address, UInt32);
 131         return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed);
 132 }
 133
 134 #undef OSAddAtomic64
 135 SInt64
 136 OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
 137 {
 138         _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
 139
 140         ALIGN_TEST(address, SInt64);
 141         return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed);
 142 }
 143
 144 #undef OSAddAtomicLong
 145 long
 146 OSAddAtomicLong(long theAmount, volatile long *address)
 147 {
 148 #ifdef __LP64__
 149         return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address);
 150 #else
 151         return (long)OSAddAtomic((SInt32)theAmount, address);
 152 #endif
 153 }
 154
 155 #undef OSIncrementAtomic
 156 SInt32
 157 OSIncrementAtomic(volatile SInt32 * value)
 158 {
 159         return OSAddAtomic(1, value);
 160 }
 161
 162 #undef OSDecrementAtomic
 163 SInt32
 164 OSDecrementAtomic(volatile SInt32 * value)
 165 {
 166         return OSAddAtomic(-1, value);
 167 }
 168
 169 #undef OSBitAndAtomic
 170 UInt32
 171 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
 172 {
 173         return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed);
 174 }
 175
 176 #undef OSBitOrAtomic
 177 UInt32
 178 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
 179 {
 180         return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed);
 181 }
 182
 183 #undef OSBitXorAtomic
 184 UInt32
 185 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
 186 {
 187         return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed);
 188 }
 189
 190 static Boolean
 191 OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
 192 {
 193         UInt8           mask = 1;
 194         UInt8           oldValue;
 195         UInt8           wantValue;
 196
 197         startAddress += (bit / 8);
 198         mask <<= (7 - (bit % 8));
 199         wantValue = wantSet ? mask : 0;
 200
 201         do {
 202                 oldValue = *startAddress;
 203                 if ((oldValue & mask) == wantValue) {
 204                         break;
 205                 }
 206         } while (!__c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress,
 207             &oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed));
 208
 209         return (oldValue & mask) == wantValue;
 210 }
 211
 212 Boolean
 213 OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
 214 {
 215         return OSTestAndSetClear(bit, true, startAddress);
 216 }
 217
 218 Boolean
 219 OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
 220 {
 221         return OSTestAndSetClear(bit, false, startAddress);
 222 }
 223
 224 /*
 225  * silly unaligned versions
 226  */
 227
 228 SInt8
 229 OSIncrementAtomic8(volatile SInt8 * value)
 230 {
 231         return OSAddAtomic8(1, value);
 232 }
 233
 234 SInt8
 235 OSDecrementAtomic8(volatile SInt8 * value)
 236 {
 237         return OSAddAtomic8(-1, value);
 238 }
 239
 240 UInt8
 241 OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
 242 {
 243         return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed);
 244 }
 245
 246 UInt8
 247 OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
 248 {
 249         return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed);
 250 }
 251
 252 UInt8
 253 OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
 254 {
 255         return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed);
 256 }
 257
 258 SInt16
 259 OSIncrementAtomic16(volatile SInt16 * value)
 260 {
 261         return OSAddAtomic16(1, value);
 262 }
 263
 264 SInt16
 265 OSDecrementAtomic16(volatile SInt16 * value)
 266 {
 267         return OSAddAtomic16(-1, value);
 268 }
 269
 270 UInt16
 271 OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
 272 {
 273         return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed);
 274 }
 275
 276 UInt16
 277 OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
 278 {
 279         return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed);
 280 }
 281
 282 UInt16
 283 OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
 284 {
 285         return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed);
 286 }
 287
 288 // 19831745 - end of big hammer!
 289 #pragma clang diagnostic pop