libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <libkern/OSAtomic.h>
  30
  31 enum {
  32         false   = 0,
  33         true    = 1
  34 };
  35
  36 #ifndef NULL
  37 #define NULL ((void *)0)
  38 #endif
  39
  40 /*
  41  * atomic operations
  42  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  43  *      which is implemented in assembler.      if we are worried about the cost of
  44  *      this layering (we shouldn't be), then all this stuff could be
  45  *      implemented in assembler, as it is in MacOS8/9
  46  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  47  *      which I wrote for NuKernel in a previous life with a different last name...)
  48  *
  49  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  50  *
  51  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  52  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  53  * commpage atomic operations, and the platform specific versions.
  54  * Like standards, there are a lot of atomic ops to choose from!
  55  */
  56
  57 #if !defined(__i386__) && !defined(__x86_64__)
  58 /* Implemented in assembly for i386 and x86_64 */
  59 #undef OSAddAtomic
  60 SInt32
  61 OSAddAtomic(SInt32 amount, volatile SInt32 * value)
  62 {
  63         SInt32 oldValue;
  64         SInt32 newValue;
  65
  66         do {
  67                 oldValue = *value;
  68                 newValue = oldValue + amount;
  69         } while (!OSCompareAndSwap((UInt32)oldValue,
  70                                 (UInt32)newValue,
  71                                 (volatile UInt32 *) value));
  72         return oldValue;
  73 }
  74
  75 #undef OSAddAtomicLong
  76 long
  77 OSAddAtomicLong(long theAmount, volatile long *address)
  78 {
  79 #if __LP64__
  80 #error Unimplemented
  81 #else
  82         return (long)OSAddAtomic((SInt32)theAmount, address);
  83 #endif
  84 }
  85
  86 /* Implemented as an assembly alias for i386 */
  87 #undef OSCompareAndSwapPtr
  88 Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue,
  89                             void * volatile *address)
  90 {
  91 #if __LP64__
  92   return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue,
  93                           (volatile UInt64 *)address);
  94 #else
  95   return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue,
  96                           (volatile UInt32 *)address);
  97 #endif
  98 }
  99 #endif
 100
 101 #undef OSIncrementAtomic
 102 SInt32  OSIncrementAtomic(volatile SInt32 * value)
 103 {
 104         return OSAddAtomic(1, value);
 105 }
 106
 107 #undef OSDecrementAtomic
 108 SInt32  OSDecrementAtomic(volatile SInt32 * value)
 109 {
 110         return OSAddAtomic(-1, value);
 111 }
 112
 113 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt32 * value)
 114 {
 115         UInt32  oldValue;
 116         UInt32  newValue;
 117
 118         do {
 119                 oldValue = *value;
 120                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 121         } while (! OSCompareAndSwap(oldValue, newValue, value));
 122
 123         return oldValue;
 124 }
 125
 126 #undef OSBitAndAtomic
 127 UInt32  OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
 128 {
 129         return OSBitwiseAtomic(mask, 0, 0, value);
 130 }
 131
 132 #undef OSBitOrAtomic
 133 UInt32  OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
 134 {
 135         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 136 }
 137
 138 #undef OSBitXorAtomic
 139 UInt32  OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
 140 {
 141         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 142 }
 143
 144 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, volatile UInt8 * value8)
 145 {
 146         UInt32                          mask            = 0x000000ff;
 147         UInt32                          alignment       = (UInt32)((unsigned long) value8) & (sizeof(UInt32) - 1);
 148         UInt32                          shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 149         int                                     shift           = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 150         volatile UInt32 *       value32         = (volatile UInt32 *) ((uintptr_t)value8 - alignment);
 151         UInt32                          oldValue;
 152         UInt32                          newValue;
 153
 154         mask <<= shift;
 155
 156         oldValue = *value32;
 157         oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 158         newValue = (oldValue & ~mask) | (newValue8 << shift);
 159
 160         return OSCompareAndSwap(oldValue, newValue, value32);
 161 }
 162
 163 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
 164 {
 165         UInt8           mask = 1;
 166         UInt8           oldValue;
 167         UInt8           wantValue;
 168
 169         startAddress += (bit / 8);
 170         mask <<= (7 - (bit % 8));
 171         wantValue = wantSet ? mask : 0;
 172
 173         do {
 174                 oldValue = *startAddress;
 175                 if ((oldValue & mask) == wantValue) {
 176                         break;
 177                 }
 178         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 179
 180         return (oldValue & mask) == wantValue;
 181 }
 182
 183 Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
 184 {
 185         return OSTestAndSetClear(bit, true, startAddress);
 186 }
 187
 188 Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
 189 {
 190         return OSTestAndSetClear(bit, false, startAddress);
 191 }
 192
 193 /*
 194  * silly unaligned versions
 195  */
 196
 197 SInt8   OSIncrementAtomic8(volatile SInt8 * value)
 198 {
 199         return OSAddAtomic8(1, value);
 200 }
 201
 202 SInt8   OSDecrementAtomic8(volatile SInt8 * value)
 203 {
 204         return OSAddAtomic8(-1, value);
 205 }
 206
 207 SInt8   OSAddAtomic8(SInt32 amount, volatile SInt8 * value)
 208 {
 209         SInt8   oldValue;
 210         SInt8   newValue;
 211
 212         do {
 213                 oldValue = *value;
 214                 newValue = oldValue + amount;
 215         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (volatile UInt8 *) value));
 216
 217         return oldValue;
 218 }
 219
 220 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt8 * value)
 221 {
 222         UInt8   oldValue;
 223         UInt8   newValue;
 224
 225         do {
 226                 oldValue = *value;
 227                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 228         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 229
 230         return oldValue;
 231 }
 232
 233 UInt8   OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
 234 {
 235         return OSBitwiseAtomic8(mask, 0, 0, value);
 236 }
 237
 238 UInt8   OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
 239 {
 240         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 241 }
 242
 243 UInt8   OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
 244 {
 245         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 246 }
 247
 248 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, volatile UInt16 * value16)
 249 {
 250         UInt32                          mask            = 0x0000ffff;
 251         UInt32                          alignment       = (UInt32)((unsigned long) value16) & (sizeof(UInt32) - 1);
 252         UInt32                          shiftValues = (16 << 24) | (16 << 16);
 253         UInt32                          shift           = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 254         volatile UInt32 *       value32         = (volatile UInt32 *) (((unsigned long) value16) - alignment);
 255         UInt32                          oldValue;
 256         UInt32                          newValue;
 257
 258         mask <<= shift;
 259
 260         oldValue = *value32;
 261         oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 262         newValue = (oldValue & ~mask) | (newValue16 << shift);
 263
 264         return OSCompareAndSwap(oldValue, newValue, value32);
 265 }
 266
 267 SInt16  OSIncrementAtomic16(volatile SInt16 * value)
 268 {
 269         return OSAddAtomic16(1, value);
 270 }
 271
 272 SInt16  OSDecrementAtomic16(volatile SInt16 * value)
 273 {
 274         return OSAddAtomic16(-1, value);
 275 }
 276
 277 SInt16  OSAddAtomic16(SInt32 amount, volatile SInt16 * value)
 278 {
 279         SInt16  oldValue;
 280         SInt16  newValue;
 281
 282         do {
 283                 oldValue = *value;
 284                 newValue = oldValue + amount;
 285         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (volatile UInt16 *) value));
 286
 287         return oldValue;
 288 }
 289
 290 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt16 * value)
 291 {
 292         UInt16  oldValue;
 293         UInt16  newValue;
 294
 295         do {
 296                 oldValue = *value;
 297                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 298         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 299
 300         return oldValue;
 301 }
 302
 303 UInt16  OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
 304 {
 305         return OSBitwiseAtomic16(mask, 0, 0, value);
 306 }
 307
 308 UInt16  OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
 309 {
 310         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 311 }
 312
 313 UInt16  OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
 314 {
 315         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 316 }
 317