libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <libkern/OSAtomic.h>
  30
  31 enum {
  32         false   = 0,
  33         true    = 1
  34 };
  35
  36 #ifndef NULL
  37 #define NULL ((void *)0)
  38 #endif
  39
  40 /*
  41  * atomic operations
  42  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  43  *      which is implemented in assembler.      if we are worried about the cost of
  44  *      this layering (we shouldn't be), then all this stuff could be
  45  *      implemented in assembler, as it is in MacOS8/9
  46  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  47  *      which I wrote for NuKernel in a previous life with a different last name...)
  48  *
  49  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  50  *
  51  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  52  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  53  * commpage atomic operations, and the platform specific versions.
  54  * Like standards, there are a lot of atomic ops to choose from!
  55  */
  56
  57 #if !defined(__ppc__) && !defined(__i386__) && !defined(__x86_64__)
  58 /* Implemented in assembly for ppc and i386 and x86_64 */
  59 #undef OSAddAtomic
  60 SInt32
  61 OSAddAtomic(SInt32 amount, volatile SInt32 * value)
  62 {
  63         SInt32 oldValue;
  64         SInt32 newValue;
  65
  66         do {
  67                 oldValue = *value;
  68                 newValue = oldValue + amount;
  69         } while (!OSCompareAndSwap((UInt32)oldValue,
  70                                 (UInt32)newValue,
  71                                 (volatile UInt32 *) value));
  72         return oldValue;
  73 }
  74
  75 long
  76 OSAddAtomicLong(long theAmount, volatile long *address)
  77 {
  78 #if __LP64__
  79 #error Unimplemented
  80 #else
  81         return (long)OSAddAtomic((SInt32)theAmount, address);
  82 #endif
  83 }
  84
  85 /* Implemented as an assembly alias for i386 and linker alias for ppc */
  86 #undef OSCompareAndSwapPtr
  87 Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue,
  88                             void * volatile *address)
  89 {
  90 #if __LP64__
  91   return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue,
  92                           (volatile UInt64 *)address);
  93 #else
  94   return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue,
  95                           (volatile UInt32 *)address);
  96 #endif
  97 }
  98 #endif
  99
 100 #ifndef __ppc__
 101 /* Implemented as assembly for ppc */
 102
 103 #undef OSIncrementAtomic
 104 SInt32  OSIncrementAtomic(volatile SInt32 * value)
 105 {
 106         return OSAddAtomic(1, value);
 107 }
 108
 109 #undef OSDecrementAtomic
 110 SInt32  OSDecrementAtomic(volatile SInt32 * value)
 111 {
 112         return OSAddAtomic(-1, value);
 113 }
 114 #endif  /* !__ppc__ */
 115
 116 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt32 * value)
 117 {
 118         UInt32  oldValue;
 119         UInt32  newValue;
 120
 121         do {
 122                 oldValue = *value;
 123                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 124         } while (! OSCompareAndSwap(oldValue, newValue, value));
 125
 126         return oldValue;
 127 }
 128
 129 #undef OSBitAndAtomic
 130 UInt32  OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
 131 {
 132         return OSBitwiseAtomic(mask, 0, 0, value);
 133 }
 134
 135 #undef OSBitOrAtomic
 136 UInt32  OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
 137 {
 138         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 139 }
 140
 141 #undef OSBitXorAtomic
 142 UInt32  OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
 143 {
 144         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 145 }
 146
 147 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, volatile UInt8 * value8)
 148 {
 149         UInt32                          mask            = 0x000000ff;
 150         UInt32                          alignment       = (UInt32)((unsigned long) value8) & (sizeof(UInt32) - 1);
 151         UInt32                          shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 152         int                                     shift           = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 153         volatile UInt32 *       value32         = (volatile UInt32 *) ((uintptr_t)value8 - alignment);
 154         UInt32                          oldValue;
 155         UInt32                          newValue;
 156
 157         mask <<= shift;
 158
 159         oldValue = *value32;
 160         oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 161         newValue = (oldValue & ~mask) | (newValue8 << shift);
 162
 163         return OSCompareAndSwap(oldValue, newValue, value32);
 164 }
 165
 166 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
 167 {
 168         UInt8           mask = 1;
 169         UInt8           oldValue;
 170         UInt8           wantValue;
 171
 172         startAddress += (bit / 8);
 173         mask <<= (7 - (bit % 8));
 174         wantValue = wantSet ? mask : 0;
 175
 176         do {
 177                 oldValue = *startAddress;
 178                 if ((oldValue & mask) == wantValue) {
 179                         break;
 180                 }
 181         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 182
 183         return (oldValue & mask) == wantValue;
 184 }
 185
 186 Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
 187 {
 188         return OSTestAndSetClear(bit, true, startAddress);
 189 }
 190
 191 Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
 192 {
 193         return OSTestAndSetClear(bit, false, startAddress);
 194 }
 195
 196 /*
 197  * silly unaligned versions
 198  */
 199
 200 SInt8   OSIncrementAtomic8(volatile SInt8 * value)
 201 {
 202         return OSAddAtomic8(1, value);
 203 }
 204
 205 SInt8   OSDecrementAtomic8(volatile SInt8 * value)
 206 {
 207         return OSAddAtomic8(-1, value);
 208 }
 209
 210 SInt8   OSAddAtomic8(SInt32 amount, volatile SInt8 * value)
 211 {
 212         SInt8   oldValue;
 213         SInt8   newValue;
 214
 215         do {
 216                 oldValue = *value;
 217                 newValue = oldValue + amount;
 218         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (volatile UInt8 *) value));
 219
 220         return oldValue;
 221 }
 222
 223 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt8 * value)
 224 {
 225         UInt8   oldValue;
 226         UInt8   newValue;
 227
 228         do {
 229                 oldValue = *value;
 230                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 231         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 232
 233         return oldValue;
 234 }
 235
 236 UInt8   OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
 237 {
 238         return OSBitwiseAtomic8(mask, 0, 0, value);
 239 }
 240
 241 UInt8   OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
 242 {
 243         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 244 }
 245
 246 UInt8   OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
 247 {
 248         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 249 }
 250
 251 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, volatile UInt16 * value16)
 252 {
 253         UInt32                          mask            = 0x0000ffff;
 254         UInt32                          alignment       = (UInt32)((unsigned long) value16) & (sizeof(UInt32) - 1);
 255         UInt32                          shiftValues = (16 << 24) | (16 << 16);
 256         UInt32                          shift           = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 257         volatile UInt32 *       value32         = (volatile UInt32 *) (((unsigned long) value16) - alignment);
 258         UInt32                          oldValue;
 259         UInt32                          newValue;
 260
 261         mask <<= shift;
 262
 263         oldValue = *value32;
 264         oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 265         newValue = (oldValue & ~mask) | (newValue16 << shift);
 266
 267         return OSCompareAndSwap(oldValue, newValue, value32);
 268 }
 269
 270 SInt16  OSIncrementAtomic16(volatile SInt16 * value)
 271 {
 272         return OSAddAtomic16(1, value);
 273 }
 274
 275 SInt16  OSDecrementAtomic16(volatile SInt16 * value)
 276 {
 277         return OSAddAtomic16(-1, value);
 278 }
 279
 280 SInt16  OSAddAtomic16(SInt32 amount, volatile SInt16 * value)
 281 {
 282         SInt16  oldValue;
 283         SInt16  newValue;
 284
 285         do {
 286                 oldValue = *value;
 287                 newValue = oldValue + amount;
 288         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (volatile UInt16 *) value));
 289
 290         return oldValue;
 291 }
 292
 293 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt16 * value)
 294 {
 295         UInt16  oldValue;
 296         UInt16  newValue;
 297
 298         do {
 299                 oldValue = *value;
 300                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 301         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 302
 303         return oldValue;
 304 }
 305
 306 UInt16  OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
 307 {
 308         return OSBitwiseAtomic16(mask, 0, 0, value);
 309 }
 310
 311 UInt16  OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
 312 {
 313         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 314 }
 315
 316 UInt16  OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
 317 {
 318         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 319 }
 320