libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <libkern/OSAtomic.h>
  30
  31 enum {
  32         false   = 0,
  33         true    = 1
  34 };
  35
  36 #ifndef NULL
  37 #define NULL ((void *)0)
  38 #endif
  39
  40 /*
  41  * atomic operations
  42  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  43  *      which is implemented in assembler.      if we are worried about the cost of
  44  *      this layering (we shouldn't be), then all this stuff could be
  45  *      implemented in assembler, as it is in MacOS8/9
  46  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  47  *      which I wrote for NuKernel in a previous life with a different last name...)
  48  *
  49  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  50  *
  51  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  52  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  53  * commpage atomic operations, and the platform specific versions.
  54  * Like standards, there are a lot of atomic ops to choose from!
  55  */
  56
  57 #if defined(__i386__) || defined(__x86_64__)
  58 /* Implemented in assembly for i386 and x86_64 */
  59 #else
  60 #error Unsupported arch
  61 #endif
  62
  63 #undef OSIncrementAtomic
  64 SInt32  OSIncrementAtomic(volatile SInt32 * value)
  65 {
  66         return OSAddAtomic(1, value);
  67 }
  68
  69 #undef OSDecrementAtomic
  70 SInt32  OSDecrementAtomic(volatile SInt32 * value)
  71 {
  72         return OSAddAtomic(-1, value);
  73 }
  74
  75 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt32 * value)
  76 {
  77         UInt32  oldValue;
  78         UInt32  newValue;
  79
  80         do {
  81                 oldValue = *value;
  82                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
  83         } while (! OSCompareAndSwap(oldValue, newValue, value));
  84
  85         return oldValue;
  86 }
  87
  88 #undef OSBitAndAtomic
  89 UInt32  OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
  90 {
  91         return OSBitwiseAtomic(mask, 0, 0, value);
  92 }
  93
  94 #undef OSBitOrAtomic
  95 UInt32  OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
  96 {
  97         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
  98 }
  99
 100 #undef OSBitXorAtomic
 101 UInt32  OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
 102 {
 103         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 104 }
 105
 106 #if defined(__i386__) || defined(__x86_64__)
 107 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, volatile UInt8 * value8)
 108 {
 109         UInt32                          mask            = 0x000000ff;
 110         UInt32                          alignment       = (UInt32)((unsigned long) value8) & (sizeof(UInt32) - 1);
 111         UInt32                          shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 112         int                                     shift           = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 113         volatile UInt32 *       value32         = (volatile UInt32 *) ((uintptr_t)value8 - alignment);
 114         UInt32                          oldValue;
 115         UInt32                          newValue;
 116
 117         mask <<= shift;
 118
 119         oldValue = *value32;
 120         oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 121         newValue = (oldValue & ~mask) | (newValue8 << shift);
 122
 123         return OSCompareAndSwap(oldValue, newValue, value32);
 124 }
 125 #endif
 126
 127 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
 128 {
 129         UInt8           mask = 1;
 130         UInt8           oldValue;
 131         UInt8           wantValue;
 132
 133         startAddress += (bit / 8);
 134         mask <<= (7 - (bit % 8));
 135         wantValue = wantSet ? mask : 0;
 136
 137         do {
 138                 oldValue = *startAddress;
 139                 if ((oldValue & mask) == wantValue) {
 140                         break;
 141                 }
 142         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 143
 144         return (oldValue & mask) == wantValue;
 145 }
 146
 147 Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
 148 {
 149         return OSTestAndSetClear(bit, true, startAddress);
 150 }
 151
 152 Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
 153 {
 154         return OSTestAndSetClear(bit, false, startAddress);
 155 }
 156
 157 /*
 158  * silly unaligned versions
 159  */
 160
 161 SInt8   OSIncrementAtomic8(volatile SInt8 * value)
 162 {
 163         return OSAddAtomic8(1, value);
 164 }
 165
 166 SInt8   OSDecrementAtomic8(volatile SInt8 * value)
 167 {
 168         return OSAddAtomic8(-1, value);
 169 }
 170
 171 #if defined(__i386__) || defined(__x86_64__)
 172 SInt8   OSAddAtomic8(SInt32 amount, volatile SInt8 * value)
 173 {
 174         SInt8   oldValue;
 175         SInt8   newValue;
 176
 177         do {
 178                 oldValue = *value;
 179                 newValue = oldValue + amount;
 180         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (volatile UInt8 *) value));
 181
 182         return oldValue;
 183 }
 184 #endif
 185
 186 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt8 * value)
 187 {
 188         UInt8   oldValue;
 189         UInt8   newValue;
 190
 191         do {
 192                 oldValue = *value;
 193                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 194         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 195
 196         return oldValue;
 197 }
 198
 199 UInt8   OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
 200 {
 201         return OSBitwiseAtomic8(mask, 0, 0, value);
 202 }
 203
 204 UInt8   OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
 205 {
 206         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 207 }
 208
 209 UInt8   OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
 210 {
 211         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 212 }
 213
 214 #if defined(__i386__) || defined(__x86_64__)
 215 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, volatile UInt16 * value16)
 216 {
 217         UInt32                          mask            = 0x0000ffff;
 218         UInt32                          alignment       = (UInt32)((unsigned long) value16) & (sizeof(UInt32) - 1);
 219         UInt32                          shiftValues = (16 << 24) | (16 << 16);
 220         UInt32                          shift           = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 221         volatile UInt32 *       value32         = (volatile UInt32 *) (((unsigned long) value16) - alignment);
 222         UInt32                          oldValue;
 223         UInt32                          newValue;
 224
 225         mask <<= shift;
 226
 227         oldValue = *value32;
 228         oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 229         newValue = (oldValue & ~mask) | (newValue16 << shift);
 230
 231         return OSCompareAndSwap(oldValue, newValue, value32);
 232 }
 233 #endif
 234
 235 SInt16  OSIncrementAtomic16(volatile SInt16 * value)
 236 {
 237         return OSAddAtomic16(1, value);
 238 }
 239
 240 SInt16  OSDecrementAtomic16(volatile SInt16 * value)
 241 {
 242         return OSAddAtomic16(-1, value);
 243 }
 244
 245 #if defined(__i386__) || defined(__x86_64__)
 246 SInt16  OSAddAtomic16(SInt32 amount, volatile SInt16 * value)
 247 {
 248         SInt16  oldValue;
 249         SInt16  newValue;
 250
 251         do {
 252                 oldValue = *value;
 253                 newValue = oldValue + amount;
 254         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (volatile UInt16 *) value));
 255
 256         return oldValue;
 257 }
 258 #endif
 259
 260 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt16 * value)
 261 {
 262         UInt16  oldValue;
 263         UInt16  newValue;
 264
 265         do {
 266                 oldValue = *value;
 267                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 268         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 269
 270         return oldValue;
 271 }
 272
 273 UInt16  OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
 274 {
 275         return OSBitwiseAtomic16(mask, 0, 0, value);
 276 }
 277
 278 UInt16  OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
 279 {
 280         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 281 }
 282
 283 UInt16  OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
 284 {
 285         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 286 }
 287