libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <libkern/OSAtomic.h>
  24
  25 enum {
  26         false   = 0,
  27         true    = 1
  28 };
  29 #define NULL 0L
  30
  31 /*
  32  * atomic operations
  33  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  34  *      which is implemented in assembler.  if we are worried about the cost of
  35  *      this layering (we shouldn't be), then all this stuff could be
  36  *      implemented in assembler, as it is in MacOS8/9
  37  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  38  *      which I wrote for NuKernel in a previous life with a different last name...)
  39  *
  40  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  41  *
  42  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  43  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  44  * commpage atomic operations, and the platform specific versions.
  45  * Like standards, there are a lot of atomic ops to choose from!
  46  */
  47
  48 #ifndef __ppc__
  49
  50 SInt32  OSAddAtomic(SInt32 amount, SInt32 * value)
  51 {
  52         SInt32  oldValue;
  53         SInt32  newValue;
  54
  55         do {
  56                 oldValue = *value;
  57                 newValue = oldValue + amount;
  58         } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
  59
  60         return oldValue;
  61 }
  62
  63 SInt32  OSIncrementAtomic(SInt32 * value)
  64 {
  65         return OSAddAtomic(1, value);
  66 }
  67
  68 SInt32  OSDecrementAtomic(SInt32 * value)
  69 {
  70         return OSAddAtomic(-1, value);
  71 }
  72
  73 #ifdef CMPXCHG8B
  74 void *  OSDequeueAtomic(void ** inList, SInt32 inOffset)
  75 {
  76         void *  oldListHead;
  77         void *  newListHead;
  78
  79         do {
  80                 oldListHead = *inList;
  81                 if (oldListHead == NULL) {
  82                         break;
  83                 }
  84
  85                 newListHead = *(void **) (((char *) oldListHead) + inOffset);
  86         } while (! OSCompareAndSwap((UInt32)oldListHead,
  87                                         (UInt32)newListHead, (UInt32 *)inList));
  88         return oldListHead;
  89 }
  90
  91 void    OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
  92 {
  93         void *  oldListHead;
  94         void *  newListHead = inNewLink;
  95         void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
  96
  97         do {
  98                 oldListHead = *inList;
  99                 *newLinkNextPtr = oldListHead;
 100         } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 101                                         (UInt32 *)inList));
 102 }
 103 #endif /* CMPXCHG8B */
 104 #endif  /* !__ppc__ */
 105
 106 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
 107 {
 108         UInt32  oldValue;
 109         UInt32  newValue;
 110
 111         do {
 112                 oldValue = *value;
 113                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 114         } while (! OSCompareAndSwap(oldValue, newValue, value));
 115
 116         return oldValue;
 117 }
 118
 119 UInt32  OSBitAndAtomic(UInt32 mask, UInt32 * value)
 120 {
 121         return OSBitwiseAtomic(mask, 0, 0, value);
 122 }
 123
 124 UInt32  OSBitOrAtomic(UInt32 mask, UInt32 * value)
 125 {
 126         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 127 }
 128
 129 UInt32  OSBitXorAtomic(UInt32 mask, UInt32 * value)
 130 {
 131         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 132 }
 133
 134 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
 135 {
 136         UInt32          mask        = 0x000000ff;
 137         UInt32          alignment   = ((UInt32) value8) & (sizeof(UInt32) - 1);
 138     UInt32      shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 139     int                 shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 140         UInt32 *        value32     = (UInt32 *) (value8 - alignment);
 141     UInt32      oldValue;
 142     UInt32      newValue;
 143
 144     mask <<= shift;
 145
 146     oldValue = *value32;
 147     oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 148     newValue = (oldValue & ~mask) | (newValue8 << shift);
 149
 150         return OSCompareAndSwap(oldValue, newValue, value32);
 151 }
 152
 153 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
 154 {
 155         UInt8           mask = 1;
 156         UInt8           oldValue;
 157         UInt8           wantValue;
 158
 159         startAddress += (bit / 8);
 160         mask <<= (7 - (bit % 8));
 161         wantValue = wantSet ? mask : 0;
 162
 163         do {
 164                 oldValue = *startAddress;
 165                 if ((oldValue & mask) == wantValue) {
 166                         break;
 167                 }
 168         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 169
 170         return (oldValue & mask) == wantValue;
 171 }
 172
 173 Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress)
 174 {
 175         return OSTestAndSetClear(bit, true, startAddress);
 176 }
 177
 178 Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress)
 179 {
 180         return OSTestAndSetClear(bit, false, startAddress);
 181 }
 182
 183 /*
 184  * silly unaligned versions
 185  */
 186
 187 SInt8   OSIncrementAtomic8(SInt8 * value)
 188 {
 189         return OSAddAtomic8(1, value);
 190 }
 191
 192 SInt8   OSDecrementAtomic8(SInt8 * value)
 193 {
 194         return OSAddAtomic8(-1, value);
 195 }
 196
 197 SInt8   OSAddAtomic8(SInt32 amount, SInt8 * value)
 198 {
 199         SInt8   oldValue;
 200         SInt8   newValue;
 201
 202         do {
 203                 oldValue = *value;
 204                 newValue = oldValue + amount;
 205         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
 206
 207         return oldValue;
 208 }
 209
 210 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
 211 {
 212         UInt8   oldValue;
 213         UInt8   newValue;
 214
 215         do {
 216                 oldValue = *value;
 217                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 218         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 219
 220         return oldValue;
 221 }
 222
 223 UInt8   OSBitAndAtomic8(UInt32 mask, UInt8 * value)
 224 {
 225         return OSBitwiseAtomic8(mask, 0, 0, value);
 226 }
 227
 228 UInt8   OSBitOrAtomic8(UInt32 mask, UInt8 * value)
 229 {
 230         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 231 }
 232
 233 UInt8   OSBitXorAtomic8(UInt32 mask, UInt8 * value)
 234 {
 235         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 236 }
 237
 238 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
 239 {
 240         UInt32          mask        = 0x0000ffff;
 241         UInt32          alignment   = ((UInt32) value16) & (sizeof(UInt32) - 1);
 242     UInt32      shiftValues = (16 << 24) | (16 << 16);
 243     UInt32              shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 244         UInt32 *        value32     = (UInt32 *) (((UInt32) value16) - alignment);
 245     UInt32      oldValue;
 246     UInt32      newValue;
 247
 248     mask <<= shift;
 249
 250     oldValue = *value32;
 251     oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 252     newValue = (oldValue & ~mask) | (newValue16 << shift);
 253
 254         return OSCompareAndSwap(oldValue, newValue, value32);
 255 }
 256
 257 SInt16  OSIncrementAtomic16(SInt16 * value)
 258 {
 259         return OSAddAtomic16(1, value);
 260 }
 261
 262 SInt16  OSDecrementAtomic16(SInt16 * value)
 263 {
 264         return OSAddAtomic16(-1, value);
 265 }
 266
 267 SInt16  OSAddAtomic16(SInt32 amount, SInt16 * value)
 268 {
 269         SInt16  oldValue;
 270         SInt16  newValue;
 271
 272         do {
 273                 oldValue = *value;
 274                 newValue = oldValue + amount;
 275         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
 276
 277         return oldValue;
 278 }
 279
 280 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
 281 {
 282         UInt16  oldValue;
 283         UInt16  newValue;
 284
 285         do {
 286                 oldValue = *value;
 287                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 288         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 289
 290         return oldValue;
 291 }
 292
 293 UInt16  OSBitAndAtomic16(UInt32 mask, UInt16 * value)
 294 {
 295         return OSBitwiseAtomic16(mask, 0, 0, value);
 296 }
 297
 298 UInt16  OSBitOrAtomic16(UInt32 mask, UInt16 * value)
 299 {
 300         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 301 }
 302
 303 UInt16  OSBitXorAtomic16(UInt32 mask, UInt16 * value)
 304 {
 305         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 306 }
 307