libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <libkern/OSAtomic.h>
  30
  31 enum {
  32         false   = 0,
  33         true    = 1
  34 };
  35 #define NULL 0L
  36
  37 /*
  38  * atomic operations
  39  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  40  *      which is implemented in assembler.  if we are worried about the cost of
  41  *      this layering (we shouldn't be), then all this stuff could be
  42  *      implemented in assembler, as it is in MacOS8/9
  43  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  44  *      which I wrote for NuKernel in a previous life with a different last name...)
  45  *
  46  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  47  *
  48  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  49  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  50  * commpage atomic operations, and the platform specific versions.
  51  * Like standards, there are a lot of atomic ops to choose from!
  52  */
  53
  54 #ifndef __ppc__
  55
  56 SInt32  OSAddAtomic(SInt32 amount, SInt32 * value)
  57 {
  58         SInt32  oldValue;
  59         SInt32  newValue;
  60
  61         do {
  62                 oldValue = *value;
  63                 newValue = oldValue + amount;
  64         } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
  65
  66         return oldValue;
  67 }
  68
  69 SInt32  OSIncrementAtomic(SInt32 * value)
  70 {
  71         return OSAddAtomic(1, value);
  72 }
  73
  74 SInt32  OSDecrementAtomic(SInt32 * value)
  75 {
  76         return OSAddAtomic(-1, value);
  77 }
  78
  79 #ifdef CMPXCHG8B
  80 void *  OSDequeueAtomic(void ** inList, SInt32 inOffset)
  81 {
  82         void *  oldListHead;
  83         void *  newListHead;
  84
  85         do {
  86                 oldListHead = *inList;
  87                 if (oldListHead == NULL) {
  88                         break;
  89                 }
  90
  91                 newListHead = *(void **) (((char *) oldListHead) + inOffset);
  92         } while (! OSCompareAndSwap((UInt32)oldListHead,
  93                                         (UInt32)newListHead, (UInt32 *)inList));
  94         return oldListHead;
  95 }
  96
  97 void    OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
  98 {
  99         void *  oldListHead;
 100         void *  newListHead = inNewLink;
 101         void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
 102
 103         do {
 104                 oldListHead = *inList;
 105                 *newLinkNextPtr = oldListHead;
 106         } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 107                                         (UInt32 *)inList));
 108 }
 109 #endif /* CMPXCHG8B */
 110 #endif  /* !__ppc__ */
 111
 112 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
 113 {
 114         UInt32  oldValue;
 115         UInt32  newValue;
 116
 117         do {
 118                 oldValue = *value;
 119                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 120         } while (! OSCompareAndSwap(oldValue, newValue, value));
 121
 122         return oldValue;
 123 }
 124
 125 UInt32  OSBitAndAtomic(UInt32 mask, UInt32 * value)
 126 {
 127         return OSBitwiseAtomic(mask, 0, 0, value);
 128 }
 129
 130 UInt32  OSBitOrAtomic(UInt32 mask, UInt32 * value)
 131 {
 132         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 133 }
 134
 135 UInt32  OSBitXorAtomic(UInt32 mask, UInt32 * value)
 136 {
 137         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 138 }
 139
 140 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
 141 {
 142         UInt32          mask        = 0x000000ff;
 143         UInt32          alignment   = ((UInt32) value8) & (sizeof(UInt32) - 1);
 144     UInt32      shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 145     int                 shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 146         UInt32 *        value32     = (UInt32 *) (value8 - alignment);
 147     UInt32      oldValue;
 148     UInt32      newValue;
 149
 150     mask <<= shift;
 151
 152     oldValue = *value32;
 153     oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 154     newValue = (oldValue & ~mask) | (newValue8 << shift);
 155
 156         return OSCompareAndSwap(oldValue, newValue, value32);
 157 }
 158
 159 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
 160 {
 161         UInt8           mask = 1;
 162         UInt8           oldValue;
 163         UInt8           wantValue;
 164
 165         startAddress += (bit / 8);
 166         mask <<= (7 - (bit % 8));
 167         wantValue = wantSet ? mask : 0;
 168
 169         do {
 170                 oldValue = *startAddress;
 171                 if ((oldValue & mask) == wantValue) {
 172                         break;
 173                 }
 174         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 175
 176         return (oldValue & mask) == wantValue;
 177 }
 178
 179 Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress)
 180 {
 181         return OSTestAndSetClear(bit, true, startAddress);
 182 }
 183
 184 Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress)
 185 {
 186         return OSTestAndSetClear(bit, false, startAddress);
 187 }
 188
 189 /*
 190  * silly unaligned versions
 191  */
 192
 193 SInt8   OSIncrementAtomic8(SInt8 * value)
 194 {
 195         return OSAddAtomic8(1, value);
 196 }
 197
 198 SInt8   OSDecrementAtomic8(SInt8 * value)
 199 {
 200         return OSAddAtomic8(-1, value);
 201 }
 202
 203 SInt8   OSAddAtomic8(SInt32 amount, SInt8 * value)
 204 {
 205         SInt8   oldValue;
 206         SInt8   newValue;
 207
 208         do {
 209                 oldValue = *value;
 210                 newValue = oldValue + amount;
 211         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
 212
 213         return oldValue;
 214 }
 215
 216 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
 217 {
 218         UInt8   oldValue;
 219         UInt8   newValue;
 220
 221         do {
 222                 oldValue = *value;
 223                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 224         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 225
 226         return oldValue;
 227 }
 228
 229 UInt8   OSBitAndAtomic8(UInt32 mask, UInt8 * value)
 230 {
 231         return OSBitwiseAtomic8(mask, 0, 0, value);
 232 }
 233
 234 UInt8   OSBitOrAtomic8(UInt32 mask, UInt8 * value)
 235 {
 236         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 237 }
 238
 239 UInt8   OSBitXorAtomic8(UInt32 mask, UInt8 * value)
 240 {
 241         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 242 }
 243
 244 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
 245 {
 246         UInt32          mask        = 0x0000ffff;
 247         UInt32          alignment   = ((UInt32) value16) & (sizeof(UInt32) - 1);
 248     UInt32      shiftValues = (16 << 24) | (16 << 16);
 249     UInt32              shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 250         UInt32 *        value32     = (UInt32 *) (((UInt32) value16) - alignment);
 251     UInt32      oldValue;
 252     UInt32      newValue;
 253
 254     mask <<= shift;
 255
 256     oldValue = *value32;
 257     oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 258     newValue = (oldValue & ~mask) | (newValue16 << shift);
 259
 260         return OSCompareAndSwap(oldValue, newValue, value32);
 261 }
 262
 263 SInt16  OSIncrementAtomic16(SInt16 * value)
 264 {
 265         return OSAddAtomic16(1, value);
 266 }
 267
 268 SInt16  OSDecrementAtomic16(SInt16 * value)
 269 {
 270         return OSAddAtomic16(-1, value);
 271 }
 272
 273 SInt16  OSAddAtomic16(SInt32 amount, SInt16 * value)
 274 {
 275         SInt16  oldValue;
 276         SInt16  newValue;
 277
 278         do {
 279                 oldValue = *value;
 280                 newValue = oldValue + amount;
 281         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
 282
 283         return oldValue;
 284 }
 285
 286 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
 287 {
 288         UInt16  oldValue;
 289         UInt16  newValue;
 290
 291         do {
 292                 oldValue = *value;
 293                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 294         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 295
 296         return oldValue;
 297 }
 298
 299 UInt16  OSBitAndAtomic16(UInt32 mask, UInt16 * value)
 300 {
 301         return OSBitwiseAtomic16(mask, 0, 0, value);
 302 }
 303
 304 UInt16  OSBitOrAtomic16(UInt32 mask, UInt16 * value)
 305 {
 306         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 307 }
 308
 309 UInt16  OSBitXorAtomic16(UInt32 mask, UInt16 * value)
 310 {
 311         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 312 }
 313