libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #include <libkern/OSAtomic.h>
  32
  33 enum {
  34         false   = 0,
  35         true    = 1
  36 };
  37 #define NULL 0L
  38
  39 /*
  40  * atomic operations
  41  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  42  *      which is implemented in assembler.  if we are worried about the cost of
  43  *      this layering (we shouldn't be), then all this stuff could be
  44  *      implemented in assembler, as it is in MacOS8/9
  45  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  46  *      which I wrote for NuKernel in a previous life with a different last name...)
  47  *
  48  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  49  *
  50  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  51  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  52  * commpage atomic operations, and the platform specific versions.
  53  * Like standards, there are a lot of atomic ops to choose from!
  54  */
  55
  56 #ifndef __ppc__
  57
  58 SInt32  OSAddAtomic(SInt32 amount, SInt32 * value)
  59 {
  60         SInt32  oldValue;
  61         SInt32  newValue;
  62
  63         do {
  64                 oldValue = *value;
  65                 newValue = oldValue + amount;
  66         } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
  67
  68         return oldValue;
  69 }
  70
  71 SInt32  OSIncrementAtomic(SInt32 * value)
  72 {
  73         return OSAddAtomic(1, value);
  74 }
  75
  76 SInt32  OSDecrementAtomic(SInt32 * value)
  77 {
  78         return OSAddAtomic(-1, value);
  79 }
  80
  81 #ifdef CMPXCHG8B
  82 void *  OSDequeueAtomic(void ** inList, SInt32 inOffset)
  83 {
  84         void *  oldListHead;
  85         void *  newListHead;
  86
  87         do {
  88                 oldListHead = *inList;
  89                 if (oldListHead == NULL) {
  90                         break;
  91                 }
  92
  93                 newListHead = *(void **) (((char *) oldListHead) + inOffset);
  94         } while (! OSCompareAndSwap((UInt32)oldListHead,
  95                                         (UInt32)newListHead, (UInt32 *)inList));
  96         return oldListHead;
  97 }
  98
  99 void    OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
 100 {
 101         void *  oldListHead;
 102         void *  newListHead = inNewLink;
 103         void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
 104
 105         do {
 106                 oldListHead = *inList;
 107                 *newLinkNextPtr = oldListHead;
 108         } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 109                                         (UInt32 *)inList));
 110 }
 111 #endif /* CMPXCHG8B */
 112 #endif  /* !__ppc__ */
 113
 114 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
 115 {
 116         UInt32  oldValue;
 117         UInt32  newValue;
 118
 119         do {
 120                 oldValue = *value;
 121                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 122         } while (! OSCompareAndSwap(oldValue, newValue, value));
 123
 124         return oldValue;
 125 }
 126
 127 UInt32  OSBitAndAtomic(UInt32 mask, UInt32 * value)
 128 {
 129         return OSBitwiseAtomic(mask, 0, 0, value);
 130 }
 131
 132 UInt32  OSBitOrAtomic(UInt32 mask, UInt32 * value)
 133 {
 134         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 135 }
 136
 137 UInt32  OSBitXorAtomic(UInt32 mask, UInt32 * value)
 138 {
 139         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 140 }
 141
 142 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
 143 {
 144         UInt32          mask        = 0x000000ff;
 145         UInt32          alignment   = ((UInt32) value8) & (sizeof(UInt32) - 1);
 146     UInt32      shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 147     int                 shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 148         UInt32 *        value32     = (UInt32 *) (value8 - alignment);
 149     UInt32      oldValue;
 150     UInt32      newValue;
 151
 152     mask <<= shift;
 153
 154     oldValue = *value32;
 155     oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 156     newValue = (oldValue & ~mask) | (newValue8 << shift);
 157
 158         return OSCompareAndSwap(oldValue, newValue, value32);
 159 }
 160
 161 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
 162 {
 163         UInt8           mask = 1;
 164         UInt8           oldValue;
 165         UInt8           wantValue;
 166
 167         startAddress += (bit / 8);
 168         mask <<= (7 - (bit % 8));
 169         wantValue = wantSet ? mask : 0;
 170
 171         do {
 172                 oldValue = *startAddress;
 173                 if ((oldValue & mask) == wantValue) {
 174                         break;
 175                 }
 176         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 177
 178         return (oldValue & mask) == wantValue;
 179 }
 180
 181 Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress)
 182 {
 183         return OSTestAndSetClear(bit, true, startAddress);
 184 }
 185
 186 Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress)
 187 {
 188         return OSTestAndSetClear(bit, false, startAddress);
 189 }
 190
 191 /*
 192  * silly unaligned versions
 193  */
 194
 195 SInt8   OSIncrementAtomic8(SInt8 * value)
 196 {
 197         return OSAddAtomic8(1, value);
 198 }
 199
 200 SInt8   OSDecrementAtomic8(SInt8 * value)
 201 {
 202         return OSAddAtomic8(-1, value);
 203 }
 204
 205 SInt8   OSAddAtomic8(SInt32 amount, SInt8 * value)
 206 {
 207         SInt8   oldValue;
 208         SInt8   newValue;
 209
 210         do {
 211                 oldValue = *value;
 212                 newValue = oldValue + amount;
 213         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
 214
 215         return oldValue;
 216 }
 217
 218 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
 219 {
 220         UInt8   oldValue;
 221         UInt8   newValue;
 222
 223         do {
 224                 oldValue = *value;
 225                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 226         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 227
 228         return oldValue;
 229 }
 230
 231 UInt8   OSBitAndAtomic8(UInt32 mask, UInt8 * value)
 232 {
 233         return OSBitwiseAtomic8(mask, 0, 0, value);
 234 }
 235
 236 UInt8   OSBitOrAtomic8(UInt32 mask, UInt8 * value)
 237 {
 238         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 239 }
 240
 241 UInt8   OSBitXorAtomic8(UInt32 mask, UInt8 * value)
 242 {
 243         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 244 }
 245
 246 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
 247 {
 248         UInt32          mask        = 0x0000ffff;
 249         UInt32          alignment   = ((UInt32) value16) & (sizeof(UInt32) - 1);
 250     UInt32      shiftValues = (16 << 24) | (16 << 16);
 251     UInt32              shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 252         UInt32 *        value32     = (UInt32 *) (((UInt32) value16) - alignment);
 253     UInt32      oldValue;
 254     UInt32      newValue;
 255
 256     mask <<= shift;
 257
 258     oldValue = *value32;
 259     oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 260     newValue = (oldValue & ~mask) | (newValue16 << shift);
 261
 262         return OSCompareAndSwap(oldValue, newValue, value32);
 263 }
 264
 265 SInt16  OSIncrementAtomic16(SInt16 * value)
 266 {
 267         return OSAddAtomic16(1, value);
 268 }
 269
 270 SInt16  OSDecrementAtomic16(SInt16 * value)
 271 {
 272         return OSAddAtomic16(-1, value);
 273 }
 274
 275 SInt16  OSAddAtomic16(SInt32 amount, SInt16 * value)
 276 {
 277         SInt16  oldValue;
 278         SInt16  newValue;
 279
 280         do {
 281                 oldValue = *value;
 282                 newValue = oldValue + amount;
 283         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
 284
 285         return oldValue;
 286 }
 287
 288 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
 289 {
 290         UInt16  oldValue;
 291         UInt16  newValue;
 292
 293         do {
 294                 oldValue = *value;
 295                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 296         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 297
 298         return oldValue;
 299 }
 300
 301 UInt16  OSBitAndAtomic16(UInt32 mask, UInt16 * value)
 302 {
 303         return OSBitwiseAtomic16(mask, 0, 0, value);
 304 }
 305
 306 UInt16  OSBitOrAtomic16(UInt32 mask, UInt16 * value)
 307 {
 308         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 309 }
 310
 311 UInt16  OSBitXorAtomic16(UInt32 mask, UInt16 * value)
 312 {
 313         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 314 }
 315