libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include <libkern/OSAtomic.h>
  25
  26 enum {
  27         false   = 0,
  28         true    = 1
  29 };
  30 #define NULL 0L
  31
  32
  33 /*
  34  * atomic operations
  35  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  36  *      which is implemented in assembler.  if we are worried about the cost of
  37  *      this layering (we shouldn't be), then all this stuff could be
  38  *      implemented in assembler, as it is in MacOS8/9
  39  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  40  *      which I wrote for NuKernel in a previous life with a different last name...)
  41  *
  42  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  43  *
  44  * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
  45  * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
  46  * commpage atomic operations, and the platform specific versions.
  47  * Like standards, there are a lot of atomic ops to choose from!
  48  */
  49
  50 #ifndef __ppc__
  51
  52 SInt32  OSAddAtomic(SInt32 amount, SInt32 * value)
  53 {
  54         SInt32  oldValue;
  55         SInt32  newValue;
  56
  57         do {
  58                 oldValue = *value;
  59                 newValue = oldValue + amount;
  60         } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
  61
  62         return oldValue;
  63 }
  64
  65 SInt32  OSIncrementAtomic(SInt32 * value)
  66 {
  67         return OSAddAtomic(1, value);
  68 }
  69
  70 SInt32  OSDecrementAtomic(SInt32 * value)
  71 {
  72         return OSAddAtomic(-1, value);
  73 }
  74
  75 void *  OSDequeueAtomic(void ** inList, SInt32 inOffset)
  76 {
  77         void *  oldListHead;
  78         void *  newListHead;
  79
  80         do {
  81                 oldListHead = *inList;
  82                 if (oldListHead == NULL) {
  83                         break;
  84                 }
  85
  86                 newListHead = *(void **) (((char *) oldListHead) + inOffset);
  87         } while (! OSCompareAndSwap((UInt32)oldListHead,
  88                                         (UInt32)newListHead, (UInt32 *)inList));
  89
  90         return oldListHead;
  91 }
  92
  93 void    OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
  94 {
  95         void *  oldListHead;
  96         void *  newListHead = inNewLink;
  97         void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
  98
  99         do {
 100                 oldListHead = *inList;
 101                 *newLinkNextPtr = oldListHead;
 102         } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 103                                         (UInt32 *)inList));
 104 }
 105
 106 #endif  /* !__ppc__ */
 107
 108 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
 109 {
 110         UInt32  oldValue;
 111         UInt32  newValue;
 112
 113         do {
 114                 oldValue = *value;
 115                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 116         } while (! OSCompareAndSwap(oldValue, newValue, value));
 117
 118         return oldValue;
 119 }
 120
 121 UInt32  OSBitAndAtomic(UInt32 mask, UInt32 * value)
 122 {
 123         return OSBitwiseAtomic(mask, 0, 0, value);
 124 }
 125
 126 UInt32  OSBitOrAtomic(UInt32 mask, UInt32 * value)
 127 {
 128         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 129 }
 130
 131 UInt32  OSBitXorAtomic(UInt32 mask, UInt32 * value)
 132 {
 133         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 134 }
 135
 136 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
 137 {
 138         UInt32          mask        = 0x000000ff;
 139         UInt32          alignment   = ((UInt32) value8) & (sizeof(UInt32) - 1);
 140     UInt32      shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 141     int                 shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 142         UInt32 *        value32     = (UInt32 *) (value8 - alignment);
 143     UInt32      oldValue;
 144     UInt32      newValue;
 145
 146     mask <<= shift;
 147
 148     oldValue = *value32;
 149     oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 150     newValue = (oldValue & ~mask) | (newValue8 << shift);
 151
 152         return OSCompareAndSwap(oldValue, newValue, value32);
 153 }
 154
 155 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
 156 {
 157         UInt8           mask = 1;
 158         UInt8           oldValue;
 159         UInt8           wantValue;
 160
 161         startAddress += (bit / 8);
 162         mask <<= (7 - (bit % 8));
 163         wantValue = wantSet ? mask : 0;
 164
 165         do {
 166                 oldValue = *startAddress;
 167                 if ((oldValue & mask) == wantValue) {
 168                         break;
 169                 }
 170         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 171
 172         return (oldValue & mask) == wantValue;
 173 }
 174
 175 Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress)
 176 {
 177         return OSTestAndSetClear(bit, true, startAddress);
 178 }
 179
 180 Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress)
 181 {
 182         return OSTestAndSetClear(bit, false, startAddress);
 183 }
 184
 185 /*
 186  * silly unaligned versions
 187  */
 188
 189 SInt8   OSIncrementAtomic8(SInt8 * value)
 190 {
 191         return OSAddAtomic8(1, value);
 192 }
 193
 194 SInt8   OSDecrementAtomic8(SInt8 * value)
 195 {
 196         return OSAddAtomic8(-1, value);
 197 }
 198
 199 SInt8   OSAddAtomic8(SInt32 amount, SInt8 * value)
 200 {
 201         SInt8   oldValue;
 202         SInt8   newValue;
 203
 204         do {
 205                 oldValue = *value;
 206                 newValue = oldValue + amount;
 207         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
 208
 209         return oldValue;
 210 }
 211
 212 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
 213 {
 214         UInt8   oldValue;
 215         UInt8   newValue;
 216
 217         do {
 218                 oldValue = *value;
 219                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 220         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 221
 222         return oldValue;
 223 }
 224
 225 UInt8   OSBitAndAtomic8(UInt32 mask, UInt8 * value)
 226 {
 227         return OSBitwiseAtomic8(mask, 0, 0, value);
 228 }
 229
 230 UInt8   OSBitOrAtomic8(UInt32 mask, UInt8 * value)
 231 {
 232         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 233 }
 234
 235 UInt8   OSBitXorAtomic8(UInt32 mask, UInt8 * value)
 236 {
 237         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 238 }
 239
 240 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
 241 {
 242         UInt32          mask        = 0x0000ffff;
 243         UInt32          alignment   = ((UInt32) value16) & (sizeof(UInt32) - 1);
 244     UInt32      shiftValues = (16 << 24) | (16 << 16);
 245     UInt32              shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 246         UInt32 *        value32     = (UInt32 *) (((UInt32) value16) - alignment);
 247     UInt32      oldValue;
 248     UInt32      newValue;
 249
 250     mask <<= shift;
 251
 252     oldValue = *value32;
 253     oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 254     newValue = (oldValue & ~mask) | (newValue16 << shift);
 255
 256         return OSCompareAndSwap(oldValue, newValue, value32);
 257 }
 258
 259 SInt16  OSIncrementAtomic16(SInt16 * value)
 260 {
 261         return OSAddAtomic16(1, value);
 262 }
 263
 264 SInt16  OSDecrementAtomic16(SInt16 * value)
 265 {
 266         return OSAddAtomic16(-1, value);
 267 }
 268
 269 SInt16  OSAddAtomic16(SInt32 amount, SInt16 * value)
 270 {
 271         SInt16  oldValue;
 272         SInt16  newValue;
 273
 274         do {
 275                 oldValue = *value;
 276                 newValue = oldValue + amount;
 277         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
 278
 279         return oldValue;
 280 }
 281
 282 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
 283 {
 284         UInt16  oldValue;
 285         UInt16  newValue;
 286
 287         do {
 288                 oldValue = *value;
 289                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 290         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 291
 292         return oldValue;
 293 }
 294
 295 UInt16  OSBitAndAtomic16(UInt32 mask, UInt16 * value)
 296 {
 297         return OSBitwiseAtomic16(mask, 0, 0, value);
 298 }
 299
 300 UInt16  OSBitOrAtomic16(UInt32 mask, UInt16 * value)
 301 {
 302         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 303 }
 304
 305 UInt16  OSBitXorAtomic16(UInt32 mask, UInt16 * value)
 306 {
 307         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 308 }
 309