libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <libkern/OSAtomic.h>
  24
  25 enum {
  26         false   = 0,
  27         true    = 1
  28 };
  29 #define NULL 0L
  30
  31
  32 /*
  33  * atomic operations
  34  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  35  *      which is implemented in assembler.  if we are worried about the cost of
  36  *      this layering (we shouldn't be), then all this stuff could be
  37  *      implemented in assembler, as it is in MacOS8/9
  38  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  39  *      which I wrote for NuKernel in a previous life with a different last name...)
  40  *
  41  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  42  */
  43
  44 #ifndef __ppc__
  45
  46 SInt32  OSAddAtomic(SInt32 amount, SInt32 * value)
  47 {
  48         SInt32  oldValue;
  49         SInt32  newValue;
  50
  51         do {
  52                 oldValue = *value;
  53                 newValue = oldValue + amount;
  54         } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
  55
  56         return oldValue;
  57 }
  58
  59 SInt32  OSIncrementAtomic(SInt32 * value)
  60 {
  61         return OSAddAtomic(1, value);
  62 }
  63
  64 SInt32  OSDecrementAtomic(SInt32 * value)
  65 {
  66         return OSAddAtomic(-1, value);
  67 }
  68
  69 #endif  /* !__ppc__ */
  70
  71 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
  72 {
  73         UInt32  oldValue;
  74         UInt32  newValue;
  75
  76         do {
  77                 oldValue = *value;
  78                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
  79         } while (! OSCompareAndSwap(oldValue, newValue, value));
  80
  81         return oldValue;
  82 }
  83
  84 UInt32  OSBitAndAtomic(UInt32 mask, UInt32 * value)
  85 {
  86         return OSBitwiseAtomic(mask, 0, 0, value);
  87 }
  88
  89 UInt32  OSBitOrAtomic(UInt32 mask, UInt32 * value)
  90 {
  91         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
  92 }
  93
  94 UInt32  OSBitXorAtomic(UInt32 mask, UInt32 * value)
  95 {
  96         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
  97 }
  98
  99
 100 static Boolean  OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
 101 {
 102         UInt32          mask = 0x000000ff;
 103         UInt32          newbits = (UInt32) newValue8;
 104         int                     shift;
 105         UInt32          alignment = ((UInt32) value8) & (sizeof(UInt32) - 1);
 106         UInt32          oldValue;
 107         UInt32          newValue;
 108         UInt32 *        value;
 109
 110         switch (alignment) {
 111                 default:
 112                         // assert(false);
 113                 case 0:
 114                         value = (UInt32 *) value8;
 115                         shift = 24;
 116                         break;
 117                 case 1:
 118                         value = (UInt32 *) (value8 + 1);
 119                         shift = 16;
 120                         break;
 121                 case 2:
 122                         value = (UInt32 *) (value8 + 2);
 123                         shift = 8;
 124                         break;
 125                 case 3:
 126                         value = (UInt32 *) (value8 + 3);
 127                         shift = 0;
 128                         break;
 129         }
 130
 131         mask <<= shift;
 132         newbits <<= shift;
 133
 134         oldValue = *value;
 135         newValue = (oldValue & ~mask) | (newbits & mask);
 136
 137         return OSCompareAndSwap(oldValue, newValue, value);
 138 }
 139
 140 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
 141 {
 142         UInt8           mask = 1;
 143         UInt8           oldValue;
 144         UInt8           wantValue;
 145
 146         startAddress += (bit / 8);
 147         mask <<= (7 - (bit % 8));
 148         wantValue = wantSet ? mask : 0;
 149
 150         do {
 151                 oldValue = *startAddress;
 152                 if ((oldValue & mask) == wantValue) {
 153                         break;
 154                 }
 155         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 156
 157         return (oldValue & mask) == wantValue;
 158 }
 159
 160 Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress)
 161 {
 162         return OSTestAndSetClear(bit, true, startAddress);
 163 }
 164
 165 Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress)
 166 {
 167         return OSTestAndSetClear(bit, false, startAddress);
 168 }
 169
 170 void *  OSDequeueAtomic(void ** inList, SInt32 inOffset)
 171 {
 172         void *  oldListHead;
 173         void *  newListHead;
 174
 175         do {
 176                 oldListHead = *inList;
 177                 if (oldListHead == NULL) {
 178                         break;
 179                 }
 180
 181                 newListHead = *(void **) (((char *) oldListHead) + inOffset);
 182         } while (! OSCompareAndSwap((UInt32)oldListHead,
 183                                         (UInt32)newListHead, (UInt32 *)inList));
 184
 185         return oldListHead;
 186 }
 187
 188 void    OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
 189 {
 190         void *  oldListHead;
 191         void *  newListHead = inNewLink;
 192         void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
 193
 194         do {
 195                 oldListHead = *inList;
 196                 *newLinkNextPtr = oldListHead;
 197         } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 198                                         (UInt32 *)inList));
 199 }
 200
 201 /*
 202  * silly unaligned versions
 203  */
 204
 205 SInt8   OSIncrementAtomic8(SInt8 * value)
 206 {
 207         return OSAddAtomic8(1, value);
 208 }
 209
 210 SInt8   OSDecrementAtomic8(SInt8 * value)
 211 {
 212         return OSAddAtomic8(-1, value);
 213 }
 214
 215 SInt8   OSAddAtomic8(SInt32 amount, SInt8 * value)
 216 {
 217         SInt8   oldValue;
 218         SInt8   newValue;
 219
 220         do {
 221                 oldValue = *value;
 222                 newValue = oldValue + amount;
 223         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
 224
 225         return oldValue;
 226 }
 227
 228 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
 229 {
 230         UInt8   oldValue;
 231         UInt8   newValue;
 232
 233         do {
 234                 oldValue = *value;
 235                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 236         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 237
 238         return oldValue;
 239 }
 240
 241 UInt8   OSBitAndAtomic8(UInt32 mask, UInt8 * value)
 242 {
 243         return OSBitwiseAtomic8(mask, 0, 0, value);
 244 }
 245
 246 UInt8   OSBitOrAtomic8(UInt32 mask, UInt8 * value)
 247 {
 248         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 249 }
 250
 251 UInt8   OSBitXorAtomic8(UInt32 mask, UInt8 * value)
 252 {
 253         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 254 }
 255
 256
 257 static Boolean  OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
 258 {
 259         UInt32          mask = 0x0000ffff;
 260         UInt32          newbits = (UInt32) newValue16;
 261         int                     shift;
 262         UInt32          alignment = ((UInt32) value16) & (sizeof(UInt32) - 1);
 263         UInt32          oldValue;
 264         UInt32          newValue;
 265         UInt32 *        value;
 266
 267         if (alignment == 2) {
 268                 value = (UInt32 *) (value16 - 1);
 269                 shift = 0;
 270         }
 271         else {
 272                 // assert(alignment == 0);
 273                 value = (UInt32 *) value16;
 274                 shift = 16;
 275         }
 276
 277         mask <<= shift;
 278         newbits <<= shift;
 279
 280         oldValue = *value;
 281         newValue = (oldValue & ~mask) | (newbits & mask);
 282
 283         return OSCompareAndSwap(oldValue, newValue, value);
 284 }
 285
 286 SInt16  OSIncrementAtomic16(SInt16 * value)
 287 {
 288         return OSAddAtomic16(1, value);
 289 }
 290
 291 SInt16  OSDecrementAtomic16(SInt16 * value)
 292 {
 293         return OSAddAtomic16(-1, value);
 294 }
 295
 296 SInt16  OSAddAtomic16(SInt32 amount, SInt16 * value)
 297 {
 298         SInt16  oldValue;
 299         SInt16  newValue;
 300
 301         do {
 302                 oldValue = *value;
 303                 newValue = oldValue + amount;
 304         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
 305
 306         return oldValue;
 307 }
 308
 309 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
 310 {
 311         UInt16  oldValue;
 312         UInt16  newValue;
 313
 314         do {
 315                 oldValue = *value;
 316                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 317         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 318
 319         return oldValue;
 320 }
 321
 322 UInt16  OSBitAndAtomic16(UInt32 mask, UInt16 * value)
 323 {
 324         return OSBitwiseAtomic16(mask, 0, 0, value);
 325 }
 326
 327 UInt16  OSBitOrAtomic16(UInt32 mask, UInt16 * value)
 328 {
 329         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 330 }
 331
 332 UInt16  OSBitXorAtomic16(UInt32 mask, UInt16 * value)
 333 {
 334         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 335 }
 336