libkern/gen/OSAtomicOperations.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25
  26 #include <libkern/OSAtomic.h>
  27
  28 enum {
  29         false   = 0,
  30         true    = 1
  31 };
  32 #define NULL 0L
  33
  34
  35 /*
  36  * atomic operations
  37  *      these are _the_ atomic operations, currently cast atop CompareAndSwap,
  38  *      which is implemented in assembler.  if we are worried about the cost of
  39  *      this layering (we shouldn't be), then all this stuff could be
  40  *      implemented in assembler, as it is in MacOS8/9
  41  *      (derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
  42  *      which I wrote for NuKernel in a previous life with a different last name...)
  43  *
  44  * native Boolean       CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
  45  */
  46
  47 #ifndef __ppc__
  48
  49 SInt32  OSAddAtomic(SInt32 amount, SInt32 * value)
  50 {
  51         SInt32  oldValue;
  52         SInt32  newValue;
  53
  54         do {
  55                 oldValue = *value;
  56                 newValue = oldValue + amount;
  57         } while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
  58
  59         return oldValue;
  60 }
  61
  62 SInt32  OSIncrementAtomic(SInt32 * value)
  63 {
  64         return OSAddAtomic(1, value);
  65 }
  66
  67 SInt32  OSDecrementAtomic(SInt32 * value)
  68 {
  69         return OSAddAtomic(-1, value);
  70 }
  71
  72 #endif  /* !__ppc__ */
  73
  74 static UInt32   OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
  75 {
  76         UInt32  oldValue;
  77         UInt32  newValue;
  78
  79         do {
  80                 oldValue = *value;
  81                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
  82         } while (! OSCompareAndSwap(oldValue, newValue, value));
  83
  84         return oldValue;
  85 }
  86
  87 UInt32  OSBitAndAtomic(UInt32 mask, UInt32 * value)
  88 {
  89         return OSBitwiseAtomic(mask, 0, 0, value);
  90 }
  91
  92 UInt32  OSBitOrAtomic(UInt32 mask, UInt32 * value)
  93 {
  94         return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
  95 }
  96
  97 UInt32  OSBitXorAtomic(UInt32 mask, UInt32 * value)
  98 {
  99         return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 100 }
 101
 102 static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
 103 {
 104         UInt32          mask        = 0x000000ff;
 105         UInt32          alignment   = ((UInt32) value8) & (sizeof(UInt32) - 1);
 106     UInt32      shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
 107     int                 shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 108         UInt32 *        value32     = (UInt32 *) (value8 - alignment);
 109     UInt32      oldValue;
 110     UInt32      newValue;
 111
 112     mask <<= shift;
 113
 114     oldValue = *value32;
 115     oldValue = (oldValue & ~mask) | (oldValue8 << shift);
 116     newValue = (oldValue & ~mask) | (newValue8 << shift);
 117
 118         return OSCompareAndSwap(oldValue, newValue, value32);
 119 }
 120
 121 static Boolean  OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
 122 {
 123         UInt8           mask = 1;
 124         UInt8           oldValue;
 125         UInt8           wantValue;
 126
 127         startAddress += (bit / 8);
 128         mask <<= (7 - (bit % 8));
 129         wantValue = wantSet ? mask : 0;
 130
 131         do {
 132                 oldValue = *startAddress;
 133                 if ((oldValue & mask) == wantValue) {
 134                         break;
 135                 }
 136         } while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
 137
 138         return (oldValue & mask) == wantValue;
 139 }
 140
 141 Boolean OSTestAndSet(UInt32 bit, UInt8 * startAddress)
 142 {
 143         return OSTestAndSetClear(bit, true, startAddress);
 144 }
 145
 146 Boolean OSTestAndClear(UInt32 bit, UInt8 * startAddress)
 147 {
 148         return OSTestAndSetClear(bit, false, startAddress);
 149 }
 150
 151 void *  OSDequeueAtomic(void ** inList, SInt32 inOffset)
 152 {
 153         void *  oldListHead;
 154         void *  newListHead;
 155
 156         do {
 157                 oldListHead = *inList;
 158                 if (oldListHead == NULL) {
 159                         break;
 160                 }
 161
 162                 newListHead = *(void **) (((char *) oldListHead) + inOffset);
 163         } while (! OSCompareAndSwap((UInt32)oldListHead,
 164                                         (UInt32)newListHead, (UInt32 *)inList));
 165
 166         return oldListHead;
 167 }
 168
 169 void    OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
 170 {
 171         void *  oldListHead;
 172         void *  newListHead = inNewLink;
 173         void ** newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
 174
 175         do {
 176                 oldListHead = *inList;
 177                 *newLinkNextPtr = oldListHead;
 178         } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 179                                         (UInt32 *)inList));
 180 }
 181
 182 /*
 183  * silly unaligned versions
 184  */
 185
 186 SInt8   OSIncrementAtomic8(SInt8 * value)
 187 {
 188         return OSAddAtomic8(1, value);
 189 }
 190
 191 SInt8   OSDecrementAtomic8(SInt8 * value)
 192 {
 193         return OSAddAtomic8(-1, value);
 194 }
 195
 196 SInt8   OSAddAtomic8(SInt32 amount, SInt8 * value)
 197 {
 198         SInt8   oldValue;
 199         SInt8   newValue;
 200
 201         do {
 202                 oldValue = *value;
 203                 newValue = oldValue + amount;
 204         } while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
 205
 206         return oldValue;
 207 }
 208
 209 static UInt8    OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
 210 {
 211         UInt8   oldValue;
 212         UInt8   newValue;
 213
 214         do {
 215                 oldValue = *value;
 216                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 217         } while (! OSCompareAndSwap8(oldValue, newValue, value));
 218
 219         return oldValue;
 220 }
 221
 222 UInt8   OSBitAndAtomic8(UInt32 mask, UInt8 * value)
 223 {
 224         return OSBitwiseAtomic8(mask, 0, 0, value);
 225 }
 226
 227 UInt8   OSBitOrAtomic8(UInt32 mask, UInt8 * value)
 228 {
 229         return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 230 }
 231
 232 UInt8   OSBitXorAtomic8(UInt32 mask, UInt8 * value)
 233 {
 234         return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 235 }
 236
 237 static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
 238 {
 239         UInt32          mask        = 0x0000ffff;
 240         UInt32          alignment   = ((UInt32) value16) & (sizeof(UInt32) - 1);
 241     UInt32      shiftValues = (16 << 24) | (16 << 16);
 242     UInt32              shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
 243         UInt32 *        value32     = (UInt32 *) (((UInt32) value16) - alignment);
 244     UInt32      oldValue;
 245     UInt32      newValue;
 246
 247     mask <<= shift;
 248
 249     oldValue = *value32;
 250     oldValue = (oldValue & ~mask) | (oldValue16 << shift);
 251     newValue = (oldValue & ~mask) | (newValue16 << shift);
 252
 253         return OSCompareAndSwap(oldValue, newValue, value32);
 254 }
 255
 256 SInt16  OSIncrementAtomic16(SInt16 * value)
 257 {
 258         return OSAddAtomic16(1, value);
 259 }
 260
 261 SInt16  OSDecrementAtomic16(SInt16 * value)
 262 {
 263         return OSAddAtomic16(-1, value);
 264 }
 265
 266 SInt16  OSAddAtomic16(SInt32 amount, SInt16 * value)
 267 {
 268         SInt16  oldValue;
 269         SInt16  newValue;
 270
 271         do {
 272                 oldValue = *value;
 273                 newValue = oldValue + amount;
 274         } while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
 275
 276         return oldValue;
 277 }
 278
 279 static UInt16   OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
 280 {
 281         UInt16  oldValue;
 282         UInt16  newValue;
 283
 284         do {
 285                 oldValue = *value;
 286                 newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
 287         } while (! OSCompareAndSwap16(oldValue, newValue, value));
 288
 289         return oldValue;
 290 }
 291
 292 UInt16  OSBitAndAtomic16(UInt32 mask, UInt16 * value)
 293 {
 294         return OSBitwiseAtomic16(mask, 0, 0, value);
 295 }
 296
 297 UInt16  OSBitOrAtomic16(UInt32 mask, UInt16 * value)
 298 {
 299         return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 300 }
 301
 302 UInt16  OSBitXorAtomic16(UInt32 mask, UInt16 * value)
 303 {
 304         return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 305 }
 306