darwin/OSAtomicLoadStoreEx.c

   1 /*
   2  * Copyright (c) 2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 // OSAtomic.h is included by files that include this C file.
  25 /* #include "OSAtomic.h" */
  26
  27 int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAdd32);
  28 int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p)
  29 {
  30         _OSATOMIC_ALIAS_NB(OSAtomicAdd32);
  31         int32_t r;
  32         uint32_t t;
  33
  34         _osatomic_store_barrier();
  35         do {
  36                 _osatomic_load_exclusive(p, r);
  37                 r += v;
  38                 _osatomic_store_exclusive(p, r, t);
  39         } while (slowpath(t));
  40
  41         _osatomic_barrier();
  42         return r;
  43 }
  44
  45 int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p) _OSATOMIC_VARIANT_B(OSAtomicAdd64);
  46 int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p)
  47 {
  48         _OSATOMIC_ALIAS_NB(OSAtomicAdd64);
  49
  50         int64_t r;
  51         uint32_t t;
  52
  53         _osatomic_store_barrier();
  54         do {
  55                 _osatomic_load_exclusive64(p, r);
  56                 r += v;
  57                 _osatomic_store_exclusive64(p, r, t);
  58         } while (slowpath(t));
  59
  60         _osatomic_barrier();
  61         return r;
  62 }
  63
  64 int32_t OSAtomicOr32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicOr32);
  65 int32_t OSAtomicOr32(int32_t v, volatile int32_t *p)
  66 {
  67         _OSATOMIC_ALIAS_NB(OSAtomicOr32);
  68         int32_t r;
  69         uint32_t t;
  70
  71         _osatomic_store_barrier();
  72         do {
  73                 _osatomic_load_exclusive(p, r);
  74                 r |= v;
  75                 _osatomic_store_exclusive(p, r, t);
  76         } while (slowpath(t));
  77
  78         _osatomic_barrier();
  79         return r;
  80 }
  81
  82 int32_t OSAtomicOr32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicOr32Orig);
  83 int32_t OSAtomicOr32Orig(int32_t v, volatile int32_t *p)
  84 {
  85         _OSATOMIC_ALIAS_NB(OSAtomicOr32Orig);
  86         int32_t r, n;
  87         uint32_t t;
  88
  89         _osatomic_store_barrier();
  90         do {
  91                 _osatomic_load_exclusive(p, r);
  92                 n = r | v;
  93                 _osatomic_store_exclusive(p, n, t);
  94         } while (slowpath(t));
  95
  96         _osatomic_barrier();
  97         return r;
  98 }
  99
 100 int32_t OSAtomicAnd32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAnd32);
 101 int32_t OSAtomicAnd32(int32_t v, volatile int32_t *p)
 102 {
 103         _OSATOMIC_ALIAS_NB(OSAtomicAnd32);
 104         int32_t r;
 105         uint32_t t;
 106
 107         _osatomic_store_barrier();
 108         do {
 109                 _osatomic_load_exclusive(p, r);
 110                 r &= v;
 111                 _osatomic_store_exclusive(p, r, t);
 112         } while (slowpath(t));
 113
 114         _osatomic_barrier();
 115         return r;
 116 }
 117
 118 int32_t OSAtomicAnd32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicAnd32Orig);
 119 int32_t OSAtomicAnd32Orig(int32_t v, volatile int32_t *p)
 120 {
 121         _OSATOMIC_ALIAS_NB(OSAtomicAnd32Orig);
 122         int32_t r, n;
 123         uint32_t t;
 124
 125         _osatomic_store_barrier();
 126         do {
 127                 _osatomic_load_exclusive(p, r);
 128                 n = r & v;
 129                 _osatomic_store_exclusive(p, n, t);
 130         } while (slowpath(t));
 131
 132         _osatomic_barrier();
 133         return r;
 134 }
 135
 136 int32_t OSAtomicXor32(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicXor32);
 137 int32_t OSAtomicXor32(int32_t v, volatile int32_t *p)
 138 {
 139         _OSATOMIC_ALIAS_NB(OSAtomicXor32);
 140         int32_t r;
 141         uint32_t t;
 142
 143         _osatomic_store_barrier();
 144         do {
 145                 _osatomic_load_exclusive(p, r);
 146                 r ^= v;
 147                 _osatomic_store_exclusive(p, r, t);
 148         } while (slowpath(t));
 149
 150         _osatomic_barrier();
 151         return r;
 152 }
 153
 154 int32_t OSAtomicXor32Orig(int32_t v, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicXor32Orig);
 155 int32_t OSAtomicXor32Orig(int32_t v, volatile int32_t *p)
 156 {
 157         _OSATOMIC_ALIAS_NB(OSAtomicXor32Orig);
 158         int32_t r, n;
 159         uint32_t t;
 160
 161         _osatomic_store_barrier();
 162         do {
 163                 _osatomic_load_exclusive(p, r);
 164                 n = r ^ v;
 165                 _osatomic_store_exclusive(p, n, t);
 166         } while (slowpath(t));
 167
 168         _osatomic_barrier();
 169         return r;
 170 }
 171
 172 bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p) _OSATOMIC_VARIANT_B(OSAtomicCompareAndSwap32);
 173 bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p)
 174 {
 175         _OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapInt, OSAtomicCompareAndSwap32);
 176
 177         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwap32);
 178         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapInt);
 179 #ifndef __LP64__
 180         _OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapLong, OSAtomicCompareAndSwap32);
 181         _OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapPtr, OSAtomicCompareAndSwap32);
 182         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapLong);
 183         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapPtr);
 184 #endif
 185
 186         int32_t r;
 187         uint32_t t;
 188
 189         do {
 190                 _osatomic_load_exclusive(p, r);
 191                 if (r != o) return false;
 192                 _osatomic_store_barrier();
 193                 _osatomic_store_exclusive(p, n, t);
 194         } while (slowpath(t));
 195
 196         _osatomic_barrier();
 197         return true;
 198 }
 199
 200 bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p) _OSATOMIC_VARIANT_B(OSAtomicCompareAndSwap64);
 201 bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p)
 202 {
 203         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwap64);
 204 #ifdef __LP64__
 205         _OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapLong, OSAtomicCompareAndSwap64);
 206         _OSATOMIC_ALIAS_B(OSAtomicCompareAndSwapPtr, OSAtomicCompareAndSwap64);
 207         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapLong);
 208         _OSATOMIC_ALIAS_NB(OSAtomicCompareAndSwapPtr);
 209 #endif
 210
 211         int64_t r;
 212         uint32_t t;
 213
 214         do {
 215                 _osatomic_load_exclusive64(p, r);
 216                 if (r != o) return false;
 217                 _osatomic_store_barrier();
 218                 _osatomic_store_exclusive64(p, n, t);
 219         } while (slowpath(t));
 220
 221         _osatomic_barrier();
 222         return true;
 223 }
 224
 225 #if defined(_OSATOMIC_EXTRAS)
 226
 227 void OSMemoryBarrier(void) _OSATOMIC_VARIANT(OSMemoryBarrier);
 228 void OSMemoryBarrier(void)
 229 {
 230         _osatomic_barrier();
 231 }
 232
 233 typedef volatile struct {
 234         void    *item;
 235         long     unused;
 236 } OSQueueHead;
 237
 238 void OSAtomicEnqueue(OSQueueHead *l, void *n, size_t o) _OSATOMIC_VARIANT(OSAtomicEnqueue);
 239 void OSAtomicEnqueue(OSQueueHead *l, void *n, size_t o)
 240 {
 241         void ** r = (void **)((char *)n + o);
 242         void * volatile * i = (void **)&(l->item);
 243         void * q;
 244         uint32_t t;
 245
 246         _osatomic_store_barrier();
 247         do {
 248                 _osatomic_load_exclusive(i, q);
 249                 *r = q;
 250                 _osatomic_store_exclusive(i, n, t);
 251         } while (slowpath(t));
 252         _osatomic_barrier();
 253 }
 254
 255 void* OSAtomicDequeue(OSQueueHead *l, size_t o) _OSATOMIC_VARIANT(OSAtomicDequeue);
 256 void* OSAtomicDequeue(OSQueueHead *l, size_t o)
 257 {
 258         void * h;
 259         void ** r;
 260         void * volatile * i = (void **)&(l->item);
 261         uint32_t t;
 262
 263         _osatomic_store_barrier();
 264         do {
 265                 _osatomic_load_exclusive(i, r);
 266                 if (!r) return NULL;
 267                 h = *(void **)((char *)r + o);
 268                 _osatomic_store_exclusive(i, h, t);
 269         } while (slowpath(t));
 270
 271         _osatomic_barrier();
 272         return r;
 273 }
 274
 275 #endif
 276
 277 bool OSAtomicTestAndSet(uint32_t n, volatile void * p) _OSATOMIC_VARIANT_B(OSAtomicTestAndSet);
 278 bool OSAtomicTestAndSet(uint32_t n, volatile void * p)
 279 {
 280         _OSATOMIC_ALIAS_NB(OSAtomicTestAndSet);
 281         uint32_t * ptr = (uint32_t *)((char *)p + (4 * (n / 32)));
 282         uint32_t i, r, t;
 283
 284         n = (0x80 >> (n & 7)) << (n & ~7 & 31);
 285
 286         _osatomic_store_barrier();
 287         do {
 288                 _osatomic_load_exclusive(ptr, r);
 289                 i = r | n;
 290                 if (i == r) break;
 291                 _osatomic_store_exclusive(ptr, i, t);
 292         } while (slowpath(t));
 293
 294         _osatomic_barrier();
 295         return ((r & n) != 0);
 296 }
 297
 298 bool OSAtomicTestAndClear(uint32_t n, volatile void * p) _OSATOMIC_VARIANT_B(OSAtomicTestAndClear);
 299 bool OSAtomicTestAndClear(uint32_t n, volatile void * p)
 300 {
 301         _OSATOMIC_ALIAS_NB(OSAtomicTestAndClear);
 302         uint32_t * ptr = (uint32_t *)((char *)p + (4 * (n / 32)));
 303         uint32_t i, r, t;
 304
 305         n = (0x80 >> (n & 7)) << (n & ~7 & 31);
 306
 307         _osatomic_store_barrier();
 308         do {
 309                 _osatomic_load_exclusive(ptr, r);
 310                 i = r & ~n;
 311                 if (i == r) break;
 312                 _osatomic_store_exclusive(ptr, i, t);
 313         } while (slowpath(t));
 314
 315         _osatomic_barrier();
 316         return ((r & n) != 0);
 317 }
 318