arm/sys/OSAtomic.h

   1 /*
   2  * Copyright (c) 2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #ifndef __ARM_OSATOMIC_SHIMS__
  25 #define __ARM_OSATOMIC_SHIMS__
  26
  27 #include <sys/cdefs.h>
  28 #include <sys/types.h>
  29 #include <unistd.h>
  30 #include <stdint.h>
  31 #include <stdbool.h>
  32
  33 #define fastpath(x)     ((typeof(x))__builtin_expect((long)(x), ~0l))
  34 #define slowpath(x)     ((typeof(x))__builtin_expect((long)(x), 0l))
  35
  36 #if defined(__arm__)
  37 #include <arm/arch.h>
  38
  39 #define _osatomic_load_exclusive(p, r) \
  40         __asm__ __volatile__( \
  41         "ldrex  %[_r], %[_p]" \
  42         : [_r] "=&r" (r) \
  43         : [_p] "m" (*(p)) : "memory")
  44
  45 #define _osatomic_load_exclusive64(p, r) \
  46         do { \
  47                 register uint32_t rl asm("r4"), rh asm("r5"); \
  48                 __asm__ __volatile__( \
  49                         "ldrexd %[_rl], %[_rh], %[_p]" \
  50                         : [_rl] "=r" (rl), [_rh] "=r" (rh) \
  51                         : [_p] "m" (*(p)) : "memory"); \
  52                 r = (typeof(r))(((uint64_t)rh << 32) | (uint64_t)rl); \
  53         } while (0);
  54
  55 #define _osatomic_store_exclusive(p, r, t) \
  56         __asm__ __volatile__( \
  57         "strex  %[_t], %[_r], %[_p]" \
  58         : [_t] "=&r" (t) \
  59         : [_p] "m" (*(p)), [_r] "r" (r) : "memory")
  60
  61 #define _osatomic_store_exclusive64(p, r, t) \
  62         do { \
  63                 register uint32_t rl asm("r4"), rh asm("r5"); \
  64                 rh = (uint32_t)(r >> 32); \
  65                 rl = (uint32_t)(r); \
  66                 __asm__ __volatile__( \
  67                         "strexd %[_t], %[_rl], %[_rh], %[_p]" \
  68                         : [_t] "=&r" (t) \
  69                         : [_p] "m" (*(p)), [_rl] "r" (rl), [_rh] "r" (rh) : "memory"); \
  70         } while (0);
  71
  72 #if defined(_ARM_ARCH_7)
  73
  74 #ifdef _OSATOMIC_WFE
  75 #       define MP_SPIN_TRIES            10
  76 #       define _osatomic_pause()        __asm__ __volatile__("wfe")
  77 #else
  78 #       define MP_SPIN_TRIES            1000
  79 #       define _osatomic_pause()        __asm__ __volatile__("yield")
  80 #endif // _OSATOMIC_WFE
  81
  82 #define _osatomic_barrier() \
  83         __asm__ __volatile__( \
  84         "dmb    ish" \
  85         : : : "memory")
  86
  87 #define _osatomic_store_barrier() \
  88         __asm__ __volatile__( \
  89         "dmb    ishst" \
  90         : : : "memory")
  91
  92 #elif defined(_ARM_ARCH_6) && !defined(__thumb__) // _ARM_ARCH_7
  93
  94 #define _osatomic_barrier() \
  95         __asm__ __volatile__( \
  96         "mcr    p15, 0, %0, c7, c10, 5" \
  97         : : "r" (0) : "memory")
  98
  99 #define _osatomic_store_barrier() _osatomic_barrier()
 100 #define _osatomic_pause()
 101
 102 #elif defined(_ARM_ARCH_6) && defined(__thumb__)
 103 #error Atomic operations not available on ARMv6 Thumb
 104 #endif // _ARM_ARCH_7
 105
 106 #if !defined(_ARM_ARCH_7)
 107 #       ifndef _OSATOMIC_NO_BARRIERS
 108 #               define __OSATOMIC_SUFFIX        ""
 109 #               define __OSATOMIC_BARRIER       "Barrier"
 110 #               define _OSATOMIC_ALIAS_NB(x)
 111 #       else
 112 #               define _OSATOMIC_EXTRAS                 1
 113 #               define __OSATOMIC_SUFFIX        ""
 114 #               define __OSATOMIC_BARRIER       ""
 115 #               define _OSATOMIC_ALIAS_NB(x)
 116 #       endif // _OSATOMIC_NO_BARRIERS
 117 #else
 118 #       define _OSATOMIC_EXTRAS                 1
 119 #       ifndef _OSATOMIC_NO_BARRIERS
 120 #               ifdef _OSATOMIC_WFE
 121 #                       define __OSATOMIC_SUFFIX "$VARIANT$wfe"
 122 #               else
 123 #                       define __OSATOMIC_SUFFIX "$VARIANT$mp"
 124 #               endif // _OSATOMIC_ONLY_WFE
 125 #               define __OSATOMIC_BARRIER "Barrier" __OSATOMIC_SUFFIX
 126 #               define _OSATOMIC_ALIAS_NB(x)
 127 #       else
 128 #               define __OSATOMIC_SUFFIX "$VARIANT$up"
 129 #               define __OSATOMIC_BARRIER "Barrier" __OSATOMIC_SUFFIX
 130 // Undefine the barrier operations
 131 #               undef _osatomic_barrier
 132 #               define _osatomic_barrier()
 133 #               undef _osatomic_store_barrier
 134 #               define _osatomic_store_barrier()
 135 // Aliases to alias non-barrier operations to UP variant names
 136 #               if defined(__thumb__)
 137 #                       define _OSATOMIC_ALIAS_NB(sym)  __asm __volatile__(".globl _" __STRING(sym) "; .thumb_func _" __STRING(sym) " ; .set _" __STRING(sym) ", _" __STRING(sym) __OSATOMIC_BARRIER)
 138 #               else
 139 #                       define _OSATOMIC_ALIAS_NB(sym)  __asm __volatile__(".globl _" __STRING(sym) "; .set _" __STRING(sym) ", _" __STRING(sym) __OSATOMIC_BARRIER)
 140 #               endif // __thumb__
 141 #       endif // _OSATOMIC_NO_BARRIERS
 142 #endif // !_ARM_ARCH_7
 143
 144 // Allow us to mark up functions as being uni/multiprocessor.
 145 #define _OSATOMIC_VARIANT(sym)  __asm("_" __STRING(sym) __OSATOMIC_SUFFIX)
 146 #define _OSATOMIC_VARIANT_B(sym)        __asm("_" __STRING(sym) __OSATOMIC_BARRIER)
 147
 148 #if defined(__thumb__)
 149         // For aliasing CompareAndSwap operations of the same size
 150         #define _OSATOMIC_ALIAS_B(newsym, sym)  \
 151                 __asm __volatile__( \
 152                         ".globl _" __STRING(newsym) __OSATOMIC_BARRIER ";" \
 153                         ".thumb_func _" __STRING(newsym) __OSATOMIC_BARRIER ";" \
 154                         ".set _" __STRING(newsym) __OSATOMIC_BARRIER ", _" __STRING(sym) __OSATOMIC_BARRIER \
 155                 )
 156
 157         #define _OSATOMIC_ALIAS(newsym, sym)    \
 158                 __asm __volatile__( \
 159                         ".globl _" __STRING(newsym) __OSATOMIC_SUFFIX ";" \
 160                         ".thumb_func _" __STRING(newsym) __OSATOMIC_SUFFIX ";" \
 161                         ".set _" __STRING(newsym) __OSATOMIC_SUFFIX ", _" __STRING(sym) __OSATOMIC_SUFFIX \
 162                 )
 163 #else
 164         // For aliasing CompareAndSwap operations of the same size
 165         #define _OSATOMIC_ALIAS_B(newsym, sym)  \
 166                 __asm __volatile__( \
 167                         ".globl _" __STRING(newsym) __OSATOMIC_BARRIER ";" \
 168                         ".set _" __STRING(newsym) __OSATOMIC_BARRIER ", _" __STRING(sym) __OSATOMIC_BARRIER \
 169                 )
 170
 171         #define _OSATOMIC_ALIAS(newsym, sym)    \
 172                 __asm __volatile__( \
 173                         ".globl _" __STRING(newsym) __OSATOMIC_SUFFIX ";" \
 174                         ".set _" __STRING(newsym) __OSATOMIC_SUFFIX ", _" __STRING(sym) __OSATOMIC_SUFFIX \
 175                 )
 176 #endif // __thumb__
 177
 178 #endif // __arm__
 179
 180 #endif // __ARM_OSATOMIC_SHIMS__