x86_64/sys/OSAtomic.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <machine/cpu_capabilities.h>
  26 #include <platfunc.h>
  27
  28 #define DECLARE(x)   \
  29         .align 2, 0x90      ; \
  30         .globl x            ; \
  31         .globl x ## Barrier ; \
  32 x:                  ; \
  33 x ## Barrier:
  34
  35 .text
  36
  37 #define ATOMIC_UP       0
  38 #define ATOMIC_MP       1
  39 #define ATOMIC_RET_ORIG 0
  40 #define ATOMIC_RET_NEW  1
  41
  42 // compare and exchange 32-bit
  43 // xchg32 <new> <dst> <mp>
  44 .macro xchg32
  45         .if $2 == ATOMIC_MP
  46         lock
  47         .endif
  48         cmpxchgl        $0, ($1)
  49 .endm
  50
  51 // xchg64 <new> <dst> <mp>
  52 .macro xchg64
  53         .if $2 == ATOMIC_MP
  54         lock
  55         .endif
  56         cmpxchg         $0, ($1)
  57 .endm
  58
  59 #define ATOMIC_ARITHMETIC(instr, orig, mp)      \
  60         movl    (%rsi), %eax    /* get 2nd arg -> eax */ ;\
  61 1:      movl    %eax, %edx      /* copy value to new reg */ ;\
  62         instr   %edi, %edx      /* apply instr to %edx with arg2 */ ;\
  63         xchg32  %edx, %rsi, mp  /* do the compare swap (see macro above) */ ;\
  64         jnz     1b              /* jump if failed */ ;\
  65         .if orig == 1           /* to return the new value, overwrite eax */ ;\
  66         movl    %edx, %eax      /* return the new value */ ;\
  67         .endif
  68
  69 // Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
  70 // Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
  71 #define ATOMIC_BIT_OP(instr, mp)        \
  72         xorl    $7, %edi        /* bit position is numbered big endian so convert to little endian */ ;\
  73         shlq    $3, %rsi        ;\
  74         addq    %rdi, %rsi      /* generate bit address */ ;\
  75         movq    %rsi, %rdi      ;\
  76         andq    $31, %rdi       /* keep bit offset in range 0..31 */ ;\
  77         xorq    %rdi, %rsi      /* 4-byte align address */ ;\
  78         shrq    $3, %rsi        /* get 4-byte aligned address */ ;\
  79         .if mp == ATOMIC_MP     /* don't plant the lock in UP code */ ;\
  80         lock                    /* lock the bit test */ ;\
  81         .endif                  ;\
  82         instr   %edi, (%rsi)    /* do the bit test, supplied into the macro */ ;\
  83         setc    %al             ;\
  84         movzbl  %al,%eax        /* widen in case caller assumes we return an int */
  85
  86 // uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
  87 PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 64, 2)
  88 PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
  89         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
  90         ret
  91
  92 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
  93 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
  94         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
  95         ret
  96
  97 // uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
  98 PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 64, 2)
  99 PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
 100         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
 101         ret
 102
 103 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
 104 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
 105         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
 106         ret
 107
 108 // uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
 109 PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 64, 2)
 110 PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
 111         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
 112         ret
 113
 114 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
 115 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
 116         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
 117         ret
 118
 119 // uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
 120 PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 64, 2)
 121 PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
 122         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
 123         ret
 124
 125 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
 126 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
 127         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
 128         ret
 129
 130 // uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
 131 PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 64, 2)
 132 PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
 133         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
 134         ret
 135
 136 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
 137 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
 138         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
 139         ret
 140
 141 // uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
 142 PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 64, 2)
 143 PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
 144         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
 145         ret
 146
 147 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
 148 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
 149         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
 150         ret
 151
 152 // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
 153 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 64, 2)
 154 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
 155 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
 156 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
 157         movl    %edi, %eax
 158         xchg32  %esi, %rdx, ATOMIC_UP
 159         sete    %al
 160         movzbl  %al,%eax        // widen in case caller assumes we return an int
 161         ret
 162
 163 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
 164 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
 165 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
 166 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
 167         movl    %edi, %eax
 168         xchg32  %esi, %rdx, ATOMIC_MP
 169         sete    %al
 170         movzbl  %al,%eax        // widen in case caller assumes we return an int
 171         ret
 172
 173 // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
 174 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 64, 2)
 175 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
 176 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
 177 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
 178 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
 179 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
 180         mov     %rdi, %rax
 181         xchg64  %rsi, %rdx, ATOMIC_UP
 182         sete    %al
 183         movzbl  %al,%eax        // widen in case caller assumes we return an int
 184         ret
 185
 186 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
 187 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
 188 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
 189 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
 190 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
 191 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
 192         mov     %rdi, %rax
 193         xchg64  %rsi, %rdx, ATOMIC_MP
 194         sete    %al
 195         movzbl  %al,%eax        // widen in case caller assumes we return an int
 196         ret
 197
 198 // int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
 199 PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 64, 2)
 200 PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
 201         movl    %edi, %eax      // save amt to add
 202         xaddl   %edi, (%rsi)    // swap and add value, returns old value in %edi
 203         addl    %edi, %eax      // add old value to amt as return value
 204         ret
 205
 206 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
 207 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
 208         movl    %edi, %eax      // save amt to add
 209         lock                    // lock prefix breaks tabs ;)
 210         xaddl   %edi, (%rsi)    // swap and add value, returns old value in %edi
 211         addl    %edi, %eax      // add old value to amt as return value
 212         ret
 213
 214 // int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
 215 PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 64, 2)
 216 PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
 217         movq    %rdi, %rax      // save amt to add
 218         xadd    %rdi, (%rsi)    // swap and add value, returns old value in %rsi
 219         addq    %rdi, %rax      // add old value to amt as return value
 220         ret
 221
 222 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
 223 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
 224         movq    %rdi, %rax      // save amt to add
 225         lock
 226         xadd    %rdi, (%rsi)    // swap and add value, returns old value in %rsi
 227         addq    %rdi, %rax      // add old value to amt as return value
 228         ret
 229
 230 // bool OSAtomicTestAndSet( uint32_t n, void *value );
 231 PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 64, 2)
 232 PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
 233         ATOMIC_BIT_OP(btsl, ATOMIC_UP)
 234         ret
 235
 236 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
 237 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
 238         ATOMIC_BIT_OP(btsl, ATOMIC_MP)
 239         ret
 240
 241 // bool OSAtomicTestAndClear( uint32_t n, void *value );
 242 PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 64, 2)
 243 PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
 244         ATOMIC_BIT_OP(btrl, ATOMIC_UP)
 245         ret
 246
 247 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
 248 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
 249         ATOMIC_BIT_OP(btrl, ATOMIC_MP)
 250         ret
 251
 252 // void OSMemoryBarrier( void );
 253         .align  2, 0x90
 254         .globl  _OSMemoryBarrier
 255 _OSMemoryBarrier:
 256         mfence
 257         ret
 258
 259 /*
 260  *      typedef volatile struct {
 261  *              void    *opaque1;  <-- ptr to 1st queue element or null
 262  *              long     opaque2;  <-- generation count
 263  *      } OSQueueHead;
 264  *
 265  * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 266  */
 267         .align  2
 268         .globl  _OSAtomicEnqueue
 269 _OSAtomicEnqueue:               // %rdi == list head, %rsi == new, %rdx == offset
 270         pushq   %rbx
 271         movq    %rsi,%rbx       // %rbx == new
 272         movq    %rdx,%rsi       // %rsi == offset
 273         movq    (%rdi),%rax     // %rax == ptr to 1st element in Q
 274         movq    8(%rdi),%rdx    // %rdx == current generation count
 275 1:
 276         movq    %rax,(%rbx,%rsi)// link to old list head from new element
 277         movq    %rdx,%rcx
 278         incq    %rcx            // increment generation count
 279         lock                    // always lock for now...
 280         cmpxchg16b (%rdi)       // ...push on new element
 281         jnz     1b
 282         popq    %rbx
 283         ret
 284
 285
 286         /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 287         .align  2
 288         .globl  _OSAtomicDequeue
 289 _OSAtomicDequeue:               // %rdi == list head, %rsi == offset
 290         pushq   %rbx
 291         movq    (%rdi),%rax     // %rax == ptr to 1st element in Q
 292         movq    8(%rdi),%rdx    // %rdx == current generation count
 293 1:
 294         testq   %rax,%rax       // list empty?
 295         jz      2f              // yes
 296         movq    (%rax,%rsi),%rbx // point to 2nd in Q
 297         movq    %rdx,%rcx
 298         incq    %rcx            // increment generation count
 299         lock                    // always lock for now...
 300         cmpxchg16b (%rdi)       // ...pop off 1st element
 301         jnz     1b
 302 2:
 303         popq    %rbx
 304         ret                     // ptr to 1st element in Q still in %rax
 305
 306 /*
 307  *      typedef volatile struct {
 308  *              void    *opaque1;  <-- ptr to first queue element or null
 309  *              void    *opaque2;  <-- ptr to last queue element or null
 310  *              int      opaque3;  <-- spinlock
 311  *      } OSFifoQueueHead;
 312  *
 313  * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 314  */
 315         .align  2
 316         .globl  _OSAtomicFifoEnqueue
 317 _OSAtomicFifoEnqueue:
 318         pushq   %rbx
 319         xorl    %ebx,%ebx       // clear "preemption pending" flag
 320         movq    $(_COMM_PAGE_PFZ_ENQUEUE), %rcx
 321         call    *%rcx
 322         testl   %ebx,%ebx       // pending preemption?
 323         jz      1f
 324         call    _preempt        // call into the kernel to pfz_exit
 325 1:
 326         popq    %rbx
 327         ret
 328
 329
 330 /* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
 331         .align  2
 332         .globl  _OSAtomicFifoDequeue
 333 _OSAtomicFifoDequeue:
 334         pushq   %rbx
 335         xorl    %ebx,%ebx       // clear "preemption pending" flag
 336         movq    %rsi,%rdx       // move offset to %rdx to be like the Enqueue case
 337         movq    $(_COMM_PAGE_PFZ_DEQUEUE), %rcx
 338         call    *%rcx
 339         testl   %ebx,%ebx       // pending preemption?
 340         jz      1f
 341         call    _preempt        // call into the kernel to pfz_exit
 342 1:
 343         popq    %rbx
 344         ret                     // ptr to 1st element in Q in %rax
 345
 346 // Local Variables:
 347 // tab-width: 8
 348 // End: