x86_64/sys/OSAtomic.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <machine/cpu_capabilities.h>
  26
  27 #define DECLARE(x)   \
  28 .align 2, 0x90      ; \
  29 .globl x            ; \
  30 .globl x ## Barrier ; \
  31 x:                  ; \
  32 x ## Barrier:
  33
  34 .text
  35
  36
  37 // uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value);
  38 DECLARE(_OSAtomicAnd32)
  39         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
  40         movl    %edi, %r11d     // save mask
  41         movl    (%rsi), %eax    // get value
  42         movq    %rsi, %rdx      // put ptr where compare-and-swap expects it
  43 1:
  44         movl    %r11d, %esi     // original mask
  45         movl    %eax, %edi      // old value
  46         andl    %eax, %esi      // new value
  47         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
  48         jnz     1b
  49         movl    %esi, %eax
  50         ret
  51
  52
  53 // uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value);
  54 DECLARE(_OSAtomicOr32)
  55         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
  56         movl    %edi, %r11d     // save mask
  57         movl    (%rsi), %eax    // get value
  58         movq    %rsi, %rdx      // put ptr where compare-and-swap expects it
  59 1:
  60         movl    %r11d, %esi     // original mask
  61         movl    %eax, %edi      // old value
  62         orl     %eax, %esi      // new value
  63         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
  64         jnz     1b
  65         movl    %esi, %eax
  66         ret
  67
  68
  69 // uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value);
  70 DECLARE(_OSAtomicXor32)
  71         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
  72         movl    %edi, %r11d     // save mask
  73         movl    (%rsi), %eax    // get value
  74         movq    %rsi, %rdx      // put ptr where compare-and-swap expects it
  75 1:
  76         movl    %r11d, %esi     // original mask
  77         movl    %eax, %edi      // old value
  78         xorl    %eax, %esi      // new value
  79         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
  80         jnz     1b
  81         movl    %esi, %eax
  82         ret
  83
  84
  85 // uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value);
  86 DECLARE(_OSAtomicAnd32Orig)
  87         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
  88         movl    %edi, %r11d     // save mask
  89         movl    (%rsi), %eax    // get value
  90         movq    %rsi, %rdx      // put ptr where compare-and-swap expects it
  91 1:
  92         movl    %r11d, %esi     // original mask
  93         movl    %eax, %edi      // old value
  94         andl    %eax, %esi      // new value
  95         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
  96         jnz     1b
  97         movl    %edi, %eax
  98         ret
  99
 100
 101 // uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value);
 102 DECLARE(_OSAtomicOr32Orig)
 103         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
 104         movl    %edi, %r11d     // save mask
 105         movl    (%rsi), %eax    // get value
 106         movq    %rsi, %rdx      // put ptr where compare-and-swap expects it
 107 1:
 108         movl    %r11d, %esi     // original mask
 109         movl    %eax, %edi      // old value
 110         orl     %eax, %esi      // new value
 111         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
 112         jnz     1b
 113         movl    %edi, %eax
 114         ret
 115
 116
 117 // uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value);
 118 DECLARE(_OSAtomicXor32Orig)
 119         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
 120         movl    %edi, %r11d     // save mask
 121         movl    (%rsi), %eax    // get value
 122         movq    %rsi, %rdx      // put ptr where compare-and-swap expects it
 123 1:
 124         movl    %r11d, %esi     // original mask
 125         movl    %eax, %edi      // old value
 126         xorl    %eax, %esi      // new value
 127         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
 128         jnz     1b
 129         movl    %edi, %eax
 130         ret
 131
 132
 133 // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
 134 DECLARE(_OSAtomicCompareAndSwapInt)
 135 DECLARE(_OSAtomicCompareAndSwap32)
 136         movq    $(_COMM_PAGE_COMPARE_AND_SWAP32), %rcx
 137         call    *%rcx           // %edi=old value,  %esi=new value. %rdx=ptr
 138         sete    %al
 139         movzbl  %al,%eax        // widen in case caller assumes we return an int
 140         ret
 141
 142
 143 // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
 144 DECLARE(_OSAtomicCompareAndSwapPtr)
 145 DECLARE(_OSAtomicCompareAndSwapLong)
 146 DECLARE(_OSAtomicCompareAndSwap64)
 147         movq    $(_COMM_PAGE_COMPARE_AND_SWAP64), %rcx
 148         call    *%rcx           // %rdi=old value,  %rsi=new value. %rdx=ptr
 149         sete    %al
 150         movzbl  %al,%eax        // widen in case caller assumes we return an int
 151         ret
 152
 153
 154 // int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
 155 DECLARE(_OSAtomicAdd32)
 156         movq    $(_COMM_PAGE_ATOMIC_ADD32), %rcx
 157         movl    %edi, %eax      // save amt to add
 158         call    *%rcx
 159         addl    %edi,%eax       // new value
 160         ret
 161
 162
 163 // int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
 164 DECLARE(_OSAtomicAdd64)
 165         movq    $(_COMM_PAGE_ATOMIC_ADD64), %rcx
 166         movq    %rdi, %rax      // save amt to add
 167         call    *%rcx
 168         addq    %rdi, %rax      // new value
 169         ret
 170
 171
 172 // bool OSAtomicTestAndSet( uint32_t n, void *value );
 173 DECLARE(_OSAtomicTestAndSet)
 174         movq    $(_COMM_PAGE_BTS), %rax
 175         xorl    $7, %edi        // bit position is numbered big endian
 176         call    *%rax
 177         setc    %al
 178         movzbl  %al,%eax        // widen in case caller assumes we return an int
 179         ret
 180
 181
 182 // bool OSAtomicTestAndClear( uint32_t n, void *value );
 183 DECLARE(_OSAtomicTestAndClear)
 184         movq    $(_COMM_PAGE_BTC), %rax
 185         xorl    $7, %edi        // bit position is numbered big endian
 186         call    *%rax
 187         setc    %al
 188         movzbl  %al,%eax        // widen in case caller assumes we return an int
 189         ret
 190
 191 // bool OSSpinLockTry( OSSpinLock *lock );
 192         .align  2, 0x90
 193         .globl  _OSSpinLockTry
 194         .globl  __spin_lock_try
 195 _OSSpinLockTry:
 196 __spin_lock_try:
 197         movq    $(_COMM_PAGE_SPINLOCK_TRY), %rax
 198         jmp     *%rax
 199
 200
 201 // void OSSpinLockLock( OSSpinLock *lock );
 202         .align  2, 0x90
 203         .globl  _OSSpinLockLock
 204         .globl  _spin_lock
 205         .globl  __spin_lock
 206 _OSSpinLockLock:
 207 _spin_lock:
 208 __spin_lock:
 209         movq    $(_COMM_PAGE_SPINLOCK_LOCK), %rax
 210         jmp     *%rax
 211
 212
 213 // void OSSpinLockUnlock( OSSpinLock *lock );
 214         .align  2, 0x90
 215         .globl  _OSSpinLockUnlock
 216         .globl  _spin_unlock
 217         .globl  __spin_unlock
 218 _OSSpinLockUnlock:
 219 _spin_unlock:
 220 __spin_unlock:
 221         movl    $0, (%rdi)
 222         ret
 223
 224
 225 // void OSMemoryBarrier( void );
 226         .align  2, 0x90
 227         .globl  _OSMemoryBarrier
 228 _OSMemoryBarrier:
 229         movq    $(_COMM_PAGE_MEMORY_BARRIER), %rax
 230         jmp     *%rax
 231
 232
 233 /*
 234  *      typedef volatile struct {
 235  *              void    *opaque1;  <-- ptr to 1st queue element or null
 236  *              long     opaque2;  <-- generation count
 237  *      } OSQueueHead;
 238  *
 239  * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 240  */
 241         .align  2
 242         .globl  _OSAtomicEnqueue
 243 _OSAtomicEnqueue:               // %rdi == list head, %rsi == new, %rdx == offset
 244         pushq   %rbx
 245         movq    %rsi,%rbx       // %rbx == new
 246         movq    %rdx,%rsi       // %rsi == offset
 247         movq    (%rdi),%rax     // %rax == ptr to 1st element in Q
 248         movq    8(%rdi),%rdx    // %rdx == current generation count
 249 1:
 250         movq    %rax,(%rbx,%rsi)// link to old list head from new element
 251         movq    %rdx,%rcx
 252         incq    %rcx            // increment generation count
 253         lock                    // always lock for now...
 254         cmpxchg16b (%rdi)       // ...push on new element
 255         jnz     1b
 256         popq    %rbx
 257         ret
 258
 259
 260 /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 261         .align  2
 262         .globl  _OSAtomicDequeue
 263 _OSAtomicDequeue:               // %rdi == list head, %rsi == offset
 264         pushq   %rbx
 265         movq    (%rdi),%rax     // %rax == ptr to 1st element in Q
 266         movq    8(%rdi),%rdx    // %rdx == current generation count
 267 1:
 268         testq   %rax,%rax       // list empty?
 269         jz      2f              // yes
 270         movq    (%rax,%rsi),%rbx // point to 2nd in Q
 271         movq    %rdx,%rcx
 272         incq    %rcx            // increment generation count
 273         lock                    // always lock for now...
 274         cmpxchg16b (%rdi)       // ...pop off 1st element
 275         jnz     1b
 276 2:
 277         popq    %rbx
 278         ret                     // ptr to 1st element in Q still in %rax