i386/sys/OSAtomic.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <machine/cpu_capabilities.h>
  26
  27 #define DECLARE(x)   \
  28 .align 2, 0x90      ; \
  29 .globl x            ; \
  30 .globl x ## Barrier ; \
  31 x:                  ; \
  32 x ## Barrier:
  33
  34 .text
  35
  36 DECLARE(_OSAtomicAnd32)
  37         movl 8(%esp), %ecx
  38         movl (%ecx), %eax
  39 1:
  40         movl 4(%esp), %edx
  41         andl %eax, %edx
  42         call *_COMM_PAGE_COMPARE_AND_SWAP32
  43         jnz  1b
  44         movl %edx, %eax
  45         ret
  46
  47 DECLARE(_OSAtomicOr32)
  48         movl 8(%esp), %ecx
  49         movl (%ecx), %eax
  50 1:
  51         movl 4(%esp), %edx
  52         orl %eax, %edx
  53         call *_COMM_PAGE_COMPARE_AND_SWAP32
  54         jnz  1b
  55         movl %edx, %eax
  56         ret
  57
  58 DECLARE(_OSAtomicXor32)
  59         movl 8(%esp), %ecx
  60         movl (%ecx), %eax
  61 1:
  62         movl 4(%esp), %edx
  63         xorl %eax, %edx
  64         call *_COMM_PAGE_COMPARE_AND_SWAP32
  65         jnz  1b
  66         movl %edx, %eax
  67         ret
  68
  69 DECLARE(_OSAtomicAnd32Orig)
  70         movl 8(%esp), %ecx
  71         movl (%ecx), %eax
  72 1:
  73         movl 4(%esp), %edx
  74         andl %eax, %edx
  75         call *_COMM_PAGE_COMPARE_AND_SWAP32
  76         jnz  1b
  77         ret
  78
  79 DECLARE(_OSAtomicOr32Orig)
  80         movl 8(%esp), %ecx
  81         movl (%ecx), %eax
  82 1:
  83         movl 4(%esp), %edx
  84         orl %eax, %edx
  85         call *_COMM_PAGE_COMPARE_AND_SWAP32
  86         jnz  1b
  87         ret
  88
  89 DECLARE(_OSAtomicXor32Orig)
  90         movl 8(%esp), %ecx
  91         movl (%ecx), %eax
  92 1:
  93         movl 4(%esp), %edx
  94         xorl %eax, %edx
  95         call *_COMM_PAGE_COMPARE_AND_SWAP32
  96         jnz  1b
  97         ret
  98
  99 DECLARE(_OSAtomicCompareAndSwapPtr)
 100 DECLARE(_OSAtomicCompareAndSwapInt)
 101 DECLARE(_OSAtomicCompareAndSwapLong)
 102 DECLARE(_OSAtomicCompareAndSwap32)
 103         movl     4(%esp), %eax
 104         movl     8(%esp), %edx
 105         movl    12(%esp), %ecx
 106         call    *_COMM_PAGE_COMPARE_AND_SWAP32
 107         sete    %al
 108         movzbl  %al,%eax        // widen in case caller assumes we return an int
 109         ret
 110
 111 DECLARE(_OSAtomicCompareAndSwap64)
 112         pushl   %ebx
 113         pushl   %esi
 114         movl    12(%esp), %eax
 115         movl    16(%esp), %edx
 116         movl    20(%esp), %ebx
 117         movl    24(%esp), %ecx
 118         movl    28(%esp), %esi
 119         call    *_COMM_PAGE_COMPARE_AND_SWAP64
 120         sete    %al
 121         movzbl  %al,%eax        // widen in case caller assumes we return an int
 122         popl    %esi
 123         popl    %ebx
 124         ret
 125
 126 DECLARE(_OSAtomicAdd32)
 127         movl    4(%esp), %eax
 128         movl    8(%esp), %edx
 129         movl    %eax, %ecx
 130         call    *_COMM_PAGE_ATOMIC_ADD32
 131         addl    %ecx, %eax
 132         ret
 133
 134 DECLARE(_OSAtomicAdd64)
 135         pushl   %ebx
 136         pushl   %esi
 137         movl    20(%esp), %esi
 138         movl    0(%esi), %eax
 139         movl    4(%esi), %edx
 140 1:      movl    12(%esp), %ebx
 141         movl    16(%esp), %ecx
 142         addl    %eax, %ebx
 143         adcl    %edx, %ecx
 144         call    *_COMM_PAGE_COMPARE_AND_SWAP64
 145         jnz     1b
 146         movl    %ebx, %eax
 147         movl    %ecx, %edx
 148         popl    %esi
 149         popl    %ebx
 150         ret
 151
 152 DECLARE(_OSAtomicTestAndSet)
 153         movl    4(%esp), %eax
 154         movl    8(%esp), %edx
 155         movl    %eax, %ecx
 156         andl    $-8, %ecx
 157         notl    %eax
 158         andl    $7, %eax
 159         orl     %ecx, %eax
 160         call    *_COMM_PAGE_BTS
 161         setc    %al
 162         movzbl  %al,%eax        // widen in case caller assumes we return an int
 163         ret
 164
 165 DECLARE(_OSAtomicTestAndClear)
 166         movl    4(%esp), %eax
 167         movl    8(%esp), %edx
 168         movl    %eax, %ecx
 169         andl    $-8, %ecx
 170         notl    %eax
 171         andl    $7, %eax
 172         orl     %ecx, %eax
 173         call    *_COMM_PAGE_BTC
 174         setc    %al
 175         movzbl  %al,%eax        // widen in case caller assumes we return an int
 176         ret
 177
 178         .align  2, 0x90
 179         .globl  _OSSpinLockTry
 180         .globl  __spin_lock_try
 181 _OSSpinLockTry:
 182 __spin_lock_try:
 183         movl    $(_COMM_PAGE_SPINLOCK_TRY), %eax
 184         jmpl    *%eax
 185
 186         .align  2, 0x90
 187         .globl  _OSSpinLockLock
 188         .globl  _spin_lock
 189         .globl  __spin_lock
 190 _OSSpinLockLock:
 191 _spin_lock:
 192 __spin_lock:
 193         movl    $(_COMM_PAGE_SPINLOCK_LOCK), %eax
 194         jmpl    *%eax
 195
 196         .align  2, 0x90
 197         .globl  _OSSpinLockUnlock
 198         .globl  _spin_unlock
 199         .globl  __spin_unlock
 200 _OSSpinLockUnlock:
 201 _spin_unlock:
 202 __spin_unlock:
 203         movl    4(%esp), %eax
 204         movl    $0, (%eax)
 205         ret
 206
 207         .align 2, 0x90
 208         .globl _OSMemoryBarrier
 209 _OSMemoryBarrier:
 210         movl    $(_COMM_PAGE_MEMORY_BARRIER), %eax
 211         jmpl    *%eax
 212
 213 /*
 214  *      typedef volatile struct {
 215  *              void    *opaque1;  <-- ptr to 1st queue element or null
 216  *              long     opaque2;  <-- generation count
 217  *      } OSQueueHead;
 218  *
 219  * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 220  */
 221         .align  2
 222         .globl  _OSAtomicEnqueue
 223 _OSAtomicEnqueue:
 224         pushl   %edi
 225         pushl   %esi
 226         pushl   %ebx
 227         movl    16(%esp),%edi   // %edi == ptr to list head
 228         movl    20(%esp),%ebx   // %ebx == new
 229         movl    24(%esp),%esi   // %esi == offset
 230         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 231         movl    4(%edi),%edx    // %edx == current generation count
 232 1:
 233         movl    %eax,(%ebx,%esi)// link to old list head from new element
 234         movl    %edx,%ecx
 235         incl    %ecx            // increment generation count
 236         lock                    // always lock for now...
 237         cmpxchg8b (%edi)        // ...push on new element
 238         jnz     1b
 239         popl    %ebx
 240         popl    %esi
 241         popl    %edi
 242         ret
 243
 244
 245 /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 246         .align  2
 247         .globl  _OSAtomicDequeue
 248 _OSAtomicDequeue:
 249         pushl   %edi
 250         pushl   %esi
 251         pushl   %ebx
 252         movl    16(%esp),%edi   // %edi == ptr to list head
 253         movl    20(%esp),%esi   // %esi == offset
 254         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 255         movl    4(%edi),%edx    // %edx == current generation count
 256 1:
 257         testl   %eax,%eax       // list empty?
 258         jz      2f              // yes
 259         movl    (%eax,%esi),%ebx // point to 2nd in Q
 260         movl    %edx,%ecx
 261         incl    %ecx            // increment generation count
 262         lock                    // always lock for now...
 263         cmpxchg8b (%edi)        // ...pop off 1st element
 264         jnz     1b
 265 2:
 266         popl    %ebx
 267         popl    %esi
 268         popl    %edi
 269         ret                     // ptr to 1st element in Q still in %eax
 270