i386/sys/OSAtomic.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <machine/cpu_capabilities.h>
  26 #include <platfunc.h>
  27
  28 #define ATOMIC_UP       0
  29 #define ATOMIC_MP       1
  30 #define ATOMIC_RET_ORIG 0
  31 #define ATOMIC_RET_NEW  1
  32
  33 // compare and exchange 32-bit
  34 // xchg32 <new> <dst> <mp>
  35 .macro xchg32
  36         .if $2 == ATOMIC_MP
  37         lock
  38         .endif
  39         cmpxchgl        $0, ($1)
  40 .endm
  41
  42 // compare and exchange 64-bit
  43 // xchg64 <dst> <mp>
  44 .macro xchg64
  45         .if $1 == ATOMIC_MP
  46         lock
  47         .endif
  48         cmpxchg8b       ($0)
  49 .endm
  50
  51
  52 // int32_t OSAtomicAdd32(int32_t theAmount, volatile int32_t *theValue);
  53 #define ATOMIC_ARITHMETIC(instr, orig, mp) \
  54         movl    8(%esp), %ecx   /* load 2nd arg ptr into ecx */ ;\
  55         movl    (%ecx), %eax    /* load contents of ecx into eax */ ;\
  56 1:      movl    4(%esp), %edx   /* load 1st arg into edx */ ;\
  57         instr   %eax, %edx      /* do the operation */ ;\
  58         xchg32  %edx, %ecx, mp  /* old in %eax, new in %edx, exchange into %ecx */ ;\
  59         jnz     1b              /* go back if we failed to exchange */ ;\
  60         .if orig == ATOMIC_RET_NEW ;\
  61         movl    %edx, %eax      /* return new value */ ;\
  62         .endif
  63
  64 // bool OSAtomicTestAndSet(uint32_t n, volatile void *theAddress);
  65 #define ATOMIC_BIT_OP(instr, mp) \
  66         movl    4(%esp), %eax   ;\
  67         movl    8(%esp), %edx   ;\
  68         shldl   $3,%edx,%ecx    /* save top 3 bits of address in %ecx */ ;\
  69         shll    $3,%edx         ;\
  70         xorl    $7,%eax         /* bit position is numbered big endian so convert to little endian */ ;\
  71         addl    %eax,%edx       /* generate bit address */ ;\
  72         adcl    $0,%ecx         /* handle carry out of lower half of address */ ;\
  73         movl    %edx,%eax       /* copy lower half of bit address */ ;\
  74         andl    $31,%eax        /* keep bit offset in range 0..31 */ ;\
  75         xorl    %eax,%edx       /* 4-byte align address */ ;\
  76         shrdl   $3,%ecx,%edx    /* restore 32-bit byte address in %edx */ ;\
  77         .if mp == ATOMIC_MP     ;\
  78         lock                    ;\
  79         .endif                  ;\
  80         instr   %eax, (%edx)    ;\
  81         setc    %al             ;\
  82         movzbl  %al,%eax        // widen in case caller assumes we return an int
  83
  84 // int64_t OSAtomicAdd64(int64_t theAmount, volatile int64_t *theValue);
  85 #define ATOMIC_ADD64(mp) \
  86         pushl   %ebx            ;\
  87         pushl   %esi            ;\
  88         movl    20(%esp), %esi  ;\
  89         movl    0(%esi), %eax   ;\
  90         movl    4(%esi), %edx   ;\
  91 1:      movl    12(%esp), %ebx  ;\
  92         movl    16(%esp), %ecx  ;\
  93         addl    %eax, %ebx      ;\
  94         adcl    %edx, %ecx      ;\
  95         xchg64  %esi, mp        ;\
  96         jnz     1b              ;\
  97         movl    %ebx, %eax      ;\
  98         movl    %ecx, %edx      ;\
  99         popl    %esi            ;\
 100         popl    %ebx
 101
 102         .text
 103
 104 PLATFUNC_FUNCTION_START(OSAtomicAnd32, up, 32, 2)
 105 PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 32, 2)
 106         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
 107         ret
 108
 109 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 32, 2)
 110 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 32, 2)
 111         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
 112         ret
 113
 114 PLATFUNC_FUNCTION_START(OSAtomicOr32, up, 32, 2)
 115 PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 32, 2)
 116         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
 117         ret
 118
 119 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 32, 2)
 120 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 32, 2)
 121         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
 122         ret
 123
 124 PLATFUNC_FUNCTION_START(OSAtomicXor32, up, 32, 2)
 125 PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 32, 2)
 126         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
 127         ret
 128
 129 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 32, 2)
 130 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 32, 2)
 131         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
 132         ret
 133
 134 PLATFUNC_FUNCTION_START(OSAtomicAnd32Orig, up, 32, 2)
 135 PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 32, 2)
 136         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
 137         ret
 138
 139 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 32, 2)
 140 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 32, 2)
 141         ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
 142         ret
 143
 144 PLATFUNC_FUNCTION_START(OSAtomicOr32Orig, up, 32, 2)
 145 PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 32, 2)
 146         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
 147         ret
 148
 149 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 32, 2)
 150 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 32, 2)
 151         ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
 152         ret
 153
 154 PLATFUNC_FUNCTION_START(OSAtomicXor32Orig, up, 32, 2)
 155 PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 32, 2)
 156         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
 157         ret
 158
 159 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 32, 2)
 160 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 32, 2)
 161         ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
 162         ret
 163
 164 // bool OSAtomicCompareAndSwapInt(int oldValue, int newValue, volatile int *theValue);
 165 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtr, up, 32, 2)
 166 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 32, 2)
 167 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapInt, up, 32, 2)
 168 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 32, 2)
 169 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 32, 2)
 170 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 32, 2)
 171 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 32, 2)
 172 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 32, 2)
 173         movl    4(%esp), %eax
 174         movl    8(%esp), %edx
 175         movl    12(%esp), %ecx
 176         xchg32  %edx, %ecx, ATOMIC_UP
 177         sete    %al
 178         movzbl  %al,%eax        // widen in case caller assumes we return an int
 179         ret
 180
 181 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 32, 2)
 182 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 32, 2)
 183 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 32, 2)
 184 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 32, 2)
 185 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 32, 2)
 186 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 32, 2)
 187 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 32, 2)
 188 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 32, 2)
 189         movl    4(%esp), %eax
 190         movl    8(%esp), %edx
 191         movl    12(%esp), %ecx
 192         xchg32  %edx, %ecx, ATOMIC_MP
 193         sete    %al
 194         movzbl  %al,%eax        // widen in case caller assumes we return an int
 195         ret
 196
 197 // bool OSAtomicCompareAndSwap64(int64_t oldValue, int64_t newValue, volatile int64_t *theValue);
 198 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 32, 2)
 199 PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 32, 2)
 200         pushl   %ebx            // push out spare stuff for space
 201         pushl   %esi
 202         movl    12(%esp), %eax  // load in 1st 64-bit parameter
 203         movl    16(%esp), %edx
 204         movl    20(%esp), %ebx  // load in 2nd 64-bit parameter
 205         movl    24(%esp), %ecx
 206         movl    28(%esp), %esi  // laod in destination address
 207         xchg64  %esi, ATOMIC_UP // compare and swap 64-bit
 208         sete    %al
 209         movzbl  %al,%eax        // widen in case caller assumes we return an int
 210         popl    %esi
 211         popl    %ebx
 212         ret
 213
 214 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 32, 2)
 215 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 32, 2)
 216         pushl   %ebx            // push out spare stuff for space
 217         pushl   %esi
 218         movl    12(%esp), %eax  // load in 1st 64-bit parameter
 219         movl    16(%esp), %edx
 220         movl    20(%esp), %ebx  // load in 2nd 64-bit parameter
 221         movl    24(%esp), %ecx
 222         movl    28(%esp), %esi  // laod in destination address
 223         xchg64  %esi, ATOMIC_MP // compare and swap 64-bit
 224         sete    %al
 225         movzbl  %al,%eax        // widen in case caller assumes we return an int
 226         popl    %esi
 227         popl    %ebx
 228         ret
 229
 230 PLATFUNC_FUNCTION_START(OSAtomicAdd32, up, 32, 2)
 231 PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 32, 2)
 232         movl    4(%esp), %eax
 233         movl    8(%esp), %edx
 234         movl    %eax, %ecx
 235         xaddl   %eax, (%edx)
 236         addl    %ecx, %eax
 237         ret
 238
 239 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 32, 2)
 240 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 32, 2)
 241         movl    4(%esp), %eax
 242         movl    8(%esp), %edx
 243         movl    %eax, %ecx
 244         lock
 245         xaddl   %eax, (%edx)
 246         addl    %ecx, %eax
 247         ret
 248
 249 PLATFUNC_FUNCTION_START(OSAtomicAdd64, up, 32, 2)
 250 PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 32, 2)
 251         ATOMIC_ADD64(ATOMIC_UP)
 252         ret
 253
 254 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 32, 2)
 255 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 32, 2)
 256         ATOMIC_ADD64(ATOMIC_MP)
 257         ret
 258
 259 PLATFUNC_FUNCTION_START(OSAtomicTestAndSet, up, 32, 2)
 260 PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 32, 2)
 261         ATOMIC_BIT_OP(btsl, ATOMIC_UP)
 262         ret
 263
 264 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 32, 2)
 265 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 32, 2)
 266         ATOMIC_BIT_OP(btsl, ATOMIC_MP)
 267         ret
 268
 269 PLATFUNC_FUNCTION_START(OSAtomicTestAndClear, up, 32, 2)
 270 PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 32, 2)
 271         ATOMIC_BIT_OP(btrl, ATOMIC_UP)
 272         ret
 273
 274 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 32, 2)
 275 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 32, 2)
 276         ATOMIC_BIT_OP(btrl, ATOMIC_MP)
 277         ret
 278
 279 // OSMemoryBarrier()
 280 // These are used both in 32 and 64-bit mode.  We use a fence even on UP
 281 // machines, so this function can be used with nontemporal stores.
 282
 283 PLATFUNC_FUNCTION_START_GENERIC(OSMemoryBarrier, all, 32, 4)
 284         lock
 285         addl    $0,(%esp)
 286         ret
 287 PLATFUNC_DESCRIPTOR(OSMemoryBarrier,all,0,kHasSSE2);
 288
 289 PLATFUNC_FUNCTION_START(OSMemoryBarrier, sse2, 32, 4)
 290         mfence
 291         ret
 292 PLATFUNC_DESCRIPTOR(OSMemoryBarrier,sse2,kHasSSE2,0);
 293
 294         /*
 295          *      typedef volatile struct {
 296          *              void    *opaque1;  <-- ptr to 1st queue element or null
 297          *              long     opaque2;  <-- generation count
 298          *      } OSQueueHead;
 299          *
 300          * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 301          */
 302 PLATFUNC_FUNCTION_START(OSAtomicEnqueue, up, 32, 2)
 303         pushl   %edi
 304         pushl   %esi
 305         pushl   %ebx
 306         movl    16(%esp),%edi   // %edi == ptr to list head
 307         movl    20(%esp),%ebx   // %ebx == new
 308         movl    24(%esp),%esi   // %esi == offset
 309         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 310         movl    4(%edi),%edx    // %edx == current generation count
 311 1:      movl    %eax,(%ebx,%esi)// link to old list head from new element
 312         movl    %edx,%ecx
 313         incl    %ecx            // increment generation count
 314         xchg64  %edi, ATOMIC_UP // ...push on new element
 315         jnz     1b
 316         popl    %ebx
 317         popl    %esi
 318         popl    %edi
 319         ret
 320
 321 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicEnqueue, mp, 32, 2)
 322         pushl   %edi
 323         pushl   %esi
 324         pushl   %ebx
 325         movl    16(%esp),%edi   // %edi == ptr to list head
 326         movl    20(%esp),%ebx   // %ebx == new
 327         movl    24(%esp),%esi   // %esi == offset
 328         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 329         movl    4(%edi),%edx    // %edx == current generation count
 330 1:      movl    %eax,(%ebx,%esi)// link to old list head from new element
 331         movl    %edx,%ecx
 332         incl    %ecx            // increment generation count
 333         xchg64  %edi, ATOMIC_MP // ...push on new element
 334         jnz     1b
 335         popl    %ebx
 336         popl    %esi
 337         popl    %edi
 338         ret
 339
 340 /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 341 PLATFUNC_FUNCTION_START(OSAtomicDequeue, up, 32, 2)
 342         pushl   %edi
 343         pushl   %esi
 344         pushl   %ebx
 345         movl    16(%esp),%edi   // %edi == ptr to list head
 346         movl    20(%esp),%esi   // %esi == offset
 347         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 348         movl    4(%edi),%edx    // %edx == current generation count
 349 1:      testl   %eax,%eax       // list empty?
 350         jz      2f              // yes
 351         movl    (%eax,%esi),%ebx // point to 2nd in Q
 352         movl    %edx,%ecx
 353         incl    %ecx            // increment generation count
 354         xchg64  %edi, ATOMIC_UP // ...pop off 1st element
 355         jnz     1b
 356 2:      popl    %ebx
 357         popl    %esi
 358         popl    %edi
 359         ret                     // ptr to 1st element in Q still in %eax
 360
 361 PLATFUNC_FUNCTION_START_GENERIC(OSAtomicDequeue, mp, 32, 2)
 362         pushl   %edi
 363         pushl   %esi
 364         pushl   %ebx
 365         movl    16(%esp),%edi   // %edi == ptr to list head
 366         movl    20(%esp),%esi   // %esi == offset
 367         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 368         movl    4(%edi),%edx    // %edx == current generation count
 369 1:      testl   %eax,%eax       // list empty?
 370         jz      2f              // yes
 371         movl    (%eax,%esi),%ebx // point to 2nd in Q
 372         movl    %edx,%ecx
 373         incl    %ecx            // increment generation count
 374         xchg64  %edi, ATOMIC_MP // ...pop off 1st element
 375         jnz     1b
 376 2:      popl    %ebx
 377         popl    %esi
 378         popl    %edi
 379         ret                     // ptr to 1st element in Q still in %eax
 380
 381 /*
 382  *      typedef volatile struct {
 383  *              void    *opaque1;  <-- ptr to first queue element or null
 384  *              void    *opaque2;  <-- ptr to last queue element or null
 385  *              int      opaque3;  <-- spinlock
 386  *      } OSFifoQueueHead;
 387  *
 388  * void  OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset);
 389  */
 390         .align  2
 391         .globl  _OSAtomicFifoEnqueue
 392 _OSAtomicFifoEnqueue:
 393         pushl   %edi
 394         pushl   %esi
 395         pushl   %ebx
 396         xorl    %ebx,%ebx       // clear "preemption pending" flag
 397         movl    16(%esp),%edi   // %edi == ptr to list head
 398         movl    20(%esp),%esi   // %esi == new
 399         movl    24(%esp),%edx   // %edx == offset
 400         movl    $(_COMM_PAGE_PFZ_ENQUEUE), %ecx
 401         call    *%ecx
 402         testl   %ebx,%ebx       // pending preemption?
 403         jz      1f
 404         call    _preempt
 405 1:      popl    %ebx
 406         popl    %esi
 407         popl    %edi
 408         ret
 409
 410 /* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset); */
 411         .align  2
 412         .globl  _OSAtomicFifoDequeue
 413 _OSAtomicFifoDequeue:
 414         pushl   %edi
 415         pushl   %esi
 416         pushl   %ebx
 417         xorl    %ebx,%ebx       // clear "preemption pending" flag
 418         movl    16(%esp),%edi   // %edi == ptr to list head
 419         movl    20(%esp),%edx   // %edx == offset
 420         movl    $(_COMM_PAGE_PFZ_DEQUEUE), %ecx
 421         call    *%ecx
 422         testl   %ebx,%ebx       // pending preemption?
 423         jz      1f
 424         pushl   %eax            // save return value across sysenter
 425         call    _preempt
 426         popl    %eax
 427 1:      popl    %ebx
 428         popl    %esi
 429         popl    %edi
 430         ret                     // ptr to 1st element in Q still in %eax
 431
 432 // Local Variables:
 433 // tab-width: 8
 434 // End: