osfmk/i386/commpage/atomic.s

   1 /*
   2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/appleapiopts.h>
  30 #include <machine/cpu_capabilities.h>
  31 #include <machine/commpage.h>
  32
  33 /* OSAtomic.h library native implementations. */
  34
  35 // This is a regparm(3) subroutine used by:
  36
  37 // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
  38 // int32_t OSAtomicAnd32( int32_t mask, int32_t *value);
  39 // int32_t OSAtomicOr32( int32_t mask, int32_t *value);
  40 // int32_t OSAtomicXor32( int32_t mask, int32_t *value);
  41
  42 // It assumes old -> %eax, new -> %edx, value -> %ecx
  43 // on success: returns with ZF set
  44 // on failure: returns with *value in %eax, ZF clear
  45
  46 // The first word of the routine contains the address of the first instruction,
  47 // so callers can pass parameters in registers by using the absolute:
  48
  49 //      call *_COMPARE_AND_SWAP32
  50
  51 //      TODO: move the .long onto a separate page to reduce icache pollution (?)
  52
  53 COMMPAGE_FUNCTION_START(compare_and_swap32_mp, 32, 4)
  54 .long   _COMM_PAGE_COMPARE_AND_SWAP32+4
  55         lock
  56         cmpxchgl  %edx, (%ecx)
  57         ret
  58 COMMPAGE_DESCRIPTOR(compare_and_swap32_mp,_COMM_PAGE_COMPARE_AND_SWAP32,0,kUP)
  59
  60 COMMPAGE_FUNCTION_START(compare_and_swap32_up, 32, 4)
  61 .long   _COMM_PAGE_COMPARE_AND_SWAP32+4
  62         cmpxchgl %edx, (%ecx)
  63         ret
  64 COMMPAGE_DESCRIPTOR(compare_and_swap32_up,_COMM_PAGE_COMPARE_AND_SWAP32,kUP,0)
  65
  66 // This is a subroutine used by:
  67 // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
  68
  69 // It assumes old -> %eax/%edx, new -> %ebx/%ecx, value -> %esi
  70 // on success: returns with ZF set
  71 // on failure: returns with *value in %eax/%edx, ZF clear
  72
  73 COMMPAGE_FUNCTION_START(compare_and_swap64_mp, 32, 4)
  74 .long   _COMM_PAGE_COMPARE_AND_SWAP64+4
  75         lock
  76         cmpxchg8b (%esi)
  77         ret
  78 COMMPAGE_DESCRIPTOR(compare_and_swap64_mp,_COMM_PAGE_COMPARE_AND_SWAP64,0,kUP)
  79
  80 COMMPAGE_FUNCTION_START(compare_and_swap64_up, 32, 4)
  81 .long   _COMM_PAGE_COMPARE_AND_SWAP64+4
  82         cmpxchg8b (%esi)
  83         ret
  84 COMMPAGE_DESCRIPTOR(compare_and_swap64_up,_COMM_PAGE_COMPARE_AND_SWAP64,kUP,0)
  85
  86 // This is a subroutine used by:
  87 // bool OSAtomicTestAndSet( uint32_t n, void *value );
  88 // It assumes n -> %eax, value -> %edx
  89
  90 // Returns: old value of bit in CF
  91
  92 COMMPAGE_FUNCTION_START(bit_test_and_set_mp, 32, 4)
  93 .long   _COMM_PAGE_BTS+4
  94         lock
  95         btsl %eax, (%edx)
  96         ret
  97 COMMPAGE_DESCRIPTOR(bit_test_and_set_mp,_COMM_PAGE_BTS,0,kUP)
  98
  99 COMMPAGE_FUNCTION_START(bit_test_and_set_up, 32, 4)
 100 .long   _COMM_PAGE_BTS+4
 101         btsl %eax, (%edx)
 102         ret
 103 COMMPAGE_DESCRIPTOR(bit_test_and_set_up,_COMM_PAGE_BTS,kUP,0)
 104
 105 // This is a subroutine used by:
 106 // bool OSAtomicTestAndClear( uint32_t n, void *value );
 107 // It assumes n -> %eax, value -> %edx
 108
 109 // Returns: old value of bit in CF
 110
 111 COMMPAGE_FUNCTION_START(bit_test_and_clear_mp, 32, 4)
 112 .long   _COMM_PAGE_BTC+4
 113         lock
 114         btrl %eax, (%edx)
 115         ret
 116 COMMPAGE_DESCRIPTOR(bit_test_and_clear_mp,_COMM_PAGE_BTC,0,kUP)
 117
 118 COMMPAGE_FUNCTION_START(bit_test_and_clear_up, 32, 4)
 119 .long   _COMM_PAGE_BTC+4
 120         btrl %eax, (%edx)
 121         ret
 122 COMMPAGE_DESCRIPTOR(bit_test_and_clear_up,_COMM_PAGE_BTC,kUP,0)
 123
 124 // This is a subroutine used by:
 125 // int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
 126 // It assumes amt -> %eax, value -> %edx
 127
 128 // Returns: old value in %eax
 129 // NB: OSAtomicAdd32 returns the new value,  so clients will add amt to %eax
 130
 131 COMMPAGE_FUNCTION_START(atomic_add32_mp, 32, 4)
 132 .long   _COMM_PAGE_ATOMIC_ADD32+4
 133         lock
 134         xaddl   %eax, (%edx)
 135         ret
 136 COMMPAGE_DESCRIPTOR(atomic_add32_mp,_COMM_PAGE_ATOMIC_ADD32,0,kUP)
 137
 138 COMMPAGE_FUNCTION_START(atomic_add32_up, 32, 4)
 139 .long   _COMM_PAGE_ATOMIC_ADD32+4
 140         xaddl   %eax, (%edx)
 141         ret
 142 COMMPAGE_DESCRIPTOR(atomic_add32_up,_COMM_PAGE_ATOMIC_ADD32,kUP,0)
 143
 144
 145 // OSMemoryBarrier()
 146 // These are used both in 32 and 64-bit mode.  We use a fence even on UP
 147 // machines, so this function can be used with nontemporal stores.
 148
 149 COMMPAGE_FUNCTION_START(memory_barrier, 32, 4)
 150         lock
 151         addl    $0,(%esp)
 152         ret
 153 COMMPAGE_DESCRIPTOR(memory_barrier,_COMM_PAGE_MEMORY_BARRIER,0,kHasSSE2);
 154
 155 COMMPAGE_FUNCTION_START(memory_barrier_sse2, 32, 4)
 156         mfence
 157         ret
 158 COMMPAGE_DESCRIPTOR(memory_barrier_sse2,_COMM_PAGE_MEMORY_BARRIER,kHasSSE2,0);
 159
 160
 161 /*
 162  *      typedef volatile struct {
 163  *              void    *opaque1;  <-- ptr to 1st queue element or null
 164  *              long     opaque2;  <-- generation count
 165  *      } OSQueueHead;
 166  *
 167  * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 168  */
 169
 170 COMMPAGE_FUNCTION_START(AtomicEnqueue, 32, 4)
 171         pushl   %edi
 172         pushl   %esi
 173         pushl   %ebx
 174         movl    16(%esp),%edi   // %edi == ptr to list head
 175         movl    20(%esp),%ebx   // %ebx == new
 176         movl    24(%esp),%esi   // %esi == offset
 177         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 178         movl    4(%edi),%edx    // %edx == current generation count
 179 1:
 180         movl    %eax,(%ebx,%esi)// link to old list head from new element
 181         movl    %edx,%ecx
 182         incl    %ecx            // increment generation count
 183         lock                    // always lock for now...
 184         cmpxchg8b (%edi)        // ...push on new element
 185         jnz     1b
 186         popl    %ebx
 187         popl    %esi
 188         popl    %edi
 189         ret
 190 COMMPAGE_DESCRIPTOR(AtomicEnqueue,_COMM_PAGE_ENQUEUE,0,0)
 191
 192
 193 /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 194
 195 COMMPAGE_FUNCTION_START(AtomicDequeue, 32, 4)
 196         pushl   %edi
 197         pushl   %esi
 198         pushl   %ebx
 199         movl    16(%esp),%edi   // %edi == ptr to list head
 200         movl    20(%esp),%esi   // %esi == offset
 201         movl    (%edi),%eax     // %eax == ptr to 1st element in Q
 202         movl    4(%edi),%edx    // %edx == current generation count
 203 1:
 204         testl   %eax,%eax       // list empty?
 205         jz      2f              // yes
 206         movl    (%eax,%esi),%ebx // point to 2nd in Q
 207         movl    %edx,%ecx
 208         incl    %ecx            // increment generation count
 209         lock                    // always lock for now...
 210         cmpxchg8b (%edi)        // ...pop off 1st element
 211         jnz     1b
 212 2:
 213         popl    %ebx
 214         popl    %esi
 215         popl    %edi
 216         ret                     // ptr to 1st element in Q still in %eax
 217 COMMPAGE_DESCRIPTOR(AtomicDequeue,_COMM_PAGE_DEQUEUE,0,0)
 218
 219
 220
 221 /************************* x86_64 versions follow **************************/
 222
 223
 224 // This is a subroutine used by:
 225
 226 // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
 227 // int32_t OSAtomicAnd32( int32_t mask, int32_t *value);
 228 // int32_t OSAtomicOr32( int32_t mask, int32_t *value);
 229 // int32_t OSAtomicXor32( int32_t mask, int32_t *value);
 230
 231 // It assumes: old -> %rdi  (ie, it follows the ABI parameter conventions)
 232 //             new -> %rsi
 233 //             value -> %rdx
 234 // on success: returns with ZF set
 235 // on failure: returns with *value in %eax, ZF clear
 236
 237 COMMPAGE_FUNCTION_START(compare_and_swap32_mp_64, 64, 4)
 238         movl    %edi,%eax                       // put old value where "cmpxchg" wants it
 239         lock
 240         cmpxchgl  %esi, (%rdx)
 241         ret
 242 COMMPAGE_DESCRIPTOR(compare_and_swap32_mp_64,_COMM_PAGE_COMPARE_AND_SWAP32,0,kUP)
 243
 244 COMMPAGE_FUNCTION_START(compare_and_swap32_up_64, 64, 4)
 245         movl    %edi,%eax                       // put old value where "cmpxchg" wants it
 246         cmpxchgl  %esi, (%rdx)
 247         ret
 248 COMMPAGE_DESCRIPTOR(compare_and_swap32_up_64,_COMM_PAGE_COMPARE_AND_SWAP32,kUP,0)
 249
 250 // This is a subroutine used by:
 251 // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
 252
 253 // It assumes: old -> %rdi  (ie, it follows the ABI parameter conventions)
 254 //             new -> %rsi
 255 //             value -> %rdx
 256 // on success: returns with ZF set
 257 // on failure: returns with *value in %rax, ZF clear
 258
 259 COMMPAGE_FUNCTION_START(compare_and_swap64_mp_64, 64, 4)
 260         movq    %rdi,%rax                       // put old value where "cmpxchg" wants it
 261         lock
 262         cmpxchgq  %rsi, (%rdx)
 263         ret
 264 COMMPAGE_DESCRIPTOR(compare_and_swap64_mp_64,_COMM_PAGE_COMPARE_AND_SWAP64,0,kUP)
 265
 266 COMMPAGE_FUNCTION_START(compare_and_swap64_up_64, 64, 4)
 267         movq    %rdi,%rax                       // put old value where "cmpxchg" wants it
 268         cmpxchgq  %rsi, (%rdx)
 269         ret
 270 COMMPAGE_DESCRIPTOR(compare_and_swap64_up_64,_COMM_PAGE_COMPARE_AND_SWAP64,kUP,0)
 271
 272 // This is a subroutine used by:
 273 // bool OSAtomicTestAndSet( uint32_t n, void *value );
 274 // It is called with standard register conventions:
 275 //                      n = %rdi
 276 //                      value = %rsi
 277 // Returns: old value of bit in CF
 278
 279 COMMPAGE_FUNCTION_START(bit_test_and_set_mp_64, 64, 4)
 280         lock
 281         btsl %edi, (%rsi)
 282         ret
 283 COMMPAGE_DESCRIPTOR(bit_test_and_set_mp_64,_COMM_PAGE_BTS,0,kUP)
 284
 285 COMMPAGE_FUNCTION_START(bit_test_and_set_up_64, 64, 4)
 286         btsl %edi, (%rsi)
 287         ret
 288 COMMPAGE_DESCRIPTOR(bit_test_and_set_up_64,_COMM_PAGE_BTS,kUP,0)
 289
 290 // This is a subroutine used by:
 291 // bool OSAtomicTestAndClear( uint32_t n, void *value );
 292 // It is called with standard register conventions:
 293 //                      n = %rdi
 294 //                      value = %rsi
 295 // Returns: old value of bit in CF
 296
 297 COMMPAGE_FUNCTION_START(bit_test_and_clear_mp_64, 64, 4)
 298         lock
 299         btrl %edi, (%rsi)
 300         ret
 301 COMMPAGE_DESCRIPTOR(bit_test_and_clear_mp_64,_COMM_PAGE_BTC,0,kUP)
 302
 303 COMMPAGE_FUNCTION_START(bit_test_and_clear_up_64, 64, 4)
 304         btrl %edi, (%rsi)
 305         ret
 306 COMMPAGE_DESCRIPTOR(bit_test_and_clear_up_64,_COMM_PAGE_BTC,kUP,0)
 307
 308 // This is a subroutine used by:
 309 // int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
 310 // It is called with standard register conventions:
 311 //                      amt = %rdi
 312 //                      value = %rsi
 313 // Returns: old value in %edi
 314 // NB: OSAtomicAdd32 returns the new value,  so clients will add amt to %edi
 315
 316 COMMPAGE_FUNCTION_START(atomic_add32_mp_64, 64, 4)
 317         lock
 318         xaddl   %edi, (%rsi)
 319         ret
 320 COMMPAGE_DESCRIPTOR(atomic_add32_mp_64,_COMM_PAGE_ATOMIC_ADD32,0,kUP)
 321
 322 COMMPAGE_FUNCTION_START(atomic_add32_up_64, 64, 4)
 323         xaddl   %edi, (%rsi)
 324         ret
 325 COMMPAGE_DESCRIPTOR(atomic_add32_up_64,_COMM_PAGE_ATOMIC_ADD32,kUP,0)
 326
 327 // This is a subroutine used by:
 328 // int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
 329 // It is called with standard register conventions:
 330 //                      amt = %rdi
 331 //                      value = %rsi
 332 // Returns: old value in %rdi
 333 // NB: OSAtomicAdd64 returns the new value,  so clients will add amt to %rdi
 334
 335 COMMPAGE_FUNCTION_START(atomic_add64_mp_64, 64, 4)
 336         lock
 337         xaddq   %rdi, (%rsi)
 338         ret
 339 COMMPAGE_DESCRIPTOR(atomic_add64_mp_64,_COMM_PAGE_ATOMIC_ADD64,0,kUP)
 340
 341 COMMPAGE_FUNCTION_START(atomic_add64_up_64, 64, 4)
 342         xaddq   %rdi, (%rsi)
 343         ret
 344 COMMPAGE_DESCRIPTOR(atomic_add64_up_64,_COMM_PAGE_ATOMIC_ADD64,kUP,0)
 345
 346
 347 /*
 348  *      typedef volatile struct {
 349  *              void    *opaque1;  <-- ptr to 1st queue element or null
 350  *              long     opaque2;  <-- generation count
 351  *      } OSQueueHead;
 352  *
 353  * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
 354  */
 355
 356 // %rdi == list head, %rsi == new, %rdx == offset
 357
 358 COMMPAGE_FUNCTION_START(AtomicEnqueue_64, 64, 4)
 359         pushq   %rbx
 360         movq    %rsi,%rbx       // %rbx == new
 361         movq    %rdx,%rsi       // %rsi == offset
 362         movq    (%rdi),%rax     // %rax == ptr to 1st element in Q
 363         movq    8(%rdi),%rdx    // %rdx == current generation count
 364 1:
 365         movq    %rax,(%rbx,%rsi)// link to old list head from new element
 366         movq    %rdx,%rcx
 367         incq    %rcx            // increment generation count
 368         lock                    // always lock for now...
 369         cmpxchg16b (%rdi)       // ...push on new element
 370         jnz     1b
 371         popq    %rbx
 372         ret
 373 COMMPAGE_DESCRIPTOR(AtomicEnqueue_64,_COMM_PAGE_ENQUEUE,0,0)
 374
 375
 376 /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 377
 378 // %rdi == list head, %rsi == offset
 379
 380 COMMPAGE_FUNCTION_START(AtomicDequeue_64, 64, 4)
 381         pushq   %rbx
 382         movq    (%rdi),%rax     // %rax == ptr to 1st element in Q
 383         movq    8(%rdi),%rdx    // %rdx == current generation count
 384 1:
 385         testq   %rax,%rax       // list empty?
 386         jz      2f              // yes
 387         movq    (%rax,%rsi),%rbx // point to 2nd in Q
 388         movq    %rdx,%rcx
 389         incq    %rcx            // increment generation count
 390         lock                    // always lock for now...
 391         cmpxchg16b (%rdi)       // ...pop off 1st element
 392         jnz     1b
 393 2:
 394         popq    %rbx
 395         ret                     // ptr to 1st element in Q still in %rax
 396 COMMPAGE_DESCRIPTOR(AtomicDequeue_64,_COMM_PAGE_DEQUEUE,0,0)