ppc/sys/OSAtomic.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #define __APPLE_API_PRIVATE
  26 #include <machine/cpu_capabilities.h>
  27 #undef  __APPLE_API_PRIVATE
  28
  29 #include <architecture/ppc/mode_independent_asm.h>
  30
  31
  32 /* These are the functions in <libkern/OSAtomic.h>.
  33  * The actual primitives are implemented on the commpage.
  34  */
  35
  36
  37 /* int32_t      OSAtomicAdd32( int32_t theAmount, int32_t *theValue ); */
  38
  39 MI_ENTRY_POINT(_OSAtomicAdd32)
  40     ba      _COMM_PAGE_ATOMIC_ADD32
  41
  42
  43 /* int32_t      OSAtomicOr32( int32_t theMask, int32_t *theValue ); */
  44
  45 MI_ENTRY_POINT(_OSAtomicOr32)
  46     mflr    r12                 // save return address
  47     mr      r5,r4               // move ptr to where compare-and-swap wants it
  48     mr      r11,r3              // copy mask
  49 1:
  50     lwz     r3,0(r5)            // get old value
  51     or      r4,r3,r11           // make new value
  52     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
  53     cmpwi   r3,0                // did swap occur?
  54     beq--   1b                  // compare-and-swap failed, try again
  55     mtlr    r12                 // restore return adddress
  56     mr      r3,r4               // return new value
  57     blr
  58
  59
  60 /* int32_t      OSAtomicAnd32( int32_t theMask, int32_t *theValue ); */
  61
  62 MI_ENTRY_POINT(_OSAtomicAnd32)
  63     mflr    r12                 // save return address
  64     mr      r5,r4               // move ptr to where compare-and-swap wants it
  65     mr      r11,r3              // copy mask
  66 1:
  67     lwz     r3,0(r5)            // get old value
  68     and     r4,r3,r11           // make new value
  69     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
  70     cmpwi   r3,0                // did swap occur?
  71     beq--   1b                  // compare-and-swap failed, try again
  72     mtlr    r12                 // restore return adddress
  73     mr      r3,r4               // return new value
  74     blr
  75
  76
  77 /* int32_t      OSAtomicXor32( int32_t theMask, int32_t *theValue ); */
  78
  79 MI_ENTRY_POINT(_OSAtomicXor32)
  80     mflr    r12                 // save return address
  81     mr      r5,r4               // move ptr to where compare-and-swap wants it
  82     mr      r11,r3              // copy mask
  83 1:
  84     lwz     r3,0(r5)            // get old value
  85     xor     r4,r3,r11           // make new value
  86     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
  87     cmpwi   r3,0                // did swap occur?
  88     beq--   1b                  // compare-and-swap failed, try again
  89     mtlr    r12                 // restore return adddress
  90     mr      r3,r4               // return new value
  91     blr
  92
  93
  94 /* int32_t      OSAtomicOr32Orig( int32_t theMask, int32_t *theValue ); */
  95
  96 MI_ENTRY_POINT(_OSAtomicOr32Orig)
  97     mflr    r12                 // save return address
  98     mr      r5,r4               // move ptr to where compare-and-swap wants it
  99     mr      r11,r3              // copy mask
 100 1:
 101     lwz     r3,0(r5)            // get old value
 102     mr      r10,r3              // save old value
 103     or      r4,r3,r11           // make new value
 104     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
 105     cmpwi   r3,0                // did swap occur?
 106     beq--   1b                  // compare-and-swap failed, try again
 107     mtlr    r12                 // restore return adddress
 108     mr      r3,r10              // return old value
 109     blr
 110
 111
 112 /* int32_t      OSAtomicAnd32Orig( int32_t theMask, int32_t *theValue ); */
 113
 114 MI_ENTRY_POINT(_OSAtomicAnd32Orig)
 115     mflr    r12                 // save return address
 116     mr      r5,r4               // move ptr to where compare-and-swap wants it
 117     mr      r11,r3              // copy mask
 118 1:
 119     lwz     r3,0(r5)            // get old value
 120     mr      r10,r3              // save old value
 121     and     r4,r3,r11           // make new value
 122     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
 123     cmpwi   r3,0                // did swap occur?
 124     beq--   1b                  // compare-and-swap failed, try again
 125     mtlr    r12                 // restore return adddress
 126     mr      r3,r10              // return old value
 127     blr
 128
 129
 130 /* int32_t      OSAtomicXor32Orig( int32_t theMask, int32_t *theValue ); */
 131
 132 MI_ENTRY_POINT(_OSAtomicXor32Orig)
 133     mflr    r12                 // save return address
 134     mr      r5,r4               // move ptr to where compare-and-swap wants it
 135     mr      r11,r3              // copy mask
 136 1:
 137     lwz     r3,0(r5)            // get old value
 138     mr      r10,r3              // save old value
 139     xor     r4,r3,r11           // make new value
 140     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
 141     cmpwi   r3,0                // did swap occur?
 142     beq--   1b                  // compare-and-swap failed, try again
 143     mtlr    r12                 // restore return adddress
 144     mr      r3,r10              // return old value
 145     blr
 146
 147
 148 /* int64_t      OSAtomicAdd64( int64_t theAmount, int64_t *theValue ); */
 149
 150 #if defined(__ppc64__)
 151 MI_ENTRY_POINT(_OSAtomicAdd64)
 152     ba      _COMM_PAGE_ATOMIC_ADD64
 153 #endif  /* defined(__ppc64__) */
 154
 155
 156 /* bool OSAtomicCompareAndSwap32( int32_t oldValue, int32_t newValue, int32_t *theValue ); */
 157 /* bool OSAtomicCompareAndSwap64( int64_t oldValue, int64_t newValue, int64_t *theValue ); */
 158 /* bool OSAtomicCompareAndSwapPtr( void* oldValue, void* newValue, void* *theValue ); */
 159 /* bool OSAtomicCompareAndSwapInt( int oldValue, int newValue, int *theValue ); */
 160 /* bool OSAtomicCompareAndSwapLong( long oldValue, long newValue, long *theValue ); */
 161
 162 MI_ENTRY_POINT(_OSAtomicCompareAndSwap32)
 163     ba      _COMM_PAGE_COMPARE_AND_SWAP32
 164 MI_ENTRY_POINT(_OSAtomicCompareAndSwapInt)
 165     ba      _COMM_PAGE_COMPARE_AND_SWAP32
 166
 167 #if defined(__ppc64__)
 168 MI_ENTRY_POINT(_OSAtomicCompareAndSwap64)
 169     ba      _COMM_PAGE_COMPARE_AND_SWAP64
 170 MI_ENTRY_POINT(_OSAtomicCompareAndSwapPtr)
 171     ba      _COMM_PAGE_COMPARE_AND_SWAP64
 172 MI_ENTRY_POINT(_OSAtomicCompareAndSwapLong)
 173     ba      _COMM_PAGE_COMPARE_AND_SWAP64
 174 #else   /* !defined(__ppc64__) */
 175 MI_ENTRY_POINT(_OSAtomicCompareAndSwapPtr)
 176     ba      _COMM_PAGE_COMPARE_AND_SWAP32
 177 MI_ENTRY_POINT(_OSAtomicCompareAndSwapLong)
 178     ba      _COMM_PAGE_COMPARE_AND_SWAP32
 179 #endif  /* defined(__ppc64__) */
 180
 181
 182 /* bool OSAtomicTestAndSet( uint32_t n, void *theAddress ); */
 183
 184 MI_ENTRY_POINT(_OSAtomicTestAndSet)
 185     mflr    r12                 // save return
 186     srwi    r5,r3,3             // get byte offset of n
 187     rlwinm  r6,r3,0,0x7         // get bit position within byte
 188     add     r4,r4,r5            // r4 points to byte containing the bit
 189     lis     r10,0x8000          // light bit 0
 190     rlwimi  r6,r4,3,0x18        // r6 is bit position within word
 191     clrrgi  r5,r4,2             // point to word containing the bit
 192     srw     r10,r10,r6          // get mask for bit
 193     addi    r9,r6,1             // save bit position + 1
 194 1:
 195     lwz     r3,0(r5)            // get old word
 196     rlwnm   r11,r3,r9,0x1       // right justify old value of bit
 197     or      r4,r3,r10           // set it in new word
 198     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
 199     cmpwi   r3,0                // did swap occur?
 200     beq--   1b                  // compare-and-swap failed, try again
 201     mtlr    r12                 // restore return adddress
 202     mr      r3,r11              // return original value of bit
 203     blr
 204
 205
 206 /* bool OSAtomicTestAndClear( uint32_t n, void *theAddress ); */
 207
 208 MI_ENTRY_POINT(_OSAtomicTestAndClear)
 209     mflr    r12                 // save return
 210     srwi    r5,r3,3             // get byte offset of n
 211     rlwinm  r6,r3,0,0x7         // get bit position within byte
 212     add     r4,r4,r5            // r4 points to byte containing the bit
 213     lis     r10,0x8000          // light bit 0
 214     rlwimi  r6,r4,3,0x18        // r6 is bit position within word
 215     clrrgi  r5,r4,2             // point to word containing the bit
 216     srw     r10,r10,r6          // get mask for bit
 217     addi    r9,r6,1             // save bit position + 1
 218 1:
 219     lwz     r3,0(r5)            // get old word
 220     rlwnm   r11,r3,r9,0x1       // right justify old value of bit
 221     andc    r4,r3,r10           // clear it in new word
 222     bla     _COMM_PAGE_COMPARE_AND_SWAP32   // preserves r4,r5,r9-r12
 223     cmpwi   r3,0                // did swap occur?
 224     beq--   1b                  // compare-and-swap failed, try again
 225     mtlr    r12                 // restore return adddress
 226     mr      r3,r11              // return original value of bit
 227     blr
 228
 229
 230 /* int32_t      OSAtomicAdd32Barrier( int32_t theAmount, int32_t *theValue ); */
 231
 232 MI_ENTRY_POINT(_OSAtomicAdd32Barrier)
 233     mflr    r12                 // save return address
 234     mr      r5,r4               // move ptr to where compare-and-swap wants it
 235     mr      r11,r3              // copy theAmount
 236 1:
 237     lwz     r3,0(r5)            // get old value
 238     add     r4,r3,r11           // make new value
 239     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 240     cmpwi   r3,0                // did swap occur?
 241     beq--   1b                  // compare-and-swap failed, try again
 242     mtlr    r12                 // restore return adddress
 243     mr      r3,r4               // return new value
 244     blr
 245
 246
 247 /* int32_t      OSAtomicOr32Barrier( int32_t theMask, int32_t *theValue ); */
 248
 249 MI_ENTRY_POINT(_OSAtomicOr32Barrier)
 250     mflr    r12                 // save return address
 251     mr      r5,r4               // move ptr to where compare-and-swap wants it
 252     mr      r11,r3              // copy mask
 253 1:
 254     lwz     r3,0(r5)            // get old value
 255     or      r4,r3,r11           // make new value
 256     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 257     cmpwi   r3,0                // did swap occur?
 258     beq--   1b                  // compare-and-swap failed, try again
 259     mtlr    r12                 // restore return adddress
 260     mr      r3,r4               // return new value
 261     blr
 262
 263
 264 /* int32_t      OSAtomicAnd32Barrier( int32_t theMask, int32_t *theValue ); */
 265
 266 MI_ENTRY_POINT(_OSAtomicAnd32Barrier)
 267     mflr    r12                 // save return address
 268     mr      r5,r4               // move ptr to where compare-and-swap wants it
 269     mr      r11,r3              // copy mask
 270 1:
 271     lwz     r3,0(r5)            // get old value
 272     and     r4,r3,r11           // make new value
 273     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 274     cmpwi   r3,0                // did swap occur?
 275     beq--   1b                  // compare-and-swap failed, try again
 276     mtlr    r12                 // restore return adddress
 277     mr      r3,r4               // return new value
 278     blr
 279
 280
 281 /* int32_t      OSAtomicXor32Barrier( int32_t theMask, int32_t *theValue ); */
 282
 283 MI_ENTRY_POINT(_OSAtomicXor32Barrier)
 284     mflr    r12                 // save return address
 285     mr      r5,r4               // move ptr to where compare-and-swap wants it
 286     mr      r11,r3              // copy mask
 287 1:
 288     lwz     r3,0(r5)            // get old value
 289     xor     r4,r3,r11           // make new value
 290     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 291     cmpwi   r3,0                // did swap occur?
 292     beq--   1b                  // compare-and-swap failed, try again
 293     mtlr    r12                 // restore return adddress
 294     mr      r3,r4               // return new value
 295     blr
 296
 297
 298 /* int32_t      OSAtomicOr32OrigBarrier( int32_t theMask, int32_t *theValue ); */
 299
 300 MI_ENTRY_POINT(_OSAtomicOr32OrigBarrier)
 301     mflr    r12                 // save return address
 302     mr      r5,r4               // move ptr to where compare-and-swap wants it
 303     mr      r11,r3              // copy mask
 304 1:
 305     lwz     r3,0(r5)            // get old value
 306     mr      r10,r3              // save old value
 307     or      r4,r3,r11           // make new value
 308     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 309     cmpwi   r3,0                // did swap occur?
 310     beq--   1b                  // compare-and-swap failed, try again
 311     mtlr    r12                 // restore return adddress
 312     mr      r3,r10              // return old value
 313     blr
 314
 315
 316 /* int32_t      OSAtomicAnd32OrigBarrier( int32_t theMask, int32_t *theValue ); */
 317
 318 MI_ENTRY_POINT(_OSAtomicAnd32OrigBarrier)
 319     mflr    r12                 // save return address
 320     mr      r5,r4               // move ptr to where compare-and-swap wants it
 321     mr      r11,r3              // copy mask
 322 1:
 323     lwz     r3,0(r5)            // get old value
 324     mr      r10,r3              // save old value
 325     and     r4,r3,r11           // make new value
 326     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 327     cmpwi   r3,0                // did swap occur?
 328     beq--   1b                  // compare-and-swap failed, try again
 329     mtlr    r12                 // restore return adddress
 330     mr      r3,r10              // return old value
 331     blr
 332
 333
 334 /* int32_t      OSAtomicXor32OrigBarrier( int32_t theMask, int32_t *theValue ); */
 335
 336 MI_ENTRY_POINT(_OSAtomicXor32OrigBarrier)
 337     mflr    r12                 // save return address
 338     mr      r5,r4               // move ptr to where compare-and-swap wants it
 339     mr      r11,r3              // copy mask
 340 1:
 341     lwz     r3,0(r5)            // get old value
 342     mr      r10,r3              // save old value
 343     xor     r4,r3,r11           // make new value
 344     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 345     cmpwi   r3,0                // did swap occur?
 346     beq--   1b                  // compare-and-swap failed, try again
 347     mtlr    r12                 // restore return adddress
 348     mr      r3,r10              // return old value
 349     blr
 350
 351
 352 /* int64_t      OSAtomicAdd64Barrier( int64_t theAmount, int64_t *theValue ); */
 353
 354 #if defined(__ppc64__)
 355 MI_ENTRY_POINT(_OSAtomicAdd64Barrier)
 356     mflr    r12                 // save return address
 357     mr      r5,r4               // move ptr to where compare-and-swap wants it
 358     mr      r11,r3              // copy theAmount
 359 1:
 360     ld      r3,0(r5)            // get old value
 361     add     r4,r3,r11           // make new value
 362     bla     _COMM_PAGE_COMPARE_AND_SWAP64B   // preserves r4,r5,r9-r12
 363     cmpwi   r3,0                // did swap occur?
 364     beq--   1b                  // compare-and-swap failed, try again
 365     mtlr    r12                 // restore return adddress
 366     mr      r3,r4               // return new value
 367     blr
 368 #endif  /* defined(__ppc64__) */
 369
 370
 371 /* bool OSAtomicCompareAndSwap32Barrier( int32_t oldValue, int32_t newValue, int32_t *theValue ); */
 372 /* bool OSAtomicCompareAndSwap64Barrier( int64_t oldValue, int64_t newValue, int64_t *theValue ); */
 373 /* bool OSAtomicCompareAndSwapPtrBarrier( void* oldValue, void* newValue, void* *theValue ); */
 374 /* bool OSAtomicCompareAndSwapIntBarrier( int oldValue, int newValue, int *theValue ); */
 375 /* bool OSAtomicCompareAndSwapLongBarrier( long oldValue, long newValue, long *theValue ); */
 376
 377 MI_ENTRY_POINT(_OSAtomicCompareAndSwap32Barrier)
 378     ba      _COMM_PAGE_COMPARE_AND_SWAP32B
 379 MI_ENTRY_POINT(_OSAtomicCompareAndSwapIntBarrier)
 380     ba      _COMM_PAGE_COMPARE_AND_SWAP32B
 381
 382 #if defined(__ppc64__)
 383 MI_ENTRY_POINT(_OSAtomicCompareAndSwap64Barrier)
 384     ba      _COMM_PAGE_COMPARE_AND_SWAP64B
 385 MI_ENTRY_POINT(_OSAtomicCompareAndSwapPtrBarrier)
 386     ba      _COMM_PAGE_COMPARE_AND_SWAP64B
 387 MI_ENTRY_POINT(_OSAtomicCompareAndSwapLongBarrier)
 388     ba      _COMM_PAGE_COMPARE_AND_SWAP64B
 389 #else   /* !defined(__ppc64__) */
 390 MI_ENTRY_POINT(_OSAtomicCompareAndSwapPtrBarrier)
 391     ba      _COMM_PAGE_COMPARE_AND_SWAP32B
 392 MI_ENTRY_POINT(_OSAtomicCompareAndSwapLongBarrier)
 393     ba      _COMM_PAGE_COMPARE_AND_SWAP32B
 394 #endif  /* defined(__ppc64__) */
 395
 396
 397 /* bool OSAtomicTestAndSetBarrier( uint32_t n, void *theAddress ); */
 398
 399 MI_ENTRY_POINT(_OSAtomicTestAndSetBarrier)
 400     mflr    r12                 // save return
 401     srwi    r5,r3,3             // get byte offset of n
 402     rlwinm  r6,r3,0,0x7         // get bit position within byte
 403     add     r4,r4,r5            // r4 points to byte containing the bit
 404     lis     r10,0x8000          // light bit 0
 405     rlwimi  r6,r4,3,0x18        // r6 is bit position within word
 406     clrrgi  r5,r4,2             // point to word containing the bit
 407     srw     r10,r10,r6          // get mask for bit
 408     addi    r9,r6,1             // save bit position + 1
 409 1:
 410     lwz     r3,0(r5)            // get old word
 411     rlwnm   r11,r3,r9,0x1       // right justify old value of bit
 412     or      r4,r3,r10           // set it in new word
 413     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 414     cmpwi   r3,0                // did swap occur?
 415     beq--   1b                  // compare-and-swap failed, try again
 416     mtlr    r12                 // restore return adddress
 417     mr      r3,r11              // return original value of bit
 418     blr
 419
 420
 421 /* bool OSAtomicTestAndClearBarrier( uint32_t n, void *theAddress ); */
 422
 423 MI_ENTRY_POINT(_OSAtomicTestAndClearBarrier)
 424     mflr    r12                 // save return
 425     srwi    r5,r3,3             // get byte offset of n
 426     rlwinm  r6,r3,0,0x7         // get bit position within byte
 427     add     r4,r4,r5            // r4 points to byte containing the bit
 428     lis     r10,0x8000          // light bit 0
 429     rlwimi  r6,r4,3,0x18        // r6 is bit position within word
 430     clrrgi  r5,r4,2             // point to word containing the bit
 431     srw     r10,r10,r6          // get mask for bit
 432     addi    r9,r6,1             // save bit position + 1
 433 1:
 434     lwz     r3,0(r5)            // get old word
 435     rlwnm   r11,r3,r9,0x1       // right justify old value of bit
 436     andc    r4,r3,r10           // clear it in new word
 437     bla     _COMM_PAGE_COMPARE_AND_SWAP32B   // preserves r4,r5,r9-r12
 438     cmpwi   r3,0                // did swap occur?
 439     beq--   1b                  // compare-and-swap failed, try again
 440     mtlr    r12                 // restore return adddress
 441     mr      r3,r11              // return original value of bit
 442     blr
 443
 444 /* bool    OSSpinLockTry( OSSpinLock *lock ); */
 445
 446 MI_ENTRY_POINT(_OSSpinLockTry)
 447     .globl  __spin_lock_try
 448 __spin_lock_try:
 449     ba      _COMM_PAGE_SPINLOCK_TRY
 450
 451
 452 /* void    OSSpinLockLock( OSSpinLock *lock ); */
 453
 454 MI_ENTRY_POINT(_OSSpinLockLock)
 455     .globl  _spin_lock
 456     .globl  __spin_lock
 457 _spin_lock:
 458 __spin_lock:
 459     ba      _COMM_PAGE_SPINLOCK_LOCK
 460
 461
 462 /* void    OSSpinLockUnlock( OSSpinLock *lock ); */
 463
 464 MI_ENTRY_POINT(_OSSpinLockUnlock)
 465     .globl  _spin_unlock
 466     .globl  __spin_unlock
 467 _spin_unlock:
 468 __spin_unlock:
 469     ba      _COMM_PAGE_SPINLOCK_UNLOCK
 470
 471
 472 /* void    OSMemoryBarrier( void ); */
 473
 474 MI_ENTRY_POINT(_OSMemoryBarrier)
 475     ba     _COMM_PAGE_MEMORY_BARRIER
 476
 477
 478 /* void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset); */
 479
 480 MI_ENTRY_POINT(_OSAtomicEnqueue)
 481     ba      _COMM_PAGE_ENQUEUE
 482
 483
 484 /* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
 485
 486 MI_ENTRY_POINT(_OSAtomicDequeue)
 487     ba      _COMM_PAGE_DEQUEUE