osfmk/ppc/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/appleapiopts.h>
  30 #include <ppc/asm.h>                                    // EXT, LEXT
  31 #include <machine/cpu_capabilities.h>
  32 #include <machine/commpage.h>
  33
  34         .text
  35         .align  2
  36
  37 #define MP_SPIN_TRIES   1000
  38
  39
  40 /* The user mode spinlock library.  There are many versions,
  41  * in order to take advantage of a few special cases:
  42  *      - no barrier instructions (SYNC,ISYNC) are needed if UP
  43  *      - 64-bit processors can use LWSYNC instead of SYNC (if MP)
  44  *  - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
  45  *      - branch hints appropriate to the processor (+ vs ++ etc)
  46  *      - potentially custom relinquish strategies (not used at present)
  47  *      - fixes for errata as necessary
  48  *
  49  * The convention for lockwords is that 0==free and -1==locked.
  50  */
  51
  52
  53 spinlock_32_try_mp:
  54                 mr              r5, r3
  55                 li              r3, 1
  56 1:
  57         lwarx   r4,0,r5
  58                 li              r6,-1           // locked == -1
  59         cmpwi   r4,0
  60         bne-    2f
  61         stwcx.  r6,0,r5
  62         isync                           // cancel speculative execution
  63         beqlr+
  64         b               1b
  65 2:
  66         li              r3,0        // we did not get the lock
  67         blr
  68
  69         COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
  70
  71
  72 spinlock_32_try_up:
  73                 mr              r5, r3
  74                 li              r3, 1
  75 1:
  76         lwarx   r4,0,r5
  77                 li              r6,-1           // locked == -1
  78         cmpwi   r4,0
  79         bne-    2f
  80         stwcx.  r6,0,r5
  81         beqlr+
  82         b               1b
  83 2:
  84         li              r3,0        // we did not get the lock
  85         blr
  86
  87     COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)
  88
  89
  90 spinlock_32_lock_mp:
  91         li              r5,MP_SPIN_TRIES
  92 1:
  93         lwarx   r4,0,r3
  94                 li              r6,-1           // locked == -1
  95         cmpwi   r4,0
  96         bne-    2f
  97         stwcx.  r6,0,r3
  98         isync                           // cancel speculative execution
  99         beqlr+                          // we return void
 100         b               1b
 101 2:
 102         subic.  r5,r5,1         // try again before relinquish?
 103         bne             1b
 104         ba              _COMM_PAGE_RELINQUISH
 105
 106     COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)
 107
 108
 109 spinlock_32_lock_up:
 110 1:
 111         lwarx   r4,0,r3
 112                 li              r6,-1           // locked == -1
 113         cmpwi   r4,0
 114         bnea-   _COMM_PAGE_RELINQUISH   // always depress on UP (let lock owner run)
 115         stwcx.  r6,0,r3
 116         beqlr+                          // we return void
 117         b               1b
 118
 119     COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)
 120
 121
 122 spinlock_32_unlock_mp:
 123         li              r4,0
 124         isync                           // complete prior stores before unlock
 125                 eieio                           // (using isync/eieio is faster than a sync)
 126         stw             r4,0(r3)
 127         blr
 128
 129     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)
 130
 131
 132 spinlock_32_unlock_up:
 133         li              r4,0
 134         stw             r4,0(r3)
 135         blr
 136
 137     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)
 138
 139
 140 spinlock_64_try_mp:
 141                 mr              r5, r3
 142                 li              r3, 1
 143 1:
 144         lwarx   r4,0,r5
 145                 li              r6,-1           // locked == -1
 146         cmpwi   r4,0
 147         bne--   2f
 148         stwcx.  r6,0,r5
 149         isync                           // cancel speculative execution
 150         beqlr++
 151         b               1b
 152 2:
 153         li              r6,-4
 154         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 155         li              r3,0        // we did not get the lock
 156         blr
 157
 158     COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)
 159
 160
 161 spinlock_64_try_up:
 162                 mr              r5, r3
 163                 li              r3, 1
 164 1:
 165         lwarx   r4,0,r5
 166                 li              r6,-1           // locked == -1
 167         cmpwi   r4,0
 168         bne--   2f
 169         stwcx.  r6,0,r5
 170         beqlr++
 171         b               1b
 172 2:
 173         li              r6,-4
 174         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 175         li              r3,0        // we did not get the lock
 176         blr
 177
 178     COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)
 179
 180
 181 spinlock_64_lock_mp:
 182         li              r5,MP_SPIN_TRIES
 183 1:
 184         lwarx   r4,0,r3
 185                 li              r6,-1           // locked == -1
 186         cmpwi   r4,0
 187         bne--   2f
 188         stwcx.  r6,0,r3
 189         isync                           // cancel speculative execution
 190         beqlr++                         // we return void
 191         b               1b
 192 2:
 193         li              r6,-4
 194         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 195         subic.  r5,r5,1         // try again before relinquish?
 196         bne--   1b                      // mispredict this one (a cheap back-off)
 197         ba              _COMM_PAGE_RELINQUISH
 198
 199     COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)
 200
 201
 202 spinlock_64_lock_up:
 203 1:
 204         lwarx   r4,0,r3
 205                 li              r6,-1           // locked == -1
 206         cmpwi   r4,0
 207         bne--   2f
 208         stwcx.  r6,0,r3
 209         beqlr++                         // we return void
 210         b               1b
 211 2:                                                      // always relinquish on UP (let lock owner run)
 212         li              r6,-4
 213         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 214                 ba              _COMM_PAGE_RELINQUISH
 215
 216     COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)
 217
 218
 219 spinlock_64_unlock_mp:
 220         lwsync                          // complete prior stores before unlock
 221         li              r4,0
 222         stw             r4,0(r3)
 223         blr
 224
 225     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)
 226
 227
 228 spinlock_64_unlock_up:
 229         li              r4,0
 230         stw             r4,0(r3)
 231         blr
 232
 233     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
 234
 235
 236 spinlock_relinquish:
 237         mr              r12,r3          // preserve lockword ptr across relinquish
 238         li              r3,0            // THREAD_NULL
 239         li              r4,1            // SWITCH_OPTION_DEPRESS
 240         li              r5,1            // timeout (ms)
 241         li              r0,-61          // SYSCALL_THREAD_SWITCH
 242         sc                                      // relinquish
 243         mr              r3,r12
 244         ba              _COMM_PAGE_SPINLOCK_LOCK
 245
 246     COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
 247