osfmk/ppc/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #include <sys/appleapiopts.h>
  32 #include <ppc/asm.h>                                    // EXT, LEXT
  33 #include <machine/cpu_capabilities.h>
  34 #include <machine/commpage.h>
  35
  36         .text
  37         .align  2
  38
  39 #define MP_SPIN_TRIES   1000
  40
  41
  42 /* The user mode spinlock library.  There are many versions,
  43  * in order to take advantage of a few special cases:
  44  *      - no barrier instructions (SYNC,ISYNC) are needed if UP
  45  *      - 64-bit processors can use LWSYNC instead of SYNC (if MP)
  46  *  - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
  47  *      - branch hints appropriate to the processor (+ vs ++ etc)
  48  *      - potentially custom relinquish strategies (not used at present)
  49  *      - fixes for errata as necessary
  50  *
  51  * The convention for lockwords is that 0==free and -1==locked.
  52  */
  53
  54
  55 spinlock_32_try_mp:
  56                 mr              r5, r3
  57                 li              r3, 1
  58 1:
  59         lwarx   r4,0,r5
  60                 li              r6,-1           // locked == -1
  61         cmpwi   r4,0
  62         bne-    2f
  63         stwcx.  r6,0,r5
  64         isync                           // cancel speculative execution
  65         beqlr+
  66         b               1b
  67 2:
  68         li              r3,0        // we did not get the lock
  69         blr
  70
  71         COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
  72
  73
  74 spinlock_32_try_up:
  75                 mr              r5, r3
  76                 li              r3, 1
  77 1:
  78         lwarx   r4,0,r5
  79                 li              r6,-1           // locked == -1
  80         cmpwi   r4,0
  81         bne-    2f
  82         stwcx.  r6,0,r5
  83         beqlr+
  84         b               1b
  85 2:
  86         li              r3,0        // we did not get the lock
  87         blr
  88
  89     COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)
  90
  91
  92 spinlock_32_lock_mp:
  93         li              r5,MP_SPIN_TRIES
  94 1:
  95         lwarx   r4,0,r3
  96                 li              r6,-1           // locked == -1
  97         cmpwi   r4,0
  98         bne-    2f
  99         stwcx.  r6,0,r3
 100         isync                           // cancel speculative execution
 101         beqlr+                          // we return void
 102         b               1b
 103 2:
 104         subic.  r5,r5,1         // try again before relinquish?
 105         bne             1b
 106         ba              _COMM_PAGE_RELINQUISH
 107
 108     COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)
 109
 110
 111 spinlock_32_lock_up:
 112 1:
 113         lwarx   r4,0,r3
 114                 li              r6,-1           // locked == -1
 115         cmpwi   r4,0
 116         bnea-   _COMM_PAGE_RELINQUISH   // always depress on UP (let lock owner run)
 117         stwcx.  r6,0,r3
 118         beqlr+                          // we return void
 119         b               1b
 120
 121     COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)
 122
 123
 124 spinlock_32_unlock_mp:
 125         li              r4,0
 126         isync                           // complete prior stores before unlock
 127                 eieio                           // (using isync/eieio is faster than a sync)
 128         stw             r4,0(r3)
 129         blr
 130
 131     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)
 132
 133
 134 spinlock_32_unlock_up:
 135         li              r4,0
 136         stw             r4,0(r3)
 137         blr
 138
 139     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)
 140
 141
 142 spinlock_64_try_mp:
 143                 mr              r5, r3
 144                 li              r3, 1
 145 1:
 146         lwarx   r4,0,r5
 147                 li              r6,-1           // locked == -1
 148         cmpwi   r4,0
 149         bne--   2f
 150         stwcx.  r6,0,r5
 151         isync                           // cancel speculative execution
 152         beqlr++
 153         b               1b
 154 2:
 155         li              r6,-4
 156         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 157         li              r3,0        // we did not get the lock
 158         blr
 159
 160     COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)
 161
 162
 163 spinlock_64_try_up:
 164                 mr              r5, r3
 165                 li              r3, 1
 166 1:
 167         lwarx   r4,0,r5
 168                 li              r6,-1           // locked == -1
 169         cmpwi   r4,0
 170         bne--   2f
 171         stwcx.  r6,0,r5
 172         beqlr++
 173         b               1b
 174 2:
 175         li              r6,-4
 176         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 177         li              r3,0        // we did not get the lock
 178         blr
 179
 180     COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)
 181
 182
 183 spinlock_64_lock_mp:
 184         li              r5,MP_SPIN_TRIES
 185 1:
 186         lwarx   r4,0,r3
 187                 li              r6,-1           // locked == -1
 188         cmpwi   r4,0
 189         bne--   2f
 190         stwcx.  r6,0,r3
 191         isync                           // cancel speculative execution
 192         beqlr++                         // we return void
 193         b               1b
 194 2:
 195         li              r6,-4
 196         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 197         subic.  r5,r5,1         // try again before relinquish?
 198         bne--   1b                      // mispredict this one (a cheap back-off)
 199         ba              _COMM_PAGE_RELINQUISH
 200
 201     COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)
 202
 203
 204 spinlock_64_lock_up:
 205 1:
 206         lwarx   r4,0,r3
 207                 li              r6,-1           // locked == -1
 208         cmpwi   r4,0
 209         bne--   2f
 210         stwcx.  r6,0,r3
 211         beqlr++                         // we return void
 212         b               1b
 213 2:                                                      // always relinquish on UP (let lock owner run)
 214         li              r6,-4
 215         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 216                 ba              _COMM_PAGE_RELINQUISH
 217
 218     COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)
 219
 220
 221 spinlock_64_unlock_mp:
 222         lwsync                          // complete prior stores before unlock
 223         li              r4,0
 224         stw             r4,0(r3)
 225         blr
 226
 227     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)
 228
 229
 230 spinlock_64_unlock_up:
 231         li              r4,0
 232         stw             r4,0(r3)
 233         blr
 234
 235     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
 236
 237
 238 spinlock_relinquish:
 239         mr              r12,r3          // preserve lockword ptr across relinquish
 240         li              r3,0            // THREAD_NULL
 241         li              r4,1            // SWITCH_OPTION_DEPRESS
 242         li              r5,1            // timeout (ms)
 243         li              r0,-61          // SYSCALL_THREAD_SWITCH
 244         sc                                      // relinquish
 245         mr              r3,r12
 246         ba              _COMM_PAGE_SPINLOCK_LOCK
 247
 248     COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
 249