osfmk/ppc/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <sys/appleapiopts.h>
  24 #include <ppc/asm.h>                                    // EXT, LEXT
  25 #include <machine/cpu_capabilities.h>
  26 #include <machine/commpage.h>
  27
  28         .text
  29         .align  2
  30
  31 #define MP_SPIN_TRIES   1000
  32
  33
  34 /* The user mode spinlock library.  There are many versions,
  35  * in order to take advantage of a few special cases:
  36  *      - no barrier instructions (SYNC,ISYNC) are needed if UP
  37  *      - 64-bit processors can use LWSYNC instead of SYNC (if MP)
  38  *  - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
  39  *      - branch hints appropriate to the processor (+ vs ++ etc)
  40  *      - potentially custom relinquish strategies (not used at present)
  41  *      - fixes for errata as necessary
  42  *
  43  * The convention for lockwords is that 0==free and -1==locked.
  44  */
  45
  46
  47 spinlock_32_try_mp:
  48                 mr              r5, r3
  49                 li              r3, 1
  50 1:
  51         lwarx   r4,0,r5
  52                 li              r6,-1           // locked == -1
  53         cmpwi   r4,0
  54         bne-    2f
  55         stwcx.  r6,0,r5
  56         isync                           // cancel speculative execution
  57         beqlr+
  58         b               1b
  59 2:
  60         li              r3,0        // we did not get the lock
  61         blr
  62
  63         COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
  64
  65
  66 spinlock_32_try_up:
  67                 mr              r5, r3
  68                 li              r3, 1
  69 1:
  70         lwarx   r4,0,r5
  71                 li              r6,-1           // locked == -1
  72         cmpwi   r4,0
  73         bne-    2f
  74         stwcx.  r6,0,r5
  75         beqlr+
  76         b               1b
  77 2:
  78         li              r3,0        // we did not get the lock
  79         blr
  80
  81     COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)
  82
  83
  84 spinlock_32_lock_mp:
  85         li              r5,MP_SPIN_TRIES
  86 1:
  87         lwarx   r4,0,r3
  88                 li              r6,-1           // locked == -1
  89         cmpwi   r4,0
  90         bne-    2f
  91         stwcx.  r6,0,r3
  92         isync                           // cancel speculative execution
  93         beqlr+                          // we return void
  94         b               1b
  95 2:
  96         subic.  r5,r5,1         // try again before relinquish?
  97         bne             1b
  98         ba              _COMM_PAGE_RELINQUISH
  99
 100     COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)
 101
 102
 103 spinlock_32_lock_up:
 104 1:
 105         lwarx   r4,0,r3
 106                 li              r6,-1           // locked == -1
 107         cmpwi   r4,0
 108         bnea-   _COMM_PAGE_RELINQUISH   // always depress on UP (let lock owner run)
 109         stwcx.  r6,0,r3
 110         beqlr+                          // we return void
 111         b               1b
 112
 113     COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)
 114
 115
 116 spinlock_32_unlock_mp:
 117         li              r4,0
 118         isync                           // complete prior stores before unlock
 119                 eieio                           // (using isync/eieio is faster than a sync)
 120         stw             r4,0(r3)
 121         blr
 122
 123     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)
 124
 125
 126 spinlock_32_unlock_up:
 127         li              r4,0
 128         stw             r4,0(r3)
 129         blr
 130
 131     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)
 132
 133
 134 spinlock_64_try_mp:
 135                 mr              r5, r3
 136                 li              r3, 1
 137 1:
 138         lwarx   r4,0,r5
 139                 li              r6,-1           // locked == -1
 140         cmpwi   r4,0
 141         bne--   2f
 142         stwcx.  r6,0,r5
 143         isync                           // cancel speculative execution
 144         beqlr++
 145         b               1b
 146 2:
 147         li              r6,-4
 148         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 149         li              r3,0        // we did not get the lock
 150         blr
 151
 152     COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)
 153
 154
 155 spinlock_64_try_up:
 156                 mr              r5, r3
 157                 li              r3, 1
 158 1:
 159         lwarx   r4,0,r5
 160                 li              r6,-1           // locked == -1
 161         cmpwi   r4,0
 162         bne--   2f
 163         stwcx.  r6,0,r5
 164         beqlr++
 165         b               1b
 166 2:
 167         li              r6,-4
 168         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 169         li              r3,0        // we did not get the lock
 170         blr
 171
 172     COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)
 173
 174
 175 spinlock_64_lock_mp:
 176         li              r5,MP_SPIN_TRIES
 177 1:
 178         lwarx   r4,0,r3
 179                 li              r6,-1           // locked == -1
 180         cmpwi   r4,0
 181         bne--   2f
 182         stwcx.  r6,0,r3
 183         isync                           // cancel speculative execution
 184         beqlr++                         // we return void
 185         b               1b
 186 2:
 187         li              r6,-4
 188         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 189         subic.  r5,r5,1         // try again before relinquish?
 190         bne--   1b                      // mispredict this one (a cheap back-off)
 191         ba              _COMM_PAGE_RELINQUISH
 192
 193     COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)
 194
 195
 196 spinlock_64_lock_up:
 197 1:
 198         lwarx   r4,0,r3
 199                 li              r6,-1           // locked == -1
 200         cmpwi   r4,0
 201         bne--   2f
 202         stwcx.  r6,0,r3
 203         beqlr++                         // we return void
 204         b               1b
 205 2:                                                      // always relinquish on UP (let lock owner run)
 206         li              r6,-4
 207         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 208                 ba              _COMM_PAGE_RELINQUISH
 209
 210     COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)
 211
 212
 213 spinlock_64_unlock_mp:
 214         lwsync                          // complete prior stores before unlock
 215         li              r4,0
 216         stw             r4,0(r3)
 217         blr
 218
 219     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)
 220
 221
 222 spinlock_64_unlock_up:
 223         li              r4,0
 224         stw             r4,0(r3)
 225         blr
 226
 227     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
 228
 229
 230 spinlock_relinquish:
 231         mr              r12,r3          // preserve lockword ptr across relinquish
 232         li              r3,0            // THREAD_NULL
 233         li              r4,1            // SWITCH_OPTION_DEPRESS
 234         li              r5,1            // timeout (ms)
 235         li              r0,-61          // SYSCALL_THREAD_SWITCH
 236         sc                                      // relinquish
 237         mr              r3,r12
 238         ba              _COMM_PAGE_SPINLOCK_LOCK
 239
 240     COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
 241