osfmk/ppc/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <sys/appleapiopts.h>
  24 #include <ppc/asm.h>                                    // EXT, LEXT
  25 #include <machine/cpu_capabilities.h>
  26 #include <machine/commpage.h>
  27
  28         .text
  29         .align  2
  30         .globl  EXT(spinlock_32_try_mp)
  31         .globl  EXT(spinlock_32_try_up)
  32         .globl  EXT(spinlock_32_lock_mp)
  33         .globl  EXT(spinlock_32_lock_up)
  34         .globl  EXT(spinlock_32_unlock_mp)
  35         .globl  EXT(spinlock_32_unlock_up)
  36
  37         .globl  EXT(spinlock_64_try_mp)
  38         .globl  EXT(spinlock_64_try_up)
  39         .globl  EXT(spinlock_64_lock_mp)
  40         .globl  EXT(spinlock_64_lock_up)
  41         .globl  EXT(spinlock_64_unlock_mp)
  42         .globl  EXT(spinlock_64_unlock_up)
  43
  44         .globl  EXT(spinlock_relinquish)
  45
  46 #define MP_SPIN_TRIES   1000
  47
  48
  49 // The user mode spinlock library.  There are many versions,
  50 // in order to take advantage of a few special cases:
  51 //      - no barrier instructions (SYNC,ISYNC) are needed if UP
  52 //      - 64-bit processors can use LWSYNC instead of SYNC (if MP)
  53 //      - branch hints appropriate to the processor (+ vs ++ etc)
  54 //      - potentially custom relinquish strategies (not used at present)
  55 //      - fixes for errata as necessary
  56
  57
  58 spinlock_32_try_mp:
  59         mr      r5, r3
  60         li      r3, 1
  61 1:
  62         lwarx   r4,0,r5
  63         cmpwi   r4,0
  64         bne-    2f
  65         stwcx.  r5,0,r5
  66         isync                           // cancel speculative execution
  67         beqlr+
  68         b               1b
  69 2:
  70         li              r3,0
  71         blr
  72
  73     COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,0)
  74
  75
  76 spinlock_32_try_up:
  77         mr      r5, r3
  78         li      r3, 1
  79 1:
  80         lwarx   r4,0,r5
  81         cmpwi   r4,0
  82         bne-    2f
  83         stwcx.  r5,0,r5
  84         beqlr+
  85         b               1b
  86 2:
  87         li              r3,0
  88         blr
  89
  90     COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,0)
  91
  92
  93 spinlock_32_lock_mp:
  94         li              r5,MP_SPIN_TRIES
  95 1:
  96         lwarx   r4,0,r3
  97         cmpwi   r4,0
  98         bne-    2f
  99         stwcx.  r3,0,r3
 100         isync                           // cancel speculative execution
 101         beqlr+                          // we return void
 102         b               1b
 103 2:
 104         subic.  r5,r5,1         // try again before relinquish?
 105         bne             1b
 106         ba              _COMM_PAGE_RELINQUISH
 107
 108     COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,0)
 109
 110
 111 spinlock_32_lock_up:
 112 1:
 113         lwarx   r4,0,r3
 114         cmpwi   r4,0
 115         bnea-   _COMM_PAGE_RELINQUISH   // always depress on UP (let lock owner run)
 116         stwcx.  r3,0,r3
 117         beqlr+                          // we return void
 118         b               1b
 119
 120     COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,0)
 121
 122
 123 spinlock_32_unlock_mp:
 124         li              r4,0
 125         sync                            // complete prior stores before unlock
 126         stw             r4,0(r3)
 127         blr
 128
 129     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,0)
 130
 131
 132 spinlock_32_unlock_up:
 133         li              r4,0
 134         stw             r4,0(r3)
 135         blr
 136
 137     COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,0)
 138
 139
 140 spinlock_64_try_mp:
 141         mr      r5, r3
 142         li      r3, 1
 143 1:
 144         lwarx   r4,0,r5
 145         cmpwi   r4,0
 146         bne--   2f
 147         stwcx.  r5,0,r5
 148         isync                           // cancel speculative execution
 149         beqlr++
 150         b               1b
 151 2:
 152         li              r6,-4
 153         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 154         li              r3,0            // Pass failure
 155         blr
 156
 157     COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,0)
 158
 159
 160 spinlock_64_try_up:
 161         mr      r5, r3
 162         li      r3, 1
 163 1:
 164         lwarx   r4,0,r5
 165         cmpwi   r4,0
 166         bne--   2f
 167         stwcx.  r5,0,r5
 168         beqlr++
 169         b               1b
 170 2:
 171         li              r6,-4
 172         stwcx.  r5,r6,r1        // clear the pending reservation (using red zone)
 173         li              r3,0
 174         blr
 175
 176     COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,0)
 177
 178
 179 spinlock_64_lock_mp:
 180         li              r5,MP_SPIN_TRIES
 181 1:
 182         lwarx   r4,0,r3
 183         cmpwi   r4,0
 184         bne--   2f
 185         stwcx.  r3,0,r3
 186         isync                           // cancel speculative execution
 187         beqlr++                         // we return void
 188         b               1b
 189 2:
 190         li              r6,-4
 191         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 192         subic.  r5,r5,1         // try again before relinquish?
 193         bne--   1b                      // mispredict this one (a cheap back-off)
 194         ba              _COMM_PAGE_RELINQUISH
 195
 196     COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,0)
 197
 198
 199 spinlock_64_lock_up:
 200 1:
 201         lwarx   r4,0,r3
 202         cmpwi   r4,0
 203         bne--   2f
 204         stwcx.  r3,0,r3
 205         beqlr++                         // we return void
 206         b               1b
 207 2:                                                      // always relinquish on UP (let lock owner run)
 208         li              r6,-4
 209         stwcx.  r3,r6,r1        // clear the pending reservation (using red zone)
 210                 ba              _COMM_PAGE_RELINQUISH
 211
 212     COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,0)
 213
 214
 215 spinlock_64_unlock_mp:
 216         li              r4,0
 217         lwsync                          // complete prior stores before unlock
 218         stw             r4,0(r3)
 219         blr
 220
 221     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,0)
 222
 223
 224 spinlock_64_unlock_up:
 225         li              r4,0
 226         stw             r4,0(r3)
 227         blr
 228
 229     COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,0)
 230
 231
 232 spinlock_relinquish:
 233         mr              r12,r3          // preserve lockword ptr across relinquish
 234         li              r3,0            // THREAD_NULL
 235         li              r4,1            // SWITCH_OPTION_DEPRESS
 236         li              r5,1            // timeout (ms)
 237         li              r0,-61          // SYSCALL_THREAD_SWITCH
 238         sc                                      // relinquish
 239         mr              r3,r12
 240         ba              _COMM_PAGE_SPINLOCK_LOCK
 241
 242     COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,0)
 243