osfmk/i386/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/appleapiopts.h>
  30 #include <machine/cpu_capabilities.h>
  31 #include <machine/commpage.h>
  32 #include <mach/i386/syscall_sw.h>
  33
  34
  35
  36 #define MP_SPIN_TRIES   1024
  37
  38         .text
  39         .align 4, 0x90
  40
  41 Lspin_lock_try_up:
  42         movl            4(%esp), %ecx
  43         xorl            %eax, %eax
  44         orl             $-1, %edx
  45         cmpxchgl        %edx, (%ecx)
  46         setz            %dl
  47         movzbl          %dl, %eax
  48         ret
  49
  50         COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  51
  52
  53         .align 4, 0x90
  54 Lspin_lock_try_mp:
  55         movl            4(%esp), %ecx
  56         xorl            %eax, %eax
  57         orl             $-1, %edx
  58         lock
  59         cmpxchgl        %edx, (%ecx)
  60         setz            %dl
  61         movzbl          %dl, %eax
  62         ret
  63
  64         COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
  65
  66
  67         .align 4, 0x90
  68 Lspin_lock_up:
  69         movl            4(%esp), %ecx
  70         xorl            %eax, %eax
  71         orl             $-1, %edx
  72         cmpxchgl        %edx, (%ecx)
  73         jnz,pn          1f              /* predict not taken */
  74         ret
  75 1:
  76         /* failed to get lock so relinquish the processor immediately on UP */
  77         pushl           $1              /* 1 ms                         */
  78         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
  79         pushl           $0              /* THREAD_NULL                  */
  80         pushl           $0              /* push dummy stack ret addr    */
  81         movl            $-61,%eax       /* SYSCALL_THREAD_SWITCH */
  82         int             $(MACH_INT)
  83         addl            $16, %esp       /* adjust stack*/
  84         jmp             Lspin_lock_up
  85
  86         COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
  87
  88
  89         .align 4, 0x90
  90 Lspin_lock_mp:
  91         movl            4(%esp), %ecx
  92         xorl            %eax, %eax
  93 0:
  94         orl             $-1, %edx
  95         lock
  96         cmpxchgl        %edx, (%ecx)
  97         jnz,pn          1f              /* predict not taken */
  98         ret
  99 1:
 100         xorl            %eax, %eax
 101         movl            $(MP_SPIN_TRIES), %edx
 102 2:
 103         pause
 104         cmpl            %eax, (%ecx)
 105         jz,pt           0b              /* favor success and slow down spin loop */
 106         decl            %edx
 107         jnz,pn          2b              /* slow down spin loop with a mispredict */
 108         /* failed to get lock after spinning so relinquish  */
 109         pushl           $1              /* 1 ms                         */
 110         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
 111         pushl           $0              /* THREAD_NULL                  */
 112         pushl           $0              /* push dummy stack ret addr    */
 113         movl            $-61,%eax       /* SYSCALL_THREAD_SWITCH */
 114         int             $(MACH_INT)
 115         addl            $16, %esp       /* adjust stack*/
 116         jmp             Lspin_lock_mp
 117
 118         COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 119
 120
 121         .align 4, 0x90
 122 Lspin_unlock:
 123         movl            4(%esp), %ecx
 124         movl            $0, (%ecx)
 125         ret
 126
 127         COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 128
 129
 130 /* ============================ 64-bit versions follow ===================== */
 131
 132
 133         .text
 134         .code64
 135         .align 4, 0x90
 136
 137 Lspin_lock_try_up_64:
 138         xorl            %eax, %eax
 139         orl             $-1, %edx
 140         cmpxchgl        %edx, (%rdi)
 141         setz            %dl
 142         movzbl          %dl, %eax
 143         ret
 144
 145         COMMPAGE_DESCRIPTOR(spin_lock_try_up_64,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
 146
 147
 148         .align 4, 0x90
 149 Lspin_lock_try_mp_64:
 150         xorl            %eax, %eax
 151         orl             $-1, %edx
 152         lock
 153         cmpxchgl        %edx, (%rdi)
 154         setz            %dl
 155         movzbl          %dl, %eax
 156         ret
 157
 158         COMMPAGE_DESCRIPTOR(spin_lock_try_mp_64,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
 159
 160
 161         .align 4, 0x90
 162 Lspin_lock_up_64:
 163         movq            %rdi,%r8
 164 0:
 165         xorl            %eax, %eax
 166         orl             $-1, %edx
 167         cmpxchgl        %edx, (%r8)
 168         jnz,pn          1f              /* predict not taken */
 169         ret
 170 1:
 171         /* failed to get lock so relinquish the processor immediately on UP */
 172         xorl            %edi,%edi       /* THREAD_NULL                  */
 173         movl            $1,%esi         /* SWITCH_OPTION_DEPRESS        */
 174         movl            $1,%edx         /* 1 ms                         */
 175         movl            $(SYSCALL_CONSTRUCT_MACH(61)),%eax      /* 61 = thread_switch */
 176         syscall
 177         jmp             0b
 178
 179         COMMPAGE_DESCRIPTOR(spin_lock_up_64,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
 180
 181
 182
 183         .align 4, 0x90
 184 Lspin_lock_mp_64:
 185         movq            %rdi,%r8
 186 0:
 187         xorl            %eax, %eax
 188         orl             $-1, %edx
 189         lock
 190         cmpxchgl        %edx, (%r8)
 191         jnz,pn          1f              /* predict not taken */
 192         ret
 193 1:
 194         xorl            %eax, %eax
 195         movl            $(MP_SPIN_TRIES), %edx
 196 2:                                      /* spin for awhile before relinquish */
 197         pause
 198         cmpl            %eax, (%r8)
 199         jz              0b
 200         decl            %edx
 201         jnz             2b
 202         /* failed to get lock after spinning so relinquish  */
 203         xorl            %edi,%edi       /* THREAD_NULL                  */
 204         movl            $1,%esi         /* SWITCH_OPTION_DEPRESS        */
 205         movl            $1,%edx         /* 1 ms                         */
 206         movl            $(SYSCALL_CONSTRUCT_MACH(61)),%eax      /* 61 = thread_switch */
 207         syscall
 208         jmp             0b
 209
 210         COMMPAGE_DESCRIPTOR(spin_lock_mp_64,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 211
 212
 213         .align 4, 0x90
 214 Lspin_unlock_64:
 215         movl            $0, (%rdi)
 216         ret
 217
 218         COMMPAGE_DESCRIPTOR(spin_unlock_64,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)