osfmk/i386/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #include <sys/appleapiopts.h>
  32 #include <machine/cpu_capabilities.h>
  33 #include <machine/commpage.h>
  34
  35 /*
  36  * We need a relative branch within the comm page, and don't want the linker
  37  * to relocate it, so we have to hand-code the instructions. LEN is to account
  38  * for the length of a .long, since the jmp is relative to the next instruction.
  39  */
  40
  41 #define JNZ .byte 0x0f, 0x85; .long
  42 #define JMP .byte 0xe9; .long
  43 #define LEN 4
  44
  45 /*
  46  * Branch prediction prefixes
  47  */
  48
  49 #define LIKELY          .byte 0x3e
  50 #define UNLIKELY        .byte 0x2e
  51
  52 #define MP_SPIN_TRIES   1024
  53
  54         .text
  55         .align 4, 0x90
  56
  57 Lspin_lock_try_up:
  58         movl            4(%esp), %ecx
  59         xorl            %eax, %eax
  60         orl             $-1, %edx
  61         cmpxchgl        %edx, (%ecx)
  62         setz            %dl
  63         movzbl          %dl, %eax
  64         ret
  65
  66         COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  67
  68         .align 4, 0x90
  69 Lspin_lock_try_mp:
  70         movl            4(%esp), %ecx
  71         xorl            %eax, %eax
  72         orl             $-1, %edx
  73         lock
  74         cmpxchgl        %edx, (%ecx)
  75         setz            %dl
  76         movzbl          %dl, %eax
  77         ret
  78
  79         COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
  80
  81 .set Lrelinquish_off,   _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
  82
  83         .align 4, 0x90
  84 Lspin_lock_up:
  85         movl            4(%esp), %ecx
  86         xorl            %eax, %eax
  87 .set Lretry,            . - Lspin_lock_up
  88         orl             $-1, %edx
  89         cmpxchgl        %edx, (%ecx)
  90         UNLIKELY
  91         JNZ             Lrelinquish_off - . + Lspin_lock_up - LEN
  92         ret
  93
  94         COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
  95
  96         .align 4, 0x90
  97 Lspin_lock_mp:
  98         movl            4(%esp), %ecx
  99         xorl            %eax, %eax
 100 0:
 101         orl             $-1, %edx
 102         lock
 103         cmpxchgl        %edx, (%ecx)
 104         UNLIKELY
 105         jnz             1f
 106         ret
 107 1:
 108         xorl            %eax, %eax
 109         movl            $(MP_SPIN_TRIES), %edx
 110 2:
 111         pause
 112         cmpl            %eax, (%ecx)
 113         LIKELY
 114         jz              0b
 115         decl            %edx
 116         LIKELY
 117         jnz             2b
 118         JMP             Lrelinquish_off - . + Lspin_lock_mp - LEN
 119
 120         COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 121
 122         .align 4, 0x90
 123 Lspin_unlock:
 124         movl            4(%esp), %ecx
 125         movl            $0, (%ecx)
 126         ret
 127
 128         COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 129
 130         .align 4, 0x90
 131 Lrelinquish:                            /* relinquish the processor     */
 132         pushl           $1              /* 1 ms                         */
 133         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
 134         pushl           $0              /* THREAD_NULL                  */
 135         pushl           $0              /* push dummy stack ret addr    */
 136         movl            $-61, %eax      /* syscall_thread_switch        */
 137         lcall           $7, $0
 138         addl            $16, %esp       /* adjust stack*/
 139         xorl            %eax, %eax      /* set %eax to 0 again          */
 140         JMP             Lretry - Lrelinquish_off - . + Lrelinquish - LEN
 141
 142         COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)