osfmk/i386/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/appleapiopts.h>
  30 #include <machine/cpu_capabilities.h>
  31 #include <machine/commpage.h>
  32
  33 /*
  34  * We need a relative branch within the comm page, and don't want the linker
  35  * to relocate it, so we have to hand-code the instructions. LEN is to account
  36  * for the length of a .long, since the jmp is relative to the next instruction.
  37  */
  38
  39 #define JNZ .byte 0x0f, 0x85; .long
  40 #define JMP .byte 0xe9; .long
  41 #define LEN 4
  42
  43 /*
  44  * Branch prediction prefixes
  45  */
  46
  47 #define LIKELY          .byte 0x3e
  48 #define UNLIKELY        .byte 0x2e
  49
  50 #define MP_SPIN_TRIES   1024
  51
  52         .text
  53         .align 4, 0x90
  54
  55 Lspin_lock_try_up:
  56         movl            4(%esp), %ecx
  57         xorl            %eax, %eax
  58         orl             $-1, %edx
  59         cmpxchgl        %edx, (%ecx)
  60         setz            %dl
  61         movzbl          %dl, %eax
  62         ret
  63
  64         COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  65
  66         .align 4, 0x90
  67 Lspin_lock_try_mp:
  68         movl            4(%esp), %ecx
  69         xorl            %eax, %eax
  70         orl             $-1, %edx
  71         lock
  72         cmpxchgl        %edx, (%ecx)
  73         setz            %dl
  74         movzbl          %dl, %eax
  75         ret
  76
  77         COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
  78
  79 .set Lrelinquish_off,   _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
  80
  81         .align 4, 0x90
  82 Lspin_lock_up:
  83         movl            4(%esp), %ecx
  84         xorl            %eax, %eax
  85 .set Lretry,            . - Lspin_lock_up
  86         orl             $-1, %edx
  87         cmpxchgl        %edx, (%ecx)
  88         UNLIKELY
  89         JNZ             Lrelinquish_off - . + Lspin_lock_up - LEN
  90         ret
  91
  92         COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
  93
  94         .align 4, 0x90
  95 Lspin_lock_mp:
  96         movl            4(%esp), %ecx
  97         xorl            %eax, %eax
  98 0:
  99         orl             $-1, %edx
 100         lock
 101         cmpxchgl        %edx, (%ecx)
 102         UNLIKELY
 103         jnz             1f
 104         ret
 105 1:
 106         xorl            %eax, %eax
 107         movl            $(MP_SPIN_TRIES), %edx
 108 2:
 109         pause
 110         cmpl            %eax, (%ecx)
 111         LIKELY
 112         jz              0b
 113         decl            %edx
 114         LIKELY
 115         jnz             2b
 116         JMP             Lrelinquish_off - . + Lspin_lock_mp - LEN
 117
 118         COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 119
 120         .align 4, 0x90
 121 Lspin_unlock:
 122         movl            4(%esp), %ecx
 123         movl            $0, (%ecx)
 124         ret
 125
 126         COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 127
 128         .align 4, 0x90
 129 Lrelinquish:                            /* relinquish the processor     */
 130         pushl           $1              /* 1 ms                         */
 131         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
 132         pushl           $0              /* THREAD_NULL                  */
 133         pushl           $0              /* push dummy stack ret addr    */
 134         movl            $-61, %eax      /* syscall_thread_switch        */
 135         lcall           $7, $0
 136         addl            $16, %esp       /* adjust stack*/
 137         xorl            %eax, %eax      /* set %eax to 0 again          */
 138         JMP             Lretry - Lrelinquish_off - . + Lrelinquish - LEN
 139
 140         COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)