osfmk/i386/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <sys/appleapiopts.h>
  24 #include <machine/cpu_capabilities.h>
  25 #include <machine/commpage.h>
  26
  27 /*
  28  * We need a relative branch within the comm page, and don't want the linker
  29  * to relocate it, so we have to hand-code the instructions. LEN is to account
  30  * for the length of a .long, since the jmp is relative to the next instruction.
  31  */
  32
  33 #define JNZ .byte 0x0f, 0x85; .long
  34 #define JMP .byte 0xe9; .long
  35 #define LEN 4
  36
  37 /*
  38  * Branch prediction prefixes
  39  */
  40
  41 #define LIKELY          .byte 0x3e
  42 #define UNLIKELY        .byte 0x2e
  43
  44 #define MP_SPIN_TRIES   1024
  45
  46         .text
  47         .align 4, 0x90
  48
  49 Lspin_lock_try_up:
  50         movl            4(%esp), %ecx
  51         xorl            %eax, %eax
  52         orl             $-1, %edx
  53         cmpxchgl        %edx, (%ecx)
  54         setz            %dl
  55         movzbl          %dl, %eax
  56         ret
  57
  58         COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  59
  60         .align 4, 0x90
  61 Lspin_lock_try_mp:
  62         movl            4(%esp), %ecx
  63         xorl            %eax, %eax
  64         orl             $-1, %edx
  65         lock
  66         cmpxchgl        %edx, (%ecx)
  67         setz            %dl
  68         movzbl          %dl, %eax
  69         ret
  70
  71         COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
  72
  73 .set Lrelinquish_off,   _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
  74
  75         .align 4, 0x90
  76 Lspin_lock_up:
  77         movl            4(%esp), %ecx
  78         xorl            %eax, %eax
  79 .set Lretry,            . - Lspin_lock_up
  80         orl             $-1, %edx
  81         cmpxchgl        %edx, (%ecx)
  82         UNLIKELY
  83         JNZ             Lrelinquish_off - . + Lspin_lock_up - LEN
  84         ret
  85
  86         COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
  87
  88         .align 4, 0x90
  89 Lspin_lock_mp:
  90         movl            4(%esp), %ecx
  91         xorl            %eax, %eax
  92 0:
  93         orl             $-1, %edx
  94         lock
  95         cmpxchgl        %edx, (%ecx)
  96         UNLIKELY
  97         jnz             1f
  98         ret
  99 1:
 100         xorl            %eax, %eax
 101         movl            $(MP_SPIN_TRIES), %edx
 102 2:
 103         pause
 104         cmpl            %eax, (%ecx)
 105         LIKELY
 106         jz              0b
 107         decl            %edx
 108         LIKELY
 109         jnz             2b
 110         JMP             Lrelinquish_off - . + Lspin_lock_mp - LEN
 111
 112         COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 113
 114         .align 4, 0x90
 115 Lspin_unlock:
 116         movl            4(%esp), %ecx
 117         movl            $0, (%ecx)
 118         ret
 119
 120         COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 121
 122         .align 4, 0x90
 123 Lrelinquish:                            /* relinquish the processor     */
 124         pushl           $1              /* 1 ms                         */
 125         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
 126         pushl           $0              /* THREAD_NULL                  */
 127         pushl           $0              /* push dummy stack ret addr    */
 128         movl            $-61, %eax      /* syscall_thread_switch        */
 129         lcall           $7, $0
 130         addl            $16, %esp       /* adjust stack*/
 131         xorl            %eax, %eax      /* set %eax to 0 again          */
 132         JMP             Lretry - Lrelinquish_off - . + Lrelinquish - LEN
 133
 134         COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)