osfmk/i386/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <sys/appleapiopts.h>
  24 #include <machine/cpu_capabilities.h>
  25 #include <machine/commpage.h>
  26
  27 /*
  28  * We need a relative branch within the comm page, and don't want the linker
  29  * to relocate it, so we have to hand-code the instructions. LEN is to account
  30  * for the length of a .long, since the jmp is relative to the next instruction.
  31  */
  32
  33 #define JNZ .byte 0x0f, 0x85; .long
  34 #define JMP .byte 0xe9; .long
  35 #define LEN 4
  36
  37 /*
  38  * Branch prediction prefixes
  39  */
  40
  41 #define LIKELY          .byte 0x3e
  42 #define UNLIKELY        .byte 0x2e
  43
  44 #define MP_SPIN_TRIES   1024
  45
  46         .text
  47         .align 4, 0x90
  48
  49 Lspin_lock_try_up:
  50         movl            4(%esp), %ecx
  51         xorl            %eax, %eax
  52         cmpxchgl        %ecx, (%ecx)
  53         setz            %dl
  54         movzbl          %dl, %eax
  55         ret
  56
  57         COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  58
  59         .align 4, 0x90
  60 Lspin_lock_try_mp:
  61         movl            4(%esp), %ecx
  62         xorl            %eax, %eax
  63         lock
  64         cmpxchgl        %ecx, (%ecx)
  65         setz            %dl
  66         movzbl          %dl, %eax
  67         ret
  68
  69         COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
  70
  71 .set Lrelinquish_off,   _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
  72
  73         .align 4, 0x90
  74 Lspin_lock_up:
  75         movl            4(%esp), %ecx
  76         xorl            %eax, %eax
  77 .set Lretry,            . - Lspin_lock_up
  78         cmpxchgl        %ecx, (%ecx)
  79         UNLIKELY
  80         JNZ             Lrelinquish_off - . + Lspin_lock_up - LEN
  81         ret
  82
  83         COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
  84
  85         .align 4, 0x90
  86 Lspin_lock_mp:
  87         movl            4(%esp), %ecx
  88         xorl            %eax, %eax
  89 0:
  90         lock
  91         cmpxchgl        %ecx, (%ecx)
  92         UNLIKELY
  93         jnz             1f
  94         ret
  95 1:
  96         xorl            %eax, %eax
  97         movl            $(MP_SPIN_TRIES), %edx
  98 2:
  99         pause
 100         cmpl            %eax, (%ecx)
 101         LIKELY
 102         jz              0b
 103         decl            %edx
 104         LIKELY
 105         jnz             2b
 106         JMP             Lrelinquish_off - . + Lspin_lock_mp - LEN
 107
 108         COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 109
 110         .align 4, 0x90
 111 Lspin_unlock:
 112         movl            4(%esp), %ecx
 113         movl            $0, (%ecx)
 114         ret
 115
 116         COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 117
 118         .align 4, 0x90
 119 Lrelinquish:                            /* relinquish the processor     */
 120         pushl           $1              /* 1 ms                         */
 121         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
 122         pushl           $0              /* THREAD_NULL                  */
 123         movl            $-61, %eax      /* syscall_thread_switch        */
 124         lcall           $7, $0
 125         popl            %eax            /* set %eax to 0 again          */
 126         popl            %edx            /* use %edx as scratch          */
 127         popl            %edx            /* reg to fixup stack           */
 128         JMP             Lretry - Lrelinquish_off - . + Lrelinquish - LEN
 129
 130         COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)