osfmk/i386/commpage/spinlocks.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25
  26 #include <sys/appleapiopts.h>
  27 #include <machine/cpu_capabilities.h>
  28 #include <machine/commpage.h>
  29
  30 /*
  31  * We need a relative branch within the comm page, and don't want the linker
  32  * to relocate it, so we have to hand-code the instructions. LEN is to account
  33  * for the length of a .long, since the jmp is relative to the next instruction.
  34  */
  35
  36 #define JNZ .byte 0x0f, 0x85; .long
  37 #define JMP .byte 0xe9; .long
  38 #define LEN 4
  39
  40 /*
  41  * Branch prediction prefixes
  42  */
  43
  44 #define LIKELY          .byte 0x3e
  45 #define UNLIKELY        .byte 0x2e
  46
  47 #define MP_SPIN_TRIES   1024
  48
  49         .text
  50         .align 4, 0x90
  51
  52 Lspin_lock_try_up:
  53         movl            4(%esp), %ecx
  54         xorl            %eax, %eax
  55         cmpxchgl        %ecx, (%ecx)
  56         setz            %dl
  57         movzbl          %dl, %eax
  58         ret
  59
  60         COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  61
  62         .align 4, 0x90
  63 Lspin_lock_try_mp:
  64         movl            4(%esp), %ecx
  65         xorl            %eax, %eax
  66         lock
  67         cmpxchgl        %ecx, (%ecx)
  68         setz            %dl
  69         movzbl          %dl, %eax
  70         ret
  71
  72         COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
  73
  74 .set Lrelinquish_off,   _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
  75
  76         .align 4, 0x90
  77 Lspin_lock_up:
  78         movl            4(%esp), %ecx
  79         xorl            %eax, %eax
  80 .set Lretry,            . - Lspin_lock_up
  81         cmpxchgl        %ecx, (%ecx)
  82         UNLIKELY
  83         JNZ             Lrelinquish_off - . + Lspin_lock_up - LEN
  84         ret
  85
  86         COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
  87
  88         .align 4, 0x90
  89 Lspin_lock_mp:
  90         movl            4(%esp), %ecx
  91         xorl            %eax, %eax
  92 0:
  93         lock
  94         cmpxchgl        %ecx, (%ecx)
  95         UNLIKELY
  96         jnz             1f
  97         ret
  98 1:
  99         xorl            %eax, %eax
 100         movl            $(MP_SPIN_TRIES), %edx
 101 2:
 102         pause
 103         cmpl            %eax, (%ecx)
 104         LIKELY
 105         jz              0b
 106         decl            %edx
 107         LIKELY
 108         jnz             2b
 109         JMP             Lrelinquish_off - . + Lspin_lock_mp - LEN
 110
 111         COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 112
 113         .align 4, 0x90
 114 Lspin_unlock:
 115         movl            4(%esp), %ecx
 116         movl            $0, (%ecx)
 117         ret
 118
 119         COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 120
 121         .align 4, 0x90
 122 Lrelinquish:                            /* relinquish the processor     */
 123         pushl           $1              /* 1 ms                         */
 124         pushl           $1              /* SWITCH_OPTION_DEPRESS        */
 125         pushl           $0              /* THREAD_NULL                  */
 126         movl            $-61, %eax      /* syscall_thread_switch        */
 127         lcall           $7, $0
 128         popl            %eax            /* set %eax to 0 again          */
 129         popl            %edx            /* use %edx as scratch          */
 130         popl            %edx            /* reg to fixup stack           */
 131         JMP             Lretry - Lrelinquish_off - . + Lrelinquish - LEN
 132
 133         COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)