osfmk/i386/commpage/pthreads.s

   1 /*
   2  * Copyright (c) 2003-2009 Apple, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/appleapiopts.h>
  30 #include <machine/cpu_capabilities.h>
  31 #include <machine/commpage.h>
  32 #include <mach/i386/syscall_sw.h>
  33
  34 /* Temporary definitions.  Replace by #including the correct file when available.  */
  35
  36 #define PTHRW_EBIT      0x01
  37 #define PTHRW_LBIT      0x02
  38 #define PTHRW_YBIT      0x04
  39 #define PTHRW_WBIT      0x08
  40 #define PTHRW_UBIT      0x10
  41 #define PTHRW_RETRYBIT      0x20
  42 #define PTHRW_TRYLKBIT      0x40
  43
  44 #define PTHRW_INC       0x100
  45 #define PTHRW_BIT_MASK  0x000000ff;
  46
  47 #define PTHRW_COUNT_SHIFT       8
  48 #define PTHRW_COUNT_MASK        0xffffff00
  49 #define PTHRW_MAX_READERS       0xffffff00
  50
  51 #define KSYN_MLWAIT 301     /* mutex lock wait syscall */
  52
  53 #define PTHRW_STATUS_ACQUIRED   0
  54 #define PTHRW_STATUS_SYSCALL    1
  55 #define PTHRW_STATUS_ERROR      2
  56
  57 #define PTHRW_LVAL    0
  58 #define PTHRW_UVAL    4
  59
  60
  61
  62 /* PREEMPTION FREE ZONE (PFZ)
  63  *
  64  * A portion of the commpage is speacial-cased by the kernel to be "preemption free",
  65  * ie as if we had disabled interrupts in user mode.  This facilitates writing
  66  * "nearly-lockless" code, for example code that must be serialized by a spinlock but
  67  * which we do not want to preempt while the spinlock is held.
  68  *
  69  * The PFZ is implemented by collecting all the "preemption-free" code into a single
  70  * contiguous region of the commpage.  Register %ebx is used as a flag register;
  71  * before entering the PFZ, %ebx is cleared.  If some event occurs that would normally
  72  * result in a premption while in the PFZ, the kernel sets %ebx nonzero instead of
  73  * preempting.  Then, when the routine leaves the PFZ we check %ebx and
  74  * if nonzero execute a special "pfz_exit" syscall to take the delayed preemption.
  75  *
  76  * PFZ code must bound the amount of time spent in the PFZ, in order to control
  77  * latency.  Backward branches are dangerous and must not be used in a way that
  78  * could inadvertently create a long-running loop.
  79  *
  80  * Because we need to avoid being preempted between changing the mutex stateword
  81  * and entering the kernel to relinquish, some low-level pthread mutex manipulations
  82  * are located in the PFZ.
  83  */
  84
  85 /* Work around 10062261 with a dummy non-local symbol */
  86 pthreads_dummy_symbol:
  87
  88 /* Internal routine to handle pthread mutex lock operation.  This is in the PFZ.
  89  *      %edi == ptr to LVAL/UVAL pair
  90  *      %esi == ptr to argument list on stack
  91  *      %ebx == preempion pending flag (kernel sets nonzero if we should preempt)
  92  */
  93 COMMPAGE_FUNCTION_START(pfz_mutex_lock, 32, 4)
  94         pushl   %ebp                        // set up frame for backtrace
  95         movl    %esp,%ebp
  96 1:
  97         movl    16(%esi),%ecx               // get mask (ie, PTHRW_EBIT etc)
  98 2:
  99         movl    PTHRW_LVAL(%edi),%eax       // get mutex LVAL
 100         testl   %eax,%ecx                   // is mutex available?
 101         jnz     5f                          // no
 102
 103         /* lock is available (if we act fast) */
 104         lea     PTHRW_INC(%eax),%edx        // copy original lval and bump sequence count
 105         orl     $PTHRW_EBIT, %edx           // set EBIT
 106         lock
 107         cmpxchgl %edx,PTHRW_LVAL(%edi)      // try to acquire lock for real
 108         jz      4f                          // got it
 109 3:
 110         testl   %ebx,%ebx                   // kernel trying to preempt us?
 111         jz      2b                          // no, so loop and try again
 112         COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock)
 113         jmp     1b                          // loop to try again
 114
 115         /* we acquired the mutex */
 116 4:
 117         movl    20(%esi),%eax               // get ptr to TID field of mutex
 118         movl    8(%esi),%ecx                // get 64-bit mtid
 119         movl    12(%esi),%edx
 120         movl    %ecx,0(%eax)                // store my TID in mutex structure
 121         movl    %edx,4(%eax)
 122         movl    $PTHRW_STATUS_ACQUIRED,%eax
 123         popl    %ebp
 124         ret
 125
 126         /* cannot acquire mutex, so update seq count, set "W", and block in kernel */
 127         /* this is where we cannot tolerate preemption or being killed */
 128 5:
 129         lea     PTHRW_INC(%eax),%edx        // copy original lval and bump sequence count
 130         orl     $PTHRW_WBIT, %edx           // set WBIT
 131         lock
 132         cmpxchgl %edx,PTHRW_LVAL(%edi)      // try to update lock status atomically
 133         jnz     3b                          // failed
 134         movl    20(%esi),%eax               // get ptr to TID field of mutex
 135         pushl   4(%esi)                     // arg 5: flags from arg list
 136         pushl   4(%eax)                     // arg 4: tid field from mutex
 137         pushl   0(%eax)
 138         pushl   PTHRW_UVAL(%edi)            // arg 3: uval field from mutex
 139         pushl   %edx                        // arg 2: new value of mutex lval field
 140         pushl   %edi                        // arg 1: ptr to LVAL/UVAL pair in mutex
 141         call    6f                          // make ksyn_mlwait call
 142         jc      6f                          // immediately reissue syscall if error
 143         movl    24(%esi),%edx               // get ptr to syscall_return arg
 144         movl    %eax,(%edx)                 // save syscall return value
 145         movl    $PTHRW_STATUS_SYSCALL,%eax  // we had to make syscall
 146         addl    $28,%esp                    // pop off syscall args and return address
 147         popl    %ebp                        // pop off frame ptr
 148         ret
 149
 150         /* subroutine to make a ksyn_mlwait syscall */
 151 6:
 152         movl    (%esp),%edx                 // get return address but leave on stack
 153         movl    %esp,%ecx                   // save stack ptr here
 154         movl    $KSYN_MLWAIT,%eax           // get syscall code
 155         orl     $0x00180000,%eax            // copy 24 bytes of arguments in trampoline
 156         xorl    %ebx,%ebx                   // clear preemption flag
 157         sysenter
 158 COMMPAGE_DESCRIPTOR(pfz_mutex_lock,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
 159
 160
 161
 162 /************************* x86_64 versions follow **************************/
 163
 164
 165
 166 /* Internal routine to handle pthread mutex lock operation.  This is in the PFZ.
 167  *      %rdi = lvalp
 168  *      %esi = flags
 169  *      %rdx = mtid
 170  *      %ecx = mask
 171  *      %r8  = tidp
 172  *      %r9  = &syscall_return
 173  *      %ebx = preempion pending flag (kernel sets nonzero if we should preempt)
 174  */
 175 COMMPAGE_FUNCTION_START(pfz_mutex_lock_64, 64, 4)
 176         pushq   %rbp                        // set up frame for backtrace
 177         movq    %rsp,%rbp
 178 1:
 179         movl    PTHRW_LVAL(%rdi),%eax       // get old lval from mutex
 180 2:
 181         testl   %eax,%ecx                   // can we acquire the lock?
 182         jnz     5f                          // no
 183
 184         /* lock is available (if we act fast) */
 185         lea     PTHRW_INC(%rax),%r11        // copy original lval and bump sequence count
 186         orl     $PTHRW_EBIT, %r11d          // set EBIT
 187         lock
 188         cmpxchgl %r11d,PTHRW_LVAL(%rdi)     // try to acquire lock
 189         jz      4f                          // got it
 190 3:
 191         testl   %ebx,%ebx                   // kernel trying to preempt us?
 192         jz      2b                          // no, so loop and try again
 193         COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock_64)
 194         jmp     1b                          // loop to try again
 195
 196         /* we acquired the mutex */
 197 4:
 198         movq    %rdx,(%r8)                  // store mtid in mutex structure
 199         movl    $PTHRW_STATUS_ACQUIRED,%eax
 200         popq    %rbp
 201         ret
 202
 203         /* cannot acquire mutex, so update seq count and block in kernel */
 204         /* this is where we cannot tolerate preemption or being killed */
 205 5:
 206         lea     PTHRW_INC(%rax),%r11        // copy original lval and bump sequence count
 207         orl     $PTHRW_WBIT, %r11d          // set WBIT
 208         lock
 209         cmpxchgl %r11d,PTHRW_LVAL(%rdi)     // try to update lock status atomically
 210         jnz     3b                          // failed
 211         movq    (%r8),%r10                  // arg 4: tid field from mutex [NB: passed in R10]
 212         movl    %esi,%r8d                   // arg 5: flags from arg list
 213         movl    PTHRW_UVAL(%rdi),%edx       // arg 3: uval field from mutex
 214         movl    %r11d,%esi                  // arg 2: new value of mutex lval field
 215                                             // arg 1: LVAL/UVAL ptr already in %rdi
 216 6:
 217         movl    $(SYSCALL_CONSTRUCT_UNIX(KSYN_MLWAIT)),%eax
 218         pushq   %rdx                        // some syscalls destroy %rdx so save it
 219         xorl    %ebx,%ebx                   // clear preemption flag
 220         syscall
 221         popq    %rdx                        // restore in case we need to re-execute syscall
 222         jc      6b                          // immediately re-execute syscall if error
 223         movl    %eax,(%r9)                  // store kernel return value
 224         movl    $PTHRW_STATUS_SYSCALL,%eax  // we made syscall
 225         popq    %rbp
 226         ret
 227 COMMPAGE_DESCRIPTOR(pfz_mutex_lock_64,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
 228