osfmk/i386/commpage/pthreads.s

   1 /*
   2  * Copyright (c) 2003-2009 Apple, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/appleapiopts.h>
  30 #include <machine/cpu_capabilities.h>
  31 #include <machine/commpage.h>
  32 #include <mach/i386/syscall_sw.h>
  33
  34 #define _PTHREAD_TSD_OFFSET32 0x48
  35 #define _PTHREAD_TSD_OFFSET64 0x60
  36
  37
  38 /* These routines do not need to be on the copmmpage on Intel.  They are for now
  39  * to avoid revlock, but the code should move to Libc, and we should eventually remove
  40  * these.
  41  */
  42 COMMPAGE_FUNCTION_START(pthread_getspecific, 32, 4)
  43         movl    4(%esp), %eax
  44         movl    %gs:_PTHREAD_TSD_OFFSET32(,%eax,4), %eax
  45         ret
  46 COMMPAGE_DESCRIPTOR(pthread_getspecific,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)
  47
  48 COMMPAGE_FUNCTION_START(pthread_self, 32, 4)
  49         movl    %gs:_PTHREAD_TSD_OFFSET32, %eax
  50         ret
  51 COMMPAGE_DESCRIPTOR(pthread_self,_COMM_PAGE_PTHREAD_SELF,0,0)
  52
  53 /* the 64-bit versions: */
  54 COMMPAGE_FUNCTION_START(pthread_getspecific_64, 64, 4)
  55         movq    %gs:_PTHREAD_TSD_OFFSET64(,%rdi,8), %rax
  56         ret
  57 COMMPAGE_DESCRIPTOR(pthread_getspecific_64,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)
  58
  59 COMMPAGE_FUNCTION_START(pthread_self_64, 64, 4)
  60         movq    %gs:_PTHREAD_TSD_OFFSET64, %rax
  61         ret
  62 COMMPAGE_DESCRIPTOR(pthread_self_64,_COMM_PAGE_PTHREAD_SELF,0,0)
  63
  64
  65 /* Temporary definitions.  Replace by #including the correct file when available.  */
  66
  67 #define PTHRW_EBIT      0x01
  68 #define PTHRW_LBIT      0x02
  69 #define PTHRW_YBIT      0x04
  70 #define PTHRW_WBIT      0x08
  71 #define PTHRW_UBIT      0x10
  72 #define PTHRW_RETRYBIT      0x20
  73 #define PTHRW_TRYLKBIT      0x40
  74
  75 #define PTHRW_INC       0x100
  76 #define PTHRW_BIT_MASK  0x000000ff;
  77
  78 #define PTHRW_COUNT_SHIFT       8
  79 #define PTHRW_COUNT_MASK        0xffffff00
  80 #define PTHRW_MAX_READERS       0xffffff00
  81
  82 #define KSYN_MLWAIT 301     /* mutex lock wait syscall */
  83
  84 #define PTHRW_STATUS_ACQUIRED   0
  85 #define PTHRW_STATUS_SYSCALL    1
  86 #define PTHRW_STATUS_ERROR      2
  87
  88 #define PTHRW_LVAL    0
  89 #define PTHRW_UVAL    4
  90
  91
  92
  93 /* PREEMPTION FREE ZONE (PFZ)
  94  *
  95  * A portion of the commpage is speacial-cased by the kernel to be "preemption free",
  96  * ie as if we had disabled interrupts in user mode.  This facilitates writing
  97  * "nearly-lockless" code, for example code that must be serialized by a spinlock but
  98  * which we do not want to preempt while the spinlock is held.
  99  *
 100  * The PFZ is implemented by collecting all the "preemption-free" code into a single
 101  * contiguous region of the commpage.  Register %ebx is used as a flag register;
 102  * before entering the PFZ, %ebx is cleared.  If some event occurs that would normally
 103  * result in a premption while in the PFZ, the kernel sets %ebx nonzero instead of
 104  * preempting.  Then, when the routine leaves the PFZ we check %ebx and
 105  * if nonzero execute a special "pfz_exit" syscall to take the delayed preemption.
 106  *
 107  * PFZ code must bound the amount of time spent in the PFZ, in order to control
 108  * latency.  Backward branches are dangerous and must not be used in a way that
 109  * could inadvertently create a long-running loop.
 110  *
 111  * Because we need to avoid being preempted between changing the mutex stateword
 112  * and entering the kernel to relinquish, some low-level pthread mutex manipulations
 113  * are located in the PFZ.
 114  */
 115
 116
 117 /* int                                                      // we return 0 on acquire, 1 on syscall
 118  * pthread_mutex_lock(  uint32_t    *lvalp,                 // ptr to mutex LVAL/UVAL pair
 119  *                      int         flags,                  // flags to pass kernel if we do syscall
 120  *                      uint64_t    mtid,                   // my Thread ID
 121  *                      uint32_t    mask,                   // bits to test in LVAL (ie, EBIT etc)
 122  *                      uint64_t    *tidp,                  // ptr to TID field of mutex
 123  *                      int         *syscall_return );      // if syscall, return value stored here
 124  */
 125 COMMPAGE_FUNCTION_START(pthread_mutex_lock, 32, 4)
 126         pushl   %ebp                        // set up frame for backtrace
 127         movl    %esp,%ebp
 128         pushl   %esi
 129         pushl   %edi
 130         pushl   %ebx
 131         xorl    %ebx,%ebx                   // clear "preemption pending" flag
 132         movl    20(%esp),%edi               // %edi == ptr to LVAL/UVAL structure
 133         lea     20(%esp),%esi               // %esi == ptr to argument list
 134         movl    _COMM_PAGE_SPIN_COUNT, %edx
 135         movl    16(%esi),%ecx               // get mask (ie, PTHRW_EBIT etc)
 136 1:
 137         testl   PTHRW_LVAL(%edi),%ecx       // is mutex available?
 138         jz      2f                          // yes, it is available
 139         pause
 140         decl    %edx                        // decrement max spin count
 141         jnz     1b                          // keep spinning
 142 2:
 143         COMMPAGE_CALL(_COMM_PAGE_PFZ_MUTEX_LOCK,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock)
 144         testl   %ebx,%ebx                   // pending preemption?
 145         jz      3f
 146         pushl   %eax                        // save return value across sysenter
 147         COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock)
 148         popl    %eax
 149 3:
 150         popl    %ebx
 151         popl    %edi
 152         popl    %esi
 153         popl    %ebp
 154         ret
 155 COMMPAGE_DESCRIPTOR(pthread_mutex_lock,_COMM_PAGE_MUTEX_LOCK,0,0)
 156
 157
 158 /* Internal routine to handle pthread mutex lock operation.  This is in the PFZ.
 159  *      %edi == ptr to LVAL/UVAL pair
 160  *      %esi == ptr to argument list on stack
 161  *      %ebx == preempion pending flag (kernel sets nonzero if we should preempt)
 162  */
 163 COMMPAGE_FUNCTION_START(pfz_mutex_lock, 32, 4)
 164         pushl   %ebp                        // set up frame for backtrace
 165         movl    %esp,%ebp
 166 1:
 167         movl    16(%esi),%ecx               // get mask (ie, PTHRW_EBIT etc)
 168 2:
 169         movl    PTHRW_LVAL(%edi),%eax       // get mutex LVAL
 170         testl   %eax,%ecx                   // is mutex available?
 171         jnz     5f                          // no
 172
 173         /* lock is available (if we act fast) */
 174         lea     PTHRW_INC(%eax),%edx        // copy original lval and bump sequence count
 175         orl     $PTHRW_EBIT, %edx           // set EBIT
 176         lock
 177         cmpxchgl %edx,PTHRW_LVAL(%edi)      // try to acquire lock for real
 178         jz      4f                          // got it
 179 3:
 180         testl   %ebx,%ebx                   // kernel trying to preempt us?
 181         jz      2b                          // no, so loop and try again
 182         COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock)
 183         jmp     1b                          // loop to try again
 184
 185         /* we acquired the mutex */
 186 4:
 187         movl    20(%esi),%eax               // get ptr to TID field of mutex
 188         movl    8(%esi),%ecx                // get 64-bit mtid
 189         movl    12(%esi),%edx
 190         movl    %ecx,0(%eax)                // store my TID in mutex structure
 191         movl    %edx,4(%eax)
 192         movl    $PTHRW_STATUS_ACQUIRED,%eax
 193         popl    %ebp
 194         ret
 195
 196         /* cannot acquire mutex, so update seq count, set "W", and block in kernel */
 197         /* this is where we cannot tolerate preemption or being killed */
 198 5:
 199         lea     PTHRW_INC(%eax),%edx        // copy original lval and bump sequence count
 200         orl     $PTHRW_WBIT, %edx           // set WBIT
 201         lock
 202         cmpxchgl %edx,PTHRW_LVAL(%edi)      // try to update lock status atomically
 203         jnz     3b                          // failed
 204         movl    20(%esi),%eax               // get ptr to TID field of mutex
 205         pushl   4(%esi)                     // arg 5: flags from arg list
 206         pushl   4(%eax)                     // arg 4: tid field from mutex
 207         pushl   0(%eax)
 208         pushl   PTHRW_UVAL(%edi)            // arg 3: uval field from mutex
 209         pushl   %edx                        // arg 2: new value of mutex lval field
 210         pushl   %edi                        // arg 1: ptr to LVAL/UVAL pair in mutex
 211         call    6f                          // make ksyn_mlwait call
 212         jc      6f                          // immediately reissue syscall if error
 213         movl    24(%esi),%edx               // get ptr to syscall_return arg
 214         movl    %eax,(%edx)                 // save syscall return value
 215         movl    $PTHRW_STATUS_SYSCALL,%eax  // we had to make syscall
 216         addl    $28,%esp                    // pop off syscall args and return address
 217         popl    %ebp                        // pop off frame ptr
 218         ret
 219
 220         /* subroutine to make a ksyn_mlwait syscall */
 221 6:
 222         movl    (%esp),%edx                 // get return address but leave on stack
 223         movl    %esp,%ecx                   // save stack ptr here
 224         movl    $KSYN_MLWAIT,%eax           // get syscall code
 225         orl     $0x00180000,%eax            // copy 24 bytes of arguments in trampoline
 226         xorl    %ebx,%ebx                   // clear preemption flag
 227         sysenter
 228 COMMPAGE_DESCRIPTOR(pfz_mutex_lock,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
 229
 230
 231
 232 /************************* x86_64 versions follow **************************/
 233
 234
 235
 236 /* int                                                      // we return 0 on acquire, 1 on syscall
 237  * pthread_mutex_lock(  uint32_t    *lvalp,                 // ptr to mutex LVAL/UVAL pair
 238  *                      int         flags,                  // flags to pass kernel if we do syscall
 239  *                      uint64_t    mtid,                   // my Thread ID
 240  *                      uint32_t    mask,                   // bits to test in LVAL (ie, EBIT etc)
 241  *                      uint64_t    *tidp,                  // ptr to TID field of mutex
 242  *                      int         *syscall_return );      // if syscall, return value stored here
 243  *
 244  *      %rdi = lvalp
 245  *      %esi = flags
 246  *      %rdx = mtid
 247  *      %ecx = mask
 248  *      %r8  = tidp
 249  *      %r9  = &syscall_return
 250  */
 251 COMMPAGE_FUNCTION_START(pthread_mutex_lock_64, 64, 4)
 252         pushq   %rbp                // set up frame for backtrace
 253         movq    %rsp,%rbp
 254         pushq   %rbx
 255         xorl    %ebx,%ebx           // clear "preemption pending" flag
 256         movl    _COMM_PAGE_32_TO_64(_COMM_PAGE_SPIN_COUNT), %eax
 257 1:
 258         testl   PTHRW_LVAL(%rdi),%ecx // is mutex available?
 259         jz      2f                  // yes, it is available
 260         pause
 261         decl    %eax                // decrement max spin count
 262         jnz     1b                  // keep spinning
 263 2:
 264         COMMPAGE_CALL(_COMM_PAGE_PFZ_MUTEX_LOCK,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock_64)
 265         testl   %ebx,%ebx           // pending preemption?
 266         jz      1f                  // no
 267         COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock_64)
 268 1:
 269         popq    %rbx
 270         popq    %rbp
 271         ret
 272 COMMPAGE_DESCRIPTOR(pthread_mutex_lock_64,_COMM_PAGE_MUTEX_LOCK,0,0)
 273
 274
 275 /* Internal routine to handle pthread mutex lock operation.  This is in the PFZ.
 276  *      %rdi = lvalp
 277  *      %esi = flags
 278  *      %rdx = mtid
 279  *      %ecx = mask
 280  *      %r8  = tidp
 281  *      %r9  = &syscall_return
 282  *      %ebx = preempion pending flag (kernel sets nonzero if we should preempt)
 283  */
 284 COMMPAGE_FUNCTION_START(pfz_mutex_lock_64, 64, 4)
 285         pushq   %rbp                        // set up frame for backtrace
 286         movq    %rsp,%rbp
 287 1:
 288         movl    PTHRW_LVAL(%rdi),%eax       // get old lval from mutex
 289 2:
 290         testl   %eax,%ecx                   // can we acquire the lock?
 291         jnz     5f                          // no
 292
 293         /* lock is available (if we act fast) */
 294         lea     PTHRW_INC(%rax),%r11        // copy original lval and bump sequence count
 295         orl     $PTHRW_EBIT, %r11d          // set EBIT
 296         lock
 297         cmpxchgl %r11d,PTHRW_LVAL(%rdi)     // try to acquire lock
 298         jz      4f                          // got it
 299 3:
 300         testl   %ebx,%ebx                   // kernel trying to preempt us?
 301         jz      2b                          // no, so loop and try again
 302         COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock_64)
 303         jmp     1b                          // loop to try again
 304
 305         /* we acquired the mutex */
 306 4:
 307         movq    %rdx,(%r8)                  // store mtid in mutex structure
 308         movl    $PTHRW_STATUS_ACQUIRED,%eax
 309         popq    %rbp
 310         ret
 311
 312         /* cannot acquire mutex, so update seq count and block in kernel */
 313         /* this is where we cannot tolerate preemption or being killed */
 314 5:
 315         lea     PTHRW_INC(%rax),%r11        // copy original lval and bump sequence count
 316         orl     $PTHRW_WBIT, %r11d          // set WBIT
 317         lock
 318         cmpxchgl %r11d,PTHRW_LVAL(%rdi)     // try to update lock status atomically
 319         jnz     3b                          // failed
 320         movq    (%r8),%r10                  // arg 4: tid field from mutex [NB: passed in R10]
 321         movl    %esi,%r8d                   // arg 5: flags from arg list
 322         movl    PTHRW_UVAL(%rdi),%edx       // arg 3: uval field from mutex
 323         movl    %r11d,%esi                  // arg 2: new value of mutex lval field
 324                                             // arg 1: LVAL/UVAL ptr already in %rdi
 325 6:
 326         movl    $(SYSCALL_CONSTRUCT_UNIX(KSYN_MLWAIT)),%eax
 327         pushq   %rdx                        // some syscalls destroy %rdx so save it
 328         xorl    %ebx,%ebx                   // clear preemption flag
 329         syscall
 330         popq    %rdx                        // restore in case we need to re-execute syscall
 331         jc      6b                          // immediately re-execute syscall if error
 332         movl    %eax,(%r9)                  // store kernel return value
 333         movl    $PTHRW_STATUS_SYSCALL,%eax  // we made syscall
 334         popq    %rbp
 335         ret
 336 COMMPAGE_DESCRIPTOR(pfz_mutex_lock_64,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
 337