[apple/xnu.git] / osfmk / i386 / commpage / pthreads.s

/*
 * Copyright (c) 2003-2009 Apple, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
#include <mach/i386/syscall_sw.h>

#define _PTHREAD_TSD_OFFSET32 0x48
#define _PTHREAD_TSD_OFFSET64 0x60


/* These routines do not need to be on the copmmpage on Intel.  They are for now
 * to avoid revlock, but the code should move to Libc, and we should eventually remove
 * these.
 */
COMMPAGE_FUNCTION_START(pthread_getspecific, 32, 4)
	movl	4(%esp), %eax
	movl	%gs:_PTHREAD_TSD_OFFSET32(,%eax,4), %eax
	ret
COMMPAGE_DESCRIPTOR(pthread_getspecific,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)

COMMPAGE_FUNCTION_START(pthread_self, 32, 4)
	movl	%gs:_PTHREAD_TSD_OFFSET32, %eax
	ret
COMMPAGE_DESCRIPTOR(pthread_self,_COMM_PAGE_PTHREAD_SELF,0,0)

/* the 64-bit versions: */
COMMPAGE_FUNCTION_START(pthread_getspecific_64, 64, 4)
	movq	%gs:_PTHREAD_TSD_OFFSET64(,%rdi,8), %rax
	ret
COMMPAGE_DESCRIPTOR(pthread_getspecific_64,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)

COMMPAGE_FUNCTION_START(pthread_self_64, 64, 4)
	movq	%gs:_PTHREAD_TSD_OFFSET64, %rax
	ret
COMMPAGE_DESCRIPTOR(pthread_self_64,_COMM_PAGE_PTHREAD_SELF,0,0)


/* Temporary definitions.  Replace by #including the correct file when available.  */

#define PTHRW_EBIT      0x01
#define PTHRW_LBIT      0x02
#define PTHRW_YBIT      0x04
#define PTHRW_WBIT      0x08
#define PTHRW_UBIT      0x10
#define PTHRW_RETRYBIT      0x20
#define PTHRW_TRYLKBIT      0x40

#define PTHRW_INC       0x100
#define PTHRW_BIT_MASK  0x000000ff;

#define PTHRW_COUNT_SHIFT       8
#define PTHRW_COUNT_MASK        0xffffff00
#define PTHRW_MAX_READERS       0xffffff00

#define	KSYN_MLWAIT 301	    /* mutex lock wait syscall */

#define	PTHRW_STATUS_ACQUIRED	0
#define	PTHRW_STATUS_SYSCALL	1
#define	PTHRW_STATUS_ERROR	2
 
#define	PTHRW_LVAL    0
#define	PTHRW_UVAL    4


/* PREEMPTION FREE ZONE (PFZ)
 *
 * A portion of the commpage is speacial-cased by the kernel to be "preemption free",
 * ie as if we had disabled interrupts in user mode.  This facilitates writing
 * "nearly-lockless" code, for example code that must be serialized by a spinlock but
 * which we do not want to preempt while the spinlock is held.
 *
 * The PFZ is implemented by collecting all the "preemption-free" code into a single
 * contiguous region of the commpage.  Register %ebx is used as a flag register;
 * before entering the PFZ, %ebx is cleared.  If some event occurs that would normally
 * result in a premption while in the PFZ, the kernel sets %ebx nonzero instead of
 * preempting.  Then, when the routine leaves the PFZ we check %ebx and
 * if nonzero execute a special "pfz_exit" syscall to take the delayed preemption.
 *
 * PFZ code must bound the amount of time spent in the PFZ, in order to control
 * latency.  Backward branches are dangerous and must not be used in a way that
 * could inadvertently create a long-running loop.
 *
 * Because we need to avoid being preempted between changing the mutex stateword
 * and entering the kernel to relinquish, some low-level pthread mutex manipulations
 * are located in the PFZ.
 */


/* int							    // we return 0 on acquire, 1 on syscall
 * pthread_mutex_lock(	uint32_t    *lvalp,		    // ptr to mutex LVAL/UVAL pair
 *			int	    flags,		    // flags to pass kernel if we do syscall
 *			uint64_t    mtid,		    // my Thread ID
 *			uint32_t    mask,		    // bits to test in LVAL (ie, EBIT etc)
 *			uint64_t    *tidp,		    // ptr to TID field of mutex
 *			int	    *syscall_return );	    // if syscall, return value stored here
 */
COMMPAGE_FUNCTION_START(pthread_mutex_lock, 32, 4)
	pushl	%ebp			    // set up frame for backtrace
	movl	%esp,%ebp
	pushl	%esi
	pushl	%edi
	pushl	%ebx
	xorl	%ebx,%ebx		    // clear "preemption pending" flag
	movl	20(%esp),%edi		    // %edi == ptr to LVAL/UVAL structure
	lea	20(%esp),%esi		    // %esi == ptr to argument list
	movl	_COMM_PAGE_SPIN_COUNT, %edx
	movl	16(%esi),%ecx		    // get mask (ie, PTHRW_EBIT etc)
1:
	testl	PTHRW_LVAL(%edi),%ecx	    // is mutex available?
	jz	2f			    // yes, it is available
	pause
	decl	%edx			    // decrement max spin count
	jnz	1b			    // keep spinning
2:
	COMMPAGE_CALL(_COMM_PAGE_PFZ_MUTEX_LOCK,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock)
	testl	%ebx,%ebx		    // pending preemption?
	jz	3f
	pushl	%eax			    // save return value across sysenter
	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock)
	popl	%eax
3:
	popl	%ebx
	popl	%edi
	popl	%esi
	popl	%ebp
	ret
COMMPAGE_DESCRIPTOR(pthread_mutex_lock,_COMM_PAGE_MUTEX_LOCK,0,0)


/* Internal routine to handle pthread mutex lock operation.  This is in the PFZ.
 *	%edi == ptr to LVAL/UVAL pair
 *	%esi == ptr to argument list on stack
 *	%ebx == preempion pending flag (kernel sets nonzero if we should preempt)
 */
COMMPAGE_FUNCTION_START(pfz_mutex_lock, 32, 4)
	pushl	%ebp			    // set up frame for backtrace
	movl	%esp,%ebp
1:	
	movl	16(%esi),%ecx		    // get mask (ie, PTHRW_EBIT etc)
2:
	movl	PTHRW_LVAL(%edi),%eax	    // get mutex LVAL
	testl	%eax,%ecx		    // is mutex available?
	jnz	5f			    // no
	
	/* lock is available (if we act fast) */
	lea	PTHRW_INC(%eax),%edx	    // copy original lval and bump sequence count
	orl	$PTHRW_EBIT, %edx	    // set EBIT
	lock
	cmpxchgl %edx,PTHRW_LVAL(%edi)	    // try to acquire lock for real
	jz	4f			    // got it
3:
	testl	%ebx,%ebx		    // kernel trying to preempt us?
	jz	2b			    // no, so loop and try again
	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock)
	jmp	1b			    // loop to try again
	
	/* we acquired the mutex */
4:
	movl	20(%esi),%eax		    // get ptr to TID field of mutex
	movl	8(%esi),%ecx		    // get 64-bit mtid
	movl	12(%esi),%edx
	movl	%ecx,0(%eax)		    // store my TID in mutex structure
	movl	%edx,4(%eax)
	movl	$PTHRW_STATUS_ACQUIRED,%eax
	popl	%ebp
	ret
	
	/* cannot acquire mutex, so update seq count, set "W", and block in kernel */
	/* this is where we cannot tolerate preemption or being killed */
5:
	lea	PTHRW_INC(%eax),%edx	    // copy original lval and bump sequence count
	orl	$PTHRW_WBIT, %edx	    // set WBIT
	lock
	cmpxchgl %edx,PTHRW_LVAL(%edi)	    // try to update lock status atomically
	jnz	3b			    // failed
	movl	20(%esi),%eax		    // get ptr to TID field of mutex
	pushl	4(%esi)			    // arg 5: flags from arg list
	pushl	4(%eax)			    // arg 4: tid field from mutex
	pushl	0(%eax)
	pushl	PTHRW_UVAL(%edi)	    // arg 3: uval field from mutex
	pushl	%edx			    // arg 2: new value of mutex lval field
	pushl	%edi			    // arg 1: ptr to LVAL/UVAL pair in mutex
	call	6f			    // make ksyn_mlwait call
	jc	6f			    // immediately reissue syscall if error
	movl	24(%esi),%edx		    // get ptr to syscall_return arg
	movl	%eax,(%edx)		    // save syscall return value
	movl	$PTHRW_STATUS_SYSCALL,%eax  // we had to make syscall
	addl	$28,%esp		    // pop off syscall args and return address
	popl	%ebp			    // pop off frame ptr
	ret

	/* subroutine to make a ksyn_mlwait syscall */
6:
	movl	(%esp),%edx		    // get return address but leave on stack
	movl	%esp,%ecx		    // save stack ptr here
	movl	$KSYN_MLWAIT,%eax	    // get syscall code
	orl	$0x00180000,%eax	    // copy 24 bytes of arguments in trampoline
	xorl	%ebx,%ebx		    // clear preemption flag
	sysenter
COMMPAGE_DESCRIPTOR(pfz_mutex_lock,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)


/************************* x86_64 versions follow **************************/


/* int							    // we return 0 on acquire, 1 on syscall
 * pthread_mutex_lock(	uint32_t    *lvalp,		    // ptr to mutex LVAL/UVAL pair
 *			int	    flags,		    // flags to pass kernel if we do syscall
 *			uint64_t    mtid,		    // my Thread ID
 *			uint32_t    mask,		    // bits to test in LVAL (ie, EBIT etc)
 *			uint64_t    *tidp,		    // ptr to TID field of mutex
 *			int	    *syscall_return );	    // if syscall, return value stored here
 *
 *	%rdi = lvalp
 *	%esi = flags
 *	%rdx = mtid
 *	%ecx = mask
 *	%r8  = tidp
 *	%r9  = &syscall_return
 */
COMMPAGE_FUNCTION_START(pthread_mutex_lock_64, 64, 4)
	pushq	%rbp		    // set up frame for backtrace
	movq	%rsp,%rbp
	pushq	%rbx
	xorl	%ebx,%ebx	    // clear "preemption pending" flag
	movl	_COMM_PAGE_32_TO_64(_COMM_PAGE_SPIN_COUNT), %eax
1:
	testl	PTHRW_LVAL(%rdi),%ecx // is mutex available?
	jz	2f		    // yes, it is available
	pause
	decl	%eax		    // decrement max spin count
	jnz	1b		    // keep spinning
2:
	COMMPAGE_CALL(_COMM_PAGE_PFZ_MUTEX_LOCK,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock_64)
	testl	%ebx,%ebx	    // pending preemption?
	jz	1f		    // no
	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock_64)
1:
	popq	%rbx
	popq	%rbp
	ret
COMMPAGE_DESCRIPTOR(pthread_mutex_lock_64,_COMM_PAGE_MUTEX_LOCK,0,0)


/* Internal routine to handle pthread mutex lock operation.  This is in the PFZ.
 *	%rdi = lvalp
 *	%esi = flags
 *	%rdx = mtid
 *	%ecx = mask
 *	%r8  = tidp
 *	%r9  = &syscall_return
 *	%ebx = preempion pending flag (kernel sets nonzero if we should preempt)
 */
COMMPAGE_FUNCTION_START(pfz_mutex_lock_64, 64, 4)
	pushq	%rbp			    // set up frame for backtrace
	movq	%rsp,%rbp
1:	
	movl	PTHRW_LVAL(%rdi),%eax	    // get old lval from mutex
2:
	testl	%eax,%ecx		    // can we acquire the lock?
	jnz	5f			    // no
	
	/* lock is available (if we act fast) */
	lea	PTHRW_INC(%rax),%r11	    // copy original lval and bump sequence count
	orl	$PTHRW_EBIT, %r11d	    // set EBIT
	lock
	cmpxchgl %r11d,PTHRW_LVAL(%rdi)	    // try to acquire lock
	jz	4f			    // got it
3:
	testl	%ebx,%ebx		    // kernel trying to preempt us?
	jz	2b			    // no, so loop and try again
	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock_64)
	jmp	1b			    // loop to try again
	
	/* we acquired the mutex */
4:
	movq	%rdx,(%r8)		    // store mtid in mutex structure
	movl	$PTHRW_STATUS_ACQUIRED,%eax
	popq	%rbp
	ret
	
	/* cannot acquire mutex, so update seq count and block in kernel */
	/* this is where we cannot tolerate preemption or being killed */
5:
	lea	PTHRW_INC(%rax),%r11	    // copy original lval and bump sequence count
	orl	$PTHRW_WBIT, %r11d	    // set WBIT
	lock
	cmpxchgl %r11d,PTHRW_LVAL(%rdi)	    // try to update lock status atomically
	jnz	3b			    // failed
	movq	(%r8),%r10		    // arg 4: tid field from mutex [NB: passed in R10]
	movl	%esi,%r8d		    // arg 5: flags from arg list
	movl	PTHRW_UVAL(%rdi),%edx	    // arg 3: uval field from mutex
	movl	%r11d,%esi		    // arg 2: new value of mutex lval field
					    // arg 1: LVAL/UVAL ptr already in %rdi
6:
	movl	$(SYSCALL_CONSTRUCT_UNIX(KSYN_MLWAIT)),%eax
	pushq	%rdx			    // some syscalls destroy %rdx so save it
	xorl	%ebx,%ebx		    // clear preemption flag
	syscall
	popq	%rdx			    // restore in case we need to re-execute syscall
	jc	6b			    // immediately re-execute syscall if error
	movl	%eax,(%r9)		    // store kernel return value
	movl	$PTHRW_STATUS_SYSCALL,%eax  // we made syscall
	popq	%rbp
	ret
COMMPAGE_DESCRIPTOR(pfz_mutex_lock_64,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
Commit	Line	Data
1c79356b	1	/*
b0d623f7	2	* Copyright (c) 2003-2009 Apple, Inc. All rights reserved.
1c79356b	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b	27	*/
1c79356b	28
55e303ae A	29	#include <sys/appleapiopts.h>
	30	#include <machine/cpu_capabilities.h>
	31	#include <machine/commpage.h>
b0d623f7	32	#include <mach/i386/syscall_sw.h>
1c79356b	33
0c530ab8 A	34	#define _PTHREAD_TSD_OFFSET32 0x48
0c530ab8 A	35	#define _PTHREAD_TSD_OFFSET64 0x60
4452a7af	36
0c530ab8 A	37
	38	/* These routines do not need to be on the copmmpage on Intel. They are for now
	39	* to avoid revlock, but the code should move to Libc, and we should eventually remove
	40	* these.
	41	*/
b0d623f7	42	COMMPAGE_FUNCTION_START(pthread_getspecific, 32, 4)
55e303ae	43	movl 4(%esp), %eax
0c530ab8	44	movl %gs:_PTHREAD_TSD_OFFSET32(,%eax,4), %eax
55e303ae	45	ret
b0d623f7	46	COMMPAGE_DESCRIPTOR(pthread_getspecific,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)
55e303ae	47
b0d623f7	48	COMMPAGE_FUNCTION_START(pthread_self, 32, 4)
0c530ab8	49	movl %gs:_PTHREAD_TSD_OFFSET32, %eax
55e303ae	50	ret
b0d623f7	51	COMMPAGE_DESCRIPTOR(pthread_self,_COMM_PAGE_PTHREAD_SELF,0,0)
0c530ab8 A	52
0c530ab8 A	53	/* the 64-bit versions: */
b0d623f7	54	COMMPAGE_FUNCTION_START(pthread_getspecific_64, 64, 4)
0c530ab8 A	55	movq %gs:_PTHREAD_TSD_OFFSET64(,%rdi,8), %rax
0c530ab8 A	56	ret
b0d623f7	57	COMMPAGE_DESCRIPTOR(pthread_getspecific_64,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)
0c530ab8	58
b0d623f7	59	COMMPAGE_FUNCTION_START(pthread_self_64, 64, 4)
0c530ab8 A	60	movq %gs:_PTHREAD_TSD_OFFSET64, %rax
0c530ab8 A	61	ret
b0d623f7 A	62	COMMPAGE_DESCRIPTOR(pthread_self_64,_COMM_PAGE_PTHREAD_SELF,0,0)
	63
	64
	65	/* Temporary definitions. Replace by #including the correct file when available. */
	66
	67	#define PTHRW_EBIT 0x01
	68	#define PTHRW_LBIT 0x02
	69	#define PTHRW_YBIT 0x04
	70	#define PTHRW_WBIT 0x08
	71	#define PTHRW_UBIT 0x10
	72	#define PTHRW_RETRYBIT 0x20
	73	#define PTHRW_TRYLKBIT 0x40
	74
	75	#define PTHRW_INC 0x100
	76	#define PTHRW_BIT_MASK 0x000000ff;
	77
	78	#define PTHRW_COUNT_SHIFT 8
	79	#define PTHRW_COUNT_MASK 0xffffff00
	80	#define PTHRW_MAX_READERS 0xffffff00
	81
	82	#define KSYN_MLWAIT 301 /* mutex lock wait syscall */
	83
	84	#define PTHRW_STATUS_ACQUIRED 0
	85	#define PTHRW_STATUS_SYSCALL 1
	86	#define PTHRW_STATUS_ERROR 2
	87
	88	#define PTHRW_LVAL 0
	89	#define PTHRW_UVAL 4
	90
	91
	92
	93	/* PREEMPTION FREE ZONE (PFZ)
	94	*
	95	* A portion of the commpage is speacial-cased by the kernel to be "preemption free",
	96	* ie as if we had disabled interrupts in user mode. This facilitates writing
	97	* "nearly-lockless" code, for example code that must be serialized by a spinlock but
	98	* which we do not want to preempt while the spinlock is held.
	99	*
	100	* The PFZ is implemented by collecting all the "preemption-free" code into a single
	101	* contiguous region of the commpage. Register %ebx is used as a flag register;
	102	* before entering the PFZ, %ebx is cleared. If some event occurs that would normally
	103	* result in a premption while in the PFZ, the kernel sets %ebx nonzero instead of
	104	* preempting. Then, when the routine leaves the PFZ we check %ebx and
	105	* if nonzero execute a special "pfz_exit" syscall to take the delayed preemption.
	106	*
	107	* PFZ code must bound the amount of time spent in the PFZ, in order to control
	108	* latency. Backward branches are dangerous and must not be used in a way that
	109	* could inadvertently create a long-running loop.
	110	*
	111	* Because we need to avoid being preempted between changing the mutex stateword
	112	* and entering the kernel to relinquish, some low-level pthread mutex manipulations
	113	* are located in the PFZ.
	114	*/
	115
	116
	117	/* int // we return 0 on acquire, 1 on syscall
	118	* pthread_mutex_lock( uint32_t *lvalp, // ptr to mutex LVAL/UVAL pair
	119	* int flags, // flags to pass kernel if we do syscall
	120	* uint64_t mtid, // my Thread ID
	121	* uint32_t mask, // bits to test in LVAL (ie, EBIT etc)
	122	* uint64_t *tidp, // ptr to TID field of mutex
	123	* int *syscall_return ); // if syscall, return value stored here
	124	*/
	125	COMMPAGE_FUNCTION_START(pthread_mutex_lock, 32, 4)
126	pushl %ebp // set up frame for backtrace
127	movl %esp,%ebp
128	pushl %esi
129	pushl %edi
130	pushl %ebx
131	xorl %ebx,%ebx // clear "preemption pending" flag
132	movl 20(%esp),%edi // %edi == ptr to LVAL/UVAL structure
133	lea 20(%esp),%esi // %esi == ptr to argument list
134	movl _COMM_PAGE_SPIN_COUNT, %edx
135	movl 16(%esi),%ecx // get mask (ie, PTHRW_EBIT etc)
136	1:
137	testl PTHRW_LVAL(%edi),%ecx // is mutex available?
138	jz 2f // yes, it is available
139	pause
140	decl %edx // decrement max spin count
141	jnz 1b // keep spinning
142	2:
143	COMMPAGE_CALL(_COMM_PAGE_PFZ_MUTEX_LOCK,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock)
144	testl %ebx,%ebx // pending preemption?
145	jz 3f
146	pushl %eax // save return value across sysenter
147	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock)
148	popl %eax
149	3:
150	popl %ebx
151	popl %edi
152	popl %esi
153	popl %ebp
154	ret
155	COMMPAGE_DESCRIPTOR(pthread_mutex_lock,_COMM_PAGE_MUTEX_LOCK,0,0)
156
157
158	/* Internal routine to handle pthread mutex lock operation. This is in the PFZ.
159	* %edi == ptr to LVAL/UVAL pair
160	* %esi == ptr to argument list on stack
161	* %ebx == preempion pending flag (kernel sets nonzero if we should preempt)
162	*/
163	COMMPAGE_FUNCTION_START(pfz_mutex_lock, 32, 4)
164	pushl %ebp // set up frame for backtrace
165	movl %esp,%ebp
166	1:
167	movl 16(%esi),%ecx // get mask (ie, PTHRW_EBIT etc)
168	2:
169	movl PTHRW_LVAL(%edi),%eax // get mutex LVAL
170	testl %eax,%ecx // is mutex available?
171	jnz 5f // no
172
173	/* lock is available (if we act fast) */
174	lea PTHRW_INC(%eax),%edx // copy original lval and bump sequence count
175	orl $PTHRW_EBIT, %edx // set EBIT
176	lock
177	cmpxchgl %edx,PTHRW_LVAL(%edi) // try to acquire lock for real
178	jz 4f // got it
179	3:
180	testl %ebx,%ebx // kernel trying to preempt us?
181	jz 2b // no, so loop and try again
182	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock)
183	jmp 1b // loop to try again
184
185	/* we acquired the mutex */
186	4:
187	movl 20(%esi),%eax // get ptr to TID field of mutex
188	movl 8(%esi),%ecx // get 64-bit mtid
189	movl 12(%esi),%edx
190	movl %ecx,0(%eax) // store my TID in mutex structure
191	movl %edx,4(%eax)
192	movl $PTHRW_STATUS_ACQUIRED,%eax
193	popl %ebp
194	ret
195
196	/* cannot acquire mutex, so update seq count, set "W", and block in kernel */
197	/* this is where we cannot tolerate preemption or being killed */
198	5:
199	lea PTHRW_INC(%eax),%edx // copy original lval and bump sequence count
200	orl $PTHRW_WBIT, %edx // set WBIT
201	lock
202	cmpxchgl %edx,PTHRW_LVAL(%edi) // try to update lock status atomically
203	jnz 3b // failed
204	movl 20(%esi),%eax // get ptr to TID field of mutex
205	pushl 4(%esi) // arg 5: flags from arg list
206	pushl 4(%eax) // arg 4: tid field from mutex
207	pushl 0(%eax)
208	pushl PTHRW_UVAL(%edi) // arg 3: uval field from mutex
209	pushl %edx // arg 2: new value of mutex lval field
210	pushl %edi // arg 1: ptr to LVAL/UVAL pair in mutex
211	call 6f // make ksyn_mlwait call
212	jc 6f // immediately reissue syscall if error
213	movl 24(%esi),%edx // get ptr to syscall_return arg
214	movl %eax,(%edx) // save syscall return value
215	movl $PTHRW_STATUS_SYSCALL,%eax // we had to make syscall
216	addl $28,%esp // pop off syscall args and return address
217	popl %ebp // pop off frame ptr
218	ret
219
220	/* subroutine to make a ksyn_mlwait syscall */
221	6:
222	movl (%esp),%edx // get return address but leave on stack
223	movl %esp,%ecx // save stack ptr here
224	movl $KSYN_MLWAIT,%eax // get syscall code
225	orl $0x00180000,%eax // copy 24 bytes of arguments in trampoline
226	xorl %ebx,%ebx // clear preemption flag
227	sysenter
228	COMMPAGE_DESCRIPTOR(pfz_mutex_lock,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
229
230
231
232	/*********************** x86_64 versions follow ************************/
233
234
235
236	/* int // we return 0 on acquire, 1 on syscall
237	* pthread_mutex_lock( uint32_t *lvalp, // ptr to mutex LVAL/UVAL pair
238	* int flags, // flags to pass kernel if we do syscall
239	* uint64_t mtid, // my Thread ID
240	* uint32_t mask, // bits to test in LVAL (ie, EBIT etc)
241	* uint64_t *tidp, // ptr to TID field of mutex
242	* int *syscall_return ); // if syscall, return value stored here
243	*
244	* %rdi = lvalp
245	* %esi = flags
246	* %rdx = mtid
247	* %ecx = mask
248	* %r8 = tidp
249	* %r9 = &syscall_return
250	*/
251	COMMPAGE_FUNCTION_START(pthread_mutex_lock_64, 64, 4)
252	pushq %rbp // set up frame for backtrace
253	movq %rsp,%rbp
254	pushq %rbx
255	xorl %ebx,%ebx // clear "preemption pending" flag
256	movl _COMM_PAGE_32_TO_64(_COMM_PAGE_SPIN_COUNT), %eax
257	1:
258	testl PTHRW_LVAL(%rdi),%ecx // is mutex available?
259	jz 2f // yes, it is available
260	pause
261	decl %eax // decrement max spin count
262	jnz 1b // keep spinning
263	2:
264	COMMPAGE_CALL(_COMM_PAGE_PFZ_MUTEX_LOCK,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock_64)
265	testl %ebx,%ebx // pending preemption?
266	jz 1f // no
267	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_MUTEX_LOCK,pthread_mutex_lock_64)
268	1:
269	popq %rbx
270	popq %rbp
271	ret
272	COMMPAGE_DESCRIPTOR(pthread_mutex_lock_64,_COMM_PAGE_MUTEX_LOCK,0,0)
273
274
275	/* Internal routine to handle pthread mutex lock operation. This is in the PFZ.
276	* %rdi = lvalp
277	* %esi = flags
278	* %rdx = mtid
279	* %ecx = mask
280	* %r8 = tidp
281	* %r9 = &syscall_return
282	* %ebx = preempion pending flag (kernel sets nonzero if we should preempt)
283	*/
284	COMMPAGE_FUNCTION_START(pfz_mutex_lock_64, 64, 4)
285	pushq %rbp // set up frame for backtrace
286	movq %rsp,%rbp
287	1:
288	movl PTHRW_LVAL(%rdi),%eax // get old lval from mutex
289	2:
290	testl %eax,%ecx // can we acquire the lock?
291	jnz 5f // no
292
293	/* lock is available (if we act fast) */
294	lea PTHRW_INC(%rax),%r11 // copy original lval and bump sequence count
295	orl $PTHRW_EBIT, %r11d // set EBIT
296	lock
297	cmpxchgl %r11d,PTHRW_LVAL(%rdi) // try to acquire lock
298	jz 4f // got it
299	3:
300	testl %ebx,%ebx // kernel trying to preempt us?
301	jz 2b // no, so loop and try again
302	COMMPAGE_CALL(_COMM_PAGE_PREEMPT,_COMM_PAGE_PFZ_MUTEX_LOCK,pfz_mutex_lock_64)
303	jmp 1b // loop to try again
304
305	/* we acquired the mutex */
306	4:
307	movq %rdx,(%r8) // store mtid in mutex structure
308	movl $PTHRW_STATUS_ACQUIRED,%eax
309	popq %rbp
310	ret
311
312	/* cannot acquire mutex, so update seq count and block in kernel */
313	/* this is where we cannot tolerate preemption or being killed */
314	5:
315	lea PTHRW_INC(%rax),%r11 // copy original lval and bump sequence count
316	orl $PTHRW_WBIT, %r11d // set WBIT
317	lock
318	cmpxchgl %r11d,PTHRW_LVAL(%rdi) // try to update lock status atomically
319	jnz 3b // failed
320	movq (%r8),%r10 // arg 4: tid field from mutex [NB: passed in R10]
321	movl %esi,%r8d // arg 5: flags from arg list
322	movl PTHRW_UVAL(%rdi),%edx // arg 3: uval field from mutex
323	movl %r11d,%esi // arg 2: new value of mutex lval field
324	// arg 1: LVAL/UVAL ptr already in %rdi
325	6:
326	movl $(SYSCALL_CONSTRUCT_UNIX(KSYN_MLWAIT)),%eax
327	pushq %rdx // some syscalls destroy %rdx so save it
328	xorl %ebx,%ebx // clear preemption flag
329	syscall
330	popq %rdx // restore in case we need to re-execute syscall
331	jc 6b // immediately re-execute syscall if error
332	movl %eax,(%r9) // store kernel return value
333	movl $PTHRW_STATUS_SYSCALL,%eax // we made syscall
334	popq %rbp
335	ret
336	COMMPAGE_DESCRIPTOR(pfz_mutex_lock_64,_COMM_PAGE_PFZ_MUTEX_LOCK,0,0)
0c530ab8	337