[apple/xnu.git] / osfmk / i386 / commpage / spinlocks.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>

/*
 * We need a relative branch within the comm page, and don't want the linker
 * to relocate it, so we have to hand-code the instructions. LEN is to account
 * for the length of a .long, since the jmp is relative to the next instruction.
 */

#define JNZ .byte 0x0f, 0x85; .long
#define JMP .byte 0xe9; .long
#define LEN 4 

/*
 * Branch prediction prefixes
 */

#define LIKELY		.byte 0x3e
#define UNLIKELY	.byte 0x2e

#define MP_SPIN_TRIES	1024

	.text
	.align 4, 0x90

Lspin_lock_try_up:
	movl		4(%esp), %ecx 
	xorl		%eax, %eax
	cmpxchgl	%ecx, (%ecx)
	setz		%dl
	movzbl		%dl, %eax
	ret

	COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
 
	.align 4, 0x90
Lspin_lock_try_mp:
	movl		4(%esp), %ecx 
	xorl		%eax, %eax
	lock
	cmpxchgl	%ecx, (%ecx)
	setz		%dl
	movzbl		%dl, %eax
	ret

	COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)

.set Lrelinquish_off,	_COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK

	.align 4, 0x90
Lspin_lock_up:
	movl		4(%esp), %ecx
	xorl		%eax, %eax
.set Lretry,		. - Lspin_lock_up
	cmpxchgl	%ecx, (%ecx)
	UNLIKELY
	JNZ		Lrelinquish_off - . + Lspin_lock_up - LEN
	ret

	COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)

	.align 4, 0x90
Lspin_lock_mp:
	movl		4(%esp), %ecx
	xorl		%eax, %eax
0:
	lock
	cmpxchgl	%ecx, (%ecx)
	UNLIKELY
	jnz		1f
	ret
1:
	xorl		%eax, %eax
	movl		$(MP_SPIN_TRIES), %edx
2:
	pause	
	cmpl		%eax, (%ecx)
	LIKELY
	jz		0b
	decl		%edx
	LIKELY
	jnz		2b
	JMP		Lrelinquish_off - . + Lspin_lock_mp - LEN
 
	COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)

	.align 4, 0x90
Lspin_unlock:
	movl		4(%esp), %ecx
	movl		$0, (%ecx)
	ret

	COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)

	.align 4, 0x90
Lrelinquish:				/* relinquish the processor	*/
	pushl		$1		/* 1 ms				*/
	pushl		$1		/* SWITCH_OPTION_DEPRESS	*/
	pushl		$0		/* THREAD_NULL			*/
	movl		$-61, %eax	/* syscall_thread_switch	*/
	lcall		$7, $0
	popl		%eax		/* set %eax to 0 again		*/
	popl		%edx		/* use %edx as scratch		*/
	popl		%edx		/* reg to fixup stack		*/
	JMP		Lretry - Lrelinquish_off - . + Lrelinquish - LEN

	COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
e5568f75 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
55e303ae	11	*
e5568f75 A	12	* This Original Code and all software distributed under the License are
e5568f75 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
55e303ae A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
55e303ae A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
55e303ae A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <machine/cpu_capabilities.h>
	25	#include <machine/commpage.h>
	26
	27	/*
	28	* We need a relative branch within the comm page, and don't want the linker
	29	* to relocate it, so we have to hand-code the instructions. LEN is to account
	30	* for the length of a .long, since the jmp is relative to the next instruction.
	31	*/
	32
	33	#define JNZ .byte 0x0f, 0x85; .long
	34	#define JMP .byte 0xe9; .long
	35	#define LEN 4
	36
	37	/*
	38	* Branch prediction prefixes
	39	*/
	40
	41	#define LIKELY .byte 0x3e
	42	#define UNLIKELY .byte 0x2e
	43
	44	#define MP_SPIN_TRIES 1024
	45
	46	.text
	47	.align 4, 0x90
	48
	49	Lspin_lock_try_up:
	50	movl 4(%esp), %ecx
	51	xorl %eax, %eax
	52	cmpxchgl %ecx, (%ecx)
	53	setz %dl
	54	movzbl %dl, %eax
	55	ret
	56
	57	COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
	58
	59	.align 4, 0x90
	60	Lspin_lock_try_mp:
	61	movl 4(%esp), %ecx
	62	xorl %eax, %eax
	63	lock
	64	cmpxchgl %ecx, (%ecx)
	65	setz %dl
	66	movzbl %dl, %eax
	67	ret
	68
	69	COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
	70
	71	.set Lrelinquish_off, _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
	72
	73	.align 4, 0x90
	74	Lspin_lock_up:
	75	movl 4(%esp), %ecx
	76	xorl %eax, %eax
	77	.set Lretry, . - Lspin_lock_up
	78	cmpxchgl %ecx, (%ecx)
	79	UNLIKELY
	80	JNZ Lrelinquish_off - . + Lspin_lock_up - LEN
	81	ret
	82
83	COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
84
85	.align 4, 0x90
86	Lspin_lock_mp:
87	movl 4(%esp), %ecx
88	xorl %eax, %eax
89	0:
90	lock
91	cmpxchgl %ecx, (%ecx)
92	UNLIKELY
93	jnz 1f
94	ret
95	1:
96	xorl %eax, %eax
97	movl $(MP_SPIN_TRIES), %edx
98	2:
99	pause
100	cmpl %eax, (%ecx)
101	LIKELY
102	jz 0b
103	decl %edx
104	LIKELY
105	jnz 2b
106	JMP Lrelinquish_off - . + Lspin_lock_mp - LEN
107
108	COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
109
110	.align 4, 0x90
111	Lspin_unlock:
112	movl 4(%esp), %ecx
113	movl $0, (%ecx)
114	ret
115
116	COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
117
118	.align 4, 0x90
119	Lrelinquish: /* relinquish the processor */
120	pushl $1 /* 1 ms */
121	pushl $1 /* SWITCH_OPTION_DEPRESS */
122	pushl $0 /* THREAD_NULL */
123	movl $-61, %eax /* syscall_thread_switch */
124	lcall $7, $0
125	popl %eax /* set %eax to 0 again */
126	popl %edx /* use %edx as scratch */
127	popl %edx /* reg to fixup stack */
128	JMP Lretry - Lrelinquish_off - . + Lrelinquish - LEN
129
130	COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)