[apple/xnu.git] / osfmk / i386 / commpage / spinlocks.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>

/*
 * We need a relative branch within the comm page, and don't want the linker
 * to relocate it, so we have to hand-code the instructions. LEN is to account
 * for the length of a .long, since the jmp is relative to the next instruction.
 */

#define JNZ .byte 0x0f, 0x85; .long
#define JMP .byte 0xe9; .long
#define LEN 4 

/*
 * Branch prediction prefixes
 */

#define LIKELY		.byte 0x3e
#define UNLIKELY	.byte 0x2e

#define MP_SPIN_TRIES	1024

	.text
	.align 4, 0x90

Lspin_lock_try_up:
	movl		4(%esp), %ecx 
	xorl		%eax, %eax
	orl		$-1, %edx
	cmpxchgl	%edx, (%ecx)
	setz		%dl
	movzbl		%dl, %eax
	ret

	COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
 
	.align 4, 0x90
Lspin_lock_try_mp:
	movl		4(%esp), %ecx 
	xorl		%eax, %eax
	orl		$-1, %edx
	lock
	cmpxchgl	%edx, (%ecx)
	setz		%dl
	movzbl		%dl, %eax
	ret

	COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)

.set Lrelinquish_off,	_COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK

	.align 4, 0x90
Lspin_lock_up:
	movl		4(%esp), %ecx
	xorl		%eax, %eax
.set Lretry,		. - Lspin_lock_up
	orl		$-1, %edx
	cmpxchgl	%edx, (%ecx)
	UNLIKELY
	JNZ		Lrelinquish_off - . + Lspin_lock_up - LEN
	ret

	COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)

	.align 4, 0x90
Lspin_lock_mp:
	movl		4(%esp), %ecx
	xorl		%eax, %eax
0:
	orl		$-1, %edx
	lock
	cmpxchgl	%edx, (%ecx)
	UNLIKELY
	jnz		1f
	ret
1:
	xorl		%eax, %eax
	movl		$(MP_SPIN_TRIES), %edx
2:
	pause	
	cmpl		%eax, (%ecx)
	LIKELY
	jz		0b
	decl		%edx
	LIKELY
	jnz		2b
	JMP		Lrelinquish_off - . + Lspin_lock_mp - LEN
 
	COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)

	.align 4, 0x90
Lspin_unlock:
	movl		4(%esp), %ecx
	movl		$0, (%ecx)
	ret

	COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)

	.align 4, 0x90
Lrelinquish:				/* relinquish the processor	*/
	pushl		$1		/* 1 ms				*/
	pushl		$1		/* SWITCH_OPTION_DEPRESS	*/
	pushl		$0		/* THREAD_NULL			*/
	pushl		$0		/* push dummy stack ret addr    */
	movl		$-61, %eax	/* syscall_thread_switch	*/
	lcall		$7, $0
	addl		$16, %esp	/* adjust stack*/
	xorl		%eax, %eax	/* set %eax to 0 again		*/
	JMP		Lretry - Lrelinquish_off - . + Lrelinquish - LEN

	COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
e5568f75 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
55e303ae	11	*
e5568f75 A	12	* This Original Code and all software distributed under the License are
e5568f75 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
55e303ae A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
55e303ae A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
55e303ae A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <machine/cpu_capabilities.h>
	25	#include <machine/commpage.h>
	26
	27	/*
	28	* We need a relative branch within the comm page, and don't want the linker
	29	* to relocate it, so we have to hand-code the instructions. LEN is to account
	30	* for the length of a .long, since the jmp is relative to the next instruction.
	31	*/
	32
	33	#define JNZ .byte 0x0f, 0x85; .long
	34	#define JMP .byte 0xe9; .long
	35	#define LEN 4
	36
	37	/*
	38	* Branch prediction prefixes
	39	*/
	40
	41	#define LIKELY .byte 0x3e
	42	#define UNLIKELY .byte 0x2e
	43
	44	#define MP_SPIN_TRIES 1024
	45
	46	.text
	47	.align 4, 0x90
	48
	49	Lspin_lock_try_up:
	50	movl 4(%esp), %ecx
	51	xorl %eax, %eax
91447636 A	52	orl $-1, %edx
91447636 A	53	cmpxchgl %edx, (%ecx)
55e303ae A	54	setz %dl
	55	movzbl %dl, %eax
	56	ret
	57
	58	COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
	59
	60	.align 4, 0x90
	61	Lspin_lock_try_mp:
	62	movl 4(%esp), %ecx
	63	xorl %eax, %eax
91447636	64	orl $-1, %edx
55e303ae	65	lock
91447636	66	cmpxchgl %edx, (%ecx)
55e303ae A	67	setz %dl
	68	movzbl %dl, %eax
	69	ret
	70
	71	COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
	72
	73	.set Lrelinquish_off, _COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
	74
	75	.align 4, 0x90
	76	Lspin_lock_up:
	77	movl 4(%esp), %ecx
	78	xorl %eax, %eax
	79	.set Lretry, . - Lspin_lock_up
91447636 A	80	orl $-1, %edx
91447636 A	81	cmpxchgl %edx, (%ecx)
55e303ae A	82	UNLIKELY
	83	JNZ Lrelinquish_off - . + Lspin_lock_up - LEN
	84	ret
	85
	86	COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
	87
	88	.align 4, 0x90
	89	Lspin_lock_mp:
	90	movl 4(%esp), %ecx
	91	xorl %eax, %eax
	92	0:
91447636	93	orl $-1, %edx
55e303ae	94	lock
91447636	95	cmpxchgl %edx, (%ecx)
55e303ae A	96	UNLIKELY
	97	jnz 1f
	98	ret
	99	1:
	100	xorl %eax, %eax
	101	movl $(MP_SPIN_TRIES), %edx
	102	2:
	103	pause
	104	cmpl %eax, (%ecx)
	105	LIKELY
	106	jz 0b
	107	decl %edx
	108	LIKELY
	109	jnz 2b
	110	JMP Lrelinquish_off - . + Lspin_lock_mp - LEN
	111
	112	COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
	113
	114	.align 4, 0x90
	115	Lspin_unlock:
	116	movl 4(%esp), %ecx
	117	movl $0, (%ecx)
	118	ret
	119
	120	COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
	121
	122	.align 4, 0x90
	123	Lrelinquish: /* relinquish the processor */
	124	pushl $1 /* 1 ms */
	125	pushl $1 /* SWITCH_OPTION_DEPRESS */
	126	pushl $0 /* THREAD_NULL */
91447636	127	pushl $0 /* push dummy stack ret addr */
55e303ae A	128	movl $-61, %eax /* syscall_thread_switch */
55e303ae A	129	lcall $7, $0
91447636 A	130	addl $16, %esp /* adjust stack*/
91447636 A	131	xorl %eax, %eax /* set %eax to 0 again */
55e303ae A	132	JMP Lretry - Lrelinquish_off - . + Lrelinquish - LEN
	133
	134	COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)