git.saurik.com Git - apple/xnu.git/blame_incremental

0 / 212 ( 0%)

Commit	Line	Data
	1	/*
	2	* Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
	11	*
	12	* This Original Code and all software distributed under the License are
	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <machine/cpu_capabilities.h>
	25	#include <machine/commpage.h>
	26	#include <mach/i386/syscall_sw.h>
	27
	28
	29
	30	#define MP_SPIN_TRIES 1024
	31
	32	.text
	33	.align 4, 0x90
	34
	35	Lspin_lock_try_up:
	36	movl 4(%esp), %ecx
	37	xorl %eax, %eax
	38	orl $-1, %edx
	39	cmpxchgl %edx, (%ecx)
	40	setz %dl
	41	movzbl %dl, %eax
	42	ret
	43
	44	COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
	45
	46
	47	.align 4, 0x90
	48	Lspin_lock_try_mp:
	49	movl 4(%esp), %ecx
	50	xorl %eax, %eax
	51	orl $-1, %edx
	52	lock
	53	cmpxchgl %edx, (%ecx)
	54	setz %dl
	55	movzbl %dl, %eax
	56	ret
	57
	58	COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
	59
	60
	61	.align 4, 0x90
	62	Lspin_lock_up:
	63	movl 4(%esp), %ecx
	64	xorl %eax, %eax
	65	orl $-1, %edx
	66	cmpxchgl %edx, (%ecx)
	67	jnz,pn 1f /* predict not taken */
	68	ret
	69	1:
	70	/* failed to get lock so relinquish the processor immediately on UP */
	71	pushl $1 /* 1 ms */
	72	pushl $1 /* SWITCH_OPTION_DEPRESS */
	73	pushl $0 /* THREAD_NULL */
	74	pushl $0 /* push dummy stack ret addr */
	75	movl $-61,%eax /* SYSCALL_THREAD_SWITCH */
	76	int $(MACH_INT)
	77	addl $16, %esp /* adjust stack*/
	78	jmp Lspin_lock_up
	79
	80	COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
	81
	82
	83	.align 4, 0x90
	84	Lspin_lock_mp:
	85	movl 4(%esp), %ecx
	86	xorl %eax, %eax
	87	0:
	88	orl $-1, %edx
	89	lock
	90	cmpxchgl %edx, (%ecx)
	91	jnz,pn 1f /* predict not taken */
	92	ret
	93	1:
	94	xorl %eax, %eax
	95	movl $(MP_SPIN_TRIES), %edx
	96	2:
	97	pause
	98	cmpl %eax, (%ecx)
	99	jz,pt 0b /* favor success and slow down spin loop */
	100	decl %edx
	101	jnz,pn 2b /* slow down spin loop with a mispredict */
	102	/* failed to get lock after spinning so relinquish */
	103	pushl $1 /* 1 ms */
	104	pushl $1 /* SWITCH_OPTION_DEPRESS */
	105	pushl $0 /* THREAD_NULL */
	106	pushl $0 /* push dummy stack ret addr */
	107	movl $-61,%eax /* SYSCALL_THREAD_SWITCH */
	108	int $(MACH_INT)
	109	addl $16, %esp /* adjust stack*/
	110	jmp Lspin_lock_mp
	111
	112	COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
	113
	114
	115	.align 4, 0x90
	116	Lspin_unlock:
	117	movl 4(%esp), %ecx
	118	movl $0, (%ecx)
	119	ret
	120
	121	COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
	122
	123
	124	/* ============================ 64-bit versions follow ===================== */
	125
	126
	127	.text
	128	.code64
	129	.align 4, 0x90
	130
	131	Lspin_lock_try_up_64:
	132	xorl %eax, %eax
	133	orl $-1, %edx
	134	cmpxchgl %edx, (%rdi)
	135	setz %dl
	136	movzbl %dl, %eax
	137	ret
	138
	139	COMMPAGE_DESCRIPTOR(spin_lock_try_up_64,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
	140
	141
	142	.align 4, 0x90
	143	Lspin_lock_try_mp_64:
	144	xorl %eax, %eax
	145	orl $-1, %edx
	146	lock
	147	cmpxchgl %edx, (%rdi)
	148	setz %dl
	149	movzbl %dl, %eax
	150	ret
	151
	152	COMMPAGE_DESCRIPTOR(spin_lock_try_mp_64,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
	153
	154
	155	.align 4, 0x90
	156	Lspin_lock_up_64:
	157	movq %rdi,%r8
	158	0:
	159	xorl %eax, %eax
	160	orl $-1, %edx
	161	cmpxchgl %edx, (%r8)
	162	jnz,pn 1f /* predict not taken */
	163	ret
	164	1:
	165	/* failed to get lock so relinquish the processor immediately on UP */
	166	xorl %edi,%edi /* THREAD_NULL */
	167	movl $1,%esi /* SWITCH_OPTION_DEPRESS */
	168	movl $1,%edx /* 1 ms */
	169	movl $(SYSCALL_CONSTRUCT_MACH(61)),%eax /* 61 = thread_switch */
	170	syscall
	171	jmp 0b
	172
	173	COMMPAGE_DESCRIPTOR(spin_lock_up_64,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
	174
	175
	176
	177	.align 4, 0x90
	178	Lspin_lock_mp_64:
	179	movq %rdi,%r8
	180	0:
	181	xorl %eax, %eax
	182	orl $-1, %edx
	183	lock
	184	cmpxchgl %edx, (%r8)
	185	jnz,pn 1f /* predict not taken */
	186	ret
	187	1:
	188	xorl %eax, %eax
	189	movl $(MP_SPIN_TRIES), %edx
	190	2: /* spin for awhile before relinquish */
	191	pause
	192	cmpl %eax, (%r8)
	193	jz 0b
	194	decl %edx
	195	jnz 2b
	196	/* failed to get lock after spinning so relinquish */
	197	xorl %edi,%edi /* THREAD_NULL */
	198	movl $1,%esi /* SWITCH_OPTION_DEPRESS */
	199	movl $1,%edx /* 1 ms */
	200	movl $(SYSCALL_CONSTRUCT_MACH(61)),%eax /* 61 = thread_switch */
	201	syscall
	202	jmp 0b
	203
	204	COMMPAGE_DESCRIPTOR(spin_lock_mp_64,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
	205
	206
	207	.align 4, 0x90
	208	Lspin_unlock_64:
	209	movl $0, (%rdi)
	210	ret
	211
	212	COMMPAGE_DESCRIPTOR(spin_unlock_64,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)