[apple/xnu.git] / osfmk / ppc / commpage / spinlocks.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>					// EXT, LEXT
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>

        .text
        .align	2
        .globl	EXT(spinlock_32_try_mp)
        .globl	EXT(spinlock_32_try_up)
        .globl	EXT(spinlock_32_lock_mp)
        .globl	EXT(spinlock_32_lock_up)
        .globl	EXT(spinlock_32_unlock_mp)
        .globl	EXT(spinlock_32_unlock_up)
        
        .globl	EXT(spinlock_64_try_mp)
        .globl	EXT(spinlock_64_try_up)
        .globl	EXT(spinlock_64_lock_mp)
        .globl	EXT(spinlock_64_lock_up)
        .globl	EXT(spinlock_64_unlock_mp)
        .globl	EXT(spinlock_64_unlock_up)
        
        .globl	EXT(spinlock_relinquish)

#define	MP_SPIN_TRIES   1000


// The user mode spinlock library.  There are many versions,
// in order to take advantage of a few special cases:
//	- no barrier instructions (SYNC,ISYNC) are needed if UP
//	- 64-bit processors can use LWSYNC instead of SYNC (if MP)
//	- branch hints appropriate to the processor (+ vs ++ etc)
//	- potentially custom relinquish strategies (not used at present)
//	- fixes for errata as necessary


spinlock_32_try_mp:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne-	2f
        stwcx.	r5,0,r5
        isync				// cancel speculative execution
        beqlr+
        b		1b
2:
        li		r3,0
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,0)
        

spinlock_32_try_up:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne-	2f
        stwcx.	r5,0,r5
        beqlr+
        b		1b
2:
        li		r3,0
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,0)


spinlock_32_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bne-	2f
        stwcx.	r3,0,r3
        isync				// cancel speculative execution
        beqlr+				// we return void
        b		1b
2:
        subic.	r5,r5,1		// try again before relinquish?
        bne		1b
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,0)


spinlock_32_lock_up:
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bnea-	_COMM_PAGE_RELINQUISH	// always depress on UP (let lock owner run)
        stwcx.	r3,0,r3
        beqlr+				// we return void
        b		1b

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,0)


spinlock_32_unlock_mp:
        li		r4,0
        sync				// complete prior stores before unlock
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,0)


spinlock_32_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,0)


spinlock_64_try_mp:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne--	2f
        stwcx.	r5,0,r5
        isync				// cancel speculative execution
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0		// Pass failure
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,0)


spinlock_64_try_up:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne--	2f
        stwcx.	r5,0,r5
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,0)


spinlock_64_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bne--	2f
        stwcx.	r3,0,r3
        isync				// cancel speculative execution
        beqlr++				// we return void
        b		1b
2:
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
        subic.	r5,r5,1		// try again before relinquish?
        bne--	1b			// mispredict this one (a cheap back-off)
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,0)


spinlock_64_lock_up:
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bne--	2f
        stwcx.	r3,0,r3
        beqlr++				// we return void
        b		1b
2:							// always relinquish on UP (let lock owner run)
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
		ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,0)


spinlock_64_unlock_mp:
        li		r4,0
        lwsync				// complete prior stores before unlock
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,0)


spinlock_64_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,0)
    

spinlock_relinquish:
        mr		r12,r3		// preserve lockword ptr across relinquish
        li		r3,0		// THREAD_NULL
        li		r4,1		// SWITCH_OPTION_DEPRESS
        li		r5,1		// timeout (ms)
        li		r0,-61		// SYSCALL_THREAD_SWITCH
        sc					// relinquish
        mr		r3,r12
        ba		_COMM_PAGE_SPINLOCK_LOCK
        
    COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,0)
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
e5568f75 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
55e303ae	11	*
e5568f75 A	12	* This Original Code and all software distributed under the License are
e5568f75 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
55e303ae A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
55e303ae A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
55e303ae A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <ppc/asm.h> // EXT, LEXT
	25	#include <machine/cpu_capabilities.h>
	26	#include <machine/commpage.h>
	27
	28	.text
	29	.align 2
	30	.globl EXT(spinlock_32_try_mp)
	31	.globl EXT(spinlock_32_try_up)
	32	.globl EXT(spinlock_32_lock_mp)
	33	.globl EXT(spinlock_32_lock_up)
	34	.globl EXT(spinlock_32_unlock_mp)
	35	.globl EXT(spinlock_32_unlock_up)
	36
	37	.globl EXT(spinlock_64_try_mp)
	38	.globl EXT(spinlock_64_try_up)
	39	.globl EXT(spinlock_64_lock_mp)
	40	.globl EXT(spinlock_64_lock_up)
	41	.globl EXT(spinlock_64_unlock_mp)
	42	.globl EXT(spinlock_64_unlock_up)
	43
	44	.globl EXT(spinlock_relinquish)
	45
	46	#define MP_SPIN_TRIES 1000
	47
	48
	49	// The user mode spinlock library. There are many versions,
	50	// in order to take advantage of a few special cases:
	51	// - no barrier instructions (SYNC,ISYNC) are needed if UP
	52	// - 64-bit processors can use LWSYNC instead of SYNC (if MP)
	53	// - branch hints appropriate to the processor (+ vs ++ etc)
	54	// - potentially custom relinquish strategies (not used at present)
	55	// - fixes for errata as necessary
	56
	57
	58	spinlock_32_try_mp:
	59	mr r5, r3
	60	li r3, 1
	61	1:
	62	lwarx r4,0,r5
	63	cmpwi r4,0
	64	bne- 2f
	65	stwcx. r5,0,r5
	66	isync // cancel speculative execution
	67	beqlr+
	68	b 1b
	69	2:
	70	li r3,0
	71	blr
	72
	73	COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,0)
	74
	75
	76	spinlock_32_try_up:
	77	mr r5, r3
	78	li r3, 1
	79	1:
	80	lwarx r4,0,r5
	81	cmpwi r4,0
	82	bne- 2f
83	stwcx. r5,0,r5
84	beqlr+
85	b 1b
86	2:
87	li r3,0
88	blr
89
90	COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,0)
91
92
93	spinlock_32_lock_mp:
94	li r5,MP_SPIN_TRIES
95	1:
96	lwarx r4,0,r3
97	cmpwi r4,0
98	bne- 2f
99	stwcx. r3,0,r3
100	isync // cancel speculative execution
101	beqlr+ // we return void
102	b 1b
103	2:
104	subic. r5,r5,1 // try again before relinquish?
105	bne 1b
106	ba _COMM_PAGE_RELINQUISH
107
108	COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,0)
109
110
111	spinlock_32_lock_up:
112	1:
113	lwarx r4,0,r3
114	cmpwi r4,0
115	bnea- _COMM_PAGE_RELINQUISH // always depress on UP (let lock owner run)
116	stwcx. r3,0,r3
117	beqlr+ // we return void
118	b 1b
119
120	COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,0)
121
122
123	spinlock_32_unlock_mp:
124	li r4,0
125	sync // complete prior stores before unlock
126	stw r4,0(r3)
127	blr
128
129	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,0)
130
131
132	spinlock_32_unlock_up:
133	li r4,0
134	stw r4,0(r3)
135	blr
136
137	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,0)
138
139
140	spinlock_64_try_mp:
141	mr r5, r3
142	li r3, 1
143	1:
144	lwarx r4,0,r5
145	cmpwi r4,0
146	bne-- 2f
147	stwcx. r5,0,r5
148	isync // cancel speculative execution
149	beqlr++
150	b 1b
151	2:
152	li r6,-4
153	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
154	li r3,0 // Pass failure
155	blr
156
157	COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,0)
158
159
160	spinlock_64_try_up:
161	mr r5, r3
162	li r3, 1
163	1:
164	lwarx r4,0,r5
165	cmpwi r4,0
166	bne-- 2f
167	stwcx. r5,0,r5
168	beqlr++
169	b 1b
170	2:
171	li r6,-4
172	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
173	li r3,0
174	blr
175
176	COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,0)
177
178
179	spinlock_64_lock_mp:
180	li r5,MP_SPIN_TRIES
181	1:
182	lwarx r4,0,r3
183	cmpwi r4,0
184	bne-- 2f
185	stwcx. r3,0,r3
186	isync // cancel speculative execution
187	beqlr++ // we return void
188	b 1b
189	2:
190	li r6,-4
191	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
192	subic. r5,r5,1 // try again before relinquish?
193	bne-- 1b // mispredict this one (a cheap back-off)
194	ba _COMM_PAGE_RELINQUISH
195
196	COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,0)
197
198
199	spinlock_64_lock_up:
200	1:
201	lwarx r4,0,r3
202	cmpwi r4,0
203	bne-- 2f
204	stwcx. r3,0,r3
205	beqlr++ // we return void
206	b 1b
207	2: // always relinquish on UP (let lock owner run)
208	li r6,-4
209	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
210	ba _COMM_PAGE_RELINQUISH
211
212	COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,0)
213
214
215	spinlock_64_unlock_mp:
216	li r4,0
217	lwsync // complete prior stores before unlock
218	stw r4,0(r3)
219	blr
220
221	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,0)
222
223
224	spinlock_64_unlock_up:
225	li r4,0
226	stw r4,0(r3)
227	blr
228
229	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,0)
230
231
232	spinlock_relinquish:
233	mr r12,r3 // preserve lockword ptr across relinquish
234	li r3,0 // THREAD_NULL
235	li r4,1 // SWITCH_OPTION_DEPRESS
236	li r5,1 // timeout (ms)
237	li r0,-61 // SYSCALL_THREAD_SWITCH
238	sc // relinquish
239	mr r3,r12
240	ba _COMM_PAGE_SPINLOCK_LOCK
241
242	COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,0)
243