[apple/xnu.git] / osfmk / ppc / commpage / spinlocks.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>					// EXT, LEXT
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>

        .text
        .align	2
        .globl	EXT(spinlock_32_try_mp)
        .globl	EXT(spinlock_32_try_up)
        .globl	EXT(spinlock_32_lock_mp)
        .globl	EXT(spinlock_32_lock_up)
        .globl	EXT(spinlock_32_unlock_mp)
        .globl	EXT(spinlock_32_unlock_up)
        
        .globl	EXT(spinlock_64_try_mp)
        .globl	EXT(spinlock_64_try_up)
        .globl	EXT(spinlock_64_lock_mp)
        .globl	EXT(spinlock_64_lock_up)
        .globl	EXT(spinlock_64_unlock_mp)
        .globl	EXT(spinlock_64_unlock_up)
        
        .globl	EXT(spinlock_relinquish)

#define	MP_SPIN_TRIES   1000


// The user mode spinlock library.  There are many versions,
// in order to take advantage of a few special cases:
//	- no barrier instructions (SYNC,ISYNC) are needed if UP
//	- 64-bit processors can use LWSYNC instead of SYNC (if MP)
//	- branch hints appropriate to the processor (+ vs ++ etc)
//	- potentially custom relinquish strategies (not used at present)
//	- fixes for errata as necessary


spinlock_32_try_mp:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne-	2f
        stwcx.	r5,0,r5
        isync				// cancel speculative execution
        beqlr+
        b		1b
2:
        li		r3,0
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,0)
        

spinlock_32_try_up:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne-	2f
        stwcx.	r5,0,r5
        beqlr+
        b		1b
2:
        li		r3,0
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,0)


spinlock_32_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bne-	2f
        stwcx.	r3,0,r3
        isync				// cancel speculative execution
        beqlr+				// we return void
        b		1b
2:
        subic.	r5,r5,1		// try again before relinquish?
        bne		1b
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,0)


spinlock_32_lock_up:
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bnea-	_COMM_PAGE_RELINQUISH	// always depress on UP (let lock owner run)
        stwcx.	r3,0,r3
        beqlr+				// we return void
        b		1b

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,0)


spinlock_32_unlock_mp:
        li		r4,0
        sync				// complete prior stores before unlock
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,0)


spinlock_32_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,0)


spinlock_64_try_mp:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne--	2f
        stwcx.	r5,0,r5
        isync				// cancel speculative execution
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0		// Pass failure
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,0)


spinlock_64_try_up:
	mr	r5, r3
	li	r3, 1
1:
        lwarx	r4,0,r5
        cmpwi	r4,0
        bne--	2f
        stwcx.	r5,0,r5
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,0)


spinlock_64_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bne--	2f
        stwcx.	r3,0,r3
        isync				// cancel speculative execution
        beqlr++				// we return void
        b		1b
2:
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
        subic.	r5,r5,1		// try again before relinquish?
        bne--	1b			// mispredict this one (a cheap back-off)
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,0)


spinlock_64_lock_up:
1:
        lwarx	r4,0,r3
        cmpwi	r4,0
        bne--	2f
        stwcx.	r3,0,r3
        beqlr++				// we return void
        b		1b
2:							// always relinquish on UP (let lock owner run)
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
		ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,0)


spinlock_64_unlock_mp:
        li		r4,0
        lwsync				// complete prior stores before unlock
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,0)


spinlock_64_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,0)
    

spinlock_relinquish:
        mr		r12,r3		// preserve lockword ptr across relinquish
        li		r3,0		// THREAD_NULL
        li		r4,1		// SWITCH_OPTION_DEPRESS
        li		r5,1		// timeout (ms)
        li		r0,-61		// SYSCALL_THREAD_SWITCH
        sc					// relinquish
        mr		r3,r12
        ba		_COMM_PAGE_SPINLOCK_LOCK
        
    COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,0)
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
	7	*
	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
	25
	26	#include <sys/appleapiopts.h>
	27	#include <ppc/asm.h> // EXT, LEXT
	28	#include <machine/cpu_capabilities.h>
	29	#include <machine/commpage.h>
	30
	31	.text
	32	.align 2
	33	.globl EXT(spinlock_32_try_mp)
	34	.globl EXT(spinlock_32_try_up)
	35	.globl EXT(spinlock_32_lock_mp)
	36	.globl EXT(spinlock_32_lock_up)
	37	.globl EXT(spinlock_32_unlock_mp)
	38	.globl EXT(spinlock_32_unlock_up)
	39
	40	.globl EXT(spinlock_64_try_mp)
	41	.globl EXT(spinlock_64_try_up)
	42	.globl EXT(spinlock_64_lock_mp)
	43	.globl EXT(spinlock_64_lock_up)
	44	.globl EXT(spinlock_64_unlock_mp)
	45	.globl EXT(spinlock_64_unlock_up)
	46
	47	.globl EXT(spinlock_relinquish)
	48
	49	#define MP_SPIN_TRIES 1000
	50
	51
	52	// The user mode spinlock library. There are many versions,
	53	// in order to take advantage of a few special cases:
	54	// - no barrier instructions (SYNC,ISYNC) are needed if UP
	55	// - 64-bit processors can use LWSYNC instead of SYNC (if MP)
	56	// - branch hints appropriate to the processor (+ vs ++ etc)
	57	// - potentially custom relinquish strategies (not used at present)
	58	// - fixes for errata as necessary
	59
	60
	61	spinlock_32_try_mp:
	62	mr r5, r3
	63	li r3, 1
	64	1:
65	lwarx r4,0,r5
66	cmpwi r4,0
67	bne- 2f
68	stwcx. r5,0,r5
69	isync // cancel speculative execution
70	beqlr+
71	b 1b
72	2:
73	li r3,0
74	blr
75
76	COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,0)
77
78
79	spinlock_32_try_up:
80	mr r5, r3
81	li r3, 1
82	1:
83	lwarx r4,0,r5
84	cmpwi r4,0
85	bne- 2f
86	stwcx. r5,0,r5
87	beqlr+
88	b 1b
89	2:
90	li r3,0
91	blr
92
93	COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,0)
94
95
96	spinlock_32_lock_mp:
97	li r5,MP_SPIN_TRIES
98	1:
99	lwarx r4,0,r3
100	cmpwi r4,0
101	bne- 2f
102	stwcx. r3,0,r3
103	isync // cancel speculative execution
104	beqlr+ // we return void
105	b 1b
106	2:
107	subic. r5,r5,1 // try again before relinquish?
108	bne 1b
109	ba _COMM_PAGE_RELINQUISH
110
111	COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,0)
112
113
114	spinlock_32_lock_up:
115	1:
116	lwarx r4,0,r3
117	cmpwi r4,0
118	bnea- _COMM_PAGE_RELINQUISH // always depress on UP (let lock owner run)
119	stwcx. r3,0,r3
120	beqlr+ // we return void
121	b 1b
122
123	COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,0)
124
125
126	spinlock_32_unlock_mp:
127	li r4,0
128	sync // complete prior stores before unlock
129	stw r4,0(r3)
130	blr
131
132	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,0)
133
134
135	spinlock_32_unlock_up:
136	li r4,0
137	stw r4,0(r3)
138	blr
139
140	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,0)
141
142
143	spinlock_64_try_mp:
144	mr r5, r3
145	li r3, 1
146	1:
147	lwarx r4,0,r5
148	cmpwi r4,0
149	bne-- 2f
150	stwcx. r5,0,r5
151	isync // cancel speculative execution
152	beqlr++
153	b 1b
154	2:
155	li r6,-4
156	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
157	li r3,0 // Pass failure
158	blr
159
160	COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,0)
161
162
163	spinlock_64_try_up:
164	mr r5, r3
165	li r3, 1
166	1:
167	lwarx r4,0,r5
168	cmpwi r4,0
169	bne-- 2f
170	stwcx. r5,0,r5
171	beqlr++
172	b 1b
173	2:
174	li r6,-4
175	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
176	li r3,0
177	blr
178
179	COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,0)
180
181
182	spinlock_64_lock_mp:
183	li r5,MP_SPIN_TRIES
184	1:
185	lwarx r4,0,r3
186	cmpwi r4,0
187	bne-- 2f
188	stwcx. r3,0,r3
189	isync // cancel speculative execution
190	beqlr++ // we return void
191	b 1b
192	2:
193	li r6,-4
194	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
195	subic. r5,r5,1 // try again before relinquish?
196	bne-- 1b // mispredict this one (a cheap back-off)
197	ba _COMM_PAGE_RELINQUISH
198
199	COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,0)
200
201
202	spinlock_64_lock_up:
203	1:
204	lwarx r4,0,r3
205	cmpwi r4,0
206	bne-- 2f
207	stwcx. r3,0,r3
208	beqlr++ // we return void
209	b 1b
210	2: // always relinquish on UP (let lock owner run)
211	li r6,-4
212	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
213	ba _COMM_PAGE_RELINQUISH
214
215	COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,0)
216
217
218	spinlock_64_unlock_mp:
219	li r4,0
220	lwsync // complete prior stores before unlock
221	stw r4,0(r3)
222	blr
223
224	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,0)
225
226
227	spinlock_64_unlock_up:
228	li r4,0
229	stw r4,0(r3)
230	blr
231
232	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,0)
233
234
235	spinlock_relinquish:
236	mr r12,r3 // preserve lockword ptr across relinquish
237	li r3,0 // THREAD_NULL
238	li r4,1 // SWITCH_OPTION_DEPRESS
239	li r5,1 // timeout (ms)
240	li r0,-61 // SYSCALL_THREAD_SWITCH
241	sc // relinquish
242	mr r3,r12
243	ba _COMM_PAGE_SPINLOCK_LOCK
244
245	COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,0)
246