[apple/xnu.git] / osfmk / ppc / commpage / spinlocks.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>					// EXT, LEXT
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>

        .text
        .align	2

#define	MP_SPIN_TRIES   1000


/* The user mode spinlock library.  There are many versions,
 * in order to take advantage of a few special cases:
 *	- no barrier instructions (SYNC,ISYNC) are needed if UP
 *	- 64-bit processors can use LWSYNC instead of SYNC (if MP)
 *  - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
 *	- branch hints appropriate to the processor (+ vs ++ etc)
 *	- potentially custom relinquish strategies (not used at present)
 *	- fixes for errata as necessary
 *
 * The convention for lockwords is that 0==free and -1==locked.
 */ 


spinlock_32_try_mp:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne-	2f
        stwcx.	r6,0,r5
        isync				// cancel speculative execution
        beqlr+
        b		1b
2:
        li		r3,0        // we did not get the lock
        blr

	COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
        

spinlock_32_try_up:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne-	2f
        stwcx.	r6,0,r5
        beqlr+
        b		1b
2:
        li		r3,0        // we did not get the lock
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)


spinlock_32_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne-	2f
        stwcx.	r6,0,r3
        isync				// cancel speculative execution
        beqlr+				// we return void
        b		1b
2:
        subic.	r5,r5,1		// try again before relinquish?
        bne		1b
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)


spinlock_32_lock_up:
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bnea-	_COMM_PAGE_RELINQUISH	// always depress on UP (let lock owner run)
        stwcx.	r6,0,r3
        beqlr+				// we return void
        b		1b

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)


spinlock_32_unlock_mp:
        li		r4,0
        isync				// complete prior stores before unlock
		eieio				// (using isync/eieio is faster than a sync)
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)


spinlock_32_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)


spinlock_64_try_mp:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r5
        isync				// cancel speculative execution
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0        // we did not get the lock
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)


spinlock_64_try_up:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r5
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0        // we did not get the lock
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)


spinlock_64_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r3
        isync				// cancel speculative execution
        beqlr++				// we return void
        b		1b
2:
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
        subic.	r5,r5,1		// try again before relinquish?
        bne--	1b			// mispredict this one (a cheap back-off)
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)


spinlock_64_lock_up:
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r3
        beqlr++				// we return void
        b		1b
2:							// always relinquish on UP (let lock owner run)
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
		ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)


spinlock_64_unlock_mp:
        lwsync				// complete prior stores before unlock
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)


spinlock_64_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
    

spinlock_relinquish:
        mr		r12,r3		// preserve lockword ptr across relinquish
        li		r3,0		// THREAD_NULL
        li		r4,1		// SWITCH_OPTION_DEPRESS
        li		r5,1		// timeout (ms)
        li		r0,-61		// SYSCALL_THREAD_SWITCH
        sc					// relinquish
        mr		r3,r12
        ba		_COMM_PAGE_SPINLOCK_LOCK
        
    COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
55e303ae	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
55e303ae A	27	*/
	28
	29	#include <sys/appleapiopts.h>
	30	#include <ppc/asm.h> // EXT, LEXT
	31	#include <machine/cpu_capabilities.h>
	32	#include <machine/commpage.h>
	33
	34	.text
	35	.align 2
55e303ae A	36
	37	#define MP_SPIN_TRIES 1000
	38
	39
91447636 A	40	/* The user mode spinlock library. There are many versions,
	41	* in order to take advantage of a few special cases:
	42	* - no barrier instructions (SYNC,ISYNC) are needed if UP
	43	* - 64-bit processors can use LWSYNC instead of SYNC (if MP)
	44	* - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
	45	* - branch hints appropriate to the processor (+ vs ++ etc)
	46	* - potentially custom relinquish strategies (not used at present)
	47	* - fixes for errata as necessary
	48	*
	49	* The convention for lockwords is that 0==free and -1==locked.
	50	*/
55e303ae A	51
	52
	53	spinlock_32_try_mp:
91447636 A	54	mr r5, r3
91447636 A	55	li r3, 1
55e303ae A	56	1:
55e303ae A	57	lwarx r4,0,r5
91447636	58	li r6,-1 // locked == -1
55e303ae A	59	cmpwi r4,0
55e303ae A	60	bne- 2f
91447636	61	stwcx. r6,0,r5
55e303ae A	62	isync // cancel speculative execution
	63	beqlr+
	64	b 1b
	65	2:
91447636	66	li r3,0 // we did not get the lock
55e303ae A	67	blr
55e303ae A	68
91447636	69	COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
55e303ae A	70
	71
	72	spinlock_32_try_up:
91447636 A	73	mr r5, r3
91447636 A	74	li r3, 1
55e303ae A	75	1:
55e303ae A	76	lwarx r4,0,r5
91447636	77	li r6,-1 // locked == -1
55e303ae A	78	cmpwi r4,0
55e303ae A	79	bne- 2f
91447636	80	stwcx. r6,0,r5
55e303ae A	81	beqlr+
	82	b 1b
	83	2:
91447636	84	li r3,0 // we did not get the lock
55e303ae A	85	blr
55e303ae A	86
91447636	87	COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)
55e303ae A	88
	89
	90	spinlock_32_lock_mp:
	91	li r5,MP_SPIN_TRIES
	92	1:
	93	lwarx r4,0,r3
91447636	94	li r6,-1 // locked == -1
55e303ae A	95	cmpwi r4,0
55e303ae A	96	bne- 2f
91447636	97	stwcx. r6,0,r3
55e303ae A	98	isync // cancel speculative execution
	99	beqlr+ // we return void
	100	b 1b
	101	2:
	102	subic. r5,r5,1 // try again before relinquish?
	103	bne 1b
	104	ba _COMM_PAGE_RELINQUISH
	105
91447636	106	COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)
55e303ae A	107
	108
	109	spinlock_32_lock_up:
	110	1:
	111	lwarx r4,0,r3
91447636	112	li r6,-1 // locked == -1
55e303ae A	113	cmpwi r4,0
55e303ae A	114	bnea- _COMM_PAGE_RELINQUISH // always depress on UP (let lock owner run)
91447636	115	stwcx. r6,0,r3
55e303ae A	116	beqlr+ // we return void
	117	b 1b
	118
91447636	119	COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)
55e303ae A	120
	121
	122	spinlock_32_unlock_mp:
	123	li r4,0
91447636 A	124	isync // complete prior stores before unlock
91447636 A	125	eieio // (using isync/eieio is faster than a sync)
55e303ae A	126	stw r4,0(r3)
	127	blr
	128
91447636	129	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)
55e303ae A	130
	131
	132	spinlock_32_unlock_up:
	133	li r4,0
	134	stw r4,0(r3)
	135	blr
	136
91447636	137	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)
55e303ae A	138
	139
	140	spinlock_64_try_mp:
91447636 A	141	mr r5, r3
91447636 A	142	li r3, 1
55e303ae A	143	1:
55e303ae A	144	lwarx r4,0,r5
91447636	145	li r6,-1 // locked == -1
55e303ae A	146	cmpwi r4,0
55e303ae A	147	bne-- 2f
91447636	148	stwcx. r6,0,r5
55e303ae A	149	isync // cancel speculative execution
	150	beqlr++
	151	b 1b
	152	2:
	153	li r6,-4
	154	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
91447636	155	li r3,0 // we did not get the lock
55e303ae A	156	blr
55e303ae A	157
91447636	158	COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)
55e303ae A	159
	160
	161	spinlock_64_try_up:
91447636 A	162	mr r5, r3
91447636 A	163	li r3, 1
55e303ae A	164	1:
55e303ae A	165	lwarx r4,0,r5
91447636	166	li r6,-1 // locked == -1
55e303ae A	167	cmpwi r4,0
55e303ae A	168	bne-- 2f
91447636	169	stwcx. r6,0,r5
55e303ae A	170	beqlr++
	171	b 1b
	172	2:
	173	li r6,-4
	174	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
91447636	175	li r3,0 // we did not get the lock
55e303ae A	176	blr
55e303ae A	177
91447636	178	COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)
55e303ae A	179
	180
	181	spinlock_64_lock_mp:
	182	li r5,MP_SPIN_TRIES
	183	1:
	184	lwarx r4,0,r3
91447636	185	li r6,-1 // locked == -1
55e303ae A	186	cmpwi r4,0
55e303ae A	187	bne-- 2f
91447636	188	stwcx. r6,0,r3
55e303ae A	189	isync // cancel speculative execution
	190	beqlr++ // we return void
	191	b 1b
	192	2:
	193	li r6,-4
	194	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
	195	subic. r5,r5,1 // try again before relinquish?
	196	bne-- 1b // mispredict this one (a cheap back-off)
	197	ba _COMM_PAGE_RELINQUISH
	198
91447636	199	COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)
55e303ae A	200
	201
	202	spinlock_64_lock_up:
	203	1:
	204	lwarx r4,0,r3
91447636	205	li r6,-1 // locked == -1
55e303ae A	206	cmpwi r4,0
55e303ae A	207	bne-- 2f
91447636	208	stwcx. r6,0,r3
55e303ae A	209	beqlr++ // we return void
	210	b 1b
	211	2: // always relinquish on UP (let lock owner run)
	212	li r6,-4
	213	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
	214	ba _COMM_PAGE_RELINQUISH
	215
91447636	216	COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)
55e303ae A	217
	218
	219	spinlock_64_unlock_mp:
55e303ae	220	lwsync // complete prior stores before unlock
91447636	221	li r4,0
55e303ae A	222	stw r4,0(r3)
	223	blr
	224
91447636	225	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)
55e303ae A	226
	227
	228	spinlock_64_unlock_up:
	229	li r4,0
	230	stw r4,0(r3)
	231	blr
	232
91447636	233	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
55e303ae A	234
	235
	236	spinlock_relinquish:
	237	mr r12,r3 // preserve lockword ptr across relinquish
	238	li r3,0 // THREAD_NULL
	239	li r4,1 // SWITCH_OPTION_DEPRESS
	240	li r5,1 // timeout (ms)
	241	li r0,-61 // SYSCALL_THREAD_SWITCH
	242	sc // relinquish
	243	mr r3,r12
	244	ba _COMM_PAGE_SPINLOCK_LOCK
	245
91447636	246	COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
55e303ae	247