[apple/xnu.git] / osfmk / ppc / commpage / spinlocks.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>					// EXT, LEXT
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>

        .text
        .align	2

#define	MP_SPIN_TRIES   1000


/* The user mode spinlock library.  There are many versions,
 * in order to take advantage of a few special cases:
 *	- no barrier instructions (SYNC,ISYNC) are needed if UP
 *	- 64-bit processors can use LWSYNC instead of SYNC (if MP)
 *  - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
 *	- branch hints appropriate to the processor (+ vs ++ etc)
 *	- potentially custom relinquish strategies (not used at present)
 *	- fixes for errata as necessary
 *
 * The convention for lockwords is that 0==free and -1==locked.
 */ 


spinlock_32_try_mp:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne-	2f
        stwcx.	r6,0,r5
        isync				// cancel speculative execution
        beqlr+
        b		1b
2:
        li		r3,0        // we did not get the lock
        blr

	COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
        

spinlock_32_try_up:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne-	2f
        stwcx.	r6,0,r5
        beqlr+
        b		1b
2:
        li		r3,0        // we did not get the lock
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)


spinlock_32_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne-	2f
        stwcx.	r6,0,r3
        isync				// cancel speculative execution
        beqlr+				// we return void
        b		1b
2:
        subic.	r5,r5,1		// try again before relinquish?
        bne		1b
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)


spinlock_32_lock_up:
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bnea-	_COMM_PAGE_RELINQUISH	// always depress on UP (let lock owner run)
        stwcx.	r6,0,r3
        beqlr+				// we return void
        b		1b

    COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)


spinlock_32_unlock_mp:
        li		r4,0
        isync				// complete prior stores before unlock
		eieio				// (using isync/eieio is faster than a sync)
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)


spinlock_32_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)


spinlock_64_try_mp:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r5
        isync				// cancel speculative execution
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0        // we did not get the lock
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)


spinlock_64_try_up:
		mr		r5, r3
		li		r3, 1
1:
        lwarx	r4,0,r5
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r5
        beqlr++
        b		1b
2:
        li		r6,-4
        stwcx.	r5,r6,r1	// clear the pending reservation (using red zone)
        li		r3,0        // we did not get the lock
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)


spinlock_64_lock_mp:
        li		r5,MP_SPIN_TRIES
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r3
        isync				// cancel speculative execution
        beqlr++				// we return void
        b		1b
2:
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
        subic.	r5,r5,1		// try again before relinquish?
        bne--	1b			// mispredict this one (a cheap back-off)
        ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)


spinlock_64_lock_up:
1:
        lwarx	r4,0,r3
		li		r6,-1		// locked == -1
        cmpwi	r4,0
        bne--	2f
        stwcx.	r6,0,r3
        beqlr++				// we return void
        b		1b
2:							// always relinquish on UP (let lock owner run)
        li		r6,-4
        stwcx.	r3,r6,r1	// clear the pending reservation (using red zone)
		ba		_COMM_PAGE_RELINQUISH

    COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)


spinlock_64_unlock_mp:
        lwsync				// complete prior stores before unlock
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)


spinlock_64_unlock_up:
        li		r4,0
        stw		r4,0(r3)
        blr

    COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
    

spinlock_relinquish:
        mr		r12,r3		// preserve lockword ptr across relinquish
        li		r3,0		// THREAD_NULL
        li		r4,1		// SWITCH_OPTION_DEPRESS
        li		r5,1		// timeout (ms)
        li		r0,-61		// SYSCALL_THREAD_SWITCH
        sc					// relinquish
        mr		r3,r12
        ba		_COMM_PAGE_SPINLOCK_LOCK
        
    COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
e5568f75 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
55e303ae	11	*
e5568f75 A	12	* This Original Code and all software distributed under the License are
e5568f75 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
55e303ae A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
55e303ae A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
55e303ae A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <ppc/asm.h> // EXT, LEXT
	25	#include <machine/cpu_capabilities.h>
	26	#include <machine/commpage.h>
	27
	28	.text
	29	.align 2
55e303ae A	30
	31	#define MP_SPIN_TRIES 1000
	32
	33
91447636 A	34	/* The user mode spinlock library. There are many versions,
	35	* in order to take advantage of a few special cases:
	36	* - no barrier instructions (SYNC,ISYNC) are needed if UP
	37	* - 64-bit processors can use LWSYNC instead of SYNC (if MP)
	38	* - 32-bit processors can use ISYNC/EIEIO instead of SYNC (if MP)
	39	* - branch hints appropriate to the processor (+ vs ++ etc)
	40	* - potentially custom relinquish strategies (not used at present)
	41	* - fixes for errata as necessary
	42	*
	43	* The convention for lockwords is that 0==free and -1==locked.
	44	*/
55e303ae A	45
	46
	47	spinlock_32_try_mp:
91447636 A	48	mr r5, r3
91447636 A	49	li r3, 1
55e303ae A	50	1:
55e303ae A	51	lwarx r4,0,r5
91447636	52	li r6,-1 // locked == -1
55e303ae A	53	cmpwi r4,0
55e303ae A	54	bne- 2f
91447636	55	stwcx. r6,0,r5
55e303ae A	56	isync // cancel speculative execution
	57	beqlr+
	58	b 1b
	59	2:
91447636	60	li r3,0 // we did not get the lock
55e303ae A	61	blr
55e303ae A	62
91447636	63	COMMPAGE_DESCRIPTOR(spinlock_32_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,k64Bit+kUP,kCommPage32)
55e303ae A	64
	65
	66	spinlock_32_try_up:
91447636 A	67	mr r5, r3
91447636 A	68	li r3, 1
55e303ae A	69	1:
55e303ae A	70	lwarx r4,0,r5
91447636	71	li r6,-1 // locked == -1
55e303ae A	72	cmpwi r4,0
55e303ae A	73	bne- 2f
91447636	74	stwcx. r6,0,r5
55e303ae A	75	beqlr+
	76	b 1b
	77	2:
91447636	78	li r3,0 // we did not get the lock
55e303ae A	79	blr
55e303ae A	80
91447636	81	COMMPAGE_DESCRIPTOR(spinlock_32_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,k64Bit,kCommPage32)
55e303ae A	82
	83
	84	spinlock_32_lock_mp:
	85	li r5,MP_SPIN_TRIES
	86	1:
	87	lwarx r4,0,r3
91447636	88	li r6,-1 // locked == -1
55e303ae A	89	cmpwi r4,0
55e303ae A	90	bne- 2f
91447636	91	stwcx. r6,0,r3
55e303ae A	92	isync // cancel speculative execution
	93	beqlr+ // we return void
	94	b 1b
	95	2:
	96	subic. r5,r5,1 // try again before relinquish?
	97	bne 1b
	98	ba _COMM_PAGE_RELINQUISH
	99
91447636	100	COMMPAGE_DESCRIPTOR(spinlock_32_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,k64Bit+kUP,kCommPage32)
55e303ae A	101
	102
	103	spinlock_32_lock_up:
	104	1:
	105	lwarx r4,0,r3
91447636	106	li r6,-1 // locked == -1
55e303ae A	107	cmpwi r4,0
55e303ae A	108	bnea- _COMM_PAGE_RELINQUISH // always depress on UP (let lock owner run)
91447636	109	stwcx. r6,0,r3
55e303ae A	110	beqlr+ // we return void
	111	b 1b
	112
91447636	113	COMMPAGE_DESCRIPTOR(spinlock_32_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,k64Bit,kCommPage32)
55e303ae A	114
	115
	116	spinlock_32_unlock_mp:
	117	li r4,0
91447636 A	118	isync // complete prior stores before unlock
91447636 A	119	eieio // (using isync/eieio is faster than a sync)
55e303ae A	120	stw r4,0(r3)
	121	blr
	122
91447636	123	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,0,k64Bit+kUP,kCommPage32)
55e303ae A	124
	125
	126	spinlock_32_unlock_up:
	127	li r4,0
	128	stw r4,0(r3)
	129	blr
	130
91447636	131	COMMPAGE_DESCRIPTOR(spinlock_32_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,kUP,k64Bit,kCommPage32)
55e303ae A	132
	133
	134	spinlock_64_try_mp:
91447636 A	135	mr r5, r3
91447636 A	136	li r3, 1
55e303ae A	137	1:
55e303ae A	138	lwarx r4,0,r5
91447636	139	li r6,-1 // locked == -1
55e303ae A	140	cmpwi r4,0
55e303ae A	141	bne-- 2f
91447636	142	stwcx. r6,0,r5
55e303ae A	143	isync // cancel speculative execution
	144	beqlr++
	145	b 1b
	146	2:
	147	li r6,-4
	148	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
91447636	149	li r3,0 // we did not get the lock
55e303ae A	150	blr
55e303ae A	151
91447636	152	COMMPAGE_DESCRIPTOR(spinlock_64_try_mp,_COMM_PAGE_SPINLOCK_TRY,k64Bit,kUP,kCommPageBoth)
55e303ae A	153
	154
	155	spinlock_64_try_up:
91447636 A	156	mr r5, r3
91447636 A	157	li r3, 1
55e303ae A	158	1:
55e303ae A	159	lwarx r4,0,r5
91447636	160	li r6,-1 // locked == -1
55e303ae A	161	cmpwi r4,0
55e303ae A	162	bne-- 2f
91447636	163	stwcx. r6,0,r5
55e303ae A	164	beqlr++
	165	b 1b
	166	2:
	167	li r6,-4
	168	stwcx. r5,r6,r1 // clear the pending reservation (using red zone)
91447636	169	li r3,0 // we did not get the lock
55e303ae A	170	blr
55e303ae A	171
91447636	172	COMMPAGE_DESCRIPTOR(spinlock_64_try_up,_COMM_PAGE_SPINLOCK_TRY,k64Bit+kUP,0,kCommPageBoth)
55e303ae A	173
	174
	175	spinlock_64_lock_mp:
	176	li r5,MP_SPIN_TRIES
	177	1:
	178	lwarx r4,0,r3
91447636	179	li r6,-1 // locked == -1
55e303ae A	180	cmpwi r4,0
55e303ae A	181	bne-- 2f
91447636	182	stwcx. r6,0,r3
55e303ae A	183	isync // cancel speculative execution
	184	beqlr++ // we return void
	185	b 1b
	186	2:
	187	li r6,-4
	188	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
	189	subic. r5,r5,1 // try again before relinquish?
	190	bne-- 1b // mispredict this one (a cheap back-off)
	191	ba _COMM_PAGE_RELINQUISH
	192
91447636	193	COMMPAGE_DESCRIPTOR(spinlock_64_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,k64Bit,kUP,kCommPageBoth)
55e303ae A	194
	195
	196	spinlock_64_lock_up:
	197	1:
	198	lwarx r4,0,r3
91447636	199	li r6,-1 // locked == -1
55e303ae A	200	cmpwi r4,0
55e303ae A	201	bne-- 2f
91447636	202	stwcx. r6,0,r3
55e303ae A	203	beqlr++ // we return void
	204	b 1b
	205	2: // always relinquish on UP (let lock owner run)
	206	li r6,-4
	207	stwcx. r3,r6,r1 // clear the pending reservation (using red zone)
	208	ba _COMM_PAGE_RELINQUISH
	209
91447636	210	COMMPAGE_DESCRIPTOR(spinlock_64_lock_up,_COMM_PAGE_SPINLOCK_LOCK,k64Bit+kUP,0,kCommPageBoth)
55e303ae A	211
	212
	213	spinlock_64_unlock_mp:
55e303ae	214	lwsync // complete prior stores before unlock
91447636	215	li r4,0
55e303ae A	216	stw r4,0(r3)
	217	blr
	218
91447636	219	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_mp,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit,kUP,kCommPageBoth)
55e303ae A	220
	221
	222	spinlock_64_unlock_up:
	223	li r4,0
	224	stw r4,0(r3)
	225	blr
	226
91447636	227	COMMPAGE_DESCRIPTOR(spinlock_64_unlock_up,_COMM_PAGE_SPINLOCK_UNLOCK,k64Bit+kUP,0,kCommPageBoth)
55e303ae A	228
	229
	230	spinlock_relinquish:
	231	mr r12,r3 // preserve lockword ptr across relinquish
	232	li r3,0 // THREAD_NULL
	233	li r4,1 // SWITCH_OPTION_DEPRESS
	234	li r5,1 // timeout (ms)
	235	li r0,-61 // SYSCALL_THREAD_SWITCH
	236	sc // relinquish
	237	mr r3,r12
	238	ba _COMM_PAGE_SPINLOCK_LOCK
	239
91447636	240	COMMPAGE_DESCRIPTOR(spinlock_relinquish,_COMM_PAGE_RELINQUISH,0,0,kCommPageBoth)
55e303ae	241