[apple/xnu.git] / osfmk / ppc / commpage / atomic.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>					// EXT, LEXT
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>


/* OSAtomic.h library native implementations. */

        .text
        .align	2

atomic_add32:						// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
1:
		lwarx   r5,0,r4
		add		r6,r3,r5
		stwcx.  r6,0,r4
		bne--   1b
		mr		r3,r6
		blr
		
    COMMPAGE_DESCRIPTOR(atomic_add32,_COMM_PAGE_ATOMIC_ADD32,0,0,kCommPageBoth)


atomic_add64:						// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
1:
		ldarx   r5,0,r4
		add		r6,r3,r5
		stdcx.  r6,0,r4
		bne--   1b
		mr		r3,r6
		blr
		
    COMMPAGE_DESCRIPTOR(atomic_add64,_COMM_PAGE_ATOMIC_ADD64,k64Bit,0,kCommPage64)

/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This is the no-barrier version */
compare_and_swap32_on32:			// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
1:
		lwarx   r7,0,r5
		cmplw   r7,r3
		bne-	2f
		stwcx.  r4,0,r5
		bne-	1b
		li		r3,1
		blr
2:
		li		r3,0				// return failure
		blr

    COMMPAGE_DESCRIPTOR(compare_and_swap32_on32,_COMM_PAGE_COMPARE_AND_SWAP32,0,k64Bit,kCommPageBoth)


/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This is the no-barrier version */
compare_and_swap32_on64:			// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
1:
		lwarx   r7,0,r5
		cmplw   r7,r3
		bne--	2f
		stwcx.  r4,0,r5
		bne--	1b
		li		r3,1
		blr
2:
		li		r8,-8				// on 970, must release reservation
		li		r3,0				// return failure
		stwcx.  r4,r8,r1			// store into red zone to release
		blr

    COMMPAGE_DESCRIPTOR(compare_and_swap32_on64,_COMM_PAGE_COMPARE_AND_SWAP32,k64Bit,0,kCommPageBoth)


/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This is the no-barrier version */
compare_and_swap64:					// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
1:
		ldarx   r7,0,r5
		cmpld   r7,r3
		bne--	2f
		stdcx.  r4,0,r5
		bne--	1b
		li		r3,1
		blr
2:
		li		r8,-8				// on 970, must release reservation
		li		r3,0				// return failure
		stdcx.  r4,r8,r1			// store into red zone to release
		blr

    COMMPAGE_DESCRIPTOR(compare_and_swap64,_COMM_PAGE_COMPARE_AND_SWAP64,k64Bit,0,kCommPage64)

/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This version of compare-and-swap incorporates a memory barrier. */
compare_and_swap32_on32b:			// bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value);
        eieio                       // write barrier, NOP'd on a UP
1:
		lwarx   r7,0,r5
		cmplw   r7,r3
		bne-	2f
		stwcx.  r4,0,r5
		bne-	1b
        isync                       // read barrier, NOP'd on a UP
		li		r3,1
		blr
2:
		li		r3,0				// return failure
		blr

    COMMPAGE_DESCRIPTOR(compare_and_swap32_on32b,_COMM_PAGE_COMPARE_AND_SWAP32B,0,k64Bit,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)


/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This version of compare-and-swap incorporates a memory barrier. */
compare_and_swap32_on64b:			// bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value);
        lwsync                      // write barrier, NOP'd on a UP
1:
		lwarx   r7,0,r5
		cmplw   r7,r3
		bne--	2f
		stwcx.  r4,0,r5
		bne--	1b
        isync                       // read barrier, NOP'd on a UP
		li		r3,1
		blr
2:
		li		r8,-8				// on 970, must release reservation
		li		r3,0				// return failure
		stwcx.  r4,r8,r1			// store into red zone to release
		blr

    COMMPAGE_DESCRIPTOR(compare_and_swap32_on64b,_COMM_PAGE_COMPARE_AND_SWAP32B,k64Bit,0,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)


/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This version of compare-and-swap incorporates a memory barrier. */
compare_and_swap64b:				// bool OSAtomicCompareAndSwapBarrier64( int64_t old, int64_t new, int64_t *value);
        lwsync                      // write barrier, NOP'd on a UP
1:
		ldarx   r7,0,r5
		cmpld   r7,r3
		bne--	2f
		stdcx.  r4,0,r5
		bne--	1b
        isync                       // read barrier, NOP'd on a UP
		li		r3,1
		blr
2:
		li		r8,-8				// on 970, must release reservation
		li		r3,0				// return failure
		stdcx.  r4,r8,r1			// store into red zone to release
		blr

    COMMPAGE_DESCRIPTOR(compare_and_swap64b,_COMM_PAGE_COMPARE_AND_SWAP64B,k64Bit,0,kCommPage64+kCommPageSYNC+kCommPageISYNC)


atomic_enqueue32:					// void OSAtomicEnqueue( void **list, void *new, size_t offset);
1:
		lwarx   r6,0,r3				// get link to 1st on list
		stwx	r6,r4,r5			// hang list off new node
		eieio						// make sure the "stwx" comes before "stwcx." (nop'd on UP)
		stwcx.  r4,0,r3				// make new 1st on list
		beqlr++
		b		1b
		
    COMMPAGE_DESCRIPTOR(atomic_enqueue32,_COMM_PAGE_ENQUEUE,0,0,kCommPageSYNC+kCommPage32)


atomic_enqueue64:					// void OSAtomicEnqueue( void **list, void *new, size_t offset);
1:
		ldarx   r6,0,r3				// get link to 1st on list
		stdx	r6,r4,r5			// hang list off new node
		lwsync						// make sure the "stdx" comes before the "stdcx." (nop'd on UP)
		stdcx.  r4,0,r3				// make new 1st on list
		beqlr++
		b		1b
		
    COMMPAGE_DESCRIPTOR(atomic_enqueue64,_COMM_PAGE_ENQUEUE,k64Bit,0,kCommPageSYNC+kCommPage64)


atomic_dequeue32_on32:              // void* OSAtomicDequeue( void **list, size_t offset);
        mr      r5,r3
1:
		lwarx   r3,0,r5				// get 1st in list
        cmpwi   r3,0                // null?
        beqlr                       // yes, list empty
		lwzx	r6,r3,r4			// get 2nd
		stwcx.  r6,0,r5				// make 2nd first
		bne--	1b
		isync						// cancel read-aheads (nop'd on UP)
		blr

    COMMPAGE_DESCRIPTOR(atomic_dequeue32_on32,_COMM_PAGE_DEQUEUE,0,k64Bit,kCommPageISYNC+kCommPage32)


atomic_dequeue32_on64:              // void* OSAtomicDequeue( void **list, size_t offset);
        mr      r5,r3
        li      r7,-8               // use red zone to release reservation if necessary
1:
		lwarx   r3,0,r5				// get 1st in list
        cmpwi   r3,0                // null?
        beq     2f
		lwzx	r6,r3,r4			// get 2nd
		stwcx.  r6,0,r5				// make 2nd first
		isync						// cancel read-aheads (nop'd on UP)
		beqlr++                     // return next element in r2
        b       1b                  // retry (lost reservation)
2:
        stwcx.  r0,r7,r1            // on 970, release reservation using red zone
		blr                         // return null

    COMMPAGE_DESCRIPTOR(atomic_dequeue32_on64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage32)


atomic_dequeue64:					// void* OSAtomicDequeue( void **list, size_t offset);
        mr      r5,r3
        li      r7,-8               // use red zone to release reservation if necessary
1:
		ldarx   r3,0,r5				// get 1st in list
        cmpdi   r3,0                // null?
        beq     2f
		ldx     r6,r3,r4			// get 2nd
		stdcx.  r6,0,r5				// make 2nd first
		isync						// cancel read-aheads (nop'd on UP)
		beqlr++                     // return next element in r2
        b       1b                  // retry (lost reservation)
2:
        stdcx.  r0,r7,r1            // on 970, release reservation using red zone
		blr                         // return null

    COMMPAGE_DESCRIPTOR(atomic_dequeue64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage64)


memory_barrier_up:					// void OSMemoryBarrier( void )
		blr							// nothing to do on UP
		
    COMMPAGE_DESCRIPTOR(memory_barrier_up,_COMM_PAGE_MEMORY_BARRIER,kUP,0,kCommPageBoth)


memory_barrier_mp32:				// void OSMemoryBarrier( void )
		isync						// we use eieio in preference to sync...
		eieio						// ...because it is faster
		blr
		
    COMMPAGE_DESCRIPTOR(memory_barrier_mp32,_COMM_PAGE_MEMORY_BARRIER,0,kUP+k64Bit,kCommPage32)


memory_barrier_mp64:				// void OSMemoryBarrier( void )
		isync
		lwsync						// on 970, lwsync is faster than eieio
		blr
		
    COMMPAGE_DESCRIPTOR(memory_barrier_mp64,_COMM_PAGE_MEMORY_BARRIER,k64Bit,kUP,kCommPageBoth)
Commit	Line	Data
91447636 A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
	11	*
	12	* This Original Code and all software distributed under the License are
	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <ppc/asm.h> // EXT, LEXT
	25	#include <machine/cpu_capabilities.h>
	26	#include <machine/commpage.h>
	27
	28
	29	/* OSAtomic.h library native implementations. */
	30
	31	.text
	32	.align 2
	33
	34	atomic_add32: // int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
	35	1:
	36	lwarx r5,0,r4
	37	add r6,r3,r5
	38	stwcx. r6,0,r4
	39	bne-- 1b
	40	mr r3,r6
	41	blr
	42
	43	COMMPAGE_DESCRIPTOR(atomic_add32,_COMM_PAGE_ATOMIC_ADD32,0,0,kCommPageBoth)
	44
	45
	46	atomic_add64: // int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
	47	1:
	48	ldarx r5,0,r4
	49	add r6,r3,r5
	50	stdcx. r6,0,r4
	51	bne-- 1b
	52	mr r3,r6
	53	blr
	54
	55	COMMPAGE_DESCRIPTOR(atomic_add64,_COMM_PAGE_ATOMIC_ADD64,k64Bit,0,kCommPage64)
	56
	57	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	58	/* This is the no-barrier version */
	59	compare_and_swap32_on32: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
	60	1:
	61	lwarx r7,0,r5
	62	cmplw r7,r3
	63	bne- 2f
	64	stwcx. r4,0,r5
65	bne- 1b
66	li r3,1
67	blr
68	2:
69	li r3,0 // return failure
70	blr
71
72	COMMPAGE_DESCRIPTOR(compare_and_swap32_on32,_COMM_PAGE_COMPARE_AND_SWAP32,0,k64Bit,kCommPageBoth)
73
74
75	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
76	/* This is the no-barrier version */
77	compare_and_swap32_on64: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
78	1:
79	lwarx r7,0,r5
80	cmplw r7,r3
81	bne-- 2f
82	stwcx. r4,0,r5
83	bne-- 1b
84	li r3,1
85	blr
86	2:
87	li r8,-8 // on 970, must release reservation
88	li r3,0 // return failure
89	stwcx. r4,r8,r1 // store into red zone to release
90	blr
91
92	COMMPAGE_DESCRIPTOR(compare_and_swap32_on64,_COMM_PAGE_COMPARE_AND_SWAP32,k64Bit,0,kCommPageBoth)
93
94
95	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
96	/* This is the no-barrier version */
97	compare_and_swap64: // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
98	1:
99	ldarx r7,0,r5
100	cmpld r7,r3
101	bne-- 2f
102	stdcx. r4,0,r5
103	bne-- 1b
104	li r3,1
105	blr
106	2:
107	li r8,-8 // on 970, must release reservation
108	li r3,0 // return failure
109	stdcx. r4,r8,r1 // store into red zone to release
110	blr
111
112	COMMPAGE_DESCRIPTOR(compare_and_swap64,_COMM_PAGE_COMPARE_AND_SWAP64,k64Bit,0,kCommPage64)
113
114	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
115	/* This version of compare-and-swap incorporates a memory barrier. */
116	compare_and_swap32_on32b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value);
117	eieio // write barrier, NOP'd on a UP
118	1:
119	lwarx r7,0,r5
120	cmplw r7,r3
121	bne- 2f
122	stwcx. r4,0,r5
123	bne- 1b
124	isync // read barrier, NOP'd on a UP
125	li r3,1
126	blr
127	2:
128	li r3,0 // return failure
129	blr
130
131	COMMPAGE_DESCRIPTOR(compare_and_swap32_on32b,_COMM_PAGE_COMPARE_AND_SWAP32B,0,k64Bit,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)
132
133
134	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
135	/* This version of compare-and-swap incorporates a memory barrier. */
136	compare_and_swap32_on64b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value);
137	lwsync // write barrier, NOP'd on a UP
138	1:
139	lwarx r7,0,r5
140	cmplw r7,r3
141	bne-- 2f
142	stwcx. r4,0,r5
143	bne-- 1b
144	isync // read barrier, NOP'd on a UP
145	li r3,1
146	blr
147	2:
148	li r8,-8 // on 970, must release reservation
149	li r3,0 // return failure
150	stwcx. r4,r8,r1 // store into red zone to release
151	blr
152
153	COMMPAGE_DESCRIPTOR(compare_and_swap32_on64b,_COMM_PAGE_COMPARE_AND_SWAP32B,k64Bit,0,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)
154
155
156	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
157	/* This version of compare-and-swap incorporates a memory barrier. */
158	compare_and_swap64b: // bool OSAtomicCompareAndSwapBarrier64( int64_t old, int64_t new, int64_t *value);
159	lwsync // write barrier, NOP'd on a UP
160	1:
161	ldarx r7,0,r5
162	cmpld r7,r3
163	bne-- 2f
164	stdcx. r4,0,r5
165	bne-- 1b
166	isync // read barrier, NOP'd on a UP
167	li r3,1
168	blr
169	2:
170	li r8,-8 // on 970, must release reservation
171	li r3,0 // return failure
172	stdcx. r4,r8,r1 // store into red zone to release
173	blr
174
175	COMMPAGE_DESCRIPTOR(compare_and_swap64b,_COMM_PAGE_COMPARE_AND_SWAP64B,k64Bit,0,kCommPage64+kCommPageSYNC+kCommPageISYNC)
176
177
178	atomic_enqueue32: // void OSAtomicEnqueue( void *list, void new, size_t offset);
179	1:
180	lwarx r6,0,r3 // get link to 1st on list
181	stwx r6,r4,r5 // hang list off new node
182	eieio // make sure the "stwx" comes before "stwcx." (nop'd on UP)
183	stwcx. r4,0,r3 // make new 1st on list
184	beqlr++
185	b 1b
186
187	COMMPAGE_DESCRIPTOR(atomic_enqueue32,_COMM_PAGE_ENQUEUE,0,0,kCommPageSYNC+kCommPage32)
188
189
190	atomic_enqueue64: // void OSAtomicEnqueue( void *list, void new, size_t offset);
191	1:
192	ldarx r6,0,r3 // get link to 1st on list
193	stdx r6,r4,r5 // hang list off new node
194	lwsync // make sure the "stdx" comes before the "stdcx." (nop'd on UP)
195	stdcx. r4,0,r3 // make new 1st on list
196	beqlr++
197	b 1b
198
199	COMMPAGE_DESCRIPTOR(atomic_enqueue64,_COMM_PAGE_ENQUEUE,k64Bit,0,kCommPageSYNC+kCommPage64)
200
201
202	atomic_dequeue32_on32: // void* OSAtomicDequeue( void **list, size_t offset);
203	mr r5,r3
204	1:
205	lwarx r3,0,r5 // get 1st in list
206	cmpwi r3,0 // null?
207	beqlr // yes, list empty
208	lwzx r6,r3,r4 // get 2nd
209	stwcx. r6,0,r5 // make 2nd first
210	bne-- 1b
211	isync // cancel read-aheads (nop'd on UP)
212	blr
213
214	COMMPAGE_DESCRIPTOR(atomic_dequeue32_on32,_COMM_PAGE_DEQUEUE,0,k64Bit,kCommPageISYNC+kCommPage32)
215
216
217	atomic_dequeue32_on64: // void* OSAtomicDequeue( void **list, size_t offset);
218	mr r5,r3
219	li r7,-8 // use red zone to release reservation if necessary
220	1:
221	lwarx r3,0,r5 // get 1st in list
222	cmpwi r3,0 // null?
223	beq 2f
224	lwzx r6,r3,r4 // get 2nd
225	stwcx. r6,0,r5 // make 2nd first
226	isync // cancel read-aheads (nop'd on UP)
227	beqlr++ // return next element in r2
228	b 1b // retry (lost reservation)
229	2:
230	stwcx. r0,r7,r1 // on 970, release reservation using red zone
231	blr // return null
232
233	COMMPAGE_DESCRIPTOR(atomic_dequeue32_on64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage32)
234
235
236	atomic_dequeue64: // void* OSAtomicDequeue( void **list, size_t offset);
237	mr r5,r3
238	li r7,-8 // use red zone to release reservation if necessary
239	1:
240	ldarx r3,0,r5 // get 1st in list
241	cmpdi r3,0 // null?
242	beq 2f
243	ldx r6,r3,r4 // get 2nd
244	stdcx. r6,0,r5 // make 2nd first
245	isync // cancel read-aheads (nop'd on UP)
246	beqlr++ // return next element in r2
247	b 1b // retry (lost reservation)
248	2:
249	stdcx. r0,r7,r1 // on 970, release reservation using red zone
250	blr // return null
251
252	COMMPAGE_DESCRIPTOR(atomic_dequeue64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage64)
253
254
255	memory_barrier_up: // void OSMemoryBarrier( void )
256	blr // nothing to do on UP
257
258	COMMPAGE_DESCRIPTOR(memory_barrier_up,_COMM_PAGE_MEMORY_BARRIER,kUP,0,kCommPageBoth)
259
260
261	memory_barrier_mp32: // void OSMemoryBarrier( void )
262	isync // we use eieio in preference to sync...
263	eieio // ...because it is faster
264	blr
265
266	COMMPAGE_DESCRIPTOR(memory_barrier_mp32,_COMM_PAGE_MEMORY_BARRIER,0,kUP+k64Bit,kCommPage32)
267
268
269	memory_barrier_mp64: // void OSMemoryBarrier( void )
270	isync
271	lwsync // on 970, lwsync is faster than eieio
272	blr
273
274	COMMPAGE_DESCRIPTOR(memory_barrier_mp64,_COMM_PAGE_MEMORY_BARRIER,k64Bit,kUP,kCommPageBoth)