git.saurik.com Git - apple/xnu.git/blame_incremental

... / ...

Commit	Line	Data
	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <sys/appleapiopts.h>
	30	#include <ppc/asm.h> // EXT, LEXT
	31	#include <machine/cpu_capabilities.h>
	32	#include <machine/commpage.h>
	33
	34
	35	/* OSAtomic.h library native implementations. */
	36
	37	.text
	38	.align 2
	39
	40	atomic_add32: // int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
	41	1:
	42	lwarx r5,0,r4
	43	add r6,r3,r5
	44	stwcx. r6,0,r4
	45	bne-- 1b
	46	mr r3,r6
	47	blr
	48
	49	COMMPAGE_DESCRIPTOR(atomic_add32,_COMM_PAGE_ATOMIC_ADD32,0,0,kCommPageBoth)
	50
	51
	52	atomic_add64: // int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
	53	1:
	54	ldarx r5,0,r4
	55	add r6,r3,r5
	56	stdcx. r6,0,r4
	57	bne-- 1b
	58	mr r3,r6
	59	blr
	60
	61	COMMPAGE_DESCRIPTOR(atomic_add64,_COMM_PAGE_ATOMIC_ADD64,k64Bit,0,kCommPage64)
	62
	63	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	64	/* This is the no-barrier version */
	65	compare_and_swap32_on32: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
	66	1:
	67	lwarx r7,0,r5
	68	cmplw r7,r3
	69	bne- 2f
	70	stwcx. r4,0,r5
	71	bne- 1b
	72	li r3,1
	73	blr
	74	2:
	75	li r3,0 // return failure
	76	blr
	77
	78	COMMPAGE_DESCRIPTOR(compare_and_swap32_on32,_COMM_PAGE_COMPARE_AND_SWAP32,0,k64Bit,kCommPageBoth)
	79
	80
	81	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	82	/* This is the no-barrier version */
	83	compare_and_swap32_on64: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
	84	1:
	85	lwarx r7,0,r5
	86	cmplw r7,r3
	87	bne-- 2f
	88	stwcx. r4,0,r5
	89	bne-- 1b
	90	li r3,1
	91	blr
	92	2:
	93	li r8,-8 // on 970, must release reservation
	94	li r3,0 // return failure
	95	stwcx. r4,r8,r1 // store into red zone to release
	96	blr
	97
	98	COMMPAGE_DESCRIPTOR(compare_and_swap32_on64,_COMM_PAGE_COMPARE_AND_SWAP32,k64Bit,0,kCommPageBoth)
	99
	100
	101	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	102	/* This is the no-barrier version */
	103	compare_and_swap64: // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
	104	1:
	105	ldarx r7,0,r5
	106	cmpld r7,r3
	107	bne-- 2f
	108	stdcx. r4,0,r5
	109	bne-- 1b
	110	li r3,1
	111	blr
	112	2:
	113	li r8,-8 // on 970, must release reservation
	114	li r3,0 // return failure
	115	stdcx. r4,r8,r1 // store into red zone to release
	116	blr
	117
	118	COMMPAGE_DESCRIPTOR(compare_and_swap64,_COMM_PAGE_COMPARE_AND_SWAP64,k64Bit,0,kCommPage64)
	119
	120	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	121	/* This version of compare-and-swap incorporates a memory barrier. */
	122	compare_and_swap32_on32b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value);
	123	eieio // write barrier, NOP'd on a UP
	124	1:
	125	lwarx r7,0,r5
	126	cmplw r7,r3
	127	bne- 2f
	128	stwcx. r4,0,r5
	129	bne- 1b
	130	isync // read barrier, NOP'd on a UP
	131	li r3,1
	132	blr
	133	2:
	134	li r3,0 // return failure
	135	blr
	136
	137	COMMPAGE_DESCRIPTOR(compare_and_swap32_on32b,_COMM_PAGE_COMPARE_AND_SWAP32B,0,k64Bit,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)
	138
	139
	140	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	141	/* This version of compare-and-swap incorporates a memory barrier. */
	142	compare_and_swap32_on64b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value);
	143	lwsync // write barrier, NOP'd on a UP
	144	1:
	145	lwarx r7,0,r5
	146	cmplw r7,r3
	147	bne-- 2f
	148	stwcx. r4,0,r5
	149	bne-- 1b
	150	isync // read barrier, NOP'd on a UP
	151	li r3,1
	152	blr
	153	2:
	154	li r8,-8 // on 970, must release reservation
	155	li r3,0 // return failure
	156	stwcx. r4,r8,r1 // store into red zone to release
	157	blr
	158
	159	COMMPAGE_DESCRIPTOR(compare_and_swap32_on64b,_COMM_PAGE_COMPARE_AND_SWAP32B,k64Bit,0,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)
	160
	161
	162	/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
	163	/* This version of compare-and-swap incorporates a memory barrier. */
	164	compare_and_swap64b: // bool OSAtomicCompareAndSwapBarrier64( int64_t old, int64_t new, int64_t *value);
	165	lwsync // write barrier, NOP'd on a UP
	166	1:
	167	ldarx r7,0,r5
	168	cmpld r7,r3
	169	bne-- 2f
	170	stdcx. r4,0,r5
	171	bne-- 1b
	172	isync // read barrier, NOP'd on a UP
	173	li r3,1
	174	blr
	175	2:
	176	li r8,-8 // on 970, must release reservation
	177	li r3,0 // return failure
	178	stdcx. r4,r8,r1 // store into red zone to release
	179	blr
	180
	181	COMMPAGE_DESCRIPTOR(compare_and_swap64b,_COMM_PAGE_COMPARE_AND_SWAP64B,k64Bit,0,kCommPage64+kCommPageSYNC+kCommPageISYNC)
	182
	183
	184	atomic_enqueue32: // void OSAtomicEnqueue( void *list, void new, size_t offset);
	185	1:
	186	lwarx r6,0,r3 // get link to 1st on list
	187	stwx r6,r4,r5 // hang list off new node
	188	eieio // make sure the "stwx" comes before "stwcx." (nop'd on UP)
	189	stwcx. r4,0,r3 // make new 1st on list
	190	beqlr++
	191	b 1b
	192
	193	COMMPAGE_DESCRIPTOR(atomic_enqueue32,_COMM_PAGE_ENQUEUE,0,0,kCommPageSYNC+kCommPage32)
	194
	195
	196	atomic_enqueue64: // void OSAtomicEnqueue( void *list, void new, size_t offset);
	197	1:
	198	ldarx r6,0,r3 // get link to 1st on list
	199	stdx r6,r4,r5 // hang list off new node
	200	lwsync // make sure the "stdx" comes before the "stdcx." (nop'd on UP)
	201	stdcx. r4,0,r3 // make new 1st on list
	202	beqlr++
	203	b 1b
	204
	205	COMMPAGE_DESCRIPTOR(atomic_enqueue64,_COMM_PAGE_ENQUEUE,k64Bit,0,kCommPageSYNC+kCommPage64)
	206
	207
	208	atomic_dequeue32_on32: // void* OSAtomicDequeue( void **list, size_t offset);
	209	mr r5,r3
	210	1:
	211	lwarx r3,0,r5 // get 1st in list
	212	cmpwi r3,0 // null?
	213	beqlr // yes, list empty
	214	lwzx r6,r3,r4 // get 2nd
	215	stwcx. r6,0,r5 // make 2nd first
	216	bne-- 1b
	217	isync // cancel read-aheads (nop'd on UP)
	218	blr
	219
	220	COMMPAGE_DESCRIPTOR(atomic_dequeue32_on32,_COMM_PAGE_DEQUEUE,0,k64Bit,kCommPageISYNC+kCommPage32)
	221
	222
	223	atomic_dequeue32_on64: // void* OSAtomicDequeue( void **list, size_t offset);
	224	mr r5,r3
	225	li r7,-8 // use red zone to release reservation if necessary
	226	1:
	227	lwarx r3,0,r5 // get 1st in list
	228	cmpwi r3,0 // null?
	229	beq 2f
	230	lwzx r6,r3,r4 // get 2nd
	231	stwcx. r6,0,r5 // make 2nd first
	232	isync // cancel read-aheads (nop'd on UP)
	233	beqlr++ // return next element in r2
	234	b 1b // retry (lost reservation)
	235	2:
	236	stwcx. r0,r7,r1 // on 970, release reservation using red zone
	237	blr // return null
	238
	239	COMMPAGE_DESCRIPTOR(atomic_dequeue32_on64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage32)
	240
	241
	242	atomic_dequeue64: // void* OSAtomicDequeue( void **list, size_t offset);
	243	mr r5,r3
	244	li r7,-8 // use red zone to release reservation if necessary
	245	1:
	246	ldarx r3,0,r5 // get 1st in list
	247	cmpdi r3,0 // null?
	248	beq 2f
	249	ldx r6,r3,r4 // get 2nd
	250	stdcx. r6,0,r5 // make 2nd first
	251	isync // cancel read-aheads (nop'd on UP)
	252	beqlr++ // return next element in r2
	253	b 1b // retry (lost reservation)
	254	2:
	255	stdcx. r0,r7,r1 // on 970, release reservation using red zone
	256	blr // return null
	257
	258	COMMPAGE_DESCRIPTOR(atomic_dequeue64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage64)
	259
	260
	261	memory_barrier_up: // void OSMemoryBarrier( void )
	262	blr // nothing to do on UP
	263
	264	COMMPAGE_DESCRIPTOR(memory_barrier_up,_COMM_PAGE_MEMORY_BARRIER,kUP,0,kCommPageBoth)
	265
	266
	267	memory_barrier_mp32: // void OSMemoryBarrier( void )
	268	isync // we use eieio in preference to sync...
	269	eieio // ...because it is faster
	270	blr
	271
	272	COMMPAGE_DESCRIPTOR(memory_barrier_mp32,_COMM_PAGE_MEMORY_BARRIER,0,kUP+k64Bit,kCommPage32)
	273
	274
	275	memory_barrier_mp64: // void OSMemoryBarrier( void )
	276	isync
	277	lwsync // on 970, lwsync is faster than eieio
	278	blr
	279
	280	COMMPAGE_DESCRIPTOR(memory_barrier_mp64,_COMM_PAGE_MEMORY_BARRIER,k64Bit,kUP,kCommPageBoth)