git.saurik.com Git - apple/xnu.git/blame_incremental

... / ...

Commit	Line	Data
	1	/*
	2	* Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach_assert.h>
	30	#include <mach_ldebug.h>
	31	#include <ppc/asm.h>
	32	#include <ppc/proc_reg.h>
	33	#include <assym.s>
	34
	35
	36	#include <config_dtrace.h>
	37	#if CONFIG_DTRACE
	38	#define LOCKSTAT_LABEL(lab) \
	39	.data __ASMNL__ \
	40	.globl lab __ASMNL__ \
	41	lab: __ASMNL__ \
	42	.long 9f __ASMNL__ \
	43	.text __ASMNL__ \
	44	9: __ASMNL__ \
	45
	46	.globl _dtrace_probe, _lockstat_probemap
	47	#define LOCKSTAT_RECORD(id) \
	48	lis r6,hi16(_lockstat_probemap) __ASMNL__ \
	49	ori r6,r6,lo16(_lockstat_probemap) __ASMNL__ \
	50	lwz r5,4*id(r6) __ASMNL__ \
	51	mr. r5,r5 __ASMNL__ \
	52	beqlr-- __ASMNL__ \
	53	mr r4,r3 __ASMNL__ \
	54	mr r3,r5 __ASMNL__ \
	55	li r5,0 __ASMNL__ \
	56	li r6,0 __ASMNL__ \
	57	li r7,0 __ASMNL__ \
	58	li r8,0 __ASMNL__ \
	59	PROLOG(0) __ASMNL__ \
	60	bl _dtrace_probe __ASMNL__ \
	61	EPILOG
	62	#endif
	63
	64
	65
	66	#define STRING ascii
	67
	68	#define ILK_LOCKED 0x01
	69	#define WAIT_FLAG 0x02
	70	#define WANT_UPGRADE 0x04
	71	#define WANT_EXCL 0x08
	72	#define PRIV_EXCL 0x8000
	73
	74	#define TH_FN_OWNED 0x01
	75
	76	# volatile CR bits
	77	#define hwtimeout 20
	78	#define mlckmiss 21
	79
	80	#define RW_DATA 0
	81
	82	#define PROLOG(space) \
	83	stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \
	84	mfcr r2 __ASMNL__ \
	85	mflr r0 __ASMNL__ \
	86	stw r3,FM_ARG0(r1) __ASMNL__ \
	87	stw r11,FM_ARG0+0x04(r1) __ASMNL__ \
	88	stw r2,(FM_ALIGN(space)+FM_SIZE+FM_CR_SAVE)(r1) __ASMNL__ \
	89	stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__
	90
	91	#define EPILOG \
	92	lwz r1,0(r1) __ASMNL__ \
	93	lwz r0,FM_LR_SAVE(r1) __ASMNL__ \
	94	mtlr r0 __ASMNL__
	95
	96	/*
	97	* void hw_lock_init(hw_lock_t)
	98	*
	99	* Initialize a hardware lock.
	100	*/
	101	.align 5
	102	.globl EXT(hw_lock_init)
	103
	104	LEXT(hw_lock_init)
	105
	106	li r0, 0 ; set lock to free == 0
	107	stw r0, 0(r3) ; Initialize the lock
	108	blr
	109
	110	/*
	111	* unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout)
	112	*
	113	* Try to acquire spin-lock. The second parameter is the bit mask to test and set.
	114	* multiple bits may be set. Return success (1) or failure (0).
	115	* Attempt will fail after timeout ticks of the timebase.
	116	*/
	117	.align 5
	118	.globl EXT(hw_lock_bit)
	119
	120	LEXT(hw_lock_bit)
	121
	122	crset hwtimeout ; timeout option
	123	mr r12,r4 ; Load bit mask
	124	mr r4,r5 ; Load timeout value
	125	b lckcomm ; Join on up...
	126
	127	/*
	128	* void hw_lock_lock(hw_lock_t)
	129	*
	130	* Acquire lock, spinning until it becomes available.
	131	* Return with preemption disabled.
	132	* We will just set a default timeout and jump into the NORMAL timeout lock.
	133	*/
	134	.align 5
	135	.globl EXT(hw_lock_lock)
	136
	137	LEXT(hw_lock_lock)
	138	crclr hwtimeout ; no timeout option
	139	li r4,0 ; request default timeout value
	140	li r12,ILK_LOCKED ; Load bit mask
	141	b lckcomm ; Join on up...
	142
	143	lockDisa:
	144	crset hwtimeout ; timeout option
	145	li r4,0 ; request default timeout value
	146	li r12,ILK_LOCKED ; Load bit mask
	147	b lckcomm ; Join on up...
	148
	149	/*
	150	* unsigned int hw_lock_to(hw_lock_t, unsigned int timeout)
	151	*
	152	* Try to acquire spin-lock. Return success (1) or failure (0).
	153	* Attempt will fail after timeout ticks of the timebase.
	154	* We try fairly hard to get this lock. We disable for interruptions, but
	155	* reenable after a "short" timeout (128 ticks, we may want to change this).
	156	* After checking to see if the large timeout value (passed in) has expired and a
	157	* sufficient number of cycles have gone by (to insure pending 'rupts are taken),
	158	* we return either in abject failure, or disable and go back to the lock sniff routine.
	159	* If the sniffer finds the lock free, it jumps right up and tries to grab it.
	160	*/
	161	.align 5
	162	.globl EXT(hw_lock_to)
	163
	164	LEXT(hw_lock_to)
	165	crset hwtimeout ; timeout option
	166	li r12,ILK_LOCKED ; Load bit mask
	167	lckcomm:
	168	mfsprg r6,1 ; Get the current activation
	169	lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
	170	addi r5,r5,1 ; Bring up the disable count
	171	stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
	172	mr r5,r3 ; Get the address of the lock
	173	li r8,0 ; Set r8 to zero
	174
	175	lcktry: lwarx r6,0,r5 ; Grab the lock value
	176	and. r3,r6,r12 ; Is it locked?
	177	or r6,r6,r12 ; Set interlock
	178	bne-- lckspin ; Yeah, wait for it to clear...
	179	stwcx. r6,0,r5 ; Try to seize that there durn lock
	180	bne-- lcktry ; Couldn't get it...
	181	li r3,1 ; return true
	182	.globl EXT(hwllckPatch_isync)
	183	LEXT(hwllckPatch_isync)
	184	isync ; Make sure we don't use a speculativily loaded value
	185	blr ; Go on home...
	186
	187	lckspin: li r6,lgKillResv ; Get killing field
	188	stwcx. r6,0,r6 ; Kill reservation
	189
	190	mr. r4,r4 ; Test timeout value
	191	bne++ lockspin0
	192	lis r4,hi16(EXT(LockTimeOut)) ; Get the high part
	193	ori r4,r4,lo16(EXT(LockTimeOut)) ; And the low part
	194	lwz r4,0(r4) ; Get the timeout value
	195	lockspin0:
	196	mr. r8,r8 ; Is r8 set to zero
	197	bne++ lockspin1 ; If yes, first spin attempt
	198	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	199	mfmsr r9 ; Get the MSR value
	200	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	201	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	202	andc r9,r9,r0 ; Clear FP and VEC
	203	andc r7,r9,r7 ; Clear EE as well
	204	mtmsr r7 ; Turn off interruptions
	205	isync ; May have turned off vec and fp here
	206	mftb r8 ; Get timestamp on entry
	207	b lcksniff
	208
	209	lockspin1: mtmsr r7 ; Turn off interruptions
	210	mftb r8 ; Get timestamp on entry
	211
	212	lcksniff: lwz r3,0(r5) ; Get that lock in here
	213	and. r3,r3,r12 ; Is it free yet?
	214	beq++ lckretry ; Yeah, try for it again...
	215
	216	mftb r10 ; Time stamp us now
	217	sub r10,r10,r8 ; Get the elapsed time
	218	cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
	219	blt++ lcksniff ; Not yet...
	220
	221	mtmsr r9 ; Say, any interrupts pending?
	222
	223	; The following instructions force the pipeline to be interlocked to that only one
	224	; instruction is issued per cycle. The insures that we stay enabled for a long enough
	225	; time; if it's too short, pending interruptions will not have a chance to be taken
	226
	227	subi r4,r4,128 ; Back off elapsed time from timeout value
	228	or r4,r4,r4 ; Do nothing here but force a single cycle delay
	229	mr. r4,r4 ; See if we used the whole timeout
	230	li r3,0 ; Assume a timeout return code
	231	or r4,r4,r4 ; Do nothing here but force a single cycle delay
	232
	233	ble-- lckfail ; We failed
	234	b lockspin1 ; Now that we've opened an enable window, keep trying...
	235	lckretry:
	236	mtmsr r9 ; Restore interrupt state
	237	li r8,1 ; Insure that R8 is not 0
	238	b lcktry
	239	lckfail: ; We couldn't get the lock
	240	bf hwtimeout,lckpanic
	241	li r3,0 ; Set failure return code
	242	blr ; Return, head hanging low...
	243	lckpanic:
	244	mr r4,r5
	245	mr r5,r3
	246	lis r3,hi16(lckpanic_str) ; Get the failed lck message
	247	ori r3,r3,lo16(lckpanic_str) ; Get the failed lck message
	248	bl EXT(panic)
	249	BREAKPOINT_TRAP ; We die here anyway
	250	.data
	251	lckpanic_str:
	252	STRINGD "timeout on attempt to acquire lock (0x%08X), value = 0x%08X\n\000"
	253	.text
	254
	255	/*
	256	* void hw_lock_unlock(hw_lock_t)
	257	*
	258	* Unconditionally release lock.
	259	* Release preemption level.
	260	*/
	261	.align 5
	262	.globl EXT(hw_lock_unlock)
	263
	264	LEXT(hw_lock_unlock)
	265
	266	.globl EXT(hwulckPatch_isync)
	267	LEXT(hwulckPatch_isync)
	268	isync
	269	.globl EXT(hwulckPatch_eieio)
	270	LEXT(hwulckPatch_eieio)
	271	eieio
	272	li r0, 0 ; set lock to free
	273	stw r0, 0(r3)
	274
	275	b epStart ; Go enable preemption...
	276
	277	/*
	278	* unsigned int hw_unlock_bit(hw_lock_t, unsigned int bit)
	279	*
	280	* Release bit based spin-lock. The second parameter is the bit mask to clear.
	281	* Multiple bits may be cleared.
	282	*
	283	*/
	284	.align 5
	285	.globl EXT(hw_unlock_bit)
	286
	287	LEXT(hw_unlock_bit)
	288
	289	.globl EXT(hwulckbPatch_isync)
	290	LEXT(hwulckbPatch_isync)
	291	isync
	292	.globl EXT(hwulckbPatch_eieio)
	293	LEXT(hwulckbPatch_eieio)
	294	eieio
	295	ubittry: lwarx r0,0,r3 ; Grab the lock value
	296	andc r0,r0,r4 ; Clear the lock bits
	297	stwcx. r0,0,r3 ; Try to clear that there durn lock
	298	bne- ubittry ; Try again, couldn't save it...
	299
	300	b epStart ; Go enable preemption...
	301
	302	/*
	303	* unsigned int hw_lock_mbits(hw_lock_t, unsigned int bits, unsigned int value,
	304	* unsigned int newb, unsigned int timeout)
	305	*
	306	* Try to acquire spin-lock. The second parameter is the bit mask to check.
	307	* The third is the value of those bits and the 4th is what to set them to.
	308	* Return success (1) or failure (0).
	309	* Attempt will fail after timeout ticks of the timebase.
	310	* We try fairly hard to get this lock. We disable for interruptions, but
	311	* reenable after a "short" timeout (128 ticks, we may want to shorten this).
	312	* After checking to see if the large timeout value (passed in) has expired and a
	313	* sufficient number of cycles have gone by (to insure pending 'rupts are taken),
	314	* we return either in abject failure, or disable and go back to the lock sniff routine.
	315	* If the sniffer finds the lock free, it jumps right up and tries to grab it.
	316	*/
	317	.align 5
	318	.globl EXT(hw_lock_mbits)
	319
	320	LEXT(hw_lock_mbits)
	321
	322	li r10,0
	323
	324	mbittry: lwarx r12,0,r3 ; Grab the lock value
	325	and r0,r12,r4 ; Clear extra bits
	326	andc r12,r12,r4 ; Clear all bits in the bit mask
	327	or r12,r12,r6 ; Turn on the lock bits
	328	cmplw r0,r5 ; Are these the right bits?
	329	bne-- mbitspin ; Nope, wait for it to clear...
	330	stwcx. r12,0,r3 ; Try to seize that there durn lock
	331	beq++ mbitgot ; We got it, yahoo...
	332	b mbittry ; Just start up again if the store failed...
	333
	334	.align 5
	335	mbitspin: li r11,lgKillResv ; Point to killing field
	336	stwcx. r11,0,r11 ; Kill it
	337
	338	mr. r10,r10 ; Is r10 set to zero
	339	bne++ mbitspin0 ; If yes, first spin attempt
	340	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	341	mfmsr r9 ; Get the MSR value
	342	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	343	ori r8,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	344	andc r9,r9,r0 ; Clear FP and VEC
	345	andc r8,r9,r8 ; Clear EE as well
	346	mtmsr r8 ; Turn off interruptions
	347	isync ; May have turned off vectors or float here
	348	mftb r10 ; Get the low part of the time base
	349	b mbitsniff
	350	mbitspin0:
	351	mtmsr r8 ; Turn off interruptions
	352	mftb r10 ; Get the low part of the time base
	353	mbitsniff:
	354	lwz r12,0(r3) ; Get that lock in here
	355	and r0,r12,r4 ; Clear extra bits
	356	cmplw r0,r5 ; Are these the right bits?
	357	beq++ mbitretry ; Yeah, try for it again...
	358
	359	mftb r11 ; Time stamp us now
	360	sub r11,r11,r10 ; Get the elapsed time
	361	cmplwi r11,128 ; Have we been spinning for 128 tb ticks?
	362	blt++ mbitsniff ; Not yet...
	363
	364	mtmsr r9 ; Say, any interrupts pending?
	365
	366	; The following instructions force the pipeline to be interlocked to that only one
	367	; instruction is issued per cycle. The insures that we stay enabled for a long enough
	368	; time. If it is too short, pending interruptions will not have a chance to be taken
	369
	370	subi r7,r7,128 ; Back off elapsed time from timeout value
	371	or r7,r7,r7 ; Do nothing here but force a single cycle delay
	372	mr. r7,r7 ; See if we used the whole timeout
	373	or r7,r7,r7 ; Do nothing here but force a single cycle delay
	374
	375	ble-- mbitfail ; We failed
	376	b mbitspin0 ; Now that we have opened an enable window, keep trying...
	377	mbitretry:
	378	mtmsr r9 ; Enable for interruptions
	379	li r10,1 ; Make sure this is non-zero
	380	b mbittry
	381
	382	.align 5
	383	mbitgot:
	384	li r3,1 ; Set good return code
	385	.globl EXT(hwlmlckPatch_isync)
	386	LEXT(hwlmlckPatch_isync)
	387	isync ; Make sure we do not use a speculativily loaded value
	388	blr
	389
	390	mbitfail: li r3,0 ; Set failure return code
	391	blr ; Return, head hanging low...
	392
	393	/*
	394	* unsigned int hw_cpu_sync(unsigned int *, unsigned int timeout)
	395	*
	396	* Spin until word hits 0 or timeout.
	397	* Return success (1) or failure (0).
	398	* Attempt will fail after timeout ticks of the timebase.
	399	*
	400	* The theory is that a processor will bump a counter as it signals
	401	* other processors. Then it will spin untl the counter hits 0 (or
	402	* times out). The other processors, as it receives the signal will
	403	* decrement the counter.
	404	*
	405	* The other processors use interlocked update to decrement, this one
	406	* does not need to interlock.
	407	*/
	408	.align 5
	409	.globl EXT(hw_cpu_sync)
	410
	411	LEXT(hw_cpu_sync)
	412
	413	mftb r10 ; Get the low part of the time base
	414	mr r9,r3 ; Save the sync word address
	415	li r3,1 ; Assume we work
	416
	417	csynctry: lwz r11,0(r9) ; Grab the sync value
	418	mr. r11,r11 ; Counter hit 0?
	419	beqlr- ; Yeah, we are sunk...
	420	mftb r12 ; Time stamp us now
	421
	422	sub r12,r12,r10 ; Get the elapsed time
	423	cmplw r4,r12 ; Have we gone too long?
	424	bge+ csynctry ; Not yet...
	425
	426	li r3,0 ; Set failure...
	427	blr ; Return, head hanging low...
	428
	429	/*
	430	* unsigned int hw_cpu_wcng(unsigned int *, unsigned int, unsigned int timeout)
	431	*
	432	* Spin until word changes or timeout.
	433	* Return success (1) or failure (0).
	434	* Attempt will fail after timeout ticks of the timebase.
	435	*
	436	* This is used to insure that a processor passes a certain point.
	437	* An example of use is to monitor the last interrupt time in the
	438	* per_proc block. This can be used to insure that the other processor
	439	* has seen at least one interrupt since a specific time.
	440	*/
	441	.align 5
	442	.globl EXT(hw_cpu_wcng)
	443
	444	LEXT(hw_cpu_wcng)
	445
	446	mftb r10 ; Get the low part of the time base
	447	mr r9,r3 ; Save the sync word address
	448	li r3,1 ; Assume we work
	449
	450	wcngtry: lwz r11,0(r9) ; Grab the value
	451	cmplw r11,r4 ; Do they still match?
	452	bnelr- ; Nope, cool...
	453	mftb r12 ; Time stamp us now
	454
	455	sub r12,r12,r10 ; Get the elapsed time
	456	cmplw r5,r12 ; Have we gone too long?
	457	bge+ wcngtry ; Not yet...
	458
	459	li r3,0 ; Set failure...
	460	blr ; Return, head hanging low...
	461
	462
	463	/*
	464	* unsigned int hw_lock_try(hw_lock_t)
	465	*
	466	* Try to acquire spin-lock. Return success (1) or failure (0)
	467	* Returns with preemption disabled on success.
	468	*
	469	*/
	470	.align 5
	471	.globl EXT(hw_lock_try)
	472
	473	LEXT(hw_lock_try)
	474
	475	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	476	mfmsr r9 ; Get the MSR value
	477	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	478	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	479	andc r9,r9,r0 ; Clear FP and VEC
	480	andc r7,r9,r7 ; Clear EE as well
	481
	482	mtmsr r7 ; Disable interruptions and thus, preemption
	483
	484	lwz r5,0(r3) ; Quick load
	485	andi. r6,r5,ILK_LOCKED ; TEST...
	486	bne-- .L_lock_try_failed ; No go...
	487
	488	.L_lock_try_loop:
	489	lwarx r5,0,r3 ; Ld from addr of arg and reserve
	490
	491	andi. r6,r5,ILK_LOCKED ; TEST...
	492	ori r5,r5,ILK_LOCKED
	493	bne-- .L_lock_try_failedX ; branch if taken. Predict free
	494
	495	stwcx. r5,0,r3 ; And SET (if still reserved)
	496	bne-- .L_lock_try_loop ; If set failed, loop back
	497
	498	.globl EXT(hwltlckPatch_isync)
	499	LEXT(hwltlckPatch_isync)
	500	isync
	501
	502	mfsprg r6,1 ; Get current activation
	503	lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
	504	addi r5,r5,1 ; Bring up the disable count
	505	stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
	506
	507	mtmsr r9 ; Allow interruptions now
	508	li r3,1 ; Set that the lock was free
	509	blr
	510
	511	.L_lock_try_failedX:
	512	li r6,lgKillResv ; Killing field
	513	stwcx. r6,0,r6 ; Kill reservation
	514
	515	.L_lock_try_failed:
	516	mtmsr r9 ; Allow interruptions now
	517	li r3,0 ; FAILURE - lock was taken
	518	blr
	519
	520	/*
	521	* unsigned int hw_lock_held(hw_lock_t)
	522	*
	523	* Return 1 if lock is held
	524	* Doesn't change preemption state.
	525	* N.B. Racy, of course.
	526	*/
	527	.align 5
	528	.globl EXT(hw_lock_held)
	529
	530	LEXT(hw_lock_held)
	531
	532	isync ; Make sure we don't use a speculativily fetched lock
	533	lwz r3, 0(r3) ; Get lock value
	534	andi. r6,r3,ILK_LOCKED ; Extract the ILK_LOCKED bit
	535	blr
	536
	537	/*
	538	* uint32_t hw_compare_and_store(uint32_t oldval, uint32_t newval, uint32_t *dest)
	539	*
	540	* Compare old to area if equal, store new, and return true
	541	* else return false and no store
	542	* This is an atomic operation
	543	*/
	544	.align 5
	545	.globl EXT(hw_compare_and_store)
	546	.globl EXT(OSCompareAndSwap)
	547	.globl EXT(OSCompareAndSwapPtr)
	548
	549	LEXT(hw_compare_and_store)
	550	LEXT(OSCompareAndSwap)
	551	LEXT(OSCompareAndSwapPtr)
	552
	553	mr r6,r3 ; Save the old value
	554
	555	cstry: lwarx r9,0,r5 ; Grab the area value
	556	li r3,1 ; Assume it works
	557	cmplw cr0,r9,r6 ; Does it match the old value?
	558	bne-- csfail ; No, it must have changed...
	559	stwcx. r4,0,r5 ; Try to save the new value
	560	bne-- cstry ; Didn't get it, try again...
	561	.globl EXT(hwcsatomicPatch_isync)
	562	LEXT(hwcsatomicPatch_isync)
	563	isync ; Just hold up prefetch
	564	blr ; Return...
	565
	566	csfail: li r3,lgKillResv ; Killing field
	567	stwcx. r3,0,r3 ; Blow reservation
	568
	569	li r3,0 ; Set failure
	570	blr ; Better luck next time...
	571
	572
	573	/*
	574	* uint32_t hw_atomic_add(uint32_t *dest, uint32_t delt)
	575	*
	576	* Atomically add the second parameter to the first.
	577	* Returns the result.
	578	*
	579	*/
	580	.align 5
	581	.globl EXT(hw_atomic_add)
	582
	583	LEXT(hw_atomic_add)
	584
	585	mr r6,r3 ; Save the area
	586
	587	addtry: lwarx r3,0,r6 ; Grab the area value
	588	add r3,r3,r4 ; Add the value
	589	stwcx. r3,0,r6 ; Try to save the new value
	590	bne-- addtry ; Didn't get it, try again...
	591	blr ; Return...
	592
	593
	594	/*
	595	* uint32_t hw_atomic_sub(uint32_t *dest, uint32_t delt)
	596	*
	597	* Atomically subtract the second parameter from the first.
	598	* Returns the result.
	599	*
	600	*/
	601	.align 5
	602	.globl EXT(hw_atomic_sub)
	603
	604	LEXT(hw_atomic_sub)
	605
	606	mr r6,r3 ; Save the area
	607
	608	subtry: lwarx r3,0,r6 ; Grab the area value
	609	sub r3,r3,r4 ; Subtract the value
	610	stwcx. r3,0,r6 ; Try to save the new value
	611	bne-- subtry ; Didn't get it, try again...
	612	blr ; Return...
	613
	614
	615	/*
	616	* uint32_t hw_atomic_or(uint32_t *dest, uint32_t mask)
	617	*
	618	* Atomically ORs the second parameter into the first.
	619	* Returns the result.
	620	*/
	621	.align 5
	622	.globl EXT(hw_atomic_or)
	623	LEXT(hw_atomic_or)
	624	.globl EXT(hw_atomic_or_noret)
	625	LEXT(hw_atomic_or_noret)
	626	mr r6,r3 ; Save the area
	627
	628	ortry: lwarx r3,0,r6 ; Grab the area value
	629	or r3,r3,r4 ; OR the value
	630	stwcx. r3,0,r6 ; Try to save the new value
	631	bne-- ortry ; Did not get it, try again...
	632	blr ; Return...
	633
	634
	635	/*
	636	* uint32_t hw_atomic_and(uint32_t *dest, uint32_t mask)
	637	*
	638	* Atomically ANDs the second parameter with the first.
	639	* Returns the result.
	640	*
	641	*/
	642	.align 5
	643	.globl EXT(hw_atomic_and)
	644	LEXT(hw_atomic_and)
	645	.globl EXT(hw_atomic_and_noret)
	646	LEXT(hw_atomic_and_noret)
	647	mr r6,r3 ; Save the area
	648
	649	andtry: lwarx r3,0,r6 ; Grab the area value
	650	and r3,r3,r4 ; AND the value
	651	stwcx. r3,0,r6 ; Try to save the new value
	652	bne-- andtry ; Did not get it, try again...
	653	blr ; Return...
	654
	655
	656	/*
	657	* void hw_queue_atomic(unsigned int * anchor, unsigned int * elem, unsigned int disp)
	658	*
	659	* Atomically inserts the element at the head of the list
	660	* anchor is the pointer to the first element
	661	* element is the pointer to the element to insert
	662	* disp is the displacement into the element to the chain pointer
	663	*/
	664	.align 5
	665	.globl EXT(hw_queue_atomic)
	666	.globl EXT(OSEnqueueAtomic)
	667
	668	LEXT(hw_queue_atomic)
	669	LEXT(OSEnqueueAtomic)
	670
	671	mr r7,r4 ; Make end point the same as start
	672	mr r8,r5 ; Copy the displacement also
	673	b hw_queue_comm ; Join common code...
	674
	675	/*
	676	* void hw_queue_atomic_list(unsigned int * anchor, unsigned int * first, unsigned int * last, unsigned int disp)
	677	*
	678	* Atomically inserts the list of elements at the head of the list
	679	* anchor is the pointer to the first element
	680	* first is the pointer to the first element to insert
	681	* last is the pointer to the last element to insert
	682	* disp is the displacement into the element to the chain pointer
	683	*/
	684	.align 5
	685	.globl EXT(hw_queue_atomic_list)
	686
	687	LEXT(hw_queue_atomic_list)
	688
	689	mr r7,r5 ; Make end point the same as start
	690	mr r8,r6 ; Copy the displacement also
	691
	692	hw_queue_comm:
	693	lwarx r9,0,r3 ; Pick up the anchor
	694	stwx r9,r8,r7 ; Chain that to the end of the new stuff
	695	eieio ; Make sure this store makes it before the anchor update
	696	stwcx. r4,0,r3 ; Try to chain into the front
	697	bne-- hw_queue_comm ; Didn't make it, try again...
	698
	699	blr ; Return...
	700
	701	/*
	702	* unsigned int hw_dequeue_atomic(unsigned int anchor, unsigned int disp)
	703	*
	704	* Atomically removes the first element in a list and returns it.
	705	* anchor is the pointer to the first element
	706	* disp is the displacement into the element to the chain pointer
	707	* Returns element if found, 0 if empty.
	708	*/
	709	.align 5
	710	.globl EXT(hw_dequeue_atomic)
	711	.globl EXT(OSDequeueAtomic)
	712
	713	LEXT(hw_dequeue_atomic)
	714	LEXT(OSDequeueAtomic)
	715
	716	mr r5,r3 ; Save the anchor
	717
	718	hw_dequeue_comm:
	719	lwarx r3,0,r5 ; Pick up the anchor
	720	mr. r3,r3 ; Is the list empty?
	721	beq-- hdcFail ; Leave it list empty...
	722	lwzx r9,r4,r3 ; Get the next in line
	723	stwcx. r9,0,r5 ; Try to chain into the front
	724	beqlr++ ; Got the thing, go away with it...
	725	b hw_dequeue_comm ; Did not make it, try again...
	726
	727	hdcFail: li r4,lgKillResv ; Killing field
	728	stwcx. r4,0,r4 ; Dump reservation
	729	blr ; Leave...
	730
	731
	732	/*
	733	* Routines for mutex lock debugging.
	734	*/
	735
	736	/*
	737	* Gets lock check flags in CR6: CR bits 24-27
	738	*/
	739	#define CHECK_SETUP(rg) \
	740	lbz rg,lglcksWork(0) __ASMNL__ \
	741	mtcrf 2,rg __ASMNL__
	742
	743
	744	/*
	745	* Checks for expected lock type.
	746	*/
	747	#define CHECK_MUTEX_TYPE() \
	748	bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \
	749	bt 24+disLktypeb,1f __ASMNL__ \
	750	lwz r10,MUTEX_TYPE(r3) __ASMNL__ \
	751	cmpwi r10,MUTEX_TAG __ASMNL__ \
	752	beq++ 1f __ASMNL__ \
	753	PROLOG(0) __ASMNL__ \
	754	mr r4,r11 __ASMNL__ \
	755	mr r5,r10 __ASMNL__ \
	756	lis r3,hi16(not_a_mutex) __ASMNL__ \
	757	ori r3,r3,lo16(not_a_mutex) __ASMNL__ \
	758	bl EXT(panic) __ASMNL__ \
	759	BREAKPOINT_TRAP __ASMNL__ \
	760	1:
	761
	762	.data
	763	not_a_mutex:
	764	STRINGD "mutex (0x%08X) not a mutex type (0x%08X)\n\000"
	765	.text
	766
	767	/*
	768	* Verifies return to the correct thread in "unlock" situations.
	769	*/
	770	#define CHECK_THREAD(thread_offset) \
	771	bf MUTEX_ATTR_DEBUGb,3f __ASMNL__ \
	772	bt 24+disLkThreadb,3f __ASMNL__ \
	773	mfsprg r10,1 __ASMNL__ \
	774	lwz r5,MUTEX_DATA(r3) __ASMNL__ \
	775	rlwinm. r9,r5,0,0,29 __ASMNL__ \
	776	bne++ 1f __ASMNL__ \
	777	lis r3,hi16(not_held) __ASMNL__ \
	778	ori r3,r3,lo16(not_held) __ASMNL__ \
	779	b 2f __ASMNL__ \
	780	1: __ASMNL__ \
	781	cmpw r9,r10 __ASMNL__ \
	782	beq++ 3f __ASMNL__ \
	783	mr r5,r10 __ASMNL__ \
	784	mr r6,r9 __ASMNL__ \
	785	lis r3,hi16(wrong_thread) __ASMNL__ \
	786	ori r3,r3,lo16(wrong_thread) __ASMNL__ \
	787	2: __ASMNL__ \
	788	mr r4,r11 __ASMNL__ \
	789	PROLOG(0) __ASMNL__ \
	790	bl EXT(panic) __ASMNL__ \
	791	BREAKPOINT_TRAP __ASMNL__ \
	792	3:
	793
	794	.data
	795	not_held:
	796	STRINGD "mutex (0x%08X) not held\n\000"
	797	wrong_thread:
	798	STRINGD "mutex (0x%08X) unlocked by non-owner(0x%08X), current owner(0x%08X)\n\000"
	799	.text
	800
	801	#define CHECK_MYLOCK() \
	802	bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \
	803	bt 24+disLkMyLckb,1f __ASMNL__ \
	804	mfsprg r10,1 __ASMNL__ \
	805	lwz r9,MUTEX_DATA(r3) __ASMNL__ \
	806	rlwinm r9,r9,0,0,29 __ASMNL__ \
	807	cmpw r9,r10 __ASMNL__ \
	808	bne++ 1f __ASMNL__ \
	809	mr r4,r11 __ASMNL__ \
	810	lis r3, hi16(mylock_attempt) __ASMNL__ \
	811	ori r3,r3,lo16(mylock_attempt) __ASMNL__ \
	812	bl EXT(panic) __ASMNL__ \
	813	BREAKPOINT_TRAP __ASMNL__ \
	814	1:
	815
	816	.data
	817	mylock_attempt:
	818	STRINGD "mutex (0x%08X) recursive lock attempt\n\000"
	819	.text
	820
	821	#define LCK_STACK(lck, stack, lck_stack, frame_cnt, lr_save, tmp) \
	822	bf 24+enaLkExtStckb,3f __ASMNL__ \
	823	addi lck_stack,lck,MUTEX_STACK __ASMNL__ \
	824	li frame_cnt,MUTEX_FRAMES-1 __ASMNL__ \
	825	1: __ASMNL__ \
	826	mr tmp,stack __ASMNL__ \
	827	lwz stack,0(stack) __ASMNL__ \
	828	xor tmp,stack,tmp __ASMNL__ \
	829	cmplwi tmp,8192 __ASMNL__ \
	830	bge-- 2f __ASMNL__ \
	831	lwz lr_save,FM_LR_SAVE(stack) __ASMNL__ \
	832	stwu lr_save,4(lck_stack) __ASMNL__ \
	833	subi frame_cnt,frame_cnt,1 __ASMNL__ \
	834	cmpi cr0,frame_cnt,0 __ASMNL__ \
	835	bne 1b __ASMNL__ \
	836	b 3f __ASMNL__ \
	837	2: __ASMNL__ \
	838	li tmp,0 __ASMNL__ \
	839	stwu tmp,4(lck_stack) __ASMNL__ \
	840	subi frame_cnt,frame_cnt,1 __ASMNL__ \
	841	cmpi cr0,frame_cnt,0 __ASMNL__ \
	842	bne 2b __ASMNL__ \
	843	3:
	844
	845	.align 5
	846	mr r11,r3 ; Save lock addr
	847	mlckeEnter:
	848	lwz r0,MUTEX_ATTR(r3)
	849	mtcrf 1,r0 ; Set cr7
	850	CHECK_SETUP(r12)
	851	CHECK_MUTEX_TYPE()
	852
	853	bf MUTEX_ATTR_DEBUGb,L_mtx_lock_assert_wait_2
	854	PROLOG(0)
	855	bl EXT(assert_wait_possible)
	856	mr. r3,r3
	857	bne L_mtx_lock_assert_wait_1
	858	lis r3,hi16(L_mtx_lock_assert_wait_panic_str)
	859	ori r3,r3,lo16(L_mtx_lock_assert_wait_panic_str)
	860	bl EXT(panic)
	861	BREAKPOINT_TRAP ; We die here anyway
	862
	863	.data
	864	L_mtx_lock_assert_wait_panic_str:
	865	STRINGD "mutex lock attempt with assert_wait_possible false\n\000"
	866	.text
	867
	868	L_mtx_lock_assert_wait_1:
	869	lwz r3,FM_ARG0(r1)
	870	lwz r11,FM_ARG0+0x04(r1)
	871	lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
	872	mtcr r2
	873	EPILOG
	874	L_mtx_lock_assert_wait_2:
	875
	876	mfsprg r6,1 ; load the current thread
	877	bf MUTEX_ATTR_STATb,mlckestatskip ; Branch if no stat
	878	lwz r5,MUTEX_GRP(r3) ; Load lock group
	879	li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset
	880	mlckestatloop:
	881	lwarx r8,r7,r5 ; Load stat util cnt
	882	addi r8,r8,1 ; Increment stat util cnt
	883	stwcx. r8,r7,r5 ; Store stat util cnt
	884	bne-- mlckestatloop ; Retry if failed
	885	mr. r8,r8 ; Test for zero
	886	bne++ mlckestatskip ; Did stat util cnt wrapped?
	887	lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt
	888	addi r8,r8,1 ; Increment upper stat util cnt
	889	stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt
	890	mlckestatskip:
	891	lwz r5,MUTEX_DATA(r3) ; Get the lock quickly
	892	li r4,0
	893	li r8,0
	894	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	895	mfmsr r9 ; Get the MSR value
	896	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	897	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	898	andc r9,r9,r0 ; Clear FP and VEC
	899	andc r7,r9,r7 ; Clear EE as well
	900	mtmsr r7 ; Turn off interruptions
	901	isync ; May have turned off vec and fp here
	902	mr. r5,r5 ; Quick check
	903	bne-- mlckespin01 ; Can not get it right now...
	904
	905	mlcketry:
	906	lwarx r5,MUTEX_DATA,r3 ; load the mutex lock
	907	mr. r5,r5
	908	bne-- mlckespin0 ; Can not get it right now...
	909	stwcx. r6,MUTEX_DATA,r3 ; grab the lock
	910	bne-- mlcketry ; loop back if failed
	911	.globl EXT(mlckePatch_isync)
	912	LEXT(mlckePatch_isync)
	913	isync ; stop prefeteching
	914	mflr r12
	915	bf MUTEX_ATTR_DEBUGb,mlckedebskip
	916	mr r8,r6 ; Get the active thread
	917	stw r12,MUTEX_STACK(r3) ; Save our caller
	918	stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
	919	mr r5,r1
	920	LCK_STACK(r3,r5,r6,r7,r8,r10)
	921	mlckedebskip:
	922	mtmsr r9 ; Say, any interrupts pending?
	923	blr
	924
	925	mlckespin0:
	926	li r5,lgKillResv ; Killing field
	927	stwcx. r5,0,r5 ; Kill reservation
	928	mlckespin01:
	929	mflr r12
	930	mtmsr r9 ; Say, any interrupts pending?
	931	bl mlckspin1
	932	mtmsr r7 ; Turn off interruptions, vec and fp off already
	933	mtlr r12
	934	b mlcketry
	935
	936	/*
	937	* void lck_mtx_lock(lck_mtx_t*)
	938	*
	939	*/
	940	.align 5
	941	.globl EXT(lck_mtx_lock)
	942	LEXT(lck_mtx_lock)
	943
	944	mfsprg r6,1 ; load the current thread
	945	lwz r5,MUTEX_DATA(r3) ; Get the lock quickly
	946	mr r11,r3 ; Save lock addr
	947	li r4,0
	948	li r8,0
	949	li r9,0
	950	mr. r5,r5 ; Quick check
	951	bne-- mlckspin00 ; Indirect or Can not get it right now...
	952
	953	mlcktry:
	954	lwarx r5,MUTEX_DATA,r3 ; load the mutex lock
	955	mr. r5,r5
	956	bne-- mlckspin01 ; Can not get it right now...
	957	stwcx. r6,MUTEX_DATA,r3 ; grab the lock
	958	bne-- mlcktry ; loop back if failed
	959	.globl EXT(mlckPatch_isync)
	960	LEXT(mlckPatch_isync)
	961	isync ; stop prefeteching
	962	blr
	963	; Need to debug making blr above a patch point and record:
	964	; LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE)
	965
	966	mlckspin00:
	967	cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
	968	bne-- mlckspin02 ; No, go handle contention
	969	lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
	970	b mlckeEnter
	971	mlckspin01:
	972	li r5,lgKillResv ; Killing field
	973	stwcx. r5,0,r5 ; Kill reservation
	974	mlckspin02:
	975	mflr r12
	976	li r0,0
	977	mtcrf 1,r0 ; Set cr7 to zero
	978	bl mlckspin1
	979	mtlr r12
	980	b mlcktry
	981
	982
	983	mlckspin1:
	984	mr. r4,r4 ; Test timeout value
	985	bne++ mlckspin2
	986	lis r4,hi16(EXT(MutexSpin)) ; Get the high part
	987	ori r4,r4,lo16(EXT(MutexSpin) ) ; And the low part
	988	lwz r4,0(r4) ; Get spin timerout value
	989	mr. r4,r4 ; Test spin timeout value
	990	bne++ mlckspin2 ; Is spin timeout requested
	991	crclr mlckmiss ; Clear miss test
	992	b mlckslow1 ; Don't try to spin
	993
	994	mlckspin2: mr. r8,r8 ; Is r8 set to zero
	995	bne++ mlckspin3 ; If yes, first spin attempt
	996	crclr mlckmiss ; Clear miss test
	997	mr. r9,r9 ; Is r9 set to zero
	998	bne++ mlckspin3 ; If yes, r9 set with msr value
	999	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	1000	mfmsr r9 ; Get the MSR value
	1001	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	1002	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	1003	andc r9,r9,r0 ; Clear FP and VEC
	1004	andc r7,r9,r7 ; Clear EE as well
	1005	mtmsr r7 ; Turn off interruptions
	1006	isync ; May have turned off vec and fp here
	1007	mftb r8 ; Get timestamp on entry
	1008	b mlcksniff
	1009
	1010	mlckspin3: mtmsr r7 ; Turn off interruptions
	1011	mftb r8 ; Get timestamp on entry
	1012
	1013	mlcksniff: lwz r5,MUTEX_DATA(r3) ; Get that lock in here
	1014	mr. r5,r5 ; Is the lock held
	1015	beq++ mlckretry ; No, try for it again...
	1016	rlwinm. r10,r5,0,0,29 ; Extract the lock owner
	1017	beq++ mlckslow0 ; InterLock is held
	1018	bf MUTEX_ATTR_STATb,mlStatSkip ; Branch if no stat
	1019	andi. r5,r5,ILK_LOCKED ; extract interlocked?
	1020	bne mlStatSkip ; yes, skip
	1021	bt mlckmiss,mlStatSkip ; miss already counted
	1022	crset mlckmiss ; Remember miss recorded
	1023	lwz r5,MUTEX_GRP(r3) ; Load lock group
	1024	addi r5,r5,GRP_MTX_STAT_MISS+4 ; Add stat miss offset
	1025	mlStatLoop:
	1026	lwarx r6,0,r5 ; Load stat miss cnt
	1027	addi r6,r6,1 ; Increment stat miss cnt
	1028	stwcx. r6,0,r5 ; Update stat miss cnt
	1029	bne-- mlStatLoop ; Retry if failed
	1030	mfsprg r6,1 ; Reload current thread
	1031	mlStatSkip:
	1032	lwz r2,ACT_MACT_SPF(r10) ; Get the special flags
	1033	rlwinm. r2,r2,0,OnProcbit,OnProcbit ; Is OnProcbit set?
	1034	beq mlckslow0 ; Lock owner isn't running
	1035	lis r2,hi16(TH_IDLE) ; Get thread idle state
	1036	ori r2,r2,lo16(TH_IDLE) ; Get thread idle state
	1037	lwz r10,THREAD_STATE(r10) ; Get the thread state
	1038	and. r10,r10,r2 ; Is idle set?
	1039	bne mlckslow0 ; Lock owner is idling
	1040
	1041	mftb r10 ; Time stamp us now
	1042	sub r10,r10,r8 ; Get the elapsed time
	1043	cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
	1044	blt++ mlcksniff ; Not yet...
	1045
	1046	mtmsr r9 ; Say, any interrupts pending?
	1047
	1048	; The following instructions force the pipeline to be interlocked to that only one
	1049	; instruction is issued per cycle. The insures that we stay enabled for a long enough
	1050	; time; if it's too short, pending interruptions will not have a chance to be taken
	1051
	1052	subi r4,r4,128 ; Back off elapsed time from timeout value
	1053	or r4,r4,r4 ; Do nothing here but force a single cycle delay
	1054	mr. r4,r4 ; See if we used the whole timeout
	1055	or r4,r4,r4 ; Do nothing here but force a single cycle delay
	1056
	1057	ble-- mlckslow1 ; We failed
	1058	b mlckspin3 ; Now that we've opened an enable window, keep trying...
	1059	mlckretry:
	1060	mtmsr r9 ; Restore interrupt state
	1061	li r8,1 ; Show already through once
	1062	blr
	1063
	1064	mlckslow0: ; We couldn't get the lock
	1065	mtmsr r9 ; Restore interrupt state
	1066
	1067	mlckslow1:
	1068	mtlr r12
	1069
	1070	PROLOG(0)
	1071	.L_ml_retry:
	1072	bl lockDisa ; Go get a lock on the mutex's interlock lock
	1073	mr. r4,r3 ; Did we get it?
	1074	lwz r3,FM_ARG0(r1) ; Restore the lock address
	1075	bne++ mlGotInt ; We got it just fine...
	1076	mr r4,r11 ; Saved lock addr
	1077	lis r3,hi16(mutex_failed1) ; Get the failed mutex message
	1078	ori r3,r3,lo16(mutex_failed1) ; Get the failed mutex message
	1079	bl EXT(panic) ; Call panic
	1080	BREAKPOINT_TRAP ; We die here anyway, can not get the lock
	1081
	1082	.data
	1083	mutex_failed1:
	1084	STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock\n\000"
	1085	.text
	1086
	1087	mlGotInt:
	1088
	1089	; Note that there is no reason to do a load and reserve here. We already
	1090	; hold the interlock lock and no one can touch this field unless they
	1091	; have that, so, we're free to play
	1092
	1093	lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field
	1094	rlwinm. r9,r4,30,2,31 ; So, can we have it?
	1095	bne- mlInUse ; Nope, sombody's playing already...
	1096
	1097	bf++ MUTEX_ATTR_DEBUGb,mlDebSkip
	1098	CHECK_SETUP(r5)
	1099	mfsprg r9,1 ; Get the current activation
	1100	lwz r5,0(r1) ; Get previous save frame
	1101	lwz r6,FM_LR_SAVE(r5) ; Get our caller's address
	1102	mr r8,r9 ; Get the active thread
	1103	stw r6,MUTEX_STACK(r3) ; Save our caller
	1104	stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
	1105	LCK_STACK(r3,r5,r6,r7,r8,r10)
	1106	mlDebSkip:
	1107	mr r3,r11 ; Get the based lock address
	1108	bl EXT(lck_mtx_lock_acquire)
	1109	lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
	1110	mfsprg r5,1
	1111	mtcr r2
	1112	mr. r4,r3
	1113	lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
	1114	lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
	1115	beq mlUnlock
	1116	ori r5,r5,WAIT_FLAG
	1117
	1118	mlUnlock: eieio
	1119	stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock
	1120
	1121	EPILOG ; Restore all saved registers
	1122	b epStart ; Go enable preemption...
	1123
	1124	; We come to here when we have a resource conflict. In other words,
	1125	; the mutex is held.
	1126
	1127	mlInUse:
	1128
	1129	CHECK_SETUP(r12)
	1130	CHECK_MYLOCK() ; Assert we don't own the lock already */
	1131
	1132	; Note that we come in here with the interlock set. The wait routine
	1133	; will unlock it before waiting.
	1134
	1135	bf MUTEX_ATTR_STATb,mlStatSkip2 ; Branch if no stat
	1136	lwz r5,MUTEX_GRP(r3) ; Load lck group
	1137	bt mlckmiss,mlStatSkip1 ; Skip miss already counted
	1138	crset mlckmiss ; Remember miss recorded
	1139	li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset
	1140	mlStatLoop1:
	1141	lwarx r8,r9,r5 ; Load stat miss cnt
	1142	addi r8,r8,1 ; Increment stat miss cnt
	1143	stwcx. r8,r9,r5 ; Store stat miss cnt
	1144	bne-- mlStatLoop1 ; Retry if failed
	1145	mlStatSkip1:
	1146	lwz r9,GRP_MTX_STAT_WAIT+4(r5) ; Load wait cnt
	1147	addi r9,r9,1 ; Increment wait cnt
	1148	stw r9,GRP_MTX_STAT_WAIT+4(r5) ; Update miss cnt
	1149	mlStatSkip2:
	1150	ori r4,r4,WAIT_FLAG ; Set the wait flag
	1151	stw r4,MUTEX_DATA(r3)
	1152	rlwinm r4,r4,0,0,29 ; Extract the lock owner
	1153	mfcr r2
	1154	stw r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
	1155	mr r3,r11 ; Get the based lock address
	1156	bl EXT(lck_mtx_lock_wait) ; Wait for our turn at the lock
	1157
	1158	lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
	1159	lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
	1160	lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
	1161	mtcr r2
	1162	b .L_ml_retry ; and try again...
	1163
	1164
	1165	/*
	1166	* void lck_mtx_try_lock(_extlck_mtx_ext_t*)
	1167	*
	1168	*/
	1169	.align 5
	1170	.globl EXT(lck_mtx_try_lock_ext)
	1171	LEXT(lck_mtx_try_lock_ext)
	1172	mr r11,r3 ; Save lock addr
	1173	mlteEnter:
	1174	lwz r0,MUTEX_ATTR(r3)
	1175	mtcrf 1,r0 ; Set cr7
	1176	CHECK_SETUP(r12)
	1177	CHECK_MUTEX_TYPE()
	1178
	1179	bf MUTEX_ATTR_STATb,mlteStatSkip ; Branch if no stat
	1180	lwz r5,MUTEX_GRP(r3) ; Load lock group
	1181	li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset
	1182	mlteStatLoop:
	1183	lwarx r8,r7,r5 ; Load stat util cnt
	1184	addi r8,r8,1 ; Increment stat util cnt
	1185	stwcx. r8,r7,r5 ; Store stat util cnt
	1186	bne-- mlteStatLoop ; Retry if failed
	1187	mr. r8,r8 ; Test for zero
	1188	bne++ mlteStatSkip ; Did stat util cnt wrapped?
	1189	lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt
	1190	addi r8,r8,1 ; Increment upper stat util cnt
	1191	stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt
	1192	mlteStatSkip:
	1193	mfsprg r6,1 ; load the current thread
	1194	lwz r5,MUTEX_DATA(r3) ; Get the lock value
	1195	mr. r5,r5 ; Quick check
	1196	bne-- L_mtx_try_slow ; Can not get it now...
	1197	mfmsr r9 ; Get the MSR value
	1198	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	1199	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	1200	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	1201	andc r9,r9,r0 ; Clear FP and VEC
	1202	andc r7,r9,r7 ; Clear EE as well
	1203	mtmsr r7 ; Turn off interruptions
	1204	isync ; May have turned off vec and fp here
	1205
	1206	mlteLoopTry:
	1207	lwarx r5,MUTEX_DATA,r3 ; load the lock value
	1208	mr. r5,r5
	1209	bne-- mlteSlowX ; branch to the slow path
	1210	stwcx. r6,MUTEX_DATA,r3 ; grab the lock
	1211	bne-- mlteLoopTry ; retry if failed
	1212	.globl EXT(mltelckPatch_isync)
	1213	LEXT(mltelckPatch_isync)
	1214	isync ; stop prefetching
	1215	mflr r12
	1216	bf MUTEX_ATTR_DEBUGb,mlteDebSkip
	1217	mr r8,r6 ; Get the active thread
	1218	stw r12,MUTEX_STACK(r3) ; Save our caller
	1219	stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
	1220	mr r5,r1
	1221	LCK_STACK(r3,r5,r6,r7,r8,r10)
	1222	mlteDebSkip:
	1223	li r3, 1
	1224	mtmsr r9 ; Say, any interrupts pending?
	1225	blr
	1226	mlteSlowX:
	1227	li r5,lgKillResv ; Killing field
	1228	stwcx. r5,0,r5 ; Kill reservation
	1229	mtmsr r9 ; Say, any interrupts pending?
	1230	b L_mtx_try_slow
	1231
	1232
	1233	/*
	1234	* void lck_mtx_try_lock(lck_mtx_t*)
	1235	*
	1236	*/
	1237	.align 5
	1238	.globl EXT(lck_mtx_try_lock)
	1239	LEXT(lck_mtx_try_lock)
	1240
	1241	mfsprg r6,1 ; load the current thread
	1242	lwz r5,MUTEX_DATA(r3) ; Get the lock value
	1243	mr r11,r3 ; Save lock addr
	1244	mr. r5,r5 ; Quick check
	1245	bne-- mltSlow00 ; Indirect or Can not get it now...
	1246
	1247	mltLoopTry:
	1248	lwarx r5,MUTEX_DATA,r3 ; load the lock value
	1249	mr. r5,r5
	1250	bne-- mltSlow01 ; branch to the slow path
	1251	stwcx. r6,MUTEX_DATA,r3 ; grab the lock
	1252	bne-- mltLoopTry ; retry if failed
	1253	.globl EXT(mltlckPatch_isync)
	1254	LEXT(mltlckPatch_isync)
	1255	isync ; stop prefetching
	1256	li r3, 1
	1257	blr
	1258
	1259	mltSlow00:
	1260	cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
	1261	bne-- mltSlow02 ; No, go handle contention
	1262	lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
	1263	b mlteEnter
	1264	mltSlow01:
	1265	li r5,lgKillResv ; Killing field
	1266	stwcx. r5,0,r5 ; Kill reservation
	1267
	1268	mltSlow02:
	1269	li r0,0
	1270	mtcrf 1,r0 ; Set cr7 to zero
	1271
	1272	L_mtx_try_slow:
	1273	PROLOG(0)
	1274
	1275	lwz r6,MUTEX_DATA(r3) ; Quick check
	1276	rlwinm. r6,r6,30,2,31 ; to see if someone has this lock already
	1277	bne- mtFail ; Someone's got it already...
	1278
	1279	bl lockDisa ; Go get a lock on the mutex's interlock lock
	1280	mr. r4,r3 ; Did we get it?
	1281	lwz r3,FM_ARG0(r1) ; Restore the lock address
	1282	bne++ mtGotInt ; We got it just fine...
	1283	mr r4,r11 ; Saved lock addr
	1284	lis r3,hi16(mutex_failed2) ; Get the failed mutex message
	1285	ori r3,r3,lo16(mutex_failed2) ; Get the failed mutex message
	1286	bl EXT(panic) ; Call panic
	1287	BREAKPOINT_TRAP ; We die here anyway, can not get the lock
	1288
	1289	.data
	1290	mutex_failed2:
	1291	STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock try\n\000"
	1292	.text
	1293
	1294	mtGotInt:
	1295
	1296	; Note that there is no reason to do a load and reserve here. We already
	1297	; hold the interlock and no one can touch at this field unless they
	1298	; have that, so, we're free to play
	1299
	1300	lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field
	1301	rlwinm. r9,r4,30,2,31 ; So, can we have it?
	1302	bne- mtInUse ; Nope, sombody's playing already...
	1303
	1304	bf++ MUTEX_ATTR_DEBUGb,mtDebSkip
	1305	CHECK_SETUP(r5)
	1306	mfsprg r9,1 ; Get the current activation
	1307	lwz r5,0(r1) ; Get previous save frame
	1308	lwz r6,FM_LR_SAVE(r5) ; Get our caller's address
	1309	mr r8,r9 ; Get the active thread
	1310	stw r6,MUTEX_STACK(r3) ; Save our caller
	1311	stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
	1312	LCK_STACK(r3,r5,r6,r7,r8,r10)
	1313	mtDebSkip:
	1314	mr r3,r11 ; Get the based lock address
	1315	bl EXT(lck_mtx_lock_acquire)
	1316	mfsprg r5,1
	1317	mr. r4,r3
	1318	lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
	1319	lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
	1320	beq mtUnlock
	1321	ori r5,r5,WAIT_FLAG
	1322
	1323	mtUnlock: eieio
	1324	stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock
	1325
	1326	bl epStart ; Go enable preemption...
	1327
	1328	li r3, 1
	1329	EPILOG ; Restore all saved registers
	1330	blr ; Return...
	1331
	1332	; We come to here when we have a resource conflict. In other words,
	1333	; the mutex is held.
	1334
	1335	mtInUse:
	1336	bf++ MUTEX_ATTR_STATb,mtStatSkip ; Branch if no stat
	1337	lwz r5,MUTEX_GRP(r3) ; Load lock group
	1338	li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset
	1339	mtStatLoop:
	1340	lwarx r8,r9,r5 ; Load stat miss cnt
	1341	addi r8,r8,1 ; Increment stat miss cnt
	1342	stwcx. r8,r9,r5 ; Store stat miss cnt
	1343	bne-- mtStatLoop ; Retry if failed
	1344	mtStatSkip:
	1345	rlwinm r4,r4,0,0,30 ; Get the unlock value
	1346	stw r4,MUTEX_DATA(r3) ; free the interlock
	1347	bl epStart ; Go enable preemption...
	1348
	1349	mtFail: li r3,0 ; Set failure code
	1350	EPILOG ; Restore all saved registers
	1351	blr ; Return...
	1352
	1353
	1354
	1355	/*
	1356	* void lck_mtx_ext_unlock(lck_mtx_ext_t* l)
	1357	*
	1358	*/
	1359	.align 5
	1360	.globl EXT(lck_mtx_ext_unlock)
	1361	LEXT(lck_mtx_ext_unlock)
	1362	mlueEnter:
	1363	.globl EXT(mulckePatch_isync)
	1364	LEXT(mulckePatch_isync)
	1365	isync
	1366	.globl EXT(mulckePatch_eieio)
	1367	LEXT(mulckePatch_eieio)
	1368	eieio
	1369	mr r11,r3 ; Save lock addr
	1370	mlueEnter1:
	1371	lwz r0,MUTEX_ATTR(r3)
	1372	mtcrf 1,r0 ; Set cr7
	1373	CHECK_SETUP(r12)
	1374	CHECK_MUTEX_TYPE()
	1375	CHECK_THREAD(MUTEX_THREAD)
	1376
	1377	lwz r5,MUTEX_DATA(r3) ; Get the lock
	1378	rlwinm. r4,r5,0,30,31 ; Quick check
	1379	bne-- L_mtx_unlock_slow ; Can not get it now...
	1380	mfmsr r9 ; Get the MSR value
	1381	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	1382	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	1383	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	1384	andc r9,r9,r0 ; Clear FP and VEC
	1385	andc r7,r9,r7 ; Clear EE as well
	1386	mtmsr r7 ; Turn off interruptions
	1387	isync ; May have turned off vec and fp here
	1388
	1389	mlueLoop:
	1390	lwarx r5,MUTEX_DATA,r3
	1391	rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
	1392	li r5,0 ; Clear the mutexlock
	1393	bne-- mlueSlowX
	1394	stwcx. r5,MUTEX_DATA,r3
	1395	bne-- mlueLoop
	1396	mtmsr r9 ; Say, any interrupts pending?
	1397	blr
	1398
	1399	mlueSlowX:
	1400	li r5,lgKillResv ; Killing field
	1401	stwcx. r5,0,r5 ; Dump reservation
	1402	mtmsr r9 ; Say, any interrupts pending?
	1403	b L_mtx_unlock_slow ; Join slow path...
	1404
	1405	/*
	1406	* void lck_mtx_unlock(lck_mtx_t* l)
	1407	*
	1408	*/
	1409	.align 5
	1410	.globl EXT(lck_mtx_unlock)
	1411	LEXT(lck_mtx_unlock)
	1412	mluEnter:
	1413	.globl EXT(mulckPatch_isync)
	1414	LEXT(mulckPatch_isync)
	1415	isync
	1416	.globl EXT(mulckPatch_eieio)
	1417	LEXT(mulckPatch_eieio)
	1418	eieio
	1419	mr r11,r3 ; Save lock addr
	1420	mluEnter1:
	1421	lwz r5,MUTEX_DATA(r3) ; Get the lock
	1422	rlwinm. r4,r5,0,30,31 ; Quick check
	1423	bne-- mluSlow0 ; Indirect or Can not get it now...
	1424
	1425	mluLoop:
	1426	lwarx r5,MUTEX_DATA,r3
	1427	rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
	1428	li r5,0 ; Clear the mutexlock
	1429	bne-- mluSlowX
	1430	stwcx. r5,MUTEX_DATA,r3
	1431	bne-- mluLoop
	1432	#if CONFIG_DTRACE
	1433	/* lock released - LS_LCK_MTX_UNLOCK_RELEASE */
	1434	LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
	1435	blr
	1436
	1437	LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE)
	1438	#endif
	1439	blr
	1440
	1441
	1442	mluSlow0:
	1443	cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
	1444	bne-- L_mtx_unlock_slow ; No, go handle contention
	1445	lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
	1446	b mlueEnter1
	1447	mluSlowX:
	1448	li r5,lgKillResv ; Killing field
	1449	stwcx. r5,0,r5 ; Dump reservation
	1450
	1451	L_mtx_unlock_slow:
	1452
	1453	PROLOG(0)
	1454
	1455	bl lockDisa ; Go get a lock on the mutex's interlock lock
	1456	mr. r4,r3 ; Did we get it?
	1457	lwz r3,FM_ARG0(r1) ; Restore the lock address
	1458	bne++ muGotInt ; We got it just fine...
	1459	mr r4,r11 ; Saved lock addr
	1460	lis r3,hi16(mutex_failed3) ; Get the failed mutex message
	1461	ori r3,r3,lo16(mutex_failed3) ; Get the failed mutex message
	1462	bl EXT(panic) ; Call panic
	1463	BREAKPOINT_TRAP ; We die here anyway, can not get the lock
	1464
	1465	.data
	1466	mutex_failed3:
	1467	STRINGD "attempt to interlock mutex (0x%08X) failed on mutex unlock\n\000"
	1468	.text
	1469
	1470
	1471	muGotInt:
	1472	lwz r4,MUTEX_DATA(r3)
	1473	andi. r5,r4,WAIT_FLAG ; are there any waiters ?
	1474	rlwinm r4,r4,0,0,29
	1475	beq+ muUnlock ; Nope, we're done...
	1476
	1477	mr r3,r11 ; Get the based lock address
	1478	bl EXT(lck_mtx_unlock_wakeup) ; yes, wake a thread
	1479	lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
	1480	lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
	1481	lwz r5,MUTEX_DATA(r3) ; load the lock
	1482
	1483	muUnlock:
	1484	andi. r5,r5,WAIT_FLAG ; Get the unlock value
	1485	eieio
	1486	stw r5,MUTEX_DATA(r3) ; unlock the interlock and lock
	1487
	1488	EPILOG ; Deal with the stack now, enable_preemption doesn't always want one
	1489	b epStart ; Go enable preemption...
	1490
	1491	/*
	1492	* void lck_mtx_assert(lck_mtx_t* l, unsigned int)
	1493	*
	1494	*/
	1495	.align 5
	1496	.globl EXT(lck_mtx_assert)
	1497	LEXT(lck_mtx_assert)
	1498	mr r11,r3
	1499	maEnter:
	1500	lwz r5,MUTEX_DATA(r3)
	1501	cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
	1502	bne-- maCheck ; No, go check the assertion
	1503	lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
	1504	b maEnter
	1505	maCheck:
	1506	mfsprg r6,1 ; load the current thread
	1507	rlwinm r5,r5,0,0,29 ; Extract the lock owner
	1508	cmpwi r4,MUTEX_ASSERT_OWNED
	1509	cmplw cr1,r6,r5 ; Is the lock held by current act
	1510	crandc cr0_eq,cr0_eq,cr1_eq ; Check owned assertion
	1511	bne-- maNext
	1512	mr r4,r11
	1513	lis r3,hi16(mutex_assert1) ; Get the failed mutex message
	1514	ori r3,r3,lo16(mutex_assert1) ; Get the failed mutex message
	1515	b maPanic ; Panic path
	1516	maNext:
	1517	cmpwi r4,MUTEX_ASSERT_NOTOWNED ; Check not owned assertion
	1518	crand cr0_eq,cr0_eq,cr1_eq ;
	1519	bnelr++
	1520	maPanic:
	1521	PROLOG(0)
	1522	mr r4,r11
	1523	lis r3,hi16(mutex_assert2) ; Get the failed mutex message
	1524	ori r3,r3,lo16(mutex_assert2) ; Get the failed mutex message
	1525	bl EXT(panic) ; Call panic
	1526	BREAKPOINT_TRAP ; We die here anyway
	1527
	1528	.data
	1529	mutex_assert1:
	1530	STRINGD "mutex (0x%08X) not owned\n\000"
	1531	mutex_assert2:
	1532	STRINGD "mutex (0x%08X) owned\n\000"
	1533	.text
	1534
	1535
	1536	/*
	1537	* void lck_mtx_ilk_unlock(lck_mtx *lock)
	1538	*/
	1539	.globl EXT(lck_mtx_ilk_unlock)
	1540	LEXT(lck_mtx_ilk_unlock)
	1541
	1542	lwz r10,MUTEX_DATA(r3)
	1543	rlwinm r10,r10,0,0,30
	1544	eieio
	1545	stw r10,MUTEX_DATA(r3)
	1546
	1547	b epStart ; Go enable preemption...
	1548
	1549	/*
	1550	* void _enable_preemption_no_check(void)
	1551	*
	1552	* This version does not check if we get preempted or not
	1553	*/
	1554	.align 4
	1555	.globl EXT(_enable_preemption_no_check)
	1556
	1557	LEXT(_enable_preemption_no_check)
	1558
	1559	cmplw cr1,r1,r1 ; Force zero cr so we know not to check if preempted
	1560	b epCommn ; Join up with the other enable code...
	1561
	1562	/*
	1563	* void _enable_preemption(void)
	1564	*
	1565	* This version checks if we get preempted or not
	1566	*/
	1567	.align 5
	1568	.globl EXT(_enable_preemption)
	1569
	1570	LEXT(_enable_preemption)
	1571
	1572	; Here is where we enable preemption.
	1573
	1574	epStart:
	1575	cmplwi cr1,r1,0 ; Force non-zero cr so we know to check if preempted
	1576
	1577	epCommn:
	1578	mfsprg r3,1 ; Get current activation
	1579	li r8,-1 ; Get a decrementer
	1580	lwz r5,ACT_PREEMPT_CNT(r3) ; Get the preemption level
	1581	add. r5,r5,r8 ; Bring down the disable count
	1582	blt- epTooFar ; Yeah, we did...
	1583	stw r5,ACT_PREEMPT_CNT(r3) ; Save it back
	1584	crandc cr0_eq,cr0_eq,cr1_eq
	1585	beq+ epCheckPreempt ; Go check if we need to be preempted...
	1586	blr ; Leave...
	1587	epTooFar:
	1588	mr r4,r5
	1589	lis r3,hi16(epTooFarStr) ; First half of panic string
	1590	ori r3,r3,lo16(epTooFarStr) ; Second half of panic string
	1591	PROLOG(0)
	1592	bl EXT(panic)
	1593	BREAKPOINT_TRAP ; We die here anyway
	1594
	1595	.data
	1596	epTooFarStr:
	1597	STRINGD "enable_preemption: preemption_level %d\n\000"
	1598
	1599	.text
	1600	.align 5
	1601	epCheckPreempt:
	1602	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	1603	mfmsr r9 ; Get the MSR value
	1604	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	1605	andi. r4,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off
	1606	beq+ epCPno ; No preemption here...
	1607	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	1608	andc r9,r9,r0 ; Clear FP and VEC
	1609	andc r7,r9,r7 ; Clear EE as well
	1610	mtmsr r7 ; Turn off interruptions
	1611	isync ; May have turned off vec and fp here
	1612	lwz r3,ACT_PER_PROC(r3) ; Get the per_proc block
	1613	lwz r7,PP_PENDING_AST(r3) ; Get pending AST mask
	1614	li r5,AST_URGENT ; Get the requests we do honor
	1615	lis r0,hi16(DoPreemptCall) ; Just in case, get the top of firmware call
	1616	and. r7,r7,r5 ; Should we preempt?
	1617	ori r0,r0,lo16(DoPreemptCall) ; Merge in bottom part
	1618	mtmsr r9 ; Allow interrupts if we can
	1619	epCPno:
	1620	beqlr+ ; We probably will not preempt...
	1621	sc ; Do the preemption
	1622	blr ; Now, go away now...
	1623
	1624	/*
	1625	* void disable_preemption(void)
	1626	*
	1627	* Here is where we disable preemption.
	1628	*/
	1629	.align 5
	1630	.globl EXT(_disable_preemption)
	1631
	1632	LEXT(_disable_preemption)
	1633
	1634	mfsprg r6,1 ; Get the current activation
	1635	lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
	1636	addi r5,r5,1 ; Bring up the disable count
	1637	stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
	1638	blr ; Return...
	1639
	1640	/*
	1641	* int get_preemption_level(void)
	1642	*
	1643	* Return the current preemption level
	1644	*/
	1645	.align 5
	1646	.globl EXT(get_preemption_level)
	1647
	1648	LEXT(get_preemption_level)
	1649
	1650	mfsprg r6,1 ; Get current activation
	1651	lwz r3,ACT_PREEMPT_CNT(r6) ; Get the preemption level
	1652	blr ; Return...
	1653
	1654	/*
	1655	* void ppc_usimple_lock_init(simple_lock_t, etap_event_t)
	1656	*
	1657	* Initialize a simple lock.
	1658	*/
	1659	.align 5
	1660	.globl EXT(ppc_usimple_lock_init)
	1661
	1662	LEXT(ppc_usimple_lock_init)
	1663
	1664	li r0, 0 ; set lock to free == 0
	1665	stw r0, 0(r3) ; Initialize the lock
	1666	blr
	1667
	1668	/*
	1669	* void lck_spin_lock(lck_spin_t *)
	1670	* void ppc_usimple_lock(simple_lock_t *)
	1671	*
	1672	*/
	1673	.align 5
	1674	.globl EXT(lck_spin_lock)
	1675	LEXT(lck_spin_lock)
	1676	.globl EXT(ppc_usimple_lock)
	1677	LEXT(ppc_usimple_lock)
	1678
	1679	mfsprg r6,1 ; Get the current activation
	1680	lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
	1681	addi r5,r5,1 ; Bring up the disable count
	1682	stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
	1683	mr r5,r3 ; Get the address of the lock
	1684	li r8,0 ; Set r8 to zero
	1685	li r4,0 ; Set r4 to zero
	1686
	1687	slcktry: lwarx r11,SLOCK_ILK,r5 ; Grab the lock value
	1688	andi. r3,r11,ILK_LOCKED ; Is it locked?
	1689	ori r11,r6,ILK_LOCKED ; Set interlock
	1690	bne-- slckspin ; Yeah, wait for it to clear...
	1691	stwcx. r11,SLOCK_ILK,r5 ; Try to seize that there durn lock
	1692	bne-- slcktry ; Couldn't get it...
	1693	.globl EXT(slckPatch_isync)
	1694	LEXT(slckPatch_isync)
	1695	isync ; Make sure we don't use a speculativily loaded value
	1696	blr ; Go on home...
	1697
	1698	slckspin: li r11,lgKillResv ; Killing field
	1699	stwcx. r11,0,r11 ; Kill reservation
	1700
	1701	mr. r4,r4 ; Test timeout value
	1702	bne++ slockspin0
	1703	lis r4,hi16(EXT(LockTimeOut)) ; Get the high part
	1704	ori r4,r4,lo16(EXT(LockTimeOut)) ; And the low part
	1705	lwz r4,0(r4) ; Get the timerout value
	1706
	1707	slockspin0: mr. r8,r8 ; Is r8 set to zero
	1708	bne++ slockspin1 ; If yes, first spin attempt
	1709	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	1710	mfmsr r9 ; Get the MSR value
	1711	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	1712	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	1713	andc r9,r9,r0 ; Clear FP and VEC
	1714	andc r7,r9,r7 ; Clear EE as well
	1715	mtmsr r7 ; Turn off interruptions
	1716	isync ; May have turned off vec and fp here
	1717	mftb r8 ; Get timestamp on entry
	1718	b slcksniff
	1719
	1720	slockspin1: mtmsr r7 ; Turn off interruptions
	1721	mftb r8 ; Get timestamp on entry
	1722
	1723	slcksniff: lwz r3,SLOCK_ILK(r5) ; Get that lock in here
	1724	andi. r3,r3,ILK_LOCKED ; Is it free yet?
	1725	beq++ slckretry ; Yeah, try for it again...
	1726
	1727	mftb r10 ; Time stamp us now
	1728	sub r10,r10,r8 ; Get the elapsed time
	1729	cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
	1730	blt++ slcksniff ; Not yet...
	1731
	1732	mtmsr r9 ; Say, any interrupts pending?
	1733
	1734	; The following instructions force the pipeline to be interlocked to that only one
	1735	; instruction is issued per cycle. The insures that we stay enabled for a long enough
	1736	; time; if it's too short, pending interruptions will not have a chance to be taken
	1737
	1738	subi r4,r4,128 ; Back off elapsed time from timeout value
	1739	or r4,r4,r4 ; Do nothing here but force a single cycle delay
	1740	mr. r4,r4 ; See if we used the whole timeout
	1741	li r3,0 ; Assume a timeout return code
	1742	or r4,r4,r4 ; Do nothing here but force a single cycle delay
	1743
	1744	ble-- slckfail ; We failed
	1745	b slockspin1 ; Now that we've opened an enable window, keep trying...
	1746	slckretry:
	1747	mtmsr r9 ; Restore interrupt state
	1748	li r8,1 ; Show already through once
	1749	b slcktry
	1750	slckfail: ; We couldn't get the lock
	1751	lis r3,hi16(slckpanic_str)
	1752	ori r3,r3,lo16(slckpanic_str)
	1753	mr r4,r5
	1754	mflr r5
	1755	PROLOG(0)
	1756	bl EXT(panic)
	1757	BREAKPOINT_TRAP ; We die here anyway
	1758
	1759	.data
	1760	slckpanic_str:
	1761	STRINGD "simple lock (0x%08X) deadlock detection, pc=0x%08X\n\000"
	1762	.text
	1763
	1764	/*
	1765	* boolean_t lck_spin_try_lock(lck_spin_t *)
	1766	* unsigned int ppc_usimple_lock_try(simple_lock_t *)
	1767	*
	1768	*/
	1769	.align 5
	1770	.globl EXT(lck_spin_try_lock)
	1771	LEXT(lck_spin_try_lock)
	1772	.globl EXT(ppc_usimple_lock_try)
	1773	LEXT(ppc_usimple_lock_try)
	1774
	1775	lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
	1776	mfmsr r9 ; Get the MSR value
	1777	ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
	1778	ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
	1779	andc r9,r9,r0 ; Clear FP and VEC
	1780	andc r7,r9,r7 ; Clear EE as well
	1781	mtmsr r7 ; Disable interruptions and thus, preemption
	1782	mfsprg r6,1 ; Get current activation
	1783
	1784	lwz r11,SLOCK_ILK(r3) ; Get the lock
	1785	andi. r5,r11,ILK_LOCKED ; Check it...
	1786	bne-- slcktryfail ; Quickly fail...
	1787
	1788	slcktryloop:
	1789	lwarx r11,SLOCK_ILK,r3 ; Ld from addr of arg and reserve
	1790
	1791	andi. r5,r11,ILK_LOCKED ; TEST...
	1792	ori r5,r6,ILK_LOCKED
	1793	bne-- slcktryfailX ; branch if taken. Predict free
	1794
	1795	stwcx. r5,SLOCK_ILK,r3 ; And SET (if still reserved)
	1796	bne-- slcktryloop ; If set failed, loop back
	1797
	1798	.globl EXT(stlckPatch_isync)
	1799	LEXT(stlckPatch_isync)
	1800	isync
	1801
	1802	lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
	1803	addi r5,r5,1 ; Bring up the disable count
	1804	stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
	1805
	1806	mtmsr r9 ; Allow interruptions now
	1807	li r3,1 ; Set that the lock was free
	1808	blr
	1809
	1810	slcktryfailX:
	1811	li r5,lgKillResv ; Killing field
	1812	stwcx. r5,0,r5 ; Kill reservation
	1813
	1814	slcktryfail:
	1815	mtmsr r9 ; Allow interruptions now
	1816	li r3,0 ; FAILURE - lock was taken
	1817	blr
	1818
	1819
	1820	/*
	1821	* void lck_spin_unlock(lck_spin_t *)
	1822	* void ppc_usimple_unlock_rwcmb(simple_lock_t *)
	1823	*
	1824	*/
	1825	.align 5
	1826	.globl EXT(lck_spin_unlock)
	1827	LEXT(lck_spin_unlock)
	1828	.globl EXT(ppc_usimple_unlock_rwcmb)
	1829	LEXT(ppc_usimple_unlock_rwcmb)
	1830
	1831	li r0,0
	1832	.globl EXT(sulckPatch_isync)
	1833	LEXT(sulckPatch_isync)
	1834	isync
	1835	.globl EXT(sulckPatch_eieio)
	1836	LEXT(sulckPatch_eieio)
	1837	eieio
	1838	stw r0, SLOCK_ILK(r3)
	1839
	1840	b epStart ; Go enable preemption...
	1841
	1842	/*
	1843	* void ppc_usimple_unlock_rwmb(simple_lock_t *)
	1844	*
	1845	*/
	1846	.align 5
	1847	.globl EXT(ppc_usimple_unlock_rwmb)
	1848
	1849	LEXT(ppc_usimple_unlock_rwmb)
	1850
	1851	li r0,0
	1852	sync
	1853	stw r0, SLOCK_ILK(r3)
	1854
	1855	b epStart ; Go enable preemption...
	1856
	1857	/*
	1858	* void lck_rw_lock_exclusive(lck_rw_t*)
	1859	*
	1860	*/
	1861	.align 5
	1862	.globl EXT(lck_rw_lock_exclusive)
	1863	LEXT(lck_rw_lock_exclusive)
	1864	#if !MACH_LDEBUG
	1865	.globl EXT(lock_write)
	1866	LEXT(lock_write)
	1867	#endif
	1868	lis r7,0xFFFF
	1869	ori r7,r7,(WANT_EXCL\|WANT_UPGRADE\|ILK_LOCKED)
	1870	rwleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	1871	and. r8,r5,r7 ; Can we have it?
	1872	ori r6,r5,WANT_EXCL ; Mark Exclusive
	1873	bne-- rwlespin ; Branch if cannot be held
	1874	stwcx. r6,RW_DATA,r3 ; Update lock word
	1875	bne-- rwleloop
	1876	.globl EXT(rwlePatch_isync)
	1877	LEXT(rwlePatch_isync)
	1878	isync
	1879	blr
	1880	rwlespin:
	1881	li r4,lgKillResv ; Killing field
	1882	stwcx. r4,0,r4 ; Kill it
	1883	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	1884	bne-- rwlespin1 ; No, go handle contention
	1885	mr r4,r3 ; pass lock pointer
	1886	lwz r3,RW_PTR(r3) ; load lock ext pointer
	1887	b EXT(lck_rw_lock_exclusive_ext)
	1888	rwlespin1:
	1889	b EXT(lck_rw_lock_exclusive_gen)
	1890
	1891	/*
	1892	* void lck_rw_lock_shared(lck_rw_t*)
	1893	*
	1894	*/
	1895	.align 5
	1896	.globl EXT(lck_rw_lock_shared)
	1897	LEXT(lck_rw_lock_shared)
	1898	#if !MACH_LDEBUG
	1899	.globl EXT(lock_read)
	1900	LEXT(lock_read)
	1901	#endif
	1902	rwlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	1903	andi. r7,r5,WANT_EXCL\|WANT_UPGRADE\|ILK_LOCKED ; Can we have it?
	1904	bne-- rwlsopt ; Branch if cannot be held
	1905	rwlsloopres:
	1906	addis r6,r5,1 ; Increment read cnt
	1907	stwcx. r6,RW_DATA,r3 ; Update lock word
	1908	bne-- rwlsloop
	1909	.globl EXT(rwlsPatch_isync)
	1910	LEXT(rwlsPatch_isync)
	1911	isync
	1912	blr
	1913	rwlsopt:
	1914	andi. r7,r5,PRIV_EXCL\|ILK_LOCKED ; Can we have it?
	1915	bne-- rwlsspin ; Branch if cannot be held
	1916	lis r7,0xFFFF ; Get read cnt mask
	1917	and. r8,r5,r7 ; Is it shared
	1918	bne rwlsloopres ; Branch if can be held
	1919	rwlsspin:
	1920	li r4,lgKillResv ; Killing field
	1921	stwcx. r4,0,r4 ; Kill it
	1922	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	1923	bne-- rwlsspin1 ; No, go handle contention
	1924	mr r4,r3 ; pass lock pointer
	1925	lwz r3,RW_PTR(r3) ; load lock ext pointer
	1926	b EXT(lck_rw_lock_shared_ext)
	1927	rwlsspin1:
	1928	b EXT(lck_rw_lock_shared_gen)
	1929
	1930	/*
	1931	* boolean_t lck_rw_lock_shared_to_exclusive(lck_rw_t*)
	1932	*
	1933	*/
	1934	.align 5
	1935	.globl EXT(lck_rw_lock_shared_to_exclusive)
	1936	LEXT(lck_rw_lock_shared_to_exclusive)
	1937	#if !MACH_LDEBUG
	1938	.globl EXT(lock_read_to_write)
	1939	LEXT(lock_read_to_write)
	1940	#endif
	1941	rwlseloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	1942	addis r6,r5,0xFFFF ; Decrement read cnt
	1943	lis r8,0xFFFF ; Get read count mask
	1944	ori r8,r8,WANT_UPGRADE\|ILK_LOCKED ; Include Interlock and upgrade flags
	1945	and. r7,r6,r8 ; Can we have it?
	1946	ori r9,r6,WANT_UPGRADE ; Mark Exclusive
	1947	bne-- rwlsespin ; Branch if cannot be held
	1948	stwcx. r9,RW_DATA,r3 ; Update lock word
	1949	bne-- rwlseloop
	1950	.globl EXT(rwlsePatch_isync)
	1951	LEXT(rwlsePatch_isync)
	1952	isync
	1953	li r3,1 ; Succeed, return TRUE...
	1954	blr
	1955	rwlsespin:
	1956	li r4,lgKillResv ; Killing field
	1957	stwcx. r4,0,r4 ; Kill it
	1958	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	1959	bne-- rwlsespin1 ; No, go handle contention
	1960	mr r4,r3 ; pass lock pointer
	1961	lwz r3,RW_PTR(r3) ; load lock ext pointer
	1962	b EXT(lck_rw_lock_shared_to_exclusive_ext)
	1963	rwlsespin1:
	1964	b EXT(lck_rw_lock_shared_to_exclusive_gen)
	1965
	1966
	1967
	1968	/*
	1969	* void lck_rw_lock_exclusive_to_shared(lck_rw_t*)
	1970	*
	1971	*/
	1972	.align 5
	1973	.globl EXT(lck_rw_lock_exclusive_to_shared)
	1974	LEXT(lck_rw_lock_exclusive_to_shared)
	1975	#if !MACH_LDEBUG
	1976	.globl EXT(lock_write_to_read)
	1977	LEXT(lock_write_to_read)
	1978	#endif
	1979	.globl EXT(rwlesPatch_isync)
	1980	LEXT(rwlesPatch_isync)
	1981	isync
	1982	.globl EXT(rwlesPatch_eieio)
	1983	LEXT(rwlesPatch_eieio)
	1984	eieio
	1985	rwlesloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	1986	andi. r7,r5,ILK_LOCKED ; Test interlock flag
	1987	bne-- rwlesspin ; Branch if interlocked
	1988	lis r6,1 ; Get 1 for read count
	1989	andi. r10,r5,WANT_UPGRADE ; Is it held with upgrade
	1990	li r9,WANT_UPGRADE\|WAIT_FLAG ; Get upgrade and wait flags mask
	1991	bne rwlesexcl1 ; Skip if held with upgrade
	1992	li r9,WANT_EXCL\|WAIT_FLAG ; Get exclusive and wait flags mask
	1993	rwlesexcl1:
	1994	andc r7,r5,r9 ; Marked free
	1995	rlwimi r6,r7,0,16,31 ; Set shared cnt to one
	1996	stwcx. r6,RW_DATA,r3 ; Update lock word
	1997	bne-- rwlesloop
	1998	andi. r7,r5,WAIT_FLAG ; Test wait flag
	1999	beqlr++ ; Return of no waiters
	2000	addi r3,r3,RW_EVENT ; Get lock event address
	2001	b EXT(thread_wakeup) ; wakeup waiters
	2002	rwlesspin:
	2003	li r4,lgKillResv ; Killing field
	2004	stwcx. r4,0,r4 ; Kill it
	2005	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	2006	bne-- rwlesspin1 ; No, go handle contention
	2007	mr r4,r3 ; pass lock pointer
	2008	lwz r3,RW_PTR(r3) ; load lock ext pointer
	2009	b EXT(lck_rw_lock_exclusive_to_shared_ext)
	2010	rwlesspin1:
	2011	b EXT(lck_rw_lock_exclusive_to_shared_gen)
	2012
	2013
	2014
	2015	/*
	2016	* boolean_t lck_rw_try_lock_exclusive(lck_rw_t*)
	2017	*
	2018	*/
	2019	.align 5
	2020	.globl EXT(lck_rw_try_lock_exclusive)
	2021	LEXT(lck_rw_try_lock_exclusive)
	2022	lis r10,0xFFFF ; Load read count mask
	2023	ori r10,r10,WANT_EXCL\|WANT_UPGRADE ; Include exclusive and upgrade flags
	2024	rwtleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	2025	andi. r7,r5,ILK_LOCKED ; Test interlock flag
	2026	bne-- rwtlespin ; Branch if interlocked
	2027	and. r7,r5,r10 ; Can we have it
	2028	ori r6,r5,WANT_EXCL ; Mark Exclusive
	2029	bne-- rwtlefail ;
	2030	stwcx. r6,RW_DATA,r3 ; Update lock word
	2031	bne-- rwtleloop
	2032	.globl EXT(rwtlePatch_isync)
	2033	LEXT(rwtlePatch_isync)
	2034	isync
	2035	li r3,1 ; Return TRUE
	2036	blr
	2037	rwtlefail:
	2038	li r4,lgKillResv ; Killing field
	2039	stwcx. r4,0,r4 ; Kill it
	2040	li r3,0 ; Return FALSE
	2041	blr
	2042	rwtlespin:
	2043	li r4,lgKillResv ; Killing field
	2044	stwcx. r4,0,r4 ; Kill it
	2045	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	2046	bne-- rwtlespin1 ; No, go handle contention
	2047	mr r4,r3 ; pass lock pointer
	2048	lwz r3,RW_PTR(r3) ; load lock ext pointer
	2049	b EXT(lck_rw_try_lock_exclusive_ext)
	2050	rwtlespin1:
	2051	b EXT(lck_rw_try_lock_exclusive_gen)
	2052
	2053
	2054	/*
	2055	* boolean_t lck_rw_try_lock_shared(lck_rw_t*)
	2056	*
	2057	*/
	2058	.align 5
	2059	.globl EXT(lck_rw_try_lock_shared)
	2060	LEXT(lck_rw_try_lock_shared)
	2061	rwtlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	2062	andi. r7,r5,ILK_LOCKED ; Test interlock flag
	2063	bne-- rwtlsspin ; Branch if interlocked
	2064	andi. r7,r5,WANT_EXCL\|WANT_UPGRADE ; So, can we have it?
	2065	bne-- rwtlsopt ; Branch if held exclusive
	2066	rwtlsloopres:
	2067	addis r6,r5,1 ; Increment read cnt
	2068	stwcx. r6,RW_DATA,r3 ; Update lock word
	2069	bne-- rwtlsloop
	2070	.globl EXT(rwtlsPatch_isync)
	2071	LEXT(rwtlsPatch_isync)
	2072	isync
	2073	li r3,1 ; Return TRUE
	2074	blr
	2075	rwtlsopt:
	2076	andi. r7,r5,PRIV_EXCL ; Can we have it?
	2077	bne-- rwtlsfail ; Branch if cannot be held
	2078	lis r7,0xFFFF ; Get read cnt mask
	2079	and. r8,r5,r7 ; Is it shared
	2080	bne rwtlsloopres ; Branch if can be held
	2081	rwtlsfail:
	2082	li r3,0 ; Return FALSE
	2083	blr
	2084	rwtlsspin:
	2085	li r4,lgKillResv ; Killing field
	2086	stwcx. r4,0,r4 ; Kill it
	2087	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	2088	bne-- rwtlsspin1 ; No, go handle contention
	2089	mr r4,r3 ; pass lock pointer
	2090	lwz r3,RW_PTR(r3) ; load lock ext pointer
	2091	b EXT(lck_rw_try_lock_shared_ext)
	2092	rwtlsspin1:
	2093	b EXT(lck_rw_try_lock_shared_gen)
	2094
	2095
	2096
	2097	/*
	2098	* lck_rw_type_t lck_rw_done(lck_rw_t*)
	2099	*
	2100	*/
	2101	.align 5
	2102	.globl EXT(lck_rw_done)
	2103	LEXT(lck_rw_done)
	2104	#if !MACH_LDEBUG
	2105	.globl EXT(lock_done)
	2106	LEXT(lock_done)
	2107	#endif
	2108	.globl EXT(rwldPatch_isync)
	2109	LEXT(rwldPatch_isync)
	2110	isync
	2111	.globl EXT(rwldPatch_eieio)
	2112	LEXT(rwldPatch_eieio)
	2113	eieio
	2114	li r10,WAIT_FLAG ; Get wait flag
	2115	lis r7,0xFFFF ; Get read cnt mask
	2116	mr r12,r3 ; Save lock addr
	2117	rwldloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
	2118	andi. r8,r5,ILK_LOCKED ; Test interlock flag
	2119	bne-- rwldspin ; Branch if interlocked
	2120	and. r8,r5,r7 ; Is it shared
	2121	cmpi cr1,r8,0 ; Is it shared
	2122	beq cr1,rwldexcl ; No, check exclusive
	2123	li r11,RW_SHARED ; Set return value
	2124	addis r6,r5,0xFFFF ; Decrement read count
	2125	and. r8,r6,r7 ; Is it still shared
	2126	li r8,0 ; Assume no wakeup
	2127	bne rwldshared1 ; Skip if still held shared
	2128	and r8,r6,r10 ; Extract wait flag
	2129	andc r6,r6,r10 ; Clear wait flag
	2130	rwldshared1:
	2131	b rwldstore
	2132	rwldexcl:
	2133	li r11,RW_EXCL ; Set return value
	2134	li r9,WANT_UPGRADE ; Get upgrade flag
	2135	and. r6,r5,r9 ; Is it held with upgrade
	2136	li r9,WANT_UPGRADE\|WAIT_FLAG ; Mask upgrade abd wait flags
	2137	bne rwldexcl1 ; Skip if held with upgrade
	2138	li r9,WANT_EXCL\|WAIT_FLAG ; Mask exclusive and wait flags
	2139	rwldexcl1:
	2140	andc r6,r5,r9 ; Marked free
	2141	and r8,r5,r10 ; Null if no waiter
	2142	rwldstore:
	2143	stwcx. r6,RW_DATA,r3 ; Update lock word
	2144	bne-- rwldloop
	2145	mr. r8,r8 ; wakeup needed?
	2146	mr r3,r11 ; Return lock held type
	2147	beqlr++
	2148	mr r3,r12 ; Restore lock address
	2149	PROLOG(0)
	2150	addi r3,r3,RW_EVENT ; Get lock event address
	2151	bl EXT(thread_wakeup) ; wakeup threads
	2152	lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
	2153	mtcr r2
	2154	EPILOG
	2155	li r3,RW_SHARED ; Assume lock type shared
	2156	bne cr1,rwldret ; Branch if was held exclusive
	2157	li r3,RW_EXCL ; Return lock type exclusive
	2158	rwldret:
	2159	blr
	2160	rwldspin:
	2161	li r4,lgKillResv ; Killing field
	2162	stwcx. r4,0,r4 ; Kill it
	2163	cmpli cr0,r5,RW_IND ; Is it a lock indirect
	2164	bne-- rwldspin1 ; No, go handle contention
	2165	mr r4,r3 ; pass lock pointer
	2166	lwz r3,RW_PTR(r3) ; load lock ext pointer
	2167	b EXT(lck_rw_done_ext)
	2168	rwldspin1:
	2169	b EXT(lck_rw_done_gen)
	2170
	2171	/*
	2172	* void lck_rw_ilk_lock(lck_rw_t *lock)
	2173	*/
	2174	.globl EXT(lck_rw_ilk_lock)
	2175	LEXT(lck_rw_ilk_lock)
	2176	crclr hwtimeout ; no timeout option
	2177	li r4,0 ; request default timeout value
	2178	li r12,ILK_LOCKED ; Load bit mask
	2179	b lckcomm ; Join on up...
	2180
	2181	/*
	2182	* void lck_rw_ilk_unlock(lck_rw_t *lock)
	2183	*/
	2184	.globl EXT(lck_rw_ilk_unlock)
	2185	LEXT(lck_rw_ilk_unlock)
	2186	li r4,1
	2187	b EXT(hw_unlock_bit)