/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
- *
- * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* Please see the License for the specific language governing rights and
* limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
-#include <cpus.h>
#include <mach_assert.h>
#include <mach_ldebug.h>
-#include <mach_rt.h>
-
-#include <kern/etap_options.h>
-
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <assym.s>
-#define STRING ascii
-
-#define SWT_HI 0+FM_SIZE
-#define SWT_LO 4+FM_SIZE
-#define MISSED 8+FM_SIZE
-
-#define ILK_LOCKED 0x01
-#define WAIT_FLAG 0x02
-#define SLOCK_FAST 0x02
-#define TH_FN_OWNED 0x01
-
-;
-; NOTE: make sure that PREEMPTSTACK in aligned_data is
-; set the same as it is here. This is the number of
-; traceback entries we can handle per processor
-;
-; A value of 0 disables the stack.
-;
-#define PREEMPTSTACK 0
-#define CHECKNMI 0
-#define CHECKLOCKS 1
-
-#include <ppc/POWERMAC/mp/mp.h>
-
-#define PROLOG(space) \
- stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \
- mflr r0 __ASMNL__ \
- stw r3,FM_ARG0(r1) __ASMNL__ \
- stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__
-
-#define EPILOG \
- lwz r1,0(r1) __ASMNL__ \
- lwz r0,FM_LR_SAVE(r1) __ASMNL__ \
- mtlr r0 __ASMNL__
-
-#if MACH_LDEBUG && CHECKLOCKS
-/*
- * Routines for general lock debugging.
- */
-
-/* Gets lock check flags in CR6: CR bits 24-27 */
-#define CHECK_SETUP(rg) \
- lis rg,hi16(EXT(dgWork)) __ASMNL__ \
- ori rg,rg,lo16(EXT(dgWork)) __ASMNL__ \
- lbz rg,dgFlags(rg) __ASMNL__ \
- mtcrf 2,rg __ASMNL__
+#include <config_dtrace.h>
+#if CONFIG_DTRACE
+ #define LOCKSTAT_LABEL(lab) \
+ .data __ASMNL__ \
+ .globl lab __ASMNL__ \
+ lab: __ASMNL__ \
+ .long 9f __ASMNL__ \
+ .text __ASMNL__ \
+ 9: __ASMNL__ \
+
+ .globl _dtrace_probe, _lockstat_probemap
+#define LOCKSTAT_RECORD(id) \
+ lis r6,hi16(_lockstat_probemap) __ASMNL__ \
+ ori r6,r6,lo16(_lockstat_probemap) __ASMNL__ \
+ lwz r5,4*id(r6) __ASMNL__ \
+ mr. r5,r5 __ASMNL__ \
+ beqlr-- __ASMNL__ \
+ mr r4,r3 __ASMNL__ \
+ mr r3,r5 __ASMNL__ \
+ li r5,0 __ASMNL__ \
+ li r6,0 __ASMNL__ \
+ li r7,0 __ASMNL__ \
+ li r8,0 __ASMNL__ \
+ PROLOG(0) __ASMNL__ \
+ bl _dtrace_probe __ASMNL__ \
+ EPILOG
+#endif
+
-/*
- * Checks for expected lock types and calls "panic" on
- * mismatch. Detects calls to Mutex functions with
- * type simplelock and vice versa.
- */
-#define CHECK_MUTEX_TYPE() \
- bt 24+disLktypeb,1f __ASMNL__ \
- lwz r10,MUTEX_TYPE(r3) __ASMNL__ \
- cmpwi r10,MUTEX_TAG __ASMNL__ \
- beq+ 1f __ASMNL__ \
- lis r3,hi16(not_a_mutex) __ASMNL__ \
- ori r3,r3,lo16(not_a_mutex) __ASMNL__ \
- bl EXT(panic) __ASMNL__ \
- lwz r3,FM_ARG0(r1) __ASMNL__ \
-1:
-
- .data
-not_a_mutex:
- STRINGD "not a mutex!\n\000"
- .text
+#define STRING ascii
-#define CHECK_SIMPLE_LOCK_TYPE() \
- bt 24+disLktypeb,1f __ASMNL__ \
- lwz r10,SLOCK_TYPE(r3) __ASMNL__ \
- cmpwi r10,USLOCK_TAG __ASMNL__ \
- beq+ 1f __ASMNL__ \
- lis r3,hi16(not_a_slock) __ASMNL__ \
- ori r3,r3,lo16(not_a_slock) __ASMNL__ \
- bl EXT(panic) __ASMNL__ \
- lwz r3,FM_ARG0(r1) __ASMNL__ \
-1:
-
- .data
-not_a_slock:
- STRINGD "not a simple lock!\n\000"
- .text
+#define ILK_LOCKED 0x01
+#define WAIT_FLAG 0x02
+#define WANT_UPGRADE 0x04
+#define WANT_EXCL 0x08
+#define PRIV_EXCL 0x8000
-#define CHECK_NO_SIMPLELOCKS() \
- bt 24+disLkNmSimpb,2f __ASMNL__ \
- mfmsr r11 __ASMNL__ \
- rlwinm r11,r11,0,MSR_FP_BIT+1,MSR_FP_BIT-1 __ASMNL__ \
- rlwinm r11,r11,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 __ASMNL__ \
- rlwinm r10,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1 __ASMNL__ \
- mtmsr r10 __ASMNL__ \
- isync __ASMNL__ \
- mfsprg r10,0 __ASMNL__ \
- lwz r10,PP_SIMPLE_LOCK_CNT(r10) __ASMNL__ \
- cmpwi r10,0 __ASMNL__ \
- beq+ 1f __ASMNL__ \
- lis r3,hi16(simple_locks_held) __ASMNL__ \
- ori r3,r3,lo16(simple_locks_held) __ASMNL__ \
- bl EXT(panic) __ASMNL__ \
- lwz r3,FM_ARG0(r1) __ASMNL__ \
-1: __ASMNL__ \
- mtmsr r11 __ASMNL__ \
-2:
-
- .data
-simple_locks_held:
- STRINGD "simple locks held!\n\000"
- .text
+#define TH_FN_OWNED 0x01
-/*
- * Verifies return to the correct thread in "unlock" situations.
- */
+# volatile CR bits
+#define hwtimeout 20
+#define mlckmiss 21
-#define CHECK_THREAD(thread_offset) \
- bt 24+disLkThreadb,2f __ASMNL__ \
- mfmsr r11 __ASMNL__ \
- rlwinm r11,r11,0,MSR_FP_BIT+1,MSR_FP_BIT-1 __ASMNL__ \
- rlwinm r11,r11,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 __ASMNL__ \
- rlwinm r10,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1 __ASMNL__ \
- mtmsr r10 __ASMNL__ \
- isync __ASMNL__ \
- mfsprg r10,0 __ASMNL__ \
- lwz r10,PP_ACTIVE_THREAD(r10) __ASMNL__ \
- cmpwi r10,0 __ASMNL__ \
- beq- 1f __ASMNL__ \
- lwz r9,thread_offset(r3) __ASMNL__ \
- cmpw r9,r10 __ASMNL__ \
- beq+ 1f __ASMNL__ \
- lis r3,hi16(wrong_thread) __ASMNL__ \
- ori r3,r3,lo16(wrong_thread) __ASMNL__ \
- bl EXT(panic) __ASMNL__ \
- lwz r3,FM_ARG0(r1) __ASMNL__ \
-1: __ASMNL__ \
- mtmsr r11 __ASMNL__ \
-2:
- .data
-wrong_thread:
- STRINGD "wrong thread!\n\000"
- .text
+#define RW_DATA 0
-#define CHECK_MYLOCK(thread_offset) \
- bt 24+disLkMyLckb,2f __ASMNL__ \
- mfmsr r11 __ASMNL__ \
- rlwinm r11,r11,0,MSR_FP_BIT+1,MSR_FP_BIT-1 __ASMNL__ \
- rlwinm r11,r11,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 __ASMNL__ \
- rlwinm r10,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1 __ASMNL__ \
- mtmsr r10 __ASMNL__ \
- isync __ASMNL__ \
- mfsprg r10,0 __ASMNL__ \
- lwz r10,PP_ACTIVE_THREAD(r10) __ASMNL__ \
- cmpwi r10,0 __ASMNL__ \
- beq- 1f __ASMNL__ \
- lwz r9, thread_offset(r3) __ASMNL__ \
- cmpw r9,r10 __ASMNL__ \
- bne+ 1f __ASMNL__ \
- lis r3, HIGH_ADDR(mylock_attempt) __ASMNL__ \
- ori r3,r3,LOW_ADDR(mylock_attempt) __ASMNL__ \
- bl EXT(panic) __ASMNL__ \
- lwz r3,FM_ARG0(r1) __ASMNL__ \
-1: __ASMNL__ \
- mtmsr r11 __ASMNL__ \
-2:
+#define PROLOG(space) \
+ stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \
+ mfcr r2 __ASMNL__ \
+ mflr r0 __ASMNL__ \
+ stw r3,FM_ARG0(r1) __ASMNL__ \
+ stw r11,FM_ARG0+0x04(r1) __ASMNL__ \
+ stw r2,(FM_ALIGN(space)+FM_SIZE+FM_CR_SAVE)(r1) __ASMNL__ \
+ stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__
- .data
-mylock_attempt:
- STRINGD "mylock attempt!\n\000"
- .text
-
-#else /* MACH_LDEBUG */
+#define EPILOG \
+ lwz r1,0(r1) __ASMNL__ \
+ lwz r0,FM_LR_SAVE(r1) __ASMNL__ \
+ mtlr r0 __ASMNL__
-#define CHECK_SETUP(rg)
-#define CHECK_MUTEX_TYPE()
-#define CHECK_SIMPLE_LOCK_TYPE()
-#define CHECK_THREAD(thread_offset)
-#define CHECK_NO_SIMPLELOCKS()
-#define CHECK_MYLOCK(thread_offset)
-
-#endif /* MACH_LDEBUG */
-
/*
- * void hw_lock_init(hw_lock_t)
+ * void hw_lock_init(hw_lock_t)
*
- * Initialize a hardware lock. These locks should be cache aligned and a multiple
- * of cache size.
+ * Initialize a hardware lock.
*/
+ .align 5
+ .globl EXT(hw_lock_init)
-ENTRY(hw_lock_init, TAG_NO_FRAME_USED)
+LEXT(hw_lock_init)
- li r0, 0 /* set lock to free == 0 */
- stw r0, 0(r3) /* Initialize the lock */
+ li r0, 0 ; set lock to free == 0
+ stw r0, 0(r3) ; Initialize the lock
blr
/*
- * void hw_lock_unlock(hw_lock_t)
+ * unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout)
*
- * Unconditionally release lock.
- * Release preemption level.
+ * Try to acquire spin-lock. The second parameter is the bit mask to test and set.
+ * multiple bits may be set. Return success (1) or failure (0).
+ * Attempt will fail after timeout ticks of the timebase.
*/
-
-
.align 5
- .globl EXT(hw_lock_unlock)
-
-LEXT(hw_lock_unlock)
-
-#if 0
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lis r5,0xFFFF /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
-#endif
- sync /* Flush writes done under lock */
- li r0, 0 /* set lock to free */
- stw r0, 0(r3)
+ .globl EXT(hw_lock_bit)
- b epStart /* Go enable preemption... */
+LEXT(hw_lock_bit)
+ crset hwtimeout ; timeout option
+ mr r12,r4 ; Load bit mask
+ mr r4,r5 ; Load timeout value
+ b lckcomm ; Join on up...
-/*
- * Special case for internal use. Uses same lock code, but sets up so
- * that there will be no disabling of preemption after locking. Generally
- * used for mutex locks when obtaining the interlock although there is
- * nothing stopping other uses.
- */
-
-lockLock: lis r4,HIGH_ADDR(EXT(LockTimeOut)) /* Get the high part */
- ori r4,r4,LOW_ADDR(EXT(LockTimeOut)) /* And the low part */
- cmplwi cr1,r1,0 /* Set flag to disable disable preemption */
- lwz r4,0(r4) /* Get the timerout value */
- b lockComm /* Join on up... */
-
/*
* void hw_lock_lock(hw_lock_t)
*
- * Acquire lock, spinning until it becomes available.
- * Return with preemption disabled.
- * Apparently not used except by mach_perf.
- * We will just set a default timeout and jump into the NORMAL timeout lock.
+ * Acquire lock, spinning until it becomes available.
+ * Return with preemption disabled.
+ * We will just set a default timeout and jump into the NORMAL timeout lock.
*/
-
.align 5
.globl EXT(hw_lock_lock)
LEXT(hw_lock_lock)
+ crclr hwtimeout ; no timeout option
+ li r4,0 ; request default timeout value
+ li r12,ILK_LOCKED ; Load bit mask
+ b lckcomm ; Join on up...
-lockDisa: lis r4,HIGH_ADDR(EXT(LockTimeOut)) /* Get the high part */
- ori r4,r4,LOW_ADDR(EXT(LockTimeOut)) /* And the low part */
- cmplw cr1,r1,r1 /* Set flag to enable disable preemption */
- lwz r4,0(r4) /* Get the timerout value */
- b lockComm /* Join on up... */
+lockDisa:
+ crset hwtimeout ; timeout option
+ li r4,0 ; request default timeout value
+ li r12,ILK_LOCKED ; Load bit mask
+ b lckcomm ; Join on up...
/*
- * unsigned int hw_lock_to(hw_lock_t, unsigned int timeout)
- *
- * Try to acquire spin-lock. Return success (1) or failure (0).
- * Attempt will fail after timeout ticks of the timebase.
- * We try fairly hard to get this lock. We disable for interruptions, but
- * reenable after a "short" timeout (128 ticks, we may want to change this).
- * After checking to see if the large timeout value (passed in) has expired and a
- * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
- * we return either in abject failure, or disable and go back to the lock sniff routine.
- * If the sniffer finds the lock free, it jumps right up and tries to grab it.
- *
- * One programming note: NEVER DO NOTHING IN HERE NO HOW THAT WILL FORCE US TO CALL
- * THIS WITH TRANSLATION OR INTERRUPTIONS EITHER ON OR OFF, GOSH DARN IT!
+ * unsigned int hw_lock_to(hw_lock_t, unsigned int timeout)
*
+ * Try to acquire spin-lock. Return success (1) or failure (0).
+ * Attempt will fail after timeout ticks of the timebase.
+ * We try fairly hard to get this lock. We disable for interruptions, but
+ * reenable after a "short" timeout (128 ticks, we may want to change this).
+ * After checking to see if the large timeout value (passed in) has expired and a
+ * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
+ * we return either in abject failure, or disable and go back to the lock sniff routine.
+ * If the sniffer finds the lock free, it jumps right up and tries to grab it.
*/
.align 5
.globl EXT(hw_lock_to)
LEXT(hw_lock_to)
-
-#if 0
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lis r5,0xEEEE /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
-#endif
-
-#if CHECKNMI
- mflr r12 ; (TEST/DEBUG)
- bl EXT(ml_sense_nmi) ; (TEST/DEBUG)
- mtlr r12 ; (TEST/DEBUG)
-#endif
-
- cmplw cr1,r1,r1 /* Set flag to enable disable preemption */
-
-lockComm: mfmsr r9 /* Get the MSR value */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- mr r5,r3 /* Get the address of the lock */
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Get MSR that is uninterruptible */
-
- mtmsr r7 /* Turn off interruptions */
- isync ; May have turned off vec and fp here
- mftb r8 /* Get the low part of the time base */
+ crset hwtimeout ; timeout option
+ li r12,ILK_LOCKED ; Load bit mask
+lckcomm:
+ mfsprg r6,1 ; Get the current activation
+ lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
+ addi r5,r5,1 ; Bring up the disable count
+ stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
+ mr r5,r3 ; Get the address of the lock
+ li r8,0 ; Set r8 to zero
+
+lcktry: lwarx r6,0,r5 ; Grab the lock value
+ and. r3,r6,r12 ; Is it locked?
+ or r6,r6,r12 ; Set interlock
+ bne-- lckspin ; Yeah, wait for it to clear...
+ stwcx. r6,0,r5 ; Try to seize that there durn lock
+ bne-- lcktry ; Couldn't get it...
+ li r3,1 ; return true
+ .globl EXT(hwllckPatch_isync)
+LEXT(hwllckPatch_isync)
+ isync ; Make sure we don't use a speculativily loaded value
+ blr ; Go on home...
+
+lckspin: li r6,lgKillResv ; Get killing field
+ stwcx. r6,0,r6 ; Kill reservation
-lcktry: lwarx r6,0,r5 /* Grab the lock value */
- andi. r3,r6,ILK_LOCKED /* Is it locked? */
- ori r6,r6,ILK_LOCKED /* Set interlock */
- bne- lcksniff /* Yeah, wait for it to clear... */
- stwcx. r6,0,r5 /* Try to seize that there durn lock */
- bne- lcktry /* Couldn't get it... */
- li r3,1 /* return true */
- isync /* Make sure we don't use a speculativily loaded value */
- beq+ cr1,daPreComm /* We got it, go disable preemption if we're supposed to... */
- mtmsr r9 ; Restore interrupt state
- blr /* Go on home... */
+ mr. r4,r4 ; Test timeout value
+ bne++ lockspin0
+ lis r4,hi16(EXT(LockTimeOut)) ; Get the high part
+ ori r4,r4,lo16(EXT(LockTimeOut)) ; And the low part
+ lwz r4,0(r4) ; Get the timeout value
+lockspin0:
+ mr. r8,r8 ; Is r8 set to zero
+ bne++ lockspin1 ; If yes, first spin attempt
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+ mftb r8 ; Get timestamp on entry
+ b lcksniff
+
+lockspin1: mtmsr r7 ; Turn off interruptions
+ mftb r8 ; Get timestamp on entry
+
+lcksniff: lwz r3,0(r5) ; Get that lock in here
+ and. r3,r3,r12 ; Is it free yet?
+ beq++ lckretry ; Yeah, try for it again...
- .align 5
-
-lcksniff: lwz r3,0(r5) /* Get that lock in here */
- andi. r3,r3,ILK_LOCKED /* Is it free yet? */
- beq+ lcktry /* Yeah, try for it again... */
+ mftb r10 ; Time stamp us now
+ sub r10,r10,r8 ; Get the elapsed time
+ cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
+ blt++ lcksniff ; Not yet...
- mftb r10 /* Time stamp us now */
- sub r10,r10,r8 /* Get the elapsed time */
- cmplwi r10,128 /* Have we been spinning for 128 tb ticks? */
- blt+ lcksniff /* Not yet... */
-
- mtmsr r9 /* Say, any interrupts pending? */
-
-/* The following instructions force the pipeline to be interlocked to that only one
- instruction is issued per cycle. The insures that we stay enabled for a long enough
- time; if it's too short, pending interruptions will not have a chance to be taken */
-
- subi r4,r4,128 /* Back off elapsed time from timeout value */
- or r4,r4,r4 /* Do nothing here but force a single cycle delay */
- mr. r4,r4 /* See if we used the whole timeout */
- li r3,0 /* Assume a timeout return code */
- or r4,r4,r4 /* Do nothing here but force a single cycle delay */
-
- ble- lckfail /* We failed */
- mtmsr r7 /* Disable for interruptions */
- mftb r8 /* Get the low part of the time base */
- b lcksniff /* Now that we've opened an enable window, keep trying... */
+ mtmsr r9 ; Say, any interrupts pending?
-lckfail: /* We couldn't get the lock */
- li r3,0 /* Set failure return code */
- blr /* Return, head hanging low... */
+; The following instructions force the pipeline to be interlocked to that only one
+; instruction is issued per cycle. The insures that we stay enabled for a long enough
+; time; if it's too short, pending interruptions will not have a chance to be taken
+ subi r4,r4,128 ; Back off elapsed time from timeout value
+ or r4,r4,r4 ; Do nothing here but force a single cycle delay
+ mr. r4,r4 ; See if we used the whole timeout
+ li r3,0 ; Assume a timeout return code
+ or r4,r4,r4 ; Do nothing here but force a single cycle delay
+
+ ble-- lckfail ; We failed
+ b lockspin1 ; Now that we've opened an enable window, keep trying...
+lckretry:
+ mtmsr r9 ; Restore interrupt state
+ li r8,1 ; Insure that R8 is not 0
+ b lcktry
+lckfail: ; We couldn't get the lock
+ bf hwtimeout,lckpanic
+ li r3,0 ; Set failure return code
+ blr ; Return, head hanging low...
+lckpanic:
+ mr r4,r5
+ mr r5,r3
+ lis r3,hi16(lckpanic_str) ; Get the failed lck message
+ ori r3,r3,lo16(lckpanic_str) ; Get the failed lck message
+ bl EXT(panic)
+ BREAKPOINT_TRAP ; We die here anyway
+ .data
+lckpanic_str:
+ STRINGD "timeout on attempt to acquire lock (0x%08X), value = 0x%08X\n\000"
+ .text
/*
- * unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout)
- *
- * Try to acquire spin-lock. The second parameter is the bit mask to test and set.
- * multiple bits may be set. Return success (1) or failure (0).
- * Attempt will fail after timeout ticks of the timebase.
- * We try fairly hard to get this lock. We disable for interruptions, but
- * reenable after a "short" timeout (128 ticks, we may want to shorten this).
- * After checking to see if the large timeout value (passed in) has expired and a
- * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
- * we return either in abject failure, or disable and go back to the lock sniff routine.
- * If the sniffer finds the lock free, it jumps right up and tries to grab it.
- *
- * NOTE WELL!!!! THE ROUTINE hw_lock_phys_vir KNOWS WHAT REGISTERS THIS GUY
- * USES. THIS SAVES A TRANSLATION OFF TO ON TRANSITION AND BACK AND A SAVE AND
- * RESTORE FROM THE STACK.
+ * void hw_lock_unlock(hw_lock_t)
*
+ * Unconditionally release lock.
+ * Release preemption level.
*/
-
.align 5
-
- nop ; Force loop alignment to cache line
- nop
- nop
- nop
-
- .globl EXT(hw_lock_bit)
-
-LEXT(hw_lock_bit)
-
- mfmsr r9 /* Get the MSR value */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Get MSR that is uninterruptible */
-
- mtmsr r7 /* Turn off interruptions */
- isync ; May have turned off vec and fp here
-
- mftb r8 /* Get the low part of the time base */
-
-bittry: lwarx r6,0,r3 /* Grab the lock value */
- and. r0,r6,r4 /* See if any of the lock bits are on */
- or r6,r6,r4 /* Turn on the lock bits */
- bne- bitsniff /* Yeah, wait for it to clear... */
- stwcx. r6,0,r3 /* Try to seize that there durn lock */
- beq+ bitgot /* We got it, yahoo... */
- b bittry /* Just start up again if the store failed... */
-
- .align 5
-
-bitsniff: lwz r6,0(r3) /* Get that lock in here */
- and. r0,r6,r4 /* See if any of the lock bits are on */
- beq+ bittry /* Yeah, try for it again... */
-
- mftb r6 /* Time stamp us now */
- sub r6,r6,r8 /* Get the elapsed time */
- cmplwi r6,128 /* Have we been spinning for 128 tb ticks? */
- blt+ bitsniff /* Not yet... */
-
- mtmsr r9 /* Say, any interrupts pending? */
-
-/* The following instructions force the pipeline to be interlocked to that only one
- instruction is issued per cycle. The insures that we stay enabled for a long enough
- time. If it's too short, pending interruptions will not have a chance to be taken
-*/
-
- subi r5,r5,128 /* Back off elapsed time from timeout value */
- or r5,r5,r5 /* Do nothing here but force a single cycle delay */
- mr. r5,r5 /* See if we used the whole timeout */
- or r5,r5,r5 /* Do nothing here but force a single cycle delay */
-
- ble- bitfail /* We failed */
- mtmsr r7 /* Disable for interruptions */
- mftb r8 /* Get the low part of the time base */
- b bitsniff /* Now that we've opened an enable window, keep trying... */
+ .globl EXT(hw_lock_unlock)
- .align 5
+LEXT(hw_lock_unlock)
-bitgot: mtmsr r9 /* Enable for interruptions */
- li r3,1 /* Set good return code */
- isync /* Make sure we don't use a speculativily loaded value */
- blr
+ .globl EXT(hwulckPatch_isync)
+LEXT(hwulckPatch_isync)
+ isync
+ .globl EXT(hwulckPatch_eieio)
+LEXT(hwulckPatch_eieio)
+ eieio
+ li r0, 0 ; set lock to free
+ stw r0, 0(r3)
-bitfail: li r3,0 /* Set failure return code */
- blr /* Return, head hanging low... */
-
+ b epStart ; Go enable preemption...
/*
- * unsigned int hw_unlock_bit(hw_lock_t, unsigned int bit)
+ * unsigned int hw_unlock_bit(hw_lock_t, unsigned int bit)
*
- * Release bit based spin-lock. The second parameter is the bit mask to clear.
- * Multiple bits may be cleared.
+ * Release bit based spin-lock. The second parameter is the bit mask to clear.
+ * Multiple bits may be cleared.
*
- * NOTE WELL!!!! THE ROUTINE hw_lock_phys_vir KNOWS WHAT REGISTERS THIS GUY
- * USES. THIS SAVES A TRANSLATION OFF TO ON TRANSITION AND BACK AND A SAVE AND
- * RESTORE FROM THE STACK.
*/
-
.align 5
.globl EXT(hw_unlock_bit)
LEXT(hw_unlock_bit)
- sync
-
-ubittry: lwarx r0,0,r3 /* Grab the lock value */
- andc r0,r0,r4 /* Clear the lock bits */
- stwcx. r0,0,r3 /* Try to clear that there durn lock */
- bne- ubittry /* Try again, couldn't save it... */
+ .globl EXT(hwulckbPatch_isync)
+LEXT(hwulckbPatch_isync)
+ isync
+ .globl EXT(hwulckbPatch_eieio)
+LEXT(hwulckbPatch_eieio)
+ eieio
+ubittry: lwarx r0,0,r3 ; Grab the lock value
+ andc r0,r0,r4 ; Clear the lock bits
+ stwcx. r0,0,r3 ; Try to clear that there durn lock
+ bne- ubittry ; Try again, couldn't save it...
- blr /* Leave... */
+ b epStart ; Go enable preemption...
/*
- * unsigned int hw_lock_mbits(hw_lock_t, unsigned int bits, unsigned int value,
+ * unsigned int hw_lock_mbits(hw_lock_t, unsigned int bits, unsigned int value,
* unsigned int newb, unsigned int timeout)
*
- * Try to acquire spin-lock. The second parameter is the bit mask to check.
- * The third is the value of those bits and the 4th is what to set them to.
- * Return success (1) or failure (0).
- * Attempt will fail after timeout ticks of the timebase.
- * We try fairly hard to get this lock. We disable for interruptions, but
- * reenable after a "short" timeout (128 ticks, we may want to shorten this).
- * After checking to see if the large timeout value (passed in) has expired and a
- * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
- * we return either in abject failure, or disable and go back to the lock sniff routine.
- * If the sniffer finds the lock free, it jumps right up and tries to grab it.
- *
+ * Try to acquire spin-lock. The second parameter is the bit mask to check.
+ * The third is the value of those bits and the 4th is what to set them to.
+ * Return success (1) or failure (0).
+ * Attempt will fail after timeout ticks of the timebase.
+ * We try fairly hard to get this lock. We disable for interruptions, but
+ * reenable after a "short" timeout (128 ticks, we may want to shorten this).
+ * After checking to see if the large timeout value (passed in) has expired and a
+ * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
+ * we return either in abject failure, or disable and go back to the lock sniff routine.
+ * If the sniffer finds the lock free, it jumps right up and tries to grab it.
*/
-
.align 5
-
- nop ; Force loop alignment to cache line
- nop
- nop
- nop
-
.globl EXT(hw_lock_mbits)
LEXT(hw_lock_mbits)
- mfmsr r9 ; Get the MSR value
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Get MSR that is uninterruptible
+ li r10,0
- mtmsr r8 ; Turn off interruptions
- isync ; May have turned off vectors or float here
- mftb r10 ; Get the low part of the time base
-
-mbittry: lwarx r12,0,r3 ; Grab the lock value
- and r0,r12,r4 ; Clear extra bits
- andc r12,r12,r4 ; Clear all bits in the bit mask
- or r12,r12,r6 ; Turn on the lock bits
- cmplw r0,r5 ; Are these the right bits?
- bne- mbitsniff ; Nope, wait for it to clear...
- stwcx. r12,0,r3 ; Try to seize that there durn lock
- beq+ mbitgot ; We got it, yahoo...
- b mbittry ; Just start up again if the store failed...
+mbittry: lwarx r12,0,r3 ; Grab the lock value
+ and r0,r12,r4 ; Clear extra bits
+ andc r12,r12,r4 ; Clear all bits in the bit mask
+ or r12,r12,r6 ; Turn on the lock bits
+ cmplw r0,r5 ; Are these the right bits?
+ bne-- mbitspin ; Nope, wait for it to clear...
+ stwcx. r12,0,r3 ; Try to seize that there durn lock
+ beq++ mbitgot ; We got it, yahoo...
+ b mbittry ; Just start up again if the store failed...
.align 5
+mbitspin: li r11,lgKillResv ; Point to killing field
+ stwcx. r11,0,r11 ; Kill it
-mbitsniff: lwz r12,0(r3) ; Get that lock in here
- and r0,r12,r4 ; Clear extra bits
- cmplw r0,r5 ; Are these the right bits?
- beq+ mbittry ; Yeah, try for it again...
+ mr. r10,r10 ; Is r10 set to zero
+ bne++ mbitspin0 ; If yes, first spin attempt
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r8,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r8,r9,r8 ; Clear EE as well
+ mtmsr r8 ; Turn off interruptions
+ isync ; May have turned off vectors or float here
+ mftb r10 ; Get the low part of the time base
+ b mbitsniff
+mbitspin0:
+ mtmsr r8 ; Turn off interruptions
+ mftb r10 ; Get the low part of the time base
+mbitsniff:
+ lwz r12,0(r3) ; Get that lock in here
+ and r0,r12,r4 ; Clear extra bits
+ cmplw r0,r5 ; Are these the right bits?
+ beq++ mbitretry ; Yeah, try for it again...
- mftb r11 ; Time stamp us now
- sub r11,r11,r10 ; Get the elapsed time
- cmplwi r11,128 ; Have we been spinning for 128 tb ticks?
- blt+ mbitsniff ; Not yet...
+ mftb r11 ; Time stamp us now
+ sub r11,r11,r10 ; Get the elapsed time
+ cmplwi r11,128 ; Have we been spinning for 128 tb ticks?
+ blt++ mbitsniff ; Not yet...
- mtmsr r9 ; Say, any interrupts pending?
+ mtmsr r9 ; Say, any interrupts pending?
; The following instructions force the pipeline to be interlocked to that only one
; instruction is issued per cycle. The insures that we stay enabled for a long enough
; time. If it is too short, pending interruptions will not have a chance to be taken
- subi r7,r7,128 ; Back off elapsed time from timeout value
- or r7,r7,r7 ; Do nothing here but force a single cycle delay
- mr. r7,r7 ; See if we used the whole timeout
- or r7,r7,r7 ; Do nothing here but force a single cycle delay
+ subi r7,r7,128 ; Back off elapsed time from timeout value
+ or r7,r7,r7 ; Do nothing here but force a single cycle delay
+ mr. r7,r7 ; See if we used the whole timeout
+ or r7,r7,r7 ; Do nothing here but force a single cycle delay
- ble- mbitfail ; We failed
- mtmsr r8 ; Disable for interruptions
- mftb r10 ; Get the low part of the time base
- b mbitsniff ; Now that we have opened an enable window, keep trying...
+ ble-- mbitfail ; We failed
+ b mbitspin0 ; Now that we have opened an enable window, keep trying...
+mbitretry:
+ mtmsr r9 ; Enable for interruptions
+ li r10,1 ; Make sure this is non-zero
+ b mbittry
.align 5
-
-mbitgot: mtmsr r9 ; Enable for interruptions
- li r3,1 ; Set good return code
- isync ; Make sure we do not use a speculativily loaded value
+mbitgot:
+ li r3,1 ; Set good return code
+ .globl EXT(hwlmlckPatch_isync)
+LEXT(hwlmlckPatch_isync)
+ isync ; Make sure we do not use a speculativily loaded value
blr
-mbitfail: li r3,0 ; Set failure return code
- blr ; Return, head hanging low...
-
+mbitfail: li r3,0 ; Set failure return code
+ blr ; Return, head hanging low...
/*
* unsigned int hw_cpu_sync(unsigned int *, unsigned int timeout)
*
- * Spin until word hits 0 or timeout.
- * Return success (1) or failure (0).
- * Attempt will fail after timeout ticks of the timebase.
- *
- * The theory is that a processor will bump a counter as it signals
- * other processors. Then it will spin untl the counter hits 0 (or
- * times out). The other processors, as it receives the signal will
- * decrement the counter.
+ * Spin until word hits 0 or timeout.
+ * Return success (1) or failure (0).
+ * Attempt will fail after timeout ticks of the timebase.
*
- * The other processors use interlocked update to decrement, this one
- * does not need to interlock.
+ * The theory is that a processor will bump a counter as it signals
+ * other processors. Then it will spin untl the counter hits 0 (or
+ * times out). The other processors, as it receives the signal will
+ * decrement the counter.
*
+ * The other processors use interlocked update to decrement, this one
+ * does not need to interlock.
*/
-
.align 5
-
.globl EXT(hw_cpu_sync)
LEXT(hw_cpu_sync)
- mftb r10 ; Get the low part of the time base
- mr r9,r3 ; Save the sync word address
- li r3,1 ; Assume we work
+ mftb r10 ; Get the low part of the time base
+ mr r9,r3 ; Save the sync word address
+ li r3,1 ; Assume we work
-csynctry: lwz r11,0(r9) ; Grab the sync value
- mr. r11,r11 ; Counter hit 0?
- beqlr- ; Yeah, we are sunk...
- mftb r12 ; Time stamp us now
+csynctry: lwz r11,0(r9) ; Grab the sync value
+ mr. r11,r11 ; Counter hit 0?
+ beqlr- ; Yeah, we are sunk...
+ mftb r12 ; Time stamp us now
- sub r12,r12,r10 ; Get the elapsed time
- cmplw r4,r12 ; Have we gone too long?
- bge+ csynctry ; Not yet...
+ sub r12,r12,r10 ; Get the elapsed time
+ cmplw r4,r12 ; Have we gone too long?
+ bge+ csynctry ; Not yet...
- li r3,0 ; Set failure...
- blr ; Return, head hanging low...
+ li r3,0 ; Set failure...
+ blr ; Return, head hanging low...
/*
* unsigned int hw_cpu_wcng(unsigned int *, unsigned int, unsigned int timeout)
*
- * Spin until word changes or timeout.
- * Return success (1) or failure (0).
- * Attempt will fail after timeout ticks of the timebase.
- *
- * This is used to insure that a processor passes a certain point.
- * An example of use is to monitor the last interrupt time in the
- * per_proc block. This can be used to insure that the other processor
- * has seen at least one interrupt since a specific time.
+ * Spin until word changes or timeout.
+ * Return success (1) or failure (0).
+ * Attempt will fail after timeout ticks of the timebase.
*
+ * This is used to insure that a processor passes a certain point.
+ * An example of use is to monitor the last interrupt time in the
+ * per_proc block. This can be used to insure that the other processor
+ * has seen at least one interrupt since a specific time.
*/
-
.align 5
-
.globl EXT(hw_cpu_wcng)
LEXT(hw_cpu_wcng)
- mftb r10 ; Get the low part of the time base
- mr r9,r3 ; Save the sync word address
- li r3,1 ; Assume we work
+ mftb r10 ; Get the low part of the time base
+ mr r9,r3 ; Save the sync word address
+ li r3,1 ; Assume we work
-wcngtry: lwz r11,0(r9) ; Grab the value
- cmplw r11,r4 ; Do they still match?
- bnelr- ; Nope, cool...
- mftb r12 ; Time stamp us now
+wcngtry: lwz r11,0(r9) ; Grab the value
+ cmplw r11,r4 ; Do they still match?
+ bnelr- ; Nope, cool...
+ mftb r12 ; Time stamp us now
- sub r12,r12,r10 ; Get the elapsed time
- cmplw r5,r12 ; Have we gone too long?
- bge+ wcngtry ; Not yet...
+ sub r12,r12,r10 ; Get the elapsed time
+ cmplw r5,r12 ; Have we gone too long?
+ bge+ wcngtry ; Not yet...
- li r3,0 ; Set failure...
- blr ; Return, head hanging low...
+ li r3,0 ; Set failure...
+ blr ; Return, head hanging low...
/*
- * unsigned int hw_lock_try(hw_lock_t)
+ * unsigned int hw_lock_try(hw_lock_t)
*
- * Try to acquire spin-lock. Return success (1) or failure (0)
- * Returns with preemption disabled on success.
+ * Try to acquire spin-lock. Return success (1) or failure (0)
+ * Returns with preemption disabled on success.
*
*/
.align 5
LEXT(hw_lock_try)
-#if 0
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lis r5,0x9999 /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
-#endif
- mfmsr r9 /* Save the MSR value */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruption bit */
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
-#if MACH_LDEBUG
- lis r5, 0x10 /* roughly 1E6 */
- mtctr r5
-#endif /* MACH_LDEBUG */
-
- mtmsr r7 /* Disable interruptions and thus, preemption */
- isync ; May have turned off fp/vec here
-.L_lock_try_loop:
+ mtmsr r7 ; Disable interruptions and thus, preemption
-#if MACH_LDEBUG
- bdnz+ 0f /* Count attempts */
- mtmsr r9 /* Restore enablement */
- BREAKPOINT_TRAP /* Get to debugger */
- mtmsr r7 /* Disable interruptions and thus, preemption */
-0:
-#endif /* MACH_LDEBUG */
+ lwz r5,0(r3) ; Quick load
+ andi. r6,r5,ILK_LOCKED ; TEST...
+ bne-- .L_lock_try_failed ; No go...
- lwarx r5,0,r3 /* Ld from addr of arg and reserve */
+.L_lock_try_loop:
+ lwarx r5,0,r3 ; Ld from addr of arg and reserve
- andi. r6,r5,ILK_LOCKED /* TEST... */
+ andi. r6,r5,ILK_LOCKED ; TEST...
ori r5,r5,ILK_LOCKED
- bne- .L_lock_try_failed /* branch if taken. Predict free */
+ bne-- .L_lock_try_failedX ; branch if taken. Predict free
- stwcx. r5,0,r3 /* And SET (if still reserved) */
- mfsprg r6,0 /* Get the per_proc block */
- bne- .L_lock_try_loop /* If set failed, loop back */
+ stwcx. r5,0,r3 ; And SET (if still reserved)
+ bne-- .L_lock_try_loop ; If set failed, loop back
+ .globl EXT(hwltlckPatch_isync)
+LEXT(hwltlckPatch_isync)
isync
- lwz r5,PP_PREEMPT_CNT(r6) /* Get the preemption level */
- addi r5,r5,1 /* Bring up the disable count */
- stw r5,PP_PREEMPT_CNT(r6) /* Save it back */
+ mfsprg r6,1 ; Get current activation
+ lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
+ addi r5,r5,1 ; Bring up the disable count
+ stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
- mtmsr r9 /* Allow interruptions now */
- li r3,1 /* Set that the lock was free */
+ mtmsr r9 ; Allow interruptions now
+ li r3,1 ; Set that the lock was free
blr
+.L_lock_try_failedX:
+ li r6,lgKillResv ; Killing field
+ stwcx. r6,0,r6 ; Kill reservation
+
.L_lock_try_failed:
- mtmsr r9 /* Allow interruptions now */
- li r3,0 /* FAILURE - lock was taken */
+ mtmsr r9 ; Allow interruptions now
+ li r3,0 ; FAILURE - lock was taken
blr
/*
- * unsigned int hw_lock_held(hw_lock_t)
- *
- * Return 1 if lock is held
- * Doesn't change preemption state.
- * N.B. Racy, of course.
+ * unsigned int hw_lock_held(hw_lock_t)
*
+ * Return 1 if lock is held
+ * Doesn't change preemption state.
+ * N.B. Racy, of course.
*/
.align 5
.globl EXT(hw_lock_held)
LEXT(hw_lock_held)
-#if 0
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lis r5,0x8888 /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
-#endif
- isync /* Make sure we don't use a speculativily fetched lock */
- lwz r3, 0(r3) /* Return value of lock */
+ isync ; Make sure we don't use a speculativily fetched lock
+ lwz r3, 0(r3) ; Get lock value
+ andi. r6,r3,ILK_LOCKED ; Extract the ILK_LOCKED bit
blr
/*
* uint32_t hw_compare_and_store(uint32_t oldval, uint32_t newval, uint32_t *dest)
*
- * Compare old to area if equal, store new, and return true
- * else return false and no store
- * This is an atomic operation
- *
+ * Compare old to area if equal, store new, and return true
+ * else return false and no store
+ * This is an atomic operation
*/
.align 5
.globl EXT(hw_compare_and_store)
LEXT(hw_compare_and_store)
- mr r6,r3 /* Save the old value */
-
-cstry: lwarx r9,0,r5 /* Grab the area value */
- li r3,1 /* Assume it works */
- cmplw cr0,r9,r6 /* Does it match the old value? */
- bne- csfail /* No, it must have changed... */
- stwcx. r4,0,r5 /* Try to save the new value */
- bne- cstry /* Didn't get it, try again... */
- isync /* Just hold up prefetch */
- blr /* Return... */
+ mr r6,r3 ; Save the old value
+
+cstry: lwarx r9,0,r5 ; Grab the area value
+ li r3,1 ; Assume it works
+ cmplw cr0,r9,r6 ; Does it match the old value?
+ bne-- csfail ; No, it must have changed...
+ stwcx. r4,0,r5 ; Try to save the new value
+ bne-- cstry ; Didn't get it, try again...
+ .globl EXT(hwcsatomicPatch_isync)
+LEXT(hwcsatomicPatch_isync)
+ isync ; Just hold up prefetch
+ blr ; Return...
+
+csfail: li r3,lgKillResv ; Killing field
+ stwcx. r3,0,r3 ; Blow reservation
-csfail: li r3,0 /* Set failure */
- blr /* Better luck next time... */
+ li r3,0 ; Set failure
+ blr ; Better luck next time...
/*
* uint32_t hw_atomic_add(uint32_t *dest, uint32_t delt)
*
- * Atomically add the second parameter to the first.
- * Returns the result.
+ * Atomically add the second parameter to the first.
+ * Returns the result.
*
*/
.align 5
LEXT(hw_atomic_add)
- mr r6,r3 /* Save the area */
+ mr r6,r3 ; Save the area
-addtry: lwarx r3,0,r6 /* Grab the area value */
- add r3,r3,r4 /* Add the value */
- stwcx. r3,0,r6 /* Try to save the new value */
- bne- addtry /* Didn't get it, try again... */
- blr /* Return... */
+addtry: lwarx r3,0,r6 ; Grab the area value
+ add r3,r3,r4 ; Add the value
+ stwcx. r3,0,r6 ; Try to save the new value
+ bne-- addtry ; Didn't get it, try again...
+ blr ; Return...
/*
* uint32_t hw_atomic_sub(uint32_t *dest, uint32_t delt)
*
- * Atomically subtract the second parameter from the first.
- * Returns the result.
+ * Atomically subtract the second parameter from the first.
+ * Returns the result.
*
*/
.align 5
LEXT(hw_atomic_sub)
- mr r6,r3 /* Save the area */
+ mr r6,r3 ; Save the area
-subtry: lwarx r3,0,r6 /* Grab the area value */
- sub r3,r3,r4 /* Subtract the value */
- stwcx. r3,0,r6 /* Try to save the new value */
- bne- subtry /* Didn't get it, try again... */
- blr /* Return... */
+subtry: lwarx r3,0,r6 ; Grab the area value
+ sub r3,r3,r4 ; Subtract the value
+ stwcx. r3,0,r6 ; Try to save the new value
+ bne-- subtry ; Didn't get it, try again...
+ blr ; Return...
/*
* uint32_t hw_atomic_or(uint32_t *dest, uint32_t mask)
*
- * Atomically ORs the second parameter into the first.
- * Returns the result.
- *
+ * Atomically ORs the second parameter into the first.
+ * Returns the result.
*/
.align 5
.globl EXT(hw_atomic_or)
-
LEXT(hw_atomic_or)
+ .globl EXT(hw_atomic_or_noret)
+LEXT(hw_atomic_or_noret)
+ mr r6,r3 ; Save the area
- mr r6,r3 ; Save the area
-
-ortry: lwarx r3,0,r6 ; Grab the area value
- or r3,r3,r4 ; OR the value
- stwcx. r3,0,r6 ; Try to save the new value
- bne- ortry ; Did not get it, try again...
- blr ; Return...
+ortry: lwarx r3,0,r6 ; Grab the area value
+ or r3,r3,r4 ; OR the value
+ stwcx. r3,0,r6 ; Try to save the new value
+ bne-- ortry ; Did not get it, try again...
+ blr ; Return...
/*
* uint32_t hw_atomic_and(uint32_t *dest, uint32_t mask)
*
- * Atomically ANDs the second parameter with the first.
- * Returns the result.
+ * Atomically ANDs the second parameter with the first.
+ * Returns the result.
*
*/
.align 5
.globl EXT(hw_atomic_and)
-
LEXT(hw_atomic_and)
+ .globl EXT(hw_atomic_and_noret)
+LEXT(hw_atomic_and_noret)
+ mr r6,r3 ; Save the area
- mr r6,r3 ; Save the area
-
-andtry: lwarx r3,0,r6 ; Grab the area value
- and r3,r3,r4 ; AND the value
- stwcx. r3,0,r6 ; Try to save the new value
- bne- andtry ; Did not get it, try again...
- blr ; Return...
+andtry: lwarx r3,0,r6 ; Grab the area value
+ and r3,r3,r4 ; AND the value
+ stwcx. r3,0,r6 ; Try to save the new value
+ bne-- andtry ; Did not get it, try again...
+ blr ; Return...
/*
* void hw_queue_atomic(unsigned int * anchor, unsigned int * elem, unsigned int disp)
*
- * Atomically inserts the element at the head of the list
- * anchor is the pointer to the first element
- * element is the pointer to the element to insert
- * disp is the displacement into the element to the chain pointer
+ * Atomically inserts the element at the head of the list
+ * anchor is the pointer to the first element
+ * element is the pointer to the element to insert
+ * disp is the displacement into the element to the chain pointer
*
+ * NOTE: OSEnqueueAtomic() is aliased to this, see xnu/libkern/Makefile
*/
.align 5
.globl EXT(hw_queue_atomic)
LEXT(hw_queue_atomic)
- mr r7,r4 /* Make end point the same as start */
- mr r8,r5 /* Copy the displacement also */
- b hw_queue_comm /* Join common code... */
+ mr r7,r4 ; Make end point the same as start
+ mr r8,r5 ; Copy the displacement also
+ b hw_queue_comm ; Join common code...
/*
* void hw_queue_atomic_list(unsigned int * anchor, unsigned int * first, unsigned int * last, unsigned int disp)
*
- * Atomically inserts the list of elements at the head of the list
- * anchor is the pointer to the first element
- * first is the pointer to the first element to insert
- * last is the pointer to the last element to insert
- * disp is the displacement into the element to the chain pointer
- *
+ * Atomically inserts the list of elements at the head of the list
+ * anchor is the pointer to the first element
+ * first is the pointer to the first element to insert
+ * last is the pointer to the last element to insert
+ * disp is the displacement into the element to the chain pointer
*/
.align 5
.globl EXT(hw_queue_atomic_list)
LEXT(hw_queue_atomic_list)
- mr r7,r5 /* Make end point the same as start */
- mr r8,r6 /* Copy the displacement also */
+ mr r7,r5 ; Make end point the same as start
+ mr r8,r6 ; Copy the displacement also
hw_queue_comm:
- lwarx r9,0,r3 /* Pick up the anchor */
- stwx r9,r8,r7 /* Chain that to the end of the new stuff */
- eieio ; Make sure this store makes it before the anchor update
- stwcx. r4,0,r3 /* Try to chain into the front */
- bne- hw_queue_comm /* Didn't make it, try again... */
-
- blr /* Return... */
+ lwarx r9,0,r3 ; Pick up the anchor
+ stwx r9,r8,r7 ; Chain that to the end of the new stuff
+ eieio ; Make sure this store makes it before the anchor update
+ stwcx. r4,0,r3 ; Try to chain into the front
+ bne-- hw_queue_comm ; Didn't make it, try again...
+
+ blr ; Return...
/*
* unsigned int *hw_dequeue_atomic(unsigned int *anchor, unsigned int disp)
*
- * Atomically removes the first element in a list and returns it.
- * anchor is the pointer to the first element
- * disp is the displacement into the element to the chain pointer
- * Returns element if found, 0 if empty.
+ * Atomically removes the first element in a list and returns it.
+ * anchor is the pointer to the first element
+ * disp is the displacement into the element to the chain pointer
+ * Returns element if found, 0 if empty.
*
+ * NOTE: OSDequeueAtomic() is aliased to this, see xnu/libkern/Makefile
*/
.align 5
.globl EXT(hw_dequeue_atomic)
LEXT(hw_dequeue_atomic)
- mr r5,r3 /* Save the anchor */
+ mr r5,r3 ; Save the anchor
hw_dequeue_comm:
- lwarx r3,0,r5 /* Pick up the anchor */
- mr. r3,r3 /* Is the list empty? */
- beqlr- /* Leave it list empty... */
- lwzx r9,r4,r3 /* Get the next in line */
- stwcx. r9,0,r5 /* Try to chain into the front */
- beqlr+ ; Got the thing, go away with it...
- b hw_dequeue_comm ; Did not make it, try again...
+ lwarx r3,0,r5 ; Pick up the anchor
+ mr. r3,r3 ; Is the list empty?
+ beq-- hdcFail ; Leave it list empty...
+ lwzx r9,r4,r3 ; Get the next in line
+ stwcx. r9,0,r5 ; Try to chain into the front
+ beqlr++ ; Got the thing, go away with it...
+ b hw_dequeue_comm ; Did not make it, try again...
+
+hdcFail: li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Dump reservation
+ blr ; Leave...
+
/*
- * void mutex_init(mutex_t* l, etap_event_t etap)
+ * Routines for mutex lock debugging.
+ */
+
+/*
+ * Gets lock check flags in CR6: CR bits 24-27
*/
+#define CHECK_SETUP(rg) \
+ lbz rg,lglcksWork(0) __ASMNL__ \
+ mtcrf 2,rg __ASMNL__
-ENTRY(mutex_init,TAG_NO_FRAME_USED)
+
+/*
+ * Checks for expected lock type.
+ */
+#define CHECK_MUTEX_TYPE() \
+ bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \
+ bt 24+disLktypeb,1f __ASMNL__ \
+ lwz r10,MUTEX_TYPE(r3) __ASMNL__ \
+ cmpwi r10,MUTEX_TAG __ASMNL__ \
+ beq++ 1f __ASMNL__ \
+ PROLOG(0) __ASMNL__ \
+ mr r4,r11 __ASMNL__ \
+ mr r5,r10 __ASMNL__ \
+ lis r3,hi16(not_a_mutex) __ASMNL__ \
+ ori r3,r3,lo16(not_a_mutex) __ASMNL__ \
+ bl EXT(panic) __ASMNL__ \
+ BREAKPOINT_TRAP __ASMNL__ \
+1:
+
+ .data
+not_a_mutex:
+ STRINGD "mutex (0x%08X) not a mutex type (0x%08X)\n\000"
+ .text
+
+/*
+ * Verifies return to the correct thread in "unlock" situations.
+ */
+#define CHECK_THREAD(thread_offset) \
+ bf MUTEX_ATTR_DEBUGb,3f __ASMNL__ \
+ bt 24+disLkThreadb,3f __ASMNL__ \
+ mfsprg r10,1 __ASMNL__ \
+ lwz r5,MUTEX_DATA(r3) __ASMNL__ \
+ rlwinm. r9,r5,0,0,29 __ASMNL__ \
+ bne++ 1f __ASMNL__ \
+ lis r3,hi16(not_held) __ASMNL__ \
+ ori r3,r3,lo16(not_held) __ASMNL__ \
+ b 2f __ASMNL__ \
+1: __ASMNL__ \
+ cmpw r9,r10 __ASMNL__ \
+ beq++ 3f __ASMNL__ \
+ mr r5,r10 __ASMNL__ \
+ mr r6,r9 __ASMNL__ \
+ lis r3,hi16(wrong_thread) __ASMNL__ \
+ ori r3,r3,lo16(wrong_thread) __ASMNL__ \
+2: __ASMNL__ \
+ mr r4,r11 __ASMNL__ \
+ PROLOG(0) __ASMNL__ \
+ bl EXT(panic) __ASMNL__ \
+ BREAKPOINT_TRAP __ASMNL__ \
+3:
+
+ .data
+not_held:
+ STRINGD "mutex (0x%08X) not held\n\000"
+wrong_thread:
+ STRINGD "mutex (0x%08X) unlocked by non-owner(0x%08X), current owner(0x%08X)\n\000"
+ .text
+
+#define CHECK_MYLOCK() \
+ bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \
+ bt 24+disLkMyLckb,1f __ASMNL__ \
+ mfsprg r10,1 __ASMNL__ \
+ lwz r9,MUTEX_DATA(r3) __ASMNL__ \
+ rlwinm r9,r9,0,0,29 __ASMNL__ \
+ cmpw r9,r10 __ASMNL__ \
+ bne++ 1f __ASMNL__ \
+ mr r4,r11 __ASMNL__ \
+ lis r3, hi16(mylock_attempt) __ASMNL__ \
+ ori r3,r3,lo16(mylock_attempt) __ASMNL__ \
+ bl EXT(panic) __ASMNL__ \
+ BREAKPOINT_TRAP __ASMNL__ \
+1:
+
+ .data
+mylock_attempt:
+ STRINGD "mutex (0x%08X) recursive lock attempt\n\000"
+ .text
+
+#define LCK_STACK(lck, stack, lck_stack, frame_cnt, lr_save, tmp) \
+ bf 24+enaLkExtStckb,3f __ASMNL__ \
+ addi lck_stack,lck,MUTEX_STACK __ASMNL__ \
+ li frame_cnt,MUTEX_FRAMES-1 __ASMNL__ \
+1: __ASMNL__ \
+ mr tmp,stack __ASMNL__ \
+ lwz stack,0(stack) __ASMNL__ \
+ xor tmp,stack,tmp __ASMNL__ \
+ cmplwi tmp,8192 __ASMNL__ \
+ bge-- 2f __ASMNL__ \
+ lwz lr_save,FM_LR_SAVE(stack) __ASMNL__ \
+ stwu lr_save,4(lck_stack) __ASMNL__ \
+ subi frame_cnt,frame_cnt,1 __ASMNL__ \
+ cmpi cr0,frame_cnt,0 __ASMNL__ \
+ bne 1b __ASMNL__ \
+ b 3f __ASMNL__ \
+2: __ASMNL__ \
+ li tmp,0 __ASMNL__ \
+ stwu tmp,4(lck_stack) __ASMNL__ \
+ subi frame_cnt,frame_cnt,1 __ASMNL__ \
+ cmpi cr0,frame_cnt,0 __ASMNL__ \
+ bne 2b __ASMNL__ \
+3:
+
+/*
+ * void mutex_init(mutex_t* l, etap_event_t etap)
+ *
+ */
+ .align 5
+ .globl EXT(mutex_init)
+LEXT(mutex_init)
PROLOG(0)
- li r10, 0
- stw r10, LOCK_DATA(r3) /* clear lock word */
- sth r10, MUTEX_WAITERS(r3) /* init waiter count */
- sth r10, MUTEX_PROMOTED_PRI(r3)
+ li r10,0
+ stw r10,MUTEX_DATA(r3) ; clear lock word
+ sth r10,MUTEX_WAITERS(r3) ; init waiter count
+ sth r10,MUTEX_PROMOTED_PRI(r3)
#if MACH_LDEBUG
- stw r10, MUTEX_PC(r3) /* init caller pc */
- stw r10, MUTEX_THREAD(r3) /* and owning thread */
- li r10, MUTEX_TAG
- stw r10, MUTEX_TYPE(r3) /* set lock type */
+ li r11,MUTEX_ATTR_DEBUG
+ stw r10,MUTEX_STACK(r3) ; init caller pc
+ stw r10,MUTEX_THREAD(r3) ; and owning thread
+ li r9, MUTEX_TAG
+ stw r9, MUTEX_TYPE(r3) ; set lock type
+ stw r11,MUTEX_ATTR(r3)
+ addi r8,r3,MUTEX_STACK-4
+ li r9,MUTEX_FRAMES
+mlistck:
+ stwu r10,4(r8) ; init stack
+ subi r9,r9,1
+ cmpi cr0,r9,0
+ bne mlistck
#endif /* MACH_LDEBUG */
-
-#if ETAP_LOCK_TRACE
- bl EXT(etap_mutex_init) /* init ETAP data */
-#endif /* ETAP_LOCK_TRACE */
-
EPILOG
blr
/*
- * void mutex_lock(mutex_t*)
+ * void lck_mtx_lock_ext(lck_mtx_ext_t*)
+ *
*/
-
.align 5
+ .globl EXT(lck_mtx_lock_ext)
+LEXT(lck_mtx_lock_ext)
+#if MACH_LDEBUG
.globl EXT(mutex_lock)
LEXT(mutex_lock)
.globl EXT(_mutex_lock)
LEXT(_mutex_lock)
-
-#if !MACH_LDEBUG
- mfsprg r6,1 /* load the current thread */
-L_mutex_lock_loop:
- lwarx r5,0,r3 /* load the mutex lock */
- mr. r5,r5
- bne- L_mutex_lock_slow /* go to the slow path */
- stwcx. r6,0,r3 /* grab the lock */
- bne- L_mutex_lock_loop /* loop back if failed */
- isync /* stop prefeteching */
- blr
-L_mutex_lock_slow:
-#endif
-#if CHECKNMI
- mflr r12 ; (TEST/DEBUG)
- bl EXT(ml_sense_nmi) ; (TEST/DEBUG)
- mtlr r12 ; (TEST/DEBUG)
#endif
+ mr r11,r3 ; Save lock addr
+mlckeEnter:
+ lwz r0,MUTEX_ATTR(r3)
+ mtcrf 1,r0 ; Set cr7
+ CHECK_SETUP(r12)
+ CHECK_MUTEX_TYPE()
- PROLOG(12)
-#if MACH_LDEBUG
+ bf MUTEX_ATTR_DEBUGb,L_mutex_lock_assert_wait_2
+ PROLOG(0)
bl EXT(assert_wait_possible)
mr. r3,r3
bne L_mutex_lock_assert_wait_1
lis r3,hi16(L_mutex_lock_assert_wait_panic_str)
ori r3,r3,lo16(L_mutex_lock_assert_wait_panic_str)
bl EXT(panic)
+ BREAKPOINT_TRAP ; We die here anyway
.data
L_mutex_lock_assert_wait_panic_str:
- STRINGD "mutex_lock: assert_wait_possible false\n\000"
+ STRINGD "mutex lock attempt with assert_wait_possible false\n\000"
.text
L_mutex_lock_assert_wait_1:
lwz r3,FM_ARG0(r1)
-#endif
-
-#if ETAP_LOCK_TRACE
- li r0, 0
- stw r0,SWT_HI(r1) /* set wait time to 0 (HI) */
- stw r0,SWT_LO(r1) /* set wait time to 0 (LO) */
- stw r0,MISSED(r1) /* clear local miss marker */
-#endif /* ETAP_LOCK_TRACE */
+ lwz r11,FM_ARG0+0x04(r1)
+ lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
+ mtcr r2
+ EPILOG
+L_mutex_lock_assert_wait_2:
+
+ mfsprg r6,1 ; load the current thread
+ bf MUTEX_ATTR_STATb,mlckestatskip ; Branch if no stat
+ lwz r5,MUTEX_GRP(r3) ; Load lock group
+ li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset
+mlckestatloop:
+ lwarx r8,r7,r5 ; Load stat util cnt
+ addi r8,r8,1 ; Increment stat util cnt
+ stwcx. r8,r7,r5 ; Store stat util cnt
+ bne-- mlckestatloop ; Retry if failed
+ mr. r8,r8 ; Test for zero
+ bne++ mlckestatskip ; Did stat util cnt wrapped?
+ lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt
+ addi r8,r8,1 ; Increment upper stat util cnt
+ stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt
+mlckestatskip:
+ lwz r5,MUTEX_DATA(r3) ; Get the lock quickly
+ li r4,0
+ li r8,0
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+ mr. r5,r5 ; Quick check
+ bne-- mlckespin01 ; Can not get it right now...
+
+mlcketry:
+ lwarx r5,MUTEX_DATA,r3 ; load the mutex lock
+ mr. r5,r5
+ bne-- mlckespin0 ; Can not get it right now...
+ stwcx. r6,MUTEX_DATA,r3 ; grab the lock
+ bne-- mlcketry ; loop back if failed
+ .globl EXT(mlckePatch_isync)
+LEXT(mlckePatch_isync)
+ isync ; stop prefeteching
+ mflr r12
+ bf MUTEX_ATTR_DEBUGb,mlckedebskip
+ mr r8,r6 ; Get the active thread
+ stw r12,MUTEX_STACK(r3) ; Save our caller
+ stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
+ mr r5,r1
+ LCK_STACK(r3,r5,r6,r7,r8,r10)
+mlckedebskip:
+ mtmsr r9 ; Say, any interrupts pending?
+ blr
- CHECK_SETUP(r12)
- CHECK_MUTEX_TYPE()
- CHECK_NO_SIMPLELOCKS()
+mlckespin0:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Kill reservation
+mlckespin01:
+ mflr r12
+ mtmsr r9 ; Say, any interrupts pending?
+ bl mlckspin1
+ mtmsr r7 ; Turn off interruptions, vec and fp off already
+ mtlr r12
+ b mlcketry
-.L_ml_retry:
-#if 0
- mfsprg r4,0 /* (TEST/DEBUG) */
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lwz r4,PP_ACTIVE_THREAD(r4) /* (TEST/DEBUG) */
- lis r5,0xAAAA /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
+/*
+ * void lck_mtx_lock(lck_mtx_t*)
+ *
+ */
+ .align 5
+ .globl EXT(lck_mtx_lock)
+LEXT(lck_mtx_lock)
+
+#if !MACH_LDEBUG
+ .globl EXT(mutex_lock)
+LEXT(mutex_lock)
+
+ .globl EXT(_mutex_lock)
+LEXT(_mutex_lock)
#endif
- bl lockDisa /* Go get a lock on the mutex's interlock lock */
- mr. r4,r3 /* Did we get it? */
- lwz r3,FM_ARG0(r1) /* Restore the lock address */
- bne+ mlGotInt /* We got it just fine... */
+ mfsprg r6,1 ; load the current thread
+ lwz r5,MUTEX_DATA(r3) ; Get the lock quickly
+ mr r11,r3 ; Save lock addr
+ li r4,0
+ li r8,0
+ li r9,0
+ mr. r5,r5 ; Quick check
+ bne-- mlckspin00 ; Indirect or Can not get it right now...
+
+mlcktry:
+ lwarx r5,MUTEX_DATA,r3 ; load the mutex lock
+ mr. r5,r5
+ bne-- mlckspin01 ; Can not get it right now...
+ stwcx. r6,MUTEX_DATA,r3 ; grab the lock
+ bne-- mlcktry ; loop back if failed
+ .globl EXT(mlckPatch_isync)
+LEXT(mlckPatch_isync)
+ isync ; stop prefeteching
+ blr
+; Need to debug making blr above a patch point and record:
+; LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE)
+
+mlckspin00:
+ cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
+ bne-- mlckspin02 ; No, go handle contention
+ lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
+ b mlckeEnter
+mlckspin01:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Kill reservation
+mlckspin02:
+ mflr r12
+ li r0,0
+ mtcrf 1,r0 ; Set cr7 to zero
+ bl mlckspin1
+ mtlr r12
+ b mlcktry
+
+
+mlckspin1:
+ mr. r4,r4 ; Test timeout value
+ bne++ mlckspin2
+ lis r4,hi16(EXT(MutexSpin)) ; Get the high part
+ ori r4,r4,lo16(EXT(MutexSpin) ) ; And the low part
+ lwz r4,0(r4) ; Get spin timerout value
+ mr. r4,r4 ; Test spin timeout value
+ bne++ mlckspin2 ; Is spin timeout requested
+ crclr mlckmiss ; Clear miss test
+ b mlckslow1 ; Don't try to spin
+
+mlckspin2: mr. r8,r8 ; Is r8 set to zero
+ bne++ mlckspin3 ; If yes, first spin attempt
+ crclr mlckmiss ; Clear miss test
+ mr. r9,r9 ; Is r9 set to zero
+ bne++ mlckspin3 ; If yes, r9 set with msr value
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+ mftb r8 ; Get timestamp on entry
+ b mlcksniff
+
+mlckspin3: mtmsr r7 ; Turn off interruptions
+ mftb r8 ; Get timestamp on entry
+
+mlcksniff: lwz r5,MUTEX_DATA(r3) ; Get that lock in here
+ mr. r5,r5 ; Is the lock held
+ beq++ mlckretry ; No, try for it again...
+ rlwinm. r10,r5,0,0,29 ; Extract the lock owner
+ beq++ mlckslow0 ; InterLock is held
+ bf MUTEX_ATTR_STATb,mlStatSkip ; Branch if no stat
+ andi. r5,r5,ILK_LOCKED ; extract interlocked?
+ bne mlStatSkip ; yes, skip
+ bt mlckmiss,mlStatSkip ; miss already counted
+ crset mlckmiss ; Remember miss recorded
+ lwz r5,MUTEX_GRP(r3) ; Load lock group
+ addi r5,r5,GRP_MTX_STAT_MISS+4 ; Add stat miss offset
+mlStatLoop:
+ lwarx r6,0,r5 ; Load stat miss cnt
+ addi r6,r6,1 ; Increment stat miss cnt
+ stwcx. r6,0,r5 ; Update stat miss cnt
+ bne-- mlStatLoop ; Retry if failed
+ mfsprg r6,1 ; Reload current thread
+mlStatSkip:
+ lwz r2,ACT_MACT_SPF(r10) ; Get the special flags
+ rlwinm. r2,r2,0,OnProcbit,OnProcbit ; Is OnProcbit set?
+ beq mlckslow0 ; Lock owner isn't running
+ lis r2,hi16(TH_IDLE) ; Get thread idle state
+ ori r2,r2,lo16(TH_IDLE) ; Get thread idle state
+ lwz r10,THREAD_STATE(r10) ; Get the thread state
+ and. r10,r10,r2 ; Is idle set?
+ bne mlckslow0 ; Lock owner is idling
+
+ mftb r10 ; Time stamp us now
+ sub r10,r10,r8 ; Get the elapsed time
+ cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
+ blt++ mlcksniff ; Not yet...
+
+ mtmsr r9 ; Say, any interrupts pending?
+
+; The following instructions force the pipeline to be interlocked to that only one
+; instruction is issued per cycle. The insures that we stay enabled for a long enough
+; time; if it's too short, pending interruptions will not have a chance to be taken
+
+ subi r4,r4,128 ; Back off elapsed time from timeout value
+ or r4,r4,r4 ; Do nothing here but force a single cycle delay
+ mr. r4,r4 ; See if we used the whole timeout
+ or r4,r4,r4 ; Do nothing here but force a single cycle delay
+
+ ble-- mlckslow1 ; We failed
+ b mlckspin3 ; Now that we've opened an enable window, keep trying...
+mlckretry:
+ mtmsr r9 ; Restore interrupt state
+ li r8,1 ; Show already through once
+ blr
+
+mlckslow0: ; We couldn't get the lock
+ mtmsr r9 ; Restore interrupt state
- lis r3,HIGH_ADDR(mutex_failed1) ; Get the failed mutex message
- ori r3,r3,LOW_ADDR(mutex_failed1) ; Get the failed mutex message
- bl EXT(panic) ; Call panic
- BREAKPOINT_TRAP ; We die here anyway, can not get the lock
+mlckslow1:
+ mtlr r12
+
+ PROLOG(0)
+.L_ml_retry:
+ bl lockDisa ; Go get a lock on the mutex's interlock lock
+ mr. r4,r3 ; Did we get it?
+ lwz r3,FM_ARG0(r1) ; Restore the lock address
+ bne++ mlGotInt ; We got it just fine...
+ mr r4,r11 ; Saved lock addr
+ lis r3,hi16(mutex_failed1) ; Get the failed mutex message
+ ori r3,r3,lo16(mutex_failed1) ; Get the failed mutex message
+ bl EXT(panic) ; Call panic
+ BREAKPOINT_TRAP ; We die here anyway, can not get the lock
.data
mutex_failed1:
- STRINGD "We can't get a mutex interlock lock on mutex_lock\n\000"
+ STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock\n\000"
.text
mlGotInt:
-/* Note that there is no reason to do a load and reserve here. We already
- hold the interlock lock and no one can touch this field unless they
- have that, so, we're free to play */
-
- lwz r4,LOCK_DATA(r3) /* Get the mutex's lock field */
- rlwinm. r9,r4,30,2,31 /* So, can we have it? */
- bne- mlInUse /* Nope, sombody's playing already... */
-
-#if MACH_LDEBUG
- mfmsr r11 ; Note: no need to deal with fp or vec here
- rlwinm r5,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1
- mtmsr r5
- mfsprg r9,0 /* Get the per_proc block */
- lwz r5,0(r1) /* Get previous save frame */
- lwz r5,FM_LR_SAVE(r5) /* Get our caller's address */
- lwz r8, PP_ACTIVE_THREAD(r9) /* Get the active thread */
- stw r5,MUTEX_PC(r3) /* Save our caller */
- mr. r8,r8 /* Is there any thread? */
- stw r8,MUTEX_THREAD(r3) /* Set the mutex's holding thread */
- beq- .L_ml_no_active_thread /* No owning thread... */
- lwz r9,THREAD_MUTEX_COUNT(r8) /* Get the mutex count */
- addi r9,r9,1 /* Bump it up */
- stw r9,THREAD_MUTEX_COUNT(r8) /* Stash it back */
-.L_ml_no_active_thread:
- mtmsr r11
-#endif /* MACH_LDEBUG */
-
- bl EXT(mutex_lock_acquire)
+; Note that there is no reason to do a load and reserve here. We already
+; hold the interlock lock and no one can touch this field unless they
+; have that, so, we're free to play
+
+ lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field
+ rlwinm. r9,r4,30,2,31 ; So, can we have it?
+ bne- mlInUse ; Nope, sombody's playing already...
+
+ bf++ MUTEX_ATTR_DEBUGb,mlDebSkip
+ CHECK_SETUP(r5)
+ mfsprg r9,1 ; Get the current activation
+ lwz r5,0(r1) ; Get previous save frame
+ lwz r6,FM_LR_SAVE(r5) ; Get our caller's address
+ mr r8,r9 ; Get the active thread
+ stw r6,MUTEX_STACK(r3) ; Save our caller
+ stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
+ LCK_STACK(r3,r5,r6,r7,r8,r10)
+mlDebSkip:
+ mr r3,r11 ; Get the based lock address
+ bl EXT(lck_mtx_lock_acquire)
+ lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
mfsprg r5,1
+ mtcr r2
mr. r4,r3
- lwz r3,FM_ARG0(r1)
+ lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
+ lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
beq mlUnlock
ori r5,r5,WAIT_FLAG
-mlUnlock:
- sync
- stw r5,LOCK_DATA(r3) /* grab the mutexlock and free the interlock */
-#if ETAP_LOCK_TRACE
- mflr r4
- lwz r5,SWT_HI(r1)
- lwz r6,SWT_LO(r1)
- bl EXT(etap_mutex_hold) /* collect hold timestamp */
-#endif /* ETAP_LOCK_TRACE */
+mlUnlock: eieio
+ stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock
- EPILOG /* Restore all saved registers */
+ EPILOG ; Restore all saved registers
+ b epStart ; Go enable preemption...
- b epStart /* Go enable preemption... */
-
-/*
- * We come to here when we have a resource conflict. In other words,
- * the mutex is held.
- */
+; We come to here when we have a resource conflict. In other words,
+; the mutex is held.
mlInUse:
-#if ETAP_LOCK_TRACE
- lwz r7,MISSED(r1)
- cmpwi r7,0 /* did we already take a wait timestamp ? */
- bne .L_ml_block /* yup. carry-on */
- bl EXT(etap_mutex_miss) /* get wait timestamp */
- stw r3,SWT_HI(r1) /* store timestamp */
- stw r4,SWT_LO(r1)
- li r7, 1 /* mark wait timestamp as taken */
- stw r7,MISSED(r1)
- lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
-.L_ml_block:
-#endif /* ETAP_LOCK_TRACE */
-
CHECK_SETUP(r12)
- CHECK_MYLOCK(MUTEX_THREAD) /* Assert we don't own the lock already */
-
-
-/* Note that we come in here with the interlock set. The wait routine
- * will unlock it before waiting.
- */
- ori r4,r4,WAIT_FLAG /* Set the wait flag */
- stw r4,LOCK_DATA(r3)
- rlwinm r4,r4,0,0,29 /* Extract the lock owner */
- bl EXT(mutex_lock_wait) /* Wait for our turn at the lock */
+ CHECK_MYLOCK() ; Assert we don't own the lock already */
+
+; Note that we come in here with the interlock set. The wait routine
+; will unlock it before waiting.
+
+ bf MUTEX_ATTR_STATb,mlStatSkip2 ; Branch if no stat
+ lwz r5,MUTEX_GRP(r3) ; Load lck group
+ bt mlckmiss,mlStatSkip1 ; Skip miss already counted
+ crset mlckmiss ; Remember miss recorded
+ li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset
+mlStatLoop1:
+ lwarx r8,r9,r5 ; Load stat miss cnt
+ addi r8,r8,1 ; Increment stat miss cnt
+ stwcx. r8,r9,r5 ; Store stat miss cnt
+ bne-- mlStatLoop1 ; Retry if failed
+mlStatSkip1:
+ lwz r9,GRP_MTX_STAT_WAIT+4(r5) ; Load wait cnt
+ addi r9,r9,1 ; Increment wait cnt
+ stw r9,GRP_MTX_STAT_WAIT+4(r5) ; Update miss cnt
+mlStatSkip2:
+ ori r4,r4,WAIT_FLAG ; Set the wait flag
+ stw r4,MUTEX_DATA(r3)
+ rlwinm r4,r4,0,0,29 ; Extract the lock owner
+ mfcr r2
+ stw r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
+ mr r3,r11 ; Get the based lock address
+ bl EXT(lck_mtx_lock_wait) ; Wait for our turn at the lock
- lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
- b .L_ml_retry /* and try again... */
+ lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
+ lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
+ lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
+ mtcr r2
+ b .L_ml_retry ; and try again...
/*
- * void _mutex_try(mutex_t*)
+ * void lck_mtx_try_lock(_extlck_mtx_ext_t*)
*
*/
-
.align 5
+ .globl EXT(lck_mtx_try_lock_ext)
+LEXT(lck_mtx_try_lock_ext)
+#if MACH_LDEBUG
.globl EXT(mutex_try)
LEXT(mutex_try)
.globl EXT(_mutex_try)
LEXT(_mutex_try)
-#if !MACH_LDEBUG
- mfsprg r6,1 /* load the current thread */
-L_mutex_try_loop:
- lwarx r5,0,r3 /* load the lock value */
+#endif
+ mr r11,r3 ; Save lock addr
+mlteEnter:
+ lwz r0,MUTEX_ATTR(r3)
+ mtcrf 1,r0 ; Set cr7
+ CHECK_SETUP(r12)
+ CHECK_MUTEX_TYPE()
+
+ bf MUTEX_ATTR_STATb,mlteStatSkip ; Branch if no stat
+ lwz r5,MUTEX_GRP(r3) ; Load lock group
+ li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset
+mlteStatLoop:
+ lwarx r8,r7,r5 ; Load stat util cnt
+ addi r8,r8,1 ; Increment stat util cnt
+ stwcx. r8,r7,r5 ; Store stat util cnt
+ bne-- mlteStatLoop ; Retry if failed
+ mr. r8,r8 ; Test for zero
+ bne++ mlteStatSkip ; Did stat util cnt wrapped?
+ lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt
+ addi r8,r8,1 ; Increment upper stat util cnt
+ stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt
+mlteStatSkip:
+ mfsprg r6,1 ; load the current thread
+ lwz r5,MUTEX_DATA(r3) ; Get the lock value
+ mr. r5,r5 ; Quick check
+ bne-- L_mutex_try_slow ; Can not get it now...
+ mfmsr r9 ; Get the MSR value
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+
+mlteLoopTry:
+ lwarx r5,MUTEX_DATA,r3 ; load the lock value
mr. r5,r5
- bne- L_mutex_try_slow /* branch to the slow path */
- stwcx. r6,0,r3 /* grab the lock */
- bne- L_mutex_try_loop /* retry if failed */
- isync /* stop prefetching */
+ bne-- mlteSlowX ; branch to the slow path
+ stwcx. r6,MUTEX_DATA,r3 ; grab the lock
+ bne-- mlteLoopTry ; retry if failed
+ .globl EXT(mltelckPatch_isync)
+LEXT(mltelckPatch_isync)
+ isync ; stop prefetching
+ mflr r12
+ bf MUTEX_ATTR_DEBUGb,mlteDebSkip
+ mr r8,r6 ; Get the active thread
+ stw r12,MUTEX_STACK(r3) ; Save our caller
+ stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
+ mr r5,r1
+ LCK_STACK(r3,r5,r6,r7,r8,r10)
+mlteDebSkip:
li r3, 1
+ mtmsr r9 ; Say, any interrupts pending?
blr
-L_mutex_try_slow:
+mlteSlowX:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Kill reservation
+ mtmsr r9 ; Say, any interrupts pending?
+ b L_mutex_try_slow
+
+
+/*
+ * void lck_mtx_try_lock(lck_mtx_t*)
+ *
+ */
+ .align 5
+ .globl EXT(lck_mtx_try_lock)
+LEXT(lck_mtx_try_lock)
+#if !MACH_LDEBUG
+ .globl EXT(mutex_try)
+LEXT(mutex_try)
+ .globl EXT(_mutex_try)
+LEXT(_mutex_try)
#endif
- PROLOG(8) /* reserve space for SWT_HI and SWT_LO */
+ mfsprg r6,1 ; load the current thread
+ lwz r5,MUTEX_DATA(r3) ; Get the lock value
+ mr r11,r3 ; Save lock addr
+ mr. r5,r5 ; Quick check
+ bne-- mltSlow00 ; Indirect or Can not get it now...
+
+mltLoopTry:
+ lwarx r5,MUTEX_DATA,r3 ; load the lock value
+ mr. r5,r5
+ bne-- mltSlow01 ; branch to the slow path
+ stwcx. r6,MUTEX_DATA,r3 ; grab the lock
+ bne-- mltLoopTry ; retry if failed
+ .globl EXT(mltlckPatch_isync)
+LEXT(mltlckPatch_isync)
+ isync ; stop prefetching
+ li r3, 1
+ blr
+
+mltSlow00:
+ cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
+ bne-- mltSlow02 ; No, go handle contention
+ lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
+ b mlteEnter
+mltSlow01:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Kill reservation
+
+mltSlow02:
+ li r0,0
+ mtcrf 1,r0 ; Set cr7 to zero
+
+L_mutex_try_slow:
+ PROLOG(0)
-#if ETAP_LOCK_TRACE
- li r5, 0
- stw r5, STW_HI(r1) /* set wait time to 0 (HI) */
- stw r5, SWT_LO(r1) /* set wait time to 0 (LO) */
-#endif /* ETAP_LOCK_TRACE */
-
-#if 0
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lis r5,0xBBBB /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
-#endif
- CHECK_SETUP(r12)
- CHECK_MUTEX_TYPE()
- CHECK_NO_SIMPLELOCKS()
-
- lwz r6,LOCK_DATA(r3) /* Quick check */
- rlwinm. r6,r6,30,2,31 /* to see if someone has this lock already */
- bne- mtFail /* Someone's got it already... */
-
- bl lockDisa /* Go get a lock on the mutex's interlock lock */
- mr. r4,r3 /* Did we get it? */
- lwz r3,FM_ARG0(r1) /* Restore the lock address */
- bne+ mtGotInt /* We got it just fine... */
-
- lis r3,HIGH_ADDR(mutex_failed2) ; Get the failed mutex message
- ori r3,r3,LOW_ADDR(mutex_failed2) ; Get the failed mutex message
- bl EXT(panic) ; Call panic
- BREAKPOINT_TRAP ; We die here anyway, can not get the lock
+ lwz r6,MUTEX_DATA(r3) ; Quick check
+ rlwinm. r6,r6,30,2,31 ; to see if someone has this lock already
+ bne- mtFail ; Someone's got it already...
+
+ bl lockDisa ; Go get a lock on the mutex's interlock lock
+ mr. r4,r3 ; Did we get it?
+ lwz r3,FM_ARG0(r1) ; Restore the lock address
+ bne++ mtGotInt ; We got it just fine...
+ mr r4,r11 ; Saved lock addr
+ lis r3,hi16(mutex_failed2) ; Get the failed mutex message
+ ori r3,r3,lo16(mutex_failed2) ; Get the failed mutex message
+ bl EXT(panic) ; Call panic
+ BREAKPOINT_TRAP ; We die here anyway, can not get the lock
.data
mutex_failed2:
- STRINGD "We can't get a mutex interlock lock on mutex_try\n\000"
+ STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock try\n\000"
.text
mtGotInt:
-/* Note that there is no reason to do a load and reserve here. We already
- hold the interlock and no one can touch at this field unless they
- have that, so, we're free to play */
+; Note that there is no reason to do a load and reserve here. We already
+; hold the interlock and no one can touch at this field unless they
+; have that, so, we're free to play
- lwz r4,LOCK_DATA(r3) /* Get the mutex's lock field */
- rlwinm. r9,r4,30,2,31 /* So, can we have it? */
- bne- mtInUse /* Nope, sombody's playing already... */
+ lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field
+ rlwinm. r9,r4,30,2,31 ; So, can we have it?
+ bne- mtInUse ; Nope, sombody's playing already...
-#if MACH_LDEBUG
- mfmsr r11
- rlwinm r5,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1
- mtmsr r5
- mfsprg r9,0 /* Get the per_proc block */
- lwz r5,0(r1) /* Get previous save frame */
- lwz r5,FM_LR_SAVE(r5) /* Get our caller's address */
- lwz r8, PP_ACTIVE_THREAD(r9) /* Get the active thread */
- stw r5,MUTEX_PC(r3) /* Save our caller */
- mr. r8,r8 /* Is there any thread? */
- stw r8,MUTEX_THREAD(r3) /* Set the mutex's holding thread */
- beq- .L_mt_no_active_thread /* No owning thread... */
- lwz r9, THREAD_MUTEX_COUNT(r8) /* Get the mutex count */
- addi r9, r9, 1 /* Bump it up */
- stw r9, THREAD_MUTEX_COUNT(r8) /* Stash it back */
-.L_mt_no_active_thread:
- mtmsr r11
-#endif /* MACH_LDEBUG */
-
- bl EXT(mutex_lock_acquire)
+ bf++ MUTEX_ATTR_DEBUGb,mtDebSkip
+ CHECK_SETUP(r5)
+ mfsprg r9,1 ; Get the current activation
+ lwz r5,0(r1) ; Get previous save frame
+ lwz r6,FM_LR_SAVE(r5) ; Get our caller's address
+ mr r8,r9 ; Get the active thread
+ stw r6,MUTEX_STACK(r3) ; Save our caller
+ stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
+ LCK_STACK(r3,r5,r6,r7,r8,r10)
+mtDebSkip:
+ mr r3,r11 ; Get the based lock address
+ bl EXT(lck_mtx_lock_acquire)
mfsprg r5,1
mr. r4,r3
- lwz r3,FM_ARG0(r1)
+ lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
+ lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
beq mtUnlock
ori r5,r5,WAIT_FLAG
-mtUnlock:
- sync /* Push it all out */
- stw r5,LOCK_DATA(r3) /* grab the mutexlock and free the interlock */
-#if ETAP_LOCK_TRACE
- lwz r4,0(r1) /* Back chain the stack */
- lwz r5,SWT_HI(r1)
- lwz r4,FM_LR_SAVE(r4) /* Get our caller's address */
- lwz r6,SWT_LO(r1)
- bl EXT(etap_mutex_hold) /* collect hold timestamp */
-#endif /* ETAP_LOCK_TRACE */
+mtUnlock: eieio
+ stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock
- bl epStart /* Go enable preemption... */
+ bl epStart ; Go enable preemption...
li r3, 1
- EPILOG /* Restore all saved registers */
- blr /* Return... */
+ EPILOG ; Restore all saved registers
+ blr ; Return...
-/*
- * We come to here when we have a resource conflict. In other words,
- * the mutex is held.
- */
+; We come to here when we have a resource conflict. In other words,
+; the mutex is held.
mtInUse:
- rlwinm r4,r4,0,0,30 /* Get the unlock value */
- stw r4,LOCK_DATA(r3) /* free the interlock */
- bl epStart /* Go enable preemption... */
-
-mtFail: li r3,0 /* Set failure code */
- EPILOG /* Restore all saved registers */
- blr /* Return... */
+ bf++ MUTEX_ATTR_STATb,mtStatSkip ; Branch if no stat
+ lwz r5,MUTEX_GRP(r3) ; Load lock group
+ li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset
+mtStatLoop:
+ lwarx r8,r9,r5 ; Load stat miss cnt
+ addi r8,r8,1 ; Increment stat miss cnt
+ stwcx. r8,r9,r5 ; Store stat miss cnt
+ bne-- mtStatLoop ; Retry if failed
+mtStatSkip:
+ rlwinm r4,r4,0,0,30 ; Get the unlock value
+ stw r4,MUTEX_DATA(r3) ; free the interlock
+ bl epStart ; Go enable preemption...
+
+mtFail: li r3,0 ; Set failure code
+ EPILOG ; Restore all saved registers
+ blr ; Return...
/*
- * void mutex_unlock(mutex_t* l)
+ * void mutex_unlock(mutex_t* l)
+ *
*/
-
.align 5
.globl EXT(mutex_unlock)
-
LEXT(mutex_unlock)
-#if !MACH_LDEBUG
+
sync
-L_mutex_unlock_loop:
- lwarx r5,0,r3
- rlwinm. r4,r5,0,30,31 /* Bail if pending waiter or interlock set */
- li r5,0 /* Clear the mutexlock */
- bne- L_mutex_unlock_slow
- stwcx. r5,0,r3
- bne- L_mutex_unlock_loop
- blr
-L_mutex_unlock_slow:
+ mr r11,r3 ; Save lock addr
+#if MACH_LDEBUG
+ b mlueEnter1
+#else
+ b mluEnter1
#endif
- PROLOG(0)
-
-#if ETAP_LOCK_TRACE
- bl EXT(etap_mutex_unlock) /* collect ETAP data */
- lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
-#endif /* ETAP_LOCK_TRACE */
+/*
+ * void lck_mtx_ext_unlock(lck_mtx_ext_t* l)
+ *
+ */
+ .align 5
+ .globl EXT(lck_mtx_ext_unlock)
+LEXT(lck_mtx_ext_unlock)
+#if MACH_LDEBUG
+ .globl EXT(mutex_unlock_rwcmb)
+LEXT(mutex_unlock_rwcmb)
+#endif
+mlueEnter:
+ .globl EXT(mulckePatch_isync)
+LEXT(mulckePatch_isync)
+ isync
+ .globl EXT(mulckePatch_eieio)
+LEXT(mulckePatch_eieio)
+ eieio
+ mr r11,r3 ; Save lock addr
+mlueEnter1:
+ lwz r0,MUTEX_ATTR(r3)
+ mtcrf 1,r0 ; Set cr7
CHECK_SETUP(r12)
CHECK_MUTEX_TYPE()
CHECK_THREAD(MUTEX_THREAD)
-#if 0
- mfsprg r4,0 /* (TEST/DEBUG) */
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lwz r4,PP_ACTIVE_THREAD(r4) /* (TEST/DEBUG) */
- lis r5,0xCCCC /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
+ lwz r5,MUTEX_DATA(r3) ; Get the lock
+ rlwinm. r4,r5,0,30,31 ; Quick check
+ bne-- L_mutex_unlock_slow ; Can not get it now...
+ mfmsr r9 ; Get the MSR value
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+
+mlueLoop:
+ lwarx r5,MUTEX_DATA,r3
+ rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
+ li r5,0 ; Clear the mutexlock
+ bne-- mlueSlowX
+ stwcx. r5,MUTEX_DATA,r3
+ bne-- mlueLoop
+ mtmsr r9 ; Say, any interrupts pending?
+ blr
+
+mlueSlowX:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Dump reservation
+ mtmsr r9 ; Say, any interrupts pending?
+ b L_mutex_unlock_slow ; Join slow path...
+
+/*
+ * void lck_mtx_unlock(lck_mtx_t* l)
+ *
+ */
+ .align 5
+ .globl EXT(lck_mtx_unlock)
+LEXT(lck_mtx_unlock)
+#if !MACH_LDEBUG
+ .globl EXT(mutex_unlock_rwcmb)
+LEXT(mutex_unlock_rwcmb)
#endif
- bl lockDisa /* Go get a lock on the mutex's interlock lock */
- mr. r4,r3 /* Did we get it? */
- lwz r3,FM_ARG0(r1) /* Restore the lock address */
- bne+ muGotInt /* We got it just fine... */
-
- lis r3,HIGH_ADDR(mutex_failed3) ; Get the failed mutex message
- ori r3,r3,LOW_ADDR(mutex_failed3) ; Get the failed mutex message
- bl EXT(panic) ; Call panic
- BREAKPOINT_TRAP ; We die here anyway, can not get the lock
+mluEnter:
+ .globl EXT(mulckPatch_isync)
+LEXT(mulckPatch_isync)
+ isync
+ .globl EXT(mulckPatch_eieio)
+LEXT(mulckPatch_eieio)
+ eieio
+ mr r11,r3 ; Save lock addr
+mluEnter1:
+ lwz r5,MUTEX_DATA(r3) ; Get the lock
+ rlwinm. r4,r5,0,30,31 ; Quick check
+ bne-- mluSlow0 ; Indirect or Can not get it now...
+
+mluLoop:
+ lwarx r5,MUTEX_DATA,r3
+ rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
+ li r5,0 ; Clear the mutexlock
+ bne-- mluSlowX
+ stwcx. r5,MUTEX_DATA,r3
+ bne-- mluLoop
+#if CONFIG_DTRACE
+/* lock released - LS_LCK_MTX_UNLOCK_RELEASE */
+ LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
+ blr
+
+ LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE)
+#endif
+ blr
+
+
+mluSlow0:
+ cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
+ bne-- L_mutex_unlock_slow ; No, go handle contention
+ lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
+ b mlueEnter1
+mluSlowX:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Dump reservation
+
+L_mutex_unlock_slow:
+
+ PROLOG(0)
+
+ bl lockDisa ; Go get a lock on the mutex's interlock lock
+ mr. r4,r3 ; Did we get it?
+ lwz r3,FM_ARG0(r1) ; Restore the lock address
+ bne++ muGotInt ; We got it just fine...
+ mr r4,r11 ; Saved lock addr
+ lis r3,hi16(mutex_failed3) ; Get the failed mutex message
+ ori r3,r3,lo16(mutex_failed3) ; Get the failed mutex message
+ bl EXT(panic) ; Call panic
+ BREAKPOINT_TRAP ; We die here anyway, can not get the lock
.data
mutex_failed3:
- STRINGD "We can't get a mutex interlock lock on mutex_unlock\n\000"
+ STRINGD "attempt to interlock mutex (0x%08X) failed on mutex unlock\n\000"
.text
muGotInt:
- lwz r4,LOCK_DATA(r3)
- andi. r5,r4,WAIT_FLAG /* are there any waiters ? */
+ lwz r4,MUTEX_DATA(r3)
+ andi. r5,r4,WAIT_FLAG ; are there any waiters ?
rlwinm r4,r4,0,0,29
- beq+ muUnlock /* Nope, we're done... */
+ beq+ muUnlock ; Nope, we're done...
- bl EXT(mutex_unlock_wakeup) /* yes, wake a thread */
- lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
- lwz r5,LOCK_DATA(r3) /* load the lock */
+ mr r3,r11 ; Get the based lock address
+ bl EXT(lck_mtx_unlock_wakeup) ; yes, wake a thread
+ lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
+ lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
+ lwz r5,MUTEX_DATA(r3) ; load the lock
muUnlock:
-#if MACH_LDEBUG
- mfmsr r11
- rlwinm r9,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1
- mtmsr r9
- mfsprg r9,0
- lwz r9,PP_ACTIVE_THREAD(r9)
- stw r9,MUTEX_THREAD(r3) /* disown thread */
- cmpwi r9,0
- beq- .L_mu_no_active_thread
- lwz r8,THREAD_MUTEX_COUNT(r9)
- subi r8,r8,1
- stw r8,THREAD_MUTEX_COUNT(r9)
-.L_mu_no_active_thread:
- mtmsr r11
-#endif /* MACH_LDEBUG */
+ andi. r5,r5,WAIT_FLAG ; Get the unlock value
+ eieio
+ stw r5,MUTEX_DATA(r3) ; unlock the interlock and lock
- andi. r5,r5,WAIT_FLAG /* Get the unlock value */
- sync /* Make sure it's all there before we release */
- stw r5,LOCK_DATA(r3) /* unlock the interlock and lock */
-
- EPILOG /* Deal with the stack now, enable_preemption doesn't always want one */
- b epStart /* Go enable preemption... */
+ EPILOG ; Deal with the stack now, enable_preemption doesn't always want one
+ b epStart ; Go enable preemption...
/*
- * void interlock_unlock(hw_lock_t lock)
+ * void lck_mtx_assert(lck_mtx_t* l, unsigned int)
+ *
*/
-
.align 5
- .globl EXT(interlock_unlock)
+ .globl EXT(lck_mtx_assert)
+LEXT(lck_mtx_assert)
+ .globl EXT(_mutex_assert)
+LEXT(_mutex_assert)
+ mr r11,r3
+maEnter:
+ lwz r5,MUTEX_DATA(r3)
+ cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
+ bne-- maCheck ; No, go check the assertion
+ lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
+ b maEnter
+maCheck:
+ mfsprg r6,1 ; load the current thread
+ rlwinm r5,r5,0,0,29 ; Extract the lock owner
+ cmpwi r4,MUTEX_ASSERT_OWNED
+ cmplw cr1,r6,r5 ; Is the lock held by current act
+ crandc cr0_eq,cr0_eq,cr1_eq ; Check owned assertion
+ bne-- maNext
+ mr r4,r11
+ lis r3,hi16(mutex_assert1) ; Get the failed mutex message
+ ori r3,r3,lo16(mutex_assert1) ; Get the failed mutex message
+ b maPanic ; Panic path
+maNext:
+ cmpwi r4,MUTEX_ASSERT_NOTOWNED ; Check not owned assertion
+ crand cr0_eq,cr0_eq,cr1_eq ;
+ bnelr++
+maPanic:
+ PROLOG(0)
+ mr r4,r11
+ lis r3,hi16(mutex_assert2) ; Get the failed mutex message
+ ori r3,r3,lo16(mutex_assert2) ; Get the failed mutex message
+ bl EXT(panic) ; Call panic
+ BREAKPOINT_TRAP ; We die here anyway
-LEXT(interlock_unlock)
+ .data
+mutex_assert1:
+ STRINGD "mutex (0x%08X) not owned\n\000"
+mutex_assert2:
+ STRINGD "mutex (0x%08X) owned\n\000"
+ .text
+
+
+/*
+ * void lck_mtx_ilk_unlock(lck_mtx *lock)
+ */
+ .globl EXT(lck_mtx_ilk_unlock)
+LEXT(lck_mtx_ilk_unlock)
-#if 0
- lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
- lis r5,0xDDDD /* (TEST/DEBUG) */
- oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
- sc /* (TEST/DEBUG) */
-#endif
- lwz r10,LOCK_DATA(r3)
+ lwz r10,MUTEX_DATA(r3)
rlwinm r10,r10,0,0,30
- sync
- stw r10,LOCK_DATA(r3)
+ eieio
+ stw r10,MUTEX_DATA(r3)
- b epStart /* Go enable preemption... */
+ b epStart ; Go enable preemption...
-/*
- * Here is where we enable preemption. We need to be protected
- * against ourselves, we can't chance getting interrupted and modifying
- * our processor wide preemption count after we'sve loaded it up. So,
- * we need to disable all 'rupts. Actually, we could use a compare
- * and swap to do this, but, since there are no MP considerations
- * (we are dealing with a CPU local field) it is much, much faster
- * to disable.
+/*
+ * void _enable_preemption_no_check(void)
*
- * Note that if we are not genned MP, the calls here will be no-opped via
- * a #define and since the _mp forms are the same, likewise a #define
- * will be used to route to the other forms
+ * This version does not check if we get preempted or not
*/
-
-/* This version does not check if we get preempted or not */
-
-
.align 4
.globl EXT(_enable_preemption_no_check)
LEXT(_enable_preemption_no_check)
- cmplw cr1,r1,r1 /* Force zero cr so we know not to check if preempted */
- b epCommn /* Join up with the other enable code... */
-
-/* This version checks if we get preempted or not */
+ cmplw cr1,r1,r1 ; Force zero cr so we know not to check if preempted
+ b epCommn ; Join up with the other enable code...
+/*
+ * void _enable_preemption(void)
+ *
+ * This version checks if we get preempted or not
+ */
.align 5
.globl EXT(_enable_preemption)
LEXT(_enable_preemption)
-epStart: cmplwi cr1,r1,0 /* Force non-zero cr so we know to check if preempted */
-
-/*
- * Common enable preemption code
- */
-
-epCommn: mfmsr r9 /* Save the old MSR */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtmsr r8 /* Interrupts off */
- isync ; May have mess with vec/fp here
-
- mfsprg r3,0 /* Get the per_proc block */
- li r8,-1 /* Get a decrimenter */
- lwz r5,PP_PREEMPT_CNT(r3) /* Get the preemption level */
- add. r5,r5,r8 /* Bring down the disable count */
-#if 0
- mfsprg r4,1 ; (TEST/DEBUG) Note the next 3 keep from interrpting too early
- mr. r4,r4 ; (TEST/DEBUG)
- beq- epskptrc0 ; (TEST/DEBUG)
- lis r0,hi16(CutTrace) ; (TEST/DEBUG)
- lis r4,0xBBBB ; (TEST/DEBUG)
- oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
- sc ; (TEST/DEBUG)
-epskptrc0: mr. r5,r5 ; (TEST/DEBUG)
-#endif
-#if MACH_LDEBUG
- blt- epTooFar /* Yeah, we did... */
-#endif /* MACH_LDEBUG */
- stw r5,PP_PREEMPT_CNT(r3) /* Save it back */
-
- beq+ epCheckPreempt /* Go check if we need to be preempted... */
-
-epNoCheck: mtmsr r9 /* Restore the interrupt level */
- blr /* Leave... */
-
-#if MACH_LDEBUG
+; Here is where we enable preemption.
+
+epStart:
+ cmplwi cr1,r1,0 ; Force non-zero cr so we know to check if preempted
+
+epCommn:
+ mfsprg r3,1 ; Get current activation
+ li r8,-1 ; Get a decrementer
+ lwz r5,ACT_PREEMPT_CNT(r3) ; Get the preemption level
+ add. r5,r5,r8 ; Bring down the disable count
+ blt- epTooFar ; Yeah, we did...
+ stw r5,ACT_PREEMPT_CNT(r3) ; Save it back
+ crandc cr0_eq,cr0_eq,cr1_eq
+ beq+ epCheckPreempt ; Go check if we need to be preempted...
+ blr ; Leave...
epTooFar:
- lis r6,HIGH_ADDR(EXT(panic)) /* First half of panic call */
- lis r3,HIGH_ADDR(epTooFarStr) /* First half of panic string */
- ori r6,r6,LOW_ADDR(EXT(panic)) /* Second half of panic call */
- ori r3,r3,LOW_ADDR(epTooFarStr) /* Second half of panic string */
- mtlr r6 /* Get the address of the panic routine */
- mtmsr r9 /* Restore interruptions */
- blrl /* Panic... */
+ mr r4,r5
+ lis r3,hi16(epTooFarStr) ; First half of panic string
+ ori r3,r3,lo16(epTooFarStr) ; Second half of panic string
+ PROLOG(0)
+ bl EXT(panic)
+ BREAKPOINT_TRAP ; We die here anyway
.data
epTooFarStr:
- STRINGD "_enable_preemption: preemption_level <= 0!\000"
- .text
-#endif /* MACH_LDEBUG */
+ STRINGD "enable_preemption: preemption_level %d\n\000"
+ .text
.align 5
-
epCheckPreempt:
- lwz r7,PP_NEED_AST(r3) /* Get the AST request address */
- li r5,AST_URGENT /* Get the requests we do honor */
- lwz r7,0(r7) /* Get the actual, real live, extra special AST word */
- lis r0,HIGH_ADDR(DoPreemptCall) /* Just in case, get the top of firmware call */
- and. r7,r7,r5 ; Should we preempt?
- ori r0,r0,LOW_ADDR(DoPreemptCall) /* Merge in bottom part */
- beq+ epCPno ; No preemption here...
-
- andi. r3,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off
-
-epCPno: mtmsr r9 /* Allow interrupts if we can */
- beqlr+ ; We probably will not preempt...
- sc /* Do the preemption */
- blr /* Now, go away now... */
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ andi. r4,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off
+ beq+ epCPno ; No preemption here...
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+ lwz r3,ACT_PER_PROC(r3) ; Get the per_proc block
+ lwz r7,PP_PENDING_AST(r3) ; Get pending AST mask
+ li r5,AST_URGENT ; Get the requests we do honor
+ lis r0,hi16(DoPreemptCall) ; Just in case, get the top of firmware call
+ and. r7,r7,r5 ; Should we preempt?
+ ori r0,r0,lo16(DoPreemptCall) ; Merge in bottom part
+ mtmsr r9 ; Allow interrupts if we can
+epCPno:
+ beqlr+ ; We probably will not preempt...
+ sc ; Do the preemption
+ blr ; Now, go away now...
/*
- * Here is where we disable preemption. Since preemption is on a
- * per processor basis (a thread runs on one CPU at a time) we don't
- * need any cross-processor synchronization. We do, however, need to
- * be interrupt safe, so we don't preempt while in the process of
- * disabling it. We could use SPLs, but since we always want complete
- * disablement, and this is platform specific code, we'll just kick the
- * MSR. We'll save a couple of orders of magnitude over using SPLs.
+ * void disable_preemption(void)
+ *
+ * Here is where we disable preemption.
*/
-
.align 5
-
- nop ; Use these 5 nops to force daPreComm
- nop ; to a line boundary.
- nop
- nop
- nop
-
.globl EXT(_disable_preemption)
LEXT(_disable_preemption)
-daPreAll: mfmsr r9 /* Save the old MSR */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtmsr r8 /* Interrupts off */
- isync ; May have mess with fp/vec
-
-daPreComm: mfsprg r6,0 /* Get the per_proc block */
- lwz r5,PP_PREEMPT_CNT(r6) /* Get the preemption level */
- addi r5,r5,1 /* Bring up the disable count */
- stw r5,PP_PREEMPT_CNT(r6) /* Save it back */
-#if 0
- mfsprg r4,1 ; (TEST/DEBUG) Note the next 3 keep from interrpting too early
- mr. r4,r4 ; (TEST/DEBUG)
- beq- epskptrc1 ; (TEST/DEBUG)
- lis r0,hi16(CutTrace) ; (TEST/DEBUG)
- lis r4,0xAAAA ; (TEST/DEBUG)
- oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
- sc ; (TEST/DEBUG)
-epskptrc1: ; (TEST/DEBUG)
-#endif
-
-;
-; Set PREEMPTSTACK above to enable a preemption traceback stack.
-;
-; NOTE: make sure that PREEMPTSTACK in aligned_data is
-; set the same as it is here. This is the number of
-; traceback entries we can handle per processor
-;
-; A value of 0 disables the stack.
-;
-#if PREEMPTSTACK
- cmplwi r5,PREEMPTSTACK ; Maximum depth
- lwz r6,CPU_ACTIVE_THREAD(r6) ; Get the pointer to the currently active thread
- bgt- nopredeb ; Too many to stack...
- mr. r6,r6 ; During boot?
- beq- nopredeb ; Yes, do not do backtrace...
- lwz r6,THREAD_TOP_ACT(r6) ; Point to the active activation
- lwz r6,ACT_MACT_PCB(r6) ; Get the last savearea used
- mr. r0,r6 ; Any saved context?
- beq- nosaveds ; No...
- lwz r0,saver1(r6) ; Get end of savearea chain
-
-nosaveds: li r11,0 ; Clear callers callers callers return
- li r10,0 ; Clear callers callers callers callers return
- li r8,0 ; Clear callers callers callers callers callers return
- lwz r2,0(r1) ; Get callers callers stack frame
- lwz r12,8(r2) ; Get our callers return
- lwz r4,0(r2) ; Back chain
-
- xor r2,r4,r2 ; Form difference
- cmplwi r2,8192 ; Within a couple of pages?
- mr r2,r4 ; Move register
- bge- nosaveher2 ; No, no back chain then...
- lwz r11,8(r2) ; Get our callers return
- lwz r4,0(r2) ; Back chain
-
- xor r2,r4,r2 ; Form difference
- cmplwi r2,8192 ; Within a couple of pages?
- mr r2,r4 ; Move register
- bge- nosaveher2 ; No, no back chain then...
- lwz r10,8(r2) ; Get our callers return
- lwz r4,0(r2) ; Back chain
-
- xor r2,r4,r2 ; Form difference
- cmplwi r2,8192 ; Within a couple of pages?
- mr r2,r4 ; Move register
- bge- nosaveher2 ; No, no back chain then...
- lwz r8,8(r2) ; Get our callers return
-
-nosaveher2:
- addi r5,r5,-1 ; Get index to slot
- mfspr r6,pir ; Get our processor
- mflr r4 ; Get our return
- rlwinm r6,r6,8,0,23 ; Index to processor slot
- lis r2,hi16(EXT(DBGpreempt)) ; Stack high order
- rlwinm r5,r5,4,0,27 ; Index to stack slot
- ori r2,r2,lo16(EXT(DBGpreempt)) ; Stack low order
- add r2,r2,r5 ; Point to slot
- add r2,r2,r6 ; Move to processor
- stw r4,0(r2) ; Save our return
- stw r11,4(r2) ; Save callers caller
- stw r10,8(r2) ; Save callers callers caller
- stw r8,12(r2) ; Save callers callers callers caller
-nopredeb:
-#endif
- mtmsr r9 /* Allow interruptions now */
-
- blr /* Return... */
+ mfsprg r6,1 ; Get the current activation
+ lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
+ addi r5,r5,1 ; Bring up the disable count
+ stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
+ blr ; Return...
/*
- * Return the active thread for both inside and outside osfmk consumption
+ * int get_preemption_level(void)
+ *
+ * Return the current preemption level
*/
-
.align 5
- .globl EXT(current_thread)
-
-LEXT(current_thread)
-
-#if 1
- mfsprg r3,1
- lwz r3,ACT_THREAD(r3)
- blr
-#else
- mfmsr r9 /* Save the old MSR */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtmsr r8 /* Interrupts off */
- isync
- mfsprg r6,0 /* Get the per_proc */
- lwz r3,PP_ACTIVE_THREAD(r6) /* Get the active thread */
- mfsprg r4,1
- lwz r4,ACT_THREAD(r4)
- cmplw cr0,r4,r3
- beq current_thread_cont
- lis r5,hi16(L_current_thread_paniced)
- ori r5,r5,lo16(L_current_thread_paniced)
- lwz r6,0(r5)
- mr. r6,r6
- bne current_thread_cont
- stw r9,0(r5)
- mr r5,r4
- mr r4,r3
- lis r3,hi16(L_current_thread_panic)
- ori r3,r3,lo16(L_current_thread_panic)
- bl EXT(panic)
+ .globl EXT(get_preemption_level)
- .data
-L_current_thread_panic:
- STRINGD "current_thread: spr1 not sync %x %x %x\n\000"
-L_current_thread_paniced:
- .long 0
- .text
-current_thread_cont:
-#endif
- mtmsr r9 /* Restore interruptions to entry */
- blr /* Return... */
+LEXT(get_preemption_level)
+
+ mfsprg r6,1 ; Get current activation
+ lwz r3,ACT_PREEMPT_CNT(r6) ; Get the preemption level
+ blr ; Return...
/*
- * Set the active thread
+ * void ppc_usimple_lock_init(simple_lock_t, etap_event_t)
+ *
+ * Initialize a simple lock.
*/
.align 5
- .globl EXT(set_machine_current_thread)
-LEXT(set_machine_current_thread)
-
- mfmsr r9 /* Save the old MSR */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtmsr r8 /* Interrupts off */
- isync ; May have messed with fp/vec
- mfsprg r6,0 /* Get the per_proc */
- stw r3,PP_ACTIVE_THREAD(r6) /* Set the active thread */
- mtmsr r9 /* Restore interruptions to entry */
- blr /* Return... */
+ .globl EXT(ppc_usimple_lock_init)
+LEXT(ppc_usimple_lock_init)
+
+ li r0, 0 ; set lock to free == 0
+ stw r0, 0(r3) ; Initialize the lock
+ blr
+
/*
- * Set the current activation
+ * void lck_spin_lock(lck_spin_t *)
+ * void ppc_usimple_lock(simple_lock_t *)
+ *
*/
.align 5
- .globl EXT(set_machine_current_act)
-LEXT(set_machine_current_act)
- mtsprg 1,r3 /* Set spr1 with the active thread */
- blr /* Return... */
+ .globl EXT(lck_spin_lock)
+LEXT(lck_spin_lock)
+ .globl EXT(ppc_usimple_lock)
+LEXT(ppc_usimple_lock)
+
+ mfsprg r6,1 ; Get the current activation
+ lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
+ addi r5,r5,1 ; Bring up the disable count
+ stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
+ mr r5,r3 ; Get the address of the lock
+ li r8,0 ; Set r8 to zero
+ li r4,0 ; Set r4 to zero
+
+slcktry: lwarx r11,SLOCK_ILK,r5 ; Grab the lock value
+ andi. r3,r11,ILK_LOCKED ; Is it locked?
+ ori r11,r6,ILK_LOCKED ; Set interlock
+ bne-- slckspin ; Yeah, wait for it to clear...
+ stwcx. r11,SLOCK_ILK,r5 ; Try to seize that there durn lock
+ bne-- slcktry ; Couldn't get it...
+ .globl EXT(slckPatch_isync)
+LEXT(slckPatch_isync)
+ isync ; Make sure we don't use a speculativily loaded value
+ blr ; Go on home...
+
+slckspin: li r11,lgKillResv ; Killing field
+ stwcx. r11,0,r11 ; Kill reservation
+
+ mr. r4,r4 ; Test timeout value
+ bne++ slockspin0
+ lis r4,hi16(EXT(LockTimeOut)) ; Get the high part
+ ori r4,r4,lo16(EXT(LockTimeOut)) ; And the low part
+ lwz r4,0(r4) ; Get the timerout value
+
+slockspin0: mr. r8,r8 ; Is r8 set to zero
+ bne++ slockspin1 ; If yes, first spin attempt
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Turn off interruptions
+ isync ; May have turned off vec and fp here
+ mftb r8 ; Get timestamp on entry
+ b slcksniff
+
+slockspin1: mtmsr r7 ; Turn off interruptions
+ mftb r8 ; Get timestamp on entry
+
+slcksniff: lwz r3,SLOCK_ILK(r5) ; Get that lock in here
+ andi. r3,r3,ILK_LOCKED ; Is it free yet?
+ beq++ slckretry ; Yeah, try for it again...
+
+ mftb r10 ; Time stamp us now
+ sub r10,r10,r8 ; Get the elapsed time
+ cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
+ blt++ slcksniff ; Not yet...
+
+ mtmsr r9 ; Say, any interrupts pending?
+
+; The following instructions force the pipeline to be interlocked to that only one
+; instruction is issued per cycle. The insures that we stay enabled for a long enough
+; time; if it's too short, pending interruptions will not have a chance to be taken
+
+ subi r4,r4,128 ; Back off elapsed time from timeout value
+ or r4,r4,r4 ; Do nothing here but force a single cycle delay
+ mr. r4,r4 ; See if we used the whole timeout
+ li r3,0 ; Assume a timeout return code
+ or r4,r4,r4 ; Do nothing here but force a single cycle delay
+
+ ble-- slckfail ; We failed
+ b slockspin1 ; Now that we've opened an enable window, keep trying...
+slckretry:
+ mtmsr r9 ; Restore interrupt state
+ li r8,1 ; Show already through once
+ b slcktry
+slckfail: ; We couldn't get the lock
+ lis r3,hi16(slckpanic_str)
+ ori r3,r3,lo16(slckpanic_str)
+ mr r4,r5
+ mflr r5
+ PROLOG(0)
+ bl EXT(panic)
+ BREAKPOINT_TRAP ; We die here anyway
+
+ .data
+slckpanic_str:
+ STRINGD "simple lock (0x%08X) deadlock detection, pc=0x%08X\n\000"
+ .text
/*
- * Return the current activation
+ * boolean_t lck_spin_try_lock(lck_spin_t *)
+ * unsigned int ppc_usimple_lock_try(simple_lock_t *)
+ *
*/
.align 5
- .globl EXT(current_act)
-LEXT(current_act)
- mfsprg r3,1
+ .globl EXT(lck_spin_try_lock)
+LEXT(lck_spin_try_lock)
+ .globl EXT(ppc_usimple_lock_try)
+LEXT(ppc_usimple_lock_try)
+
+ lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
+ mfmsr r9 ; Get the MSR value
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
+ ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
+ andc r9,r9,r0 ; Clear FP and VEC
+ andc r7,r9,r7 ; Clear EE as well
+ mtmsr r7 ; Disable interruptions and thus, preemption
+ mfsprg r6,1 ; Get current activation
+
+ lwz r11,SLOCK_ILK(r3) ; Get the lock
+ andi. r5,r11,ILK_LOCKED ; Check it...
+ bne-- slcktryfail ; Quickly fail...
+
+slcktryloop:
+ lwarx r11,SLOCK_ILK,r3 ; Ld from addr of arg and reserve
+
+ andi. r5,r11,ILK_LOCKED ; TEST...
+ ori r5,r6,ILK_LOCKED
+ bne-- slcktryfailX ; branch if taken. Predict free
+
+ stwcx. r5,SLOCK_ILK,r3 ; And SET (if still reserved)
+ bne-- slcktryloop ; If set failed, loop back
+
+ .globl EXT(stlckPatch_isync)
+LEXT(stlckPatch_isync)
+ isync
+
+ lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
+ addi r5,r5,1 ; Bring up the disable count
+ stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
+
+ mtmsr r9 ; Allow interruptions now
+ li r3,1 ; Set that the lock was free
blr
+slcktryfailX:
+ li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Kill reservation
+
+slcktryfail:
+ mtmsr r9 ; Allow interruptions now
+ li r3,0 ; FAILURE - lock was taken
+ blr
/*
- * Return the current preemption level
+ * void lck_spin_unlock(lck_spin_t *)
+ * void ppc_usimple_unlock_rwcmb(simple_lock_t *)
+ *
*/
-
.align 5
- .globl EXT(get_preemption_level)
-
-LEXT(get_preemption_level)
-
- mfmsr r9 /* Save the old MSR */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtmsr r8 /* Interrupts off */
- isync
- mfsprg r6,0 /* Get the per_proc */
- lwz r3,PP_PREEMPT_CNT(r6) /* Get the preemption level */
- mtmsr r9 /* Restore interruptions to entry */
- blr /* Return... */
+ .globl EXT(lck_spin_unlock)
+LEXT(lck_spin_unlock)
+ .globl EXT(ppc_usimple_unlock_rwcmb)
+LEXT(ppc_usimple_unlock_rwcmb)
+
+ li r0,0
+ .globl EXT(sulckPatch_isync)
+LEXT(sulckPatch_isync)
+ isync
+ .globl EXT(sulckPatch_eieio)
+LEXT(sulckPatch_eieio)
+ eieio
+ stw r0, SLOCK_ILK(r3)
+ b epStart ; Go enable preemption...
/*
- * Return the cpu_data
+ * void ppc_usimple_unlock_rwmb(simple_lock_t *)
+ *
*/
-
.align 5
- .globl EXT(get_cpu_data)
+ .globl EXT(ppc_usimple_unlock_rwmb)
-LEXT(get_cpu_data)
-
- mfsprg r3,0 /* Get the per_proc */
- addi r3,r3,PP_ACTIVE_THREAD /* Get the pointer to the CPU data from per proc */
- blr /* Return... */
+LEXT(ppc_usimple_unlock_rwmb)
+ li r0,0
+ sync
+ stw r0, SLOCK_ILK(r3)
+
+ b epStart ; Go enable preemption...
/*
- * Return the simple lock count
+ * void lck_rw_lock_exclusive(lck_rw_t*)
+ *
*/
-
.align 5
- .globl EXT(get_simple_lock_count)
-
-LEXT(get_simple_lock_count)
-
- mfmsr r9 /* Save the old MSR */
- rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtmsr r8 /* Interrupts off */
- isync ; May have messed with vec/fp
- mfsprg r6,0 /* Get the per_proc */
- lwz r3,PP_SIMPLE_LOCK_CNT(r6) /* Get the simple lock count */
- mtmsr r9 /* Restore interruptions to entry */
- blr /* Return... */
+ .globl EXT(lck_rw_lock_exclusive)
+LEXT(lck_rw_lock_exclusive)
+#if !MACH_LDEBUG
+ .globl EXT(lock_write)
+LEXT(lock_write)
+#endif
+ lis r7,0xFFFF
+ ori r7,r7,(WANT_EXCL|WANT_UPGRADE|ILK_LOCKED)
+rwleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ and. r8,r5,r7 ; Can we have it?
+ ori r6,r5,WANT_EXCL ; Mark Exclusive
+ bne-- rwlespin ; Branch if cannot be held
+ stwcx. r6,RW_DATA,r3 ; Update lock word
+ bne-- rwleloop
+ .globl EXT(rwlePatch_isync)
+LEXT(rwlePatch_isync)
+ isync
+ blr
+rwlespin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwlespin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_lock_exclusive_ext)
+rwlespin1:
+ b EXT(lck_rw_lock_exclusive_gen)
/*
- * fast_usimple_lock():
+ * void lck_rw_lock_shared(lck_rw_t*)
*
- * If EE is off, get the simple lock without incrementing the preemption count and
- * mark The simple lock with SLOCK_FAST.
- * If EE is on, call usimple_lock().
*/
.align 5
- .globl EXT(fast_usimple_lock)
-
-LEXT(fast_usimple_lock)
-
-#if CHECKNMI
- b EXT(usimple_lock) ; (TEST/DEBUG)
-#endif
- mfmsr r9
- andi. r7,r9,lo16(MASK(MSR_EE))
- bne- L_usimple_lock_c
-L_usimple_lock_loop:
- lwarx r4,0,r3
- li r5,ILK_LOCKED|SLOCK_FAST
- mr. r4,r4
- bne- L_usimple_lock_c
- stwcx. r5,0,r3
- bne- L_usimple_lock_loop
- isync
- blr
-L_usimple_lock_c:
- b EXT(usimple_lock)
+ .globl EXT(lck_rw_lock_shared)
+LEXT(lck_rw_lock_shared)
+#if !MACH_LDEBUG
+ .globl EXT(lock_read)
+LEXT(lock_read)
+#endif
+rwlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ andi. r7,r5,WANT_EXCL|WANT_UPGRADE|ILK_LOCKED ; Can we have it?
+ bne-- rwlsopt ; Branch if cannot be held
+rwlsloopres:
+ addis r6,r5,1 ; Increment read cnt
+ stwcx. r6,RW_DATA,r3 ; Update lock word
+ bne-- rwlsloop
+ .globl EXT(rwlsPatch_isync)
+LEXT(rwlsPatch_isync)
+ isync
+ blr
+rwlsopt:
+ andi. r7,r5,PRIV_EXCL|ILK_LOCKED ; Can we have it?
+ bne-- rwlsspin ; Branch if cannot be held
+ lis r7,0xFFFF ; Get read cnt mask
+ and. r8,r5,r7 ; Is it shared
+ bne rwlsloopres ; Branch if can be held
+rwlsspin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwlsspin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_lock_shared_ext)
+rwlsspin1:
+ b EXT(lck_rw_lock_shared_gen)
/*
- * fast_usimple_lock_try():
+ * boolean_t lck_rw_lock_shared_to_exclusive(lck_rw_t*)
*
- * If EE is off, try to get the simple lock. The preemption count doesn't get incremented and
- * if successfully held, the simple lock is marked with SLOCK_FAST.
- * If EE is on, call usimple_lock_try()
*/
.align 5
- .globl EXT(fast_usimple_lock_try)
-
-LEXT(fast_usimple_lock_try)
-
-#if CHECKNMI
- b EXT(usimple_lock_try) ; (TEST/DEBUG)
-#endif
- mfmsr r9
- andi. r7,r9,lo16(MASK(MSR_EE))
- bne- L_usimple_lock_try_c
-L_usimple_lock_try_loop:
- lwarx r4,0,r3
- li r5,ILK_LOCKED|SLOCK_FAST
- mr. r4,r4
- bne- L_usimple_lock_try_fail
- stwcx. r5,0,r3
- bne- L_usimple_lock_try_loop
- li r3,1
- isync
- blr
-L_usimple_lock_try_fail:
- li r3,0
- blr
-L_usimple_lock_try_c:
- b EXT(usimple_lock_try)
+ .globl EXT(lck_rw_lock_shared_to_exclusive)
+LEXT(lck_rw_lock_shared_to_exclusive)
+#if !MACH_LDEBUG
+ .globl EXT(lock_read_to_write)
+LEXT(lock_read_to_write)
+#endif
+rwlseloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ addis r6,r5,0xFFFF ; Decrement read cnt
+ lis r8,0xFFFF ; Get read count mask
+ ori r8,r8,WANT_UPGRADE|ILK_LOCKED ; Include Interlock and upgrade flags
+ and. r7,r6,r8 ; Can we have it?
+ ori r9,r6,WANT_UPGRADE ; Mark Exclusive
+ bne-- rwlsespin ; Branch if cannot be held
+ stwcx. r9,RW_DATA,r3 ; Update lock word
+ bne-- rwlseloop
+ .globl EXT(rwlsePatch_isync)
+LEXT(rwlsePatch_isync)
+ isync
+ li r3,1 ; Succeed, return TRUE...
+ blr
+rwlsespin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwlsespin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_lock_shared_to_exclusive_ext)
+rwlsespin1:
+ b EXT(lck_rw_lock_shared_to_exclusive_gen)
+
+
/*
- * fast_usimple_unlock():
+ * void lck_rw_lock_exclusive_to_shared(lck_rw_t*)
*
- * If the simple lock is marked SLOCK_FAST, release it without decrementing the preemption count.
- * Call usimple_unlock() otherwise.
*/
.align 5
- .globl EXT(fast_usimple_unlock)
-
-LEXT(fast_usimple_unlock)
-
-#if CHECKNMI
- b EXT(usimple_unlock) ; (TEST/DEBUG)
-#endif
- lwz r5,LOCK_DATA(r3)
- li r0,0
- cmpi cr0,r5,ILK_LOCKED|SLOCK_FAST
- bne- L_usimple_unlock_c
- sync
-#if 0
- mfmsr r9
- andi. r7,r9,lo16(MASK(MSR_EE))
- beq L_usimple_unlock_cont
- lis r3,hi16(L_usimple_unlock_panic)
- ori r3,r3,lo16(L_usimple_unlock_panic)
- bl EXT(panic)
-
- .data
-L_usimple_unlock_panic:
- STRINGD "fast_usimple_unlock: interrupts not disabled\n\000"
- .text
-L_usimple_unlock_cont:
+ .globl EXT(lck_rw_lock_exclusive_to_shared)
+LEXT(lck_rw_lock_exclusive_to_shared)
+#if !MACH_LDEBUG
+ .globl EXT(lock_write_to_read)
+LEXT(lock_write_to_read)
#endif
- stw r0, LOCK_DATA(r3)
- blr
-L_usimple_unlock_c:
- b EXT(usimple_unlock)
+ .globl EXT(rwlesPatch_isync)
+LEXT(rwlesPatch_isync)
+ isync
+ .globl EXT(rwlesPatch_eieio)
+LEXT(rwlesPatch_eieio)
+ eieio
+rwlesloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ andi. r7,r5,ILK_LOCKED ; Test interlock flag
+ bne-- rwlesspin ; Branch if interlocked
+ lis r6,1 ; Get 1 for read count
+ andi. r10,r5,WANT_UPGRADE ; Is it held with upgrade
+ li r9,WANT_UPGRADE|WAIT_FLAG ; Get upgrade and wait flags mask
+ bne rwlesexcl1 ; Skip if held with upgrade
+ li r9,WANT_EXCL|WAIT_FLAG ; Get exclusive and wait flags mask
+rwlesexcl1:
+ andc r7,r5,r9 ; Marked free
+ rlwimi r6,r7,0,16,31 ; Set shared cnt to one
+ stwcx. r6,RW_DATA,r3 ; Update lock word
+ bne-- rwlesloop
+ andi. r7,r5,WAIT_FLAG ; Test wait flag
+ beqlr++ ; Return of no waiters
+ addi r3,r3,RW_EVENT ; Get lock event address
+ b EXT(thread_wakeup) ; wakeup waiters
+rwlesspin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwlesspin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_lock_exclusive_to_shared_ext)
+rwlesspin1:
+ b EXT(lck_rw_lock_exclusive_to_shared_gen)
+
+
/*
- * enter_funnel_section():
+ * boolean_t lck_rw_try_lock_exclusive(lck_rw_t*)
*
*/
.align 5
- .globl EXT(enter_funnel_section)
-
-LEXT(enter_funnel_section)
+ .globl EXT(lck_rw_try_lock_exclusive)
+LEXT(lck_rw_try_lock_exclusive)
+ lis r10,0xFFFF ; Load read count mask
+ ori r10,r10,WANT_EXCL|WANT_UPGRADE ; Include exclusive and upgrade flags
+rwtleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ andi. r7,r5,ILK_LOCKED ; Test interlock flag
+ bne-- rwtlespin ; Branch if interlocked
+ and. r7,r5,r10 ; Can we have it
+ ori r6,r5,WANT_EXCL ; Mark Exclusive
+ bne-- rwtlefail ;
+ stwcx. r6,RW_DATA,r3 ; Update lock word
+ bne-- rwtleloop
+ .globl EXT(rwtlePatch_isync)
+LEXT(rwtlePatch_isync)
+ isync
+ li r3,1 ; Return TRUE
+ blr
+rwtlefail:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ li r3,0 ; Return FALSE
+ blr
+rwtlespin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwtlespin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_try_lock_exclusive_ext)
+rwtlespin1:
+ b EXT(lck_rw_try_lock_exclusive_gen)
-#if !MACH_LDEBUG
- lis r10,hi16(EXT(kdebug_enable))
- ori r10,r10,lo16(EXT(kdebug_enable))
- lwz r10,0(r10)
- lis r11,hi16(EXT(split_funnel_off))
- ori r11,r11,lo16(EXT(split_funnel_off))
- lwz r11,0(r11)
- or. r10,r11,r10 ; Check kdebug_enable or split_funnel_off
- bne- L_enter_funnel_section_slow1 ; If set, call the slow path
- mfsprg r6,1 ; Get the current activation
- lwz r7,LOCK_FNL_MUTEX(r3)
- mfmsr r11
- rlwinm r11,r11,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r11,r11,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r10,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1
- mtmsr r10 ; Turn off EE
- isync ; May have messed with vec/fp
- mr r9,r6
-L_enter_funnel_section_loop:
- lwarx r5,0,r7 ; Load the mutex lock
- mr. r5,r5
- bne- L_enter_funnel_section_slow ; Go to the slow path
- stwcx. r6,0,r7 ; Grab the lock
- bne- L_enter_funnel_section_loop ; Loop back if failed
- isync ; Stop prefeteching
- lwz r6,ACT_THREAD(r6) ; Get the current thread
- li r7,TH_FN_OWNED
- stw r7,THREAD_FUNNEL_STATE(r6) ; Set the funnel state
- stw r3,THREAD_FUNNEL_LOCK(r6) ; Set the funnel lock reference
- mtmsr r11
- blr
-
-L_enter_funnel_section_slow:
- mtmsr r11
-L_enter_funnel_section_slow1:
-#endif
- li r4,TRUE
- b EXT(thread_funnel_set)
/*
- * exit_funnel_section():
+ * boolean_t lck_rw_try_lock_shared(lck_rw_t*)
*
*/
.align 5
- .globl EXT(exit_funnel_section)
+ .globl EXT(lck_rw_try_lock_shared)
+LEXT(lck_rw_try_lock_shared)
+rwtlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ andi. r7,r5,ILK_LOCKED ; Test interlock flag
+ bne-- rwtlsspin ; Branch if interlocked
+ andi. r7,r5,WANT_EXCL|WANT_UPGRADE ; So, can we have it?
+ bne-- rwtlsopt ; Branch if held exclusive
+rwtlsloopres:
+ addis r6,r5,1 ; Increment read cnt
+ stwcx. r6,RW_DATA,r3 ; Update lock word
+ bne-- rwtlsloop
+ .globl EXT(rwtlsPatch_isync)
+LEXT(rwtlsPatch_isync)
+ isync
+ li r3,1 ; Return TRUE
+ blr
+rwtlsopt:
+ andi. r7,r5,PRIV_EXCL ; Can we have it?
+ bne-- rwtlsfail ; Branch if cannot be held
+ lis r7,0xFFFF ; Get read cnt mask
+ and. r8,r5,r7 ; Is it shared
+ bne rwtlsloopres ; Branch if can be held
+rwtlsfail:
+ li r3,0 ; Return FALSE
+ blr
+rwtlsspin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwtlsspin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_try_lock_shared_ext)
+rwtlsspin1:
+ b EXT(lck_rw_try_lock_shared_gen)
-LEXT(exit_funnel_section)
+
+/*
+ * lck_rw_type_t lck_rw_done(lck_rw_t*)
+ *
+ */
+ .align 5
+ .globl EXT(lck_rw_done)
+LEXT(lck_rw_done)
#if !MACH_LDEBUG
- mfsprg r6,1 ; Get the current activation
- lwz r6,ACT_THREAD(r6) ; Get the current thread
- lwz r3,THREAD_FUNNEL_LOCK(r6) ; Get the funnel lock
- mr. r3,r3 ; Check on funnel held
- beq- L_exit_funnel_section_ret ;
- lis r10,hi16(EXT(kdebug_enable))
- ori r10,r10,lo16(EXT(kdebug_enable))
- lwz r10,0(r10)
- mr. r10,r10
- bne- L_exit_funnel_section_slow1 ; If set, call the slow path
- lwz r7,LOCK_FNL_MUTEX(r3) ; Get the funnel mutex lock
- mfmsr r11
- rlwinm r11,r11,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r11,r11,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r10,r11,0,MSR_EE_BIT+1,MSR_EE_BIT-1
- mtmsr r10 ; Turn off EE
- isync ; May have messed with fp/vec
- sync
-L_exit_funnel_section_loop:
- lwarx r5,0,r7
- rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
- li r5,0 ; Clear the mutexlock
- bne- L_exit_funnel_section_slow
- stwcx. r5,0,r7 ; Release the funnel mutexlock
- bne- L_exit_funnel_section_loop
- li r7,0
- stw r7,THREAD_FUNNEL_STATE(r6) ; Clear the funnel state
- stw r7,THREAD_FUNNEL_LOCK(r6) ; Clear the funnel lock reference
- mtmsr r11
-L_exit_funnel_section_ret:
- blr
-L_exit_funnel_section_slow:
- mtmsr r11
-L_exit_funnel_section_slow1:
+ .globl EXT(lock_done)
+LEXT(lock_done)
#endif
- li r4,FALSE
- b EXT(thread_funnel_set)
+ .globl EXT(rwldPatch_isync)
+LEXT(rwldPatch_isync)
+ isync
+ .globl EXT(rwldPatch_eieio)
+LEXT(rwldPatch_eieio)
+ eieio
+ li r10,WAIT_FLAG ; Get wait flag
+ lis r7,0xFFFF ; Get read cnt mask
+ mr r12,r3 ; Save lock addr
+rwldloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
+ andi. r8,r5,ILK_LOCKED ; Test interlock flag
+ bne-- rwldspin ; Branch if interlocked
+ and. r8,r5,r7 ; Is it shared
+ cmpi cr1,r8,0 ; Is it shared
+ beq cr1,rwldexcl ; No, check exclusive
+ li r11,RW_SHARED ; Set return value
+ addis r6,r5,0xFFFF ; Decrement read count
+ and. r8,r6,r7 ; Is it still shared
+ li r8,0 ; Assume no wakeup
+ bne rwldshared1 ; Skip if still held shared
+ and r8,r6,r10 ; Extract wait flag
+ andc r6,r6,r10 ; Clear wait flag
+rwldshared1:
+ b rwldstore
+rwldexcl:
+ li r11,RW_EXCL ; Set return value
+ li r9,WANT_UPGRADE ; Get upgrade flag
+ and. r6,r5,r9 ; Is it held with upgrade
+ li r9,WANT_UPGRADE|WAIT_FLAG ; Mask upgrade abd wait flags
+ bne rwldexcl1 ; Skip if held with upgrade
+ li r9,WANT_EXCL|WAIT_FLAG ; Mask exclusive and wait flags
+rwldexcl1:
+ andc r6,r5,r9 ; Marked free
+ and r8,r5,r10 ; Null if no waiter
+rwldstore:
+ stwcx. r6,RW_DATA,r3 ; Update lock word
+ bne-- rwldloop
+ mr. r8,r8 ; wakeup needed?
+ mr r3,r11 ; Return lock held type
+ beqlr++
+ mr r3,r12 ; Restore lock address
+ PROLOG(0)
+ addi r3,r3,RW_EVENT ; Get lock event address
+ bl EXT(thread_wakeup) ; wakeup threads
+ lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
+ mtcr r2
+ EPILOG
+ li r3,RW_SHARED ; Assume lock type shared
+ bne cr1,rwldret ; Branch if was held exclusive
+ li r3,RW_EXCL ; Return lock type exclusive
+rwldret:
+ blr
+rwldspin:
+ li r4,lgKillResv ; Killing field
+ stwcx. r4,0,r4 ; Kill it
+ cmpli cr0,r5,RW_IND ; Is it a lock indirect
+ bne-- rwldspin1 ; No, go handle contention
+ mr r4,r3 ; pass lock pointer
+ lwz r3,RW_PTR(r3) ; load lock ext pointer
+ b EXT(lck_rw_done_ext)
+rwldspin1:
+ b EXT(lck_rw_done_gen)
+/*
+ * void lck_rw_ilk_lock(lck_rw_t *lock)
+ */
+ .globl EXT(lck_rw_ilk_lock)
+LEXT(lck_rw_ilk_lock)
+ crclr hwtimeout ; no timeout option
+ li r4,0 ; request default timeout value
+ li r12,ILK_LOCKED ; Load bit mask
+ b lckcomm ; Join on up...
+
+/*
+ * void lck_rw_ilk_unlock(lck_rw_t *lock)
+ */
+ .globl EXT(lck_rw_ilk_unlock)
+LEXT(lck_rw_ilk_unlock)
+ li r4,1
+ b EXT(hw_unlock_bit)
projects / apple / xnu.git / blobdiff