X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d7e50217d7adf6e52786a38bcaa4cd698cb9a79e..4a3eedf9ecc9bbe3f3a5c6ce5e53ad199d639d32:/osfmk/ppc/hw_lock.s diff --git a/osfmk/ppc/hw_lock.s b/osfmk/ppc/hw_lock.s index 08605f5f7..4fae9fe6e 100644 --- a/osfmk/ppc/hw_lock.s +++ b/osfmk/ppc/hw_lock.s @@ -1,16 +1,19 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ - * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -20,192 +23,76 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include #include #include -#include #include #include #include -#define STRING ascii - -#define SWT_HI 0+FM_SIZE -#define SWT_LO 4+FM_SIZE -#define MISSED 8+FM_SIZE - -#define ILK_LOCKED 0x01 -#define WAIT_FLAG 0x02 -#define TH_FN_OWNED 0x01 - -#define CHECKNMI 0 -#define CHECKLOCKS 1 - -#define PROLOG(space) \ - stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \ - mflr r0 __ASMNL__ \ - stw r3,FM_ARG0(r1) __ASMNL__ \ - stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__ - -#define EPILOG \ - lwz r1,0(r1) __ASMNL__ \ - lwz r0,FM_LR_SAVE(r1) __ASMNL__ \ - mtlr r0 __ASMNL__ - -#if MACH_LDEBUG && CHECKLOCKS -/* - * Routines for general lock debugging. - */ -/* - * Gets lock check flags in CR6: CR bits 24-27 - */ +#include +#if CONFIG_DTRACE + #define LOCKSTAT_LABEL(lab) \ + .data __ASMNL__ \ + .globl lab __ASMNL__ \ + lab: __ASMNL__ \ + .long 9f __ASMNL__ \ + .text __ASMNL__ \ + 9: __ASMNL__ \ + + .globl _dtrace_probe, _lockstat_probemap +#define LOCKSTAT_RECORD(id) \ + lis r6,hi16(_lockstat_probemap) __ASMNL__ \ + ori r6,r6,lo16(_lockstat_probemap) __ASMNL__ \ + lwz r5,4*id(r6) __ASMNL__ \ + mr. r5,r5 __ASMNL__ \ + beqlr-- __ASMNL__ \ + mr r4,r3 __ASMNL__ \ + mr r3,r5 __ASMNL__ \ + li r5,0 __ASMNL__ \ + li r6,0 __ASMNL__ \ + li r7,0 __ASMNL__ \ + li r8,0 __ASMNL__ \ + PROLOG(0) __ASMNL__ \ + bl _dtrace_probe __ASMNL__ \ + EPILOG +#endif + -#define CHECK_SETUP(rg) \ - lbz rg,dgFlags(0) __ASMNL__ \ - mtcrf 2,rg __ASMNL__ +#define STRING ascii -/* - * Checks for expected lock types and calls "panic" on - * mismatch. Detects calls to Mutex functions with - * type simplelock and vice versa. - */ -#define CHECK_MUTEX_TYPE() \ - bt 24+disLktypeb,1f __ASMNL__ \ - lwz r10,MUTEX_TYPE(r3) __ASMNL__ \ - cmpwi r10,MUTEX_TAG __ASMNL__ \ - beq+ 1f __ASMNL__ \ - lis r3,hi16(not_a_mutex) __ASMNL__ \ - ori r3,r3,lo16(not_a_mutex) __ASMNL__ \ - bl EXT(panic) __ASMNL__ \ - lwz r3,FM_ARG0(r1) __ASMNL__ \ -1: - - .data -not_a_mutex: - STRINGD "not a mutex!\n\000" - .text +#define ILK_LOCKED 0x01 +#define WAIT_FLAG 0x02 +#define WANT_UPGRADE 0x04 +#define WANT_EXCL 0x08 +#define PRIV_EXCL 0x8000 -#define CHECK_SIMPLE_LOCK_TYPE() \ - bt 24+disLktypeb,1f __ASMNL__ \ - lhz r10,SLOCK_TYPE(r3) __ASMNL__ \ - cmpwi r10,USLOCK_TAG __ASMNL__ \ - beq+ 1f __ASMNL__ \ - lis r3,hi16(not_a_slock) __ASMNL__ \ - ori r3,r3,lo16(not_a_slock) __ASMNL__ \ - bl EXT(panic) __ASMNL__ \ - lwz r3,FM_ARG0(r1) __ASMNL__ \ -1: - - .data -not_a_slock: - STRINGD "not a simple lock!\n\000" - .text +#define TH_FN_OWNED 0x01 -#define CHECK_NO_SIMPLELOCKS() \ - bt 24+disLkNmSimpb,2f __ASMNL__ \ - lis r10,hi16(MASK(MSR_VEC)) __ASMNL__ \ - ori r10,r10,lo16(MASK(MSR_FP)) __ASMNL__ \ - mfmsr r11 __ASMNL__ \ - andc r11,r11,r10 __ASMNL__ \ - ori r10,r10,lo16(MASK(MSR_EE)) __ASMNL__ \ - andc r10,r11,r10 __ASMNL__ \ - mtmsr r10 __ASMNL__ \ - isync __ASMNL__ \ - mfsprg r10,0 __ASMNL__ \ - lwz r10,PP_SIMPLE_LOCK_CNT(r10) __ASMNL__ \ - cmpwi r10,0 __ASMNL__ \ - beq+ 1f __ASMNL__ \ - lis r3,hi16(simple_locks_held) __ASMNL__ \ - ori r3,r3,lo16(simple_locks_held) __ASMNL__ \ - bl EXT(panic) __ASMNL__ \ - lwz r3,FM_ARG0(r1) __ASMNL__ \ -1: __ASMNL__ \ - mtmsr r11 __ASMNL__ \ -2: - - .data -simple_locks_held: - STRINGD "simple locks held!\n\000" - .text +# volatile CR bits +#define hwtimeout 20 +#define mlckmiss 21 -/* - * Verifies return to the correct thread in "unlock" situations. - */ -#define CHECK_THREAD(thread_offset) \ - bt 24+disLkThreadb,2f __ASMNL__ \ - lis r10,hi16(MASK(MSR_VEC)) __ASMNL__ \ - ori r10,r10,lo16(MASK(MSR_FP)) __ASMNL__ \ - mfmsr r11 __ASMNL__ \ - andc r11,r11,r10 __ASMNL__ \ - ori r10,r10,lo16(MASK(MSR_EE)) __ASMNL__ \ - andc r10,r11,r10 __ASMNL__ \ - mtmsr r10 __ASMNL__ \ - isync __ASMNL__ \ - mfsprg r10,0 __ASMNL__ \ - lwz r10,PP_ACTIVE_THREAD(r10) __ASMNL__ \ - cmpwi r10,0 __ASMNL__ \ - beq- 1f __ASMNL__ \ - lwz r9,thread_offset(r3) __ASMNL__ \ - cmpw r9,r10 __ASMNL__ \ - beq+ 1f __ASMNL__ \ - lis r3,hi16(wrong_thread) __ASMNL__ \ - ori r3,r3,lo16(wrong_thread) __ASMNL__ \ - bl EXT(panic) __ASMNL__ \ - lwz r3,FM_ARG0(r1) __ASMNL__ \ -1: __ASMNL__ \ - mtmsr r11 __ASMNL__ \ -2: - .data -wrong_thread: - STRINGD "wrong thread!\n\000" - .text +#define RW_DATA 0 -#define CHECK_MYLOCK(thread_offset) \ - bt 24+disLkMyLckb,2f __ASMNL__ \ - lis r10,hi16(MASK(MSR_VEC)) __ASMNL__ \ - ori r10,r10,lo16(MASK(MSR_FP)) __ASMNL__ \ - mfmsr r11 __ASMNL__ \ - andc r11,r11,r10 __ASMNL__ \ - ori r10,r10,lo16(MASK(MSR_EE)) __ASMNL__ \ - andc r10,r11,r10 __ASMNL__ \ - mtmsr r10 __ASMNL__ \ - isync __ASMNL__ \ - mfsprg r10,0 __ASMNL__ \ - lwz r10,PP_ACTIVE_THREAD(r10) __ASMNL__ \ - cmpwi r10,0 __ASMNL__ \ - beq- 1f __ASMNL__ \ - lwz r9, thread_offset(r3) __ASMNL__ \ - cmpw r9,r10 __ASMNL__ \ - bne+ 1f __ASMNL__ \ - lis r3, hi16(mylock_attempt) __ASMNL__ \ - ori r3,r3,lo16(mylock_attempt) __ASMNL__ \ - bl EXT(panic) __ASMNL__ \ - lwz r3,FM_ARG0(r1) __ASMNL__ \ -1: __ASMNL__ \ - mtmsr r11 __ASMNL__ \ -2: +#define PROLOG(space) \ + stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \ + mfcr r2 __ASMNL__ \ + mflr r0 __ASMNL__ \ + stw r3,FM_ARG0(r1) __ASMNL__ \ + stw r11,FM_ARG0+0x04(r1) __ASMNL__ \ + stw r2,(FM_ALIGN(space)+FM_SIZE+FM_CR_SAVE)(r1) __ASMNL__ \ + stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__ - .data -mylock_attempt: - STRINGD "mylock attempt!\n\000" - .text +#define EPILOG \ + lwz r1,0(r1) __ASMNL__ \ + lwz r0,FM_LR_SAVE(r1) __ASMNL__ \ + mtlr r0 __ASMNL__ -#else /* MACH_LDEBUG */ - -#define CHECK_SETUP(rg) -#define CHECK_MUTEX_TYPE() -#define CHECK_SIMPLE_LOCK_TYPE() -#define CHECK_THREAD(thread_offset) -#define CHECK_NO_SIMPLELOCKS() -#define CHECK_MYLOCK(thread_offset) - -#endif /* MACH_LDEBUG */ - /* * void hw_lock_init(hw_lock_t) * @@ -221,26 +108,21 @@ LEXT(hw_lock_init) blr /* - * void hw_lock_unlock(hw_lock_t) + * unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout) * - * Unconditionally release lock. - * Release preemption level. + * Try to acquire spin-lock. The second parameter is the bit mask to test and set. + * multiple bits may be set. Return success (1) or failure (0). + * Attempt will fail after timeout ticks of the timebase. */ .align 5 - .globl EXT(hw_lock_unlock) - -LEXT(hw_lock_unlock) + .globl EXT(hw_lock_bit) - .globl EXT(hwulckPatch_isync) -LEXT(hwulckPatch_isync) - isync - .globl EXT(hwulckPatch_eieio) -LEXT(hwulckPatch_eieio) - eieio - li r0, 0 ; set lock to free - stw r0, 0(r3) +LEXT(hw_lock_bit) - b epStart ; Go enable preemption... + crset hwtimeout ; timeout option + mr r12,r4 ; Load bit mask + mr r4,r5 ; Load timeout value + b lckcomm ; Join on up... /* * void hw_lock_lock(hw_lock_t) @@ -253,8 +135,15 @@ LEXT(hwulckPatch_eieio) .globl EXT(hw_lock_lock) LEXT(hw_lock_lock) + crclr hwtimeout ; no timeout option + li r4,0 ; request default timeout value + li r12,ILK_LOCKED ; Load bit mask + b lckcomm ; Join on up... + lockDisa: - li r4,0 ; no timeout value + crset hwtimeout ; timeout option + li r4,0 ; request default timeout value + li r12,ILK_LOCKED ; Load bit mask b lckcomm ; Join on up... /* @@ -273,13 +162,8 @@ lockDisa: .globl EXT(hw_lock_to) LEXT(hw_lock_to) - -#if CHECKNMI - mflr r12 ; (TEST/DEBUG) - bl EXT(ml_sense_nmi) ; (TEST/DEBUG) - mtlr r12 ; (TEST/DEBUG) -#endif - + crset hwtimeout ; timeout option + li r12,ILK_LOCKED ; Load bit mask lckcomm: mfsprg r6,1 ; Get the current activation lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level @@ -289,12 +173,14 @@ lckcomm: li r8,0 ; Set r8 to zero lcktry: lwarx r6,0,r5 ; Grab the lock value - andi. r3,r6,ILK_LOCKED ; Is it locked? - ori r6,r6,ILK_LOCKED ; Set interlock + and. r3,r6,r12 ; Is it locked? + or r6,r6,r12 ; Set interlock bne-- lckspin ; Yeah, wait for it to clear... stwcx. r6,0,r5 ; Try to seize that there durn lock bne-- lcktry ; Couldn't get it... li r3,1 ; return true + .globl EXT(hwllckPatch_isync) +LEXT(hwllckPatch_isync) isync ; Make sure we don't use a speculativily loaded value blr ; Go on home... @@ -324,7 +210,7 @@ lockspin1: mtmsr r7 ; Turn off interruptions mftb r8 ; Get timestamp on entry lcksniff: lwz r3,0(r5) ; Get that lock in here - andi. r3,r3,ILK_LOCKED ; Is it free yet? + and. r3,r3,r12 ; Is it free yet? beq++ lckretry ; Yeah, try for it again... mftb r10 ; Time stamp us now @@ -351,94 +237,42 @@ lckretry: li r8,1 ; Insure that R8 is not 0 b lcktry lckfail: ; We couldn't get the lock + bf hwtimeout,lckpanic li r3,0 ; Set failure return code blr ; Return, head hanging low... - +lckpanic: + mr r4,r5 + mr r5,r3 + lis r3,hi16(lckpanic_str) ; Get the failed lck message + ori r3,r3,lo16(lckpanic_str) ; Get the failed lck message + bl EXT(panic) + BREAKPOINT_TRAP ; We die here anyway + .data +lckpanic_str: + STRINGD "timeout on attempt to acquire lock (0x%08X), value = 0x%08X\n\000" + .text /* - * unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout) + * void hw_lock_unlock(hw_lock_t) * - * Try to acquire spin-lock. The second parameter is the bit mask to test and set. - * multiple bits may be set. Return success (1) or failure (0). - * Attempt will fail after timeout ticks of the timebase. - * We try fairly hard to get this lock. We disable for interruptions, but - * reenable after a "short" timeout (128 ticks, we may want to shorten this). - * After checking to see if the large timeout value (passed in) has expired and a - * sufficient number of cycles have gone by (to insure pending 'rupts are taken), - * we return either in abject failure, or disable and go back to the lock sniff routine. - * If the sniffer finds the lock free, it jumps right up and tries to grab it. + * Unconditionally release lock. + * Release preemption level. */ .align 5 - .globl EXT(hw_lock_bit) - -LEXT(hw_lock_bit) - - li r10,0 - -bittry: lwarx r6,0,r3 ; Grab the lock value - and. r0,r6,r4 ; See if any of the lock bits are on - or r6,r6,r4 ; Turn on the lock bits - bne-- bitspin ; Yeah, wait for it to clear... - stwcx. r6,0,r3 ; Try to seize that there durn lock - bne-- bittry ; Just start up again if the store failed... - - li r3,1 ; Set good return code - isync ; Make sure we don't use a speculativily loaded value - blr - - .align 5 - -bitspin: li r11,lgKillResv ; Get killing field - stwcx. r11,0,r11 ; Kill reservation - - mr. r10,r10 ; Is r8 set to zero - li r10,1 ; Close gate - beq-- bit1sttime ; If yes, first spin attempt - -bitspin0: mtmsr r7 ; Turn off interruptions - mftb r8 ; Get the low part of the time base - -bitsniff: lwz r6,0(r3) ; Get that lock in here - and. r0,r6,r4 ; See if any of the lock bits are on - beq++ bitretry ; Yeah, try for it again... - - mftb r6 ; Time stamp us now - sub r6,r6,r8 ; Get the elapsed time - cmplwi r6,128 ; Have we been spinning for 128 tb ticks? - blt++ bitsniff ; Not yet... - - mtmsr r9 ; Say, any interrupts pending? - -; The following instructions force the pipeline to be interlocked to that only one -; instruction is issued per cycle. The insures that we stay enabled for a long enough -; time. If it's too short, pending interruptions will not have a chance to be taken - - subi r5,r5,128 ; Back off elapsed time from timeout value - or r5,r5,r5 ; Do nothing here but force a single cycle delay - mr. r5,r5 ; See if we used the whole timeout - or r5,r5,r5 ; Do nothing here but force a single cycle delay - - bgt++ bitspin0 ; Now that we've opened an enable window, keep trying... - - li r3,0 ; Set failure return code - blr ; Return, head hanging low... - -bitretry: mtmsr r9 ; Enable for interruptions - b bittry + .globl EXT(hw_lock_unlock) -bit1sttime: lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable - mfmsr r9 ; Get the MSR value - ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable - ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too - andc r9,r9,r0 ; Clear FP and VEC - andc r7,r9,r7 ; Clear EE as well - mtmsr r7 ; Turn off interruptions - isync ; May have turned off vec and fp here - mftb r8 ; Get the low part of the time base - b bitsniff +LEXT(hw_lock_unlock) - .align 5 + .globl EXT(hwulckPatch_isync) +LEXT(hwulckPatch_isync) + isync + .globl EXT(hwulckPatch_eieio) +LEXT(hwulckPatch_eieio) + eieio + li r0, 0 ; set lock to free + stw r0, 0(r3) + b epStart ; Go enable preemption... /* * unsigned int hw_unlock_bit(hw_lock_t, unsigned int bit) @@ -463,7 +297,7 @@ ubittry: lwarx r0,0,r3 ; Grab the lock value stwcx. r0,0,r3 ; Try to clear that there durn lock bne- ubittry ; Try again, couldn't save it... - blr ; Leave... + b epStart ; Go enable preemption... /* * unsigned int hw_lock_mbits(hw_lock_t, unsigned int bits, unsigned int value, @@ -548,6 +382,8 @@ mbitretry: .align 5 mbitgot: li r3,1 ; Set good return code + .globl EXT(hwlmlckPatch_isync) +LEXT(hwlmlckPatch_isync) isync ; Make sure we do not use a speculativily loaded value blr @@ -659,6 +495,8 @@ LEXT(hw_lock_try) stwcx. r5,0,r3 ; And SET (if still reserved) bne-- .L_lock_try_loop ; If set failed, loop back + .globl EXT(hwltlckPatch_isync) +LEXT(hwltlckPatch_isync) isync mfsprg r6,1 ; Get current activation @@ -716,6 +554,8 @@ cstry: lwarx r9,0,r5 ; Grab the area value bne-- csfail ; No, it must have changed... stwcx. r4,0,r5 ; Try to save the new value bne-- cstry ; Didn't get it, try again... + .globl EXT(hwcsatomicPatch_isync) +LEXT(hwcsatomicPatch_isync) isync ; Just hold up prefetch blr ; Return... @@ -776,9 +616,9 @@ subtry: lwarx r3,0,r6 ; Grab the area value */ .align 5 .globl EXT(hw_atomic_or) - LEXT(hw_atomic_or) - + .globl EXT(hw_atomic_or_noret) +LEXT(hw_atomic_or_noret) mr r6,r3 ; Save the area ortry: lwarx r3,0,r6 ; Grab the area value @@ -797,9 +637,9 @@ ortry: lwarx r3,0,r6 ; Grab the area value */ .align 5 .globl EXT(hw_atomic_and) - LEXT(hw_atomic_and) - + .globl EXT(hw_atomic_and_noret) +LEXT(hw_atomic_and_noret) mr r6,r3 ; Save the area andtry: lwarx r3,0,r6 ; Grab the area value @@ -817,6 +657,7 @@ andtry: lwarx r3,0,r6 ; Grab the area value * element is the pointer to the element to insert * disp is the displacement into the element to the chain pointer * + * NOTE: OSEnqueueAtomic() is aliased to this, see xnu/libkern/Makefile */ .align 5 .globl EXT(hw_queue_atomic) @@ -860,6 +701,8 @@ hw_queue_comm: * anchor is the pointer to the first element * disp is the displacement into the element to the chain pointer * Returns element if found, 0 if empty. + * + * NOTE: OSDequeueAtomic() is aliased to this, see xnu/libkern/Makefile */ .align 5 .globl EXT(hw_dequeue_atomic) @@ -882,101 +725,411 @@ hdcFail: li r4,lgKillResv ; Killing field blr ; Leave... +/* + * Routines for mutex lock debugging. + */ + +/* + * Gets lock check flags in CR6: CR bits 24-27 + */ +#define CHECK_SETUP(rg) \ + lbz rg,lglcksWork(0) __ASMNL__ \ + mtcrf 2,rg __ASMNL__ + + +/* + * Checks for expected lock type. + */ +#define CHECK_MUTEX_TYPE() \ + bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \ + bt 24+disLktypeb,1f __ASMNL__ \ + lwz r10,MUTEX_TYPE(r3) __ASMNL__ \ + cmpwi r10,MUTEX_TAG __ASMNL__ \ + beq++ 1f __ASMNL__ \ + PROLOG(0) __ASMNL__ \ + mr r4,r11 __ASMNL__ \ + mr r5,r10 __ASMNL__ \ + lis r3,hi16(not_a_mutex) __ASMNL__ \ + ori r3,r3,lo16(not_a_mutex) __ASMNL__ \ + bl EXT(panic) __ASMNL__ \ + BREAKPOINT_TRAP __ASMNL__ \ +1: + + .data +not_a_mutex: + STRINGD "mutex (0x%08X) not a mutex type (0x%08X)\n\000" + .text + +/* + * Verifies return to the correct thread in "unlock" situations. + */ +#define CHECK_THREAD(thread_offset) \ + bf MUTEX_ATTR_DEBUGb,3f __ASMNL__ \ + bt 24+disLkThreadb,3f __ASMNL__ \ + mfsprg r10,1 __ASMNL__ \ + lwz r5,MUTEX_DATA(r3) __ASMNL__ \ + rlwinm. r9,r5,0,0,29 __ASMNL__ \ + bne++ 1f __ASMNL__ \ + lis r3,hi16(not_held) __ASMNL__ \ + ori r3,r3,lo16(not_held) __ASMNL__ \ + b 2f __ASMNL__ \ +1: __ASMNL__ \ + cmpw r9,r10 __ASMNL__ \ + beq++ 3f __ASMNL__ \ + mr r5,r10 __ASMNL__ \ + mr r6,r9 __ASMNL__ \ + lis r3,hi16(wrong_thread) __ASMNL__ \ + ori r3,r3,lo16(wrong_thread) __ASMNL__ \ +2: __ASMNL__ \ + mr r4,r11 __ASMNL__ \ + PROLOG(0) __ASMNL__ \ + bl EXT(panic) __ASMNL__ \ + BREAKPOINT_TRAP __ASMNL__ \ +3: + + .data +not_held: + STRINGD "mutex (0x%08X) not held\n\000" +wrong_thread: + STRINGD "mutex (0x%08X) unlocked by non-owner(0x%08X), current owner(0x%08X)\n\000" + .text + +#define CHECK_MYLOCK() \ + bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \ + bt 24+disLkMyLckb,1f __ASMNL__ \ + mfsprg r10,1 __ASMNL__ \ + lwz r9,MUTEX_DATA(r3) __ASMNL__ \ + rlwinm r9,r9,0,0,29 __ASMNL__ \ + cmpw r9,r10 __ASMNL__ \ + bne++ 1f __ASMNL__ \ + mr r4,r11 __ASMNL__ \ + lis r3, hi16(mylock_attempt) __ASMNL__ \ + ori r3,r3,lo16(mylock_attempt) __ASMNL__ \ + bl EXT(panic) __ASMNL__ \ + BREAKPOINT_TRAP __ASMNL__ \ +1: + + .data +mylock_attempt: + STRINGD "mutex (0x%08X) recursive lock attempt\n\000" + .text + +#define LCK_STACK(lck, stack, lck_stack, frame_cnt, lr_save, tmp) \ + bf 24+enaLkExtStckb,3f __ASMNL__ \ + addi lck_stack,lck,MUTEX_STACK __ASMNL__ \ + li frame_cnt,MUTEX_FRAMES-1 __ASMNL__ \ +1: __ASMNL__ \ + mr tmp,stack __ASMNL__ \ + lwz stack,0(stack) __ASMNL__ \ + xor tmp,stack,tmp __ASMNL__ \ + cmplwi tmp,8192 __ASMNL__ \ + bge-- 2f __ASMNL__ \ + lwz lr_save,FM_LR_SAVE(stack) __ASMNL__ \ + stwu lr_save,4(lck_stack) __ASMNL__ \ + subi frame_cnt,frame_cnt,1 __ASMNL__ \ + cmpi cr0,frame_cnt,0 __ASMNL__ \ + bne 1b __ASMNL__ \ + b 3f __ASMNL__ \ +2: __ASMNL__ \ + li tmp,0 __ASMNL__ \ + stwu tmp,4(lck_stack) __ASMNL__ \ + subi frame_cnt,frame_cnt,1 __ASMNL__ \ + cmpi cr0,frame_cnt,0 __ASMNL__ \ + bne 2b __ASMNL__ \ +3: + /* * void mutex_init(mutex_t* l, etap_event_t etap) * */ .align 5 .globl EXT(mutex_init) - LEXT(mutex_init) PROLOG(0) - li r10, 0 - stw r10, LOCK_DATA(r3) ; clear lock word - sth r10, MUTEX_WAITERS(r3) ; init waiter count - sth r10, MUTEX_PROMOTED_PRI(r3) + li r10,0 + stw r10,MUTEX_DATA(r3) ; clear lock word + sth r10,MUTEX_WAITERS(r3) ; init waiter count + sth r10,MUTEX_PROMOTED_PRI(r3) #if MACH_LDEBUG - stw r10, MUTEX_PC(r3) ; init caller pc - stw r10, MUTEX_THREAD(r3) ; and owning thread - li r10, MUTEX_TAG - stw r10, MUTEX_TYPE(r3) ; set lock type + li r11,MUTEX_ATTR_DEBUG + stw r10,MUTEX_STACK(r3) ; init caller pc + stw r10,MUTEX_THREAD(r3) ; and owning thread + li r9, MUTEX_TAG + stw r9, MUTEX_TYPE(r3) ; set lock type + stw r11,MUTEX_ATTR(r3) + addi r8,r3,MUTEX_STACK-4 + li r9,MUTEX_FRAMES +mlistck: + stwu r10,4(r8) ; init stack + subi r9,r9,1 + cmpi cr0,r9,0 + bne mlistck #endif /* MACH_LDEBUG */ EPILOG blr /* - * void mutex_lock(mutex_t*) + * void lck_mtx_lock_ext(lck_mtx_ext_t*) * */ .align 5 + .globl EXT(lck_mtx_lock_ext) +LEXT(lck_mtx_lock_ext) +#if MACH_LDEBUG .globl EXT(mutex_lock) LEXT(mutex_lock) .globl EXT(_mutex_lock) LEXT(_mutex_lock) - -#if !MACH_LDEBUG - mfsprg r6,1 ; load the current thread - lwz r5,0(r3) ; Get the lock quickly - mr. r5,r5 ; Quick check - bne-- L_mutex_lock_slow ; Can not get it right now... - -L_mutex_lock_loop: - lwarx r5,0,r3 ; load the mutex lock - mr. r5,r5 - bne-- L_mutex_lock_slowX ; go to the slow path - stwcx. r6,0,r3 ; grab the lock - bne-- L_mutex_lock_loop ; loop back if failed - isync ; stop prefeteching - blr - -L_mutex_lock_slowX: - li r5,lgKillResv ; Killing field - stwcx. r5,0,r5 ; Kill reservation - -L_mutex_lock_slow: -#endif -#if CHECKNMI - mflr r12 ; (TEST/DEBUG) - bl EXT(ml_sense_nmi) ; (TEST/DEBUG) - mtlr r12 ; (TEST/DEBUG) #endif + mr r11,r3 ; Save lock addr +mlckeEnter: + lwz r0,MUTEX_ATTR(r3) + mtcrf 1,r0 ; Set cr7 + CHECK_SETUP(r12) + CHECK_MUTEX_TYPE() - PROLOG(12) -#if MACH_LDEBUG + bf MUTEX_ATTR_DEBUGb,L_mutex_lock_assert_wait_2 + PROLOG(0) bl EXT(assert_wait_possible) mr. r3,r3 bne L_mutex_lock_assert_wait_1 lis r3,hi16(L_mutex_lock_assert_wait_panic_str) ori r3,r3,lo16(L_mutex_lock_assert_wait_panic_str) bl EXT(panic) + BREAKPOINT_TRAP ; We die here anyway .data L_mutex_lock_assert_wait_panic_str: - STRINGD "mutex_lock: assert_wait_possible false\n\000" + STRINGD "mutex lock attempt with assert_wait_possible false\n\000" .text L_mutex_lock_assert_wait_1: lwz r3,FM_ARG0(r1) -#endif - CHECK_SETUP(r12) - CHECK_MUTEX_TYPE() - CHECK_NO_SIMPLELOCKS() -.L_ml_retry: - bl lockDisa ; Go get a lock on the mutex's interlock lock - mr. r4,r3 ; Did we get it? - lwz r3,FM_ARG0(r1) ; Restore the lock address - bne+ mlGotInt ; We got it just fine... + lwz r11,FM_ARG0+0x04(r1) + lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1) + mtcr r2 + EPILOG +L_mutex_lock_assert_wait_2: - lis r3,hi16(mutex_failed1) ; Get the failed mutex message - ori r3,r3,lo16(mutex_failed1) ; Get the failed mutex message - bl EXT(panic) ; Call panic - BREAKPOINT_TRAP ; We die here anyway, can not get the lock - + mfsprg r6,1 ; load the current thread + bf MUTEX_ATTR_STATb,mlckestatskip ; Branch if no stat + lwz r5,MUTEX_GRP(r3) ; Load lock group + li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset +mlckestatloop: + lwarx r8,r7,r5 ; Load stat util cnt + addi r8,r8,1 ; Increment stat util cnt + stwcx. r8,r7,r5 ; Store stat util cnt + bne-- mlckestatloop ; Retry if failed + mr. r8,r8 ; Test for zero + bne++ mlckestatskip ; Did stat util cnt wrapped? + lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt + addi r8,r8,1 ; Increment upper stat util cnt + stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt +mlckestatskip: + lwz r5,MUTEX_DATA(r3) ; Get the lock quickly + li r4,0 + li r8,0 + lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable + mfmsr r9 ; Get the MSR value + ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable + ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too + andc r9,r9,r0 ; Clear FP and VEC + andc r7,r9,r7 ; Clear EE as well + mtmsr r7 ; Turn off interruptions + isync ; May have turned off vec and fp here + mr. r5,r5 ; Quick check + bne-- mlckespin01 ; Can not get it right now... + +mlcketry: + lwarx r5,MUTEX_DATA,r3 ; load the mutex lock + mr. r5,r5 + bne-- mlckespin0 ; Can not get it right now... + stwcx. r6,MUTEX_DATA,r3 ; grab the lock + bne-- mlcketry ; loop back if failed + .globl EXT(mlckePatch_isync) +LEXT(mlckePatch_isync) + isync ; stop prefeteching + mflr r12 + bf MUTEX_ATTR_DEBUGb,mlckedebskip + mr r8,r6 ; Get the active thread + stw r12,MUTEX_STACK(r3) ; Save our caller + stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread + mr r5,r1 + LCK_STACK(r3,r5,r6,r7,r8,r10) +mlckedebskip: + mtmsr r9 ; Say, any interrupts pending? + blr + +mlckespin0: + li r5,lgKillResv ; Killing field + stwcx. r5,0,r5 ; Kill reservation +mlckespin01: + mflr r12 + mtmsr r9 ; Say, any interrupts pending? + bl mlckspin1 + mtmsr r7 ; Turn off interruptions, vec and fp off already + mtlr r12 + b mlcketry + +/* + * void lck_mtx_lock(lck_mtx_t*) + * + */ + .align 5 + .globl EXT(lck_mtx_lock) +LEXT(lck_mtx_lock) + +#if !MACH_LDEBUG + .globl EXT(mutex_lock) +LEXT(mutex_lock) + + .globl EXT(_mutex_lock) +LEXT(_mutex_lock) +#endif + + mfsprg r6,1 ; load the current thread + lwz r5,MUTEX_DATA(r3) ; Get the lock quickly + mr r11,r3 ; Save lock addr + li r4,0 + li r8,0 + li r9,0 + mr. r5,r5 ; Quick check + bne-- mlckspin00 ; Indirect or Can not get it right now... + +mlcktry: + lwarx r5,MUTEX_DATA,r3 ; load the mutex lock + mr. r5,r5 + bne-- mlckspin01 ; Can not get it right now... + stwcx. r6,MUTEX_DATA,r3 ; grab the lock + bne-- mlcktry ; loop back if failed + .globl EXT(mlckPatch_isync) +LEXT(mlckPatch_isync) + isync ; stop prefeteching + blr +; Need to debug making blr above a patch point and record: +; LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE) + +mlckspin00: + cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect + bne-- mlckspin02 ; No, go handle contention + lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer + b mlckeEnter +mlckspin01: + li r5,lgKillResv ; Killing field + stwcx. r5,0,r5 ; Kill reservation +mlckspin02: + mflr r12 + li r0,0 + mtcrf 1,r0 ; Set cr7 to zero + bl mlckspin1 + mtlr r12 + b mlcktry + + +mlckspin1: + mr. r4,r4 ; Test timeout value + bne++ mlckspin2 + lis r4,hi16(EXT(MutexSpin)) ; Get the high part + ori r4,r4,lo16(EXT(MutexSpin) ) ; And the low part + lwz r4,0(r4) ; Get spin timerout value + mr. r4,r4 ; Test spin timeout value + bne++ mlckspin2 ; Is spin timeout requested + crclr mlckmiss ; Clear miss test + b mlckslow1 ; Don't try to spin + +mlckspin2: mr. r8,r8 ; Is r8 set to zero + bne++ mlckspin3 ; If yes, first spin attempt + crclr mlckmiss ; Clear miss test + mr. r9,r9 ; Is r9 set to zero + bne++ mlckspin3 ; If yes, r9 set with msr value + lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable + mfmsr r9 ; Get the MSR value + ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable + ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too + andc r9,r9,r0 ; Clear FP and VEC + andc r7,r9,r7 ; Clear EE as well + mtmsr r7 ; Turn off interruptions + isync ; May have turned off vec and fp here + mftb r8 ; Get timestamp on entry + b mlcksniff + +mlckspin3: mtmsr r7 ; Turn off interruptions + mftb r8 ; Get timestamp on entry + +mlcksniff: lwz r5,MUTEX_DATA(r3) ; Get that lock in here + mr. r5,r5 ; Is the lock held + beq++ mlckretry ; No, try for it again... + rlwinm. r10,r5,0,0,29 ; Extract the lock owner + beq++ mlckslow0 ; InterLock is held + bf MUTEX_ATTR_STATb,mlStatSkip ; Branch if no stat + andi. r5,r5,ILK_LOCKED ; extract interlocked? + bne mlStatSkip ; yes, skip + bt mlckmiss,mlStatSkip ; miss already counted + crset mlckmiss ; Remember miss recorded + lwz r5,MUTEX_GRP(r3) ; Load lock group + addi r5,r5,GRP_MTX_STAT_MISS+4 ; Add stat miss offset +mlStatLoop: + lwarx r6,0,r5 ; Load stat miss cnt + addi r6,r6,1 ; Increment stat miss cnt + stwcx. r6,0,r5 ; Update stat miss cnt + bne-- mlStatLoop ; Retry if failed + mfsprg r6,1 ; Reload current thread +mlStatSkip: + lwz r2,ACT_MACT_SPF(r10) ; Get the special flags + rlwinm. r2,r2,0,OnProcbit,OnProcbit ; Is OnProcbit set? + beq mlckslow0 ; Lock owner isn't running + lis r2,hi16(TH_IDLE) ; Get thread idle state + ori r2,r2,lo16(TH_IDLE) ; Get thread idle state + lwz r10,THREAD_STATE(r10) ; Get the thread state + and. r10,r10,r2 ; Is idle set? + bne mlckslow0 ; Lock owner is idling + + mftb r10 ; Time stamp us now + sub r10,r10,r8 ; Get the elapsed time + cmplwi r10,128 ; Have we been spinning for 128 tb ticks? + blt++ mlcksniff ; Not yet... + + mtmsr r9 ; Say, any interrupts pending? + +; The following instructions force the pipeline to be interlocked to that only one +; instruction is issued per cycle. The insures that we stay enabled for a long enough +; time; if it's too short, pending interruptions will not have a chance to be taken + + subi r4,r4,128 ; Back off elapsed time from timeout value + or r4,r4,r4 ; Do nothing here but force a single cycle delay + mr. r4,r4 ; See if we used the whole timeout + or r4,r4,r4 ; Do nothing here but force a single cycle delay + + ble-- mlckslow1 ; We failed + b mlckspin3 ; Now that we've opened an enable window, keep trying... +mlckretry: + mtmsr r9 ; Restore interrupt state + li r8,1 ; Show already through once + blr + +mlckslow0: ; We couldn't get the lock + mtmsr r9 ; Restore interrupt state + +mlckslow1: + mtlr r12 + + PROLOG(0) +.L_ml_retry: + bl lockDisa ; Go get a lock on the mutex's interlock lock + mr. r4,r3 ; Did we get it? + lwz r3,FM_ARG0(r1) ; Restore the lock address + bne++ mlGotInt ; We got it just fine... + mr r4,r11 ; Saved lock addr + lis r3,hi16(mutex_failed1) ; Get the failed mutex message + ori r3,r3,lo16(mutex_failed1) ; Get the failed mutex message + bl EXT(panic) ; Call panic + BREAKPOINT_TRAP ; We die here anyway, can not get the lock + .data mutex_failed1: - STRINGD "We can't get a mutex interlock lock on mutex_lock\n\000" + STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock\n\000" .text mlGotInt: @@ -985,39 +1138,33 @@ mlGotInt: ; hold the interlock lock and no one can touch this field unless they ; have that, so, we're free to play - lwz r4,LOCK_DATA(r3) ; Get the mutex's lock field + lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field rlwinm. r9,r4,30,2,31 ; So, can we have it? bne- mlInUse ; Nope, sombody's playing already... -#if MACH_LDEBUG - li r5,lo16(MASK(MSR_EE)) ; Get the EE bit - mfmsr r11 ; Note: no need to deal with fp or vec here - andc r5,r11,r5 - mtmsr r5 - mfsprg r9,0 ; Get the per_proc block + bf++ MUTEX_ATTR_DEBUGb,mlDebSkip + CHECK_SETUP(r5) + mfsprg r9,1 ; Get the current activation lwz r5,0(r1) ; Get previous save frame - lwz r5,FM_LR_SAVE(r5) ; Get our caller's address - lwz r8, PP_ACTIVE_THREAD(r9) ; Get the active thread - stw r5,MUTEX_PC(r3) ; Save our caller - mr. r8,r8 ; Is there any thread? + lwz r6,FM_LR_SAVE(r5) ; Get our caller's address + mr r8,r9 ; Get the active thread + stw r6,MUTEX_STACK(r3) ; Save our caller stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread - beq- .L_ml_no_active_thread ; No owning thread... - lwz r9,THREAD_MUTEX_COUNT(r8) ; Get the mutex count - addi r9,r9,1 ; Bump it up - stw r9,THREAD_MUTEX_COUNT(r8) ; Stash it back -.L_ml_no_active_thread: - mtmsr r11 -#endif /* MACH_LDEBUG */ - - bl EXT(mutex_lock_acquire) + LCK_STACK(r3,r5,r6,r7,r8,r10) +mlDebSkip: + mr r3,r11 ; Get the based lock address + bl EXT(lck_mtx_lock_acquire) + lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1) mfsprg r5,1 + mtcr r2 mr. r4,r3 - lwz r3,FM_ARG0(r1) + lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog) + lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog) beq mlUnlock ori r5,r5,WAIT_FLAG mlUnlock: eieio - stw r5,LOCK_DATA(r3) ; grab the mutexlock and free the interlock + stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock EPILOG ; Restore all saved registers b epStart ; Go enable preemption... @@ -1028,68 +1175,172 @@ mlUnlock: eieio mlInUse: CHECK_SETUP(r12) - CHECK_MYLOCK(MUTEX_THREAD) ; Assert we don't own the lock already */ + CHECK_MYLOCK() ; Assert we don't own the lock already */ ; Note that we come in here with the interlock set. The wait routine ; will unlock it before waiting. + bf MUTEX_ATTR_STATb,mlStatSkip2 ; Branch if no stat + lwz r5,MUTEX_GRP(r3) ; Load lck group + bt mlckmiss,mlStatSkip1 ; Skip miss already counted + crset mlckmiss ; Remember miss recorded + li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset +mlStatLoop1: + lwarx r8,r9,r5 ; Load stat miss cnt + addi r8,r8,1 ; Increment stat miss cnt + stwcx. r8,r9,r5 ; Store stat miss cnt + bne-- mlStatLoop1 ; Retry if failed +mlStatSkip1: + lwz r9,GRP_MTX_STAT_WAIT+4(r5) ; Load wait cnt + addi r9,r9,1 ; Increment wait cnt + stw r9,GRP_MTX_STAT_WAIT+4(r5) ; Update miss cnt +mlStatSkip2: ori r4,r4,WAIT_FLAG ; Set the wait flag - stw r4,LOCK_DATA(r3) + stw r4,MUTEX_DATA(r3) rlwinm r4,r4,0,0,29 ; Extract the lock owner - bl EXT(mutex_lock_wait) ; Wait for our turn at the lock + mfcr r2 + stw r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1) + mr r3,r11 ; Get the based lock address + bl EXT(lck_mtx_lock_wait) ; Wait for our turn at the lock lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog) + lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog) + lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1) + mtcr r2 b .L_ml_retry ; and try again... /* - * void _mutex_try(mutex_t*) + * void lck_mtx_try_lock(_extlck_mtx_ext_t*) * */ .align 5 + .globl EXT(lck_mtx_try_lock_ext) +LEXT(lck_mtx_try_lock_ext) +#if MACH_LDEBUG .globl EXT(mutex_try) LEXT(mutex_try) .globl EXT(_mutex_try) LEXT(_mutex_try) -#if !MACH_LDEBUG +#endif + mr r11,r3 ; Save lock addr +mlteEnter: + lwz r0,MUTEX_ATTR(r3) + mtcrf 1,r0 ; Set cr7 + CHECK_SETUP(r12) + CHECK_MUTEX_TYPE() + + bf MUTEX_ATTR_STATb,mlteStatSkip ; Branch if no stat + lwz r5,MUTEX_GRP(r3) ; Load lock group + li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset +mlteStatLoop: + lwarx r8,r7,r5 ; Load stat util cnt + addi r8,r8,1 ; Increment stat util cnt + stwcx. r8,r7,r5 ; Store stat util cnt + bne-- mlteStatLoop ; Retry if failed + mr. r8,r8 ; Test for zero + bne++ mlteStatSkip ; Did stat util cnt wrapped? + lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt + addi r8,r8,1 ; Increment upper stat util cnt + stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt +mlteStatSkip: mfsprg r6,1 ; load the current thread - lwz r5,0(r3) ; Get the lock value + lwz r5,MUTEX_DATA(r3) ; Get the lock value mr. r5,r5 ; Quick check bne-- L_mutex_try_slow ; Can not get it now... + mfmsr r9 ; Get the MSR value + lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable + ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable + ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too + andc r9,r9,r0 ; Clear FP and VEC + andc r7,r9,r7 ; Clear EE as well + mtmsr r7 ; Turn off interruptions + isync ; May have turned off vec and fp here -L_mutex_try_loop: - lwarx r5,0,r3 ; load the lock value +mlteLoopTry: + lwarx r5,MUTEX_DATA,r3 ; load the lock value mr. r5,r5 - bne-- L_mutex_try_slowX ; branch to the slow path - stwcx. r6,0,r3 ; grab the lock - bne-- L_mutex_try_loop ; retry if failed + bne-- mlteSlowX ; branch to the slow path + stwcx. r6,MUTEX_DATA,r3 ; grab the lock + bne-- mlteLoopTry ; retry if failed + .globl EXT(mltelckPatch_isync) +LEXT(mltelckPatch_isync) isync ; stop prefetching + mflr r12 + bf MUTEX_ATTR_DEBUGb,mlteDebSkip + mr r8,r6 ; Get the active thread + stw r12,MUTEX_STACK(r3) ; Save our caller + stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread + mr r5,r1 + LCK_STACK(r3,r5,r6,r7,r8,r10) +mlteDebSkip: li r3, 1 + mtmsr r9 ; Say, any interrupts pending? blr - -L_mutex_try_slowX: +mlteSlowX: li r5,lgKillResv ; Killing field stwcx. r5,0,r5 ; Kill reservation + mtmsr r9 ; Say, any interrupts pending? + b L_mutex_try_slow -L_mutex_try_slow: +/* + * void lck_mtx_try_lock(lck_mtx_t*) + * + */ + .align 5 + .globl EXT(lck_mtx_try_lock) +LEXT(lck_mtx_try_lock) +#if !MACH_LDEBUG + .globl EXT(mutex_try) +LEXT(mutex_try) + .globl EXT(_mutex_try) +LEXT(_mutex_try) #endif - PROLOG(8) ; reserve space for SWT_HI and SWT_LO + mfsprg r6,1 ; load the current thread + lwz r5,MUTEX_DATA(r3) ; Get the lock value + mr r11,r3 ; Save lock addr + mr. r5,r5 ; Quick check + bne-- mltSlow00 ; Indirect or Can not get it now... + +mltLoopTry: + lwarx r5,MUTEX_DATA,r3 ; load the lock value + mr. r5,r5 + bne-- mltSlow01 ; branch to the slow path + stwcx. r6,MUTEX_DATA,r3 ; grab the lock + bne-- mltLoopTry ; retry if failed + .globl EXT(mltlckPatch_isync) +LEXT(mltlckPatch_isync) + isync ; stop prefetching + li r3, 1 + blr + +mltSlow00: + cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect + bne-- mltSlow02 ; No, go handle contention + lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer + b mlteEnter +mltSlow01: + li r5,lgKillResv ; Killing field + stwcx. r5,0,r5 ; Kill reservation + +mltSlow02: + li r0,0 + mtcrf 1,r0 ; Set cr7 to zero + +L_mutex_try_slow: + PROLOG(0) - CHECK_SETUP(r12) - CHECK_MUTEX_TYPE() - CHECK_NO_SIMPLELOCKS() - - lwz r6,LOCK_DATA(r3) ; Quick check + lwz r6,MUTEX_DATA(r3) ; Quick check rlwinm. r6,r6,30,2,31 ; to see if someone has this lock already bne- mtFail ; Someone's got it already... bl lockDisa ; Go get a lock on the mutex's interlock lock - mr. r4,r3 ; Did we get it? */ + mr. r4,r3 ; Did we get it? lwz r3,FM_ARG0(r1) ; Restore the lock address - bne+ mtGotInt ; We got it just fine... - + bne++ mtGotInt ; We got it just fine... + mr r4,r11 ; Saved lock addr lis r3,hi16(mutex_failed2) ; Get the failed mutex message ori r3,r3,lo16(mutex_failed2) ; Get the failed mutex message bl EXT(panic) ; Call panic @@ -1097,7 +1348,7 @@ L_mutex_try_slow: .data mutex_failed2: - STRINGD "We can't get a mutex interlock lock on mutex_try\n\000" + STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock try\n\000" .text mtGotInt: @@ -1106,43 +1357,31 @@ mtGotInt: ; hold the interlock and no one can touch at this field unless they ; have that, so, we're free to play - lwz r4,LOCK_DATA(r3) ; Get the mutex's lock field + lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field rlwinm. r9,r4,30,2,31 ; So, can we have it? bne- mtInUse ; Nope, sombody's playing already... -#if MACH_LDEBUG - lis r9,hi16(MASK(MSR_VEC)) ; Get vector enable - mfmsr r11 ; Get the MSR value - ori r9,r9,lo16(MASK(MSR_FP)) ; Get FP enable - ori r5,r9,lo16(MASK(MSR_EE)) ; Get EE bit on too - andc r11,r11,r9 ; Clear FP and VEC - andc r5,r11,r5 ; Clear EE as well - - mtmsr r5 - mfsprg r9,0 ; Get the per_proc block + bf++ MUTEX_ATTR_DEBUGb,mtDebSkip + CHECK_SETUP(r5) + mfsprg r9,1 ; Get the current activation lwz r5,0(r1) ; Get previous save frame - lwz r5,FM_LR_SAVE(r5) ; Get our caller's address - lwz r8, PP_ACTIVE_THREAD(r9) ; Get the active thread - stw r5,MUTEX_PC(r3) ; Save our caller - mr. r8,r8 ; Is there any thread? + lwz r6,FM_LR_SAVE(r5) ; Get our caller's address + mr r8,r9 ; Get the active thread + stw r6,MUTEX_STACK(r3) ; Save our caller stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread - beq- .L_mt_no_active_thread ; No owning thread... - lwz r9, THREAD_MUTEX_COUNT(r8) ; Get the mutex count - addi r9, r9, 1 ; Bump it up - stw r9, THREAD_MUTEX_COUNT(r8) ; Stash it back -.L_mt_no_active_thread: - mtmsr r11 -#endif /* MACH_LDEBUG */ - - bl EXT(mutex_lock_acquire) + LCK_STACK(r3,r5,r6,r7,r8,r10) +mtDebSkip: + mr r3,r11 ; Get the based lock address + bl EXT(lck_mtx_lock_acquire) mfsprg r5,1 mr. r4,r3 - lwz r3,FM_ARG0(r1) + lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog) + lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog) beq mtUnlock ori r5,r5,WAIT_FLAG mtUnlock: eieio - stw r5,LOCK_DATA(r3) ; grab the mutexlock and free the interlock + stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock bl epStart ; Go enable preemption... @@ -1154,8 +1393,17 @@ mtUnlock: eieio ; the mutex is held. mtInUse: + bf++ MUTEX_ATTR_STATb,mtStatSkip ; Branch if no stat + lwz r5,MUTEX_GRP(r3) ; Load lock group + li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset +mtStatLoop: + lwarx r8,r9,r5 ; Load stat miss cnt + addi r8,r8,1 ; Increment stat miss cnt + stwcx. r8,r9,r5 ; Store stat miss cnt + bne-- mtStatLoop ; Retry if failed +mtStatSkip: rlwinm r4,r4,0,0,30 ; Get the unlock value - stw r4,LOCK_DATA(r3) ; free the interlock + stw r4,MUTEX_DATA(r3) ; free the interlock bl epStart ; Go enable preemption... mtFail: li r3,0 ; Set failure code @@ -1164,79 +1412,134 @@ mtFail: li r3,0 ; Set failure code /* - * void mutex_unlock_rwcmb(mutex_t* l) + * void mutex_unlock(mutex_t* l) * */ .align 5 - .globl EXT(mutex_unlock_rwcmb) + .globl EXT(mutex_unlock) +LEXT(mutex_unlock) + sync + mr r11,r3 ; Save lock addr +#if MACH_LDEBUG + b mlueEnter1 +#else + b mluEnter1 +#endif + +/* + * void lck_mtx_ext_unlock(lck_mtx_ext_t* l) + * + */ + .align 5 + .globl EXT(lck_mtx_ext_unlock) +LEXT(lck_mtx_ext_unlock) +#if MACH_LDEBUG + .globl EXT(mutex_unlock_rwcmb) LEXT(mutex_unlock_rwcmb) - .globl EXT(mulckPatch_isync) -LEXT(mulckPatch_isync) +#endif +mlueEnter: + .globl EXT(mulckePatch_isync) +LEXT(mulckePatch_isync) isync - .globl EXT(mulckPatch_eieio) -LEXT(mulckPatch_eieio) + .globl EXT(mulckePatch_eieio) +LEXT(mulckePatch_eieio) eieio + mr r11,r3 ; Save lock addr +mlueEnter1: + lwz r0,MUTEX_ATTR(r3) + mtcrf 1,r0 ; Set cr7 + CHECK_SETUP(r12) + CHECK_MUTEX_TYPE() + CHECK_THREAD(MUTEX_THREAD) - lwz r5,0(r3) ; Get the lock + lwz r5,MUTEX_DATA(r3) ; Get the lock rlwinm. r4,r5,0,30,31 ; Quick check bne-- L_mutex_unlock_slow ; Can not get it now... + mfmsr r9 ; Get the MSR value + lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable + ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable + ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too + andc r9,r9,r0 ; Clear FP and VEC + andc r7,r9,r7 ; Clear EE as well + mtmsr r7 ; Turn off interruptions + isync ; May have turned off vec and fp here -L_mutex_unlock_rwcmb_loop: - lwarx r5,0,r3 +mlueLoop: + lwarx r5,MUTEX_DATA,r3 rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set li r5,0 ; Clear the mutexlock - bne-- L_mutex_unlock_rwcmb_slowX - stwcx. r5,0,r3 - bne-- L_mutex_unlock_rwcmb_loop + bne-- mlueSlowX + stwcx. r5,MUTEX_DATA,r3 + bne-- mlueLoop + mtmsr r9 ; Say, any interrupts pending? blr -L_mutex_unlock_rwcmb_slowX: +mlueSlowX: li r5,lgKillResv ; Killing field stwcx. r5,0,r5 ; Dump reservation + mtmsr r9 ; Say, any interrupts pending? b L_mutex_unlock_slow ; Join slow path... /* - * void mutex_unlock(mutex_t* l) + * void lck_mtx_unlock(lck_mtx_t* l) * */ .align 5 - .globl EXT(mutex_unlock) - -LEXT(mutex_unlock) + .globl EXT(lck_mtx_unlock) +LEXT(lck_mtx_unlock) #if !MACH_LDEBUG - sync - lwz r5,0(r3) ; Get the lock + .globl EXT(mutex_unlock_rwcmb) +LEXT(mutex_unlock_rwcmb) +#endif +mluEnter: + .globl EXT(mulckPatch_isync) +LEXT(mulckPatch_isync) + isync + .globl EXT(mulckPatch_eieio) +LEXT(mulckPatch_eieio) + eieio + mr r11,r3 ; Save lock addr +mluEnter1: + lwz r5,MUTEX_DATA(r3) ; Get the lock rlwinm. r4,r5,0,30,31 ; Quick check - bne-- L_mutex_unlock_slow ; Can not get it now... + bne-- mluSlow0 ; Indirect or Can not get it now... -L_mutex_unlock_loop: - lwarx r5,0,r3 +mluLoop: + lwarx r5,MUTEX_DATA,r3 rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set li r5,0 ; Clear the mutexlock - bne-- L_mutex_unlock_slowX - stwcx. r5,0,r3 - bne-- L_mutex_unlock_loop + bne-- mluSlowX + stwcx. r5,MUTEX_DATA,r3 + bne-- mluLoop +#if CONFIG_DTRACE +/* lock released - LS_LCK_MTX_UNLOCK_RELEASE */ + LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point) blr -L_mutex_unlock_slowX: - li r5,lgKillResv ; Killing field - stwcx. r5,0,r5 ; Dump reservation + LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE) #endif + blr + + +mluSlow0: + cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect + bne-- L_mutex_unlock_slow ; No, go handle contention + lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer + b mlueEnter1 +mluSlowX: + li r5,lgKillResv ; Killing field + stwcx. r5,0,r5 ; Dump reservation L_mutex_unlock_slow: PROLOG(0) - CHECK_SETUP(r12) - CHECK_MUTEX_TYPE() - CHECK_THREAD(MUTEX_THREAD) - bl lockDisa ; Go get a lock on the mutex's interlock lock mr. r4,r3 ; Did we get it? lwz r3,FM_ARG0(r1) ; Restore the lock address - bne+ muGotInt ; We got it just fine... - + bne++ muGotInt ; We got it just fine... + mr r4,r11 ; Saved lock addr lis r3,hi16(mutex_failed3) ; Get the failed mutex message ori r3,r3,lo16(mutex_failed3) ; Get the failed mutex message bl EXT(panic) ; Call panic @@ -1244,61 +1547,87 @@ L_mutex_unlock_slow: .data mutex_failed3: - STRINGD "We can't get a mutex interlock lock on mutex_unlock\n\000" + STRINGD "attempt to interlock mutex (0x%08X) failed on mutex unlock\n\000" .text muGotInt: - lwz r4,LOCK_DATA(r3) + lwz r4,MUTEX_DATA(r3) andi. r5,r4,WAIT_FLAG ; are there any waiters ? rlwinm r4,r4,0,0,29 beq+ muUnlock ; Nope, we're done... - bl EXT(mutex_unlock_wakeup) ; yes, wake a thread + mr r3,r11 ; Get the based lock address + bl EXT(lck_mtx_unlock_wakeup) ; yes, wake a thread lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog) - lwz r5,LOCK_DATA(r3) ; load the lock + lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog) + lwz r5,MUTEX_DATA(r3) ; load the lock muUnlock: -#if MACH_LDEBUG - lis r8,hi16(MASK(MSR_VEC)) ; Get vector enable - mfmsr r11 ; Get the MSR value - ori r8,r8,lo16(MASK(MSR_FP)) ; Get FP enable - ori r9,r8,lo16(MASK(MSR_EE)) ; Get EE bit on too - andc r11,r11,r8 ; Clear FP and VEC - andc r9,r11,r9 ; Clear EE as well - - mtmsr r9 - mfsprg r9,0 - lwz r9,PP_ACTIVE_THREAD(r9) - stw r9,MUTEX_THREAD(r3) ; disown thread - cmpwi r9,0 - beq- .L_mu_no_active_thread - lwz r8,THREAD_MUTEX_COUNT(r9) - subi r8,r8,1 - stw r8,THREAD_MUTEX_COUNT(r9) -.L_mu_no_active_thread: - mtmsr r11 -#endif /* MACH_LDEBUG */ - andi. r5,r5,WAIT_FLAG ; Get the unlock value eieio - stw r5,LOCK_DATA(r3) ; unlock the interlock and lock + stw r5,MUTEX_DATA(r3) ; unlock the interlock and lock EPILOG ; Deal with the stack now, enable_preemption doesn't always want one b epStart ; Go enable preemption... /* - * void interlock_unlock(hw_lock_t lock) + * void lck_mtx_assert(lck_mtx_t* l, unsigned int) + * */ .align 5 - .globl EXT(interlock_unlock) + .globl EXT(lck_mtx_assert) +LEXT(lck_mtx_assert) + .globl EXT(_mutex_assert) +LEXT(_mutex_assert) + mr r11,r3 +maEnter: + lwz r5,MUTEX_DATA(r3) + cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect + bne-- maCheck ; No, go check the assertion + lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer + b maEnter +maCheck: + mfsprg r6,1 ; load the current thread + rlwinm r5,r5,0,0,29 ; Extract the lock owner + cmpwi r4,MUTEX_ASSERT_OWNED + cmplw cr1,r6,r5 ; Is the lock held by current act + crandc cr0_eq,cr0_eq,cr1_eq ; Check owned assertion + bne-- maNext + mr r4,r11 + lis r3,hi16(mutex_assert1) ; Get the failed mutex message + ori r3,r3,lo16(mutex_assert1) ; Get the failed mutex message + b maPanic ; Panic path +maNext: + cmpwi r4,MUTEX_ASSERT_NOTOWNED ; Check not owned assertion + crand cr0_eq,cr0_eq,cr1_eq ; + bnelr++ +maPanic: + PROLOG(0) + mr r4,r11 + lis r3,hi16(mutex_assert2) ; Get the failed mutex message + ori r3,r3,lo16(mutex_assert2) ; Get the failed mutex message + bl EXT(panic) ; Call panic + BREAKPOINT_TRAP ; We die here anyway -LEXT(interlock_unlock) + .data +mutex_assert1: + STRINGD "mutex (0x%08X) not owned\n\000" +mutex_assert2: + STRINGD "mutex (0x%08X) owned\n\000" + .text + + +/* + * void lck_mtx_ilk_unlock(lck_mtx *lock) + */ + .globl EXT(lck_mtx_ilk_unlock) +LEXT(lck_mtx_ilk_unlock) - lwz r10,LOCK_DATA(r3) + lwz r10,MUTEX_DATA(r3) rlwinm r10,r10,0,0,30 eieio - stw r10,LOCK_DATA(r3) + stw r10,MUTEX_DATA(r3) b epStart ; Go enable preemption... @@ -1325,17 +1654,7 @@ LEXT(_enable_preemption_no_check) LEXT(_enable_preemption) -; Here is where we enable preemption. We need to be protected -; against ourselves, we can't chance getting interrupted and modifying -; our processor wide preemption count after we'sve loaded it up. So, -; we need to disable all 'rupts. Actually, we could use a compare -; and swap to do this, but, since there are no MP considerations -; (we are dealing with a CPU local field) it is much, much faster -; to disable. -; -; Note that if we are not genned MP, the calls here will be no-opped via -; a #define and since the _mp forms are the same, likewise a #define -; will be used to route to the other forms +; Here is where we enable preemption. epStart: cmplwi cr1,r1,0 ; Force non-zero cr so we know to check if preempted @@ -1354,11 +1673,13 @@ epTooFar: mr r4,r5 lis r3,hi16(epTooFarStr) ; First half of panic string ori r3,r3,lo16(epTooFarStr) ; Second half of panic string + PROLOG(0) bl EXT(panic) + BREAKPOINT_TRAP ; We die here anyway .data epTooFarStr: - STRINGD "_enable_preemption: preemption_level %d\n\000" + STRINGD "enable_preemption: preemption_level %d\n\000" .text .align 5 @@ -1366,17 +1687,16 @@ epCheckPreempt: lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable mfmsr r9 ; Get the MSR value ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable - andi. r3,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off + andi. r4,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off beq+ epCPno ; No preemption here... ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too andc r9,r9,r0 ; Clear FP and VEC andc r7,r9,r7 ; Clear EE as well mtmsr r7 ; Turn off interruptions isync ; May have turned off vec and fp here - mfsprg r3,0 ; Get per_proc - lwz r7,PP_NEED_AST(r3) ; Get the AST request address + lwz r3,ACT_PER_PROC(r3) ; Get the per_proc block + lwz r7,PP_PENDING_AST(r3) ; Get pending AST mask li r5,AST_URGENT ; Get the requests we do honor - lwz r7,0(r7) ; Get the actual, real live, extra special AST word lis r0,hi16(DoPreemptCall) ; Just in case, get the top of firmware call and. r7,r7,r5 ; Should we preempt? ori r0,r0,lo16(DoPreemptCall) ; Merge in bottom part @@ -1389,13 +1709,7 @@ epCPno: /* * void disable_preemption(void) * - * Here is where we disable preemption. Since preemption is on a - * per processor basis (a thread runs on one CPU at a time) we don't - * need any cross-processor synchronization. We do, however, need to - * be interrupt safe, so we don't preempt while in the process of - * disabling it. We could use SPLs, but since we always want complete - * disablement, and this is platform specific code, we'll just kick the - * MSR. We'll save a couple of orders of magnitude over using SPLs. + * Here is where we disable preemption. */ .align 5 .globl EXT(_disable_preemption) @@ -1422,34 +1736,6 @@ LEXT(get_preemption_level) lwz r3,ACT_PREEMPT_CNT(r6) ; Get the preemption level blr ; Return... -/* - * int get_simple_lock_count(void) - * - * Return the simple lock count - * - */ - .align 5 - .globl EXT(get_simple_lock_count) - -LEXT(get_simple_lock_count) - -#if MACH_LDEBUG - lis r3,hi16(MASK(MSR_VEC)) ; Get vector enable - mfmsr r9 ; Get the MSR value - ori r3,r3,lo16(MASK(MSR_FP)) ; Get FP enable - ori r8,r3,lo16(MASK(MSR_EE)) ; Get EE bit on too - andc r9,r9,r3 ; Clear FP and VEC - andc r8,r9,r8 ; Clear EE as well - mtmsr r8 ; Interrupts off - isync ; May have messed with vec/fp - mfsprg r6,0 ; Get the per_proc - lwz r3,PP_SIMPLE_LOCK_CNT(r6) ; Get the simple lock count - mtmsr r9 ; Restore interruptions to entry -#else - li r3,0 ; simple lock count not updated -#endif - blr ; Return... - /* * void ppc_usimple_lock_init(simple_lock_t, etap_event_t) * @@ -1465,20 +1751,16 @@ LEXT(ppc_usimple_lock_init) blr /* - * void ppc_usimple_lock(simple_lock_t) + * void lck_spin_lock(lck_spin_t *) + * void ppc_usimple_lock(simple_lock_t *) * */ .align 5 + .globl EXT(lck_spin_lock) +LEXT(lck_spin_lock) .globl EXT(ppc_usimple_lock) - LEXT(ppc_usimple_lock) -#if CHECKNMI - mflr r12 ; (TEST/DEBUG) - bl EXT(ml_sense_nmi) ; (TEST/DEBUG) - mtlr r12 ; (TEST/DEBUG) -#endif - mfsprg r6,1 ; Get the current activation lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level addi r5,r5,1 ; Bring up the disable count @@ -1487,12 +1769,14 @@ LEXT(ppc_usimple_lock) li r8,0 ; Set r8 to zero li r4,0 ; Set r4 to zero -slcktry: lwarx r11,0,r5 ; Grab the lock value +slcktry: lwarx r11,SLOCK_ILK,r5 ; Grab the lock value andi. r3,r11,ILK_LOCKED ; Is it locked? ori r11,r6,ILK_LOCKED ; Set interlock bne-- slckspin ; Yeah, wait for it to clear... - stwcx. r11,0,r5 ; Try to seize that there durn lock + stwcx. r11,SLOCK_ILK,r5 ; Try to seize that there durn lock bne-- slcktry ; Couldn't get it... + .globl EXT(slckPatch_isync) +LEXT(slckPatch_isync) isync ; Make sure we don't use a speculativily loaded value blr ; Go on home... @@ -1521,7 +1805,7 @@ slockspin0: mr. r8,r8 ; Is r8 set to zero slockspin1: mtmsr r7 ; Turn off interruptions mftb r8 ; Get timestamp on entry -slcksniff: lwz r3,0(r5) ; Get that lock in here +slcksniff: lwz r3,SLOCK_ILK(r5) ; Get that lock in here andi. r3,r3,ILK_LOCKED ; Is it free yet? beq++ slckretry ; Yeah, try for it again... @@ -1553,27 +1837,26 @@ slckfail: ; We couldn't get the lock ori r3,r3,lo16(slckpanic_str) mr r4,r5 mflr r5 + PROLOG(0) bl EXT(panic) + BREAKPOINT_TRAP ; We die here anyway .data slckpanic_str: - STRINGD "ppc_usimple_lock: simple lock deadlock detection l=0x%08X, pc=0x%08X\n\000" + STRINGD "simple lock (0x%08X) deadlock detection, pc=0x%08X\n\000" .text /* - * unsigned int ppc_usimple_lock_try(simple_lock_t) + * boolean_t lck_spin_try_lock(lck_spin_t *) + * unsigned int ppc_usimple_lock_try(simple_lock_t *) * */ .align 5 + .globl EXT(lck_spin_try_lock) +LEXT(lck_spin_try_lock) .globl EXT(ppc_usimple_lock_try) - LEXT(ppc_usimple_lock_try) -#if CHECKNMI - mflr r12 ; (TEST/DEBUG) - bl EXT(ml_sense_nmi) ; (TEST/DEBUG) - mtlr r12 ; (TEST/DEBUG) -#endif lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable mfmsr r9 ; Get the MSR value ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable @@ -1583,20 +1866,22 @@ LEXT(ppc_usimple_lock_try) mtmsr r7 ; Disable interruptions and thus, preemption mfsprg r6,1 ; Get current activation - lwz r11,0(r3) ; Get the lock + lwz r11,SLOCK_ILK(r3) ; Get the lock andi. r5,r11,ILK_LOCKED ; Check it... bne-- slcktryfail ; Quickly fail... slcktryloop: - lwarx r11,0,r3 ; Ld from addr of arg and reserve + lwarx r11,SLOCK_ILK,r3 ; Ld from addr of arg and reserve andi. r5,r11,ILK_LOCKED ; TEST... ori r5,r6,ILK_LOCKED bne-- slcktryfailX ; branch if taken. Predict free - stwcx. r5,0,r3 ; And SET (if still reserved) + stwcx. r5,SLOCK_ILK,r3 ; And SET (if still reserved) bne-- slcktryloop ; If set failed, loop back + .globl EXT(stlckPatch_isync) +LEXT(stlckPatch_isync) isync lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level @@ -1618,19 +1903,16 @@ slcktryfail: /* - * void ppc_usimple_unlock_rwcmb(simple_lock_t) + * void lck_spin_unlock(lck_spin_t *) + * void ppc_usimple_unlock_rwcmb(simple_lock_t *) * */ .align 5 + .globl EXT(lck_spin_unlock) +LEXT(lck_spin_unlock) .globl EXT(ppc_usimple_unlock_rwcmb) - LEXT(ppc_usimple_unlock_rwcmb) -#if CHECKNMI - mflr r12 ; (TEST/DEBUG) - bl EXT(ml_sense_nmi) ; (TEST/DEBUG) - mtlr r12 ; (TEST/DEBUG) -#endif li r0,0 .globl EXT(sulckPatch_isync) LEXT(sulckPatch_isync) @@ -1638,12 +1920,12 @@ LEXT(sulckPatch_isync) .globl EXT(sulckPatch_eieio) LEXT(sulckPatch_eieio) eieio - stw r0, LOCK_DATA(r3) + stw r0, SLOCK_ILK(r3) b epStart ; Go enable preemption... /* - * void ppc_usimple_unlock_rwmb(simple_lock_t) + * void ppc_usimple_unlock_rwmb(simple_lock_t *) * */ .align 5 @@ -1651,128 +1933,340 @@ LEXT(sulckPatch_eieio) LEXT(ppc_usimple_unlock_rwmb) -#if CHECKNMI - mflr r12 ; (TEST/DEBUG) - bl EXT(ml_sense_nmi) ; (TEST/DEBUG) - mtlr r12 ; (TEST/DEBUG) -#endif li r0,0 sync - stw r0, LOCK_DATA(r3) + stw r0, SLOCK_ILK(r3) b epStart ; Go enable preemption... /* - * void enter_funnel_section(funnel_t *) + * void lck_rw_lock_exclusive(lck_rw_t*) * */ .align 5 - .globl EXT(enter_funnel_section) - -LEXT(enter_funnel_section) - + .globl EXT(lck_rw_lock_exclusive) +LEXT(lck_rw_lock_exclusive) #if !MACH_LDEBUG - lis r10,hi16(EXT(kdebug_enable)) - ori r10,r10,lo16(EXT(kdebug_enable)) - lwz r10,0(r10) - lis r11,hi16(EXT(split_funnel_off)) - ori r11,r11,lo16(EXT(split_funnel_off)) - lwz r11,0(r11) - or. r10,r11,r10 ; Check kdebug_enable or split_funnel_off - bne- L_enter_funnel_section_slow ; If set, call the slow path - mfsprg r6,1 ; Get the current activation - lwz r7,LOCK_FNL_MUTEX(r3) - - lwz r5,0(r7) ; Get lock quickly - mr. r5,r5 ; Locked? - bne-- L_enter_funnel_section_slow ; Yup... - -L_enter_funnel_section_loop: - lwarx r5,0,r7 ; Load the mutex lock - mr. r5,r5 - bne-- L_enter_funnel_section_slowX ; Go to the slow path - stwcx. r6,0,r7 ; Grab the lock - bne-- L_enter_funnel_section_loop ; Loop back if failed - isync ; Stop prefeteching - lwz r6,ACT_THREAD(r6) ; Get the current thread - li r7,TH_FN_OWNED - stw r3,THREAD_FUNNEL_LOCK(r6) ; Set the funnel lock reference - stw r7,THREAD_FUNNEL_STATE(r6) ; Set the funnel state + .globl EXT(lock_write) +LEXT(lock_write) +#endif + lis r7,0xFFFF + ori r7,r7,(WANT_EXCL|WANT_UPGRADE|ILK_LOCKED) +rwleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + and. r8,r5,r7 ; Can we have it? + ori r6,r5,WANT_EXCL ; Mark Exclusive + bne-- rwlespin ; Branch if cannot be held + stwcx. r6,RW_DATA,r3 ; Update lock word + bne-- rwleloop + .globl EXT(rwlePatch_isync) +LEXT(rwlePatch_isync) + isync blr - -L_enter_funnel_section_slowX: +rwlespin: li r4,lgKillResv ; Killing field - stwcx. r4,0,r4 ; Kill reservation + stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwlespin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_lock_exclusive_ext) +rwlespin1: + b EXT(lck_rw_lock_exclusive_gen) -L_enter_funnel_section_slow: +/* + * void lck_rw_lock_shared(lck_rw_t*) + * + */ + .align 5 + .globl EXT(lck_rw_lock_shared) +LEXT(lck_rw_lock_shared) +#if !MACH_LDEBUG + .globl EXT(lock_read) +LEXT(lock_read) #endif - li r4,TRUE - b EXT(thread_funnel_set) +rwlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + andi. r7,r5,WANT_EXCL|WANT_UPGRADE|ILK_LOCKED ; Can we have it? + bne-- rwlsopt ; Branch if cannot be held +rwlsloopres: + addis r6,r5,1 ; Increment read cnt + stwcx. r6,RW_DATA,r3 ; Update lock word + bne-- rwlsloop + .globl EXT(rwlsPatch_isync) +LEXT(rwlsPatch_isync) + isync + blr +rwlsopt: + andi. r7,r5,PRIV_EXCL|ILK_LOCKED ; Can we have it? + bne-- rwlsspin ; Branch if cannot be held + lis r7,0xFFFF ; Get read cnt mask + and. r8,r5,r7 ; Is it shared + bne rwlsloopres ; Branch if can be held +rwlsspin: + li r4,lgKillResv ; Killing field + stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwlsspin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_lock_shared_ext) +rwlsspin1: + b EXT(lck_rw_lock_shared_gen) /* - * void exit_funnel_section(void) + * boolean_t lck_rw_lock_shared_to_exclusive(lck_rw_t*) * */ .align 5 - .globl EXT(exit_funnel_section) + .globl EXT(lck_rw_lock_shared_to_exclusive) +LEXT(lck_rw_lock_shared_to_exclusive) +#if !MACH_LDEBUG + .globl EXT(lock_read_to_write) +LEXT(lock_read_to_write) +#endif +rwlseloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + addis r6,r5,0xFFFF ; Decrement read cnt + lis r8,0xFFFF ; Get read count mask + ori r8,r8,WANT_UPGRADE|ILK_LOCKED ; Include Interlock and upgrade flags + and. r7,r6,r8 ; Can we have it? + ori r9,r6,WANT_UPGRADE ; Mark Exclusive + bne-- rwlsespin ; Branch if cannot be held + stwcx. r9,RW_DATA,r3 ; Update lock word + bne-- rwlseloop + .globl EXT(rwlsePatch_isync) +LEXT(rwlsePatch_isync) + isync + li r3,1 ; Succeed, return TRUE... + blr +rwlsespin: + li r4,lgKillResv ; Killing field + stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwlsespin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_lock_shared_to_exclusive_ext) +rwlsespin1: + b EXT(lck_rw_lock_shared_to_exclusive_gen) -LEXT(exit_funnel_section) - mfsprg r6,1 ; Get the current activation - lwz r6,ACT_THREAD(r6) ; Get the current thread - lwz r3,THREAD_FUNNEL_LOCK(r6) ; Get the funnel lock - mr. r3,r3 ; Check on funnel held - beq- L_exit_funnel_section_ret ; + +/* + * void lck_rw_lock_exclusive_to_shared(lck_rw_t*) + * + */ + .align 5 + .globl EXT(lck_rw_lock_exclusive_to_shared) +LEXT(lck_rw_lock_exclusive_to_shared) #if !MACH_LDEBUG - lis r10,hi16(EXT(kdebug_enable)) - ori r10,r10,lo16(EXT(kdebug_enable)) - lwz r10,0(r10) - mr. r10,r10 - bne- L_exit_funnel_section_slow ; If set, call the slow path - lwz r7,LOCK_FNL_MUTEX(r3) ; Get the funnel mutex lock - .globl EXT(retfsectPatch_isync) -LEXT(retfsectPatch_isync) + .globl EXT(lock_write_to_read) +LEXT(lock_write_to_read) +#endif + .globl EXT(rwlesPatch_isync) +LEXT(rwlesPatch_isync) isync - .globl EXT(retfsectPatch_eieio) -LEXT(retfsectPatch_eieio) + .globl EXT(rwlesPatch_eieio) +LEXT(rwlesPatch_eieio) eieio +rwlesloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + andi. r7,r5,ILK_LOCKED ; Test interlock flag + bne-- rwlesspin ; Branch if interlocked + lis r6,1 ; Get 1 for read count + andi. r10,r5,WANT_UPGRADE ; Is it held with upgrade + li r9,WANT_UPGRADE|WAIT_FLAG ; Get upgrade and wait flags mask + bne rwlesexcl1 ; Skip if held with upgrade + li r9,WANT_EXCL|WAIT_FLAG ; Get exclusive and wait flags mask +rwlesexcl1: + andc r7,r5,r9 ; Marked free + rlwimi r6,r7,0,16,31 ; Set shared cnt to one + stwcx. r6,RW_DATA,r3 ; Update lock word + bne-- rwlesloop + andi. r7,r5,WAIT_FLAG ; Test wait flag + beqlr++ ; Return of no waiters + addi r3,r3,RW_EVENT ; Get lock event address + b EXT(thread_wakeup) ; wakeup waiters +rwlesspin: + li r4,lgKillResv ; Killing field + stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwlesspin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_lock_exclusive_to_shared_ext) +rwlesspin1: + b EXT(lck_rw_lock_exclusive_to_shared_gen) + - lwz r5,0(r7) ; Get lock - rlwinm. r4,r5,0,30,31 ; Quick check for bail if pending waiter or interlock set - bne-- L_exit_funnel_section_slow ; No can get... - -L_exit_funnel_section_loop: - lwarx r5,0,r7 - rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set - li r5,0 ; Clear the mutexlock - bne-- L_exit_funnel_section_slowX - stwcx. r5,0,r7 ; Release the funnel mutexlock - bne-- L_exit_funnel_section_loop - li r7,0 - stw r7,THREAD_FUNNEL_STATE(r6) ; Clear the funnel state - stw r7,THREAD_FUNNEL_LOCK(r6) ; Clear the funnel lock reference - blr ; Return - -L_exit_funnel_section_slowX: + +/* + * boolean_t lck_rw_try_lock_exclusive(lck_rw_t*) + * + */ + .align 5 + .globl EXT(lck_rw_try_lock_exclusive) +LEXT(lck_rw_try_lock_exclusive) + lis r10,0xFFFF ; Load read count mask + ori r10,r10,WANT_EXCL|WANT_UPGRADE ; Include exclusive and upgrade flags +rwtleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + andi. r7,r5,ILK_LOCKED ; Test interlock flag + bne-- rwtlespin ; Branch if interlocked + and. r7,r5,r10 ; Can we have it + ori r6,r5,WANT_EXCL ; Mark Exclusive + bne-- rwtlefail ; + stwcx. r6,RW_DATA,r3 ; Update lock word + bne-- rwtleloop + .globl EXT(rwtlePatch_isync) +LEXT(rwtlePatch_isync) + isync + li r3,1 ; Return TRUE + blr +rwtlefail: + li r4,lgKillResv ; Killing field + stwcx. r4,0,r4 ; Kill it + li r3,0 ; Return FALSE + blr +rwtlespin: li r4,lgKillResv ; Killing field stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwtlespin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_try_lock_exclusive_ext) +rwtlespin1: + b EXT(lck_rw_try_lock_exclusive_gen) -L_exit_funnel_section_slow: -#endif - li r4,FALSE - b EXT(thread_funnel_set) -L_exit_funnel_section_ret: + +/* + * boolean_t lck_rw_try_lock_shared(lck_rw_t*) + * + */ + .align 5 + .globl EXT(lck_rw_try_lock_shared) +LEXT(lck_rw_try_lock_shared) +rwtlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + andi. r7,r5,ILK_LOCKED ; Test interlock flag + bne-- rwtlsspin ; Branch if interlocked + andi. r7,r5,WANT_EXCL|WANT_UPGRADE ; So, can we have it? + bne-- rwtlsopt ; Branch if held exclusive +rwtlsloopres: + addis r6,r5,1 ; Increment read cnt + stwcx. r6,RW_DATA,r3 ; Update lock word + bne-- rwtlsloop + .globl EXT(rwtlsPatch_isync) +LEXT(rwtlsPatch_isync) + isync + li r3,1 ; Return TRUE + blr +rwtlsopt: + andi. r7,r5,PRIV_EXCL ; Can we have it? + bne-- rwtlsfail ; Branch if cannot be held + lis r7,0xFFFF ; Get read cnt mask + and. r8,r5,r7 ; Is it shared + bne rwtlsloopres ; Branch if can be held +rwtlsfail: + li r3,0 ; Return FALSE blr +rwtlsspin: + li r4,lgKillResv ; Killing field + stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwtlsspin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_try_lock_shared_ext) +rwtlsspin1: + b EXT(lck_rw_try_lock_shared_gen) -; -; This is bring up code -; - .align 5 - .globl EXT(condStop) -LEXT(condStop) -XcondStop: cmplw r3,r4 ; Check if these are equal - beq-- XcondStop ; Stop here until they are different - blr ; Return. +/* + * lck_rw_type_t lck_rw_done(lck_rw_t*) + * + */ + .align 5 + .globl EXT(lck_rw_done) +LEXT(lck_rw_done) +#if !MACH_LDEBUG + .globl EXT(lock_done) +LEXT(lock_done) +#endif + .globl EXT(rwldPatch_isync) +LEXT(rwldPatch_isync) + isync + .globl EXT(rwldPatch_eieio) +LEXT(rwldPatch_eieio) + eieio + li r10,WAIT_FLAG ; Get wait flag + lis r7,0xFFFF ; Get read cnt mask + mr r12,r3 ; Save lock addr +rwldloop: lwarx r5,RW_DATA,r3 ; Grab the lock value + andi. r8,r5,ILK_LOCKED ; Test interlock flag + bne-- rwldspin ; Branch if interlocked + and. r8,r5,r7 ; Is it shared + cmpi cr1,r8,0 ; Is it shared + beq cr1,rwldexcl ; No, check exclusive + li r11,RW_SHARED ; Set return value + addis r6,r5,0xFFFF ; Decrement read count + and. r8,r6,r7 ; Is it still shared + li r8,0 ; Assume no wakeup + bne rwldshared1 ; Skip if still held shared + and r8,r6,r10 ; Extract wait flag + andc r6,r6,r10 ; Clear wait flag +rwldshared1: + b rwldstore +rwldexcl: + li r11,RW_EXCL ; Set return value + li r9,WANT_UPGRADE ; Get upgrade flag + and. r6,r5,r9 ; Is it held with upgrade + li r9,WANT_UPGRADE|WAIT_FLAG ; Mask upgrade abd wait flags + bne rwldexcl1 ; Skip if held with upgrade + li r9,WANT_EXCL|WAIT_FLAG ; Mask exclusive and wait flags +rwldexcl1: + andc r6,r5,r9 ; Marked free + and r8,r5,r10 ; Null if no waiter +rwldstore: + stwcx. r6,RW_DATA,r3 ; Update lock word + bne-- rwldloop + mr. r8,r8 ; wakeup needed? + mr r3,r11 ; Return lock held type + beqlr++ + mr r3,r12 ; Restore lock address + PROLOG(0) + addi r3,r3,RW_EVENT ; Get lock event address + bl EXT(thread_wakeup) ; wakeup threads + lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1) + mtcr r2 + EPILOG + li r3,RW_SHARED ; Assume lock type shared + bne cr1,rwldret ; Branch if was held exclusive + li r3,RW_EXCL ; Return lock type exclusive +rwldret: + blr +rwldspin: + li r4,lgKillResv ; Killing field + stwcx. r4,0,r4 ; Kill it + cmpli cr0,r5,RW_IND ; Is it a lock indirect + bne-- rwldspin1 ; No, go handle contention + mr r4,r3 ; pass lock pointer + lwz r3,RW_PTR(r3) ; load lock ext pointer + b EXT(lck_rw_done_ext) +rwldspin1: + b EXT(lck_rw_done_gen) +/* + * void lck_rw_ilk_lock(lck_rw_t *lock) + */ + .globl EXT(lck_rw_ilk_lock) +LEXT(lck_rw_ilk_lock) + crclr hwtimeout ; no timeout option + li r4,0 ; request default timeout value + li r12,ILK_LOCKED ; Load bit mask + b lckcomm ; Join on up... + +/* + * void lck_rw_ilk_unlock(lck_rw_t *lock) + */ + .globl EXT(lck_rw_ilk_unlock) +LEXT(lck_rw_ilk_unlock) + li r4,1 + b EXT(hw_unlock_bit)