xnu-344.49.tar.gz

[apple/xnu.git] / osfmk / ppc / hw_vm.s

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
#include <assym.s>
#include <debug.h>
#include <cpus.h>
#include <db_machine_commands.h>
#include <mach_rt.h>
	
#include <mach_debug.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <ppc/exception.h>
#include <ppc/Performance.h>
#include <ppc/exception.h>
#include <ppc/pmap_internals.h>
#include <mach/ppc/vm_param.h>
#define PERFTIMES 0
	
			.text

/*
 *
 *			Random notes and musings...
 *
 *			Access to mappings via the PTEG hash must be done with the list locked.
 *			Access via the physical entries is controlled by the physent lock.
 *			Access to mappings is controlled by the PTEG lock once they are queued.
 *			If they are not on the list, they don't really exist, so
 *			only one processor at a time can find them, so no access control is needed. 
 *
 *			The second half of the PTE is kept in the physical entry.  It is done this
 *			way, because there may be multiple mappings that refer to the same physical
 *			page (i.e., address aliases or synonymns).  We must do it this way, because
 *			maintenance of the reference and change bits becomes nightmarish if each mapping
 *			has its own. One side effect of this, and not necessarily a bad one, is that
 *			all mappings for a single page can have a single WIMG, protection state, and RC bits.
 *			The only "bad" thing, is the reference bit.  With a single copy, we can not get
 *			a completely accurate working set calculation, i.e., we can't tell which mapping was
 *			used to reference the page, all we can tell is that the physical page was 
 *			referenced.
 *
 *			The master copys of the reference and change bits are kept in the phys_entry.
 *			Other than the reference and change bits, changes to the phys_entry are not
 *			allowed if it has any mappings.  The master reference and change bits must be
 *			changed via atomic update.
 *
 *			Invalidating a PTE merges the RC bits into the phys_entry.
 *
 *			Before checking the reference and/or bits, ALL mappings to the physical page are
 *			invalidated.
 *			
 *			PTEs are never explicitly validated, they are always faulted in.  They are also
 *			not visible outside of the hw_vm modules.  Complete seperation of church and state.
 *
 *			Removal of a mapping is invalidates its PTE.
 *
 *			So, how do we deal with mappings to I/O space? We don't have a physent for it.
 *			Within the mapping is a copy of the second half of the PTE.  This is used
 *			ONLY when there is no physical entry.  It is swapped into the PTE whenever
 *			it is built.  There is no need to swap it back out, because RC is not
 *			maintained for these mappings.
 *
 *			So, I'm starting to get concerned about the number of lwarx/stcwx loops in
 *			this.  Satisfying a mapped address with no stealing requires one lock.  If we 
 *			steal an entry, there's two locks and an atomic update.  Invalidation of an entry
 *			takes one lock and, if there is a PTE, another lock and an atomic update.  Other 
 *			operations are multiples (per mapping) of the above.  Maybe we should look for
 *			an alternative.  So far, I haven't found one, but I haven't looked hard.
 */


/*			hw_add_map(struct mapping *mp, space_t space, vm_offset_t va) - Adds a mapping
 *
 *			Adds a mapping to the PTEG hash list.
 *
 *			Interrupts must be disabled before calling.
 *
 *			Using the space and the virtual address, we hash into the hash table
 *			and get a lock on the PTEG hash chain.  Then we chain the 
 *			mapping to the front of the list.
 *
 */

			.align	5
			.globl	EXT(hw_add_map)

LEXT(hw_add_map)

#if PERFTIMES && DEBUG
			mr		r7,r3
			mflr	r11
			li		r3,20
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r7
			mtlr	r11
#endif

			mfmsr	r0							/* Get the MSR */
			eqv		r6,r6,r6					/* Fill the bottom with foxes */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r11,r4,6,6,25				/* Position the space for the VSID */
			mfspr	r10,sdr1					/* Get hash table base and size */
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwimi	r11,r5,30,2,5				/* Insert the segment no. to make a VSID */
			mfsprg	r12,2						; Get feature flags
			rlwimi	r6,r10,16,0,15				/* Make table size -1 out of mask */
			rlwinm	r7,r5,26,10,25				/* Isolate the page index */
			or		r8,r10,r6					/* Point to the last byte in table */
			rlwinm	r9,r5,4,0,3					; Move nybble 1 up to 0
			xor		r7,r7,r11					/* Get primary hash */
			mtcrf	0x04,r12					; Set the features			
			andi.	r12,r0,0x7FCF				/* Disable translation and interruptions */
			rlwinm	r11,r11,1,1,24				/* Position VSID for pte ID */
			addi	r8,r8,1						/* Point to the PTEG Control Area */
			xor		r9,r9,r5					; Splooch vaddr nybble 0 and 1 together
			and		r7,r7,r6					/* Wrap the hash */
			rlwimi	r11,r5,10,26,31				/* Move API into pte ID */
			rlwinm	r9,r9,6,27,29				; Get splooched bits in place
			add		r8,r8,r7					/* Point to our PCA entry */
			rlwinm	r10,r4,2,27,29				; Get low 3 bits of the VSID for look-aside hash
			
			bt		pfNoMSRirb,hamNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hamNoMSRx
			
hamNoMSR:	mr		r4,r0						; Save R0
			mr		r2,r3						; Save
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r0,r4						; Restore
			mr		r3,r2						; Restore
hamNoMSRx:

			la		r4,PCAhash(r8)				/* Point to the mapping hash area */
			xor		r9,r9,r10					; Finish splooching nybble 0, 1, and the low bits of the VSID
			isync								/* Get rid of anything prefetched before we ref storage */
/*
 *			We've now got the address of our PCA, the hash chain anchor, our API subhash,
 *			and word 0 of the PTE (the virtual part). 
 *
 *			Now, we just lock the PCA.		
 */
			
			li		r12,1						/* Get the locked value */
			dcbt	0,r4						/* We'll need the hash area in a sec, so get it */
			add		r4,r4,r9					/* Point to the right mapping hash slot */
			
ptegLckx:	lwarx	r10,0,r8					/* Get the PTEG lock */
			mr.		r10,r10						/* Is it locked? */
			bne-	ptegLckwx					/* Yeah... */
			stwcx.	r12,0,r8					/* Take take it */
			bne-	ptegLckx					/* Someone else was trying, try again... */
			b		ptegSXgx					/* All done... */

			.align	4
			
ptegLckwx:	mr.		r10,r10						/* Check if it's already held */
			beq+	ptegLckx					/* It's clear... */
			lwz		r10,0(r8)					/* Get lock word again... */
			b		ptegLckwx					/* Wait... */
			
			.align	4
			
ptegSXgx:	isync								/* Make sure we haven't used anything yet */

			lwz		r7,0(r4)					/* Pick up the anchor of hash list */
			stw		r3,0(r4)					/* Save the new head */
			stw		r7,mmhashnext(r3)			/* Chain in the old head */
			
			stw		r4,mmPTEhash(r3)			/* Point to the head of the hash list */
			
			sync								/* Make sure the chain is updated */
			stw		r10,0(r8)					/* Unlock the hash list */
			mtmsr	r0							/* Restore translation and interruptions */
			isync								/* Toss anything done with DAT off */
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,21
			bl		EXT(dbgLog2)				; end of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									/* Leave... */


/*			mp=hw_lock_phys_vir(space, va) - Finds and locks a physical entry by vaddr.
 *
 *			Returns the mapping with the associated physent locked if found, or a
 *			zero and no lock if not.  It we timed out trying to get a the lock on
 *			the physical entry, we retun a 1.  A physical entry can never be on an
 *			odd boundary, so we can distinguish between a mapping and a timeout code.
 *
 *			Interrupts must be disabled before calling.
 *
 *			Using the space and the virtual address, we hash into the hash table
 *			and get a lock on the PTEG hash chain.  Then we search the chain for the
 *			mapping for our virtual address.  From there, we extract the pointer to
 *			the physical entry.
 *
 *			Next comes a bit of monkey business.  we need to get a lock on the physical
 *			entry.  But, according to our rules, we can't get it after we've gotten the
 *			PTEG hash lock, we could deadlock if we do.  So, we need to release the
 *			hash lock.  The problem is, though, that as soon as we release it, some 
 *			other yahoo may remove our mapping between the time that we release the
 *			hash lock and obtain the phys entry lock.  So, we can't count on the 
 *			mapping once we release the lock.  Instead, after we lock the phys entry,
 *			we search the mapping list (phys_link) for our translation.  If we don't find it,
 *			we unlock the phys entry, bail out, and return a 0 for the mapping address.  If we 
 *			did find it, we keep the lock and return the address of the mapping block.
 *
 *			What happens when a mapping is found, but there is no physical entry?
 *			This is what happens when there is I/O area mapped.  It one of these mappings
 *			is found, the mapping is returned, as is usual for this call, but we don't
 *			try to lock anything.  There could possibly be some problems here if another
 *			processor releases the mapping while we still alre using it.  Hope this 
 *			ain't gonna happen.
 *
 *			Taaa-dahhh!  Easy as pie, huh?
 *
 *			So, we have a few hacks hacks for running translate off in here. 
 *			First, when we call the lock routine, we have carnel knowlege of the registers is uses. 
 *			That way, we don't need a stack frame, which we can't have 'cause the stack is in
 *			virtual storage.  But wait, as if that's not enough...  We need one more register.  So, 
 *			we cram the LR into the CTR and return from there.
 *
 */
			.align	5
			.globl	EXT(hw_lock_phys_vir)

LEXT(hw_lock_phys_vir)

#if PERFTIMES && DEBUG
			mflr	r11
			mr		r5,r3
			li		r3,22
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r5
			mtlr	r11
#endif
			mfmsr	r12							/* Get the MSR */
			eqv		r6,r6,r6					/* Fill the bottom with foxes */
			mfsprg	r9,2						; Get feature flags 
			rlwinm	r11,r3,6,6,25				/* Position the space for the VSID */
			rlwinm	r12,r12,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			mfspr	r5,sdr1						/* Get hash table base and size */
			rlwimi	r11,r4,30,2,5				/* Insert the segment no. to make a VSID */
			mtcrf	0x04,r9						; Set the features			
			rlwinm	r12,r12,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwimi	r6,r5,16,0,15				/* Make table size -1 out of mask */
			andi.	r0,r12,0x7FCF				/* Disable translation and interruptions */
			rlwinm	r9,r4,4,0,3					; Move nybble 1 up to 0
			rlwinm	r7,r4,26,10,25				/* Isolate the page index */
			or		r8,r5,r6					/* Point to the last byte in table */
			xor		r7,r7,r11					/* Get primary hash */
			rlwinm	r11,r11,1,1,24				/* Position VSID for pte ID */
			addi	r8,r8,1						/* Point to the PTEG Control Area */
			xor		r9,r9,r4					; Splooch vaddr nybble 0 and 1 together
			and		r7,r7,r6					/* Wrap the hash */
			rlwimi	r11,r4,10,26,31				/* Move API into pte ID */
			rlwinm	r9,r9,6,27,29				; Get splooched bits in place
			add		r8,r8,r7					/* Point to our PCA entry */
			rlwinm	r10,r3,2,27,29				; Get low 3 bits of the VSID for look-aside hash

			bt		pfNoMSRirb,hlpNoMSR			; No MSR...

			mtmsr	r0							; Translation and all off
			isync								; Toss prefetch
			b		hlpNoMSRx
			
hlpNoMSR:	mr		r3,r0						; Get the new MSR
			li		r0,loadMSR					; Get the MSR setter SC
			sc									; Set it
hlpNoMSRx:

			la		r3,PCAhash(r8)				/* Point to the mapping hash area */
			xor		r9,r9,r10					; Finish splooching nybble 0, 1, and the low bits of the VSID
			isync								/* Make sure translation is off before we ref storage */

/*
 *			We've now got the address of our PCA, the hash chain anchor, our API subhash,
 *			and word 0 of the PTE (the virtual part). 
 *
 *			Now, we just lock the PCA and find our mapping, if it exists.				
 */
			
			dcbt	0,r3						/* We'll need the hash area in a sec, so get it */
			add		r3,r3,r9					/* Point to the right mapping hash slot */
			
ptegLcka:	lwarx	r10,0,r8					/* Get the PTEG lock */
			li		r5,1						/* Get the locked value */
			mr.		r10,r10						/* Is it locked? */
			bne-	ptegLckwa					/* Yeah... */
			stwcx.	r5,0,r8						/* Take take it */
			bne-	ptegLcka					/* Someone else was trying, try again... */
			b		ptegSXga					/* All done... */
			
			.align	4

ptegLckwa:	mr.		r10,r10						/* Check if it's already held */
			beq+	ptegLcka					/* It's clear... */
			lwz		r10,0(r8)					/* Get lock word again... */
			b		ptegLckwa					/* Wait... */
			
			.align	4

ptegSXga:	isync								/* Make sure we haven't used anything yet */

			mflr	r0							/* Get the LR */
			lwz		r9,0(r3)					/* Pick up the first mapping block */
			mtctr	r0							/* Stuff it into the CTR */
			
findmapa:	

			mr.		r3,r9						/* Did we hit the end? */
			bne+	chkmapa						/* Nope... */
			
			stw		r3,0(r8)					/* Unlock the PTEG lock
												   Note: we never saved anything while we 
												   had the lock, so we don't need a sync 
												   before we unlock it */

vbail:		mtmsr	r12							/* Restore translation and interruptions */
			isync								/* Make sure translation is cool */
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,23
			bl		EXT(dbgLog2)				; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			bctr								/* Return in abject failure... */
			
			.align	4

chkmapa:	lwz		r10,mmPTEv(r3)				/* Pick up our virtual ID */
			lwz		r9,mmhashnext(r3)			/* Pick up next mapping block */
			cmplw	r10,r11						/* Have we found ourself? */
			bne-	findmapa					/* Nope, still wandering... */
			
			lwz		r9,mmphysent(r3)			/* Get our physical entry pointer */
			li		r5,0						/* Clear this out */
			mr.		r9,r9						/* Is there, like, a physical entry? */
			stw		r5,0(r8)					/* Unlock the PTEG lock
												   Note: we never saved anything while we 
												   had the lock, so we don't need a sync 
												   before we unlock it */
												   
			beq-	vbail						/* If there is no physical entry, it's time
												   to leave... */
												   
/*			Here we want to call hw_lock_bit.  We don't want to use the stack, 'cause it's
 *			in virtual storage, and we're in real.  So, we've carefully looked at the code
 *			in hw_lock_bit (and unlock) and cleverly don't use any of the registers that it uses.
 *			Be very, very aware of how you change this code.  By the way, it uses:
 *			R0, R6, R7, R8, and R9.  R3, R4, and R5 contain parameters
 *			Unfortunatly, we need to stash R9 still. So... Since we know we will not be interrupted
 *			('cause we turned off interruptions and translation is off) we will use SPRG3...
 */
 
			lwz		r10,mmPTEhash(r3)			/* Save the head of the hash-alike chain.  We need it to find ourselves later */
			lis		r5,HIGH_ADDR(EXT(LockTimeOut))	/* Get address of timeout value */
			la		r3,pephyslink(r9)			/* Point to the lock word */
			ori		r5,r5,LOW_ADDR(EXT(LockTimeOut))	/* Get second half of address */
			li		r4,PHYS_LOCK				/* Get the lock bit value */
			lwz		r5,0(r5)					/* Pick up the timeout value */
			mtsprg	3,r9						/* Save R9 in SPRG3 */
			
			bl		EXT(hw_lock_bit)			/* Go do the lock */
			
			mfsprg	r9,3						/* Restore pointer to the phys_entry */		
			mr.		r3,r3						/* Did we timeout? */
			lwz		r4,pephyslink(r9)			/* Pick up first mapping block */		
			beq-	penterr						/* Bad deal, we timed out... */

			rlwinm	r4,r4,0,0,26				; Clear out the flags from first link
			
findmapb:	mr.		r3,r4						/* Did we hit the end? */
			bne+	chkmapb						/* Nope... */
			
			la		r3,pephyslink(r9)			/* Point to where the lock is */						
			li		r4,PHYS_LOCK				/* Get the lock bit value */
			bl		EXT(hw_unlock_bit)			/* Go unlock the physentry */

			li		r3,0						/* Say we failed */			
			b		vbail						/* Return in abject failure... */
			
penterr:	li		r3,1						/* Set timeout */
			b		vbail						/* Return in abject failure... */
					
			.align	5

chkmapb:	lwz		r6,mmPTEv(r3)				/* Pick up our virtual ID */
			lwz		r4,mmnext(r3)				/* Pick up next mapping block */
			cmplw	r6,r11						/* Have we found ourself? */
			lwz		r5,mmPTEhash(r3)			/* Get the start of our hash chain */
			bne-	findmapb					/* Nope, still wandering... */
			cmplw	r5,r10						/* On the same hash chain? */
			bne-	findmapb					/* Nope, keep looking... */

			b		vbail						/* Return in glorious triumph... */


/*
 *			hw_rem_map(mapping) - remove a mapping from the system.
 *
 *			Upon entry, R3 contains a pointer to a mapping block and the associated
 *			physical entry is locked if there is one.
 *
 *			If the mapping entry indicates that there is a PTE entry, we invalidate
 *			if and merge the reference and change information into the phys_entry.
 *
 *			Next, we remove the mapping from the phys_ent and the PTEG hash list.
 *
 *			Unlock any locks that are left, and exit.
 *
 *			Note that this must be done with both interruptions off and VM off
 *	
 *			Note that this code depends upon the VSID being of the format 00SXXXXX
 *			where S is the segment number.
 *
 *			  
 */

			.align	5
			.globl	EXT(hw_rem_map)

LEXT(hw_rem_map)
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,24
			bl		EXT(dbgLog2)				; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
 			mfsprg	r9,2						; Get feature flags 
			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */

			bt		pfNoMSRirb,lmvNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		lmvNoMSRx
			
lmvNoMSR:	
			mr		r6,r0
			mr		r4,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r4
			mr		r0,r6

lmvNoMSRx:

		
			lwz		r6,mmPTEhash(r3)			/* Get pointer to hash list anchor */
			lwz		r5,mmPTEv(r3)				/* Get the VSID */
			dcbt	0,r6						/* We'll need that chain in a bit */

			rlwinm	r7,r6,0,0,25				/* Round hash list down to PCA boundary */
			li		r12,1						/* Get the locked value */
			subi	r6,r6,mmhashnext			/* Make the anchor look like an entry */

ptegLck1:	lwarx	r10,0,r7					/* Get the PTEG lock */
			mr.		r10,r10						/* Is it locked? */
			bne-	ptegLckw1					/* Yeah... */
			stwcx.	r12,0,r7					/* Try to take it */
			bne-	ptegLck1					/* Someone else was trying, try again... */
			b		ptegSXg1					/* All done... */
			
			.align	4

ptegLckw1:	mr.		r10,r10						/* Check if it's already held */
			beq+	ptegLck1					/* It's clear... */
			lwz		r10,0(r7)					/* Get lock word again... */
			b		ptegLckw1					/* Wait... */
			
			.align	4

ptegSXg1:	isync								/* Make sure we haven't used anything yet */

			lwz		r12,mmhashnext(r3)			/* Prime with our forward pointer */
 			lwz		r4,mmPTEent(r3)				/* Get the pointer to the PTE now that the lock's set */

srchmaps:	mr.		r10,r6						/* Save the previous entry */
			bne+	mapok						/* No error... */
			
			lis		r0,HIGH_ADDR(Choke)			/* We have a kernel choke!!! */
			ori		r0,r0,LOW_ADDR(Choke)		
			sc									/* Firmware Heimlich manuever */

			.align	4			

mapok:		lwz		r6,mmhashnext(r6)			/* Look at the next one */
			cmplwi	cr5,r4,0					/* Is there a PTE? */
			cmplw	r6,r3						/* Have we found ourselves? */
			bne+	srchmaps					/* Nope, get your head together... */
			
			stw		r12,mmhashnext(r10)			/* Remove us from the queue */
			rlwinm	r9,r5,1,0,3					/* Move in the segment */
			rlwinm	r8,r4,6,4,19				/* Line PTEG disp up to a page */
			rlwinm	r11,r5,5,4,19				/* Line up the VSID */
			lwz		r10,mmphysent(r3)			/* Point to the physical entry */
		
			beq+	cr5,nopte					/* There's no PTE to invalidate... */
			
			xor		r8,r8,r11					/* Back hash to virt index */
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))	/* Get the TLBIE lock */
			rlwimi	r9,r5,22,4,9				/* Move in the API */
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	/* Grab up the bottom part */
			mfspr	r11,pvr						/* Find out what kind of machine we are */
			rlwimi	r9,r8,0,10,19				/* Create the virtual address */
			rlwinm	r11,r11,16,16,31			/* Isolate CPU type */

			stw		r5,0(r4)					/* Make the PTE invalid */		

			cmplwi	cr1,r11,3					/* Is this a 603? */
			sync								/* Make sure the invalid is stored */
						
tlbhang1:	lwarx	r5,0,r12					/* Get the TLBIE lock */
			rlwinm	r11,r4,29,29,31				/* Get the bit position of entry */
			mr.		r5,r5						/* Is it locked? */
			lis		r6,0x8000					/* Start up a bit mask */
			li		r5,1						/* Get our lock word */
			bne-	tlbhang1					/* It's locked, go wait... */
			stwcx.	r5,0,r12					/* Try to get it */
			bne-	tlbhang1					/* We was beat... */
			
			srw		r6,r6,r11					/* Make a "free slot" mask */
			lwz		r5,PCAallo(r7)				/* Get the allocation control bits */
			rlwinm	r11,r6,24,8,15				/* Make the autogen bit to turn off */
			or		r5,r5,r6					/* turn on the free bit */
			rlwimi	r11,r11,24,16,23			/* Get lock bit mask to turn it off */
			
			andc	r5,r5,r11					/* Turn off the lock and autogen bits in allocation flags */
			li		r11,0						/* Lock clear value */

			tlbie	r9							/* Invalidate it everywhere */

			
			beq-	cr1,its603a					/* It's a 603, skip the tlbsync... */
			
			eieio								/* Make sure that the tlbie happens first */
			tlbsync								/* wait for everyone to catch up */
			isync								
			
its603a:	sync								/* Make sure of it all */
			stw		r11,0(r12)					/* Clear the tlbie lock */
			eieio								/* Make sure those RC bit are loaded */
			stw		r5,PCAallo(r7)				/* Show that the slot is free */
			stw		r11,mmPTEent(r3)			/* Clear the pointer to the PTE */

nopte:		mr.		r10,r10						/* See if there is a physical entry */
			la		r9,pephyslink(r10)			/* Point to the physical mapping chain */
			beq-	nophys						/* No physical entry, we're done... */
			beq-	cr5,nadamrg					/* No PTE to merge... */

			lwz		r6,4(r4)					/* Get the latest reference and change bits */
			la		r12,pepte1(r10)				/* Point right at the master copy */
			rlwinm	r6,r6,0,23,24				/* Extract just the RC bits */

mrgrc:		lwarx	r8,0,r12					/* Get the master copy */
			or		r8,r8,r6					/* Merge in latest RC */
			stwcx.	r8,0,r12					/* Save it back */
			bne-	mrgrc						/* If it changed, try again... */
			
nadamrg:	li		r11,0						/* Clear this out */
			lwz		r12,mmnext(r3)				/* Prime with our next */

			sync								; Make sure all is saved

			stw		r11,0(r7)					/* Unlock the hash chain now so we don't
												   lock out another processor during 
												   our next little search */		
			
srchpmap:	mr.		r10,r9						/* Save the previous entry */
			bne+	mapok1						/* No error... */
			
			lis		r0,HIGH_ADDR(Choke)			/* We have a kernel choke!!! */
			ori		r0,r0,LOW_ADDR(Choke)			
			sc									/* Firmware Heimlich maneuver */
			
			.align	4

mapok1:		lwz		r9,mmnext(r9)				/* Look at the next one */
			rlwinm	r8,r9,0,27,31				; Save the flags (including the lock)
			rlwinm	r9,r9,0,0,26				; Clear out the flags from first link
			cmplw	r9,r3						/* Have we found ourselves? */
			bne+	srchpmap					/* Nope, get your head together... */
			
			rlwimi	r12,r8,0,27,31				; Insert the lock and flags */
			stw		r12,mmnext(r10)				/* Remove us from the queue */
			
			mtmsr	r0							/* Interrupts and translation back on */
			isync
#if PERFTIMES && DEBUG
			mflr	r11
			li		r3,25
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mtlr	r11
#endif
			blr									/* Return... */

			.align	4

nophys:		li		r4,0						/* Make sure this is 0 */
			sync								/* Make sure that chain is updated */
			stw		r4,0(r7)					/* Unlock the hash chain */
			mtmsr	r0							/* Interrupts and translation back on */
			isync
#if PERFTIMES && DEBUG
			mflr	r11
			li		r3,25
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mtlr	r11
#endif
			blr									/* Return... */


/*
 *			hw_prot(physent, prot) - Change the protection of a physical page
 *
 *			Upon entry, R3 contains a pointer to a physical entry which is locked.
 *			R4 contains the PPC protection bits.
 *
 *			The first thing we do is to slam the new protection into the phys entry.
 *			Then we scan the mappings and process each one.
 *
 *			Acquire the lock on the PTEG hash list for the mapping being processed.
 *
 *			If the current mapping has a PTE entry, we invalidate
 *			it and merge the reference and change information into the phys_entry.
 *
 *			Next, slam the protection bits into the entry and unlock the hash list.
 *
 *			Note that this must be done with both interruptions off and VM off
 *	
 *			  
 */

			.align	5
			.globl	EXT(hw_prot)

LEXT(hw_prot)
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r7,r3
//			lwz		r5,4(r3)
			li		r5,0x1111
			li		r3,26
			bl		EXT(dbgLog2)				; Start of hw_add_map
			mr		r3,r7
			mtlr	r11
#endif
 			mfsprg	r9,2						; Get feature flags 
			mfmsr	r0							/* Save the MSR  */
			li		r5,pepte1					/* Get displacement to the second word of master pte */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */

			bt		pfNoMSRirb,hpNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hpNoMSRx
			
hpNoMSR:	
			mr		r10,r0
			mr		r7,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r0,r10
			mr		r3,r7
hpNoMSRx:


			
			lwz		r10,pephyslink(r3)			/* Get the first mapping block */
			rlwinm	r10,r10,0,0,26				; Clear out the flags from first link

/*
 *			Note that we need to to do the interlocked update here because another processor
 *			can be updating the reference and change bits even though the physical entry
 *			is locked.  All modifications to the PTE portion of the physical entry must be
 *			done via interlocked update.
 */

protcng:	lwarx	r8,r5,r3					/* Get the master copy */
			rlwimi	r8,r4,0,30,31				/* Move in the protection bits */
			stwcx.	r8,r5,r3					/* Save it back */
			bne-	protcng						/* If it changed, try again... */



protnext:	mr.		r10,r10						/* Are there any more mappings? */
			beq-	protdone					/* Naw... */
			
			lwz		r7,mmPTEhash(r10)			/* Get pointer to hash list anchor */
			lwz		r5,mmPTEv(r10)				/* Get the virtual address */
			rlwinm	r7,r7,0,0,25				/* Round hash list down to PCA boundary */

			li		r12,1						/* Get the locked value */

protLck1:	lwarx	r11,0,r7					/* Get the PTEG lock */
			mr.		r11,r11						/* Is it locked? */
			bne-	protLckw1					/* Yeah... */
			stwcx.	r12,0,r7					/* Try to take it */
			bne-	protLck1					/* Someone else was trying, try again... */
			b		protSXg1					/* All done... */
			
			.align	4

protLckw1:	mr.		r11,r11						/* Check if it's already held */
			beq+	protLck1					/* It's clear... */
			lwz		r11,0(r7)					/* Get lock word again... */
			b		protLckw1					/* Wait... */
			
			.align	4

protSXg1:	isync								/* Make sure we haven't used anything yet */

 			lwz		r6,mmPTEent(r10)			/* Get the pointer to the PTE now that the lock's set */

			rlwinm	r9,r5,1,0,3					/* Move in the segment */
			lwz		r2,mmPTEr(r10)				; Get the mapping copy of the PTE
			mr.		r6,r6						/* See if there is a PTE here */
			rlwinm	r8,r5,31,2,25				/* Line it up */
			rlwimi	r2,r4,0,30,31				; Move protection bits into the mapping copy
		
			beq+	protul						/* There's no PTE to invalidate... */
			
			xor		r8,r8,r6					/* Back hash to virt index */
			rlwimi	r9,r5,22,4,9				/* Move in the API */
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))	/* Get the TLBIE lock */
			rlwinm	r5,r5,0,1,31				/* Clear the valid bit */
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	/* Grab up the bottom part */
			mfspr	r11,pvr						/* Find out what kind of machine we are */
			rlwimi	r9,r8,6,10,19				/* Create the virtual address */
			rlwinm	r11,r11,16,16,31			/* Isolate CPU type */

			stw		r5,0(r6)					/* Make the PTE invalid */		
			cmplwi	cr1,r11,3					/* Is this a 603? */
			sync								/* Make sure the invalid is stored */
						
tlbhangp:	lwarx	r11,0,r12					/* Get the TLBIE lock */
			rlwinm	r8,r6,29,29,31				/* Get the bit position of entry */
			mr.		r11,r11						/* Is it locked? */
			lis		r5,0x8000					/* Start up a bit mask */
			li		r11,1						/* Get our lock word */
			bne-	tlbhangp					/* It's locked, go wait... */
			stwcx.	r11,0,r12					/* Try to get it */
			bne-	tlbhangp					/* We was beat... */
			
			li		r11,0						/* Lock clear value */

			tlbie	r9							/* Invalidate it everywhere */

			beq-	cr1,its603p					/* It's a 603, skip the tlbsync... */
			
			eieio								/* Make sure that the tlbie happens first */
			tlbsync								/* wait for everyone to catch up */
			isync								
			
its603p:	stw		r11,0(r12)					/* Clear the lock */
			srw		r5,r5,r8					/* Make a "free slot" mask */
			sync								/* Make sure of it all */

			lwz		r6,4(r6)					/* Get the latest reference and change bits */
			stw		r11,mmPTEent(r10)			/* Clear the pointer to the PTE */
			rlwinm	r6,r6,0,23,24				/* Extract the RC bits */
			lwz		r9,PCAallo(r7)				/* Get the allocation control bits */
			rlwinm	r8,r5,24,8,15				/* Make the autogen bit to turn off */
			rlwimi	r2,r6,0,23,24				; Put the latest RC bit in mapping copy
			or		r9,r9,r5					/* Set the slot free */
			rlwimi	r8,r8,24,16,23				/* Get lock bit mask to turn it off */
			andc	r9,r9,r8					/* Clear the auto and lock bits */
			li		r5,pepte1					/* Get displacement to the second word of master pte */
			stw		r9,PCAallo(r7)				/* Store the allocation controls */
			
protmod:	lwarx	r11,r5,r3					/* Get the master copy */
			or		r11,r11,r6					/* Merge in latest RC */
			stwcx.	r11,r5,r3					/* Save it back */
			bne-	protmod						/* If it changed, try again... */
			
protul:		li		r4,0						/* Get a 0 */
			stw		r2,mmPTEr(r10)				; Save the updated mapping PTE
			lwz		r10,mmnext(r10)				/* Get the next */

			sync								; Make sure stores are complete

			stw		r4,0(r7)					/* Unlock the hash chain */
			b		protnext					/* Go get the next one */
			
			.align	4

protdone:	mtmsr	r0							/* Interrupts and translation back on */
			isync
#if PERFTIMES && DEBUG
			mflr	r11
			li		r3,27
			bl		EXT(dbgLog2)				; Start of hw_add_map
			mtlr	r11
#endif
			blr									/* Return... */


/*
 *			hw_prot_virt(mapping, prot) - Change the protection of single page
 *
 *			Upon entry, R3 contains a pointer (real) to a mapping.
 *			R4 contains the PPC protection bits.
 *
 *			Acquire the lock on the PTEG hash list for the mapping being processed.
 *
 *			If the current mapping has a PTE entry, we invalidate
 *			it and merge the reference and change information into the phys_entry.
 *
 *			Next, slam the protection bits into the entry, merge the RC bits, 
 *			and unlock the hash list.
 *
 *			Note that this must be done with both interruptions off and VM off
 *	
 *			  
 */

			.align	5
			.globl	EXT(hw_prot_virt)

LEXT(hw_prot_virt)
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r7,r3
//			lwz		r5,4(r3)
			li		r5,0x1111
			li		r3,40
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r7
			mtlr	r11
#endif
 			mfsprg	r9,2						; Get feature flags 
			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */
			
			bt		pfNoMSRirb,hpvNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hpvNoMSRx
			
hpvNoMSR:	
			mr		r5,r0
			mr		r7,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r7
			mr		r0,r5
hpvNoMSRx:


			
/*
 *			Note that we need to to do the interlocked update here because another processor
 *			can be updating the reference and change bits even though the physical entry
 *			is locked.  All modifications to the PTE portion of the physical entry must be
 *			done via interlocked update.
 */
			
			lwz		r7,mmPTEhash(r3)			/* Get pointer to hash list anchor */
			lwz		r5,mmPTEv(r3)				/* Get the virtual address */
			rlwinm	r7,r7,0,0,25				/* Round hash list down to PCA boundary */

			li		r12,1						/* Get the locked value */

protvLck1:	lwarx	r11,0,r7					/* Get the PTEG lock */
			mr.		r11,r11						/* Is it locked? */
			bne-	protvLckw1					/* Yeah... */
			stwcx.	r12,0,r7					/* Try to take it */
			bne-	protvLck1					/* Someone else was trying, try again... */
			b		protvSXg1					/* All done... */
			
			.align	4

protvLckw1:	mr.		r11,r11						/* Check if it's already held */
			beq+	protvLck1					/* It's clear... */
			lwz		r11,0(r7)					/* Get lock word again... */
			b		protvLckw1					/* Wait... */
			
			.align	4

protvSXg1:	isync								/* Make sure we haven't used anything yet */

 			lwz		r6,mmPTEent(r3)				/* Get the pointer to the PTE now that the lock's set */
 			lwz		r2,mmPTEr(r3)				; Get the mapping copy if the real part

			rlwinm	r9,r5,1,0,3					/* Move in the segment */
			cmplwi	cr7,r6,0					; Any PTE to invalidate?
			rlwimi	r2,r4,0,30,31				; Move in the new protection bits
			rlwinm	r8,r5,31,2,25				/* Line it up */
		
			beq+	cr7,pvnophys				/* There's no PTE to invalidate... */
			
			xor		r8,r8,r6					/* Back hash to virt index */
			rlwimi	r9,r5,22,4,9				/* Move in the API */
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))	/* Get the TLBIE lock */
			rlwinm	r5,r5,0,1,31				/* Clear the valid bit */
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	/* Grab up the bottom part */
			mfspr	r11,pvr						/* Find out what kind of machine we are */
			rlwimi	r9,r8,6,10,19				/* Create the virtual address */
			rlwinm	r11,r11,16,16,31			/* Isolate CPU type */

			stw		r5,0(r6)					/* Make the PTE invalid */		
			cmplwi	cr1,r11,3					/* Is this a 603? */
			sync								/* Make sure the invalid is stored */
						
tlbhangpv:	lwarx	r11,0,r12					/* Get the TLBIE lock */
			rlwinm	r8,r6,29,29,31				/* Get the bit position of entry */
			mr.		r11,r11						/* Is it locked? */
			lis		r5,0x8000					/* Start up a bit mask */
			li		r11,1						/* Get our lock word */
			bne-	tlbhangpv					/* It's locked, go wait... */
			stwcx.	r11,0,r12					/* Try to get it */
			bne-	tlbhangpv					/* We was beat... */
			
			li		r11,0						/* Lock clear value */

			tlbie	r9							/* Invalidate it everywhere */

			beq-	cr1,its603pv				/* It's a 603, skip the tlbsync... */
			
			eieio								/* Make sure that the tlbie happens first */
			tlbsync								/* wait for everyone to catch up */
			isync								
			
its603pv:	stw		r11,0(r12)					/* Clear the lock */
			srw		r5,r5,r8					/* Make a "free slot" mask */
			sync								/* Make sure of it all */

			lwz		r6,4(r6)					/* Get the latest reference and change bits */
			stw		r11,mmPTEent(r3)			/* Clear the pointer to the PTE */
			rlwinm	r6,r6,0,23,24				/* Extract the RC bits */
			lwz		r9,PCAallo(r7)				/* Get the allocation control bits */
			rlwinm	r8,r5,24,8,15				/* Make the autogen bit to turn off */
			lwz		r10,mmphysent(r3)			; Get any physical entry
			or		r9,r9,r5					/* Set the slot free */
			rlwimi	r8,r8,24,16,23				/* Get lock bit mask to turn it off */
			andc	r9,r9,r8					/* Clear the auto and lock bits */
			mr.		r10,r10						; Is there a physical entry?
			li		r5,pepte1					/* Get displacement to the second word of master pte */
			stw		r9,PCAallo(r7)				/* Store the allocation controls */
			rlwimi	r2,r6,0,23,24				; Stick in RC bits
			beq-	pvnophys					; No physical entry...
			
protvmod:	lwarx	r11,r5,r10					/* Get the master copy */
			or		r11,r11,r6					/* Merge in latest RC */
			stwcx.	r11,r5,r10					/* Save it back */
			bne-	protvmod					/* If it changed, try again... */
			
pvnophys:	li		r4,0						/* Get a 0 */
			stw		r2,mmPTEr(r3)				; Set the real part of the PTE

			sync								; Make sure everything is stored

			stw		r4,0(r7)					/* Unlock the hash chain */
			mtmsr	r0							; Restore interrupts and translation
			isync

#if PERFTIMES && DEBUG
			mflr	r11
			li		r3,41
			bl		EXT(dbgLog2)				
			mtlr	r11
#endif
			blr									/* Return... */


/*
 *			hw_attr_virt(mapping, attr) - Change the attributes of single page
 *
 *			Upon entry, R3 contains a pointer (real) to a mapping.
 *			R4 contains the WIMG bits.
 *
 *			Acquire the lock on the PTEG hash list for the mapping being processed.
 *
 *			If the current mapping has a PTE entry, we invalidate
 *			it and merge the reference and change information into the phys_entry.
 *
 *			Next, slam the WIMG bits into the entry, merge the RC bits, 
 *			and unlock the hash list.
 *
 *			Note that this must be done with both interruptions off and VM off
 *	
 *			  
 */

			.align	5
			.globl	EXT(hw_attr_virt)

LEXT(hw_attr_virt)
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r7,r3
//			lwz		r5,4(r3)
			li		r5,0x1111
			li		r3,40
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r7
			mtlr	r11
#endif
			mfsprg	r9,2						; Get feature flags 
 			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			mtcrf	0x04,r9						; Set the features			
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */

			bt		pfNoMSRirb,havNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		havNoMSRx
			
havNoMSR:	
			mr		r5,r0
			mr		r7,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r7
			mr		r0,r5
havNoMSRx:

/*
 *			Note that we need to to do the interlocked update here because another processor
 *			can be updating the reference and change bits even though the physical entry
 *			is locked.  All modifications to the PTE portion of the physical entry must be
 *			done via interlocked update.
 */
			
			lwz		r7,mmPTEhash(r3)			/* Get pointer to hash list anchor */
			lwz		r5,mmPTEv(r3)				/* Get the virtual address */
			rlwinm	r7,r7,0,0,25				/* Round hash list down to PCA boundary */

			li		r12,1						/* Get the locked value */

attrvLck1:	lwarx	r11,0,r7					/* Get the PTEG lock */
			mr.		r11,r11						/* Is it locked? */
			bne-	attrvLckw1					/* Yeah... */
			stwcx.	r12,0,r7					/* Try to take it */
			bne-	attrvLck1					/* Someone else was trying, try again... */
			b		attrvSXg1					/* All done... */
			
			.align	4

attrvLckw1:	mr.		r11,r11						/* Check if it's already held */
			beq+	attrvLck1					/* It's clear... */
			lwz		r11,0(r7)					/* Get lock word again... */
			b		attrvLckw1					/* Wait... */
			
			.align	4

attrvSXg1:	isync								/* Make sure we haven't used anything yet */

 			lwz		r6,mmPTEent(r3)				/* Get the pointer to the PTE now that the lock's set */
 			lwz		r2,mmPTEr(r3)				; Get the mapping copy if the real part

			rlwinm	r9,r5,1,0,3					/* Move in the segment */
			mr.		r6,r6						/* See if there is a PTE here */
			rlwimi	r2,r4,0,25,28				; Move in the new attribute bits
			rlwinm	r8,r5,31,2,25				/* Line it up and check if empty */
		
			beq+	avnophys					/* There's no PTE to invalidate... */
			
			xor		r8,r8,r6					/* Back hash to virt index */
			rlwimi	r9,r5,22,4,9				/* Move in the API */
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))	/* Get the TLBIE lock */
			rlwinm	r5,r5,0,1,31				/* Clear the valid bit */
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	/* Grab up the bottom part */
			mfspr	r11,pvr						/* Find out what kind of machine we are */
			rlwimi	r9,r8,6,10,19				/* Create the virtual address */
			rlwinm	r11,r11,16,16,31			/* Isolate CPU type */
			stw		r5,0(r6)					/* Make the PTE invalid */		
			cmplwi	cr1,r11,3					/* Is this a 603? */
			sync								/* Make sure the invalid is stored */
						
tlbhangav:	lwarx	r11,0,r12					/* Get the TLBIE lock */
			rlwinm	r8,r6,29,29,31				/* Get the bit position of entry */
			mr.		r11,r11						/* Is it locked? */
			lis		r5,0x8000					/* Start up a bit mask */
			li		r11,1						/* Get our lock word */
			bne-	tlbhangav					/* It's locked, go wait... */
			stwcx.	r11,0,r12					/* Try to get it */
			bne-	tlbhangav					/* We was beat... */
			
			li		r11,0						/* Lock clear value */

			tlbie	r9							/* Invalidate it everywhere */

			beq-	cr1,its603av				/* It's a 603, skip the tlbsync... */
			
			eieio								/* Make sure that the tlbie happens first */
			tlbsync								/* wait for everyone to catch up */
			isync								
			
its603av:	stw		r11,0(r12)					/* Clear the lock */
			srw		r5,r5,r8					/* Make a "free slot" mask */
			sync								/* Make sure of it all */

			lwz		r6,4(r6)					/* Get the latest reference and change bits */
			stw		r11,mmPTEent(r3)			/* Clear the pointer to the PTE */
			rlwinm	r6,r6,0,23,24				/* Extract the RC bits */
			lwz		r9,PCAallo(r7)				/* Get the allocation control bits */
			rlwinm	r8,r5,24,8,15				/* Make the autogen bit to turn off */
			lwz		r10,mmphysent(r3)			; Get any physical entry
			or		r9,r9,r5					/* Set the slot free */
			rlwimi	r8,r8,24,16,23				/* Get lock bit mask to turn it off */
			andc	r9,r9,r8					/* Clear the auto and lock bits */
			mr.		r10,r10						; Is there a physical entry?
			li		r5,pepte1					/* Get displacement to the second word of master pte */
			stw		r9,PCAallo(r7)				/* Store the allocation controls */
			rlwimi	r2,r6,0,23,24				; Stick in RC bits
			beq-	avnophys					; No physical entry...			
			
attrvmod:	lwarx	r11,r5,r10					/* Get the master copy */
			or		r11,r11,r6					/* Merge in latest RC */
			stwcx.	r11,r5,r10					/* Save it back */
			bne-	attrvmod					/* If it changed, try again... */
			
avnophys:	li		r4,0						/* Get a 0 */
			stw		r2,mmPTEr(r3)				; Set the real part of the PTE

			sync								; Make sure that everything is updated

			stw		r4,0(r7)					/* Unlock the hash chain */
			
			rlwinm	r2,r2,0,0,19				; Clear back to page boundary
			
attrflsh:	cmplwi	r4,(4096-32)				; Are we about to do the last line on page?
			dcbst	r2,r4						; Flush cache because we changed attributes
			addi	r4,r4,32					; Bump up cache
			blt+	attrflsh					; Do the whole page...
			sync

			li		r4,0
attrimvl:	cmplwi	r4,(4096-32)				; Are we about to do the last line on page?
			dcbi	r2,r4						; Invalidate dcache because we changed attributes
			icbi	r2,r4						; Invalidate icache because we changed attributes
			addi	r4,r4,32					; Bump up cache
			blt+	attrimvl					; Do the whole page...
			sync

			mtmsr	r0							; Restore interrupts and translation
			isync

#if PERFTIMES && DEBUG
			mflr	r11
			li		r3,41
			bl		EXT(dbgLog2)				
			mtlr	r11
#endif
			blr									/* Return... */


/*
 *			hw_pte_comm(physent) - Do something to the PTE pointing to a physical page
 *
 *			Upon entry, R3 contains a pointer to a physical entry which is locked.
 *			Note that this must be done with both interruptions off and VM off
 *
 *			First, we set up CRs 5 and 7 to indicate which of the 7 calls this is.
 *
 *			Now we scan the mappings to invalidate any with an active PTE.
 *
 *				Acquire the lock on the PTEG hash list for the mapping being processed.
 *
 *				If the current mapping has a PTE entry, we invalidate
 *				it and merge the reference and change information into the phys_entry.
 *
 *				Next, unlock the hash list and go on to the next mapping.
 *
 *	
 *			  
 */

			.align	5
			.globl	EXT(hw_inv_all)

LEXT(hw_inv_all)
	
			li		r9,0x800					/* Indicate invalidate all */
			li		r2,0						; No inadvertant modifications please
			b		hw_pte_comm					/* Join in the fun... */


			.align	5
			.globl	EXT(hw_tst_mod)

LEXT(hw_tst_mod)

			lwz		r8,pepte1(r3)				; Get the saved PTE image
			li		r9,0x400					/* Indicate test modify */
			li		r2,0						; No inadvertant modifications please
			rlwinm.	r8,r8,25,31,31				; Make change bit into return code
			beq+	hw_pte_comm					; Assume we do not know if it is set...
			mr		r3,r8						; Set the return code
			blr									; Return quickly...

 			.align	5
			.globl	EXT(hw_tst_ref)

LEXT(hw_tst_ref)
			lwz		r8,pepte1(r3)				; Get the saved PTE image
			li		r9,0x200					/* Indicate test reference bit */
			li		r2,0						; No inadvertant modifications please
			rlwinm.	r8,r8,24,31,31				; Make reference bit into return code
			beq+	hw_pte_comm					; Assume we do not know if it is set...
			mr		r3,r8						; Set the return code
			blr									; Return quickly...

/*
 *			Note that the following are all in one CR for ease of use later
 */
			.align	4
			.globl	EXT(hw_set_mod)

LEXT(hw_set_mod)
			
			li		r9,0x008					/* Indicate set modify bit */
			li		r2,0x4						; Set set C, clear none
			b		hw_pte_comm					/* Join in the fun... */


			.align	4
			.globl	EXT(hw_clr_mod)

LEXT(hw_clr_mod)
			
			li		r9,0x004					/* Indicate clear modify bit */
			li		r2,0x1						; Set set none, clear C
			b		hw_pte_comm					/* Join in the fun... */


			.align	4
			.globl	EXT(hw_set_ref)

LEXT(hw_set_ref)
			
			li		r9,0x002					/* Indicate set reference */
			li		r2,0x8						; Set set R, clear none
			b		hw_pte_comm					/* Join in the fun... */

			.align	5
			.globl	EXT(hw_clr_ref)

LEXT(hw_clr_ref)
			
			li		r9,0x001					/* Indicate clear reference bit */
			li		r2,0x2						; Set set none, clear R
			b		hw_pte_comm					/* Join in the fun... */


/*
 *			This is the common stuff.
 */

			.align	5

hw_pte_comm:									/* Common routine for pte tests and manips */
 
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r7,r3
			lwz		r4,4(r3)
			mr		r5,r9			
			li		r3,28
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r7
			mtlr	r11
#endif
 			mfsprg	r8,2						; Get feature flags 
			lwz		r10,pephyslink(r3)			/* Get the first mapping block */
			mfmsr	r0							/* Save the MSR  */
			rlwinm.	r10,r10,0,0,26				; Clear out the flags from first link and see if we are mapped
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r8						; Set the features			
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */
			beq-	comnmap						; No mapping
			dcbt	br0,r10						; Touch the first mapping in before the isync
			
comnmap:

			bt		pfNoMSRirb,hpcNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hpcNoMSRx
			
hpcNoMSR:	
			mr		r5,r0
			mr		r7,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r7
			mr		r0,r5
hpcNoMSRx:

			mtcrf	0x05,r9						/* Set the call type flags into cr5 and 7 */

			beq-	commdone					; Nothing us mapped to this page...
			b		commnext					; Jump to first pass (jump here so we can align loop)
		
			.align	5	

commnext:	lwz		r11,mmnext(r10)				; Get the pointer to the next mapping (if any)
			lwz		r7,mmPTEhash(r10)			/* Get pointer to hash list anchor */
			lwz		r5,mmPTEv(r10)				/* Get the virtual address */
			mr.		r11,r11						; More mappings to go?
			rlwinm	r7,r7,0,0,25				/* Round hash list down to PCA boundary */
			beq-	commnxtch					; No more mappings...
			dcbt	br0,r11						; Touch the next mapping

commnxtch:	li		r12,1						/* Get the locked value */

commLck1:	lwarx	r11,0,r7					/* Get the PTEG lock */
			mr.		r11,r11						/* Is it locked? */
			bne-	commLckw1					/* Yeah... */
			stwcx.	r12,0,r7					/* Try to take it */
			bne-	commLck1					/* Someone else was trying, try again... */
			b		commSXg1					/* All done... */
			
			.align	4

commLckw1:	mr.		r11,r11						/* Check if it's already held */
			beq+	commLck1					/* It's clear... */
			lwz		r11,0(r7)					/* Get lock word again... */
			b		commLckw1					/* Wait... */
			
			.align	4

commSXg1:	isync								/* Make sure we haven't used anything yet */

 			lwz		r6,mmPTEent(r10)			/* Get the pointer to the PTE now that the lock's set */

			rlwinm	r9,r5,1,0,3					/* Move in the segment */
			mr.		r6,r6						/* See if there is a PTE entry here */
			rlwinm	r8,r5,31,2,25				/* Line it up and check if empty */
		
			beq+	commul						/* There's no PTE to invalidate... */
			
			xor		r8,r8,r6					/* Back hash to virt index */
			rlwimi	r9,r5,22,4,9				/* Move in the API */
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))		/* Get the TLBIE lock */
			rlwinm	r5,r5,0,1,31				/* Clear the valid bit */
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	/* Grab up the bottom part */
			rlwimi	r9,r8,6,10,19				/* Create the virtual address */

			stw		r5,0(r6)					/* Make the PTE invalid */		
			mfspr	r4,pvr						/* Find out what kind of machine we are */
			sync								/* Make sure the invalid is stored */
						
tlbhangco:	lwarx	r11,0,r12					/* Get the TLBIE lock */
			rlwinm	r8,r6,29,29,31				/* Get the bit position of entry */
			mr.		r11,r11						/* Is it locked? */
			lis		r5,0x8000					/* Start up a bit mask */
			li		r11,1						/* Get our lock word */
			bne-	tlbhangco					/* It's locked, go wait... */
			stwcx.	r11,0,r12					/* Try to get it */
			bne-	tlbhangco					/* We was beat... */
			
			rlwinm	r4,r4,16,16,31				/* Isolate CPU type */
			li		r11,0						/* Lock clear value */
			cmplwi	r4,3						/* Is this a 603? */

			tlbie	r9							/* Invalidate it everywhere */

			beq-	its603co					/* It's a 603, skip the tlbsync... */
			
			eieio								/* Make sure that the tlbie happens first */
			tlbsync								/* wait for everyone to catch up */
			isync								
			
its603co:	stw		r11,0(r12)					/* Clear the lock */
			srw		r5,r5,r8					/* Make a "free slot" mask */
			sync								/* Make sure of it all */

			lwz		r6,4(r6)					/* Get the latest reference and change bits */
			lwz		r9,PCAallo(r7)				/* Get the allocation control bits */
			stw		r11,mmPTEent(r10)			/* Clear the pointer to the PTE */
			rlwinm	r8,r5,24,8,15				/* Make the autogen bit to turn off */
			or		r9,r9,r5					/* Set the slot free */
			rlwimi	r8,r8,24,16,23				/* Get lock bit mask to turn it off */
			rlwinm	r4,r6,0,23,24				/* Extract the RC bits */
			andc	r9,r9,r8					/* Clear the auto and lock bits */
			li		r5,pepte1					/* Get displacement to the second word of master pte */
			stw		r9,PCAallo(r7)				/* Store the allocation controls */
			
commmod:	lwarx	r11,r5,r3					/* Get the master copy */
			or		r11,r11,r4					/* Merge in latest RC */
			stwcx.	r11,r5,r3					/* Save it back */
			bne-	commmod						/* If it changed, try again... */
			b		commulnl					; Skip loading the old real part...

commul:		lwz		r6,mmPTEr(r10)				; Get the real part

commulnl:	rlwinm	r12,r2,5,23,24				; Get the "set" bits
			rlwinm	r11,r2,7,23,24				; Get the "clear" bits
			
			or		r6,r6,r12					; Set the bits to come on
			andc	r6,r6,r11					; Clear those to come off

			stw		r6,mmPTEr(r10)				; Set the new RC

			lwz		r10,mmnext(r10)				/* Get the next */
			li		r4,0						/* Make sure this is 0 */
			mr.		r10,r10						; Is there another mapping?

			sync								; Make sure that all is saved

			stw		r4,0(r7)					/* Unlock the hash chain */
			bne+	commnext					; Go get the next if there is one...
			
/*
 *			Now that all PTEs have been invalidated and the master RC bits are updated,
 *			we go ahead and figure out what the original call was and do that.  Note that
 *			another processor could be messing around and may have entered one of the 
 *			PTEs we just removed into the hash table.  Too bad...  You takes yer chances.
 *			If there's a problem with that, it's because some higher level was trying to
 *			do something with a mapping that it shouldn't.  So, the problem's really
 *			there, nyaaa, nyaaa, nyaaa... nyaaa, nyaaa... nyaaa! So there!
 */

commdone:	li		r5,pepte1					/* Get displacement to the second word of master pte */
			blt		cr5,commfini				/* We're finished, it was invalidate all... */
			bgt		cr5,commtst					/* It was a test modified... */
			beq		cr5,commtst					/* It was a test reference... */

/*
 *			Note that we need to to do the interlocked update here because another processor
 *			can be updating the reference and change bits even though the physical entry
 *			is locked.  All modifications to the PTE portion of the physical entry must be
 *			done via interlocked update.
 */

			rlwinm	r12,r2,5,23,24				; Get the "set" bits
			rlwinm	r11,r2,7,23,24				; Get the "clear" bits

commcng:	lwarx	r8,r5,r3					/* Get the master copy */
			or		r8,r8,r12					; Set the bits to come on
			andc	r8,r8,r11					; Clear those to come off
			stwcx.	r8,r5,r3					/* Save it back */
			bne-	commcng						/* If it changed, try again... */

			mtmsr	r0							/* Interrupts and translation back on */
			isync
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,29
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									/* Return... */

			.align	4

commtst:	lwz		r8,pepte1(r3)				/* Get the PTE */
			bne-	cr5,commtcb					; This is for the change bit...
			mtmsr	r0							; Interrupts and translation back on
			rlwinm	r3,r8,24,31,31				; Copy reference bit to bit 31
			isync								; Toss prefetching
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,29
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									; Return...

			.align	4

commtcb:	rlwinm	r3,r8,25,31,31				; Copy change bit to bit 31

commfini:	mtmsr	r0							; Interrupts and translation back on
			isync								; Toss prefetching

#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,29
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									; Return...

/*
 *			unsigned int hw_test_rc(mapping *mp, boolean_t reset);
 *
 *			Test the RC bits for a specific mapping.  If reset is non-zero, clear them.
 *			We return the RC value in the mapping if there is no PTE or if C is set.
 *			(Note: R is always set with C.) Otherwise we invalidate the PTE and
 *			collect the RC bits from there, also merging them into the global copy.
 *			
 *			For now, we release the PTE slot and leave it invalid.  In the future, we
 *			may consider re-validating and not releasing the slot.  It would be faster,
 *			but our current implementation says that we will have not PTEs valid
 *			without the reference bit set.
 *
 *			We will special case C==1 && not reset to just return the RC.
 *
 *			Probable state is worst performance state: C bit is off and there is a PTE.
 */

#define		htrReset 31

			.align	5
			.globl	EXT(hw_test_rc)

LEXT(hw_test_rc)

 			mfsprg	r9,2						; Get feature flags 
			mfmsr	r0							; Save the MSR 
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
 			mr.		r4,r4						; See if we have a reset to do later
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	; Clear interruption mask
			crnot	htrReset,cr0_eq				; Remember reset
			mtcrf	0x04,r9						; Set the features			
			rlwinm	r12,r12,0,28,25				; Clear IR and DR
			
			bt		pfNoMSRirb,htrNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		htrNoMSRx
			
htrNoMSR:	
			mr		r2,r0
			mr		r7,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r7
			mr		r0,r2
htrNoMSRx:
			
			lwz		r2,mmPTEr(r3)				; Get the real part
			lwz		r7,mmPTEhash(r3)			; Get pointer to hash list anchor
			rlwinm.	r12,r2,0,24,24				; Is the change bit on?
			lwz		r5,mmPTEv(r3)				; Get the virtual address
			crnor	cr0_eq,cr0_eq,htrReset		; Set if C=1 && not reset
			rlwinm	r7,r7,0,0,25				; Round hash list down to PCA boundary 
			bt		cr0_eq,htrcset				; Special case changed but no reset case...

			li		r12,1						; Get the locked value

htrLck1:	lwarx	r11,0,r7					; Get the PTEG lock
			mr.		r11,r11						; Is it locked?
			bne-	htrLckw1					; Yeah...
			stwcx.	r12,0,r7					; Try to take it
			bne-	htrLck1						; Someone else was trying, try again...
			b		htrSXg1						; All done...
			
			.align	4

htrLckw1:	mr.		r11,r11						; Check if it is already held 
			beq+	htrLck1						; It is clear... 
			lwz		r11,0(r7)					; Get lock word again... 
			b		htrLckw1					; Wait... 
			
			.align	4

htrSXg1:	isync								; Make sure we have not used anything yet

 			lwz		r6,mmPTEent(r3)				; Get the pointer to the PTE now that the lock is set
 			lwz		r2,mmPTEr(r3)				; Get the mapping copy of the real part

			rlwinm	r9,r5,1,0,3					; Move in the segment
			mr.		r6,r6						; Any PTE to invalidate?
			rlwinm	r8,r5,31,2,25				; Line it up 
		
			beq+	htrnopte					; There is no PTE to invalidate...
			
			xor		r8,r8,r6					; Back hash to virt index
			rlwimi	r9,r5,22,4,9				; Move in the API
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))	; Get the TLBIE lock
			rlwinm	r5,r5,0,1,31				; Clear the valid bit
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	; Grab up the bottom part
			mfspr	r11,pvr						; Find out what kind of machine we are
			rlwimi	r9,r8,6,10,19				; Create the virtual address
			rlwinm	r11,r11,16,16,31			; Isolate CPU type 

			stw		r5,0(r6)					; Make the PTE invalid	
			cmplwi	cr1,r11,3					; Is this a 603?
			sync								; Make sure the invalid is stored
						
htrtlbhang:	lwarx	r11,0,r12					; Get the TLBIE lock
			rlwinm	r8,r6,29,29,31				; Get the bit position of entry 
			mr.		r11,r11						; Is it locked?
			lis		r5,0x8000					; Start up a bit mask
			li		r11,1						; Get our lock word 
			bne-	htrtlbhang					; It is locked, go wait...
			stwcx.	r11,0,r12					; Try to get it
			bne-	htrtlbhang					; We was beat...
			
			li		r11,0						; Lock clear value 

			tlbie	r9							;Invalidate it everywhere

			beq-	cr1,htr603					; It is a 603, skip the tlbsync... 
			
			eieio								; Make sure that the tlbie happens first
			tlbsync								; wait for everyone to catch up
			isync								
			
htr603:		stw		r11,0(r12)					; Clear the lock
			srw		r5,r5,r8					; Make a "free slot" mask 
			sync								; Make sure of it all 

			lwz		r6,4(r6)					; Get the latest reference and change bits
			stw		r11,mmPTEent(r3)			; Clear the pointer to the PTE 
			rlwinm	r6,r6,0,23,24				; Extract the RC bits 
			lwz		r9,PCAallo(r7)				; Get the allocation control bits 
			rlwinm	r8,r5,24,8,15				; Make the autogen bit to turn off
			lwz		r10,mmphysent(r3)			; Get any physical entry
			or		r9,r9,r5					; Set the slot free 
			rlwimi	r8,r8,24,16,23				; Get lock bit mask to turn it off
			andc	r9,r9,r8					; Clear the auto and lock bits 
			mr.		r10,r10						; Is there a physical entry?
			li		r5,pepte1					; Get displacement to the second word of master pte
			stw		r9,PCAallo(r7)				; Store the allocation controls
			rlwimi	r2,r6,0,23,24				; Stick in RC bits
			beq-	htrnopte					; No physical entry...
			
htrmrc:		lwarx	r11,r5,r10					; Get the master copy
			or		r11,r11,r6					; Merge in latest RC
			stwcx.	r11,r5,r10					; Save it back
			bne-	htrmrc						; If it changed, try again... 

htrnopte:	rlwinm	r5,r2,25,30,31				; Position RC and mask off
			bf		htrReset,htrnorst			; No reset to do...
			rlwinm	r2,r2,0,25,22				; Clear the RC if requested
			
htrnorst:	li		r4,0						; Get a 0 
			stw		r2,mmPTEr(r3)				; Set the real part of the PTE
			
			sync								; Make sure that stuff is all stored

			stw		r4,0(r7)					; Unlock the hash chain
	
			mr		r3,r5						; Get the old RC to pass back
			mtmsr	r0							; Restore interrupts and translation
			isync
			blr									; Return...

			.align	4

htrcset:	rlwinm	r3,r2,25,30,31				; Position RC and mask off
			mtmsr	r0							; Restore interrupts and translation
			isync
			blr									; Return...


/*
 *			hw_phys_attr(struct phys_entry *pp, vm_prot_t prot, unsigned int wimg) - Sets the default physical page attributes
 *
 *			Note that this must be done with both interruptions off and VM off
 *			Move the passed in attributes into the pte image in the phys entry
 *	
 *			  
 */

			.align	5
			.globl	EXT(hw_phys_attr)

LEXT(hw_phys_attr)

#if PERFTIMES && DEBUG
			mflr	r11
			mr		r8,r3
			mr		r7,r5
			mr		r5,r4
//			lwz		r4,4(r3)
			li		r4,0x1111
			li		r3,30
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r8
			mr		r4,r5
			mr		r5,r7
			mtlr	r11
#endif
			mfsprg	r9,2						; Get feature flags 
			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			andi.	r5,r5,0x0078				/* Clean up the WIMG */
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			rlwimi	r5,r4,0,30,31				/* Move the protection into the wimg register */
			la		r6,pepte1(r3)				/* Point to the default pte */
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */

			bt		pfNoMSRirb,hpaNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hpaNoMSRx
			
hpaNoMSR:	
			mr		r10,r0
			mr		r4,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r4
			mr		r0,r10
hpaNoMSRx:

atmattr:	lwarx	r10,0,r6					/* Get the pte */
			rlwimi	r10,r5,0,25,31				/* Move in the new attributes */
			stwcx.	r10,0,r6					/* Try it on for size */
			bne-	atmattr						/* Someone else was trying, try again... */
		
			mtmsr	r0							/* Interrupts and translation back on */
			isync
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r10
			li		r3,31
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mtlr	r11
#endif
			blr									/* All done... */



/*
 *			handlePF - handle a page fault interruption
 *
 *			If the fault can be handled, this routine will RFI directly,
 *			otherwise it will return with all registers as in entry.
 *
 *			Upon entry, state and all registers have been saved in savearea.
 *			This is pointed to by R13.
 *			IR and DR are off, interrupts are masked,
 *			Floating point be disabled.
 *			R3 is the interrupt code.
 *
 *			If we bail, we must restore cr5, and all registers except 6 and
 *			3.
 *
 */
	
			.align	5
			.globl	EXT(handlePF)

LEXT(handlePF)

/*
 *			This first part does a quick check to see if we can handle the fault.
 *			We can't handle any kind of protection exceptions here, so we pass
 *			them up to the next level.
 *
 *			The mapping lists are kept in MRS (most recently stolen)
 *			order on queues anchored within from the
 *			PTEG to which the virtual address hashes.  This is further segregated by
 *			the low-order 3 bits of the VSID XORed with the segment number and XORed
 *			with bits 4-7 of the vaddr in an attempt to keep the searches
 *			short.
 *			
 *			MRS is handled by moving the entry to the head of its list when stolen in the
 *			assumption that it will be revalidated soon.  Entries are created on the head 
 *			of the list because they will be used again almost immediately.
 *
 *			We need R13 set to the savearea, R3 set to the interrupt code, and R2
 *			set to the per_proc.
 *
 *			NOTE: In order for a page-fault redrive to work, the translation miss
 *			bit must be set in the DSISR (or SRR1 for IFETCH).  That must occur
 *			before we come here.
 */

			cmplwi	r3,T_INSTRUCTION_ACCESS		/* See if this is for the instruction */
			lwz		r8,savesrr1(r13)			; Get the MSR to determine mode
			beq-	gotIfetch					; We have an IFETCH here...
			
			lwz		r7,savedsisr(r13)			/* Get the DSISR */
			lwz		r6,savedar(r13)				/* Get the fault address */
			b		ckIfProt					; Go check if this is a protection fault...

gotIfetch:	mr		r7,r8						; IFETCH info is in SRR1
			lwz		r6,savesrr0(r13)			/* Get the instruction address */

ckIfProt:	rlwinm.	r7,r7,0,1,1					; Is this a protection exception?
			beqlr-								; Yes... (probably not though)

/*
 *			We will need to restore registers if we bail after this point.
 *			Note that at this point several SRs have been changed to the kernel versions.
 *			Therefore, for these we must build these values.
 */

#if PERFTIMES && DEBUG
			mflr	r11
			mr		r5,r6
			mr		r4,r3
			li		r3,32
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
			mfsprg	r2,0
#endif
			lwz		r3,PP_USERPMAP(r2)			; Get the user pmap (not needed if kernel access, but optimize for user??)
			rlwinm.	r8,r8,0,MSR_PR_BIT,MSR_PR_BIT	; Supervisor state access?
			rlwinm	r5,r6,6,26,29				; Get index to the segment slot
			eqv		r1,r1,r1					; Fill the bottom with foxes
			bne+	notsuper					; Go do the user mode interrupt stuff...
			
			cmplwi	cr1,r5,SR_COPYIN_NUM*4		; See if this is the copyin/copyout segment
			rlwinm	r3,r6,24,8,11				; Make the kernel VSID
			bne+	cr1,havevsid				; We are done if we do not want the copyin/out guy...
			
			mfsr	r3,SR_COPYIN				; Get the copy vsid
			b		havevsid					; Join up...

			.align	5

notsuper:	addi	r5,r5,PMAP_SEGS				; Get offset to table
			lwzx	r3,r3,r5					; Get the VSID

havevsid:	mfspr	r5,sdr1						/* Get hash table base and size */
			cror	cr1_eq,cr0_eq,cr0_eq		; Remember if kernel fault for later
			rlwinm	r9,r6,2,2,5					; Move nybble 1 up to 0 (keep aligned with VSID)
			rlwimi	r1,r5,16,0,15				/* Make table size -1 out of mask */
			rlwinm	r3,r3,6,2,25				/* Position the space for the VSID */
			rlwinm	r7,r6,26,10,25				/* Isolate the page index */
			xor		r9,r9,r3					; Splooch vaddr nybble 0 (from VSID) and 1 together
			or		r8,r5,r1					/* Point to the last byte in table */
			xor		r7,r7,r3					/* Get primary hash */
			rlwinm	r3,r3,1,1,24				/* Position VSID for pte ID */
			addi	r8,r8,1						/* Point to the PTEG Control Area */
			rlwinm	r9,r9,8,27,29				; Get splooched bits in place
			and		r7,r7,r1					/* Wrap the hash */
			rlwimi	r3,r6,10,26,31				/* Move API into pte ID */
			add		r8,r8,r7					/* Point to our PCA entry */
			rlwinm	r12,r3,27,27,29				; Get low 3 bits of the VSID for look-aside hash
			la		r11,PCAhash(r8)				/* Point to the mapping hash area */
			xor		r9,r9,r12					; Finish splooching nybble 0, 1, and the low bits of the VSID


/*
 *			We have about as much as we need to start searching the autogen (aka block maps)
 *			and mappings.  From here on, any kind of failure will bail, and
 *			contention will either bail or restart from here.
 *
 *			
 */
			
			li		r12,1						/* Get the locked value */
			dcbt	0,r11						/* We'll need the hash area in a sec, so get it */
			add		r11,r11,r9					/* Point to the right mapping hash slot */
			
ptegLck:	lwarx	r10,0,r8					/* Get the PTEG lock */
			mr.		r10,r10						/* Is it locked? */
			bne-	ptegLckw					/* Yeah... */
			stwcx.	r12,0,r8					/* Take take it */
			bne-	ptegLck						/* Someone else was trying, try again... */
			b		ptegSXg						/* All done... */
			
			.align	4

ptegLckw:	mr.		r10,r10						/* Check if it's already held */
			beq+	ptegLck						/* It's clear... */
			lwz		r10,0(r8)					/* Get lock word again... */
			b		ptegLckw					/* Wait... */
			
			.align	5
			
			nop									; Force ISYNC to last instruction in IFETCH
			nop									
			nop

ptegSXg:	isync								/* Make sure we haven't used anything yet */

			lwz		r9,0(r11)					/* Pick up first mapping block */
			mr		r5,r11						/* Get the address of the anchor */
			mr		r7,r9						/* Save the first in line */
			b		findmap						; Take space and force loop to cache line
		
findmap:	mr.		r12,r9						/* Are there more? */
			beq-	tryAuto						/* Nope, nothing in mapping list for us... */
			
			lwz		r10,mmPTEv(r12)				/* Get unique PTE identification */
			lwz		r9,mmhashnext(r12)			/* Get the chain, just in case */
			cmplw	r10,r3						/* Did we hit our PTE? */
 			lwz		r0,mmPTEent(r12)			/* Get the pointer to the hash table entry */
			mr		r5,r12						/* Save the current as previous */
			bne-	findmap						; Nothing here, try the next...

;			Cache line boundary here

			cmplwi	cr1,r0,0					/* Is there actually a PTE entry in the hash? */
			lwz		r2,mmphysent(r12)			/* Get the physical entry */
			bne-	cr1,MustBeOK				/* There's an entry in the hash table, so, this must 
												   have been taken care of already... */
			lis		r4,0x8000					; Tell PTE inserter that this was not an auto
			cmplwi	cr2,r2,0					/* Is there a physical entry? */
			li		r0,0x0100					/* Force on the reference bit whenever we make a PTE valid */
			bne+	cr2,gotphys					/* Skip down if we have a physical entry */
			li		r0,0x0180					/* When there is no physical entry, force on
												   both R and C bits to keep hardware from
												   updating the PTE to set them.  We don't
												   keep track of RC for I/O areas, so this is ok */
			
gotphys:	lwz		r2,mmPTEr(r12)				; Get the second part of the PTE
			b		insert						/* Go insert into the PTEG... */

MustBeOK:	li		r10,0						/* Get lock clear value */
			li		r3,T_IN_VAIN				/* Say that we handled it */
			stw		r10,PCAlock(r8)				/* Clear the PTEG lock */

#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,33
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									/* Blow back and handle exception */


			
/*
 *			We couldn't find it in the mapping list.  As a last try, we will
 *			see if we can autogen it from the block mapped list.
 *	
 *			A block mapped area is defined as a contiguous virtual area that is mapped to 
 *			a contiguous physical area.  The olde-tyme IBM VM/XA Interpretive Execution
 *			architecture referred to this as a V=F, or Virtual = Fixed area. 
 *
 *			We consider a V=F area to be a single entity, adjacent areas can not be merged
 *			or overlapped.  The protection and memory attributes are the same and reference
 *			and change indications are not kept. The areas are not considered part of the
 *			physical RAM of the machine and do not have any associated physical table
 *			entries. Their primary use is intended for mapped I/O areas (e.g., framebuffers)
 *			although certain areas of RAM, such as the kernel V=R memory, can be mapped.
 *
 *			We also have a problem in the case of copyin/out: that access is done
 *			within the kernel for a user address. Unfortunately, the user isn't
 *			necessarily the current guy.  That means that we don't have access to the
 *			right autogen list. We can't support this kind of access. So, we need to do
 *			a quick check here and cause a fault if an attempt to copyin or out to
 *			any autogenned area.
 *
 *			The lists must be kept short.
 *
 *			NOTE:  kernel_pmap_store must be in V=R storage!!!!!!!!!!!!!!
 */
 
			.align	5

tryAuto:	rlwinm.	r11,r3,0,5,24				; Check if this is a kernel VSID
			lis		r10,HIGH_ADDR(EXT(kernel_pmap_store)+PMAP_BMAPS)	; Get the top part of kernel block map anchor
			crandc	cr0_eq,cr1_eq,cr0_eq		; Set if kernel access and non-zero VSID (copyin or copyout)
			mfsprg	r11,0						; Get the per_proc area
			beq-	cr0,realFault					; Can not autogen for copyin/copyout...
			ori		r10,r10,LOW_ADDR(EXT(kernel_pmap_store)+PMAP_BMAPS)	; Get the bottom part
			beq-	cr1,bmInKernel				; We are in kernel... (cr1 set way back at entry)
			
			lwz		r10,PP_USERPMAP(r11)		; Get the user pmap
			la		r10,PMAP_BMAPS(r10)			; Point to the chain anchor
			b		bmInKernel					; Jump over alignment gap...
			nop
			nop
			nop
			nop
			nop
			nop						
bmInKernel:
#ifndef CHIP_ERRATA_MAX_V1
			lwarx	r9,0,r10	
#endif /* CHIP_ERRATA_MAX_V1 */

bmapLck:	lwarx	r9,0,r10					; Get the block map anchor and lock
			rlwinm.	r5,r9,0,31,31				; Is it locked?
			ori		r5,r5,1						; Set the lock
			bne-	bmapLckw					; Yeah...
			stwcx.	r5,0,r10					; Lock the bmap list
			bne-	bmapLck						; Someone else was trying, try again...
			b		bmapSXg						; All done...
			
			.align	4

bmapLckw:	rlwinm.	r5,r9,0,31,31				; Check if it is still held
			beq+	bmapLck						; Not no more...
			lwz		r9,0(r10)					; Get lock word again...
			b		bmapLckw					; Check it out...
			
			.align	5
			
			nop									; Force ISYNC to last instruction in IFETCH
			nop									
			nop

bmapSXg:	rlwinm.	r4,r9,0,0,26				; Clear out flags and lock
			isync								; Make sure we have not used anything yet
			bne+	findAuto					; We have something, let us go...
			
bmapNone:	stw		r9,0(r10)					; Unlock it, we have nothing here
												; No sync here because we have not changed anything
			
/*
 *			When we come here, we know that we can't handle this.  Restore whatever
 *			state that we trashed and go back to continue handling the interrupt.
 */

realFault:	li		r10,0						/* Get lock clear value */
			lwz		r3,saveexception(r13)		/* Figure out the exception code again */
			stw		r10,PCAlock(r8)				/* Clear the PTEG lock */
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,33
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									/* Blow back and handle exception */
			
			.align	5
			
findAuto:	mr.		r4,r4						; Is there more?
			beq-	bmapNone					; No more...
			lwz		r5,bmstart(r4)				; Get the bottom of range
			lwz		r11,bmend(r4)				; Get the top of range
			cmplw	cr0,r6,r5					; Are we before the entry?
			cmplw	cr1,r6,r11					; Are we after the entry?
			cror	cr1_eq,cr0_lt,cr1_gt		; Set cr1_eq if new not in range
			bne+	cr1,faGot					; Found it...
			
			lwz		r4,bmnext(r4)				; Get the next one
			b		findAuto					; Check it out...
			
faGot:		
			lwz		r7,blkFlags(r4)				; Get the flags
			rlwinm. r7,r7,0,blkRembit,blkRembit	; is this mapping partially removed
			bne		bmapNone					; Pending remove, bail out
			rlwinm	r6,r6,0,0,19				; Round to page
			lwz		r2,bmPTEr(r4)				; Get the real part of the PTE
			sub		r5,r6,r5					; Get offset into area
			stw		r9,0(r10)					; Unlock it, we are done with it (no sync needed)
			add		r2,r2,r5					; Adjust the real address
			
			lis		r4,0x8080					/* Indicate that this was autogened */
			li		r0,0x0180					/* Autogenned areas always set RC bits.
												   This keeps the hardware from having
												   to do two storage writes */
			
/*
 *			Here where we insert the PTE into the hash.  The PTE image is in R3, R2. 
 *			The PTEG allocation controls are a bit map of the state of the PTEG. The
 *			PCAlock bits are a temporary lock for the specified PTE.  PCAfree indicates that
 *			the PTE slot is empty. PCAauto means that it comes from an autogen area.  These
 *			guys do not keep track of reference and change and are actually "wired".
 *			They're easy to maintain. PCAsteal
 *			is a sliding position mask used to "randomize" PTE slot stealing.  All 4 of these
 *			fields fit in a single word and are loaded and stored under control of the
 *			PTEG control area lock (PCAlock).
 *
 *			Note that PCAauto does not contribute to the steal calculations at all.  Originally
 *			it did, autogens were second in priority.  This can result in a pathalogical
 *			case where an instruction can not make forward progress, or one PTE slot
 *			thrashes.
 *
 *			Physically, the fields are arranged:
 *				0: PCAfree
 *				1: PCAauto
 *				2: PCAlock
 *				3: PCAsteal
 */
			
insert:		lwz		r10,PCAallo(r8)				/* Get the PTEG controls */
			eqv		r6,r6,r6					/* Get all ones */		
			mr		r11,r10						/* Make a copy */
			rlwimi	r6,r10,8,16,23				/* Insert sliding steal position */
			rlwimi	r11,r11,24,24,31			/* Duplicate the locked field */
			addi	r6,r6,-256					/* Form mask */
			rlwimi	r11,r11,16,0,15				/* This gives us a quadrupled lock mask */
			rlwinm	r5,r10,31,24,0				/* Slide over the mask for next time */
			mr		r9,r10						/* Make a copy to test */
			not		r11,r11						/* Invert the quadrupled lock */
			or		r2,r2,r0					/* Force on R, and maybe C bit */
			and		r9,r9,r11					/* Remove the locked guys */
			rlwimi	r5,r5,8,24,24				/* Wrap bottom bit to top in mask */
			rlwimi	r9,r11,0,16,31				/* Put two copies of the unlocked entries at the end */
			rlwinm	r6,r6,0,16,7				; Remove the autogens from the priority calculations
			rlwimi	r10,r5,0,24,31				/* Move steal map back in */
			and		r9,r9,r6					/* Set the starting point for stealing */

/*			So, now we have in R9:
				byte 0 = ~locked & free 
				byte 1 = 0 
				byte 2 = ~locked & (PCAsteal - 1)
				byte 3 = ~locked

				Each bit position represents (modulo 8) a PTE. If it is 1, it is available for 
				allocation at its priority level, left to right.  
				
			Additionally, the PCA steal field in R10 has been rotated right one bit.
*/
			

			rlwinm	r21,r10,8,0,7				; Isolate just the old autogen bits
			cntlzw	r6,r9						/* Allocate a slot */
			mr		r14,r12						/* Save our mapping for later */
			cmplwi	r6,32						; Was there anything available?
			rlwinm	r7,r6,29,30,31				/* Get the priority slot we got this from */
			rlwinm	r6,r6,0,29,31				; Isolate bit position
			srw		r11,r4,r6					/* Position the PTEG control bits */
			slw		r21,r21,r6					; Move corresponding old autogen flag to bit 0
			mr		r22,r11						; Get another copy of the selected slot
			
			beq-	realFault					/* Arghh, no slots! Take the long way 'round... */
			
												/* Remember, we've already set up the mask pattern
												   depending upon how we got here:
												     if got here from simple mapping, R4=0x80000000,
												     if we got here from autogen it is 0x80800000. */
			
			rlwinm	r6,r6,3,26,28				/* Start calculating actual PTE address */
			rlwimi	r22,r22,24,8,15				; Duplicate selected slot in second byte
			rlwinm.	r11,r11,0,8,15				/* Isolate just the auto bit (remember about it too) */
			andc	r10,r10,r22					/* Turn off the free and auto bits */
			add		r6,r8,r6					/* Get position into PTEG control area */
 			cmplwi	cr1,r7,1					/* Set the condition based upon the old PTE type */
			sub		r6,r6,r1					/* Switch it to the hash table */
			or		r10,r10,r11					/* Turn auto on if it is (PTEG control all set up now) */			
			subi	r6,r6,1						/* Point right */
			stw		r10,PCAallo(r8)				/* Allocate our slot */
			dcbt	br0,r6						; Touch in the PTE
			bne		wasauto						/* This was autogenned... */
			
			stw		r6,mmPTEent(r14)			/* Link the mapping to the PTE slot */
			
/*
 *			So, now we're here and what exactly do we have?  We've got: 
 *				1)	a full PTE entry, both top and bottom words in R3 and R2
 *				2)	an allocated slot in the PTEG.
 *				3)	R8 still points to the PTEG Control Area (PCA)
 *				4)	R6 points to the PTE entry.
 *				5)	R1 contains length of the hash table-1. We use this to back-translate
 *					a PTE to a virtual address so we can invalidate TLBs.
 *				6)	R11 has a copy of the PCA controls we set.
 *				7a)	R7 indicates what the PTE slot was before we got to it. 0 shows
 *					that it was empty and 2 or 3, that it was
 *					a we've stolen a live one. CR1 is set to LT for empty and GT
 *					otherwise.
 *				7b)	Bit 0 of R21 is 1 if the stolen PTE was autogenned
 *				8)	So far as our selected PTE, it should be valid if it was stolen
 *					and invalid if not.  We could put some kind of assert here to
 *					check, but I think that I'd rather leave it in as a mysterious,
 *					non-reproducable bug.
 *				9)	The new PTE's mapping has been moved to the front of its PTEG hash list
 *					so that it's kept in some semblance of a MRU list.
 *			   10)	R14 points to the mapping we're adding.
 *
 *			So, what do we have to do yet?
 *				1)	If we stole a slot, we need to invalidate the PTE completely.
 *				2)	If we stole one AND it was not an autogen, 
 *					copy the entire old PTE (including R and C bits) to its mapping.
 *				3)	Set the new PTE in the PTEG and make sure it is valid.
 *				4)	Unlock the PTEG control area.
 *				5)	Go back to the interrupt handler, changing the interrupt
 *					code to "in vain" which will restore the registers and bail out.
 *
 */
wasauto:	oris	r3,r3,0x8000				/* Turn on the valid bit */
			blt+	cr1,slamit					/* It was empty, go slam it on in... */
			
			lwz		r10,0(r6)					/* Grab the top part of the PTE */
			rlwinm	r12,r6,6,4,19				/* Match up the hash to a page boundary */
			rlwinm	r5,r10,5,4,19				/* Extract the VSID to a page boundary */
			rlwinm	r10,r10,0,1,31				/* Make it invalid */
			xor		r12,r5,r12					/* Calculate vaddr */
			stw		r10,0(r6)					/* Invalidate the PTE */
			rlwinm	r5,r10,7,27,29				; Move nybble 0 up to subhash position
			rlwimi	r12,r10,1,0,3				/* Move in the segment portion */
			lis		r9,HIGH_ADDR(EXT(tlb_system_lock))	/* Get the TLBIE lock */
			xor		r5,r5,r10					; Splooch nybble 0 and 1
			rlwimi	r12,r10,22,4,9				/* Move in the API */
			ori		r9,r9,LOW_ADDR(EXT(tlb_system_lock))	/* Grab up the bottom part */
			rlwinm	r4,r10,27,27,29				; Get low 3 bits of the VSID for look-aside hash
			
			sync								/* Make sure the invalid is stored */

			xor		r4,r4,r5					; Finish splooching nybble 0, 1, and the low bits of the VSID
						
tlbhang:	lwarx	r5,0,r9						/* Get the TLBIE lock */
		
			rlwinm	r4,r4,0,27,29				; Clean up splooched hash value

			mr.		r5,r5						/* Is it locked? */
			add		r4,r4,r8					/* Point to the offset into the PCA area */
			li		r5,1						/* Get our lock word */
			bne-	tlbhang						/* It's locked, go wait... */
			
			la		r4,PCAhash(r4)				/* Point to the start of the hash chain for the PTE we're replacing */
			
			stwcx.	r5,0,r9						/* Try to get it */
			bne-	tlbhang						/* We was beat... */
			
			mfspr	r7,pvr						/* Find out what kind of machine we are */
			li		r5,0						/* Lock clear value */
			rlwinm	r7,r7,16,16,31				/* Isolate CPU type */

			tlbie	r12							/* Invalidate it everywhere */

			cmplwi	r7,3						/* Is this a 603? */
			stw		r5,0(r9)					/* Clear the lock */
			
			beq-	its603						/* It's a 603, skip the tlbsync... */
			
			eieio								/* Make sure that the tlbie happens first */
			tlbsync								/* wait for everyone to catch up */
			isync								
			
its603:		rlwinm.	r21,r21,0,0,0				; See if we just stole an autogenned entry
			sync								/* Make sure of it all */

			bne		slamit						; The old was an autogen, time to slam the new in...
			
			lwz		r9,4(r6)					/* Get the real portion of old PTE */
			lwz		r7,0(r4)					/* Get the first element.  We can't get to here
												   if we aren't working with a mapping... */
			mr		r0,r7						; Save pointer to first element
												   
findold:	mr		r1,r11						; Save the previous guy
			mr.		r11,r7						/* Copy and test the chain */
			beq-	bebad						/* Assume it's not zero... */
			
			lwz		r5,mmPTEv(r11)				/* See if this is the old active one */
			cmplw	cr2,r11,r14					/* Check if this is actually the new one */
			cmplw	r5,r10						/* Is this us?  (Note: valid bit kept off in mappings) */
			lwz		r7,mmhashnext(r11)			/* Get the next one in line */
			beq-	cr2,findold					/* Don't count the new one... */
			cmplw	cr2,r11,r0					; Check if we are first on the list
			bne+	findold						/* Not it (and assume the worst)... */
			
			lwz		r12,mmphysent(r11)			/* Get the pointer to the physical entry */
			beq-	cr2,nomove					; We are first, no need to requeue...

			stw		r11,0(r4)					; Chain us to the head
			stw		r0,mmhashnext(r11)			; Chain the old head to us
			stw		r7,mmhashnext(r1)			; Unlink us

nomove:		li		r5,0						/* Clear this on out */
			
			mr.		r12,r12						/* Is there a physical entry? */
			stw		r5,mmPTEent(r11)			; Clear the PTE entry pointer
			li		r5,pepte1					/* Point to the PTE last half */
			stw		r9,mmPTEr(r11)				; Squirrel away the whole thing (RC bits are in here)
			
			beq-	mrgmrcx						; No physical entry for this one...
			
			rlwinm	r11,r9,0,23,24				/* Keep only the RC bits */

mrgmrcx:	lwarx	r9,r5,r12					/* Get the master copy */
			or		r9,r9,r11					/* Merge in latest RC */
			stwcx.	r9,r5,r12					/* Save it back */
			bne-	mrgmrcx						/* If it changed, try again... */

/*
 *			Here's where we finish up.  We save the real part of the PTE, eieio it, to make sure it's
 *			out there before the top half (with the valid bit set).
 */

slamit:		stw		r2,4(r6)					/* Stash the real part */
			li		r4,0						/* Get a lock clear value */
			eieio								/* Erect a barricade */
			stw		r3,0(r6)					/* Stash the virtual part and set valid on */

			stw		r4,PCAlock(r8)				/* Clear the PCA lock */

			li		r3,T_IN_VAIN				/* Say that we handled it */
			sync								/* Go no further until the stores complete */
#if PERFTIMES && DEBUG
			mflr	r11
			mr		r4,r3
			li		r3,33
			bl		EXT(dbgLog2)						; Start of hw_add_map
			mr		r3,r4
			mtlr	r11
#endif
			blr									/* Back to the fold... */
					
bebad:		lis		r0,HIGH_ADDR(Choke)			/* We have a kernel choke!!! */
			ori		r0,r0,LOW_ADDR(Choke)				
			sc									/* Firmware Heimlich maneuver */
			
/*
 *			This walks the hash table or DBATs to locate the physical address of a virtual one.
 *			The space is provided.  If it is the kernel space, the DBATs are searched first.  Failing
 *			that, the hash table is accessed. Zero is returned for failure, so it must be special cased.
 *			This is usually used for debugging, so we try not to rely
 *			on anything that we don't have to.
 */

ENTRY(LRA, TAG_NO_FRAME_USED)

			mfsprg	r8,2						; Get feature flags 
			mfmsr	r10							/* Save the current MSR */
			rlwinm	r10,r10,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r10,r10,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			mtcrf	0x04,r8						; Set the features			
			xoris	r5,r3,HIGH_ADDR(PPC_SID_KERNEL)		/* Clear the top half if equal */
			andi.	r9,r10,0x7FCF				/* Turn off interrupts and translation */
			eqv		r12,r12,r12					/* Fill the bottom with foxes */

			bt		pfNoMSRirb,lraNoMSR			; No MSR...

			mtmsr	r9							; Translation and all off
			isync								; Toss prefetch
			b		lraNoMSRx
			
lraNoMSR:	
			mr		r7,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r9						; Get new MSR
			sc									; Set it
			mr		r3,r7
lraNoMSRx:

			cmplwi	r5,LOW_ADDR(PPC_SID_KERNEL)	/* See if this is kernel space */
			rlwinm	r11,r3,6,6,25				/* Position the space for the VSID */
			isync								/* Purge pipe */
			bne-	notkernsp					/* This is not for the kernel... */		
			
			mfspr	r5,dbat0u					/* Get the virtual address and length */
			eqv		r8,r8,r8					/* Get all foxes */
			rlwinm.	r0,r5,0,30,30				/* Check if valid for supervisor state */
			rlwinm	r7,r5,0,0,14				/* Clean up the base virtual address */
			beq-	ckbat1						/* not valid, skip this one... */
			sub		r7,r4,r7					/* Subtract out the base */
			rlwimi	r8,r5,15,0,14				/* Get area length - 1 */
			mfspr	r6,dbat0l					/* Get the real part */
			cmplw	r7,r8						/* Check if it is in the range */
			bng+	fndbat						/* Yup, she's a good un... */

ckbat1:		mfspr	r5,dbat1u					/* Get the virtual address and length */			
			eqv		r8,r8,r8					/* Get all foxes */
			rlwinm.	r0,r5,0,30,30				/* Check if valid for supervisor state */
			rlwinm	r7,r5,0,0,14				/* Clean up the base virtual address */
			beq-	ckbat2						/* not valid, skip this one... */
			sub		r7,r4,r7					/* Subtract out the base */
			rlwimi	r8,r5,15,0,14				/* Get area length - 1 */
			mfspr	r6,dbat1l					/* Get the real part */
			cmplw	r7,r8						/* Check if it is in the range */
			bng+	fndbat						/* Yup, she's a good un... */
			
ckbat2:		mfspr	r5,dbat2u					/* Get the virtual address and length */
			eqv		r8,r8,r8					/* Get all foxes */
			rlwinm.	r0,r5,0,30,30				/* Check if valid for supervisor state */
			rlwinm	r7,r5,0,0,14				/* Clean up the base virtual address */
			beq-	ckbat3						/* not valid, skip this one... */
			sub		r7,r4,r7					/* Subtract out the base */
			rlwimi	r8,r5,15,0,14				/* Get area length - 1 */
			mfspr	r6,dbat2l					/* Get the real part */
			cmplw	r7,r8						/* Check if it is in the range */
			bng-	fndbat						/* Yup, she's a good un... */
			
ckbat3:		mfspr	r5,dbat3u					/* Get the virtual address and length */
			eqv		r8,r8,r8					/* Get all foxes */
			rlwinm.	r0,r5,0,30,30				/* Check if valid for supervisor state */
			rlwinm	r7,r5,0,0,14				/* Clean up the base virtual address */
			beq-	notkernsp					/* not valid, skip this one... */
			sub		r7,r4,r7					/* Subtract out the base */
			rlwimi	r8,r5,15,0,14				/* Get area length - 1 */
			mfspr	r6,dbat3l					/* Get the real part */
			cmplw	r7,r8						/* Check if it is in the range */
			bgt+	notkernsp					/* No good... */
			
fndbat:		rlwinm	r6,r6,0,0,14				/* Clean up the real address */
			mtmsr	r10							/* Restore state */
			add		r3,r7,r6					/* Relocate the offset to real */
			isync								/* Purge pipe */
			blr									/* Bye, bye... */

notkernsp:	mfspr	r5,sdr1						/* Get hash table base and size */
			rlwimi	r11,r4,30,2,5				/* Insert the segment no. to make a VSID */
			rlwimi	r12,r5,16,0,15				/* Make table size -1 out of mask */
			rlwinm	r7,r4,26,10,25				/* Isolate the page index */
			andc	r5,r5,r12					/* Clean up the hash table */
			xor		r7,r7,r11					/* Get primary hash */
			rlwinm	r11,r11,1,1,24				/* Position VSID for pte ID */
			and		r7,r7,r12					/* Wrap the hash */
			rlwimi	r11,r4,10,26,31				/* Move API into pte ID */
			add		r5,r7,r5					/* Point to the PTEG */
			oris	r11,r11,0x8000				/* Slam on valid bit so's we don't match an invalid one */

			li		r9,8						/* Get the number of PTEs to check */
			lwz		r6,0(r5)					/* Preload the virtual half */
			
fndpte:		subi	r9,r9,1						/* Count the pte */
			lwz		r3,4(r5)					/* Get the real half */
			cmplw	cr1,r6,r11					/* Is this what we want? */
			lwz		r6,8(r5)					/* Start to get the next virtual half */
			mr.		r9,r9						/* Any more to try? */
			addi	r5,r5,8						/* Bump to next slot */
			beq		cr1,gotxlate				/* We found what we were looking for... */
			bne+	fndpte						/* Go try the next PTE... */
			
			mtmsr	r10							/* Restore state */
			li		r3,0						/* Show failure */
			isync								/* Purge pipe */
			blr									/* Leave... */

gotxlate:	mtmsr	r10							/* Restore state */
			rlwimi	r3,r4,0,20,31				/* Cram in the page displacement */
			isync								/* Purge pipe */
			blr									/* Return... */



/*
 *			struct blokmap *hw_add_blk(pmap_t pmap, struct blokmap *bmr)
 *	
 *			This is used to add a block mapping entry to the MRU list whose top
 *			node is anchored at bmaps.  This is a real address and is also used as
 *			the lock.
 *
 *			Overlapping areas are not allowed.  If we find one, we return it's address and
 *			expect the upper layers to panic.  We only check this for a debug build...
 *
 */

			.align	5
			.globl	EXT(hw_add_blk)

LEXT(hw_add_blk)

 			mfsprg	r9,2						; Get feature flags 
			lwz		r6,PMAP_PMAPVR(r3)			; Get the v to r translation
			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			xor		r3,r3,r6					; Get real address of bmap anchor
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */
			la		r3,PMAP_BMAPS(r3)			; Point to bmap header
			
			bt		pfNoMSRirb,habNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		habNoMSRx
			
habNoMSR:	
			mr		r9,r0
			mr		r8,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r8
			mr		r0,r9
habNoMSRx:
			
abLck:		lwarx	r9,0,r3						; Get the block map anchor and lock
			rlwinm.	r8,r9,0,31,31				; Is it locked?
			ori		r8,r9,1						; Set the lock
			bne-	abLckw						; Yeah...
			stwcx.	r8,0,r3						; Lock the bmap list
			bne-	abLck						; Someone else was trying, try again...
			b		abSXg						; All done...
			
			.align	4

abLckw:		rlwinm.	r5,r9,0,31,31				; Check if it is still held
			beq+	abLck						; Not no more...
			lwz		r9,0(r3)					; Get lock word again...
			b		abLckw						; Check it out...
			
			.align	5
			
			nop									; Force ISYNC to last instruction in IFETCH
			nop									

abSXg:		rlwinm	r11,r9,0,0,26				; Clear out flags and lock
			isync								; Make sure we have not used anything yet

;
;
;

			lwz		r7,bmstart(r4)				; Get start
			lwz		r8,bmend(r4)				; Get end		
			mr		r2,r11						; Get chain
	
abChk:		mr.		r10,r2						; End of chain?
			beq		abChkD						; Yes, chain is ok...
			lwz		r5,bmstart(r10)				; Get start of current area
			lwz		r6,bmend(r10)				; Get end of current area
			
			cmplw	cr0,r8,r5					; Is the end of the new before the old?
			cmplw	cr1,r8,r6					; Is the end of the new after the old?
			cmplw	cr6,r6,r7					; Is the end of the old before the new?
			cror	cr1_eq,cr0_lt,cr1_gt		; Set cr1_eq if new not in old
			cmplw	cr7,r6,r8					; Is the end of the old after the new?
			lwz		r2,bmnext(r10)				; Get pointer to the next
			cror	cr6_eq,cr6_lt,cr7_gt		; Set cr2_eq if old not in new
			crand	cr1_eq,cr1_eq,cr6_eq		; Set cr1_eq if no overlap
			beq+	cr1,abChk					; Ok check the next...
			
			lwz		r8,blkFlags(r10)			; Get the flags
			rlwinm.	r8,r8,0,blkRembit,blkRembit	; Check the blkRem bit
			beq		abRet						; Is the mapping partially removed
			ori		r10,r10,2					; Indicate that this block is partially removed
abRet:
			stw		r9,0(r3)					; Unlock
			mtmsr	r0							; Restore xlation and rupts
			mr		r3,r10						; Pass back the overlap
			isync								;  
			blr									; Return...

abChkD:		stw		r11,bmnext(r4)				; Chain this on in
			rlwimi	r4,r9,0,27,31				; Copy in locks and flags
			sync								; Make sure that is done
			
			stw		r4,0(r3)					; Unlock and chain the new first one
			mtmsr	r0							; Restore xlation and rupts
			li		r3,0						; Pass back a no failure return code
			isync
			blr									; Return...


/*
 *			struct blokmap *hw_rem_blk(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 *	
 *			This is used to remove a block mapping entry from the list that
 *			is anchored at bmaps.  bmaps is a virtual address and is also used as
 *			the lock.
 *
 *			Note that this function clears a single block that contains
 *			any address within the range sva to eva (inclusive).  To entirely
 *			clear any range, hw_rem_blk must be called repeatedly until it
 *			returns a 0.
 *
 *			The block is removed from the list and all hash table entries
 *			corresponding to the mapped block are invalidated and the TLB
 *			entries are purged.  If the block is large, this could take
 *			quite a while. We need to hash every possible address in the
 *			range and lock down the PCA.
 *
 *			If we attempt to remove a permanent entry, we will not do it.
 *			The block address will be ored with 1 and returned.
 *
 *
 */

			.align	5
			.globl	EXT(hw_rem_blk)

LEXT(hw_rem_blk)

  			mfsprg	r9,2						; Get feature flags
			lwz		r6,PMAP_PMAPVR(r3)			; Get the v to r translation
			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			xor		r3,r3,r6					; Get real address of bmap anchor
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */
			la		r3,PMAP_BMAPS(r3)			; Point to the bmap chain head

			bt		pfNoMSRirb,hrbNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hrbNoMSRx
			
hrbNoMSR:	
			mr		r9,r0
			mr		r8,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r8
			mr		r0,r9
hrbNoMSRx:
			li		r7,0
			cmp		cr5,r0,r7					; Request to invalidate the ptes
			b		rbLck

rbunlink:
			lwz		r4,bmstart(r10)				; Get start of current mapping
			lwz		r5,bmend(r10)				; Get end of current mapping
			cmp		cr5,r3,r3					; Request to unlink the mapping

rbLck:		lwarx	r9,0,r3						; Get the block map anchor and lock
			rlwinm.	r8,r9,0,31,31				; Is it locked?
			ori		r8,r9,1						; Set the lock
			bne-	rbLckw						; Yeah...
			stwcx.	r8,0,r3						; Lock the bmap list
			bne-	rbLck						; Someone else was trying, try again...
			b		rbSXg						; All done...
			
			.align	4

rbLckw:		rlwinm.	r11,r9,0,31,31				; Check if it is still held
			beq+	rbLck						; Not no more...
			lwz		r9,0(r3)					; Get lock word again...
			b		rbLckw						; Check it out...
			
			.align	5
			
			nop									; Force ISYNC to last instruction in IFETCH
			nop									

rbSXg:		rlwinm.	r2,r9,0,0,26				; Clear out flags and lock
			mr		r10,r3						; Keep anchor as previous pointer
			isync								; Make sure we have not used anything yet
			
			beq-	rbMT						; There is nothing in the list
			
rbChk:		mr		r12,r10						; Save the previous
			mr.		r10,r2						; End of chain?
			beq		rbMT						; Yes, nothing to do...
			lwz		r11,bmstart(r10)			; Get start of current area
			lwz		r6,bmend(r10)				; Get end of current area
			
			cmplw	cr0,r5,r11					; Is the end of range before the start of the area?
			cmplw	cr1,r4,r6					; Is the start of range after the end of the area?
			cror	cr1_eq,cr0_lt,cr1_gt		; Set cr1_eq if new not in range
			lwz		r2,bmnext(r10)				; Get the next one
			beq+	cr1,rbChk					; Not this one, check the next...

			cmplw	cr1,r12,r3					; Is the current mapping the first one?

			bne		cr5,rbblkRem				; Do we have to unchain the mapping

			bne		cr1,rbnFirst				; Yes, is this the first mapping?
			rlwimi	r9,r2,0,0,26				; Yes, Change the lock value
			ori		r2,r9,1						; Turn on the lock bit
rbnFirst:
			stw		r2,bmnext(r12)				; Unchain us
			sync
			b		rbDone

rbblkRem:
		
			lwz		r8,blkFlags(r10)			; Get the flags
			
			rlwinm.	r7,r8,0,blkPermbit,blkPermbit	; is this a permanent block?
			
			bne-	rbPerm						; This is permanent, do not remove...

			rlwinm.	r7,r8,0,blkRembit,blkRembit	; is this mapping partially removed

			beq		rbblkRemcont				; If not, check the max size
			lwz		r11,bmcurrent(r10)			; If yes, resume for the current page

			cmp		cr5,r11,r6					; No partial remove left
			beq		cr5, rbpendret				; But there is  a pending remove

rbblkRemcont:
			bne		rbblkRemcont1				; Is it the first remove

			oris	r8,r8,hi16(blkRem)			; Yes
			stw		r8,blkFlags(r10)			; set the blkRem bit in blkFlags

rbblkRemcont1:
			lis		r5,hi16(BLKREMMAX*4096)			; Load maximun size tear down
			ori		r5,r5,lo16(BLKREMMAX*4096)		; Load maximun size tear down
			sub		r7,r6,r11					; Get the remaining size to tear down
			cmp		cr5,r7,r5					; Compare against the maximun size
			ble		cr5,rbfullblk				; If less or equal, go remove the mapping

			add		r7,r11,r5					; Add the max size tear down to the current page
			stw		r7,bmcurrent(r10)			; Update the current page
			subi	r6,r7,1						; Set the current end of the partial tear down
			b		rbcont

rbfullblk:
			stw		r6,bmcurrent(r10)			; Update the current page

rbcont:
			lwz		r8,bmspace(r10)				; Get the VSID
			sync
			stw		r9,0(r3)					; Unlock and chain the new first one
			
			eqv		r4,r4,r4					; Fill the bottom with foxes
			mfspr	r12,sdr1					; Get hash table base and size
			rlwinm	r8,r8,6,0,25				; Align VSID to PTEG
			rlwimi	r4,r12,16,0,15				; Make table size - 1 out of mask
			andc	r12,r12,r4					; Clean up address of hash table
			rlwinm	r5,r11,26,6,25				; Rotate virtual start address into PTEG units
			add		r12,r12,r4					; Point to PCA - 1
			rlwinm	r6,r6,26,6,25				; Rotate virtual end address into PTEG units
			addi	r12,r12,1					; Point to PCA base
			sub		r6,r6,r5					; Get the total number of PTEGs to clear
			cmplw	r6,r4						; See if this wraps all the way around
			blt		rbHash						; Nope, length is right
			subi	r6,r4,32+31					; Back down to correct length
			
rbHash:		rlwinm	r5,r5,0,10,25				; Keep only the page index
			xor		r2,r8,r5					; Hash into table
			and		r2,r2,r4					; Wrap into the table
			add		r2,r2,r12					; Point right at the PCA

rbLcka:		lwarx	r7,0,r2						; Get the PTEG lock
			mr.		r7,r7						; Is it locked?
			bne-	rbLckwa						; Yeah...
			li		r7,1						; Get the locked value
			stwcx.	r7,0,r2						; Take it
			bne-	rbLcka						; Someone else was trying, try again...
			b		rbSXga						; All done... 

rbLckwa:	mr.		r7,r7						; Check if it is already held
			beq+	rbLcka						; It is clear...
			lwz		r7,0(r2)					; Get lock word again...
			b		rbLckwa						; Wait...
			
rbSXga:		isync								; Make sure nothing used yet
			lwz		r7,PCAallo(r2)				; Get the allocation word
			rlwinm.	r11,r7,8,0,7				; Isolate the autogenerated PTEs
			or		r7,r7,r11					; Release the autogen slots
			beq+	rbAintNone					; There are not any here
			mtcrf	0xC0,r11					; Set the branch masks for autogens
			sub		r11,r2,r4					; Move back to the hash table + 1
			rlwinm	r7,r7,0,16,7				; Clear the autogen field
			subi	r11,r11,1					; Point to the PTEG
			stw		r7,PCAallo(r2)				; Update the flags
			li		r7,0						; Get an invalid PTE value

			bf		0,rbSlot1					; No autogen here
			stw		r7,0x00(r11)				; Invalidate PTE
rbSlot1:	bf		1,rbSlot2					; No autogen here
			stw		r7,0x08(r11)				; Invalidate PTE
rbSlot2:	bf		2,rbSlot3					; No autogen here
			stw		r7,0x10(r11)				; Invalidate PTE
rbSlot3:	bf		3,rbSlot4					; No autogen here
			stw		r7,0x18(r11)				; Invalidate PTE
rbSlot4:	bf		4,rbSlot5					; No autogen here
			stw		r7,0x20(r11)				; Invalidate PTE
rbSlot5:	bf		5,rbSlot6					; No autogen here
			stw		r7,0x28(r11)				; Invalidate PTE
rbSlot6:	bf		6,rbSlot7					; No autogen here
			stw		r7,0x30(r11)				; Invalidate PTE
rbSlot7:	bf		7,rbSlotx					; No autogen here
			stw		r7,0x38(r11)				; Invalidate PTE
rbSlotx:

rbAintNone:	li		r7,0						; Clear this out
			sync								; To make SMP happy
			addic.	r6,r6,-64					; Decrement the count
			stw		r7,PCAlock(r2)				; Release the PTEG lock
			addi	r5,r5,64					; Move up by adjusted page number
			bge+	rbHash						; Not done...
	
			sync								; Make sure the memory is quiet
			
;
;			Here we take the easy way out and just purge the entire TLB. This is 
;			certainly faster and definitly easier than blasting just the correct ones
;			in the range, we only need one lock and one TLBSYNC. We would hope
;			that most blocks are more than 64 pages (256K) and on every machine
;			up to Book E, 64 TLBIEs will invalidate the entire table.
;

			li		r5,64						; Get number of TLB entries to purge
			lis		r12,HIGH_ADDR(EXT(tlb_system_lock))	; Get the TLBIE lock
			li		r6,0						; Start at 0
			ori		r12,r12,LOW_ADDR(EXT(tlb_system_lock))	; Grab up the bottom part
						
rbTlbL:		lwarx	r2,0,r12					; Get the TLBIE lock
			mr.		r2,r2						; Is it locked?
			li		r2,1						; Get our lock value
			bne-	rbTlbL						; It is locked, go wait...
			stwcx.	r2,0,r12					; Try to get it
			bne-	rbTlbL						; We was beat...
	
rbTlbN:		addic.	r5,r5,-1					; See if we did them all
			tlbie	r6							; Invalidate it everywhere
			addi	r6,r6,0x1000				; Up to the next page
			bgt+	rbTlbN						; Make sure we have done it all...
			
			mfspr	r5,pvr						; Find out what kind of machine we are
			li		r2,0						; Lock clear value
			
			rlwinm	r5,r5,16,16,31				; Isolate CPU type
			cmplwi	r5,3						; Is this a 603?
			sync								; Make sure all is quiet
			beq-	rbits603a					; It is a 603, skip the tlbsync...
			
			eieio								; Make sure that the tlbie happens first
			tlbsync								; wait for everyone to catch up
			isync								

rbits603a:	sync								; Wait for quiet again
			stw		r2,0(r12)					; Unlock invalidates
			
			sync								; Make sure that is done
			
			ble		cr5,rbunlink				; If all ptes are flush, go unlink the mapping
			mtmsr	r0							; Restore xlation and rupts
			mr		r3,r10						; Pass back the removed block in progress
			ori		r3,r3,2						; Indicate that the block remove isn't completed yet
			isync
			blr									; Return...

rbpendret:
			stw		r9,0(r3)					; Unlock
			mtmsr	r0							; Restore xlation and rupts
			mr		r3,r10						; Pass back the removed block in progress
			ori		r3,r3,2						; Indicate that the block remove isn't completed yet
			isync
			blr									; Return...


rbMT:		stw		r9,0(r3)					; Unlock
			mtmsr	r0							; Restore xlation and rupts
			li		r3,0						; Say we did not find one
			isync
			blr									; Return...
			
rbPerm:		stw		r9,0(r3)					; Unlock
			mtmsr	r0							; Restore xlation and rupts
			ori		r3,r10,1					; Say we did not remove it
			isync
			blr									; Return...

rbDone:		stw     r9,0(r3) 					; Unlock
			mtmsr	r0							; Restore xlation and rupts
			mr      r3,r10 						; Pass back the removed block
			isync
			blr									; Return...

/*
 *			hw_select_mappings(struct mappingflush *mappingflush)
 *
 *			Input: PCA addr
 *			Ouput: up to 8 user mappings
 *
 *			hw_select_mappings() scans every PCA mapping hash lists and select
 *			the last user mapping if it exists.
 *
 */

			.align	5
			.globl	EXT(hw_select_mappings)

LEXT(hw_select_mappings)
			mr		r5,r3						; Get the mapping flush addr
			mfmsr	r12							; Get the MSR 
			rlwinm	r12,r12,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r12,r12,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			mfsprg	r9,2						; Get feature flags
			andi.	r0,r12,0x7FCF				; Disable translation and interruptions
			mtcrf	0x04,r9						; Set the features
			bt		pfNoMSRirb,hvmNoMSR			; No MSR...
			mtmsr	r0
			isync
			b		hvmNoMSRx
hvmNoMSR:
			mr		r3,r0						; Get the new MSR
			li		r0,loadMSR					; Get the MSR setter SC
			sc
hvmNoMSRx:
			mr		r0,r12
			li		r11,1						; Get the locked value 

hvmptegLckx:   
			lwz		r3,MFpcaptr(r5)				; Get the PCA pointer
			lwarx	r10,0,r3					; Get the PTEG lock
			mr.		r10,r10						; Is it locked?
			bne-	hvmptegLckwx				; Yeah...
			stwcx.	r11,0,r3					; Take take it
			bne-	hvmptegLckx					; Someone else was trying, try again...
			b		hvmptegSXgx					; All done...

			.align  4

hvmptegLckwx:
			mr.		r10,r10						; Check if it is already held
			beq+	hvmptegLckx					; It's clear...
			lwz		r10,0(r3)					; Get lock word again...
			b		hvmptegLckwx				; Wait...

			.align  4

hvmptegSXgx:
			isync								; Make sure we haven't used anything yet

			li		r11,8						; set count to 8 

			lwz		r6,PCAhash(r3)				; load the first mapping hash list
			la		r12,PCAhash(r3)				; Point to the mapping hash area
			la		r4,MFmapping(r5)			; Point to the mapping flush mapping area
			li		r7,0						; Load zero
			stw		r7,MFmappingcnt(r5)			; Set the current count to 0
hvmnexthash:
			li		r10,0						; Mapping test

hvmfindmap:
			mr.		r6,r6						; Test if the hash list current pointer is zero
			beq		hvmfindmapret				; Did we hit the end of the hash list
			lwz		r7,mmPTEv(r6)				; Pick up our virtual ID
			rlwinm	r8,r7,5,0,19				; Pick VSID 20 lower bits
			mr.		r8,r8
			beq		hvmfindmapnext				; Skip Kernel VSIDs
			rlwinm	r8,r7,1,0,3					; Extract the Segment index 
			rlwinm	r9,r7,22,4,9				; Extract API 6 upper bits
			or		r8,r8,r9					; Add to the virtual address
			rlwinm	r9,r7,31,6,25				; Pick VSID 19 lower bits
			xor		r9,r9,r3					; Exclusive or with the PCA address
			rlwinm	r9,r9,6,10,19				; Extract API 10 lower bits
			or		r8,r8,r9					; Add to the virtual address

			stw		r8,4(r4)					; Store the virtual address
			lwz		r8,mmpmap(r6)				; Get the pmap
			stw		r8,0(r4)					; Store the pmap
			li		r10,1						; Found one

hvmfindmapnext:
			lwz		r6,mmhashnext(r6)			; Pick up next mapping block
			b		hvmfindmap					; Scan the next mapping
hvmfindmapret:
			mr.		r10,r10						; Found mapping
			beq		hvmnexthashprep				; If not, do not update the mappingflush array
			lwz		r7,MFmappingcnt(r5)			; Get the current count
			addi	r7,r7,1						; Increment the current count
			stw		r7,MFmappingcnt(r5)			; Store the current count
			addi	r4,r4,MFmappingSize			; Point to the next mapping flush entry
hvmnexthashprep:
			addi	r12,r12,4					; Load the next hash list
			lwz		r6,0(r12)					; Load the next hash list entry
			subi	r11,r11,1					; Decrement hash list index 
			mr.		r11,r11						; Test for a remaining hash list
			bne		hvmnexthash					; Loop to scan the next hash list

			li		r10,0
			stw		r10,0(r3)					; Unlock the hash list
			mtmsr	r0							; Restore translation and interruptions
			isync
			blr

/*
 *			vm_offset_t hw_cvp_blk(pmap_t pmap, vm_offset_t va)
 *	
 *			This is used to translate a virtual address within a block mapping entry
 *			to a physical address.  If not found, 0 is returned.
 *
 */

			.align	5
			.globl	EXT(hw_cvp_blk)

LEXT(hw_cvp_blk)

 			mfsprg	r9,2						; Get feature flags
 			lwz		r6,PMAP_PMAPVR(r3)			; Get the v to r translation
			mfmsr	r0							/* Save the MSR  */
			rlwinm	r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Clear interruptions */
			mtcrf	0x04,r9						; Set the features			
			xor		r3,r3,r6					; Get real address of bmap anchor
			rlwinm	r12,r12,0,28,25				/* Clear IR and DR */
			la		r3,PMAP_BMAPS(r3)			; Point to chain header

			bt		pfNoMSRirb,hcbNoMSR			; No MSR...

			mtmsr	r12							; Translation and all off
			isync								; Toss prefetch
			b		hcbNoMSRx
			
hcbNoMSR:	
			mr		r9,r0
			mr		r8,r3
			li		r0,loadMSR					; Get the MSR setter SC
			mr		r3,r12						; Get new MSR
			sc									; Set it
			mr		r3,r8
			mr		r0,r9
hcbNoMSRx:

cbLck:		lwarx	r9,0,r3						; Get the block map anchor and lock
			rlwinm.	r8,r9,0,31,31				; Is it locked?
			ori		r8,r9,1						; Set the lock
			bne-	cbLckw						; Yeah...
			stwcx.	r8,0,r3						; Lock the bmap list
			bne-	cbLck						; Someone else was trying, try again...
			b		cbSXg						; All done...
			
			.align	4

cbLckw:		rlwinm.	r5,r9,0,31,31				; Check if it is still held
			beq+	cbLck						; Not no more...
			lwz		r9,0(r3)					; Get lock word again...
			b		cbLckw						; Check it out...
			
			.align	5
			
			nop									; Force ISYNC to last instruction in IFETCH
			nop
			nop
			nop
			nop

cbSXg:		rlwinm.	r11,r9,0,0,26				; Clear out flags and lock
			li		r2,0						; Assume we do not find anything			
			isync								; Make sure we have not used anything yet

cbChk:		mr.		r11,r11						; Is there more?
			beq-	cbDone						; No more...
			lwz		r5,bmstart(r11)				; Get the bottom of range
			lwz		r12,bmend(r11)				; Get the top of range
			cmplw	cr0,r4,r5					; Are we before the entry?
			cmplw	cr1,r4,r12					; Are we after of the entry?
			cror	cr1_eq,cr0_lt,cr1_gt		; Set cr1_eq if new not in range
			beq-	cr1,cbNo					; We are not in the range...

			lwz		r2,bmPTEr(r11)				; Get the real part of the PTE
			sub		r5,r4,r5					; Get offset into area
			rlwinm	r2,r2,0,0,19				; Clean out everything but the page
			add		r2,r2,r5					; Adjust the real address

cbDone:		stw		r9,0(r3)					; Unlock it, we are done with it (no sync needed)
			mtmsr	r0							; Restore translation and interrupts...
			isync								; Make sure it is on
			mr		r3,r2						; Set return physical address
			blr									; Leave...
			
			.align	5
			
cbNo:		lwz		r11,bmnext(r11)				; Link next
			b		cbChk						; Check it out...
			
			
/*
 *			hw_set_user_space(pmap) 
 *			hw_set_user_space_dis(pmap) 
 *
 * 			Indicate whether memory space needs to be switched.
 *			We really need to turn off interrupts here, because we need to be non-preemptable
 *
 *			hw_set_user_space_dis is used when interruptions are already disabled. Mind the
 *			register usage here.   The VMM switch code in vmachmon.s that calls this
 *			know what registers are in use.  Check that if these change.
 */


	
			.align	5
			.globl	EXT(hw_set_user_space)

LEXT(hw_set_user_space)

			mfmsr	r10							/* Get the current MSR */
			rlwinm	r10,r10,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r10,r10,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			rlwinm	r9,r10,0,MSR_EE_BIT+1,MSR_EE_BIT-1	/* Turn off 'rupts */
			mtmsr	r9							/* Disable 'em */
 			lwz		r7,PMAP_PMAPVR(r3)			; Get the v to r translation
			lwz		r4,PMAP_SPACE(r3)			; Get the space
			mfsprg	r6,0						/* Get the per_proc_info address */
			xor		r3,r3,r7					; Get real address of bmap anchor
			stw		r4,PP_USERSPACE(r6)			/* Show our new address space */
			stw		r3,PP_USERPMAP(r6)			; Show our real pmap address
			mtmsr	r10							/* Restore interruptions */
			blr									/* Return... */
	
			.align	5
			.globl	EXT(hw_set_user_space_dis)

LEXT(hw_set_user_space_dis)

 			lwz		r7,PMAP_PMAPVR(r3)			; Get the v to r translation
			lwz		r4,PMAP_SPACE(r3)			; Get the space
			mfsprg	r6,0						; Get the per_proc_info address
			xor		r3,r3,r7					; Get real address of bmap anchor
			stw		r4,PP_USERSPACE(r6)			; Show our new address space
			stw		r3,PP_USERPMAP(r6)			; Show our real pmap address
			blr									; Return...
	

/*			struct mapping *hw_cpv(struct mapping *mp) - Converts a physcial mapping CB address to virtual
 *
 */

			.align	5
			.globl	EXT(hw_cpv)

LEXT(hw_cpv)
			
			rlwinm.	r4,r3,0,0,19				; Round back to the mapping block allocation control block
			mfmsr	r10							; Get the current MSR
			beq-	hcpvret						; Skip if we are passed a 0...
			rlwinm	r10,r10,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r10,r10,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			andi.	r9,r10,0x7FEF				; Turn off interrupts and data translation
			mtmsr	r9							; Disable DR and EE
			isync
			
			lwz		r4,mbvrswap(r4)				; Get the conversion value
			mtmsr	r10							; Interrupts and DR back on
			isync
			xor		r3,r3,r4					; Convert to physical

hcpvret:	rlwinm	r3,r3,0,0,26				; Clean out any flags
			blr


/*			struct mapping *hw_cvp(struct mapping *mp) - Converts a virtual mapping CB address to physcial
 *
 *			Translation must be on for this
 *
 */

			.align	5
			.globl	EXT(hw_cvp)

LEXT(hw_cvp)
			
			rlwinm	r4,r3,0,0,19				; Round back to the mapping block allocation control block			
			rlwinm	r3,r3,0,0,26				; Clean out any flags
			lwz		r4,mbvrswap(r4)				; Get the conversion value
			xor		r3,r3,r4					; Convert to virtual
			blr


/*			int mapalc(struct mappingblok *mb) - Finds, allocates, and checks a free mapping entry in a block
 *
 *			Lock must already be held on mapping block list
 *			returns 0 if all slots filled.
 *			returns n if a slot is found and it is not the last
 *			returns -n if a slot os found and it is the last
 *			when n and -n are returned, the corresponding bit is cleared
 *
 */

			.align	5
			.globl	EXT(mapalc)

LEXT(mapalc)
			
			lwz		r4,mbfree(r3)				; Get the first mask 
			lis		r0,0x8000					; Get the mask to clear the first free bit
			lwz		r5,mbfree+4(r3)				; Get the second mask 
			mr		r12,r3						; Save the return
			cntlzw	r8,r4						; Get first free field
			lwz		r6,mbfree+8(r3)				; Get the third mask 
			srw.	r9,r0,r8					; Get bit corresponding to first free one
			lwz		r7,mbfree+12(r3)			; Get the fourth mask 
			cntlzw	r10,r5						; Get first free field in second word
			andc	r4,r4,r9					; Turn it off
			bne		malcfnd0					; Found one...
			
			srw.	r9,r0,r10					; Get bit corresponding to first free one in second word
			cntlzw	r11,r6						; Get first free field in third word
			andc	r5,r5,r9					; Turn it off
			bne		malcfnd1					; Found one...
			
			srw.	r9,r0,r11					; Get bit corresponding to first free one in third word
			cntlzw	r10,r7						; Get first free field in fourth word
			andc	r6,r6,r9					; Turn it off
			bne		malcfnd2					; Found one...
			
			srw.	r9,r0,r10					; Get bit corresponding to first free one in second word
			li		r3,0						; Assume abject failure
			andc	r7,r7,r9					; Turn it off
			beqlr								; There are none any left...
			
			addi	r3,r10,96					; Set the correct bit number
			stw		r7,mbfree+12(r12)			; Actually allocate the slot
			
mapafin:	or		r4,r4,r5					; Merge the first two allocation maps
			or		r6,r6,r7					; Then the last two
			or.		r4,r4,r6					; Merge both halves
			bnelr+								; Return if some left for next time...
			
			neg		r3,r3						; Indicate we just allocated the last one
			blr									; Leave...
			
malcfnd0:	stw		r4,mbfree(r12)				; Actually allocate the slot
			mr		r3,r8						; Set the correct bit number
			b		mapafin						; Exit now...
			
malcfnd1:	stw		r5,mbfree+4(r12)			; Actually allocate the slot
			addi	r3,r10,32					; Set the correct bit number
			b		mapafin						; Exit now...
			
malcfnd2:	stw		r6,mbfree+8(r12)			; Actually allocate the slot
			addi	r3,r11,64					; Set the correct bit number
			b		mapafin						; Exit now...
			

/*
 * Log out all memory usage
 */

			.align	5
			.globl	EXT(logmem)

LEXT(logmem)

			mfmsr	r2							; Get the MSR	
			lis		r10,hi16(EXT(DebugWork))		; High part of area
			rlwinm	r2,r2,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			lis		r12,hi16(EXT(mem_actual))	; High part of actual
			rlwinm	r2,r2,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			andi.	r0,r2,0x7FCF				; Interrupts and translation off
			ori		r10,r10,lo16(EXT(DebugWork))	; Get the entry
			mtmsr	r0							; Turn stuff off
			ori		r12,r12,lo16(EXT(mem_actual))	; Get the actual
			li		r0,1						; Get a one
	
			isync

			stw		r0,4(r10)					; Force logging off
			lwz		r0,0(r12)					; Get the end of memory
			
			lis		r12,hi16(EXT(mem_size))		; High part of defined memory
			ori		r12,r12,lo16(EXT(mem_size))	; Low part of defined memory
			lwz		r12,0(r12)					; Make it end of defined
			
			cmplw	r0,r12						; Is there room for the data?
			ble-	logmemexit					; No, do not even try...

			stw		r12,0(r12)					; Set defined memory size
			stw		r0,4(r12)					; Set the actual amount of memory
			
			lis		r3,hi16(EXT(hash_table_base))	; Hash table address
			lis		r4,hi16(EXT(hash_table_size))	; Hash table size
			lis		r5,hi16(EXT(pmap_mem_regions))	; Memory regions
			lis		r6,hi16(EXT(mapCtl))		; Mappings
			ori		r3,r3,lo16(EXT(hash_table_base))	
			ori		r4,r4,lo16(EXT(hash_table_size))	
			ori		r5,r5,lo16(EXT(pmap_mem_regions))	
			ori		r6,r6,lo16(EXT(mapCtl))	
			lwz		r3,0(r3)
			lwz		r4,0(r4)
			lwz		r5,4(r5)					; Get the pointer to the phys_ent table
			lwz		r6,0(r6)					; Get the pointer to the current mapping block
			stw		r3,8(r12)					; Save the hash table address
			stw		r4,12(r12)					; Save the hash table size
			stw		r5,16(r12)					; Save the physent pointer
			stw		r6,20(r12)					; Save the mappings
			
			addi	r11,r12,0x1000				; Point to area to move hash table and PCA
			
			add		r4,r4,r4					; Double size for both
			
copyhash:	lwz		r7,0(r3)					; Copy both of them
			lwz		r8,4(r3)
			lwz		r9,8(r3)
			lwz		r10,12(r3)
			subic.	r4,r4,0x10
			addi	r3,r3,0x10
			stw		r7,0(r11)
			stw		r8,4(r11)
			stw		r9,8(r11)
			stw		r10,12(r11)
			addi	r11,r11,0x10
			bgt+	copyhash
			
			rlwinm	r4,r12,20,12,31				; Get number of phys_ents

copyphys:	lwz		r7,0(r5)					; Copy physents
			lwz		r8,4(r5)
			subic.	r4,r4,1
			addi	r5,r5,8
			stw		r7,0(r11)
			stw		r8,4(r11)
			addi	r11,r11,8
			bgt+	copyphys
			
			addi	r11,r11,4095				; Round up to next page
			rlwinm	r11,r11,0,0,19

			lwz		r4,4(r6)					; Get the size of the mapping area
			
copymaps:	lwz		r7,0(r6)					; Copy the mappings
			lwz		r8,4(r6)
			lwz		r9,8(r6)
			lwz		r10,12(r6)
			subic.	r4,r4,0x10
			addi	r6,r6,0x10
			stw		r7,0(r11)
			stw		r8,4(r11)
			stw		r9,8(r11)
			stw		r10,12(r11)
			addi	r11,r11,0x10
			bgt+	copymaps

			sub		r11,r11,r12					; Get the total length we saved
			stw		r11,24(r12)					; Save the size
			
logmemexit:	mtmsr	r2							; Back to normal
			li		r3,0
			isync
			blr