+ .globl EXT(mapalc2)
+LEXT(mapalc2)
+ lwz r4,mbfree(r3) ; Get the first mask
+ lis r0,0x8000 ; Get the mask to clear the first free bit
+ lwz r5,mbfree+4(r3) ; Get the second mask
+ mr r12,r3 ; Save the block ptr
+ slwi r6,r4,1 ; shift first word over
+ and r6,r4,r6 ; lite start of double bit runs in 1st word
+ slwi r7,r5,1 ; shift 2nd word over
+ cntlzw r3,r6 ; Get first free 2-bit run in 1st word
+ and r7,r5,r7 ; lite start of double bit runs in 2nd word
+ srw. r9,r0,r3 ; Get bit corresponding to first run in 1st word
+ cntlzw r10,r7 ; Get first free field in second word
+ srwi r11,r9,1 ; shift over for 2nd bit in 1st word
+ andc r4,r4,r9 ; Turn off 1st bit in 1st word
+ andc r4,r4,r11 ; turn off 2nd bit in 1st word
+ bne mapalc2a ; Found two consecutive free bits in 1st word
+
+ srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
+ li r3,0 ; assume failure
+ srwi r11,r9,1 ; get mask for 2nd bit
+ andc r5,r5,r9 ; Turn off 1st bit in 2nd word
+ andc r5,r5,r11 ; turn off 2nd bit in 2nd word
+ beq-- mapalc2c ; There are no runs of 2 bits in 2nd word either
+ addi r3,r10,32 ; set the correct number
+
+mapalc2a:
+ or. r0,r4,r5 ; any more bits set?
+ stw r4,mbfree(r12) ; update bitmasks
+ stw r5,mbfree+4(r12)
+ slwi r6,r3,6 ; get (n * mpBasicSize), ie offset of mapping in block
+ addi r7,r6,32
+ addi r8,r6,64
+ addi r9,r6,96
+ dcbz r6,r12 ; zero out the 128-byte mapping
+ dcbz r7,r12 ; we use the slow 32-byte dcbz even on 64-bit machines
+ dcbz r8,r12 ; because the mapping may not be 128-byte aligned
+ dcbz r9,r12
+
+ bnelr++ ; return if another bit remains set
+
+ neg r3,r3 ; indicate we just returned the last bit
+ blr
+
+mapalc2c:
+ rlwinm r7,r5,1,31,31 ; move bit 0 of 2nd word to bit 31
+ and. r0,r4,r7 ; is the 2-bit field that spans the 2 words free?
+ beqlr ; no, we failed
+ rlwinm r4,r4,0,0,30 ; yes, turn off bit 31 of 1st word
+ rlwinm r5,r5,0,1,31 ; turn off bit 0 of 2nd word
+ li r3,31 ; get index of this field
+ b mapalc2a
+
+
+;
+; This routine initialzes the hash table and PCA.
+; It is done here because we may need to be 64-bit to do it.
+;
+
+ .align 5
+ .globl EXT(hw_hash_init)
+
+LEXT(hw_hash_init)
+
+ mfsprg r10,2 ; Get feature flags
+ lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
+ mtcrf 0x02,r10 ; move pf64Bit to cr6
+ lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
+ lis r4,0xFF01 ; Set all slots free and start steal at end
+ ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
+ ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
+
+ lwz r12,0(r12) ; Get hash table size
+ li r3,0 ; Get start
+ bt++ pf64Bitb,hhiSF ; skip if 64-bit (only they take the hint)
+
+ lwz r11,4(r11) ; Get hash table base
+
+hhiNext32: cmplw r3,r12 ; Have we reached the end?
+ bge- hhiCPCA32 ; Yes...
+ dcbz r3,r11 ; Clear the line
+ addi r3,r3,32 ; Next one...
+ b hhiNext32 ; Go on...
+
+hhiCPCA32: rlwinm r12,r12,28,4,29 ; Get number of slots * 4
+ li r3,-4 ; Displacement to first PCA entry
+ neg r12,r12 ; Get negative end of PCA
+
+hhiNPCA32: stwx r4,r3,r11 ; Initialize the PCA entry
+ subi r3,r3,4 ; Next slot
+ cmpw r3,r12 ; Have we finished?
+ bge+ hhiNPCA32 ; Not yet...
+ blr ; Leave...
+
+hhiSF: mfmsr r9 ; Save the MSR
+ li r8,1 ; Get a 1
+ mr r0,r9 ; Get a copy of the MSR
+ ld r11,0(r11) ; Get hash table base
+ rldimi r0,r8,63,MSR_SF_BIT ; Set SF bit (bit 0)
+ mtmsrd r0 ; Turn on SF
+ isync
+
+
+hhiNext64: cmpld r3,r12 ; Have we reached the end?
+ bge-- hhiCPCA64 ; Yes...
+ dcbz128 r3,r11 ; Clear the line
+ addi r3,r3,128 ; Next one...
+ b hhiNext64 ; Go on...
+
+hhiCPCA64: rlwinm r12,r12,27,5,29 ; Get number of slots * 4
+ li r3,-4 ; Displacement to first PCA entry
+ neg r12,r12 ; Get negative end of PCA
+
+hhiNPCA64: stwx r4,r3,r11 ; Initialize the PCA entry
+ subi r3,r3,4 ; Next slot
+ cmpd r3,r12 ; Have we finished?
+ bge++ hhiNPCA64 ; Not yet...
+
+ mtmsrd r9 ; Turn off SF if it was off
+ isync
+ blr ; Leave...
+
+
+;
+; This routine sets up the hardware to start translation.
+; Note that we do NOT start translation.
+;
+
+ .align 5
+ .globl EXT(hw_setup_trans)
+
+LEXT(hw_setup_trans)
+
+ mfsprg r11,0 ; Get the per_proc block
+ mfsprg r12,2 ; Get feature flags
+ li r0,0 ; Get a 0
+ li r2,1 ; And a 1
+ mtcrf 0x02,r12 ; Move pf64Bit to cr6
+ stw r0,validSegs(r11) ; Make sure we think all SR/STEs are invalid
+ stw r0,validSegs+4(r11) ; Make sure we think all SR/STEs are invalid, part deux
+ sth r2,ppInvSeg(r11) ; Force a reload of the SRs
+ sth r0,ppCurSeg(r11) ; Set that we are starting out in kernel
+
+ bt++ pf64Bitb,hstSF ; skip if 64-bit (only they take the hint)
+
+ li r9,0 ; Clear out a register
+ sync
+ isync
+ mtdbatu 0,r9 ; Invalidate maps
+ mtdbatl 0,r9 ; Invalidate maps
+ mtdbatu 1,r9 ; Invalidate maps
+ mtdbatl 1,r9 ; Invalidate maps
+ mtdbatu 2,r9 ; Invalidate maps
+ mtdbatl 2,r9 ; Invalidate maps
+ mtdbatu 3,r9 ; Invalidate maps
+ mtdbatl 3,r9 ; Invalidate maps
+
+ mtibatu 0,r9 ; Invalidate maps
+ mtibatl 0,r9 ; Invalidate maps
+ mtibatu 1,r9 ; Invalidate maps
+ mtibatl 1,r9 ; Invalidate maps
+ mtibatu 2,r9 ; Invalidate maps
+ mtibatl 2,r9 ; Invalidate maps
+ mtibatu 3,r9 ; Invalidate maps
+ mtibatl 3,r9 ; Invalidate maps
+
+ lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
+ lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
+ ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
+ ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
+ lwz r11,4(r11) ; Get hash table base
+ lwz r12,0(r12) ; Get hash table size
+ subi r12,r12,1 ; Back off by 1
+ rlwimi r11,r12,16,23,31 ; Stick the size into the sdr1 image
+
+ mtsdr1 r11 ; Ok, we now have the hash table set up
+ sync
+
+ li r12,invalSpace ; Get the invalid segment value
+ li r10,0 ; Start low
+
+hstsetsr: mtsrin r12,r10 ; Set the SR
+ addis r10,r10,0x1000 ; Bump the segment
+ mr. r10,r10 ; Are we finished?
+ bne+ hstsetsr ; Nope...
+ sync
+ blr ; Return...
+
+;
+; 64-bit version
+;
+
+hstSF: lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
+ lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
+ ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
+ ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
+ ld r11,0(r11) ; Get hash table base
+ lwz r12,0(r12) ; Get hash table size
+ cntlzw r10,r12 ; Get the number of bits
+ subfic r10,r10,13 ; Get the extra bits we need
+ or r11,r11,r10 ; Add the size field to SDR1
+
+ mtsdr1 r11 ; Ok, we now have the hash table set up
+ sync
+
+ li r0,0 ; Set an SLB slot index of 0
+ slbia ; Trash all SLB entries (except for entry 0 that is)
+ slbmfee r7,r0 ; Get the entry that is in SLB index 0
+ rldicr r7,r7,0,35 ; Clear the valid bit and the rest
+ slbie r7 ; Invalidate it
+
+ blr ; Return...
+
+
+;
+; This routine turns on translation for the first time on a processor
+;
+
+ .align 5
+ .globl EXT(hw_start_trans)
+
+LEXT(hw_start_trans)
+
+
+ mfmsr r10 ; Get the msr
+ ori r10,r10,lo16(MASK(MSR_IR) | MASK(MSR_DR)) ; Turn on translation
+
+ mtmsr r10 ; Everything falls apart here
+ isync
+
+ blr ; Back to it.
+
+
+
+;
+; This routine validates a segment register.
+; hw_map_seg(pmap_t pmap, addr64_t seg, addr64_t va)
+;
+; r3 = virtual pmap
+; r4 = segment[0:31]
+; r5 = segment[32:63]
+; r6 = va[0:31]
+; r7 = va[32:63]
+;
+; Note that we transform the addr64_t (long long) parameters into single 64-bit values.
+; Note that there is no reason to apply the key modifier here because this is only
+; used for kernel accesses.
+;
+
+ .align 5
+ .globl EXT(hw_map_seg)
+
+LEXT(hw_map_seg)
+
+ lwz r0,pmapSpace(r3) ; Get the space, we will need it soon
+ lwz r9,pmapFlags(r3) ; Get the flags for the keys now
+ mfsprg r10,2 ; Get feature flags
+
+;
+; Note: the following code would problably be easier to follow if I split it,
+; but I just wanted to see if I could write this to work on both 32- and 64-bit
+; machines combined.
+;
+
+;
+; Here we enter with va[0:31] in r6[0:31] (or r6[32:63] on 64-bit machines)
+; and va[32:63] in r7[0:31] (or r7[32:63] on 64-bit machines)
+
+ rlwinm r4,r4,0,1,0 ; Copy seg[0:31] into r4[0;31] - no-op for 32-bit
+ rlwinm r7,r7,18,14,17 ; Slide va[32:35] east to just west of space ID
+ mtcrf 0x02,r10 ; Move pf64Bit and pfNoMSRirb to cr5 and 6
+ srwi r8,r6,14 ; Slide va[0:17] east to just west of the rest
+ rlwimi r7,r6,18,0,13 ; Slide va[18:31] east to just west of slid va[32:25]
+ rlwimi r0,r0,14,4,17 ; Dup address space ID above itself
+ rlwinm r8,r8,0,1,0 ; Dup low part into high (does nothing on 32-bit machines)
+ rlwinm r2,r0,28,0,31 ; Rotate rotate low nybble to top of low half
+ rlwimi r2,r2,0,1,0 ; Replicate bottom 32 into top 32
+ rlwimi r8,r7,0,0,31 ; Join va[0:17] with va[18:35] (just like mr on 32-bit machines)
+
+ rlwimi r2,r0,0,4,31 ; We should now have 4 copies of the space
+ ; concatenated together. There is garbage
+ ; at the top for 64-bit but we will clean
+ ; that out later.
+ rlwimi r4,r5,0,0,31 ; Copy seg[32:63] into r4[32:63] - just like mr for 32-bit
+
+
+;
+; Here we exit with va[0:35] shifted into r8[14:51], zeros elsewhere, or
+; va[18:35] shifted into r8[0:17], zeros elsewhere on 32-bit machines
+;
+
+;
+; What we have now is:
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; r2 = |xxxx0000|AAAAAAAA|AAAAAABB|BBBBBBBB|BBBBCCCC|CCCCCCCC|CCDDDDDD|DDDDDDDD| - hash value
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; 0 0 1 2 3 - for 32-bit machines
+; 0 8 6 4 1
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; r8 = |00000000|000000SS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SS000000|00000000| - shifted and cleaned EA
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; 0 0 1 2 3 - for 32-bit machines
+; 0 8 6 4 1
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; r4 = |SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSS0000|00000000|00000000|00000000| - Segment
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; 0 0 1 2 3 - for 32-bit machines
+; 0 8 6 4 1
+
+
+ xor r8,r8,r2 ; Calculate VSID
+
+ bf-- pf64Bitb,hms32bit ; Skip out if 32-bit...
+ mfsprg r12,0 ; Get the per_proc
+ li r0,1 ; Prepare to set bit 0 (also to clear EE)
+ mfmsr r6 ; Get current MSR
+ li r2,MASK(MSR_IR)|MASK(MSR_DR) ; Get the translation bits
+ mtmsrd r0,1 ; Set only the EE bit to 0
+ rlwinm r6,r6,0,MSR_EE_BIT,MSR_EE_BIT ; See if EE bit is on
+ mfmsr r11 ; Get the MSR right now, after disabling EE
+ andc r2,r11,r2 ; Turn off translation now
+ rldimi r2,r0,63,0 ; Get bit 64-bit turned on
+ or r11,r11,r6 ; Turn on the EE bit if it was on
+ mtmsrd r2 ; Make sure translation and EE are off and 64-bit is on
+ isync ; Hang out a bit
+
+ ld r6,validSegs(r12) ; Get the valid SLB entry flags
+ sldi r9,r9,9 ; Position the key and noex bit
+
+ rldimi r5,r8,12,0 ; Form the VSID/key
+
+ not r3,r6 ; Make valids be 0s
+
+ cntlzd r7,r3 ; Find a free SLB
+ cmplwi r7,63 ; Did we find a free SLB entry?
+
+ slbie r4 ; Since this ESID may still be in an SLBE, kill it
+
+ oris r4,r4,0x0800 ; Turn on the valid bit in ESID
+ addi r7,r7,1 ; Make sure we skip slb 0
+ blt++ hmsFreeSeg ; Yes, go load it...
+
+;
+; No free SLB entries, select one that is in use and invalidate it
+;
+ lwz r2,ppSegSteal(r12) ; Get the next slot to steal
+ addi r7,r2,pmapSegCacheUse+1 ; Select stealee from non-cached slots only
+ addi r2,r2,1 ; Set next slot to steal
+ slbmfee r3,r7 ; Get the entry that is in the selected spot
+ subi r8,r2,64-(pmapSegCacheUse+1) ; Force steal to wrap
+ rldicr r3,r3,0,35 ; Clear the valid bit and the rest
+ srawi r8,r8,31 ; Get -1 if steal index still in range
+ slbie r3 ; Invalidate the in-use SLB entry
+ and r2,r2,r8 ; Reset steal index when it should wrap
+ isync ;
+
+ stw r2,ppSegSteal(r12) ; Set the next slot to steal
+;
+; We are now ready to stick the SLB entry in the SLB and mark it in use
+;
+
+hmsFreeSeg: subi r2,r7,1 ; Adjust for skipped slb 0
+ rldimi r4,r7,0,58 ; Copy in the SLB entry selector
+ srd r0,r0,r2 ; Set bit mask for allocation
+ rldicl r5,r5,0,15 ; Clean out the unsupported bits
+ or r6,r6,r0 ; Turn on the allocation flag
+
+ slbmte r5,r4 ; Make that SLB entry
+
+ std r6,validSegs(r12) ; Mark as valid
+ mtmsrd r11 ; Restore the MSR
+ isync
+ blr ; Back to it...
+
+ .align 5
+
+hms32bit:
+ mfsprg r12,1 ; Get the current activation
+ lwz r12,ACT_PER_PROC(r12) ; Get the per_proc block
+ rlwinm r8,r8,0,8,31 ; Clean up the VSID
+ rlwinm r2,r4,4,28,31 ; Isolate the segment we are setting
+ lis r0,0x8000 ; Set bit 0
+ rlwimi r8,r9,28,1,3 ; Insert the keys and N bit
+ srw r0,r0,r2 ; Get bit corresponding to SR
+ addi r7,r12,validSegs ; Point to the valid segment flags directly
+
+ mtsrin r8,r4 ; Set the actual SR
+ isync ; Need to make sure this is done
+
+hmsrupt: lwarx r6,0,r7 ; Get and reserve the valid segment flags
+ or r6,r6,r0 ; Show that SR is valid
+ stwcx. r6,0,r7 ; Set the valid SR flags
+ bne-- hmsrupt ; Had an interrupt, need to get flags again...
+
+ blr ; Back to it...
+
+
+;
+; This routine invalidates a segment register.
+;
+
+ .align 5
+ .globl EXT(hw_blow_seg)
+
+LEXT(hw_blow_seg)
+
+ mfsprg r10,2 ; Get feature flags
+ mtcrf 0x02,r10 ; move pf64Bit and pfNoMSRirb to cr5 and 6
+
+ rlwinm r9,r4,0,0,3 ; Save low segment address and make sure it is clean
+
+ bf-- pf64Bitb,hbs32bit ; Skip out if 32-bit...
+
+ li r0,1 ; Prepare to set bit 0 (also to clear EE)
+ mfmsr r6 ; Get current MSR
+ li r2,MASK(MSR_IR)|MASK(MSR_DR) ; Get the translation bits
+ mtmsrd r0,1 ; Set only the EE bit to 0
+ rlwinm r6,r6,0,MSR_EE_BIT,MSR_EE_BIT ; See if EE bit is on
+ mfmsr r11 ; Get the MSR right now, after disabling EE
+ andc r2,r11,r2 ; Turn off translation now
+ rldimi r2,r0,63,0 ; Get bit 64-bit turned on
+ or r11,r11,r6 ; Turn on the EE bit if it was on
+ mtmsrd r2 ; Make sure translation and EE are off and 64-bit is on
+ isync ; Hang out a bit
+
+ rldimi r9,r3,32,0 ; Insert the top part of the ESID
+
+ slbie r9 ; Invalidate the associated SLB entry
+
+ mtmsrd r11 ; Restore the MSR
+ isync
+ blr ; Back to it.
+
+ .align 5
+
+hbs32bit:
+ mfsprg r12,1 ; Get the current activation
+ lwz r12,ACT_PER_PROC(r12) ; Get the per_proc block
+ addi r7,r12,validSegs ; Point to the valid segment flags directly
+ lwarx r4,0,r7 ; Get and reserve the valid segment flags
+ rlwinm r6,r9,4,28,31 ; Convert segment to number
+ lis r2,0x8000 ; Set up a mask
+ srw r2,r2,r6 ; Make a mask
+ and. r0,r4,r2 ; See if this is even valid
+ li r5,invalSpace ; Set the invalid address space VSID
+ beqlr ; Leave if already invalid...
+
+ mtsrin r5,r9 ; Slam the segment register
+ isync ; Need to make sure this is done
+
+hbsrupt: andc r4,r4,r2 ; Clear the valid bit for this segment
+ stwcx. r4,0,r7 ; Set the valid SR flags
+ beqlr++ ; Stored ok, no interrupt, time to leave...
+
+ lwarx r4,0,r7 ; Get and reserve the valid segment flags again
+ b hbsrupt ; Try again...
+
+;
+; This routine invadates the entire pmap segment cache
+;
+; Translation is on, interrupts may or may not be enabled.
+;
+
+ .align 5
+ .globl EXT(invalidateSegs)
+
+LEXT(invalidateSegs)
+
+ la r10,pmapCCtl(r3) ; Point to the segment cache control
+ eqv r2,r2,r2 ; Get all foxes
+
+isInv: lwarx r4,0,r10 ; Get the segment cache control value
+ rlwimi r4,r2,0,0,15 ; Slam in all invalid bits
+ rlwinm. r0,r4,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
+ bne-- isInv0 ; Yes, try again...
+
+ stwcx. r4,0,r10 ; Try to invalidate it
+ bne-- isInv ; Someone else just stuffed it...
+ blr ; Leave...
+
+
+isInv0: li r4,lgKillResv ; Get reservation kill zone
+ stwcx. r4,0,r4 ; Kill reservation
+
+isInv1: lwz r4,pmapCCtl(r3) ; Get the segment cache control
+ rlwinm. r0,r4,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
+ bne-- isInv ; Nope...
+ b isInv1 ; Still locked do it again...
+
+;
+; This routine switches segment registers between kernel and user.
+; We have some assumptions and rules:
+; We are in the exception vectors
+; pf64Bitb is set up
+; R3 contains the MSR we going to
+; We can not use R4, R13, R20, R21, R25, R26, R29
+; R13 is the savearea
+; R29 has the per_proc
+;
+; We return R3 as 0 if we did not switch between kernel and user
+; We also maintain and apply the user state key modifier used by VMM support;
+; If we go to the kernel it is set to 0, otherwise it follows the bit
+; in spcFlags.
+;
+
+ .align 5
+ .globl EXT(switchSegs)
+
+LEXT(switchSegs)
+
+ lwz r22,ppInvSeg(r29) ; Get the ppInvSeg (force invalidate) and ppCurSeg (user or kernel segments indicator)
+ lwz r9,spcFlags(r29) ; Pick up the special user state flags
+ rlwinm r2,r3,MSR_PR_BIT+1,31,31 ; Isolate the problem mode bit
+ rlwinm r3,r3,MSR_RI_BIT+1,31,31 ; Isolate the recoverable interrupt bit
+ lis r8,hi16(EXT(kernel_pmap_phys)) ; Assume kernel
+ or r2,r2,r3 ; This will 1 if we will be using user segments
+ li r3,0 ; Get a selection mask
+ cmplw r2,r22 ; This will be EQ if same state and not ppInvSeg
+ ori r8,r8,lo16(EXT(kernel_pmap_phys)) ; Assume kernel (bottom of address)
+ sub r3,r3,r2 ; Form select mask - 0 if kernel, -1 if user
+ la r19,ppUserPmap(r29) ; Point to the current user pmap
+
+; The following line is an exercise of a generally unreadable but recompile-friendly programing practice
+ rlwinm r30,r9,userProtKeybit+1+(63-sgcVSKeyUsr),sgcVSKeyUsr-32,sgcVSKeyUsr-32 ; Isolate the user state protection key
+
+ andc r8,r8,r3 ; Zero kernel pmap ptr if user, untouched otherwise
+ and r19,r19,r3 ; Zero user pmap ptr if kernel, untouched otherwise
+ and r30,r30,r3 ; Clear key modifier if kernel, leave otherwise
+ or r8,r8,r19 ; Get the pointer to the pmap we are using
+
+ beqlr ; We are staying in the same mode, do not touch segs...
+
+ lwz r28,0(r8) ; Get top half of pmap address
+ lwz r10,4(r8) ; Get bottom half
+
+ stw r2,ppInvSeg(r29) ; Clear request for invalidate and save ppCurSeg
+ rlwinm r28,r28,0,1,0 ; Copy top to top
+ stw r30,ppMapFlags(r29) ; Set the key modifier
+ rlwimi r28,r10,0,0,31 ; Insert bottom
+
+ la r10,pmapCCtl(r28) ; Point to the segment cache control
+ la r9,pmapSegCache(r28) ; Point to the segment cache
+
+ssgLock: lwarx r15,0,r10 ; Get and reserve the segment cache control
+ rlwinm. r0,r15,0,pmapCCtlLckb,pmapCCtlLckb ; Someone have the lock?
+ ori r16,r15,lo16(pmapCCtlLck) ; Set lock bit
+ bne-- ssgLock0 ; Yup, this is in use...
+
+ stwcx. r16,0,r10 ; Try to set the lock
+ bne-- ssgLock ; Did we get contention?
+
+ not r11,r15 ; Invert the invalids to valids
+ li r17,0 ; Set a mask for the SRs we are loading
+ isync ; Make sure we are all caught up
+
+ bf-- pf64Bitb,ssg32Enter ; If 32-bit, jump into it...
+
+ li r0,0 ; Clear
+ slbia ; Trash all SLB entries (except for entry 0 that is)
+ li r17,1 ; Get SLB index to load (skip slb 0)
+ oris r0,r0,0x8000 ; Get set for a mask
+ b ssg64Enter ; Start on a cache line...
+
+ .align 5
+
+ssgLock0: li r15,lgKillResv ; Killing field
+ stwcx. r15,0,r15 ; Kill reservation
+
+ssgLock1: lwz r15,pmapCCtl(r28) ; Get the segment cache controls
+ rlwinm. r15,r15,0,pmapCCtlLckb,pmapCCtlLckb ; Someone have the lock?
+ beq++ ssgLock ; Yup, this is in use...
+ b ssgLock1 ; Nope, try again...
+;
+; This is the 32-bit address space switch code.
+; We take a reservation on the segment cache and walk through.
+; For each entry, we load the specified entries and remember which
+; we did with a mask. Then, we figure out which segments should be
+; invalid and then see which actually are. Then we load those with the
+; defined invalid VSID.
+; Afterwards, we unlock the segment cache.
+;
+
+ .align 5
+
+ssg32Enter: cntlzw r12,r11 ; Find the next slot in use
+ cmplwi r12,pmapSegCacheUse ; See if we are done
+ slwi r14,r12,4 ; Index to the cache slot
+ lis r0,0x8000 ; Get set for a mask
+ add r14,r14,r9 ; Point to the entry
+
+ bge- ssg32Done ; All done...
+
+ lwz r5,sgcESID+4(r14) ; Get the ESID part
+ srw r2,r0,r12 ; Form a mask for the one we are loading
+ lwz r7,sgcVSID+4(r14) ; And get the VSID bottom
+
+ andc r11,r11,r2 ; Clear the bit
+ lwz r6,sgcVSID(r14) ; And get the VSID top
+
+ rlwinm r2,r5,4,28,31 ; Change the segment number to a number
+
+ xor r7,r7,r30 ; Modify the key before we actually set it
+ srw r0,r0,r2 ; Get a mask for the SR we are loading
+ rlwinm r8,r7,19,1,3 ; Insert the keys and N bit
+ or r17,r17,r0 ; Remember the segment
+ rlwimi r8,r7,20,12,31 ; Insert 4:23 the VSID
+ rlwimi r8,r6,20,8,11 ; Get the last nybble of the SR contents
+
+ mtsrin r8,r5 ; Load the segment
+ b ssg32Enter ; Go enter the next...
+
+ .align 5
+
+ssg32Done: lwz r16,validSegs(r29) ; Get the valid SRs flags
+ stw r15,pmapCCtl(r28) ; Unlock the segment cache controls
+
+ lis r0,0x8000 ; Get set for a mask
+ li r2,invalSpace ; Set the invalid address space VSID
+
+ nop ; Align loop
+ nop ; Align loop
+ andc r16,r16,r17 ; Get list of SRs that were valid before but not now
+ nop ; Align loop
+
+ssg32Inval: cntlzw r18,r16 ; Get the first one to invalidate
+ cmplwi r18,16 ; Have we finished?
+ srw r22,r0,r18 ; Get the mask bit
+ rlwinm r23,r18,28,0,3 ; Get the segment register we need
+ andc r16,r16,r22 ; Get rid of the guy we just did
+ bge ssg32Really ; Yes, we are really done now...
+
+ mtsrin r2,r23 ; Invalidate the SR
+ b ssg32Inval ; Do the next...
+
+ .align 5
+
+ssg32Really:
+ stw r17,validSegs(r29) ; Set the valid SR flags
+ li r3,1 ; Set kernel/user transition
+ blr
+
+;
+; This is the 64-bit address space switch code.
+; First we blow away all of the SLB entries.
+; Walk through,
+; loading the SLB. Afterwards, we release the cache lock
+;
+; Note that because we have to treat SLBE 0 specially, we do not ever use it...
+; Its a performance thing...
+;
+
+ .align 5
+
+ssg64Enter: cntlzw r12,r11 ; Find the next slot in use
+ cmplwi r12,pmapSegCacheUse ; See if we are done
+ slwi r14,r12,4 ; Index to the cache slot
+ srw r16,r0,r12 ; Form a mask for the one we are loading
+ add r14,r14,r9 ; Point to the entry
+ andc r11,r11,r16 ; Clear the bit
+ bge-- ssg64Done ; All done...
+
+ ld r5,sgcESID(r14) ; Get the ESID part
+ ld r6,sgcVSID(r14) ; And get the VSID part
+ oris r5,r5,0x0800 ; Turn on the valid bit
+ or r5,r5,r17 ; Insert the SLB slot
+ xor r6,r6,r30 ; Modify the key before we actually set it
+ addi r17,r17,1 ; Bump to the next slot
+ slbmte r6,r5 ; Make that SLB entry
+ b ssg64Enter ; Go enter the next...
+
+ .align 5
+
+ssg64Done: stw r15,pmapCCtl(r28) ; Unlock the segment cache controls
+
+ eqv r16,r16,r16 ; Load up with all foxes
+ subfic r17,r17,64 ; Get the number of 1 bits we need
+
+ sld r16,r16,r17 ; Get a mask for the used SLB entries
+ li r3,1 ; Set kernel/user transition
+ std r16,validSegs(r29) ; Set the valid SR flags
+ blr
+
+;
+; mapSetUp - this function sets initial state for all mapping functions.
+; We turn off all translations (physical), disable interruptions, and
+; enter 64-bit mode if applicable.
+;
+; We also return the original MSR in r11, the feature flags in R12,
+; and CR6 set up so we can do easy branches for 64-bit
+; hw_clear_maps assumes r10, r9 will not be trashed.
+;
+
+ .align 5
+ .globl EXT(mapSetUp)
+
+LEXT(mapSetUp)
+
+ lis r0,hi16(MASK(MSR_VEC)) ; Get the vector mask
+ mfsprg r12,2 ; Get feature flags
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get the FP as well
+ mtcrf 0x04,r12 ; move pf64Bit and pfNoMSRirb to cr5 and 6
+ mfmsr r11 ; Save the MSR
+ mtcrf 0x02,r12 ; move pf64Bit and pfNoMSRirb to cr5 and 6
+ andc r11,r11,r0 ; Clear VEC and FP for good
+ ori r0,r0,lo16(MASK(MSR_EE)|MASK(MSR_DR)|MASK(MSR_IR)) ; Get rid of EE, IR, and DR
+ li r2,1 ; Prepare for 64 bit
+ andc r0,r11,r0 ; Clear the rest
+ bt pfNoMSRirb,msuNoMSR ; No MSR...
+ bt++ pf64Bitb,msuSF ; skip if 64-bit (only they take the hint)
+
+ mtmsr r0 ; Translation and all off
+ isync ; Toss prefetch
+ blr ; Return...
+
+ .align 5
+
+msuSF: rldimi r0,r2,63,MSR_SF_BIT ; set SF bit (bit 0)
+ mtmsrd r0 ; set 64-bit mode, turn off EE, DR, and IR
+ isync ; synchronize
+ blr ; Return...
+
+ .align 5
+
+msuNoMSR: mr r2,r3 ; Save R3 across call
+ mr r3,r0 ; Get the new MSR value
+ li r0,loadMSR ; Get the MSR setter SC
+ sc ; Set it
+ mr r3,r2 ; Restore R3
+ blr ; Go back all set up...
+
+
+;
+; Guest shadow assist -- remove all guest mappings
+;
+; Remove all mappings for a guest pmap from the shadow hash table.
+;
+; Parameters:
+; r3 : address of pmap, 32-bit kernel virtual address
+;
+; Non-volatile register usage:
+; r24 : host pmap's physical address
+; r25 : VMM extension block's physical address
+; r26 : physent address
+; r27 : guest pmap's space ID number
+; r28 : current hash table page index
+; r29 : guest pmap's physical address
+; r30 : saved msr image
+; r31 : current mapping
+;
+ .align 5
+ .globl EXT(hw_rem_all_gv)
+
+LEXT(hw_rem_all_gv)
+
+#define graStackSize ((31-24+1)*4)+4
+ stwu r1,-(FM_ALIGN(graStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(graStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+
+ bt++ pf64Bitb,gra64Salt ; Test for 64-bit machine
+ lwz r25,pmapVmmExtPhys+4(r3) ; r25 <- VMM pmap extension block paddr
+ lwz r9,pmapvr+4(r3) ; Get 32-bit virt<->real conversion salt
+ lwz r24,vmxHostPmapPhys+4(r11) ; r24 <- host pmap's paddr
+ b graStart ; Get to it
+gra64Salt: ld r25,pmapVmmExtPhys(r3) ; r25 <- VMM pmap extension block paddr
+ ld r9,pmapvr(r3) ; Get 64-bit virt<->real conversion salt
+ ld r24,vmxHostPmapPhys(r11) ; r24 <- host pmap's paddr
+graStart: bl EXT(mapSetUp) ; Disable 'rupts, translation, enter 64-bit mode
+ xor r29,r3,r9 ; Convert pmap_t virt->real
+ mr r30,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkExclusive ; Get lock exclusive
+
+ lwz r3,vxsGra(r25) ; Get remove all count
+ addi r3,r3,1 ; Increment remove all count
+ stw r3,vxsGra(r25) ; Update remove all count
+
+ li r28,0 ; r28 <- first hash page table index to search
+ lwz r27,pmapSpace(r29) ; r27 <- guest pmap's space ID number
+graPgLoop:
+ la r31,VMX_HPIDX_OFFSET(r25) ; Get base of hash page physical index
+ rlwinm r11,r28,GV_PGIDX_SZ_LG2,GV_HPAGE_MASK
+ ; Convert page index into page physical index offset
+ add r31,r31,r11 ; Calculate page physical index entry address
+ bt++ pf64Bitb,gra64Page ; Separate handling for 64-bit
+ lwz r31,4(r31) ; r31 <- first slot in hash table page to examine
+ b graLoop ; Examine all slots in this page
+gra64Page: ld r31,0(r31) ; r31 <- first slot in hash table page to examine
+ b graLoop ; Examine all slots in this page
+
+ .align 5
+graLoop: lwz r3,mpFlags(r31) ; Get mapping's flags
+ lhz r4,mpSpace(r31) ; Get mapping's space ID number
+ rlwinm r6,r3,0,mpgFree ; Isolate guest free mapping flag
+ xor r4,r4,r27 ; Compare space ID number
+ or. r0,r6,r4 ; cr0_eq <- !free && space id match
+ bne graMiss ; Not one of ours, skip it
+
+ lwz r11,vxsGraHits(r25) ; Get remove hit count
+ addi r11,r11,1 ; Increment remove hit count
+ stw r11,vxsGraHits(r25) ; Update remove hit count
+
+ rlwinm. r0,r3,0,mpgDormant ; Is this entry dormant?
+ bne graRemPhys ; Yes, nothing to disconnect
+
+ lwz r11,vxsGraActive(r25) ; Get remove active count
+ addi r11,r11,1 ; Increment remove hit count
+ stw r11,vxsGraActive(r25) ; Update remove hit count
+
+ bt++ pf64Bitb,graDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b graFreePTE ; Join 64-bit path to release the PTE
+graDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+graFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- graRemPhys ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx doesn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+graRemPhys:
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r26,r3 ; Got lock on our physent?
+ beq-- graBadPLock ; No, time to bail out
+
+ crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,graRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r26) ; Point to chain anchor
+ lwz r9,ppLink+4(r26) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+
+graRemLoop: beq- graRemoveMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne graRemNext ; No, chain onward
+ bt cr1_eq,graRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b graRemoved ; Exit loop
+graRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- graRemRetry ; Lost reservation, retry
+ b graRemoved ; Good work, let's get outta here
+
+graRemNext: la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b graRemLoop ; Carry on
+
+graRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r26) ; Point to chain anchor
+ ld r9,ppLink(r26) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+graRem64Lp: beq-- graRemoveMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne graRem64Nxt ; Not mapping to remove, chain on, dude
+ bt cr1_eq,graRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b graRemoved ; Exit loop
+graRem64Rt: ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- graRem64Rt ; Lost reservation, retry
+ b graRemoved ; Good work, let's go home
+
+graRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b graRem64Lp ; Carry on
+
+graRemoved:
+ mr r3,r26 ; r3 <- physent's address
+ bl mapPhysUnlock ; Unlock the physent (and its chain of mappings)
+
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ rlwinm r3,r3,0,~mpgFlags ; Clear all guest flags
+ ori r3,r3,mpgFree ; Mark mapping free
+ stw r3,mpFlags(r31) ; Update flags
+
+graMiss: addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping
+ rlwinm. r0,r31,0,GV_PAGE_MASK ; End of hash table page?
+ bne graLoop ; No, examine next slot
+ addi r28,r28,1 ; Increment hash table page index
+ cmplwi r28,GV_HPAGES ; End of hash table?
+ bne graPgLoop ; Examine next hash table page
+
+ la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkUnlock ; Release host pmap's search lock
+
+ bt++ pf64Bitb,graRtn64 ; Handle 64-bit separately
+ mtmsr r30 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b graPopFrame ; Nothing to do now but pop a frame and return
+graRtn64: mtmsrd r30 ; Restore 'rupts, translation, 32-bit mode
+graPopFrame:
+ lwz r0,(FM_ALIGN(graStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+graBadPLock:
+graRemoveMiss:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- remove local guest mappings
+;
+; Remove local mappings for a guest pmap from the shadow hash table.
+;
+; Parameters:
+; r3 : address of guest pmap, 32-bit kernel virtual address
+;
+; Non-volatile register usage:
+; r20 : current active map word's physical address
+; r21 : current hash table page address
+; r22 : updated active map word in process
+; r23 : active map word in process
+; r24 : host pmap's physical address
+; r25 : VMM extension block's physical address
+; r26 : physent address
+; r27 : guest pmap's space ID number
+; r28 : current active map index
+; r29 : guest pmap's physical address
+; r30 : saved msr image
+; r31 : current mapping
+;
+ .align 5
+ .globl EXT(hw_rem_local_gv)
+
+LEXT(hw_rem_local_gv)
+
+#define grlStackSize ((31-20+1)*4)+4
+ stwu r1,-(FM_ALIGN(grlStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(grlStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+ stw r23,FM_ARG0+0x20(r1) ; Save non-volatile r23
+ stw r22,FM_ARG0+0x24(r1) ; Save non-volatile r22
+ stw r21,FM_ARG0+0x28(r1) ; Save non-volatile r21
+ stw r20,FM_ARG0+0x2C(r1) ; Save non-volatile r20
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+
+ bt++ pf64Bitb,grl64Salt ; Test for 64-bit machine
+ lwz r25,pmapVmmExtPhys+4(r3) ; r25 <- VMM pmap extension block paddr
+ lwz r9,pmapvr+4(r3) ; Get 32-bit virt<->real conversion salt
+ lwz r24,vmxHostPmapPhys+4(r11) ; r24 <- host pmap's paddr
+ b grlStart ; Get to it
+grl64Salt: ld r25,pmapVmmExtPhys(r3) ; r25 <- VMM pmap extension block paddr
+ ld r9,pmapvr(r3) ; Get 64-bit virt<->real conversion salt
+ ld r24,vmxHostPmapPhys(r11) ; r24 <- host pmap's paddr
+
+grlStart: bl EXT(mapSetUp) ; Disable 'rupts, translation, enter 64-bit mode
+ xor r29,r3,r9 ; Convert pmap_t virt->real
+ mr r30,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r28,0 ; r28 <- index of first active map word to search
+ lwz r27,pmapSpace(r29) ; r27 <- guest pmap's space ID number
+ b grlMap1st ; Examine first map word
+
+ .align 5
+grlNextMap: stw r22,0(r21) ; Save updated map word
+ addi r28,r28,1 ; Increment map word index
+ cmplwi r28,GV_MAP_WORDS ; See if we're done
+ beq grlDone ; Yup, let's get outta here
+
+grlMap1st: la r20,VMX_ACTMAP_OFFSET(r25) ; Get base of active map word array
+ rlwinm r11,r28,GV_MAPWD_SZ_LG2,GV_MAP_MASK
+ ; Convert map index into map index offset
+ add r20,r20,r11 ; Calculate map array element address
+ lwz r22,0(r20) ; Get active map word at index
+ mr. r23,r22 ; Any active mappings indicated?
+ beq grlNextMap ; Nope, check next word
+
+ la r21,VMX_HPIDX_OFFSET(r25) ; Get base of hash page physical index
+ rlwinm r11,r28,GV_MAP_SHIFT,GV_HPAGE_MASK
+ ; Extract page index from map word index and convert
+ ; into page physical index offset
+ add r21,r21,r11 ; Calculate page physical index entry address
+ bt++ pf64Bitb,grl64Page ; Separate handling for 64-bit
+ lwz r21,4(r21) ; Get selected hash table page's address
+ b grlLoop ; Examine all slots in this page
+grl64Page: ld r21,0(r21) ; Get selected hash table page's address
+ b grlLoop ; Examine all slots in this page
+
+ .align 5
+grlLoop: cntlzw r11,r23 ; Get next active bit lit in map word
+ cmplwi r11,32 ; Any active mappings left in this word?
+ lis r12,0x8000 ; Prepare mask to reset bit
+ srw r12,r12,r11 ; Position mask bit
+ andc r23,r23,r12 ; Reset lit bit
+ beq grlNextMap ; No bits lit, examine next map word
+
+ slwi r31,r11,GV_SLOT_SZ_LG2 ; Get slot offset in slot band from lit bit number
+ rlwinm r31,r28,GV_BAND_SHIFT,GV_BAND_MASK
+ ; Extract slot band number from index and insert
+ add r31,r31,r21 ; Add hash page address yielding mapping slot address
+
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ lhz r4,mpSpace(r31) ; Get mapping's space ID number
+ rlwinm r5,r3,0,mpgGlobal ; Extract global bit
+ xor r4,r4,r27 ; Compare space ID number
+ or. r4,r4,r5 ; (space id miss || global)
+ bne grlLoop ; Not one of ours, skip it
+ andc r22,r22,r12 ; Reset active bit corresponding to this mapping
+ ori r3,r3,mpgDormant ; Mark entry dormant
+ stw r3,mpFlags(r31) ; Update mapping's flags
+
+ bt++ pf64Bitb,grlDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b grlFreePTE ; Join 64-bit path to release the PTE
+grlDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+grlFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- grlLoop ; No valid PTE, we're done with this mapping
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx doesn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+ b grlLoop ; On to next active mapping in this map word
+
+grlDone: la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkUnlock ; Release host pmap's search lock
+
+ bt++ pf64Bitb,grlRtn64 ; Handle 64-bit separately
+ mtmsr r30 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b grlPopFrame ; Nothing to do now but pop a frame and return
+grlRtn64: mtmsrd r30 ; Restore 'rupts, translation, 32-bit mode
+grlPopFrame:
+ lwz r0,(FM_ALIGN(grlStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r23,FM_ARG0+0x20(r1) ; Restore non-volatile r23
+ lwz r22,FM_ARG0+0x24(r1) ; Restore non-volatile r22
+ lwz r21,FM_ARG0+0x28(r1) ; Restore non-volatile r21
+ lwz r20,FM_ARG0+0x2C(r1) ; Restore non-volatile r20
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+
+;
+; Guest shadow assist -- resume a guest mapping
+;
+; Locates the specified dormant mapping, and if it exists validates it and makes it
+; active.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : host virtual address, high-order 32 bits
+; r6 : host virtual address, low-order 32 bits
+; r7 : guest virtual address, high-order 32 bits
+; r8 : guest virtual address, low-order 32 bits
+; r9 : guest mapping protection code
+;
+; Non-volatile register usage:
+; r23 : VMM extension block's physical address
+; r24 : physent physical address
+; r25 : caller's msr image from mapSetUp
+; r26 : guest mapping protection code
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : host virtual address
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+ .align 5
+ .globl EXT(hw_res_map_gv)
+
+LEXT(hw_res_map_gv)
+
+#define grsStackSize ((31-23+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(grsStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(grsStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+ stw r23,FM_ARG0+0x20(r1) ; Save non-volatile r23
+
+ rlwinm r29,r6,0,0xFFFFF000 ; Clean up low-order 32 bits of host vaddr
+ rlwinm r30,r8,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+ mr r26,r9 ; Copy guest mapping protection code
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+ bt++ pf64Bitb,grs64Salt ; Handle 64-bit machine separately
+ lwz r23,pmapVmmExtPhys+4(r3) ; r23 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b grsStart ; Get to it
+
+grs64Salt: rldimi r29,r5,32,0 ; Insert high-order 32 bits of 64-bit host vaddr
+ rldimi r30,r7,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r23,pmapVmmExtPhys(r3) ; r23 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+grsStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r25,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,grs64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b grs32SrchLp ; Let the search begin!
+
+ .align 5
+grs32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq grsSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz grs32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ beq grsSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b grsSrchMiss ; No joy in our hash group
+
+grs64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b grs64SrchLp ; Let the search begin!
+
+ .align 5
+grs64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq grsSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz grs64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ bne grsSrchMiss ; No joy in our hash group
+
+grsSrchHit:
+ rlwinm. r0,r6,0,mpgDormant ; Is the mapping dormant?
+ bne grsFindHost ; Yes, nothing to disconnect
+
+ bt++ pf64Bitb,grsDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b grsFreePTE ; Join 64-bit path to release the PTE
+grsDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+grsFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- grsFindHost ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+grsFindHost:
+
+// We now have a dormant guest mapping that matches our space id and virtual address. Our next
+// step is to locate the host mapping that completes the guest mapping's connection to a physical
+// frame. The guest and host mappings must connect to the same physical frame, so they must both
+// be chained on the same physent. We search the physent chain for a host mapping matching our
+// host's space id and the host virtual address. If we succeed, we know that the entire chain
+// of mappings (guest virtual->host virtual->physical) is valid, so the dormant mapping can be
+// resumed. If we fail to find the specified host virtual->physical mapping, it is because the
+// host virtual or physical address has changed since the guest mapping was suspended, so it
+// is no longer valid and cannot be resumed -- we therefore delete the guest mappping and tell
+// our caller that it will have to take its long path, translating the host virtual address
+// through the host's skiplist and installing a new guest mapping.
+
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r24,r3 ; Got lock on our physent?
+ beq-- grsBadPLock ; No, time to bail out
+
+ bt++ pf64Bitb,grsPFnd64 ; 64-bit version of physent chain search
+
+ lwz r9,ppLink+4(r24) ; Get first mapping on physent
+ lwz r6,pmapSpace(r27) ; Get host pmap's space id number
+ rlwinm r9,r9,0,~ppFlags ; Be-gone, unsightly flags
+grsPELoop: mr. r12,r9 ; Got a mapping to look at?
+ beq- grsPEMiss ; Nope, we've missed hva->phys mapping
+ lwz r7,mpFlags(r12) ; Get mapping's flags
+ lhz r4,mpSpace(r12) ; Get mapping's space id number
+ lwz r5,mpVAddr+4(r12) ; Get mapping's virtual address
+ lwz r9,mpAlias+4(r12) ; Next mapping in physent alias chain
+
+ rlwinm r0,r7,0,mpType ; Isolate mapping's type
+ rlwinm r5,r5,0,~mpHWFlags ; Bye-bye unsightly flags
+ xori r0,r0,mpNormal ; Normal mapping?
+ xor r4,r4,r6 ; Compare w/ host space id number
+ xor r5,r5,r29 ; Compare w/ host virtual address
+ or r0,r0,r4 ; r0 <- (wrong type || !space id)
+ or. r0,r0,r5 ; cr0_eq <- (right type && space id hit && hva hit)
+ beq grsPEHit ; Hit
+ b grsPELoop ; Iterate
+
+grsPFnd64: li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ ld r9,ppLink(r24) ; Get first mapping on physent
+ lwz r6,pmapSpace(r27) ; Get pmap's space id number
+ andc r9,r9,r0 ; Cleanup mapping pointer
+grsPELp64: mr. r12,r9 ; Got a mapping to look at?
+ beq-- grsPEMiss ; Nope, we've missed hva->phys mapping
+ lwz r7,mpFlags(r12) ; Get mapping's flags
+ lhz r4,mpSpace(r12) ; Get mapping's space id number
+ ld r5,mpVAddr(r12) ; Get mapping's virtual address
+ ld r9,mpAlias(r12) ; Next mapping physent alias chain
+ rlwinm r0,r7,0,mpType ; Isolate mapping's type
+ rldicr r5,r5,0,mpHWFlagsb-1 ; Bye-bye unsightly flags
+ xori r0,r0,mpNormal ; Normal mapping?
+ xor r4,r4,r6 ; Compare w/ host space id number
+ xor r5,r5,r29 ; Compare w/ host virtual address
+ or r0,r0,r4 ; r0 <- (wrong type || !space id)
+ or. r0,r0,r5 ; cr0_eq <- (right type && space id hit && hva hit)
+ beq grsPEHit ; Hit
+ b grsPELp64 ; Iterate
+
+grsPEHit: lwz r0,mpVAddr+4(r31) ; Get va byte containing protection bits
+ rlwimi r0,r26,0,mpPP ; Insert new protection bits
+ stw r0,mpVAddr+4(r31) ; Write 'em back
+
+ eieio ; Ensure previous mapping updates are visible
+ lwz r0,mpFlags(r31) ; Get flags
+ rlwinm r0,r0,0,~mpgDormant ; Turn off dormant flag
+ stw r0,mpFlags(r31) ; Set updated flags, entry is now valid
+
+ li r31,mapRtOK ; Indicate success
+ b grsRelPhy ; Exit through physent lock release
+
+grsPEMiss: crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,grsRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r24) ; Point to chain anchor
+ lwz r9,ppLink+4(r24) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+grsRemLoop: beq- grsPEMissMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne grsRemNext ; No, chain onward
+ bt cr1_eq,grsRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b grsDelete ; Finish deleting mapping
+grsRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- grsRemRetry ; Lost reservation, retry
+ b grsDelete ; Finish deleting mapping
+
+ .align 5
+grsRemNext: la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b grsRemLoop ; Carry on
+
+grsRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r24) ; Point to chain anchor
+ ld r9,ppLink(r24) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+grsRem64Lp: beq-- grsPEMissMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne grsRem64Nxt ; Not mapping to remove, chain on, dude
+ bt cr1_eq,grsRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b grsDelete ; Finish deleting mapping
+grsRem64Rt: ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- grsRem64Rt ; Lost reservation, retry
+ b grsDelete ; Finish deleting mapping
+
+ .align 5
+grsRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b grsRem64Lp ; Carry on
+
+grsDelete:
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ rlwinm r3,r3,0,~mpgFlags ; Clear all guest flags
+ ori r3,r3,mpgFree ; Mark mapping free
+ stw r3,mpFlags(r31) ; Update flags
+
+ li r31,mapRtNotFnd ; Didn't succeed
+
+grsRelPhy: mr r3,r24 ; r3 <- physent addr
+ bl mapPhysUnlock ; Unlock physent chain
+
+grsRelPmap: la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+grsRtn: mr r3,r31 ; r3 <- result code
+ bt++ pf64Bitb,grsRtn64 ; Handle 64-bit separately
+ mtmsr r25 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b grsPopFrame ; Nothing to do now but pop a frame and return
+grsRtn64: mtmsrd r25 ; Restore 'rupts, translation, 32-bit mode
+grsPopFrame:
+ lwz r0,(FM_ALIGN(grsStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r23,FM_ARG0+0x20(r1) ; Restore non-volatile r23
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+ .align 5
+grsSrchMiss:
+ li r31,mapRtNotFnd ; Could not locate requested mapping
+ b grsRelPmap ; Exit through host pmap search lock release
+
+grsBadPLock:
+grsPEMissMiss:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- add a guest mapping
+;
+; Adds a guest mapping.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : guest virtual address, high-order 32 bits
+; r6 : guest virtual address, low-order 32 bits (with mpHWFlags)
+; r7 : new mapping's flags
+; r8 : physical address, 32-bit page number
+;
+; Non-volatile register usage:
+; r22 : hash group's physical address
+; r23 : VMM extension block's physical address
+; r24 : mapping's flags
+; r25 : caller's msr image from mapSetUp
+; r26 : physent physical address
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : physical address, 32-bit 4k-page number
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_add_map_gv)
+
+
+LEXT(hw_add_map_gv)
+
+#define gadStackSize ((31-22+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gadStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gadStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+ stw r23,FM_ARG0+0x20(r1) ; Save non-volatile r23
+ stw r22,FM_ARG0+0x24(r1) ; Save non-volatile r22
+
+ rlwinm r30,r5,0,1,0 ; Get high-order 32 bits of guest vaddr
+ rlwimi r30,r6,0,0,31 ; Get low-order 32 bits of guest vaddr
+ mr r24,r7 ; Copy guest mapping's flags
+ mr r29,r8 ; Copy target frame's physical address
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+ bt++ pf64Bitb,gad64Salt ; Test for 64-bit machine
+ lwz r23,pmapVmmExtPhys+4(r3) ; r23 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r22,VMX_HPIDX_OFFSET(r11) ; r22 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r22,r22,r10 ; r22 <- hash page index entry
+ lwz r22,4(r22) ; r22 <- hash page paddr
+ rlwimi r22,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r22 <- hash group paddr
+ b gadStart ; Get to it
+
+gad64Salt: ld r23,pmapVmmExtPhys(r3) ; r23 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r22,VMX_HPIDX_OFFSET(r11) ; r22 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r22,r22,r10 ; r22 <- hash page index entry
+ ld r22,0(r22) ; r22 <- hash page paddr
+ insrdi r22,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r22 <- hash group paddr
+
+gadStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r25,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exlusive
+
+ mr r31,r22 ; Prepare to search this group
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gad64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ clrrwi r12,r30,12 ; r12 <- virtual address we're searching for
+ b gad32SrchLp ; Let the search begin!
+
+ .align 5
+gad32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r12 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq gadRelPmap ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gad32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && && space match)
+ xor r5,r5,r12 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- free && space match && virtual addr match
+ beq gadRelPmap ; Join common path on hit (r31 points to guest mapping)
+ b gadScan ; No joy in our hash group
+
+gad64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ clrrdi r12,r30,12 ; r12 <- virtual address we're searching for
+ b gad64SrchLp ; Let the search begin!
+
+ .align 5
+gad64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r12 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq gadRelPmap ; Hit, let upper-level redrive sort it out
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gad64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && && space match)
+ xor r5,r5,r12 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ bne gadScan ; No joy in our hash group
+ b gadRelPmap ; Hit, let upper-level redrive sort it out
+
+gadScan: lbz r12,mpgCursor(r22) ; Get group's cursor
+ rlwinm r12,r12,GV_SLOT_SZ_LG2,(GV_SLOT_MASK << GV_SLOT_SZ_LG2)
+ ; Prepare to address slot at cursor
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ or r2,r22,r12 ; r2 <- 1st mapping to search
+ lwz r3,mpFlags(r2) ; r3 <- 1st mapping slot's flags
+ li r11,0 ; No dormant entries found yet
+ b gadScanLoop ; Let the search begin!
+
+ .align 5
+gadScanLoop:
+ addi r12,r12,GV_SLOT_SZ ; Calculate next slot number to search
+ rlwinm r12,r12,0,(GV_SLOT_MASK << GV_SLOT_SZ_LG2)
+ ; Trim off any carry, wrapping into slot number range
+ mr r31,r2 ; r31 <- current mapping's address
+ or r2,r22,r12 ; r2 <- next mapping to search
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags(r2) ; r3 <- next mapping slot's flags
+ rlwinm. r0,r6,0,mpgFree ; Test free flag
+ bne gadFillMap ; Join common path on hit (r31 points to free mapping)
+ rlwinm r0,r6,0,mpgDormant ; Dormant entry?
+ xori r0,r0,mpgDormant ; Invert dormant flag
+ or. r0,r0,r11 ; Skip all but the first dormant entry we see
+ bne gadNotDorm ; Not dormant or we've already seen one
+ mr r11,r31 ; We'll use this dormant entry if we don't find a free one first
+gadNotDorm: bdnz gadScanLoop ; Iterate
+
+ mr r31,r2 ; r31 <- final mapping's address
+ rlwinm. r0,r6,0,mpgFree ; Test free flag in final mapping
+ bne gadFillMap ; Join common path on hit (r31 points to dormant mapping)
+ rlwinm r0,r6,0,mpgDormant ; Dormant entry?
+ xori r0,r0,mpgDormant ; Invert dormant flag
+ or. r0,r0,r11 ; Skip all but the first dormant entry we see
+ bne gadCkDormant ; Not dormant or we've already seen one
+ mr r11,r31 ; We'll use this dormant entry if we don't find a free one first
+
+gadCkDormant:
+ mr. r31,r11 ; Get dormant mapping, if any, and test
+ bne gadUpCursor ; Go update the cursor, we'll take the dormant entry
+
+gadSteal:
+ lbz r12,mpgCursor(r22) ; Get group's cursor
+ rlwinm r12,r12,GV_SLOT_SZ_LG2,(GV_SLOT_MASK << GV_SLOT_SZ_LG2)
+ ; Prepare to address slot at cursor
+ or r31,r22,r12 ; r31 <- address of mapping to steal
+
+ bt++ pf64Bitb,gadDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b gadFreePTE ; Join 64-bit path to release the PTE
+gadDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+gadFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- gadUpCursor ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+gadUpCursor:
+ rlwinm r12,r31,(32-GV_SLOT_SZ_LG2),GV_SLOT_MASK
+ ; Recover slot number from stolen mapping's address
+ addi r12,r12,1 ; Increment slot number
+ rlwinm r12,r12,0,GV_SLOT_MASK ; Clip to slot number range
+ stb r12,mpgCursor(r22) ; Update group's cursor
+
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r26,r3 ; Got lock on our physent?
+ beq-- gadBadPLock ; No, time to bail out
+
+ crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,gadRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r26) ; Point to chain anchor
+ lwz r9,ppLink+4(r26) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+gadRemLoop: beq- gadPEMissMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne gadRemNext ; No, chain onward
+ bt cr1_eq,gadRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b gadDelDone ; Finish deleting mapping
+gadRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- gadRemRetry ; Lost reservation, retry
+ b gadDelDone ; Finish deleting mapping
+
+gadRemNext: la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b gadRemLoop ; Carry on
+
+gadRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r26) ; Point to chain anchor
+ ld r9,ppLink(r26) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+gadRem64Lp: beq-- gadPEMissMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne gadRem64Nxt ; Not mapping to remove, chain on, dude
+ bt cr1_eq,gadRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b gadDelDone ; Finish deleting mapping
+gadRem64Rt: ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- gadRem64Rt ; Lost reservation, retry
+ b gadDelDone ; Finish deleting mapping
+
+ .align 5
+gadRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b gadRem64Lp ; Carry on
+
+gadDelDone:
+ mr r3,r26 ; Get physent address
+ bl mapPhysUnlock ; Unlock physent chain
+
+gadFillMap:
+ lwz r12,pmapSpace(r28) ; Get guest space id number
+ li r2,0 ; Get a zero
+ stw r24,mpFlags(r31) ; Set mapping's flags
+ sth r12,mpSpace(r31) ; Set mapping's space id number
+ stw r2,mpPte(r31) ; Set mapping's pte pointer invalid
+ stw r29,mpPAddr(r31) ; Set mapping's physical address
+ bt++ pf64Bitb,gadVA64 ; Use 64-bit version on 64-bit machine
+ stw r30,mpVAddr+4(r31) ; Set mapping's virtual address (w/flags)
+ b gadChain ; Continue with chaining mapping to physent
+gadVA64: std r30,mpVAddr(r31) ; Set mapping's virtual address (w/flags)
+
+gadChain: mr r3,r29 ; r3 <- physical frame address
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r26,r3 ; Got lock on our physent?
+ beq-- gadBadPLock ; No, time to bail out
+
+ bt++ pf64Bitb,gadChain64 ; Use 64-bit version on 64-bit machine
+ lwz r12,ppLink+4(r26) ; Get forward chain
+ rlwinm r11,r12,0,~ppFlags ; Get physent's forward pointer sans flags
+ rlwimi r12,r31,0,~ppFlags ; Insert new mapping, preserve physent flags
+ stw r11,mpAlias+4(r31) ; New mapping will head chain
+ stw r12,ppLink+4(r26) ; Point physent to new mapping
+ b gadFinish ; All over now...
+
+gadChain64: li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ ld r12,ppLink(r26) ; Get forward chain
+ andc r11,r12,r7 ; Get physent's forward chain pointer sans flags
+ and r12,r12,r7 ; Isolate pointer's flags
+ or r12,r12,r31 ; Insert new mapping's address forming pointer
+ std r11,mpAlias(r31) ; New mapping will head chain
+ std r12,ppLink(r26) ; Point physent to new mapping
+
+gadFinish: eieio ; Ensure new mapping is completely visible
+
+gadRelPhy: mr r3,r26 ; r3 <- physent addr
+ bl mapPhysUnlock ; Unlock physent chain
+
+gadRelPmap: la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+ bt++ pf64Bitb,gadRtn64 ; Handle 64-bit separately
+ mtmsr r25 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b gadPopFrame ; Nothing to do now but pop a frame and return
+gadRtn64: mtmsrd r25 ; Restore 'rupts, translation, 32-bit mode
+gadPopFrame:
+ lwz r0,(FM_ALIGN(gadStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r23,FM_ARG0+0x20(r1) ; Restore non-volatile r23
+ lwz r22,FM_ARG0+0x24(r1) ; Restore non-volatile r22
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+gadPEMissMiss:
+gadBadPLock:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- supend a guest mapping
+;
+; Suspends a guest mapping.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : guest virtual address, high-order 32 bits
+; r6 : guest virtual address, low-order 32 bits
+;
+; Non-volatile register usage:
+; r26 : VMM extension block's physical address
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : caller's msr image from mapSetUp
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_susp_map_gv)
+
+LEXT(hw_susp_map_gv)
+
+#define gsuStackSize ((31-26+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gsuStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gsuStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+
+ rlwinm r30,r6,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+ bt++ pf64Bitb,gsu64Salt ; Test for 64-bit machine
+
+ lwz r26,pmapVmmExtPhys+4(r3) ; r26 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gsuStart ; Get to it
+gsu64Salt: rldimi r30,r5,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r26,pmapVmmExtPhys(r3) ; r26 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gsuStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r29,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gsu64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gsu32SrchLp ; Let the search begin!
+
+ .align 5
+gsu32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gsuSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gsu32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gsuSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gsuSrchMiss ; No joy in our hash group
+
+gsu64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gsu64SrchLp ; Let the search begin!
+
+ .align 5
+gsu64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gsuSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gsu64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gsuSrchMiss ; No joy in our hash group
+
+gsuSrchHit:
+ bt++ pf64Bitb,gsuDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b gsuFreePTE ; Join 64-bit path to release the PTE
+gsuDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+gsuFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- gsuNoPTE ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+gsuNoPTE: lwz r3,mpFlags(r31) ; Get mapping's flags
+ ori r3,r3,mpgDormant ; Mark entry dormant
+ stw r3,mpFlags(r31) ; Save updated flags
+ eieio ; Ensure update is visible when we unlock
+
+gsuSrchMiss:
+ la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+ bt++ pf64Bitb,gsuRtn64 ; Handle 64-bit separately
+ mtmsr r29 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b gsuPopFrame ; Nothing to do now but pop a frame and return
+gsuRtn64: mtmsrd r29 ; Restore 'rupts, translation, 32-bit mode
+gsuPopFrame:
+ lwz r0,(FM_ALIGN(gsuStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+;
+; Guest shadow assist -- test guest mapping reference and change bits
+;
+; Locates the specified guest mapping, and if it exists gathers its reference
+; and change bit, optionallyÊresetting them.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : guest virtual address, high-order 32 bits
+; r6 : guest virtual address, low-order 32 bits
+; r7 : reset boolean
+;
+; Non-volatile register usage:
+; r24 : VMM extension block's physical address
+; r25 : return code (w/reference and change bits)
+; r26 : reset boolean
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : caller's msr image from mapSetUp
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_test_rc_gv)
+
+LEXT(hw_test_rc_gv)
+
+#define gtdStackSize ((31-24+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gtdStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gtdStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+
+ rlwinm r30,r6,0,0xFFFFF000 ; Clean up low-order 20 bits of guest vaddr
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+
+ bt++ pf64Bitb,gtd64Salt ; Test for 64-bit machine
+
+ lwz r24,pmapVmmExtPhys+4(r3) ; r24 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gtdStart ; Get to it
+
+gtd64Salt: rldimi r30,r5,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r24,pmapVmmExtPhys(r3) ; r24 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gtdStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ mr r26,r7 ; Save reset boolean
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r29,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gtd64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gtd32SrchLp ; Let the search begin!
+
+ .align 5
+gtd32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gtdSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gtd32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gtdSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gtdSrchMiss ; No joy in our hash group
+
+gtd64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gtd64SrchLp ; Let the search begin!
+
+ .align 5
+gtd64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gtdSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gtd64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gtdSrchMiss ; No joy in our hash group
+
+gtdSrchHit:
+ bt++ pf64Bitb,gtdDo64 ; Split for 64 bit
+
+ bl mapInvPte32 ; Invalidate and lock PTEG, also merge into physent
+
+ cmplwi cr1,r26,0 ; Do we want to clear RC?
+ lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
+ mr. r3,r3 ; Was there a previously valid PTE?
+ li r0,lo16(mpR|mpC) ; Get bits to clear
projects / apple / xnu.git / blobdiff