/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
- *
- * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* Please see the License for the specific language governing rights and
* limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#include <assym.s>
#include <debug.h>
-#include <cpus.h>
#include <db_machine_commands.h>
#include <mach_rt.h>
#include <ppc/exception.h>
#include <ppc/Performance.h>
#include <ppc/exception.h>
-#include <ppc/pmap_internals.h>
#include <mach/ppc/vm_param.h>
-#define PERFTIMES 0
.text
-/*
- *
- * Random notes and musings...
- *
- * Access to mappings via the PTEG hash must be done with the list locked.
- * Access via the physical entries is controlled by the physent lock.
- * Access to mappings is controlled by the PTEG lock once they are queued.
- * If they are not on the list, they don't really exist, so
- * only one processor at a time can find them, so no access control is needed.
- *
- * The second half of the PTE is kept in the physical entry. It is done this
- * way, because there may be multiple mappings that refer to the same physical
- * page (i.e., address aliases or synonymns). We must do it this way, because
- * maintenance of the reference and change bits becomes nightmarish if each mapping
- * has its own. One side effect of this, and not necessarily a bad one, is that
- * all mappings for a single page can have a single WIMG, protection state, and RC bits.
- * The only "bad" thing, is the reference bit. With a single copy, we can not get
- * a completely accurate working set calculation, i.e., we can't tell which mapping was
- * used to reference the page, all we can tell is that the physical page was
- * referenced.
- *
- * The master copys of the reference and change bits are kept in the phys_entry.
- * Other than the reference and change bits, changes to the phys_entry are not
- * allowed if it has any mappings. The master reference and change bits must be
- * changed via atomic update.
- *
- * Invalidating a PTE merges the RC bits into the phys_entry.
- *
- * Before checking the reference and/or bits, ALL mappings to the physical page are
- * invalidated.
- *
- * PTEs are never explicitly validated, they are always faulted in. They are also
- * not visible outside of the hw_vm modules. Complete seperation of church and state.
- *
- * Removal of a mapping is invalidates its PTE.
- *
- * So, how do we deal with mappings to I/O space? We don't have a physent for it.
- * Within the mapping is a copy of the second half of the PTE. This is used
- * ONLY when there is no physical entry. It is swapped into the PTE whenever
- * it is built. There is no need to swap it back out, because RC is not
- * maintained for these mappings.
- *
- * So, I'm starting to get concerned about the number of lwarx/stcwx loops in
- * this. Satisfying a mapped address with no stealing requires one lock. If we
- * steal an entry, there's two locks and an atomic update. Invalidation of an entry
- * takes one lock and, if there is a PTE, another lock and an atomic update. Other
- * operations are multiples (per mapping) of the above. Maybe we should look for
- * an alternative. So far, I haven't found one, but I haven't looked hard.
- */
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; |00000000|00000SSS|SSSSSSSS|SSSSSSSS|SSSSPPPP|PPPPPPPP|PPPPxxxx|xxxxxxxx| - EA
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+;
+; 0 0 1
+; 0 8 6
+; +--------+--------+--------+
+; |//////BB|BBBBBBBB|BBBB////| - SID - base
+; +--------+--------+--------+
+;
+; 0 0 1
+; 0 8 6
+; +--------+--------+--------+
+; |////////|11111111|111111//| - SID - copy 1
+; +--------+--------+--------+
+;
+; 0 0 1
+; 0 8 6
+; +--------+--------+--------+
+; |////////|//222222|22222222| - SID - copy 2
+; +--------+--------+--------+
+;
+; 0 0 1
+; 0 8 6
+; +--------+--------+--------+
+; |//////33|33333333|33//////| - SID - copy 3 - not needed
+; +--------+--------+--------+ for 65 bit VPN
+;
+; 0 0 1 2 3 4 4 5 5
+; 0 8 6 4 2 0 8 1 5
+; +--------+--------+--------+--------+--------+--------+--------+
+; |00000000|00000002|22222222|11111111|111111BB|BBBBBBBB|BBBB////| - SID Hash - this is all
+; +--------+--------+--------+--------+--------+--------+--------+ SID copies ORed
+; 0 0 1 2 3 4 4 5 5
+; 0 8 6 4 2 0 8 1 5
+; +--------+--------+--------+--------+--------+--------+--------+
+; |00000000|0000000S|SSSSSSSS|SSSSSSSS|SSSSSS00|00000000|0000////| - Shifted high order EA
+; +--------+--------+--------+--------+--------+--------+--------+ left shifted "segment"
+; part of EA to make
+; room for SID base
+;
+;
+; 0 0 1 2 3 4 4 5 5
+; 0 8 6 4 2 0 8 1 5
+; +--------+--------+--------+--------+--------+--------+--------+
+; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVV////| - VSID - SID Hash XORed
+; +--------+--------+--------+--------+--------+--------+--------+ with shifted EA
+;
+; 0 0 1 2 3 4 4 5 6 7 7
+; 0 8 6 4 2 0 8 6 4 2 9
+; +--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+
+; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVPPPP|PPPPPPPP|PPPPxxxx|xxxxxxxx| - VPN
+; +--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+
+;
-/* hw_add_map(struct mapping *mp, space_t space, vm_offset_t va) - Adds a mapping
+/* addr64_t hw_add_map(struct pmap *pmap, struct mapping *mp) - Adds a mapping
*
- * Adds a mapping to the PTEG hash list.
+ * Maps a page or block into a pmap
*
- * Interrupts must be disabled before calling.
- *
- * Using the space and the virtual address, we hash into the hash table
- * and get a lock on the PTEG hash chain. Then we chain the
- * mapping to the front of the list.
+ * Returns 0 if add worked or the vaddr of the first overlap if not
*
+ * Make mapping - not block or I/O - note: this is low-level, upper should remove duplicates
+ *
+ * 1) bump mapping busy count
+ * 2) lock pmap share
+ * 3) find mapping full path - finds all possible list previous elements
+ * 4) upgrade pmap to exclusive
+ * 5) add mapping to search list
+ * 6) find physent
+ * 7) lock physent
+ * 8) add to physent
+ * 9) unlock physent
+ * 10) unlock pmap
+ * 11) drop mapping busy count
+ *
+ *
+ * Make mapping - block or I/O - note: this is low-level, upper should remove duplicates
+ *
+ * 1) bump mapping busy count
+ * 2) lock pmap share
+ * 3) find mapping full path - finds all possible list previous elements
+ * 4) upgrade pmap to exclusive
+ * 5) add mapping to search list
+ * 6) unlock pmap
+ * 7) drop mapping busy count
+ *
*/
.align 5
.globl EXT(hw_add_map)
LEXT(hw_add_map)
-
-#if PERFTIMES && DEBUG
- mr r7,r3
- mflr r11
- li r3,20
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r7
- mtlr r11
+
+ stwu r1,-(FM_ALIGN((31-17+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r17,FM_ARG0+0x00(r1) ; Save a register
+ stw r18,FM_ARG0+0x04(r1) ; Save a register
+ stw r19,FM_ARG0+0x08(r1) ; Save a register
+ mfsprg r19,2 ; Get feature flags
+ stw r20,FM_ARG0+0x0C(r1) ; Save a register
+ stw r21,FM_ARG0+0x10(r1) ; Save a register
+ mtcrf 0x02,r19 ; move pf64Bit cr6
+ stw r22,FM_ARG0+0x14(r1) ; Save a register
+ stw r23,FM_ARG0+0x18(r1) ; Save a register
+ stw r24,FM_ARG0+0x1C(r1) ; Save a register
+ stw r25,FM_ARG0+0x20(r1) ; Save a register
+ stw r26,FM_ARG0+0x24(r1) ; Save a register
+ stw r27,FM_ARG0+0x28(r1) ; Save a register
+ stw r28,FM_ARG0+0x2C(r1) ; Save a register
+ stw r29,FM_ARG0+0x30(r1) ; Save a register
+ stw r30,FM_ARG0+0x34(r1) ; Save a register
+ stw r31,FM_ARG0+0x38(r1) ; Save a register
+ stw r0,(FM_ALIGN((31-17+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r3) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne hamPanic ; Call not valid for guest shadow assist pmap
#endif
+
+ rlwinm r11,r4,0,0,19 ; Round down to get mapping block address
+ mr r28,r3 ; Save the pmap
+ mr r31,r4 ; Save the mapping
+ bt++ pf64Bitb,hamSF1 ; skip if 64-bit (only they take the hint)
+ lwz r20,pmapvr+4(r3) ; Get conversion mask for pmap
+ lwz r21,mbvrswap+4(r11) ; Get conversion mask for mapping
- mfmsr r0 /* Get the MSR */
- eqv r6,r6,r6 /* Fill the bottom with foxes */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r11,r4,6,6,25 /* Position the space for the VSID */
- mfspr r10,sdr1 /* Get hash table base and size */
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwimi r11,r5,30,2,5 /* Insert the segment no. to make a VSID */
- mfsprg r12,2 ; Get feature flags
- rlwimi r6,r10,16,0,15 /* Make table size -1 out of mask */
- rlwinm r7,r5,26,10,25 /* Isolate the page index */
- or r8,r10,r6 /* Point to the last byte in table */
- rlwinm r9,r5,4,0,3 ; Move nybble 1 up to 0
- xor r7,r7,r11 /* Get primary hash */
- mtcrf 0x04,r12 ; Set the features
- andi. r12,r0,0x7FCF /* Disable translation and interruptions */
- rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
- addi r8,r8,1 /* Point to the PTEG Control Area */
- xor r9,r9,r5 ; Splooch vaddr nybble 0 and 1 together
- and r7,r7,r6 /* Wrap the hash */
- rlwimi r11,r5,10,26,31 /* Move API into pte ID */
- rlwinm r9,r9,6,27,29 ; Get splooched bits in place
- add r8,r8,r7 /* Point to our PCA entry */
- rlwinm r10,r4,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
-
- bt pfNoMSRirb,hamNoMSR ; No MSR...
-
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hamNoMSRx
+ b hamSF1x ; Done...
-hamNoMSR: mr r4,r0 ; Save R0
- mr r2,r3 ; Save
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r0,r4 ; Restore
- mr r3,r2 ; Restore
-hamNoMSRx:
+hamSF1: ld r20,pmapvr(r3) ; Get conversion mask for pmap
+ ld r21,mbvrswap(r11) ; Get conversion mask for mapping
- la r4,PCAhash(r8) /* Point to the mapping hash area */
- xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
- isync /* Get rid of anything prefetched before we ref storage */
-/*
- * We've now got the address of our PCA, the hash chain anchor, our API subhash,
- * and word 0 of the PTE (the virtual part).
- *
- * Now, we just lock the PCA.
- */
+hamSF1x: bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
- li r12,1 /* Get the locked value */
- dcbt 0,r4 /* We'll need the hash area in a sec, so get it */
- add r4,r4,r9 /* Point to the right mapping hash slot */
-
-ptegLckx: lwarx r10,0,r8 /* Get the PTEG lock */
- mr. r10,r10 /* Is it locked? */
- bne- ptegLckwx /* Yeah... */
- stwcx. r12,0,r8 /* Take take it */
- bne- ptegLckx /* Someone else was trying, try again... */
- b ptegSXgx /* All done... */
-
- .align 4
-
-ptegLckwx: mr. r10,r10 /* Check if it's already held */
- beq+ ptegLckx /* It's clear... */
- lwz r10,0(r8) /* Get lock word again... */
- b ptegLckwx /* Wait... */
-
- .align 4
-
-ptegSXgx: isync /* Make sure we haven't used anything yet */
-
- lwz r7,0(r4) /* Pick up the anchor of hash list */
- stw r3,0(r4) /* Save the new head */
- stw r7,mmhashnext(r3) /* Chain in the old head */
-
- stw r4,mmPTEhash(r3) /* Point to the head of the hash list */
-
- sync /* Make sure the chain is updated */
- stw r10,0(r8) /* Unlock the hash list */
- mtmsr r0 /* Restore translation and interruptions */
- isync /* Toss anything done with DAT off */
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,21
- bl EXT(dbgLog2) ; end of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- blr /* Leave... */
+ mr r17,r11 ; Save the MSR
+ xor r28,r28,r20 ; Convert the pmap to physical addressing
+ xor r31,r31,r21 ; Convert the mapping to physical addressing
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ lwz r24,mpFlags(r31) ; Pick up the flags
+ bne-- hamBadLock ; Nope...
+ li r21,0 ; Remember that we have the shared lock
+
+;
+; Note that we do a full search (i.e., no shortcut level skips, etc.)
+; here so that we will know the previous elements so we can dequeue them
+; later.
+;
-/* mp=hw_lock_phys_vir(space, va) - Finds and locks a physical entry by vaddr.
- *
- * Returns the mapping with the associated physent locked if found, or a
- * zero and no lock if not. It we timed out trying to get a the lock on
- * the physical entry, we retun a 1. A physical entry can never be on an
- * odd boundary, so we can distinguish between a mapping and a timeout code.
- *
- * Interrupts must be disabled before calling.
- *
- * Using the space and the virtual address, we hash into the hash table
- * and get a lock on the PTEG hash chain. Then we search the chain for the
- * mapping for our virtual address. From there, we extract the pointer to
- * the physical entry.
- *
- * Next comes a bit of monkey business. we need to get a lock on the physical
- * entry. But, according to our rules, we can't get it after we've gotten the
- * PTEG hash lock, we could deadlock if we do. So, we need to release the
- * hash lock. The problem is, though, that as soon as we release it, some
- * other yahoo may remove our mapping between the time that we release the
- * hash lock and obtain the phys entry lock. So, we can't count on the
- * mapping once we release the lock. Instead, after we lock the phys entry,
- * we search the mapping list (phys_link) for our translation. If we don't find it,
- * we unlock the phys entry, bail out, and return a 0 for the mapping address. If we
- * did find it, we keep the lock and return the address of the mapping block.
- *
- * What happens when a mapping is found, but there is no physical entry?
- * This is what happens when there is I/O area mapped. It one of these mappings
- * is found, the mapping is returned, as is usual for this call, but we don't
- * try to lock anything. There could possibly be some problems here if another
- * processor releases the mapping while we still alre using it. Hope this
- * ain't gonna happen.
- *
- * Taaa-dahhh! Easy as pie, huh?
- *
- * So, we have a few hacks hacks for running translate off in here.
- * First, when we call the lock routine, we have carnel knowlege of the registers is uses.
- * That way, we don't need a stack frame, which we can't have 'cause the stack is in
- * virtual storage. But wait, as if that's not enough... We need one more register. So,
- * we cram the LR into the CTR and return from there.
- *
- */
+hamRescan: lwz r4,mpVAddr(r31) ; Get the new vaddr top half
+ lwz r5,mpVAddr+4(r31) ; Get the new vaddr bottom half
+ mr r3,r28 ; Pass in pmap to search
+ lhz r23,mpBSize(r31) ; Get the block size for later
+ mr r29,r4 ; Save top half of vaddr for later
+ mr r30,r5 ; Save bottom half of vaddr for later
+
+ bl EXT(mapSearchFull) ; Go see if we can find it
+
+ li r22,lo16(0x800C) ; Get 0xFFFF800C
+ rlwinm r0,r24,mpBSub+1,31,31 ; Rotate to get 0 if 4K bsu or 1 if 32MB bsu
+ addi r23,r23,1 ; Get actual length
+ rlwnm r22,r22,r0,27,31 ; Rotate to get 12 or 25
+ lis r0,0x8000 ; Get 0xFFFFFFFF80000000
+ slw r9,r23,r22 ; Isolate the low part
+ rlwnm r22,r23,r22,22,31 ; Extract the high order
+ addic r23,r9,-4096 ; Get the length to the last page
+ add r0,r0,r0 ; Get 0xFFFFFFFF00000000 for 64-bit or 0 for 32-bit
+ addme r22,r22 ; Do high order as well...
+ mr. r3,r3 ; Did we find a mapping here?
+ or r0,r30,r0 ; Fill high word of 64-bit with 1s so we will properly carry
+ bne-- hamOverlay ; We found a mapping, this is no good, can not double map...
+
+ addc r9,r0,r23 ; Add size to get last page in new range
+ or. r0,r4,r5 ; Are we beyond the end?
+ adde r8,r29,r22 ; Add the rest of the length on
+ rlwinm r9,r9,0,0,31 ; Clean top half of sum
+ beq++ hamFits ; We are at the end...
+
+ cmplw cr1,r9,r5 ; Is the bottom part of our end less?
+ cmplw r8,r4 ; Is our end before the next (top part)
+ crand cr0_eq,cr0_eq,cr1_lt ; Is the second half less and the first half equal?
+ cror cr0_eq,cr0_eq,cr0_lt ; Or is the top half less
+
+ bf-- cr0_eq,hamOverlay ; No, we do fit, there is an overlay...
+
+;
+; Here we try to convert to an exclusive lock. This will fail if someone else
+; has it shared.
+;
+hamFits: mr. r21,r21 ; Do we already have the exclusive lock?
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+
+ bne-- hamGotX ; We already have the exclusive...
+
+ bl sxlkPromote ; Try to promote shared to exclusive
+ mr. r3,r3 ; Could we?
+ beq++ hamGotX ; Yeah...
+
+;
+; Since we could not promote our lock, we need to convert to it.
+; That means that we drop the shared lock and wait to get it
+; exclusive. Since we release the lock, we need to do the look up
+; again.
+;
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkConvert ; Convert shared to exclusive
+ mr. r3,r3 ; Could we?
+ bne-- hamBadLock ; Nope, we must have timed out...
+
+ li r21,1 ; Remember that we have the exclusive lock
+ b hamRescan ; Go look again...
+
.align 5
- .globl EXT(hw_lock_phys_vir)
-LEXT(hw_lock_phys_vir)
+hamGotX: mr r3,r28 ; Get the pmap to insert into
+ mr r4,r31 ; Point to the mapping
+ bl EXT(mapInsert) ; Insert the mapping into the list
+
+ rlwinm r11,r24,mpPcfgb+2,mpPcfg>>6 ; Get the index into the page config table
+ lhz r8,mpSpace(r31) ; Get the address space
+ lwz r11,lgpPcfg(r11) ; Get the page config
+ mfsdr1 r7 ; Get the hash table base/bounds
+
+ lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
+ lwz r12,pmapResidentMax(r28) ; r12 = pmap->stats.resident_max
+ addi r4,r4,1 ; Bump up the mapped page count
+ stw r4,pmapResidentCnt(r28) ; Set the mapped page count
+ cmplw r12,r4 ; if pmap->stats.resident_max >= pmap->stats.resident_count
+ bge+ hamSkipMax ; goto hamSkipResMax
+ stw r4,pmapResidentMax(r28) ; pmap->stats.resident_max = pmap->stats.resident_count
+
+hamSkipMax: andi. r0,r24,mpType ; Is this a normal mapping?
+
+ rlwimi r8,r8,14,4,17 ; Double address space
+ rlwinm r9,r30,0,4,31 ; Clear segment
+ rlwinm r10,r30,18,14,17 ; Shift EA[32:35] down to correct spot in VSID (actually shift up 14)
+ rlwimi r8,r8,28,0,3 ; Get the last nybble of the hash
+ rlwimi r10,r29,18,0,13 ; Shift EA[18:31] down to VSID (31-bit math works because of max hash table size)
+ rlwinm r7,r7,0,16,31 ; Isolate length mask (or count)
+ srw r9,r9,r11 ; Isolate just the page index
+ xor r10,r10,r8 ; Calculate the low 32 bits of the VSID
+
+ xor r9,r9,r10 ; Get the hash to the PTEG
+
+ bne-- hamDoneNP ; Not a normal mapping, therefore, no physent...
+
+ bl mapPhysFindLock ; Go find and lock the physent
+
+ bt++ pf64Bitb,ham64 ; This is 64-bit...
+
+ lwz r11,ppLink+4(r3) ; Get the alias chain pointer
+ rlwinm r7,r7,16,0,15 ; Get the PTEG wrap size
+ slwi r9,r9,6 ; Make PTEG offset
+ ori r7,r7,0xFFC0 ; Stick in the bottom part
+ rlwinm r12,r11,0,~ppFlags ; Clean it up
+ and r9,r9,r7 ; Wrap offset into table
+ mr r4,r31 ; Set the link to install
+ stw r9,mpPte(r31) ; Point the mapping at the PTEG (exact offset is invalid)
+ stw r12,mpAlias+4(r31) ; Move to the mapping
+ bl mapPhyCSet32 ; Install the link
+ b hamDone ; Go finish up...
+
+ .align 5
+
+ham64: li r0,ppLFAmask ; Get mask to clean up alias pointer
+ subfic r7,r7,46 ; Get number of leading zeros
+ eqv r4,r4,r4 ; Get all ones
+ ld r11,ppLink(r3) ; Get the alias chain pointer
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ srd r4,r4,r7 ; Get the wrap mask
+ sldi r9,r9,7 ; Change hash to PTEG offset
+ andc r11,r11,r0 ; Clean out the lock and flags
+ and r9,r9,r4 ; Wrap to PTEG
+ mr r4,r31
+ stw r9,mpPte(r31) ; Point the mapping at the PTEG (exact offset is invalid)
+ std r11,mpAlias(r31) ; Set the alias pointer in the mapping
+
+ bl mapPhyCSet64 ; Install the link
+
+hamDone: bl mapPhysUnlock ; Unlock the physent chain
-#if PERFTIMES && DEBUG
- mflr r11
- mr r5,r3
- li r3,22
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r5
- mtlr r11
-#endif
- mfmsr r12 /* Get the MSR */
- eqv r6,r6,r6 /* Fill the bottom with foxes */
- mfsprg r9,2 ; Get feature flags
- rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
- rlwinm r12,r12,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- mfspr r5,sdr1 /* Get hash table base and size */
- rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
- mtcrf 0x04,r9 ; Set the features
- rlwinm r12,r12,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwimi r6,r5,16,0,15 /* Make table size -1 out of mask */
- andi. r0,r12,0x7FCF /* Disable translation and interruptions */
- rlwinm r9,r4,4,0,3 ; Move nybble 1 up to 0
- rlwinm r7,r4,26,10,25 /* Isolate the page index */
- or r8,r5,r6 /* Point to the last byte in table */
- xor r7,r7,r11 /* Get primary hash */
- rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
- addi r8,r8,1 /* Point to the PTEG Control Area */
- xor r9,r9,r4 ; Splooch vaddr nybble 0 and 1 together
- and r7,r7,r6 /* Wrap the hash */
- rlwimi r11,r4,10,26,31 /* Move API into pte ID */
- rlwinm r9,r9,6,27,29 ; Get splooched bits in place
- add r8,r8,r7 /* Point to our PCA entry */
- rlwinm r10,r3,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
-
- bt pfNoMSRirb,hlpNoMSR ; No MSR...
+hamDoneNP: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
- mtmsr r0 ; Translation and all off
- isync ; Toss prefetch
- b hlpNoMSRx
+ mr r3,r31 ; Get the mapping pointer
+ bl mapDropBusy ; Drop the busy count
-hlpNoMSR: mr r3,r0 ; Get the new MSR
- li r0,loadMSR ; Get the MSR setter SC
- sc ; Set it
-hlpNoMSRx:
+ li r3,0 ; Set successful return
+ li r4,0 ; Set successful return
- la r3,PCAhash(r8) /* Point to the mapping hash area */
- xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
- isync /* Make sure translation is off before we ref storage */
+hamReturn: bt++ pf64Bitb,hamR64 ; Yes...
-/*
- * We've now got the address of our PCA, the hash chain anchor, our API subhash,
- * and word 0 of the PTE (the virtual part).
- *
- * Now, we just lock the PCA and find our mapping, if it exists.
- */
-
- dcbt 0,r3 /* We'll need the hash area in a sec, so get it */
- add r3,r3,r9 /* Point to the right mapping hash slot */
+ mtmsr r17 ; Restore enables/translation/etc.
+ isync
+ b hamReturnC ; Join common...
+
+hamR64: mtmsrd r17 ; Restore enables/translation/etc.
+ isync
-ptegLcka: lwarx r10,0,r8 /* Get the PTEG lock */
- li r5,1 /* Get the locked value */
- mr. r10,r10 /* Is it locked? */
- bne- ptegLckwa /* Yeah... */
- stwcx. r5,0,r8 /* Take take it */
- bne- ptegLcka /* Someone else was trying, try again... */
- b ptegSXga /* All done... */
+hamReturnC: lwz r0,(FM_ALIGN((31-17+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Get the return
+ lwz r17,FM_ARG0+0x00(r1) ; Save a register
+ lwz r18,FM_ARG0+0x04(r1) ; Save a register
+ lwz r19,FM_ARG0+0x08(r1) ; Save a register
+ lwz r20,FM_ARG0+0x0C(r1) ; Save a register
+ mtlr r0 ; Restore the return
+ lwz r21,FM_ARG0+0x10(r1) ; Save a register
+ lwz r22,FM_ARG0+0x14(r1) ; Save a register
+ lwz r23,FM_ARG0+0x18(r1) ; Save a register
+ lwz r24,FM_ARG0+0x1C(r1) ; Save a register
+ lwz r25,FM_ARG0+0x20(r1) ; Save a register
+ lwz r26,FM_ARG0+0x24(r1) ; Save a register
+ lwz r27,FM_ARG0+0x28(r1) ; Save a register
+ lwz r28,FM_ARG0+0x2C(r1) ; Save a register
+ lwz r29,FM_ARG0+0x30(r1) ; Save a register
+ lwz r30,FM_ARG0+0x34(r1) ; Save a register
+ lwz r31,FM_ARG0+0x38(r1) ; Save a register
+ lwz r1,0(r1) ; Pop the stack
- .align 4
+ blr ; Leave...
-ptegLckwa: mr. r10,r10 /* Check if it's already held */
- beq+ ptegLcka /* It's clear... */
- lwz r10,0(r8) /* Get lock word again... */
- b ptegLckwa /* Wait... */
- .align 4
+ .align 5
-ptegSXga: isync /* Make sure we haven't used anything yet */
+hamOverlay: lwz r22,mpFlags(r3) ; Get the overlay flags
+ li r0,mpC|mpR ; Get a mask to turn off RC bits
+ lwz r23,mpFlags(r31) ; Get the requested flags
+ lwz r20,mpVAddr(r3) ; Get the overlay address
+ lwz r8,mpVAddr(r31) ; Get the requested address
+ lwz r21,mpVAddr+4(r3) ; Get the overlay address
+ lwz r9,mpVAddr+4(r31) ; Get the requested address
+ lhz r10,mpBSize(r3) ; Get the overlay length
+ lhz r11,mpBSize(r31) ; Get the requested length
+ lwz r24,mpPAddr(r3) ; Get the overlay physical address
+ lwz r25,mpPAddr(r31) ; Get the requested physical address
+ andc r21,r21,r0 ; Clear RC bits
+ andc r9,r9,r0 ; Clear RC bits
- mflr r0 /* Get the LR */
- lwz r9,0(r3) /* Pick up the first mapping block */
- mtctr r0 /* Stuff it into the CTR */
-
-findmapa:
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
- mr. r3,r9 /* Did we hit the end? */
- bne+ chkmapa /* Nope... */
-
- stw r3,0(r8) /* Unlock the PTEG lock
- Note: we never saved anything while we
- had the lock, so we don't need a sync
- before we unlock it */
+ rlwinm. r0,r22,0,mpRIPb,mpRIPb ; Are we in the process of removing this one?
+ mr r3,r20 ; Save the top of the colliding address
+ rlwinm r4,r21,0,0,19 ; Save the bottom of the colliding address
-vbail: mtmsr r12 /* Restore translation and interruptions */
- isync /* Make sure translation is cool */
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,23
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- bctr /* Return in abject failure... */
-
- .align 4
-
-chkmapa: lwz r10,mmPTEv(r3) /* Pick up our virtual ID */
- lwz r9,mmhashnext(r3) /* Pick up next mapping block */
- cmplw r10,r11 /* Have we found ourself? */
- bne- findmapa /* Nope, still wandering... */
-
- lwz r9,mmphysent(r3) /* Get our physical entry pointer */
- li r5,0 /* Clear this out */
- mr. r9,r9 /* Is there, like, a physical entry? */
- stw r5,0(r8) /* Unlock the PTEG lock
- Note: we never saved anything while we
- had the lock, so we don't need a sync
- before we unlock it */
-
- beq- vbail /* If there is no physical entry, it's time
- to leave... */
-
-/* Here we want to call hw_lock_bit. We don't want to use the stack, 'cause it's
- * in virtual storage, and we're in real. So, we've carefully looked at the code
- * in hw_lock_bit (and unlock) and cleverly don't use any of the registers that it uses.
- * Be very, very aware of how you change this code. By the way, it uses:
- * R0, R6, R7, R8, and R9. R3, R4, and R5 contain parameters
- * Unfortunatly, we need to stash R9 still. So... Since we know we will not be interrupted
- * ('cause we turned off interruptions and translation is off) we will use SPRG3...
- */
-
- lwz r10,mmPTEhash(r3) /* Save the head of the hash-alike chain. We need it to find ourselves later */
- lis r5,HIGH_ADDR(EXT(LockTimeOut)) /* Get address of timeout value */
- la r3,pephyslink(r9) /* Point to the lock word */
- ori r5,r5,LOW_ADDR(EXT(LockTimeOut)) /* Get second half of address */
- li r4,PHYS_LOCK /* Get the lock bit value */
- lwz r5,0(r5) /* Pick up the timeout value */
- mtsprg 3,r9 /* Save R9 in SPRG3 */
+ bne++ hamRemv ; Removing, go say so so we help...
- bl EXT(hw_lock_bit) /* Go do the lock */
+ cmplw r20,r8 ; High part of vaddr the same?
+ cmplw cr1,r21,r9 ; Low part?
+ crand cr5_eq,cr0_eq,cr1_eq ; Remember if same
- mfsprg r9,3 /* Restore pointer to the phys_entry */
- mr. r3,r3 /* Did we timeout? */
- lwz r4,pephyslink(r9) /* Pick up first mapping block */
- beq- penterr /* Bad deal, we timed out... */
-
- rlwinm r4,r4,0,0,26 ; Clear out the flags from first link
+ cmplw r10,r11 ; Size the same?
+ cmplw cr1,r24,r25 ; Physical address?
+ crand cr5_eq,cr5_eq,cr0_eq ; Remember
+ crand cr5_eq,cr5_eq,cr1_eq ; Remember if same
-findmapb: mr. r3,r4 /* Did we hit the end? */
- bne+ chkmapb /* Nope... */
+ xor r23,r23,r22 ; Compare mapping flag words
+ andi. r23,r23,mpType|mpPerm ; Are mapping types and attributes the same?
+ crand cr5_eq,cr5_eq,cr0_eq ; Merge in final check
+ bf-- cr5_eq,hamSmash ; This is not the same, so we return a smash...
- la r3,pephyslink(r9) /* Point to where the lock is */
- li r4,PHYS_LOCK /* Get the lock bit value */
- bl EXT(hw_unlock_bit) /* Go unlock the physentry */
-
- li r3,0 /* Say we failed */
- b vbail /* Return in abject failure... */
+ ori r4,r4,mapRtMapDup ; Set duplicate
+ b hamReturn ; And leave...
-penterr: li r3,1 /* Set timeout */
- b vbail /* Return in abject failure... */
-
+hamRemv: ori r4,r4,mapRtRemove ; We are in the process of removing the collision
+ b hamReturn ; Come back yall...
+
+hamSmash: ori r4,r4,mapRtSmash ; Tell caller that it has some clean up to do
+ b hamReturn ; Join common epilog code
+
.align 5
+
+hamBadLock: li r3,0 ; Set lock time out error code
+ li r4,mapRtBadLk ; Set lock time out error code
+ b hamReturn ; Leave....
+
+hamPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
-chkmapb: lwz r6,mmPTEv(r3) /* Pick up our virtual ID */
- lwz r4,mmnext(r3) /* Pick up next mapping block */
- cmplw r6,r11 /* Have we found ourself? */
- lwz r5,mmPTEhash(r3) /* Get the start of our hash chain */
- bne- findmapb /* Nope, still wandering... */
- cmplw r5,r10 /* On the same hash chain? */
- bne- findmapb /* Nope, keep looking... */
- b vbail /* Return in glorious triumph... */
/*
- * hw_rem_map(mapping) - remove a mapping from the system.
- *
- * Upon entry, R3 contains a pointer to a mapping block and the associated
- * physical entry is locked if there is one.
+ * mapping *hw_rem_map(pmap, vaddr, addr64_t *next) - remove a mapping from the system.
*
- * If the mapping entry indicates that there is a PTE entry, we invalidate
- * if and merge the reference and change information into the phys_entry.
+ * Upon entry, R3 contains a pointer to a pmap. Since vaddr is
+ * a 64-bit quantity, it is a long long so it is in R4 and R5.
+ *
+ * We return the virtual address of the removed mapping as a
+ * R3.
*
- * Next, we remove the mapping from the phys_ent and the PTEG hash list.
+ * Note that this is designed to be called from 32-bit mode with a stack.
*
- * Unlock any locks that are left, and exit.
+ * We disable translation and all interruptions here. This keeps is
+ * from having to worry about a deadlock due to having anything locked
+ * and needing it to process a fault.
*
* Note that this must be done with both interruptions off and VM off
*
- * Note that this code depends upon the VSID being of the format 00SXXXXX
- * where S is the segment number.
- *
- *
+ * Remove mapping via pmap, regular page, no pte
+ *
+ * 1) lock pmap share
+ * 2) find mapping full path - finds all possible list previous elements
+ * 4) upgrade pmap to exclusive
+ * 3) bump mapping busy count
+ * 5) remove mapping from search list
+ * 6) unlock pmap
+ * 7) lock physent
+ * 8) remove from physent
+ * 9) unlock physent
+ * 10) drop mapping busy count
+ * 11) drain mapping busy count
+ *
+ *
+ * Remove mapping via pmap, regular page, with pte
+ *
+ * 1) lock pmap share
+ * 2) find mapping full path - finds all possible list previous elements
+ * 3) upgrade lock to exclusive
+ * 4) bump mapping busy count
+ * 5) lock PTEG
+ * 6) invalidate pte and tlbie
+ * 7) atomic merge rc into physent
+ * 8) unlock PTEG
+ * 9) remove mapping from search list
+ * 10) unlock pmap
+ * 11) lock physent
+ * 12) remove from physent
+ * 13) unlock physent
+ * 14) drop mapping busy count
+ * 15) drain mapping busy count
+ *
+ *
+ * Remove mapping via pmap, I/O or block
+ *
+ * 1) lock pmap share
+ * 2) find mapping full path - finds all possible list previous elements
+ * 3) upgrade lock to exclusive
+ * 4) bump mapping busy count
+ * 5) mark remove-in-progress
+ * 6) check and bump remove chunk cursor if needed
+ * 7) unlock pmap
+ * 8) if something to invalidate, go to step 11
+
+ * 9) drop busy
+ * 10) return with mapRtRemove to force higher level to call again
+
+ * 11) Lock PTEG
+ * 12) invalidate ptes, no tlbie
+ * 13) unlock PTEG
+ * 14) repeat 11 - 13 for all pages in chunk
+ * 15) if not final chunk, go to step 9
+ * 16) invalidate tlb entries for the whole block map but no more than the full tlb
+ * 17) lock pmap share
+ * 18) find mapping full path - finds all possible list previous elements
+ * 19) upgrade lock to exclusive
+ * 20) remove mapping from search list
+ * 21) drop mapping busy count
+ * 22) drain mapping busy count
+ *
*/
.align 5
.globl EXT(hw_rem_map)
LEXT(hw_rem_map)
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,24
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- mfsprg r9,2 ; Get feature flags
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
- bt pfNoMSRirb,lmvNoMSR ; No MSR...
+;
+; NOTE NOTE NOTE - IF WE CHANGE THIS STACK FRAME STUFF WE NEED TO CHANGE
+; THE HW_PURGE_* ROUTINES ALSO
+;
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b lmvNoMSRx
+#define hrmStackSize ((31-15+1)*4)+4
+ stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r15,FM_ARG0+0x00(r1) ; Save a register
+ stw r16,FM_ARG0+0x04(r1) ; Save a register
+ stw r17,FM_ARG0+0x08(r1) ; Save a register
+ stw r18,FM_ARG0+0x0C(r1) ; Save a register
+ stw r19,FM_ARG0+0x10(r1) ; Save a register
+ mfsprg r19,2 ; Get feature flags
+ stw r20,FM_ARG0+0x14(r1) ; Save a register
+ stw r21,FM_ARG0+0x18(r1) ; Save a register
+ mtcrf 0x02,r19 ; move pf64Bit cr6
+ stw r22,FM_ARG0+0x1C(r1) ; Save a register
+ stw r23,FM_ARG0+0x20(r1) ; Save a register
+ stw r24,FM_ARG0+0x24(r1) ; Save a register
+ stw r25,FM_ARG0+0x28(r1) ; Save a register
+ stw r26,FM_ARG0+0x2C(r1) ; Save a register
+ stw r27,FM_ARG0+0x30(r1) ; Save a register
+ stw r28,FM_ARG0+0x34(r1) ; Save a register
+ stw r29,FM_ARG0+0x38(r1) ; Save a register
+ stw r30,FM_ARG0+0x3C(r1) ; Save a register
+ stw r31,FM_ARG0+0x40(r1) ; Save a register
+ stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
+ stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r3) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne hrmPanic ; Call not valid for guest shadow assist pmap
+#endif
-lmvNoMSR:
- mr r6,r0
- mr r4,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r4
- mr r0,r6
+ bt++ pf64Bitb,hrmSF1 ; skip if 64-bit (only they take the hint)
+ lwz r9,pmapvr+4(r3) ; Get conversion mask
+ b hrmSF1x ; Done...
+
+hrmSF1: ld r9,pmapvr(r3) ; Get conversion mask
-lmvNoMSRx:
+hrmSF1x:
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
+
+ xor r28,r3,r9 ; Convert the pmap to physical addressing
-
- lwz r6,mmPTEhash(r3) /* Get pointer to hash list anchor */
- lwz r5,mmPTEv(r3) /* Get the VSID */
- dcbt 0,r6 /* We'll need that chain in a bit */
+;
+; Here is where we join in from the hw_purge_* routines
+;
- rlwinm r7,r6,0,0,25 /* Round hash list down to PCA boundary */
- li r12,1 /* Get the locked value */
- subi r6,r6,mmhashnext /* Make the anchor look like an entry */
+hrmJoin: lwz r3,pmapFlags(r28) ; Get pmap's flags
+ mfsprg r19,2 ; Get feature flags again (for alternate entries)
-ptegLck1: lwarx r10,0,r7 /* Get the PTEG lock */
- mr. r10,r10 /* Is it locked? */
- bne- ptegLckw1 /* Yeah... */
- stwcx. r12,0,r7 /* Try to take it */
- bne- ptegLck1 /* Someone else was trying, try again... */
- b ptegSXg1 /* All done... */
+ mr r17,r11 ; Save the MSR
+ mr r29,r4 ; Top half of vaddr
+ mr r30,r5 ; Bottom half of vaddr
- .align 4
-
-ptegLckw1: mr. r10,r10 /* Check if it's already held */
- beq+ ptegLck1 /* It's clear... */
- lwz r10,0(r7) /* Get lock word again... */
- b ptegLckw1 /* Wait... */
+ rlwinm. r3,r3,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne-- hrmGuest ; Yes, handle specially
- .align 4
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne-- hrmBadLock ; Nope...
+
+;
+; Note that we do a full search (i.e., no shortcut level skips, etc.)
+; here so that we will know the previous elements so we can dequeue them
+; later. Note: we get back mpFlags in R7.
+;
-ptegSXg1: isync /* Make sure we haven't used anything yet */
+ mr r3,r28 ; Pass in pmap to search
+ mr r4,r29 ; High order of address
+ mr r5,r30 ; Low order of address
+ bl EXT(mapSearchFull) ; Go see if we can find it
+
+ andi. r0,r7,mpPerm ; Mapping marked permanent?
+ crmove cr5_eq,cr0_eq ; Remember permanent marking
+ mr r20,r7 ; Remember mpFlags
+ mr. r31,r3 ; Did we? (And remember mapping address for later)
+ mr r15,r4 ; Save top of next vaddr
+ mr r16,r5 ; Save bottom of next vaddr
+ beq-- hrmNotFound ; Nope, not found...
+
+ bf-- cr5_eq,hrmPerm ; This one can't be removed...
+;
+; Here we try to promote to an exclusive lock. This will fail if someone else
+; has it shared.
+;
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkPromote ; Try to promote shared to exclusive
+ mr. r3,r3 ; Could we?
+ beq++ hrmGotX ; Yeah...
+
+;
+; Since we could not promote our lock, we need to convert to it.
+; That means that we drop the shared lock and wait to get it
+; exclusive. Since we release the lock, we need to do the look up
+; again.
+;
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkConvert ; Convert shared to exclusive
+ mr. r3,r3 ; Could we?
+ bne-- hrmBadLock ; Nope, we must have timed out...
+
+ mr r3,r28 ; Pass in pmap to search
+ mr r4,r29 ; High order of address
+ mr r5,r30 ; Low order of address
+ bl EXT(mapSearchFull) ; Rescan the list
+
+ andi. r0,r7,mpPerm ; Mapping marked permanent?
+ crmove cr5_eq,cr0_eq ; Remember permanent marking
+ mr. r31,r3 ; Did we lose it when we converted?
+ mr r20,r7 ; Remember mpFlags
+ mr r15,r4 ; Save top of next vaddr
+ mr r16,r5 ; Save bottom of next vaddr
+ beq-- hrmNotFound ; Yeah, we did, someone tossed it for us...
+
+ bf-- cr5_eq,hrmPerm ; This one can't be removed...
- lwz r12,mmhashnext(r3) /* Prime with our forward pointer */
- lwz r4,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
+;
+; We have an exclusive lock on the mapping chain. And we
+; also have the busy count bumped in the mapping so it can
+; not vanish on us.
+;
-srchmaps: mr. r10,r6 /* Save the previous entry */
- bne+ mapok /* No error... */
+hrmGotX: mr r3,r31 ; Get the mapping
+ bl mapBumpBusy ; Bump up the busy count
- lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
- ori r0,r0,LOW_ADDR(Choke)
- sc /* Firmware Heimlich manuever */
-
- .align 4
+;
+; Invalidate any PTEs associated with this
+; mapping (more than one if a block) and accumulate the reference
+; and change bits.
+;
+; Here is also where we need to split 32- and 64-bit processing
+;
-mapok: lwz r6,mmhashnext(r6) /* Look at the next one */
- cmplwi cr5,r4,0 /* Is there a PTE? */
- cmplw r6,r3 /* Have we found ourselves? */
- bne+ srchmaps /* Nope, get your head together... */
+ lwz r21,mpPte(r31) ; Grab the offset to the PTE
+ rlwinm r23,r29,0,1,0 ; Copy high order vaddr to high if 64-bit machine
+ mfsdr1 r29 ; Get the hash table base and size
+
+ rlwinm r0,r20,0,mpType ; Isolate mapping type
+ cmplwi cr5,r0,mpBlock ; Remember whether this is a block mapping
+ cmplwi r0,mpMinSpecial ; cr0_lt <- not a special mapping type
+
+ rlwinm r0,r21,0,mpHValidb,mpHValidb ; See if we actually have a PTE
+ ori r2,r2,0xFFFF ; Get mask to clean out hash table base (works for both 32- and 64-bit)
+ cmpwi cr1,r0,0 ; Have we made a PTE for this yet?
+ rlwinm r21,r21,0,~mpHValid ; Clear out valid bit
+ crorc cr0_eq,cr1_eq,cr0_lt ; No need to look at PTE if none or a special mapping
+ rlwimi r23,r30,0,0,31 ; Insert low under high part of address
+ andc r29,r29,r2 ; Clean up hash table base
+ li r22,0 ; Clear this on out (also sets RC to 0 if we bail)
+ mr r30,r23 ; Move the now merged vaddr to the correct register
+ add r26,r29,r21 ; Point to the PTEG slot
+
+ bt++ pf64Bitb,hrmSplit64 ; Go do 64-bit version...
+
+ rlwinm r9,r21,28,4,29 ; Convert PTEG to PCA entry
+ beq- cr5,hrmBlock32 ; Go treat block specially...
+ subfic r9,r9,-4 ; Get the PCA entry offset
+ bt- cr0_eq,hrmPysDQ32 ; Skip next if no possible PTE...
+ add r7,r9,r29 ; Point to the PCA slot
+
+ bl mapLockPteg ; Go lock up the PTEG (Note: we need to save R6 to set PCA)
+
+ lwz r21,mpPte(r31) ; Get the quick pointer again
+ lwz r5,0(r26) ; Get the top of PTE
- stw r12,mmhashnext(r10) /* Remove us from the queue */
- rlwinm r9,r5,1,0,3 /* Move in the segment */
- rlwinm r8,r4,6,4,19 /* Line PTEG disp up to a page */
- rlwinm r11,r5,5,4,19 /* Line up the VSID */
- lwz r10,mmphysent(r3) /* Point to the physical entry */
-
- beq+ cr5,nopte /* There's no PTE to invalidate... */
+ rlwinm. r0,r21,0,mpHValidb,mpHValidb ; See if we actually have a PTE
+ rlwinm r21,r21,0,~mpHValid ; Clear out valid bit
+ rlwinm r5,r5,0,1,31 ; Turn off valid bit in PTE
+ stw r21,mpPte(r31) ; Make sure we invalidate mpPte, still pointing to PTEG (keep walk_page from making a mistake)
+ beq- hrmUlckPCA32 ; Pte is gone, no need to invalidate...
- xor r8,r8,r11 /* Back hash to virt index */
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
- rlwimi r9,r5,22,4,9 /* Move in the API */
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
- mfspr r11,pvr /* Find out what kind of machine we are */
- rlwimi r9,r8,0,10,19 /* Create the virtual address */
- rlwinm r11,r11,16,16,31 /* Isolate CPU type */
+ stw r5,0(r26) ; Invalidate the PTE
- stw r5,0(r4) /* Make the PTE invalid */
+ li r9,tlbieLock ; Get the TLBIE lock
- cmplwi cr1,r11,3 /* Is this a 603? */
- sync /* Make sure the invalid is stored */
-
-tlbhang1: lwarx r5,0,r12 /* Get the TLBIE lock */
- rlwinm r11,r4,29,29,31 /* Get the bit position of entry */
- mr. r5,r5 /* Is it locked? */
- lis r6,0x8000 /* Start up a bit mask */
- li r5,1 /* Get our lock word */
- bne- tlbhang1 /* It's locked, go wait... */
- stwcx. r5,0,r12 /* Try to get it */
- bne- tlbhang1 /* We was beat... */
+ sync ; Make sure the invalid PTE is actually in memory
+
+hrmPtlb32: lwarx r5,0,r9 ; Get the TLBIE lock
+ mr. r5,r5 ; Is it locked?
+ li r5,1 ; Get locked indicator
+ bne- hrmPtlb32 ; It is locked, go spin...
+ stwcx. r5,0,r9 ; Try to get it
+ bne- hrmPtlb32 ; We was beat...
+
+ rlwinm. r0,r19,0,pfSMPcapb,pfSMPcapb ; Can this processor do SMP?
+
+ tlbie r30 ; Invalidate it all corresponding TLB entries
- srw r6,r6,r11 /* Make a "free slot" mask */
- lwz r5,PCAallo(r7) /* Get the allocation control bits */
- rlwinm r11,r6,24,8,15 /* Make the autogen bit to turn off */
- or r5,r5,r6 /* turn on the free bit */
- rlwimi r11,r11,24,16,23 /* Get lock bit mask to turn it off */
+ beq- hrmNTlbs ; Jump if we can not do a TLBSYNC....
- andc r5,r5,r11 /* Turn off the lock and autogen bits in allocation flags */
- li r11,0 /* Lock clear value */
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; Wait for everyone to catch up
+ sync ; Make sure of it all
+
+hrmNTlbs: li r0,0 ; Clear this
+ rlwinm r2,r21,29,29,31 ; Get slot number (8 byte entries)
+ stw r0,tlbieLock(0) ; Clear the tlbie lock
+ lis r0,0x8000 ; Get bit for slot 0
+ eieio ; Make sure those RC bit have been stashed in PTE
+
+ srw r0,r0,r2 ; Get the allocation hash mask
+ lwz r22,4(r26) ; Get the latest reference and change bits
+ or r6,r6,r0 ; Show that this slot is free
+
+hrmUlckPCA32:
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock the PTEG
+
+;
+; Now, it is time to remove the mapping and unlock the chain.
+; But first, we need to make sure no one else is using this
+; mapping so we drain the busy now
+;
- tlbie r9 /* Invalidate it everywhere */
+hrmPysDQ32: mr r3,r31 ; Point to the mapping
+ bl mapDrainBusy ; Go wait until mapping is unused
-
- beq- cr1,its603a /* It's a 603, skip the tlbsync... */
-
- eieio /* Make sure that the tlbie happens first */
- tlbsync /* wait for everyone to catch up */
- isync
-
-its603a: sync /* Make sure of it all */
- stw r11,0(r12) /* Clear the tlbie lock */
- eieio /* Make sure those RC bit are loaded */
- stw r5,PCAallo(r7) /* Show that the slot is free */
- stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
+ mr r3,r28 ; Get the pmap to remove from
+ mr r4,r31 ; Point to the mapping
+ bl EXT(mapRemove) ; Remove the mapping from the list
-nopte: mr. r10,r10 /* See if there is a physical entry */
- la r9,pephyslink(r10) /* Point to the physical mapping chain */
- beq- nophys /* No physical entry, we're done... */
- beq- cr5,nadamrg /* No PTE to merge... */
+ lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
+ rlwinm r0,r20,0,mpType ; Isolate mapping type
+ cmplwi cr1,r0,mpMinSpecial ; cr1_lt <- not a special mapping type
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ subi r4,r4,1 ; Drop down the mapped page count
+ stw r4,pmapResidentCnt(r28) ; Set the mapped page count
+ bl sxlkUnlock ; Unlock the search list
- lwz r6,4(r4) /* Get the latest reference and change bits */
- la r12,pepte1(r10) /* Point right at the master copy */
- rlwinm r6,r6,0,23,24 /* Extract just the RC bits */
+ bf-- cr1_lt,hrmRetn32 ; This one has no real memory associated with it so we are done...
-mrgrc: lwarx r8,0,r12 /* Get the master copy */
- or r8,r8,r6 /* Merge in latest RC */
- stwcx. r8,0,r12 /* Save it back */
- bne- mrgrc /* If it changed, try again... */
-
-nadamrg: li r11,0 /* Clear this out */
- lwz r12,mmnext(r3) /* Prime with our next */
+ bl mapPhysFindLock ; Go find and lock the physent
- sync ; Make sure all is saved
+ lwz r9,ppLink+4(r3) ; Get first mapping
- stw r11,0(r7) /* Unlock the hash chain now so we don't
- lock out another processor during
- our next little search */
+ mr r4,r22 ; Get the RC bits we just got
+ bl mapPhysMerge ; Go merge the RC bits
+
+ rlwinm r9,r9,0,~ppFlags ; Clear the flags from the mapping pointer
-srchpmap: mr. r10,r9 /* Save the previous entry */
- bne+ mapok1 /* No error... */
+ cmplw r9,r31 ; Are we the first on the list?
+ bne- hrmNot1st ; Nope...
- lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
- ori r0,r0,LOW_ADDR(Choke)
- sc /* Firmware Heimlich maneuver */
+ li r9,0 ; Get a 0
+ lwz r4,mpAlias+4(r31) ; Get our new forward pointer
+ stw r9,mpAlias+4(r31) ; Make sure we are off the chain
+ bl mapPhyCSet32 ; Go set the physent link and preserve flags
- .align 4
+ b hrmPhyDQd ; Join up and unlock it all...
-mapok1: lwz r9,mmnext(r9) /* Look at the next one */
- rlwinm r8,r9,0,27,31 ; Save the flags (including the lock)
- rlwinm r9,r9,0,0,26 ; Clear out the flags from first link
- cmplw r9,r3 /* Have we found ourselves? */
- bne+ srchpmap /* Nope, get your head together... */
+ .align 5
- rlwimi r12,r8,0,27,31 ; Insert the lock and flags */
- stw r12,mmnext(r10) /* Remove us from the queue */
+hrmPerm: li r8,-4096 ; Get the value we need to round down to a page
+ and r8,r8,r31 ; Get back to a page
+ lwz r8,mbvrswap+4(r8) ; Get last half of virtual to real swap
- mtmsr r0 /* Interrupts and translation back on */
- isync
-#if PERFTIMES && DEBUG
- mflr r11
- li r3,25
- bl EXT(dbgLog2) ; Start of hw_add_map
- mtlr r11
-#endif
- blr /* Return... */
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ xor r3,r31,r8 ; Flip mapping address to virtual
+ ori r3,r3,mapRtPerm ; Set permanent mapping error
+ b hrmErRtn
+
+hrmBadLock: li r3,mapRtBadLk ; Set bad lock
+ b hrmErRtn
+
+hrmEndInSight:
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+hrmDoneChunk:
+ mr r3,r31 ; Point to the mapping
+ bl mapDropBusy ; Drop the busy here since we need to come back
+ li r3,mapRtRemove ; Say we are still removing this
+ b hrmErRtn
- .align 4
+ .align 5
+
+hrmNotFound:
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+ li r3,mapRtNotFnd ; No mapping found
-nophys: li r4,0 /* Make sure this is 0 */
- sync /* Make sure that chain is updated */
- stw r4,0(r7) /* Unlock the hash chain */
- mtmsr r0 /* Interrupts and translation back on */
- isync
-#if PERFTIMES && DEBUG
- mflr r11
- li r3,25
- bl EXT(dbgLog2) ; Start of hw_add_map
- mtlr r11
-#endif
- blr /* Return... */
+hrmErRtn: bt++ pf64Bitb,hrmSF1z ; skip if 64-bit (only they take the hint)
+ mtmsr r17 ; Restore enables/translation/etc.
+ isync
+ b hrmRetnCmn ; Join the common return code...
-/*
- * hw_prot(physent, prot) - Change the protection of a physical page
- *
- * Upon entry, R3 contains a pointer to a physical entry which is locked.
- * R4 contains the PPC protection bits.
- *
- * The first thing we do is to slam the new protection into the phys entry.
- * Then we scan the mappings and process each one.
- *
- * Acquire the lock on the PTEG hash list for the mapping being processed.
- *
- * If the current mapping has a PTE entry, we invalidate
- * it and merge the reference and change information into the phys_entry.
- *
- * Next, slam the protection bits into the entry and unlock the hash list.
- *
- * Note that this must be done with both interruptions off and VM off
- *
- *
- */
+hrmSF1z: mtmsrd r17 ; Restore enables/translation/etc.
+ isync
+ b hrmRetnCmn ; Join the common return code...
.align 5
- .globl EXT(hw_prot)
-LEXT(hw_prot)
-#if PERFTIMES && DEBUG
- mflr r11
- mr r7,r3
-// lwz r5,4(r3)
- li r5,0x1111
- li r3,26
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r7
- mtlr r11
-#endif
- mfsprg r9,2 ; Get feature flags
- mfmsr r0 /* Save the MSR */
- li r5,pepte1 /* Get displacement to the second word of master pte */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
-
- bt pfNoMSRirb,hpNoMSR ; No MSR...
-
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hpNoMSRx
+hrmNot1st: mr. r8,r9 ; Remember and test current node
+ beq- hrmPhyDQd ; Could not find our node, someone must have unmapped us...
+ lwz r9,mpAlias+4(r9) ; Chain to the next
+ cmplw r9,r31 ; Is this us?
+ bne- hrmNot1st ; Not us...
+
+ lwz r9,mpAlias+4(r9) ; Get our forward pointer
+ stw r9,mpAlias+4(r8) ; Unchain us
-hpNoMSR:
- mr r10,r0
- mr r7,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r0,r10
- mr r3,r7
-hpNoMSRx:
+ nop ; For alignment
+
+hrmPhyDQd: bl mapPhysUnlock ; Unlock the physent chain
+hrmRetn32: rlwinm r8,r31,0,0,19 ; Find start of page
+ mr r3,r31 ; Copy the pointer to the mapping
+ lwz r8,mbvrswap+4(r8) ; Get last half of virtual to real swap
+ bl mapDrainBusy ; Go wait until mapping is unused
+ xor r3,r31,r8 ; Flip mapping address to virtual
- lwz r10,pephyslink(r3) /* Get the first mapping block */
- rlwinm r10,r10,0,0,26 ; Clear out the flags from first link
-
-/*
- * Note that we need to to do the interlocked update here because another processor
- * can be updating the reference and change bits even though the physical entry
- * is locked. All modifications to the PTE portion of the physical entry must be
- * done via interlocked update.
- */
+ mtmsr r17 ; Restore enables/translation/etc.
+ isync
-protcng: lwarx r8,r5,r3 /* Get the master copy */
- rlwimi r8,r4,0,30,31 /* Move in the protection bits */
- stwcx. r8,r5,r3 /* Save it back */
- bne- protcng /* If it changed, try again... */
+hrmRetnCmn: lwz r6,FM_ARG0+0x44(r1) ; Get address to save next mapped vaddr
+ lwz r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
+ lwz r17,FM_ARG0+0x08(r1) ; Restore a register
+ lwz r18,FM_ARG0+0x0C(r1) ; Restore a register
+ mr. r6,r6 ; Should we pass back the "next" vaddr?
+ lwz r19,FM_ARG0+0x10(r1) ; Restore a register
+ lwz r20,FM_ARG0+0x14(r1) ; Restore a register
+ mtlr r0 ; Restore the return
+
+ rlwinm r16,r16,0,0,19 ; Clean to a page boundary
+ beq hrmNoNextAdr ; Do not pass back the next vaddr...
+ stw r15,0(r6) ; Pass back the top of the next vaddr
+ stw r16,4(r6) ; Pass back the bottom of the next vaddr
+
+hrmNoNextAdr:
+ lwz r15,FM_ARG0+0x00(r1) ; Restore a register
+ lwz r16,FM_ARG0+0x04(r1) ; Restore a register
+ lwz r21,FM_ARG0+0x18(r1) ; Restore a register
+ rlwinm r3,r3,0,0,31 ; Clear top of register if 64-bit
+ lwz r22,FM_ARG0+0x1C(r1) ; Restore a register
+ lwz r23,FM_ARG0+0x20(r1) ; Restore a register
+ lwz r24,FM_ARG0+0x24(r1) ; Restore a register
+ lwz r25,FM_ARG0+0x28(r1) ; Restore a register
+ lwz r26,FM_ARG0+0x2C(r1) ; Restore a register
+ lwz r27,FM_ARG0+0x30(r1) ; Restore a register
+ lwz r28,FM_ARG0+0x34(r1) ; Restore a register
+ lwz r29,FM_ARG0+0x38(r1) ; Restore a register
+ lwz r30,FM_ARG0+0x3C(r1) ; Restore a register
+ lwz r31,FM_ARG0+0x40(r1) ; Restore a register
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
+;
+; Here is where we come when all is lost. Somehow, we failed a mapping function
+; that must work... All hope is gone. Alas, we die.......
+;
+hrmPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
-protnext: mr. r10,r10 /* Are there any more mappings? */
- beq- protdone /* Naw... */
-
- lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
- lwz r5,mmPTEv(r10) /* Get the virtual address */
- rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
- li r12,1 /* Get the locked value */
+;
+; Invalidate block mappings by invalidating a chunk of autogen PTEs in PTEGs hashed
+; in the range. Then, if we did not finish, return a code indicating that we need to
+; be called again. Eventually, we will finish and then, we will do a TLBIE for each
+; PTEG up to the point where we have cleared it all (64 for 32-bit architecture)
+;
+; A potential speed up is that we stop the invalidate loop once we have walked through
+; the hash table once. This really is not worth the trouble because we need to have
+; mapped 1/2 of physical RAM in an individual block. Way unlikely.
+;
+; We should rethink this and see if we think it will be faster to check PTE and
+; only invalidate the specific PTE rather than all block map PTEs in the PTEG.
+;
-protLck1: lwarx r11,0,r7 /* Get the PTEG lock */
- mr. r11,r11 /* Is it locked? */
- bne- protLckw1 /* Yeah... */
- stwcx. r12,0,r7 /* Try to take it */
- bne- protLck1 /* Someone else was trying, try again... */
- b protSXg1 /* All done... */
+ .align 5
- .align 4
+hrmBlock32: lis r29,0xD000 ; Get shift to 32MB bsu
+ rlwinm r24,r20,mpBSub+1+2,29,29 ; Rotate to get 0 if 4K bsu or 13 if 32MB bsu
+ lhz r25,mpBSize(r31) ; Get the number of pages in block
+ lhz r23,mpSpace(r31) ; Get the address space hash
+ lwz r9,mpBlkRemCur(r31) ; Get our current remove position
+ rlwnm r29,r29,r24,28,31 ; Rotate to get 0 or 13
+ addi r25,r25,1 ; Account for zero-based counting
+ ori r0,r20,mpRIP ; Turn on the remove in progress flag
+ slw r25,r25,r29 ; Adjust for 32MB if needed
+ mfsdr1 r29 ; Get the hash table base and size
+ rlwinm r24,r23,maxAdrSpb,32-maxAdrSpb-maxAdrSpb,31-maxAdrSpb ; Get high order of hash
+ subi r25,r25,1 ; Convert back to zero-based counting
+ lwz r27,mpVAddr+4(r31) ; Get the base vaddr
+ sub r4,r25,r9 ; Get number of pages left
+ cmplw cr1,r9,r25 ; Have we already hit the end?
+ addi r10,r9,mapRemChunk ; Point to the start of the next chunk
+ addi r2,r4,-mapRemChunk ; See if mapRemChunk or more
+ rlwinm r26,r29,16,7,15 ; Get the hash table size
+ srawi r2,r2,31 ; We have -1 if less than mapRemChunk or 0 if equal or more
+ stb r0,mpFlags+3(r31) ; Save the flags with the mpRIP bit on
+ subi r4,r4,mapRemChunk-1 ; Back off for a running start (will be negative for more than mapRemChunk)
+ cmpwi cr7,r2,0 ; Remember if we have finished
+ slwi r0,r9,12 ; Make cursor into page offset
+ or r24,r24,r23 ; Get full hash
+ and r4,r4,r2 ; If more than a chunk, bring this back to 0
+ rlwinm r29,r29,0,0,15 ; Isolate the hash table base
+ add r27,r27,r0 ; Adjust vaddr to start of current chunk
+ addi r4,r4,mapRemChunk-1 ; Add mapRemChunk-1 to get max(num left, chunksize)
+
+ bgt- cr1,hrmEndInSight ; Someone is already doing the last hunk...
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ stw r10,mpBlkRemCur(r31) ; Set next chunk to do (note: this may indicate after end)
+ bl sxlkUnlock ; Unlock the search list while we are invalidating
+
+ rlwinm r8,r27,4+maxAdrSpb,31-maxAdrSpb-3,31-maxAdrSpb ; Isolate the segment
+ rlwinm r30,r27,26,6,25 ; Shift vaddr to PTEG offset (and remember VADDR in R27)
+ xor r24,r24,r8 ; Get the proper VSID
+ rlwinm r21,r27,26,10,25 ; Shift page index to PTEG offset (and remember VADDR in R27)
+ ori r26,r26,lo16(0xFFC0) ; Stick in the rest of the length
+ rlwinm r22,r4,6,10,25 ; Shift size to PTEG offset
+ rlwinm r24,r24,6,0,25 ; Shift hash to PTEG units
+ add r22,r22,r30 ; Get end address (in PTEG units)
+
+hrmBInv32: rlwinm r23,r30,0,10,25 ; Isolate just the page index
+ xor r23,r23,r24 ; Hash it
+ and r23,r23,r26 ; Wrap it into the table
+ rlwinm r3,r23,28,4,29 ; Change to PCA offset
+ subfic r3,r3,-4 ; Get the PCA entry offset
+ add r7,r3,r29 ; Point to the PCA slot
+ cmplw cr5,r30,r22 ; Check if we reached the end of the range
+ addi r30,r30,64 ; bump to the next vaddr
+
+ bl mapLockPteg ; Lock the PTEG
+
+ rlwinm. r4,r6,16,0,7 ; Position, save, and test block mappings in PCA
+ add r5,r23,r29 ; Point to the PTEG
+ li r0,0 ; Set an invalid PTE value
+ beq+ hrmBNone32 ; No block map PTEs in this PTEG...
+ mtcrf 0x80,r4 ; Set CRs to select PTE slots
+ mtcrf 0x40,r4 ; Set CRs to select PTE slots
-protLckw1: mr. r11,r11 /* Check if it's already held */
- beq+ protLck1 /* It's clear... */
- lwz r11,0(r7) /* Get lock word again... */
- b protLckw1 /* Wait... */
-
- .align 4
+ bf 0,hrmSlot0 ; No autogen here
+ stw r0,0x00(r5) ; Invalidate PTE
-protSXg1: isync /* Make sure we haven't used anything yet */
+hrmSlot0: bf 1,hrmSlot1 ; No autogen here
+ stw r0,0x08(r5) ; Invalidate PTE
- lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
+hrmSlot1: bf 2,hrmSlot2 ; No autogen here
+ stw r0,0x10(r5) ; Invalidate PTE
- rlwinm r9,r5,1,0,3 /* Move in the segment */
- lwz r2,mmPTEr(r10) ; Get the mapping copy of the PTE
- mr. r6,r6 /* See if there is a PTE here */
- rlwinm r8,r5,31,2,25 /* Line it up */
- rlwimi r2,r4,0,30,31 ; Move protection bits into the mapping copy
-
- beq+ protul /* There's no PTE to invalidate... */
-
- xor r8,r8,r6 /* Back hash to virt index */
- rlwimi r9,r5,22,4,9 /* Move in the API */
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
- rlwinm r5,r5,0,1,31 /* Clear the valid bit */
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
- mfspr r11,pvr /* Find out what kind of machine we are */
- rlwimi r9,r8,6,10,19 /* Create the virtual address */
- rlwinm r11,r11,16,16,31 /* Isolate CPU type */
-
- stw r5,0(r6) /* Make the PTE invalid */
- cmplwi cr1,r11,3 /* Is this a 603? */
- sync /* Make sure the invalid is stored */
-
-tlbhangp: lwarx r11,0,r12 /* Get the TLBIE lock */
- rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
- mr. r11,r11 /* Is it locked? */
- lis r5,0x8000 /* Start up a bit mask */
- li r11,1 /* Get our lock word */
- bne- tlbhangp /* It's locked, go wait... */
- stwcx. r11,0,r12 /* Try to get it */
- bne- tlbhangp /* We was beat... */
-
- li r11,0 /* Lock clear value */
+hrmSlot2: bf 3,hrmSlot3 ; No autogen here
+ stw r0,0x18(r5) ; Invalidate PTE
- tlbie r9 /* Invalidate it everywhere */
+hrmSlot3: bf 4,hrmSlot4 ; No autogen here
+ stw r0,0x20(r5) ; Invalidate PTE
- beq- cr1,its603p /* It's a 603, skip the tlbsync... */
-
- eieio /* Make sure that the tlbie happens first */
- tlbsync /* wait for everyone to catch up */
- isync
-
-its603p: stw r11,0(r12) /* Clear the lock */
- srw r5,r5,r8 /* Make a "free slot" mask */
- sync /* Make sure of it all */
+hrmSlot4: bf 5,hrmSlot5 ; No autogen here
+ stw r0,0x28(r5) ; Invalidate PTE
- lwz r6,4(r6) /* Get the latest reference and change bits */
- stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
- rlwinm r6,r6,0,23,24 /* Extract the RC bits */
- lwz r9,PCAallo(r7) /* Get the allocation control bits */
- rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
- rlwimi r2,r6,0,23,24 ; Put the latest RC bit in mapping copy
- or r9,r9,r5 /* Set the slot free */
- rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
- andc r9,r9,r8 /* Clear the auto and lock bits */
- li r5,pepte1 /* Get displacement to the second word of master pte */
- stw r9,PCAallo(r7) /* Store the allocation controls */
-
-protmod: lwarx r11,r5,r3 /* Get the master copy */
- or r11,r11,r6 /* Merge in latest RC */
- stwcx. r11,r5,r3 /* Save it back */
- bne- protmod /* If it changed, try again... */
-
-protul: li r4,0 /* Get a 0 */
- stw r2,mmPTEr(r10) ; Save the updated mapping PTE
- lwz r10,mmnext(r10) /* Get the next */
+hrmSlot5: bf 6,hrmSlot6 ; No autogen here
+ stw r0,0x30(r5) ; Invalidate PTE
- sync ; Make sure stores are complete
+hrmSlot6: bf 7,hrmSlot7 ; No autogen here
+ stw r0,0x38(r5) ; Invalidate PTE
- stw r4,0(r7) /* Unlock the hash chain */
- b protnext /* Go get the next one */
-
- .align 4
+hrmSlot7: rlwinm r0,r4,16,16,23 ; Move in use to autogen
+ or r6,r6,r4 ; Flip on the free bits that corrospond to the autogens we cleared
+ andc r6,r6,r0 ; Turn off all the old autogen bits
-protdone: mtmsr r0 /* Interrupts and translation back on */
- isync
-#if PERFTIMES && DEBUG
- mflr r11
- li r3,27
- bl EXT(dbgLog2) ; Start of hw_add_map
- mtlr r11
-#endif
- blr /* Return... */
+hrmBNone32: eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock and set the PCA
+
+ bne+ cr5,hrmBInv32 ; Go invalidate the next...
-/*
- * hw_prot_virt(mapping, prot) - Change the protection of single page
- *
- * Upon entry, R3 contains a pointer (real) to a mapping.
- * R4 contains the PPC protection bits.
- *
- * Acquire the lock on the PTEG hash list for the mapping being processed.
- *
- * If the current mapping has a PTE entry, we invalidate
- * it and merge the reference and change information into the phys_entry.
- *
- * Next, slam the protection bits into the entry, merge the RC bits,
- * and unlock the hash list.
- *
- * Note that this must be done with both interruptions off and VM off
- *
- *
- */
+ bge+ cr7,hrmDoneChunk ; We have not as yet done the last chunk, go tell our caller to call again...
- .align 5
- .globl EXT(hw_prot_virt)
+ mr r3,r31 ; Copy the pointer to the mapping
+ bl mapDrainBusy ; Go wait until we are sure all other removers are done with this one
-LEXT(hw_prot_virt)
-#if PERFTIMES && DEBUG
- mflr r11
- mr r7,r3
-// lwz r5,4(r3)
- li r5,0x1111
- li r3,40
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r7
- mtlr r11
-#endif
- mfsprg r9,2 ; Get feature flags
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
+ sync ; Make sure memory is consistent
- bt pfNoMSRirb,hpvNoMSR ; No MSR...
-
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hpvNoMSRx
+ subi r5,r25,63 ; Subtract TLB size from page count (note we are 0 based here)
+ li r6,63 ; Assume full invalidate for now
+ srawi r5,r5,31 ; Make 0 if we need a full purge, -1 otherwise
+ andc r6,r6,r5 ; Clear max if we have less to do
+ and r5,r25,r5 ; Clear count if we have more than max
+ lwz r27,mpVAddr+4(r31) ; Get the base vaddr again
+ li r7,tlbieLock ; Get the TLBIE lock
+ or r5,r5,r6 ; Get number of TLBIEs needed
+
+hrmBTLBlck: lwarx r2,0,r7 ; Get the TLBIE lock
+ mr. r2,r2 ; Is it locked?
+ li r2,1 ; Get our lock value
+ bne- hrmBTLBlck ; It is locked, go wait...
+ stwcx. r2,0,r7 ; Try to get it
+ bne- hrmBTLBlck ; We was beat...
+
+hrmBTLBi: addic. r5,r5,-1 ; See if we did them all
+ tlbie r27 ; Invalidate it everywhere
+ addi r27,r27,0x1000 ; Up to the next page
+ bge+ hrmBTLBi ; Make sure we have done it all...
-hpvNoMSR:
- mr r5,r0
- mr r7,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r7
- mr r0,r5
-hpvNoMSRx:
-
+ rlwinm. r0,r19,0,pfSMPcapb,pfSMPcapb ; Can this processor do SMP?
+ li r2,0 ; Lock clear value
+
+ sync ; Make sure all is quiet
+ beq- hrmBNTlbs ; Jump if we can not do a TLBSYNC....
+
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; Wait for everyone to catch up
+ sync ; Wait for quiet again
+
+hrmBNTlbs: stw r2,tlbieLock(0) ; Clear the tlbie lock
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne- hrmPanic ; Nope...
+
+ lwz r4,mpVAddr(r31) ; High order of address
+ lwz r5,mpVAddr+4(r31) ; Low order of address
+ mr r3,r28 ; Pass in pmap to search
+ mr r29,r4 ; Save this in case we need it (only promote fails)
+ mr r30,r5 ; Save this in case we need it (only promote fails)
+ bl EXT(mapSearchFull) ; Go see if we can find it
+
+ mr. r3,r3 ; Did we? (And remember mapping address for later)
+ mr r15,r4 ; Save top of next vaddr
+ mr r16,r5 ; Save bottom of next vaddr
+ beq- hrmPanic ; Nope, not found...
+
+ cmplw r3,r31 ; Same mapping?
+ bne- hrmPanic ; Not good...
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkPromote ; Try to promote shared to exclusive
+ mr. r3,r3 ; Could we?
+ mr r3,r31 ; Restore the mapping pointer
+ beq+ hrmBDone1 ; Yeah...
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkConvert ; Convert shared to exclusive
+ mr. r3,r3 ; Could we?
+ bne-- hrmPanic ; Nope, we must have timed out...
+
+ mr r3,r28 ; Pass in pmap to search
+ mr r4,r29 ; High order of address
+ mr r5,r30 ; Low order of address
+ bl EXT(mapSearchFull) ; Rescan the list
+
+ mr. r3,r3 ; Did we lose it when we converted?
+ mr r15,r4 ; Save top of next vaddr
+ mr r16,r5 ; Save bottom of next vaddr
+ beq-- hrmPanic ; Yeah, we did, someone tossed it for us...
+
+hrmBDone1: bl mapDrainBusy ; Go wait until mapping is unused
+
+ mr r3,r28 ; Get the pmap to remove from
+ mr r4,r31 ; Point to the mapping
+ bl EXT(mapRemove) ; Remove the mapping from the list
+
+ lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ subi r4,r4,1 ; Drop down the mapped page count
+ stw r4,pmapResidentCnt(r28) ; Set the mapped page count
+ bl sxlkUnlock ; Unlock the search list
+
+ b hrmRetn32 ; We are all done, get out...
+;
+; Here we handle the 64-bit version of hw_rem_map
+;
+
+ .align 5
+
+hrmSplit64: rlwinm r9,r21,27,5,29 ; Convert PTEG to PCA entry
+ beq-- cr5,hrmBlock64 ; Go treat block specially...
+ subfic r9,r9,-4 ; Get the PCA entry offset
+ bt-- cr0_eq,hrmPysDQ64 ; Skip next if no possible PTE...
+ add r7,r9,r29 ; Point to the PCA slot
-/*
- * Note that we need to to do the interlocked update here because another processor
- * can be updating the reference and change bits even though the physical entry
- * is locked. All modifications to the PTE portion of the physical entry must be
- * done via interlocked update.
- */
+ bl mapLockPteg ; Go lock up the PTEG
+
+ lwz r21,mpPte(r31) ; Get the quick pointer again
+ ld r5,0(r26) ; Get the top of PTE
+
+ rlwinm. r0,r21,0,mpHValidb,mpHValidb ; See if we actually have a PTE
+ rlwinm r21,r21,0,~mpHValid ; Clear out valid bit
+ sldi r23,r5,16 ; Shift AVPN up to EA format
+// **** Need to adjust above shift based on the page size - large pages need to shift a bit more
+ rldicr r5,r5,0,62 ; Clear the valid bit
+ rldimi r23,r30,0,36 ; Insert the page portion of the VPN
+ stw r21,mpPte(r31) ; Make sure we invalidate mpPte but keep pointing to PTEG (keep walk_page from making a mistake)
+ beq-- hrmUlckPCA64 ; Pte is gone, no need to invalidate...
+
+ std r5,0(r26) ; Invalidate the PTE
+
+ li r9,tlbieLock ; Get the TLBIE lock
+
+ sync ; Make sure the invalid PTE is actually in memory
+
+hrmPtlb64: lwarx r5,0,r9 ; Get the TLBIE lock
+ rldicl r23,r23,0,16 ; Clear bits 0:15 cause they say to
+ mr. r5,r5 ; Is it locked?
+ li r5,1 ; Get locked indicator
+ bne-- hrmPtlb64w ; It is locked, go spin...
+ stwcx. r5,0,r9 ; Try to get it
+ bne-- hrmPtlb64 ; We was beat...
+
+ tlbie r23 ; Invalidate all corresponding TLB entries
+
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; Wait for everyone to catch up
- lwz r7,mmPTEhash(r3) /* Get pointer to hash list anchor */
- lwz r5,mmPTEv(r3) /* Get the virtual address */
- rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
+ ptesync ; Make sure of it all
+ li r0,0 ; Clear this
+ rlwinm r2,r21,28,29,31 ; Get slot number (16 byte entries)
+ stw r0,tlbieLock(0) ; Clear the tlbie lock
+ oris r0,r0,0x8000 ; Assume slot 0
- li r12,1 /* Get the locked value */
+ srw r0,r0,r2 ; Get slot mask to deallocate
-protvLck1: lwarx r11,0,r7 /* Get the PTEG lock */
- mr. r11,r11 /* Is it locked? */
- bne- protvLckw1 /* Yeah... */
- stwcx. r12,0,r7 /* Try to take it */
- bne- protvLck1 /* Someone else was trying, try again... */
- b protvSXg1 /* All done... */
+ lwz r22,12(r26) ; Get the latest reference and change bits
+ or r6,r6,r0 ; Make the guy we killed free
- .align 4
+hrmUlckPCA64:
+ eieio ; Make sure all updates come first
-protvLckw1: mr. r11,r11 /* Check if it's already held */
- beq+ protvLck1 /* It's clear... */
- lwz r11,0(r7) /* Get lock word again... */
- b protvLckw1 /* Wait... */
-
- .align 4
+ stw r6,0(r7) ; Unlock and change the PCA
+
+hrmPysDQ64: mr r3,r31 ; Point to the mapping
+ bl mapDrainBusy ; Go wait until mapping is unused
+
+ mr r3,r28 ; Get the pmap to remove from
+ mr r4,r31 ; Point to the mapping
+ bl EXT(mapRemove) ; Remove the mapping from the list
+
+ rlwinm r0,r20,0,mpType ; Isolate mapping type
+ cmplwi cr1,r0,mpMinSpecial ; cr1_lt <- not a special mapping type
+ lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ subi r4,r4,1 ; Drop down the mapped page count
+ stw r4,pmapResidentCnt(r28) ; Set the mapped page count
+ bl sxlkUnlock ; Unlock the search list
+
+ bf-- cr1_lt,hrmRetn64 ; This one has no real memory associated with it so we are done...
-protvSXg1: isync /* Make sure we haven't used anything yet */
+ bl mapPhysFindLock ; Go find and lock the physent
- lwz r6,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
- lwz r2,mmPTEr(r3) ; Get the mapping copy if the real part
+ li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ ld r9,ppLink(r3) ; Get first mapping
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ mr r4,r22 ; Get the RC bits we just got
+
+ bl mapPhysMerge ; Go merge the RC bits
+
+ andc r9,r9,r0 ; Clean up the mapping pointer
+
+ cmpld r9,r31 ; Are we the first on the list?
+ bne-- hrmNot1st64 ; Nope...
+
+ li r9,0 ; Get a 0
+ ld r4,mpAlias(r31) ; Get our forward pointer
+
+ std r9,mpAlias(r31) ; Make sure we are off the chain
+ bl mapPhyCSet64 ; Go set the physent link and preserve flags
- rlwinm r9,r5,1,0,3 /* Move in the segment */
- cmplwi cr7,r6,0 ; Any PTE to invalidate?
- rlwimi r2,r4,0,30,31 ; Move in the new protection bits
- rlwinm r8,r5,31,2,25 /* Line it up */
-
- beq+ cr7,pvnophys /* There's no PTE to invalidate... */
-
- xor r8,r8,r6 /* Back hash to virt index */
- rlwimi r9,r5,22,4,9 /* Move in the API */
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
- rlwinm r5,r5,0,1,31 /* Clear the valid bit */
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
- mfspr r11,pvr /* Find out what kind of machine we are */
- rlwimi r9,r8,6,10,19 /* Create the virtual address */
- rlwinm r11,r11,16,16,31 /* Isolate CPU type */
-
- stw r5,0(r6) /* Make the PTE invalid */
- cmplwi cr1,r11,3 /* Is this a 603? */
- sync /* Make sure the invalid is stored */
+ b hrmPhyDQd64 ; Join up and unlock it all...
+
+hrmPtlb64w: li r5,lgKillResv ; Point to some spare memory
+ stwcx. r5,0,r5 ; Clear the pending reservation
-tlbhangpv: lwarx r11,0,r12 /* Get the TLBIE lock */
- rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
- mr. r11,r11 /* Is it locked? */
- lis r5,0x8000 /* Start up a bit mask */
- li r11,1 /* Get our lock word */
- bne- tlbhangpv /* It's locked, go wait... */
- stwcx. r11,0,r12 /* Try to get it */
- bne- tlbhangpv /* We was beat... */
- li r11,0 /* Lock clear value */
+hrmPtlb64x: lwz r5,0(r9) ; Do a regular load to avoid taking reservation
+ mr. r5,r5 ; is it locked?
+ beq++ hrmPtlb64 ; Nope...
+ b hrmPtlb64x ; Sniff some more...
+
+ .align 5
+
+hrmNot1st64:
+ mr. r8,r9 ; Remember and test current node
+ beq-- hrmPhyDQd64 ; Could not find our node...
+ ld r9,mpAlias(r9) ; Chain to the next
+ cmpld r9,r31 ; Is this us?
+ bne-- hrmNot1st64 ; Not us...
+
+ ld r9,mpAlias(r9) ; Get our forward pointer
+ std r9,mpAlias(r8) ; Unchain us
+
+ nop ; For alignment
+
+hrmPhyDQd64:
+ bl mapPhysUnlock ; Unlock the physent chain
- tlbie r9 /* Invalidate it everywhere */
+hrmRetn64: rldicr r8,r31,0,51 ; Find start of page
+ mr r3,r31 ; Copy the pointer to the mapping
+ lwz r8,mbvrswap+4(r8) ; Get last half of virtual to real swap
+ bl mapDrainBusy ; Go wait until mapping is unused
- beq- cr1,its603pv /* It's a 603, skip the tlbsync... */
+ xor r3,r31,r8 ; Flip mapping address to virtual
- eieio /* Make sure that the tlbie happens first */
- tlbsync /* wait for everyone to catch up */
- isync
-
-its603pv: stw r11,0(r12) /* Clear the lock */
- srw r5,r5,r8 /* Make a "free slot" mask */
- sync /* Make sure of it all */
-
- lwz r6,4(r6) /* Get the latest reference and change bits */
- stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
- rlwinm r6,r6,0,23,24 /* Extract the RC bits */
- lwz r9,PCAallo(r7) /* Get the allocation control bits */
- rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
- lwz r10,mmphysent(r3) ; Get any physical entry
- or r9,r9,r5 /* Set the slot free */
- rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
- andc r9,r9,r8 /* Clear the auto and lock bits */
- mr. r10,r10 ; Is there a physical entry?
- li r5,pepte1 /* Get displacement to the second word of master pte */
- stw r9,PCAallo(r7) /* Store the allocation controls */
- rlwimi r2,r6,0,23,24 ; Stick in RC bits
- beq- pvnophys ; No physical entry...
-
-protvmod: lwarx r11,r5,r10 /* Get the master copy */
- or r11,r11,r6 /* Merge in latest RC */
- stwcx. r11,r5,r10 /* Save it back */
- bne- protvmod /* If it changed, try again... */
-
-pvnophys: li r4,0 /* Get a 0 */
- stw r2,mmPTEr(r3) ; Set the real part of the PTE
-
- sync ; Make sure everything is stored
-
- stw r4,0(r7) /* Unlock the hash chain */
- mtmsr r0 ; Restore interrupts and translation
+ mtmsrd r17 ; Restore enables/translation/etc.
isync
-
-#if PERFTIMES && DEBUG
- mflr r11
- li r3,41
- bl EXT(dbgLog2)
- mtlr r11
-#endif
- blr /* Return... */
-
-
-/*
- * hw_attr_virt(mapping, attr) - Change the attributes of single page
- *
- * Upon entry, R3 contains a pointer (real) to a mapping.
- * R4 contains the WIMG bits.
- *
- * Acquire the lock on the PTEG hash list for the mapping being processed.
- *
- * If the current mapping has a PTE entry, we invalidate
- * it and merge the reference and change information into the phys_entry.
- *
- * Next, slam the WIMG bits into the entry, merge the RC bits,
- * and unlock the hash list.
- *
- * Note that this must be done with both interruptions off and VM off
- *
- *
- */
-
- .align 5
- .globl EXT(hw_attr_virt)
-
-LEXT(hw_attr_virt)
-#if PERFTIMES && DEBUG
- mflr r11
- mr r7,r3
-// lwz r5,4(r3)
- li r5,0x1111
- li r3,40
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r7
- mtlr r11
-#endif
- mfsprg r9,2 ; Get feature flags
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- mtcrf 0x04,r9 ; Set the features
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
-
- bt pfNoMSRirb,havNoMSR ; No MSR...
-
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b havNoMSRx
-
-havNoMSR:
- mr r5,r0
- mr r7,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r7
- mr r0,r5
-havNoMSRx:
-
-/*
- * Note that we need to to do the interlocked update here because another processor
- * can be updating the reference and change bits even though the physical entry
- * is locked. All modifications to the PTE portion of the physical entry must be
- * done via interlocked update.
- */
- lwz r7,mmPTEhash(r3) /* Get pointer to hash list anchor */
- lwz r5,mmPTEv(r3) /* Get the virtual address */
- rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
+ b hrmRetnCmn ; Join the common return path...
- li r12,1 /* Get the locked value */
-attrvLck1: lwarx r11,0,r7 /* Get the PTEG lock */
- mr. r11,r11 /* Is it locked? */
- bne- attrvLckw1 /* Yeah... */
- stwcx. r12,0,r7 /* Try to take it */
- bne- attrvLck1 /* Someone else was trying, try again... */
- b attrvSXg1 /* All done... */
-
- .align 4
+;
+; Check hrmBlock32 for comments.
+;
-attrvLckw1: mr. r11,r11 /* Check if it's already held */
- beq+ attrvLck1 /* It's clear... */
- lwz r11,0(r7) /* Get lock word again... */
- b attrvLckw1 /* Wait... */
+ .align 5
- .align 4
-
-attrvSXg1: isync /* Make sure we haven't used anything yet */
-
- lwz r6,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
- lwz r2,mmPTEr(r3) ; Get the mapping copy if the real part
-
- rlwinm r9,r5,1,0,3 /* Move in the segment */
- mr. r6,r6 /* See if there is a PTE here */
- rlwimi r2,r4,0,25,28 ; Move in the new attribute bits
- rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
-
- beq+ avnophys /* There's no PTE to invalidate... */
-
- xor r8,r8,r6 /* Back hash to virt index */
- rlwimi r9,r5,22,4,9 /* Move in the API */
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
- rlwinm r5,r5,0,1,31 /* Clear the valid bit */
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
- mfspr r11,pvr /* Find out what kind of machine we are */
- rlwimi r9,r8,6,10,19 /* Create the virtual address */
- rlwinm r11,r11,16,16,31 /* Isolate CPU type */
- stw r5,0(r6) /* Make the PTE invalid */
- cmplwi cr1,r11,3 /* Is this a 603? */
- sync /* Make sure the invalid is stored */
+hrmBlock64: lis r29,0xD000 ; Get shift to 32MB bsu
+ rlwinm r10,r20,mpBSub+1+2,29,29 ; Rotate to get 0 if 4K bsu or 13 if 32MB bsu
+ lhz r24,mpSpace(r31) ; Get the address space hash
+ lhz r25,mpBSize(r31) ; Get the number of pages in block
+ lwz r9,mpBlkRemCur(r31) ; Get our current remove position
+ rlwnm r29,r29,r10,28,31 ; Rotate to get 0 or 13
+ addi r25,r25,1 ; Account for zero-based counting
+ ori r0,r20,mpRIP ; Turn on the remove in progress flag
+ slw r25,r25,r29 ; Adjust for 32MB if needed
+ mfsdr1 r29 ; Get the hash table base and size
+ ld r27,mpVAddr(r31) ; Get the base vaddr
+ subi r25,r25,1 ; Convert back to zero-based counting
+ rlwinm r5,r29,0,27,31 ; Isolate the size
+ sub r4,r25,r9 ; Get number of pages left
+ cmplw cr1,r9,r25 ; Have we already hit the end?
+ addi r10,r9,mapRemChunk ; Point to the start of the next chunk
+ addi r2,r4,-mapRemChunk ; See if mapRemChunk or more
+ stb r0,mpFlags+3(r31) ; Save the flags with the mpRIP bit on
+ srawi r2,r2,31 ; We have -1 if less than mapRemChunk or 0 if equal or more
+ subi r4,r4,mapRemChunk-1 ; Back off for a running start (will be negative for more than mapRemChunk)
+ cmpwi cr7,r2,0 ; Remember if we are doing the last chunk
+ and r4,r4,r2 ; If more than a chunk, bring this back to 0
+ srdi r27,r27,12 ; Change address into page index
+ addi r4,r4,mapRemChunk-1 ; Add mapRemChunk-1 to get max(num left, chunksize)
+ add r27,r27,r9 ; Adjust vaddr to start of current chunk
+
+ bgt-- cr1,hrmEndInSight ; Someone is already doing the last hunk...
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ stw r10,mpBlkRemCur(r31) ; Set next chunk to do (note: this may indicate after end)
+ bl sxlkUnlock ; Unlock the search list while we are invalidating
+
+ rlwimi r24,r24,14,4,17 ; Insert a copy of space hash
+ eqv r26,r26,r26 ; Get all foxes here
+ rldimi r24,r24,28,8 ; Make a couple copies up higher
+ rldicr r29,r29,0,47 ; Isolate just the hash table base
+ subfic r5,r5,46 ; Get number of leading zeros
+ srd r26,r26,r5 ; Shift the size bits over
+ mr r30,r27 ; Get start of chunk to invalidate
+ rldicr r26,r26,0,56 ; Make length in PTEG units
+ add r22,r4,r30 ; Get end page number
+
+hrmBInv64: srdi r0,r30,2 ; Shift page index over to form ESID
+ rldicr r0,r0,0,49 ; Clean all but segment portion
+ rlwinm r2,r30,0,16,31 ; Get the current page index
+ xor r0,r0,r24 ; Form VSID
+ xor r8,r2,r0 ; Hash the vaddr
+ sldi r8,r8,7 ; Make into PTEG offset
+ and r23,r8,r26 ; Wrap into the hash table
+ rlwinm r3,r23,27,5,29 ; Change to PCA offset (table is always 2GB or less so 32-bit instructions work here)
+ subfic r3,r3,-4 ; Get the PCA entry offset
+ add r7,r3,r29 ; Point to the PCA slot
+
+ cmplw cr5,r30,r22 ; Have we reached the end of the range?
+
+ bl mapLockPteg ; Lock the PTEG
-tlbhangav: lwarx r11,0,r12 /* Get the TLBIE lock */
- rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
- mr. r11,r11 /* Is it locked? */
- lis r5,0x8000 /* Start up a bit mask */
- li r11,1 /* Get our lock word */
- bne- tlbhangav /* It's locked, go wait... */
- stwcx. r11,0,r12 /* Try to get it */
- bne- tlbhangav /* We was beat... */
-
- li r11,0 /* Lock clear value */
+ rlwinm. r4,r6,16,0,7 ; Extract the block mappings in this here PTEG and see if there are any
+ add r5,r23,r29 ; Point to the PTEG
+ li r0,0 ; Set an invalid PTE value
+ beq++ hrmBNone64 ; No block map PTEs in this PTEG...
+ mtcrf 0x80,r4 ; Set CRs to select PTE slots
+ mtcrf 0x40,r4 ; Set CRs to select PTE slots
- tlbie r9 /* Invalidate it everywhere */
-
- beq- cr1,its603av /* It's a 603, skip the tlbsync... */
-
- eieio /* Make sure that the tlbie happens first */
- tlbsync /* wait for everyone to catch up */
- isync
-
-its603av: stw r11,0(r12) /* Clear the lock */
- srw r5,r5,r8 /* Make a "free slot" mask */
- sync /* Make sure of it all */
-
- lwz r6,4(r6) /* Get the latest reference and change bits */
- stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
- rlwinm r6,r6,0,23,24 /* Extract the RC bits */
- lwz r9,PCAallo(r7) /* Get the allocation control bits */
- rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
- lwz r10,mmphysent(r3) ; Get any physical entry
- or r9,r9,r5 /* Set the slot free */
- rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
- andc r9,r9,r8 /* Clear the auto and lock bits */
- mr. r10,r10 ; Is there a physical entry?
- li r5,pepte1 /* Get displacement to the second word of master pte */
- stw r9,PCAallo(r7) /* Store the allocation controls */
- rlwimi r2,r6,0,23,24 ; Stick in RC bits
- beq- avnophys ; No physical entry...
-
-attrvmod: lwarx r11,r5,r10 /* Get the master copy */
- or r11,r11,r6 /* Merge in latest RC */
- stwcx. r11,r5,r10 /* Save it back */
- bne- attrvmod /* If it changed, try again... */
-
-avnophys: li r4,0 /* Get a 0 */
- stw r2,mmPTEr(r3) ; Set the real part of the PTE
-
- sync ; Make sure that everything is updated
-
- stw r4,0(r7) /* Unlock the hash chain */
-
- rlwinm r2,r2,0,0,19 ; Clear back to page boundary
-
-attrflsh: cmplwi r4,(4096-32) ; Are we about to do the last line on page?
- dcbst r2,r4 ; Flush cache because we changed attributes
- addi r4,r4,32 ; Bump up cache
- blt+ attrflsh ; Do the whole page...
- sync
- li r4,0
-attrimvl: cmplwi r4,(4096-32) ; Are we about to do the last line on page?
- dcbi r2,r4 ; Invalidate dcache because we changed attributes
- icbi r2,r4 ; Invalidate icache because we changed attributes
- addi r4,r4,32 ; Bump up cache
- blt+ attrimvl ; Do the whole page...
- sync
+ bf 0,hrmSlot0s ; No autogen here
+ std r0,0x00(r5) ; Invalidate PTE
- mtmsr r0 ; Restore interrupts and translation
- isync
+hrmSlot0s: bf 1,hrmSlot1s ; No autogen here
+ std r0,0x10(r5) ; Invalidate PTE
-#if PERFTIMES && DEBUG
- mflr r11
- li r3,41
- bl EXT(dbgLog2)
- mtlr r11
-#endif
- blr /* Return... */
+hrmSlot1s: bf 2,hrmSlot2s ; No autogen here
+ std r0,0x20(r5) ; Invalidate PTE
+hrmSlot2s: bf 3,hrmSlot3s ; No autogen here
+ std r0,0x30(r5) ; Invalidate PTE
-/*
- * hw_pte_comm(physent) - Do something to the PTE pointing to a physical page
- *
- * Upon entry, R3 contains a pointer to a physical entry which is locked.
- * Note that this must be done with both interruptions off and VM off
- *
- * First, we set up CRs 5 and 7 to indicate which of the 7 calls this is.
- *
- * Now we scan the mappings to invalidate any with an active PTE.
- *
- * Acquire the lock on the PTEG hash list for the mapping being processed.
- *
- * If the current mapping has a PTE entry, we invalidate
- * it and merge the reference and change information into the phys_entry.
- *
- * Next, unlock the hash list and go on to the next mapping.
- *
- *
- *
- */
+hrmSlot3s: bf 4,hrmSlot4s ; No autogen here
+ std r0,0x40(r5) ; Invalidate PTE
- .align 5
- .globl EXT(hw_inv_all)
+hrmSlot4s: bf 5,hrmSlot5s ; No autogen here
+ std r0,0x50(r5) ; Invalidate PTE
-LEXT(hw_inv_all)
-
- li r9,0x800 /* Indicate invalidate all */
- li r2,0 ; No inadvertant modifications please
- b hw_pte_comm /* Join in the fun... */
+hrmSlot5s: bf 6,hrmSlot6s ; No autogen here
+ std r0,0x60(r5) ; Invalidate PTE
+hrmSlot6s: bf 7,hrmSlot7s ; No autogen here
+ std r0,0x70(r5) ; Invalidate PTE
- .align 5
- .globl EXT(hw_tst_mod)
+hrmSlot7s: rlwinm r0,r4,16,16,23 ; Move in use to autogen
+ or r6,r6,r4 ; Flip on the free bits that corrospond to the autogens we cleared
+ andc r6,r6,r0 ; Turn off all the old autogen bits
-LEXT(hw_tst_mod)
+hrmBNone64: eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock and set the PCA
- lwz r8,pepte1(r3) ; Get the saved PTE image
- li r9,0x400 /* Indicate test modify */
- li r2,0 ; No inadvertant modifications please
- rlwinm. r8,r8,25,31,31 ; Make change bit into return code
- beq+ hw_pte_comm ; Assume we do not know if it is set...
- mr r3,r8 ; Set the return code
- blr ; Return quickly...
+ addi r30,r30,1 ; bump to the next PTEG
+ bne++ cr5,hrmBInv64 ; Go invalidate the next...
- .align 5
- .globl EXT(hw_tst_ref)
+ bge+ cr7,hrmDoneChunk ; We have not as yet done the last chunk, go tell our caller to call again...
-LEXT(hw_tst_ref)
- lwz r8,pepte1(r3) ; Get the saved PTE image
- li r9,0x200 /* Indicate test reference bit */
- li r2,0 ; No inadvertant modifications please
- rlwinm. r8,r8,24,31,31 ; Make reference bit into return code
- beq+ hw_pte_comm ; Assume we do not know if it is set...
- mr r3,r8 ; Set the return code
- blr ; Return quickly...
+ mr r3,r31 ; Copy the pointer to the mapping
+ bl mapDrainBusy ; Go wait until we are sure all other removers are done with this one
-/*
- * Note that the following are all in one CR for ease of use later
- */
- .align 4
- .globl EXT(hw_set_mod)
+ sync ; Make sure memory is consistent
-LEXT(hw_set_mod)
+ subi r5,r25,255 ; Subtract TLB size from page count (note we are 0 based here)
+ li r6,255 ; Assume full invalidate for now
+ srawi r5,r5,31 ; Make 0 if we need a full purge, -1 otherwise
+ andc r6,r6,r5 ; Clear max if we have less to do
+ and r5,r25,r5 ; Clear count if we have more than max
+ sldi r24,r24,28 ; Get the full XOR value over to segment position
+ ld r27,mpVAddr(r31) ; Get the base vaddr
+ li r7,tlbieLock ; Get the TLBIE lock
+ or r5,r5,r6 ; Get number of TLBIEs needed
- li r9,0x008 /* Indicate set modify bit */
- li r2,0x4 ; Set set C, clear none
- b hw_pte_comm /* Join in the fun... */
-
-
- .align 4
- .globl EXT(hw_clr_mod)
+hrmBTLBlcl: lwarx r2,0,r7 ; Get the TLBIE lock
+ mr. r2,r2 ; Is it locked?
+ li r2,1 ; Get our lock value
+ bne-- hrmBTLBlcm ; It is locked, go wait...
+ stwcx. r2,0,r7 ; Try to get it
+ bne-- hrmBTLBlcl ; We was beat...
+
+hrmBTLBj: sldi r2,r27,maxAdrSpb ; Move to make room for address space ID
+ rldicr r2,r2,0,35-maxAdrSpb ; Clear out the extra
+ addic. r5,r5,-1 ; See if we did them all
+ xor r2,r2,r24 ; Make the VSID
+ rldimi r2,r27,0,36 ; Insert the page portion of the VPN
+ rldicl r2,r2,0,16 ; Clear bits 0:15 cause they say we gotta
-LEXT(hw_clr_mod)
+ tlbie r2 ; Invalidate it everywhere
+ addi r27,r27,0x1000 ; Up to the next page
+ bge++ hrmBTLBj ; Make sure we have done it all...
- li r9,0x004 /* Indicate clear modify bit */
- li r2,0x1 ; Set set none, clear C
- b hw_pte_comm /* Join in the fun... */
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; wait for everyone to catch up
+ li r2,0 ; Lock clear value
- .align 4
- .globl EXT(hw_set_ref)
+ ptesync ; Wait for quiet again
-LEXT(hw_set_ref)
+ stw r2,tlbieLock(0) ; Clear the tlbie lock
- li r9,0x002 /* Indicate set reference */
- li r2,0x8 ; Set set R, clear none
- b hw_pte_comm /* Join in the fun... */
-
- .align 5
- .globl EXT(hw_clr_ref)
-
-LEXT(hw_clr_ref)
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne- hrmPanic ; Nope...
- li r9,0x001 /* Indicate clear reference bit */
- li r2,0x2 ; Set set none, clear R
- b hw_pte_comm /* Join in the fun... */
-
-
-/*
- * This is the common stuff.
- */
-
- .align 5
-
-hw_pte_comm: /* Common routine for pte tests and manips */
-
-#if PERFTIMES && DEBUG
- mflr r11
- mr r7,r3
- lwz r4,4(r3)
- mr r5,r9
- li r3,28
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r7
- mtlr r11
-#endif
- mfsprg r8,2 ; Get feature flags
- lwz r10,pephyslink(r3) /* Get the first mapping block */
- mfmsr r0 /* Save the MSR */
- rlwinm. r10,r10,0,0,26 ; Clear out the flags from first link and see if we are mapped
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r8 ; Set the features
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
- beq- comnmap ; No mapping
- dcbt br0,r10 ; Touch the first mapping in before the isync
-
-comnmap:
-
- bt pfNoMSRirb,hpcNoMSR ; No MSR...
-
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hpcNoMSRx
+ lwz r4,mpVAddr(r31) ; High order of address
+ lwz r5,mpVAddr+4(r31) ; Low order of address
+ mr r3,r28 ; Pass in pmap to search
+ mr r29,r4 ; Save this in case we need it (only promote fails)
+ mr r30,r5 ; Save this in case we need it (only promote fails)
+ bl EXT(mapSearchFull) ; Go see if we can find it
-hpcNoMSR:
- mr r5,r0
- mr r7,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r7
- mr r0,r5
-hpcNoMSRx:
+ mr. r3,r3 ; Did we? (And remember mapping address for later)
+ mr r15,r4 ; Save top of next vaddr
+ mr r16,r5 ; Save bottom of next vaddr
+ beq- hrmPanic ; Nope, not found...
+
+ cmpld r3,r31 ; Same mapping?
+ bne- hrmPanic ; Not good...
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkPromote ; Try to promote shared to exclusive
+ mr. r3,r3 ; Could we?
+ mr r3,r31 ; Restore the mapping pointer
+ beq+ hrmBDone2 ; Yeah...
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkConvert ; Convert shared to exclusive
+ mr. r3,r3 ; Could we?
+ bne-- hrmPanic ; Nope, we must have timed out...
+
+ mr r3,r28 ; Pass in pmap to search
+ mr r4,r29 ; High order of address
+ mr r5,r30 ; Low order of address
+ bl EXT(mapSearchFull) ; Rescan the list
+
+ mr. r3,r3 ; Did we lose it when we converted?
+ mr r15,r4 ; Save top of next vaddr
+ mr r16,r5 ; Save bottom of next vaddr
+ beq-- hrmPanic ; Yeah, we did, someone tossed it for us...
- mtcrf 0x05,r9 /* Set the call type flags into cr5 and 7 */
+hrmBDone2: bl mapDrainBusy ; Go wait until mapping is unused
- beq- commdone ; Nothing us mapped to this page...
- b commnext ; Jump to first pass (jump here so we can align loop)
+ mr r3,r28 ; Get the pmap to remove from
+ mr r4,r31 ; Point to the mapping
+ bl EXT(mapRemove) ; Remove the mapping from the list
+
+ lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ subi r4,r4,1 ; Drop down the mapped page count
+ stw r4,pmapResidentCnt(r28) ; Set the mapped page count
+ bl sxlkUnlock ; Unlock the search list
- .align 5
-
-commnext: lwz r11,mmnext(r10) ; Get the pointer to the next mapping (if any)
- lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
- lwz r5,mmPTEv(r10) /* Get the virtual address */
- mr. r11,r11 ; More mappings to go?
- rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
- beq- commnxtch ; No more mappings...
- dcbt br0,r11 ; Touch the next mapping
-
-commnxtch: li r12,1 /* Get the locked value */
-
-commLck1: lwarx r11,0,r7 /* Get the PTEG lock */
- mr. r11,r11 /* Is it locked? */
- bne- commLckw1 /* Yeah... */
- stwcx. r12,0,r7 /* Try to take it */
- bne- commLck1 /* Someone else was trying, try again... */
- b commSXg1 /* All done... */
+ b hrmRetn64 ; We are all done, get out...
- .align 4
+hrmBTLBlcm: li r2,lgKillResv ; Get space unreserve line
+ stwcx. r2,0,r2 ; Unreserve it
+
+hrmBTLBlcn: lwz r2,0(r7) ; Get the TLBIE lock
+ mr. r2,r2 ; Is it held?
+ beq++ hrmBTLBlcl ; Nope...
+ b hrmBTLBlcn ; Yeah...
-commLckw1: mr. r11,r11 /* Check if it's already held */
- beq+ commLck1 /* It's clear... */
- lwz r11,0(r7) /* Get lock word again... */
- b commLckw1 /* Wait... */
+;
+; Guest shadow assist -- mapping remove
+;
+; Method of operation:
+; o Locate the VMM extension block and the host pmap
+; o Obtain the host pmap's search lock exclusively
+; o Locate the requested mapping in the shadow hash table,
+; exit if not found
+; o If connected, disconnect the PTE and gather R&C to physent
+; o Locate and lock the physent
+; o Remove mapping from physent's chain
+; o Unlock physent
+; o Unlock pmap's search lock
+;
+; Non-volatile registers on entry:
+; r17: caller's msr image
+; r19: sprg2 (feature flags)
+; r28: guest pmap's physical address
+; r29: high-order 32 bits of guest virtual address
+; r30: low-order 32 bits of guest virtual address
+;
+; Non-volatile register usage:
+; r26: VMM extension block's physical address
+; r27: host pmap's physical address
+; r28: guest pmap's physical address
+; r29: physent's physical address
+; r30: guest virtual address
+; r31: guest mapping's physical address
+;
+ .align 5
+hrmGuest:
+ rlwinm r30,r30,0,0xFFFFF000 ; Clean up low-order bits of 32-bit guest vaddr
+ bt++ pf64Bitb,hrmG64 ; Test for 64-bit machine
+ lwz r26,pmapVmmExtPhys+4(r28) ; r26 <- VMM pmap extension block paddr
+ lwz r27,vmxHostPmapPhys+4(r26) ; r27 <- host pmap's paddr
+ b hrmGStart ; Join common code
+
+hrmG64: ld r26,pmapVmmExtPhys(r28) ; r26 <- VMM pmap extension block paddr
+ ld r27,vmxHostPmapPhys(r26) ; r27 <- host pmap's paddr
+ rldimi r30,r29,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+
+hrmGStart: la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ lwz r3,vxsGrm(r26) ; Get mapping remove request count
+
+ lwz r9,pmapSpace(r28) ; r9 <- guest space ID number
+ la r31,VMX_HPIDX_OFFSET(r26) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r9,r11 ; spaceID ^ (vaddr >> 12)
+ rlwinm r12,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r12 ; r31 <- hash page index entry
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,hrmG64Search ; Separate handling for 64-bit search
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+
+ addi r3,r3,1 ; Increment remove request count
+ stw r3,vxsGrm(r26) ; Update remove request count
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b hrmG32SrchLp ; Let the search begin!
+
+ .align 5
+hrmG32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free mapping flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(free && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq hrmGSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz hrmG32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free mapping flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(free && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ beq hrmGSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b hrmGSrchMiss ; No joy in our hash group
+
+hrmG64Search:
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b hrmG64SrchLp ; Let the search begin!
- .align 4
+ .align 5
+hrmG64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free mapping flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(free && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq hrmGSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz hrmG64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free mapping flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(free && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ beq hrmGSrchHit ; Join common path on hit (r31 points to guest mapping)
+hrmGSrchMiss:
+ lwz r3,vxsGrmMiss(r26) ; Get remove miss count
+ li r25,mapRtNotFnd ; Return not found
+ addi r3,r3,1 ; Increment miss count
+ stw r3,vxsGrmMiss(r26) ; Update miss count
+ b hrmGReturn ; Join guest return
+
+ .align 5
+hrmGSrchHit:
+ rlwinm. r0,r6,0,mpgDormant ; Is this entry dormant?
+ bne hrmGDormant ; Yes, nothing to disconnect
+
+ lwz r3,vxsGrmActive(r26) ; Get active hit count
+ addi r3,r3,1 ; Increment active hit count
+ stw r3,vxsGrmActive(r26) ; Update hit count
+
+ bt++ pf64Bitb,hrmGDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b hrmGFreePTE ; Join 64-bit path to release the PTE
+hrmGDscon64:
+ bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+hrmGFreePTE:
+ mr. r3,r3 ; Was there a valid PTE?
+ beq hrmGDormant ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE offset
+ rlwinm r8,r8,0,~mpHValid ; Make the offset invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE offset
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+hrmGDormant:
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r29,r3 ; Got lock on our physent?
+ beq-- hrmGBadPLock ; No, time to bail out
+
+ crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,hrmGRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r29) ; Point to chain anchor
+ lwz r9,ppLink+4(r29) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+hrmGRemLoop:
+ beq- hrmGPEMissMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne hrmGRemNext ; No, chain onward
+ bt cr1_eq,hrmGRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b hrmGDelete ; Finish deleting mapping
+hrmGRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- hrmGRemRetry ; Lost reservation, retry
+ b hrmGDelete ; Finish deleting mapping
+
+hrmGRemNext:
+ la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b hrmGRemLoop ; Carry on
+
+hrmGRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r29) ; Point to chain anchor
+ ld r9,ppLink(r29) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+hrmGRem64Lp:
+ beq-- hrmGPEMissMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne hrmGRem64Nxt ; No mapping to remove, chain on, dude
+ bt cr1_eq,hrmGRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b hrmGDelete ; Finish deleting mapping
+hrmGRem64Rt:
+ ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- hrmGRem64Rt ; Lost reservation, retry
+ b hrmGDelete ; Finish deleting mapping
+
+ .align 5
+hrmGRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b hrmGRem64Lp ; Carry on
+
+hrmGDelete:
+ mr r3,r29 ; r3 <- physent addr
+ bl mapPhysUnlock ; Unlock physent chain
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ rlwinm r3,r3,0,~mpgFlags ; Clear all guest flags
+ ori r3,r3,mpgFree ; Mark mapping free
+ stw r3,mpFlags(r31) ; Update flags
+ li r25,mapRtGuest ; Set return code to 'found guest mapping'
+
+hrmGReturn:
+ la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+ mr r3,r25 ; r3 <- return code
+ bt++ pf64Bitb,hrmGRtn64 ; Handle 64-bit separately
+ mtmsr r17 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b hrmRetnCmn ; Nothing to do now but pop a frame and return
+hrmGRtn64: mtmsrd r17 ; Restore 'rupts, translation, 32-bit mode
+ b hrmRetnCmn ; Join common return
+
+hrmGBadPLock:
+hrmGPEMissMiss:
+ lis r0,hi16(Choke) ; Seen the arrow on the doorpost
+ ori r0,r0,lo16(Choke) ; Sayin' "THIS LAND IS CONDEMNED"
+ li r3,failMapping ; All the way from New Orleans
+ sc ; To Jeruselem
-commSXg1: isync /* Make sure we haven't used anything yet */
- lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
+/*
+ * mapping *hw_purge_phys(physent) - remove a mapping from the system
+ *
+ * Upon entry, R3 contains a pointer to a physent.
+ *
+ * This function removes the first mapping from a physical entry
+ * alias list. It locks the list, extracts the vaddr and pmap from
+ * the first entry. It then jumps into the hw_rem_map function.
+ * NOTE: since we jump into rem_map, we need to set up the stack
+ * identically. Also, we set the next parm to 0 so we do not
+ * try to save a next vaddr.
+ *
+ * We return the virtual address of the removed mapping as a
+ * R3.
+ *
+ * Note that this is designed to be called from 32-bit mode with a stack.
+ *
+ * We disable translation and all interruptions here. This keeps is
+ * from having to worry about a deadlock due to having anything locked
+ * and needing it to process a fault.
+ *
+ * Note that this must be done with both interruptions off and VM off
+ *
+ *
+ * Remove mapping via physical page (mapping_purge)
+ *
+ * 1) lock physent
+ * 2) extract vaddr and pmap
+ * 3) unlock physent
+ * 4) do "remove mapping via pmap"
+ *
+ *
+ */
- rlwinm r9,r5,1,0,3 /* Move in the segment */
- mr. r6,r6 /* See if there is a PTE entry here */
- rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
+ .align 5
+ .globl EXT(hw_purge_phys)
+
+LEXT(hw_purge_phys)
+ stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r15,FM_ARG0+0x00(r1) ; Save a register
+ stw r16,FM_ARG0+0x04(r1) ; Save a register
+ stw r17,FM_ARG0+0x08(r1) ; Save a register
+ stw r18,FM_ARG0+0x0C(r1) ; Save a register
+ stw r19,FM_ARG0+0x10(r1) ; Save a register
+ stw r20,FM_ARG0+0x14(r1) ; Save a register
+ stw r21,FM_ARG0+0x18(r1) ; Save a register
+ stw r22,FM_ARG0+0x1C(r1) ; Save a register
+ stw r23,FM_ARG0+0x20(r1) ; Save a register
+ stw r24,FM_ARG0+0x24(r1) ; Save a register
+ stw r25,FM_ARG0+0x28(r1) ; Save a register
+ li r6,0 ; Set no next address return
+ stw r26,FM_ARG0+0x2C(r1) ; Save a register
+ stw r27,FM_ARG0+0x30(r1) ; Save a register
+ stw r28,FM_ARG0+0x34(r1) ; Save a register
+ stw r29,FM_ARG0+0x38(r1) ; Save a register
+ stw r30,FM_ARG0+0x3C(r1) ; Save a register
+ stw r31,FM_ARG0+0x40(r1) ; Save a register
+ stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
+ stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
+
+ bl mapPhysLock ; Lock the physent
+
+ bt++ pf64Bitb,hppSF ; skip if 64-bit (only they take the hint)
- beq+ commul /* There's no PTE to invalidate... */
-
- xor r8,r8,r6 /* Back hash to virt index */
- rlwimi r9,r5,22,4,9 /* Move in the API */
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
- rlwinm r5,r5,0,1,31 /* Clear the valid bit */
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
- rlwimi r9,r8,6,10,19 /* Create the virtual address */
-
- stw r5,0(r6) /* Make the PTE invalid */
- mfspr r4,pvr /* Find out what kind of machine we are */
- sync /* Make sure the invalid is stored */
-
-tlbhangco: lwarx r11,0,r12 /* Get the TLBIE lock */
- rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
- mr. r11,r11 /* Is it locked? */
- lis r5,0x8000 /* Start up a bit mask */
- li r11,1 /* Get our lock word */
- bne- tlbhangco /* It's locked, go wait... */
- stwcx. r11,0,r12 /* Try to get it */
- bne- tlbhangco /* We was beat... */
+ lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
+ li r0,ppFlags ; Set the bottom stuff to clear
+ b hppJoin ; Join the common...
+
+hppSF: li r0,ppLFAmask
+ ld r12,ppLink(r3) ; Get the pointer to the first mapping
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+
+hppJoin: andc. r12,r12,r0 ; Clean and test link
+ beq-- hppNone ; There are no more mappings on physical page
+
+ lis r28,hi16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
+ lhz r7,mpSpace(r12) ; Get the address space hash
+ ori r28,r28,lo16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
+ slwi r0,r7,2 ; Multiply space by 4
+ lwz r4,mpVAddr(r12) ; Get the top of the vaddr
+ slwi r7,r7,3 ; Multiply space by 8
+ lwz r5,mpVAddr+4(r12) ; and the bottom
+ add r7,r7,r0 ; Get correct displacement into translate table
+ lwz r28,0(r28) ; Get the actual translation map
+
+ add r28,r28,r7 ; Point to the pmap translation
+
+ bl mapPhysUnlock ; Time to unlock the physical entry
- rlwinm r4,r4,16,16,31 /* Isolate CPU type */
- li r11,0 /* Lock clear value */
- cmplwi r4,3 /* Is this a 603? */
-
- tlbie r9 /* Invalidate it everywhere */
-
- beq- its603co /* It's a 603, skip the tlbsync... */
+ bt++ pf64Bitb,hppSF2 ; skip if 64-bit (only they take the hint)
- eieio /* Make sure that the tlbie happens first */
- tlbsync /* wait for everyone to catch up */
- isync
+ lwz r28,pmapPAddr+4(r28) ; Get the physical address of the pmap
+ b hrmJoin ; Go remove the mapping...
-its603co: stw r11,0(r12) /* Clear the lock */
- srw r5,r5,r8 /* Make a "free slot" mask */
- sync /* Make sure of it all */
+hppSF2: ld r28,pmapPAddr(r28) ; Get the physical address of the pmap
+ b hrmJoin ; Go remove the mapping...
- lwz r6,4(r6) /* Get the latest reference and change bits */
- lwz r9,PCAallo(r7) /* Get the allocation control bits */
- stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
- rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
- or r9,r9,r5 /* Set the slot free */
- rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
- rlwinm r4,r6,0,23,24 /* Extract the RC bits */
- andc r9,r9,r8 /* Clear the auto and lock bits */
- li r5,pepte1 /* Get displacement to the second word of master pte */
- stw r9,PCAallo(r7) /* Store the allocation controls */
+ .align 5
-commmod: lwarx r11,r5,r3 /* Get the master copy */
- or r11,r11,r4 /* Merge in latest RC */
- stwcx. r11,r5,r3 /* Save it back */
- bne- commmod /* If it changed, try again... */
- b commulnl ; Skip loading the old real part...
+hppNone: bl mapPhysUnlock ; Time to unlock the physical entry
-commul: lwz r6,mmPTEr(r10) ; Get the real part
-
-commulnl: rlwinm r12,r2,5,23,24 ; Get the "set" bits
- rlwinm r11,r2,7,23,24 ; Get the "clear" bits
-
- or r6,r6,r12 ; Set the bits to come on
- andc r6,r6,r11 ; Clear those to come off
+ bt++ pf64Bitb,hppSF3 ; skip if 64-bit (only they take the hint)...
- stw r6,mmPTEr(r10) ; Set the new RC
+ mtmsr r11 ; Restore enables/translation/etc.
+ isync
+ b hppRetnCmn ; Join the common return code...
- lwz r10,mmnext(r10) /* Get the next */
- li r4,0 /* Make sure this is 0 */
- mr. r10,r10 ; Is there another mapping?
+hppSF3: mtmsrd r11 ; Restore enables/translation/etc.
+ isync
- sync ; Make sure that all is saved
+;
+; NOTE: we have not used any registers other than the volatiles to this point
+;
- stw r4,0(r7) /* Unlock the hash chain */
- bne+ commnext ; Go get the next if there is one...
-
-/*
- * Now that all PTEs have been invalidated and the master RC bits are updated,
- * we go ahead and figure out what the original call was and do that. Note that
- * another processor could be messing around and may have entered one of the
- * PTEs we just removed into the hash table. Too bad... You takes yer chances.
- * If there's a problem with that, it's because some higher level was trying to
- * do something with a mapping that it shouldn't. So, the problem's really
- * there, nyaaa, nyaaa, nyaaa... nyaaa, nyaaa... nyaaa! So there!
- */
+hppRetnCmn: lwz r12,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
-commdone: li r5,pepte1 /* Get displacement to the second word of master pte */
- blt cr5,commfini /* We're finished, it was invalidate all... */
- bgt cr5,commtst /* It was a test modified... */
- beq cr5,commtst /* It was a test reference... */
+ li r3,mapRtEmpty ; Physent chain is empty
+ mtlr r12 ; Restore the return
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
/*
- * Note that we need to to do the interlocked update here because another processor
- * can be updating the reference and change bits even though the physical entry
- * is locked. All modifications to the PTE portion of the physical entry must be
- * done via interlocked update.
+ * mapping *hw_purge_map(pmap, vaddr, addr64_t *next) - remove a mapping from the system.
+ *
+ * Upon entry, R3 contains a pointer to a pmap. Since vaddr is
+ * a 64-bit quantity, it is a long long so it is in R4 and R5.
+ *
+ * We return the virtual address of the removed mapping as a
+ * R3.
+ *
+ * Note that this is designed to be called from 32-bit mode with a stack.
+ *
+ * We disable translation and all interruptions here. This keeps is
+ * from having to worry about a deadlock due to having anything locked
+ * and needing it to process a fault.
+ *
+ * Note that this must be done with both interruptions off and VM off
+ *
+ * Remove a mapping which can be reestablished by VM
+ *
*/
- rlwinm r12,r2,5,23,24 ; Get the "set" bits
- rlwinm r11,r2,7,23,24 ; Get the "clear" bits
-
-commcng: lwarx r8,r5,r3 /* Get the master copy */
- or r8,r8,r12 ; Set the bits to come on
- andc r8,r8,r11 ; Clear those to come off
- stwcx. r8,r5,r3 /* Save it back */
- bne- commcng /* If it changed, try again... */
-
- mtmsr r0 /* Interrupts and translation back on */
- isync
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,29
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- blr /* Return... */
-
- .align 4
-
-commtst: lwz r8,pepte1(r3) /* Get the PTE */
- bne- cr5,commtcb ; This is for the change bit...
- mtmsr r0 ; Interrupts and translation back on
- rlwinm r3,r8,24,31,31 ; Copy reference bit to bit 31
- isync ; Toss prefetching
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,29
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
+ .align 5
+ .globl EXT(hw_purge_map)
+
+LEXT(hw_purge_map)
+ stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r15,FM_ARG0+0x00(r1) ; Save a register
+ stw r16,FM_ARG0+0x04(r1) ; Save a register
+ stw r17,FM_ARG0+0x08(r1) ; Save a register
+ stw r18,FM_ARG0+0x0C(r1) ; Save a register
+ stw r19,FM_ARG0+0x10(r1) ; Save a register
+ mfsprg r19,2 ; Get feature flags
+ stw r20,FM_ARG0+0x14(r1) ; Save a register
+ stw r21,FM_ARG0+0x18(r1) ; Save a register
+ mtcrf 0x02,r19 ; move pf64Bit cr6
+ stw r22,FM_ARG0+0x1C(r1) ; Save a register
+ stw r23,FM_ARG0+0x20(r1) ; Save a register
+ stw r24,FM_ARG0+0x24(r1) ; Save a register
+ stw r25,FM_ARG0+0x28(r1) ; Save a register
+ stw r26,FM_ARG0+0x2C(r1) ; Save a register
+ stw r27,FM_ARG0+0x30(r1) ; Save a register
+ stw r28,FM_ARG0+0x34(r1) ; Save a register
+ stw r29,FM_ARG0+0x38(r1) ; Save a register
+ stw r30,FM_ARG0+0x3C(r1) ; Save a register
+ stw r31,FM_ARG0+0x40(r1) ; Save a register
+ stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
+ stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r3) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne hpmPanic ; Call not valid for guest shadow assist pmap
#endif
- blr ; Return...
+
+ bt++ pf64Bitb,hpmSF1 ; skip if 64-bit (only they take the hint)
+ lwz r9,pmapvr+4(r3) ; Get conversion mask
+ b hpmSF1x ; Done...
+
+hpmSF1: ld r9,pmapvr(r3) ; Get conversion mask
- .align 4
+hpmSF1x:
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
-commtcb: rlwinm r3,r8,25,31,31 ; Copy change bit to bit 31
+ xor r28,r3,r9 ; Convert the pmap to physical addressing
-commfini: mtmsr r0 ; Interrupts and translation back on
- isync ; Toss prefetching
+ mr r17,r11 ; Save the MSR
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,29
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- blr ; Return...
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkExclusive ; Go get an exclusive lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne-- hrmBadLock ; Nope...
+;
+; Note that we do a full search (i.e., no shortcut level skips, etc.)
+; here so that we will know the previous elements so we can dequeue them
+; later.
+;
+hpmSearch:
+ mr r3,r28 ; Pass in pmap to search
+ mr r29,r4 ; Top half of vaddr
+ mr r30,r5 ; Bottom half of vaddr
+ bl EXT(mapSearchFull) ; Rescan the list
+ mr. r31,r3 ; Did we? (And remember mapping address for later)
+ or r0,r4,r5 ; Are we beyond the end?
+ mr r15,r4 ; Save top of next vaddr
+ cmplwi cr1,r0,0 ; See if there is another
+ mr r16,r5 ; Save bottom of next vaddr
+ bne-- hpmGotOne ; We found one, go check it out...
+
+hpmCNext: bne++ cr1,hpmSearch ; There is another to check...
+ b hrmNotFound ; No more in pmap to check...
+
+hpmGotOne: lwz r20,mpFlags(r3) ; Get the flags
+ andi. r0,r20,lo16(mpType|mpPerm) ; cr0_eq <- normal mapping && !permanent
+ rlwinm r21,r20,8,24,31 ; Extract the busy count
+ cmplwi cr2,r21,0 ; Is it busy?
+ crand cr0_eq,cr2_eq,cr0_eq ; not busy and can be removed?
+ beq++ hrmGotX ; Found, branch to remove the mapping...
+ b hpmCNext ; Nope...
+
+hpmPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
/*
- * unsigned int hw_test_rc(mapping *mp, boolean_t reset);
+ * mapping *hw_purge_space(physent, pmap) - remove a mapping from the system based upon address space
+ *
+ * Upon entry, R3 contains a pointer to a pmap.
+ * pa is a pointer to the physent
*
- * Test the RC bits for a specific mapping. If reset is non-zero, clear them.
- * We return the RC value in the mapping if there is no PTE or if C is set.
- * (Note: R is always set with C.) Otherwise we invalidate the PTE and
- * collect the RC bits from there, also merging them into the global copy.
+ * This function removes the first mapping for a specific pmap from a physical entry
+ * alias list. It locks the list, extracts the vaddr and pmap from
+ * the first apporpriate entry. It then jumps into the hw_rem_map function.
+ * NOTE: since we jump into rem_map, we need to set up the stack
+ * identically. Also, we set the next parm to 0 so we do not
+ * try to save a next vaddr.
*
- * For now, we release the PTE slot and leave it invalid. In the future, we
- * may consider re-validating and not releasing the slot. It would be faster,
- * but our current implementation says that we will have not PTEs valid
- * without the reference bit set.
+ * We return the virtual address of the removed mapping as a
+ * R3.
*
- * We will special case C==1 && not reset to just return the RC.
+ * Note that this is designed to be called from 32-bit mode with a stack.
*
- * Probable state is worst performance state: C bit is off and there is a PTE.
+ * We disable translation and all interruptions here. This keeps is
+ * from having to worry about a deadlock due to having anything locked
+ * and needing it to process a fault.
+ *
+ * Note that this must be done with both interruptions off and VM off
+ *
+ *
+ * Remove mapping via physical page (mapping_purge)
+ *
+ * 1) lock physent
+ * 2) extract vaddr and pmap
+ * 3) unlock physent
+ * 4) do "remove mapping via pmap"
+ *
+ *
*/
-#define htrReset 31
-
.align 5
- .globl EXT(hw_test_rc)
+ .globl EXT(hw_purge_space)
+
+LEXT(hw_purge_space)
+ stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r15,FM_ARG0+0x00(r1) ; Save a register
+ stw r16,FM_ARG0+0x04(r1) ; Save a register
+ stw r17,FM_ARG0+0x08(r1) ; Save a register
+ mfsprg r2,2 ; Get feature flags
+ stw r18,FM_ARG0+0x0C(r1) ; Save a register
+ stw r19,FM_ARG0+0x10(r1) ; Save a register
+ stw r20,FM_ARG0+0x14(r1) ; Save a register
+ stw r21,FM_ARG0+0x18(r1) ; Save a register
+ stw r22,FM_ARG0+0x1C(r1) ; Save a register
+ mtcrf 0x02,r2 ; move pf64Bit cr6
+ stw r23,FM_ARG0+0x20(r1) ; Save a register
+ stw r24,FM_ARG0+0x24(r1) ; Save a register
+ stw r25,FM_ARG0+0x28(r1) ; Save a register
+ stw r26,FM_ARG0+0x2C(r1) ; Save a register
+ stw r27,FM_ARG0+0x30(r1) ; Save a register
+ li r6,0 ; Set no next address return
+ stw r28,FM_ARG0+0x34(r1) ; Save a register
+ stw r29,FM_ARG0+0x38(r1) ; Save a register
+ stw r30,FM_ARG0+0x3C(r1) ; Save a register
+ stw r31,FM_ARG0+0x40(r1) ; Save a register
+ stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
+ stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r4) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne hpsPanic ; Call not valid for guest shadow assist pmap
+#endif
+
+ bt++ pf64Bitb,hpsSF1 ; skip if 64-bit (only they take the hint)
-LEXT(hw_test_rc)
+ lwz r9,pmapvr+4(r4) ; Get conversion mask for pmap
- mfsprg r9,2 ; Get feature flags
- mfmsr r0 ; Save the MSR
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- mr. r4,r4 ; See if we have a reset to do later
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Clear interruption mask
- crnot htrReset,cr0_eq ; Remember reset
- mtcrf 0x04,r9 ; Set the features
- rlwinm r12,r12,0,28,25 ; Clear IR and DR
+ b hpsSF1x ; Done...
- bt pfNoMSRirb,htrNoMSR ; No MSR...
+hpsSF1: ld r9,pmapvr(r4) ; Get conversion mask for pmap
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b htrNoMSRx
-
-htrNoMSR:
- mr r2,r0
- mr r7,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r7
- mr r0,r2
-htrNoMSRx:
+hpsSF1x: bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
- lwz r2,mmPTEr(r3) ; Get the real part
- lwz r7,mmPTEhash(r3) ; Get pointer to hash list anchor
- rlwinm. r12,r2,0,24,24 ; Is the change bit on?
- lwz r5,mmPTEv(r3) ; Get the virtual address
- crnor cr0_eq,cr0_eq,htrReset ; Set if C=1 && not reset
- rlwinm r7,r7,0,0,25 ; Round hash list down to PCA boundary
- bt cr0_eq,htrcset ; Special case changed but no reset case...
-
- li r12,1 ; Get the locked value
+ xor r4,r4,r9 ; Convert the pmap to physical addressing
-htrLck1: lwarx r11,0,r7 ; Get the PTEG lock
- mr. r11,r11 ; Is it locked?
- bne- htrLckw1 ; Yeah...
- stwcx. r12,0,r7 ; Try to take it
- bne- htrLck1 ; Someone else was trying, try again...
- b htrSXg1 ; All done...
+ bl mapPhysLock ; Lock the physent
+
+ lwz r8,pmapSpace(r4) ; Get the space hash
+
+ bt++ pf64Bitb,hpsSF ; skip if 64-bit (only they take the hint)
+
+ lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
- .align 4
-
-htrLckw1: mr. r11,r11 ; Check if it is already held
- beq+ htrLck1 ; It is clear...
- lwz r11,0(r7) ; Get lock word again...
- b htrLckw1 ; Wait...
+hpsSrc32: rlwinm. r12,r12,0,~ppFlags ; Clean and test mapping address
+ beq hpsNone ; Did not find one...
- .align 4
-
-htrSXg1: isync ; Make sure we have not used anything yet
-
- lwz r6,mmPTEent(r3) ; Get the pointer to the PTE now that the lock is set
- lwz r2,mmPTEr(r3) ; Get the mapping copy of the real part
+ lhz r10,mpSpace(r12) ; Get the space
+
+ cmplw r10,r8 ; Is this one of ours?
+ beq hpsFnd ; Yes...
+
+ lwz r12,mpAlias+4(r12) ; Chain on to the next
+ b hpsSrc32 ; Check it out...
- rlwinm r9,r5,1,0,3 ; Move in the segment
- mr. r6,r6 ; Any PTE to invalidate?
- rlwinm r8,r5,31,2,25 ; Line it up
+ .align 5
- beq+ htrnopte ; There is no PTE to invalidate...
-
- xor r8,r8,r6 ; Back hash to virt index
- rlwimi r9,r5,22,4,9 ; Move in the API
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) ; Get the TLBIE lock
- rlwinm r5,r5,0,1,31 ; Clear the valid bit
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) ; Grab up the bottom part
- mfspr r11,pvr ; Find out what kind of machine we are
- rlwimi r9,r8,6,10,19 ; Create the virtual address
- rlwinm r11,r11,16,16,31 ; Isolate CPU type
-
- stw r5,0(r6) ; Make the PTE invalid
- cmplwi cr1,r11,3 ; Is this a 603?
- sync ; Make sure the invalid is stored
-
-htrtlbhang: lwarx r11,0,r12 ; Get the TLBIE lock
- rlwinm r8,r6,29,29,31 ; Get the bit position of entry
- mr. r11,r11 ; Is it locked?
- lis r5,0x8000 ; Start up a bit mask
- li r11,1 ; Get our lock word
- bne- htrtlbhang ; It is locked, go wait...
- stwcx. r11,0,r12 ; Try to get it
- bne- htrtlbhang ; We was beat...
+hpsSF: li r0,ppLFAmask
+ ld r12,ppLink(r3) ; Get the pointer to the first mapping
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
- li r11,0 ; Lock clear value
-
- tlbie r9 ;Invalidate it everywhere
-
- beq- cr1,htr603 ; It is a 603, skip the tlbsync...
+hpsSrc64: andc. r12,r12,r0 ; Clean and test mapping address
+ beq hpsNone ; Did not find one...
- eieio ; Make sure that the tlbie happens first
- tlbsync ; wait for everyone to catch up
- isync
+ lhz r10,mpSpace(r12) ; Get the space
-htr603: stw r11,0(r12) ; Clear the lock
- srw r5,r5,r8 ; Make a "free slot" mask
- sync ; Make sure of it all
-
- lwz r6,4(r6) ; Get the latest reference and change bits
- stw r11,mmPTEent(r3) ; Clear the pointer to the PTE
- rlwinm r6,r6,0,23,24 ; Extract the RC bits
- lwz r9,PCAallo(r7) ; Get the allocation control bits
- rlwinm r8,r5,24,8,15 ; Make the autogen bit to turn off
- lwz r10,mmphysent(r3) ; Get any physical entry
- or r9,r9,r5 ; Set the slot free
- rlwimi r8,r8,24,16,23 ; Get lock bit mask to turn it off
- andc r9,r9,r8 ; Clear the auto and lock bits
- mr. r10,r10 ; Is there a physical entry?
- li r5,pepte1 ; Get displacement to the second word of master pte
- stw r9,PCAallo(r7) ; Store the allocation controls
- rlwimi r2,r6,0,23,24 ; Stick in RC bits
- beq- htrnopte ; No physical entry...
-
-htrmrc: lwarx r11,r5,r10 ; Get the master copy
- or r11,r11,r6 ; Merge in latest RC
- stwcx. r11,r5,r10 ; Save it back
- bne- htrmrc ; If it changed, try again...
-
-htrnopte: rlwinm r5,r2,25,30,31 ; Position RC and mask off
- bf htrReset,htrnorst ; No reset to do...
- rlwinm r2,r2,0,25,22 ; Clear the RC if requested
-
-htrnorst: li r4,0 ; Get a 0
- stw r2,mmPTEr(r3) ; Set the real part of the PTE
-
- sync ; Make sure that stuff is all stored
-
- stw r4,0(r7) ; Unlock the hash chain
-
- mr r3,r5 ; Get the old RC to pass back
- mtmsr r0 ; Restore interrupts and translation
- isync
- blr ; Return...
+ cmplw r10,r8 ; Is this one of ours?
+ beq hpsFnd ; Yes...
+
+ ld r12,mpAlias(r12) ; Chain on to the next
+ b hpsSrc64 ; Check it out...
+
+ .align 5
+
+hpsFnd: mr r28,r4 ; Set the pmap physical address
+ lwz r4,mpVAddr(r12) ; Get the top of the vaddr
+ lwz r5,mpVAddr+4(r12) ; and the bottom
+
+ bl mapPhysUnlock ; Time to unlock the physical entry
+ b hrmJoin ; Go remove the mapping...
+
+ .align 5
+
+hpsNone: bl mapPhysUnlock ; Time to unlock the physical entry
- .align 4
+ bt++ pf64Bitb,hpsSF3 ; skip if 64-bit (only they take the hint)...
-htrcset: rlwinm r3,r2,25,30,31 ; Position RC and mask off
- mtmsr r0 ; Restore interrupts and translation
+ mtmsr r11 ; Restore enables/translation/etc.
isync
- blr ; Return...
+ b hpsRetnCmn ; Join the common return code...
+hpsSF3: mtmsrd r11 ; Restore enables/translation/etc.
+ isync
-/*
- * hw_phys_attr(struct phys_entry *pp, vm_prot_t prot, unsigned int wimg) - Sets the default physical page attributes
- *
- * Note that this must be done with both interruptions off and VM off
- * Move the passed in attributes into the pte image in the phys entry
- *
- *
- */
+;
+; NOTE: we have not used any registers other than the volatiles to this point
+;
- .align 5
- .globl EXT(hw_phys_attr)
+hpsRetnCmn: lwz r12,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
-LEXT(hw_phys_attr)
+ li r3,mapRtEmpty ; No mappings for specified pmap on physent chain
+ mtlr r12 ; Restore the return
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
-#if PERFTIMES && DEBUG
- mflr r11
- mr r8,r3
- mr r7,r5
- mr r5,r4
-// lwz r4,4(r3)
- li r4,0x1111
- li r3,30
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r8
- mr r4,r5
- mr r5,r7
- mtlr r11
-#endif
- mfsprg r9,2 ; Get feature flags
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- andi. r5,r5,0x0078 /* Clean up the WIMG */
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- rlwimi r5,r4,0,30,31 /* Move the protection into the wimg register */
- la r6,pepte1(r3) /* Point to the default pte */
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
-
- bt pfNoMSRirb,hpaNoMSR ; No MSR...
-
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hpaNoMSRx
+hpsPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
-hpaNoMSR:
- mr r10,r0
- mr r4,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r4
- mr r0,r10
-hpaNoMSRx:
-
-atmattr: lwarx r10,0,r6 /* Get the pte */
- rlwimi r10,r5,0,25,31 /* Move in the new attributes */
- stwcx. r10,0,r6 /* Try it on for size */
- bne- atmattr /* Someone else was trying, try again... */
-
- mtmsr r0 /* Interrupts and translation back on */
- isync
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r10
- li r3,31
- bl EXT(dbgLog2) ; Start of hw_add_map
- mtlr r11
-#endif
- blr /* All done... */
-
-
-
/*
- * handlePF - handle a page fault interruption
+ * mapping *hw_scrub_guest(physent, pmap) - remove first guest mapping associated with host
+ * on this physent chain
+ *
+ * Locates the first guest mapping on the physent chain that is associated with the
+ * specified host pmap. If this succeeds, the mapping is removed by joining the general
+ * remove path; otherwise, we return NULL. The caller is expected to invoke this entry
+ * repeatedly until no additional guest mappings that match our criteria are removed.
*
- * If the fault can be handled, this routine will RFI directly,
- * otherwise it will return with all registers as in entry.
+ * Because this entry point exits through hw_rem_map, our prolog pushes its frame.
*
- * Upon entry, state and all registers have been saved in savearea.
- * This is pointed to by R13.
- * IR and DR are off, interrupts are masked,
- * Floating point be disabled.
- * R3 is the interrupt code.
+ * Parameters:
+ * r3 : physent, 32-bit kernel virtual address
+ * r4 : host pmap, 32-bit kernel virtual address
*
- * If we bail, we must restore cr5, and all registers except 6 and
- * 3.
+ * Volatile register usage (for linkage through hrmJoin):
+ * r4 : high-order 32 bits of guest virtual address
+ * r5 : low-order 32 bits of guest virtual address
+ * r11: saved MSR image
*
+ * Non-volatile register usage:
+ * r26: VMM extension block's physical address
+ * r27: host pmap's physical address
+ * r28: guest pmap's physical address
+ *
*/
-
+
.align 5
- .globl EXT(handlePF)
+ .globl EXT(hw_scrub_guest)
+
+LEXT(hw_scrub_guest)
+ stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r15,FM_ARG0+0x00(r1) ; Save a register
+ stw r16,FM_ARG0+0x04(r1) ; Save a register
+ stw r17,FM_ARG0+0x08(r1) ; Save a register
+ mfsprg r2,2 ; Get feature flags
+ stw r18,FM_ARG0+0x0C(r1) ; Save a register
+ stw r19,FM_ARG0+0x10(r1) ; Save a register
+ stw r20,FM_ARG0+0x14(r1) ; Save a register
+ stw r21,FM_ARG0+0x18(r1) ; Save a register
+ stw r22,FM_ARG0+0x1C(r1) ; Save a register
+ mtcrf 0x02,r2 ; move pf64Bit cr6
+ stw r23,FM_ARG0+0x20(r1) ; Save a register
+ stw r24,FM_ARG0+0x24(r1) ; Save a register
+ stw r25,FM_ARG0+0x28(r1) ; Save a register
+ stw r26,FM_ARG0+0x2C(r1) ; Save a register
+ stw r27,FM_ARG0+0x30(r1) ; Save a register
+ li r6,0 ; Set no next address return
+ stw r28,FM_ARG0+0x34(r1) ; Save a register
+ stw r29,FM_ARG0+0x38(r1) ; Save a register
+ stw r30,FM_ARG0+0x3C(r1) ; Save a register
+ stw r31,FM_ARG0+0x40(r1) ; Save a register
+ stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
+ stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+ lwz r11,pmapVmmExt(r4) ; get VMM pmap extension block vaddr
+
+ bt++ pf64Bitb,hsg64Salt ; Test for 64-bit machine
+ lwz r26,pmapVmmExtPhys+4(r4) ; Get VMM pmap extension block paddr
+ lwz r9,pmapvr+4(r4) ; Get 32-bit virt<->real conversion salt
+ b hsgStart ; Get to work
+
+hsg64Salt: ld r26,pmapVmmExtPhys(r4) ; Get VMM pmap extension block paddr
+ ld r9,pmapvr+4(r4) ; Get 64-bit virt<->real conversion salt
+
+hsgStart: bl EXT(mapSetUp) ; Disable 'rupts, translation, enter 64-bit mode
+ xor r27,r4,r9 ; Convert host pmap_t virt->real
+ bl mapPhysLock ; Lock the physent
+
+ bt++ pf64Bitb,hsg64Scan ; Test for 64-bit machine
+
+ lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
+hsg32Loop: rlwinm. r12,r12,0,~ppFlags ; Clean and test mapping address
+ beq hsg32Miss ; Did not find one...
+ lwz r8,mpFlags(r12) ; Get mapping's flags
+ lhz r7,mpSpace(r12) ; Get mapping's space id
+ rlwinm r8,r8,0,mpType ; Extract mapping's type code
+ lis r28,hi16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
+ xori r8,r8,mpGuest ; Is it a guest mapping?
+ ori r28,r28,lo16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
+ slwi r9,r7,2 ; Multiply space by 4
+ lwz r28,0(r28) ; Get the actual translation map
+ lwz r4,mpVAddr(r12) ; Get the top of the vaddr
+ slwi r7,r7,3 ; Multiply space by 8
+ lwz r5,mpVAddr+4(r12) ; Get the bottom of the vaddr
+ add r7,r7,r9 ; Get correct displacement into translate table
+ add r28,r28,r7 ; Point to the pmap translation
+ lwz r28,pmapPAddr+4(r28) ; Get guest pmap paddr
+ lwz r7,pmapVmmExtPhys+4(r28) ; Get VMM extension block paddr
+ xor r7,r7,r26 ; Is guest associated with specified host?
+ or. r7,r7,r8 ; Guest mapping && associated with host?
+ lwz r12,mpAlias+4(r12) ; Chain on to the next
+ bne hsg32Loop ; Try next mapping on alias chain
+
+hsg32Hit: bl mapPhysUnlock ; Unlock physent chain
+ b hrmJoin ; Join common path for mapping removal
+
+ .align 5
+hsg32Miss: bl mapPhysUnlock ; Unlock physent chain
+ mtmsr r11 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ li r3,mapRtEmpty ; No mappings found matching specified criteria
+ b hrmRetnCmn ; Exit through common epilog
+
+ .align 5
+hsg64Scan: li r6,ppLFAmask ; Get lock, flag, attribute mask seed
+ ld r12,ppLink(r3) ; Grab the pointer to the first mapping
+ rotrdi r6,r6,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+hsg64Loop: andc. r12,r12,r6 ; Clean and test mapping address
+ beq hsg64Miss ; Did not find one...
+ lwz r8,mpFlags(r12) ; Get mapping's flags
+ lhz r7,mpSpace(r12) ; Get mapping's space id
+ rlwinm r8,r8,0,mpType ; Extract mapping's type code
+ lis r28,hi16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
+ xori r8,r8,mpGuest ; Is it a guest mapping?
+ ori r28,r28,lo16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
+ slwi r9,r7,2 ; Multiply space by 4
+ lwz r28,0(r28) ; Get the actual translation map
+ lwz r4,mpVAddr(r12) ; Get the top of the vaddr
+ slwi r7,r7,3 ; Multiply space by 8
+ lwz r5,mpVAddr+4(r12) ; Get the bottom of the vaddr
+ add r7,r7,r9 ; Get correct displacement into translate table
+ add r28,r28,r7 ; Point to the pmap translation
+ ld r28,pmapPAddr(r28) ; Get guest pmap paddr
+ ld r7,pmapVmmExtPhys(r28) ; Get VMM extension block paddr
+ xor r7,r7,r26 ; Is guest associated with specified host?
+ or. r7,r7,r8 ; Guest mapping && associated with host?
+ ld r12,mpAlias(r12) ; Chain on to the next
+ bne hsg64Loop ; Try next mapping on alias chain
+
+hsg64Hit: bl mapPhysUnlock ; Unlock physent chain
+ b hrmJoin ; Join common path for mapping removal
+
+ .align 5
+hsg64Miss: bl mapPhysUnlock ; Unlock physent chain
+ mtmsrd r11 ; Restore 'rupts, translation
+ li r3,mapRtEmpty ; No mappings found matching specified criteria
+ b hrmRetnCmn ; Exit through common epilog
-LEXT(handlePF)
/*
- * This first part does a quick check to see if we can handle the fault.
- * We can't handle any kind of protection exceptions here, so we pass
- * them up to the next level.
+ * mapping *hw_find_space(physent, space) - finds the first mapping on physent for specified space
*
- * The mapping lists are kept in MRS (most recently stolen)
- * order on queues anchored within from the
- * PTEG to which the virtual address hashes. This is further segregated by
- * the low-order 3 bits of the VSID XORed with the segment number and XORed
- * with bits 4-7 of the vaddr in an attempt to keep the searches
- * short.
- *
- * MRS is handled by moving the entry to the head of its list when stolen in the
- * assumption that it will be revalidated soon. Entries are created on the head
- * of the list because they will be used again almost immediately.
+ * Upon entry, R3 contains a pointer to a physent.
+ * space is the space ID from the pmap in question
*
- * We need R13 set to the savearea, R3 set to the interrupt code, and R2
- * set to the per_proc.
+ * We return the virtual address of the found mapping in
+ * R3. Note that the mapping busy is bumped.
*
- * NOTE: In order for a page-fault redrive to work, the translation miss
- * bit must be set in the DSISR (or SRR1 for IFETCH). That must occur
- * before we come here.
+ * Note that this is designed to be called from 32-bit mode with a stack.
+ *
+ * We disable translation and all interruptions here. This keeps is
+ * from having to worry about a deadlock due to having anything locked
+ * and needing it to process a fault.
+ *
*/
- cmplwi r3,T_INSTRUCTION_ACCESS /* See if this is for the instruction */
- lwz r8,savesrr1(r13) ; Get the MSR to determine mode
- beq- gotIfetch ; We have an IFETCH here...
-
- lwz r7,savedsisr(r13) /* Get the DSISR */
- lwz r6,savedar(r13) /* Get the fault address */
- b ckIfProt ; Go check if this is a protection fault...
-
-gotIfetch: mr r7,r8 ; IFETCH info is in SRR1
- lwz r6,savesrr0(r13) /* Get the instruction address */
-
-ckIfProt: rlwinm. r7,r7,0,1,1 ; Is this a protection exception?
- beqlr- ; Yes... (probably not though)
-
-/*
- * We will need to restore registers if we bail after this point.
- * Note that at this point several SRs have been changed to the kernel versions.
- * Therefore, for these we must build these values.
- */
+ .align 5
+ .globl EXT(hw_find_space)
-#if PERFTIMES && DEBUG
- mflr r11
- mr r5,r6
- mr r4,r3
- li r3,32
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
- mfsprg r2,0
-#endif
- lwz r3,PP_USERPMAP(r2) ; Get the user pmap (not needed if kernel access, but optimize for user??)
- rlwinm. r8,r8,0,MSR_PR_BIT,MSR_PR_BIT ; Supervisor state access?
- rlwinm r5,r6,6,26,29 ; Get index to the segment slot
- eqv r1,r1,r1 ; Fill the bottom with foxes
- bne+ notsuper ; Go do the user mode interrupt stuff...
-
- cmplwi cr1,r5,SR_COPYIN_NUM*4 ; See if this is the copyin/copyout segment
- rlwinm r3,r6,24,8,11 ; Make the kernel VSID
- bne+ cr1,havevsid ; We are done if we do not want the copyin/out guy...
-
- mfsr r3,SR_COPYIN ; Get the copy vsid
- b havevsid ; Join up...
-
- .align 5
-
-notsuper: addi r5,r5,PMAP_SEGS ; Get offset to table
- lwzx r3,r3,r5 ; Get the VSID
-
-havevsid: mfspr r5,sdr1 /* Get hash table base and size */
- cror cr1_eq,cr0_eq,cr0_eq ; Remember if kernel fault for later
- rlwinm r9,r6,2,2,5 ; Move nybble 1 up to 0 (keep aligned with VSID)
- rlwimi r1,r5,16,0,15 /* Make table size -1 out of mask */
- rlwinm r3,r3,6,2,25 /* Position the space for the VSID */
- rlwinm r7,r6,26,10,25 /* Isolate the page index */
- xor r9,r9,r3 ; Splooch vaddr nybble 0 (from VSID) and 1 together
- or r8,r5,r1 /* Point to the last byte in table */
- xor r7,r7,r3 /* Get primary hash */
- rlwinm r3,r3,1,1,24 /* Position VSID for pte ID */
- addi r8,r8,1 /* Point to the PTEG Control Area */
- rlwinm r9,r9,8,27,29 ; Get splooched bits in place
- and r7,r7,r1 /* Wrap the hash */
- rlwimi r3,r6,10,26,31 /* Move API into pte ID */
- add r8,r8,r7 /* Point to our PCA entry */
- rlwinm r12,r3,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
- la r11,PCAhash(r8) /* Point to the mapping hash area */
- xor r9,r9,r12 ; Finish splooching nybble 0, 1, and the low bits of the VSID
+LEXT(hw_find_space)
+ stwu r1,-(FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ mr r8,r4 ; Remember the space
+ stw r0,(FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
-/*
- * We have about as much as we need to start searching the autogen (aka block maps)
- * and mappings. From here on, any kind of failure will bail, and
- * contention will either bail or restart from here.
- *
- *
- */
+ bl mapPhysLock ; Lock the physent
+
+ bt++ pf64Bitb,hfsSF ; skip if 64-bit (only they take the hint)
+
+ lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
- li r12,1 /* Get the locked value */
- dcbt 0,r11 /* We'll need the hash area in a sec, so get it */
- add r11,r11,r9 /* Point to the right mapping hash slot */
+hfsSrc32: rlwinm. r12,r12,0,~ppFlags ; Clean and test mapping address
+ beq hfsNone ; Did not find one...
-ptegLck: lwarx r10,0,r8 /* Get the PTEG lock */
- mr. r10,r10 /* Is it locked? */
- bne- ptegLckw /* Yeah... */
- stwcx. r12,0,r8 /* Take take it */
- bne- ptegLck /* Someone else was trying, try again... */
- b ptegSXg /* All done... */
+ lhz r10,mpSpace(r12) ; Get the space
- .align 4
-
-ptegLckw: mr. r10,r10 /* Check if it's already held */
- beq+ ptegLck /* It's clear... */
- lwz r10,0(r8) /* Get lock word again... */
- b ptegLckw /* Wait... */
+ cmplw r10,r8 ; Is this one of ours?
+ beq hfsFnd ; Yes...
+ lwz r12,mpAlias+4(r12) ; Chain on to the next
+ b hfsSrc32 ; Check it out...
+
.align 5
+
+hfsSF: li r0,ppLFAmask
+ ld r12,ppLink(r3) ; Get the pointer to the first mapping
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
- nop ; Force ISYNC to last instruction in IFETCH
- nop
- nop
-
-ptegSXg: isync /* Make sure we haven't used anything yet */
-
- lwz r9,0(r11) /* Pick up first mapping block */
- mr r5,r11 /* Get the address of the anchor */
- mr r7,r9 /* Save the first in line */
- b findmap ; Take space and force loop to cache line
-
-findmap: mr. r12,r9 /* Are there more? */
- beq- tryAuto /* Nope, nothing in mapping list for us... */
-
- lwz r10,mmPTEv(r12) /* Get unique PTE identification */
- lwz r9,mmhashnext(r12) /* Get the chain, just in case */
- cmplw r10,r3 /* Did we hit our PTE? */
- lwz r0,mmPTEent(r12) /* Get the pointer to the hash table entry */
- mr r5,r12 /* Save the current as previous */
- bne- findmap ; Nothing here, try the next...
-
-; Cache line boundary here
-
- cmplwi cr1,r0,0 /* Is there actually a PTE entry in the hash? */
- lwz r2,mmphysent(r12) /* Get the physical entry */
- bne- cr1,MustBeOK /* There's an entry in the hash table, so, this must
- have been taken care of already... */
- lis r4,0x8000 ; Tell PTE inserter that this was not an auto
- cmplwi cr2,r2,0 /* Is there a physical entry? */
- li r0,0x0100 /* Force on the reference bit whenever we make a PTE valid */
- bne+ cr2,gotphys /* Skip down if we have a physical entry */
- li r0,0x0180 /* When there is no physical entry, force on
- both R and C bits to keep hardware from
- updating the PTE to set them. We don't
- keep track of RC for I/O areas, so this is ok */
-
-gotphys: lwz r2,mmPTEr(r12) ; Get the second part of the PTE
- b insert /* Go insert into the PTEG... */
-
-MustBeOK: li r10,0 /* Get lock clear value */
- li r3,T_IN_VAIN /* Say that we handled it */
- stw r10,PCAlock(r8) /* Clear the PTEG lock */
-
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,33
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- blr /* Blow back and handle exception */
-
-
-
-/*
- * We couldn't find it in the mapping list. As a last try, we will
- * see if we can autogen it from the block mapped list.
- *
- * A block mapped area is defined as a contiguous virtual area that is mapped to
- * a contiguous physical area. The olde-tyme IBM VM/XA Interpretive Execution
- * architecture referred to this as a V=F, or Virtual = Fixed area.
- *
- * We consider a V=F area to be a single entity, adjacent areas can not be merged
- * or overlapped. The protection and memory attributes are the same and reference
- * and change indications are not kept. The areas are not considered part of the
- * physical RAM of the machine and do not have any associated physical table
- * entries. Their primary use is intended for mapped I/O areas (e.g., framebuffers)
- * although certain areas of RAM, such as the kernel V=R memory, can be mapped.
- *
- * We also have a problem in the case of copyin/out: that access is done
- * within the kernel for a user address. Unfortunately, the user isn't
- * necessarily the current guy. That means that we don't have access to the
- * right autogen list. We can't support this kind of access. So, we need to do
- * a quick check here and cause a fault if an attempt to copyin or out to
- * any autogenned area.
- *
- * The lists must be kept short.
- *
- * NOTE: kernel_pmap_store must be in V=R storage!!!!!!!!!!!!!!
- */
-
- .align 5
-
-tryAuto: rlwinm. r11,r3,0,5,24 ; Check if this is a kernel VSID
- lis r10,HIGH_ADDR(EXT(kernel_pmap_store)+PMAP_BMAPS) ; Get the top part of kernel block map anchor
- crandc cr0_eq,cr1_eq,cr0_eq ; Set if kernel access and non-zero VSID (copyin or copyout)
- mfsprg r11,0 ; Get the per_proc area
- beq- cr0,realFault ; Can not autogen for copyin/copyout...
- ori r10,r10,LOW_ADDR(EXT(kernel_pmap_store)+PMAP_BMAPS) ; Get the bottom part
- beq- cr1,bmInKernel ; We are in kernel... (cr1 set way back at entry)
-
- lwz r10,PP_USERPMAP(r11) ; Get the user pmap
- la r10,PMAP_BMAPS(r10) ; Point to the chain anchor
- b bmInKernel ; Jump over alignment gap...
- nop
- nop
- nop
- nop
- nop
- nop
-bmInKernel:
-#ifndef CHIP_ERRATA_MAX_V1
- lwarx r9,0,r10
-#endif /* CHIP_ERRATA_MAX_V1 */
-
-bmapLck: lwarx r9,0,r10 ; Get the block map anchor and lock
- rlwinm. r5,r9,0,31,31 ; Is it locked?
- ori r5,r5,1 ; Set the lock
- bne- bmapLckw ; Yeah...
- stwcx. r5,0,r10 ; Lock the bmap list
- bne- bmapLck ; Someone else was trying, try again...
- b bmapSXg ; All done...
-
- .align 4
-
-bmapLckw: rlwinm. r5,r9,0,31,31 ; Check if it is still held
- beq+ bmapLck ; Not no more...
- lwz r9,0(r10) ; Get lock word again...
- b bmapLckw ; Check it out...
-
- .align 5
-
- nop ; Force ISYNC to last instruction in IFETCH
- nop
- nop
-
-bmapSXg: rlwinm. r4,r9,0,0,26 ; Clear out flags and lock
- isync ; Make sure we have not used anything yet
- bne+ findAuto ; We have something, let us go...
-
-bmapNone: stw r9,0(r10) ; Unlock it, we have nothing here
- ; No sync here because we have not changed anything
+hfsSrc64: andc. r12,r12,r0 ; Clean and test mapping address
+ beq hfsNone ; Did not find one...
-/*
- * When we come here, we know that we can't handle this. Restore whatever
- * state that we trashed and go back to continue handling the interrupt.
- */
-
-realFault: li r10,0 /* Get lock clear value */
- lwz r3,saveexception(r13) /* Figure out the exception code again */
- stw r10,PCAlock(r8) /* Clear the PTEG lock */
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,33
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- blr /* Blow back and handle exception */
-
- .align 5
-
-findAuto: mr. r4,r4 ; Is there more?
- beq- bmapNone ; No more...
- lwz r5,bmstart(r4) ; Get the bottom of range
- lwz r11,bmend(r4) ; Get the top of range
- cmplw cr0,r6,r5 ; Are we before the entry?
- cmplw cr1,r6,r11 ; Are we after the entry?
- cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in range
- bne+ cr1,faGot ; Found it...
-
- lwz r4,bmnext(r4) ; Get the next one
- b findAuto ; Check it out...
+ lhz r10,mpSpace(r12) ; Get the space
-faGot:
- lwz r7,blkFlags(r4) ; Get the flags
- rlwinm. r7,r7,0,blkRembit,blkRembit ; is this mapping partially removed
- bne bmapNone ; Pending remove, bail out
- rlwinm r6,r6,0,0,19 ; Round to page
- lwz r2,bmPTEr(r4) ; Get the real part of the PTE
- sub r5,r6,r5 ; Get offset into area
- stw r9,0(r10) ; Unlock it, we are done with it (no sync needed)
- add r2,r2,r5 ; Adjust the real address
+ cmplw r10,r8 ; Is this one of ours?
+ beq hfsFnd ; Yes...
- lis r4,0x8080 /* Indicate that this was autogened */
- li r0,0x0180 /* Autogenned areas always set RC bits.
- This keeps the hardware from having
- to do two storage writes */
+ ld r12,mpAlias(r12) ; Chain on to the next
+ b hfsSrc64 ; Check it out...
-/*
- * Here where we insert the PTE into the hash. The PTE image is in R3, R2.
- * The PTEG allocation controls are a bit map of the state of the PTEG. The
- * PCAlock bits are a temporary lock for the specified PTE. PCAfree indicates that
- * the PTE slot is empty. PCAauto means that it comes from an autogen area. These
- * guys do not keep track of reference and change and are actually "wired".
- * They're easy to maintain. PCAsteal
- * is a sliding position mask used to "randomize" PTE slot stealing. All 4 of these
- * fields fit in a single word and are loaded and stored under control of the
- * PTEG control area lock (PCAlock).
- *
- * Note that PCAauto does not contribute to the steal calculations at all. Originally
- * it did, autogens were second in priority. This can result in a pathalogical
- * case where an instruction can not make forward progress, or one PTE slot
- * thrashes.
- *
- * Physically, the fields are arranged:
- * 0: PCAfree
- * 1: PCAauto
- * 2: PCAlock
- * 3: PCAsteal
- */
-
-insert: lwz r10,PCAallo(r8) /* Get the PTEG controls */
- eqv r6,r6,r6 /* Get all ones */
- mr r11,r10 /* Make a copy */
- rlwimi r6,r10,8,16,23 /* Insert sliding steal position */
- rlwimi r11,r11,24,24,31 /* Duplicate the locked field */
- addi r6,r6,-256 /* Form mask */
- rlwimi r11,r11,16,0,15 /* This gives us a quadrupled lock mask */
- rlwinm r5,r10,31,24,0 /* Slide over the mask for next time */
- mr r9,r10 /* Make a copy to test */
- not r11,r11 /* Invert the quadrupled lock */
- or r2,r2,r0 /* Force on R, and maybe C bit */
- and r9,r9,r11 /* Remove the locked guys */
- rlwimi r5,r5,8,24,24 /* Wrap bottom bit to top in mask */
- rlwimi r9,r11,0,16,31 /* Put two copies of the unlocked entries at the end */
- rlwinm r6,r6,0,16,7 ; Remove the autogens from the priority calculations
- rlwimi r10,r5,0,24,31 /* Move steal map back in */
- and r9,r9,r6 /* Set the starting point for stealing */
-
-/* So, now we have in R9:
- byte 0 = ~locked & free
- byte 1 = 0
- byte 2 = ~locked & (PCAsteal - 1)
- byte 3 = ~locked
-
- Each bit position represents (modulo 8) a PTE. If it is 1, it is available for
- allocation at its priority level, left to right.
-
- Additionally, the PCA steal field in R10 has been rotated right one bit.
-*/
-
-
- rlwinm r21,r10,8,0,7 ; Isolate just the old autogen bits
- cntlzw r6,r9 /* Allocate a slot */
- mr r14,r12 /* Save our mapping for later */
- cmplwi r6,32 ; Was there anything available?
- rlwinm r7,r6,29,30,31 /* Get the priority slot we got this from */
- rlwinm r6,r6,0,29,31 ; Isolate bit position
- srw r11,r4,r6 /* Position the PTEG control bits */
- slw r21,r21,r6 ; Move corresponding old autogen flag to bit 0
- mr r22,r11 ; Get another copy of the selected slot
-
- beq- realFault /* Arghh, no slots! Take the long way 'round... */
-
- /* Remember, we've already set up the mask pattern
- depending upon how we got here:
- if got here from simple mapping, R4=0x80000000,
- if we got here from autogen it is 0x80800000. */
-
- rlwinm r6,r6,3,26,28 /* Start calculating actual PTE address */
- rlwimi r22,r22,24,8,15 ; Duplicate selected slot in second byte
- rlwinm. r11,r11,0,8,15 /* Isolate just the auto bit (remember about it too) */
- andc r10,r10,r22 /* Turn off the free and auto bits */
- add r6,r8,r6 /* Get position into PTEG control area */
- cmplwi cr1,r7,1 /* Set the condition based upon the old PTE type */
- sub r6,r6,r1 /* Switch it to the hash table */
- or r10,r10,r11 /* Turn auto on if it is (PTEG control all set up now) */
- subi r6,r6,1 /* Point right */
- stw r10,PCAallo(r8) /* Allocate our slot */
- dcbt br0,r6 ; Touch in the PTE
- bne wasauto /* This was autogenned... */
-
- stw r6,mmPTEent(r14) /* Link the mapping to the PTE slot */
+ .align 5
-/*
- * So, now we're here and what exactly do we have? We've got:
- * 1) a full PTE entry, both top and bottom words in R3 and R2
- * 2) an allocated slot in the PTEG.
- * 3) R8 still points to the PTEG Control Area (PCA)
- * 4) R6 points to the PTE entry.
- * 5) R1 contains length of the hash table-1. We use this to back-translate
- * a PTE to a virtual address so we can invalidate TLBs.
- * 6) R11 has a copy of the PCA controls we set.
- * 7a) R7 indicates what the PTE slot was before we got to it. 0 shows
- * that it was empty and 2 or 3, that it was
- * a we've stolen a live one. CR1 is set to LT for empty and GT
- * otherwise.
- * 7b) Bit 0 of R21 is 1 if the stolen PTE was autogenned
- * 8) So far as our selected PTE, it should be valid if it was stolen
- * and invalid if not. We could put some kind of assert here to
- * check, but I think that I'd rather leave it in as a mysterious,
- * non-reproducable bug.
- * 9) The new PTE's mapping has been moved to the front of its PTEG hash list
- * so that it's kept in some semblance of a MRU list.
- * 10) R14 points to the mapping we're adding.
- *
- * So, what do we have to do yet?
- * 1) If we stole a slot, we need to invalidate the PTE completely.
- * 2) If we stole one AND it was not an autogen,
- * copy the entire old PTE (including R and C bits) to its mapping.
- * 3) Set the new PTE in the PTEG and make sure it is valid.
- * 4) Unlock the PTEG control area.
- * 5) Go back to the interrupt handler, changing the interrupt
- * code to "in vain" which will restore the registers and bail out.
- *
- */
-wasauto: oris r3,r3,0x8000 /* Turn on the valid bit */
- blt+ cr1,slamit /* It was empty, go slam it on in... */
-
- lwz r10,0(r6) /* Grab the top part of the PTE */
- rlwinm r12,r6,6,4,19 /* Match up the hash to a page boundary */
- rlwinm r5,r10,5,4,19 /* Extract the VSID to a page boundary */
- rlwinm r10,r10,0,1,31 /* Make it invalid */
- xor r12,r5,r12 /* Calculate vaddr */
- stw r10,0(r6) /* Invalidate the PTE */
- rlwinm r5,r10,7,27,29 ; Move nybble 0 up to subhash position
- rlwimi r12,r10,1,0,3 /* Move in the segment portion */
- lis r9,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
- xor r5,r5,r10 ; Splooch nybble 0 and 1
- rlwimi r12,r10,22,4,9 /* Move in the API */
- ori r9,r9,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
- rlwinm r4,r10,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
-
- sync /* Make sure the invalid is stored */
-
- xor r4,r4,r5 ; Finish splooching nybble 0, 1, and the low bits of the VSID
-
-tlbhang: lwarx r5,0,r9 /* Get the TLBIE lock */
-
- rlwinm r4,r4,0,27,29 ; Clean up splooched hash value
+hfsFnd: mr r8,r3 ; Save the physent
+ mr r3,r12 ; Point to the mapping
+ bl mapBumpBusy ; If we found it, bump up the busy count so the mapping does not disapear
- mr. r5,r5 /* Is it locked? */
- add r4,r4,r8 /* Point to the offset into the PCA area */
- li r5,1 /* Get our lock word */
- bne- tlbhang /* It's locked, go wait... */
+ mr r3,r8 ; Get back the physical entry
+ li r7,0xFFF ; Get a page size mask
+ bl mapPhysUnlock ; Time to unlock the physical entry
+
+ andc r3,r12,r7 ; Move the mapping back down to a page
+ lwz r3,mbvrswap+4(r3) ; Get last half of virtual to real swap
+ xor r12,r3,r12 ; Convert to virtual
+ b hfsRet ; Time to return
- la r4,PCAhash(r4) /* Point to the start of the hash chain for the PTE we're replacing */
+ .align 5
- stwcx. r5,0,r9 /* Try to get it */
- bne- tlbhang /* We was beat... */
+hfsNone: bl mapPhysUnlock ; Time to unlock the physical entry
- mfspr r7,pvr /* Find out what kind of machine we are */
- li r5,0 /* Lock clear value */
- rlwinm r7,r7,16,16,31 /* Isolate CPU type */
+hfsRet: bt++ pf64Bitb,hfsSF3 ; skip if 64-bit (only they take the hint)...
- tlbie r12 /* Invalidate it everywhere */
+ mtmsr r11 ; Restore enables/translation/etc.
+ isync
+ b hfsRetnCmn ; Join the common return code...
- cmplwi r7,3 /* Is this a 603? */
- stw r5,0(r9) /* Clear the lock */
-
- beq- its603 /* It's a 603, skip the tlbsync... */
-
- eieio /* Make sure that the tlbie happens first */
- tlbsync /* wait for everyone to catch up */
- isync
-
-its603: rlwinm. r21,r21,0,0,0 ; See if we just stole an autogenned entry
- sync /* Make sure of it all */
+hfsSF3: mtmsrd r11 ; Restore enables/translation/etc.
+ isync
- bne slamit ; The old was an autogen, time to slam the new in...
-
- lwz r9,4(r6) /* Get the real portion of old PTE */
- lwz r7,0(r4) /* Get the first element. We can't get to here
- if we aren't working with a mapping... */
- mr r0,r7 ; Save pointer to first element
-
-findold: mr r1,r11 ; Save the previous guy
- mr. r11,r7 /* Copy and test the chain */
- beq- bebad /* Assume it's not zero... */
-
- lwz r5,mmPTEv(r11) /* See if this is the old active one */
- cmplw cr2,r11,r14 /* Check if this is actually the new one */
- cmplw r5,r10 /* Is this us? (Note: valid bit kept off in mappings) */
- lwz r7,mmhashnext(r11) /* Get the next one in line */
- beq- cr2,findold /* Don't count the new one... */
- cmplw cr2,r11,r0 ; Check if we are first on the list
- bne+ findold /* Not it (and assume the worst)... */
-
- lwz r12,mmphysent(r11) /* Get the pointer to the physical entry */
- beq- cr2,nomove ; We are first, no need to requeue...
+;
+; NOTE: we have not used any registers other than the volatiles to this point
+;
+
+hfsRetnCmn: mr r3,r12 ; Get the mapping or a 0 if we failed
+
+#if DEBUG
+ mr. r3,r3 ; Anything to return?
+ beq hfsRetnNull ; Nope
+ lwz r11,mpFlags(r3) ; Get mapping flags
+ rlwinm r0,r11,0,mpType ; Isolate the mapping type
+ cmplwi r0,mpGuest ; Shadow guest mapping?
+ beq hfsPanic ; Yup, kick the bucket
+hfsRetnNull:
+#endif
+
+ lwz r12,(FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
+
+ mtlr r12 ; Restore the return
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
- stw r11,0(r4) ; Chain us to the head
- stw r0,mmhashnext(r11) ; Chain the old head to us
- stw r7,mmhashnext(r1) ; Unlink us
+hfsPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
-nomove: li r5,0 /* Clear this on out */
+;
+; mapping *hw_find_map(pmap, va, *nextva) - Looks up a vaddr in a pmap
+; Returns 0 if not found or the virtual address of the mapping if
+; if is. Also, the mapping has the busy count bumped.
+;
+ .align 5
+ .globl EXT(hw_find_map)
+
+LEXT(hw_find_map)
+ stwu r1,-(FM_ALIGN((31-25+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r25,FM_ARG0+0x00(r1) ; Save a register
+ stw r26,FM_ARG0+0x04(r1) ; Save a register
+ mr r25,r6 ; Remember address of next va
+ stw r27,FM_ARG0+0x08(r1) ; Save a register
+ stw r28,FM_ARG0+0x0C(r1) ; Save a register
+ stw r29,FM_ARG0+0x10(r1) ; Save a register
+ stw r30,FM_ARG0+0x14(r1) ; Save a register
+ stw r31,FM_ARG0+0x18(r1) ; Save a register
+ stw r0,(FM_ALIGN((31-26+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r3) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne hfmPanic ; Call not valid for guest shadow assist pmap
+#endif
- mr. r12,r12 /* Is there a physical entry? */
- stw r5,mmPTEent(r11) ; Clear the PTE entry pointer
- li r5,pepte1 /* Point to the PTE last half */
- stw r9,mmPTEr(r11) ; Squirrel away the whole thing (RC bits are in here)
+ lwz r6,pmapvr(r3) ; Get the first part of the VR translation for pmap
+ lwz r7,pmapvr+4(r3) ; Get the second part
+
+
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
+
+ mr r27,r11 ; Remember the old MSR
+ mr r26,r12 ; Remember the feature bits
+
+ xor r28,r3,r7 ; Change the common 32- and 64-bit half
+
+ bf-- pf64Bitb,hfmSF1 ; skip if 32-bit...
- beq- mrgmrcx ; No physical entry for this one...
+ rldimi r28,r6,32,0 ; Shift the fixed upper part of the physical over and cram in top
+
+hfmSF1: mr r29,r4 ; Save top half of vaddr
+ mr r30,r5 ; Save the bottom half
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne-- hfmBadLock ; Nope...
+
+ mr r3,r28 ; get the pmap address
+ mr r4,r29 ; Get bits 0:31 to look for
+ mr r5,r30 ; Get bits 32:64
- rlwinm r11,r9,0,23,24 /* Keep only the RC bits */
+ bl EXT(mapSearch) ; Go see if we can find it (note: R7 comes back with mpFlags)
-mrgmrcx: lwarx r9,r5,r12 /* Get the master copy */
- or r9,r9,r11 /* Merge in latest RC */
- stwcx. r9,r5,r12 /* Save it back */
- bne- mrgmrcx /* If it changed, try again... */
+ rlwinm r0,r7,0,mpRIPb,mpRIPb ; Find remove in progress bit
+ mr. r31,r3 ; Save the mapping if we found it
+ cmplwi cr1,r0,0 ; Are we removing?
+ mr r29,r4 ; Save next va high half
+ crorc cr0_eq,cr0_eq,cr1_eq ; Not found or removing
+ mr r30,r5 ; Save next va low half
+ li r6,0 ; Assume we did not find it
+ li r26,0xFFF ; Get a mask to relocate to start of mapping page
-/*
- * Here's where we finish up. We save the real part of the PTE, eieio it, to make sure it's
- * out there before the top half (with the valid bit set).
- */
+ bt-- cr0_eq,hfmNotFnd ; We did not find it...
-slamit: stw r2,4(r6) /* Stash the real part */
- li r4,0 /* Get a lock clear value */
- eieio /* Erect a barricade */
- stw r3,0(r6) /* Stash the virtual part and set valid on */
-
- stw r4,PCAlock(r8) /* Clear the PCA lock */
-
- li r3,T_IN_VAIN /* Say that we handled it */
- sync /* Go no further until the stores complete */
-#if PERFTIMES && DEBUG
- mflr r11
- mr r4,r3
- li r3,33
- bl EXT(dbgLog2) ; Start of hw_add_map
- mr r3,r4
- mtlr r11
-#endif
- blr /* Back to the fold... */
-
-bebad: lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
- ori r0,r0,LOW_ADDR(Choke)
- sc /* Firmware Heimlich maneuver */
-
-/*
- * This walks the hash table or DBATs to locate the physical address of a virtual one.
- * The space is provided. If it is the kernel space, the DBATs are searched first. Failing
- * that, the hash table is accessed. Zero is returned for failure, so it must be special cased.
- * This is usually used for debugging, so we try not to rely
- * on anything that we don't have to.
- */
+ bl mapBumpBusy ; If we found it, bump up the busy count so the mapping does not disapear
-ENTRY(LRA, TAG_NO_FRAME_USED)
+ andc r4,r31,r26 ; Get back to the mapping page start
- mfsprg r8,2 ; Get feature flags
- mfmsr r10 /* Save the current MSR */
- rlwinm r10,r10,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r10,r10,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- mtcrf 0x04,r8 ; Set the features
- xoris r5,r3,HIGH_ADDR(PPC_SID_KERNEL) /* Clear the top half if equal */
- andi. r9,r10,0x7FCF /* Turn off interrupts and translation */
- eqv r12,r12,r12 /* Fill the bottom with foxes */
+; Note: we can treat 32- and 64-bit the same here. Because we are going from
+; physical to virtual and we only do 32-bit virtual, we only need the low order
+; word of the xor.
- bt pfNoMSRirb,lraNoMSR ; No MSR...
+ lwz r4,mbvrswap+4(r4) ; Get last half of virtual to real swap
+ li r6,-1 ; Indicate we found it and it is not being removed
+ xor r31,r31,r4 ; Flip to virtual
- mtmsr r9 ; Translation and all off
- isync ; Toss prefetch
- b lraNoMSRx
+hfmNotFnd: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ rlwinm r3,r31,0,0,31 ; Move mapping to return register and clear top of register if 64-bit
+ and r3,r3,r6 ; Clear if not found or removing
+
+hfmReturn: bt++ pf64Bitb,hfmR64 ; Yes...
+
+ mtmsr r27 ; Restore enables/translation/etc.
+ isync
+ b hfmReturnC ; Join common...
+
+hfmR64: mtmsrd r27 ; Restore enables/translation/etc.
+ isync
-lraNoMSR:
- mr r7,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r9 ; Get new MSR
- sc ; Set it
- mr r3,r7
-lraNoMSRx:
-
- cmplwi r5,LOW_ADDR(PPC_SID_KERNEL) /* See if this is kernel space */
- rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
- isync /* Purge pipe */
- bne- notkernsp /* This is not for the kernel... */
-
- mfspr r5,dbat0u /* Get the virtual address and length */
- eqv r8,r8,r8 /* Get all foxes */
- rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
- rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
- beq- ckbat1 /* not valid, skip this one... */
- sub r7,r4,r7 /* Subtract out the base */
- rlwimi r8,r5,15,0,14 /* Get area length - 1 */
- mfspr r6,dbat0l /* Get the real part */
- cmplw r7,r8 /* Check if it is in the range */
- bng+ fndbat /* Yup, she's a good un... */
-
-ckbat1: mfspr r5,dbat1u /* Get the virtual address and length */
- eqv r8,r8,r8 /* Get all foxes */
- rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
- rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
- beq- ckbat2 /* not valid, skip this one... */
- sub r7,r4,r7 /* Subtract out the base */
- rlwimi r8,r5,15,0,14 /* Get area length - 1 */
- mfspr r6,dbat1l /* Get the real part */
- cmplw r7,r8 /* Check if it is in the range */
- bng+ fndbat /* Yup, she's a good un... */
-
-ckbat2: mfspr r5,dbat2u /* Get the virtual address and length */
- eqv r8,r8,r8 /* Get all foxes */
- rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
- rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
- beq- ckbat3 /* not valid, skip this one... */
- sub r7,r4,r7 /* Subtract out the base */
- rlwimi r8,r5,15,0,14 /* Get area length - 1 */
- mfspr r6,dbat2l /* Get the real part */
- cmplw r7,r8 /* Check if it is in the range */
- bng- fndbat /* Yup, she's a good un... */
-
-ckbat3: mfspr r5,dbat3u /* Get the virtual address and length */
- eqv r8,r8,r8 /* Get all foxes */
- rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
- rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
- beq- notkernsp /* not valid, skip this one... */
- sub r7,r4,r7 /* Subtract out the base */
- rlwimi r8,r5,15,0,14 /* Get area length - 1 */
- mfspr r6,dbat3l /* Get the real part */
- cmplw r7,r8 /* Check if it is in the range */
- bgt+ notkernsp /* No good... */
-
-fndbat: rlwinm r6,r6,0,0,14 /* Clean up the real address */
- mtmsr r10 /* Restore state */
- add r3,r7,r6 /* Relocate the offset to real */
- isync /* Purge pipe */
- blr /* Bye, bye... */
-
-notkernsp: mfspr r5,sdr1 /* Get hash table base and size */
- rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
- rlwimi r12,r5,16,0,15 /* Make table size -1 out of mask */
- rlwinm r7,r4,26,10,25 /* Isolate the page index */
- andc r5,r5,r12 /* Clean up the hash table */
- xor r7,r7,r11 /* Get primary hash */
- rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
- and r7,r7,r12 /* Wrap the hash */
- rlwimi r11,r4,10,26,31 /* Move API into pte ID */
- add r5,r7,r5 /* Point to the PTEG */
- oris r11,r11,0x8000 /* Slam on valid bit so's we don't match an invalid one */
-
- li r9,8 /* Get the number of PTEs to check */
- lwz r6,0(r5) /* Preload the virtual half */
-
-fndpte: subi r9,r9,1 /* Count the pte */
- lwz r3,4(r5) /* Get the real half */
- cmplw cr1,r6,r11 /* Is this what we want? */
- lwz r6,8(r5) /* Start to get the next virtual half */
- mr. r9,r9 /* Any more to try? */
- addi r5,r5,8 /* Bump to next slot */
- beq cr1,gotxlate /* We found what we were looking for... */
- bne+ fndpte /* Go try the next PTE... */
-
- mtmsr r10 /* Restore state */
- li r3,0 /* Show failure */
- isync /* Purge pipe */
- blr /* Leave... */
-
-gotxlate: mtmsr r10 /* Restore state */
- rlwimi r3,r4,0,20,31 /* Cram in the page displacement */
- isync /* Purge pipe */
- blr /* Return... */
+hfmReturnC: stw r29,0(r25) ; Save the top of the next va
+ stw r30,4(r25) ; Save the bottom of the next va
+ lwz r0,(FM_ALIGN((31-25+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+ lwz r25,FM_ARG0+0x00(r1) ; Restore a register
+ lwz r26,FM_ARG0+0x04(r1) ; Restore a register
+ and r3,r3,r6 ; Clear return if the mapping is being removed
+ lwz r27,FM_ARG0+0x08(r1) ; Restore a register
+ mtlr r0 ; Restore the return
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore a register
+ lwz r29,FM_ARG0+0x10(r1) ; Restore a register
+ lwz r30,FM_ARG0+0x14(r1) ; Restore a register
+ lwz r31,FM_ARG0+0x18(r1) ; Restore a register
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
+
+ .align 5
+
+hfmBadLock: li r3,1 ; Set lock time out error code
+ b hfmReturn ; Leave....
+hfmPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
/*
- * struct blokmap *hw_add_blk(pmap_t pmap, struct blokmap *bmr)
- *
- * This is used to add a block mapping entry to the MRU list whose top
- * node is anchored at bmaps. This is a real address and is also used as
- * the lock.
- *
- * Overlapping areas are not allowed. If we find one, we return it's address and
- * expect the upper layers to panic. We only check this for a debug build...
+ * void hw_clear_maps(void)
*
+ * Remove all mappings for all phys entries.
+ *
+ *
*/
.align 5
- .globl EXT(hw_add_blk)
+ .globl EXT(hw_clear_maps)
-LEXT(hw_add_blk)
+LEXT(hw_clear_maps)
+ mflr r10 ; Save the link register
+ mfcr r9 ; Save the condition register
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
- mfsprg r9,2 ; Get feature flags
- lwz r6,PMAP_PMAPVR(r3) ; Get the v to r translation
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- xor r3,r3,r6 ; Get real address of bmap anchor
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
- la r3,PMAP_BMAPS(r3) ; Point to bmap header
-
- bt pfNoMSRirb,habNoMSR ; No MSR...
+ lis r5,hi16(EXT(pmap_mem_regions)) ; Point to the start of the region table
+ ori r5,r5,lo16(EXT(pmap_mem_regions)) ; Point to the start of the region table
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b habNoMSRx
-
-habNoMSR:
- mr r9,r0
- mr r8,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r8
- mr r0,r9
-habNoMSRx:
-
-abLck: lwarx r9,0,r3 ; Get the block map anchor and lock
- rlwinm. r8,r9,0,31,31 ; Is it locked?
- ori r8,r9,1 ; Set the lock
- bne- abLckw ; Yeah...
- stwcx. r8,0,r3 ; Lock the bmap list
- bne- abLck ; Someone else was trying, try again...
- b abSXg ; All done...
-
- .align 4
+hcmNextRegion:
+ lwz r3,mrPhysTab(r5) ; Get the actual table address
+ lwz r0,mrStart(r5) ; Get start of table entry
+ lwz r4,mrEnd(r5) ; Get end of table entry
+ addi r5,r5,mrSize ; Point to the next regions
+
+ cmplwi r3,0 ; No more regions?
+ beq-- hcmDone ; Leave...
+
+ sub r4,r4,r0 ; Calculate physical entry count
+ addi r4,r4,1
+ mtctr r4
+
+ bt++ pf64Bitb,hcmNextPhys64 ; 64-bit version
-abLckw: rlwinm. r5,r9,0,31,31 ; Check if it is still held
- beq+ abLck ; Not no more...
- lwz r9,0(r3) ; Get lock word again...
- b abLckw ; Check it out...
+
+hcmNextPhys32:
+ lwz r4,ppLink+4(r3) ; Grab the pointer to the first mapping
+ addi r3,r3,physEntrySize ; Next phys_entry
+hcmNextMap32:
+ rlwinm. r4,r4,0,~ppFlags ; Clean and test mapping address
+ beq hcmNoMap32 ; Did not find one...
+
+ lwz r0,mpPte(r4) ; Grab the offset to the PTE
+ rlwinm r0,r0,0,~mpHValid ; Clear out valid bit
+ stw r0,mpPte(r4) ; Get the quick pointer again
+
+ lwz r4,mpAlias+4(r4) ; Chain on to the next
+ b hcmNextMap32 ; Check it out...
+hcmNoMap32:
+ bdnz hcmNextPhys32
+ b hcmNextRegion
+
+
.align 5
+hcmNextPhys64:
+ li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ ld r4,ppLink(r3) ; Get the pointer to the first mapping
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ addi r3,r3,physEntrySize ; Next phys_entry
- nop ; Force ISYNC to last instruction in IFETCH
- nop
+hcmNextMap64:
+ andc. r4,r4,r0 ; Clean and test mapping address
+ beq hcmNoMap64 ; Did not find one...
-abSXg: rlwinm r11,r9,0,0,26 ; Clear out flags and lock
- isync ; Make sure we have not used anything yet
+ lwz r0,mpPte(r4) ; Grab the offset to the PTE
+ rlwinm r0,r0,0,~mpHValid ; Clear out valid bit
+ stw r0,mpPte(r4) ; Get the quick pointer again
-;
-;
-;
+ ld r4,mpAlias(r4) ; Chain on to the next
+ li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ b hcmNextMap64 ; Check it out...
+hcmNoMap64:
+ bdnz hcmNextPhys64
+ b hcmNextRegion
- lwz r7,bmstart(r4) ; Get start
- lwz r8,bmend(r4) ; Get end
- mr r2,r11 ; Get chain
-
-abChk: mr. r10,r2 ; End of chain?
- beq abChkD ; Yes, chain is ok...
- lwz r5,bmstart(r10) ; Get start of current area
- lwz r6,bmend(r10) ; Get end of current area
-
- cmplw cr0,r8,r5 ; Is the end of the new before the old?
- cmplw cr1,r8,r6 ; Is the end of the new after the old?
- cmplw cr6,r6,r7 ; Is the end of the old before the new?
- cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in old
- cmplw cr7,r6,r8 ; Is the end of the old after the new?
- lwz r2,bmnext(r10) ; Get pointer to the next
- cror cr6_eq,cr6_lt,cr7_gt ; Set cr2_eq if old not in new
- crand cr1_eq,cr1_eq,cr6_eq ; Set cr1_eq if no overlap
- beq+ cr1,abChk ; Ok check the next...
-
- lwz r8,blkFlags(r10) ; Get the flags
- rlwinm. r8,r8,0,blkRembit,blkRembit ; Check the blkRem bit
- beq abRet ; Is the mapping partially removed
- ori r10,r10,2 ; Indicate that this block is partially removed
-abRet:
- stw r9,0(r3) ; Unlock
- mtmsr r0 ; Restore xlation and rupts
- mr r3,r10 ; Pass back the overlap
- isync ;
- blr ; Return...
-abChkD: stw r11,bmnext(r4) ; Chain this on in
- rlwimi r4,r9,0,27,31 ; Copy in locks and flags
- sync ; Make sure that is done
-
- stw r4,0(r3) ; Unlock and chain the new first one
- mtmsr r0 ; Restore xlation and rupts
- li r3,0 ; Pass back a no failure return code
+ .align 5
+hcmDone:
+ mtlr r10 ; Restore the return
+ mtcr r9 ; Restore the condition register
+ bt++ pf64Bitb,hcmDone64 ; 64-bit version
+hcmDone32:
+ mtmsr r11 ; Restore translation/mode/etc.
isync
- blr ; Return...
+ blr ; Leave...
+
+hcmDone64:
+ mtmsrd r11 ; Restore translation/mode/etc.
+ isync
+ blr ; Leave...
+
/*
- * struct blokmap *hw_rem_blk(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
- *
- * This is used to remove a block mapping entry from the list that
- * is anchored at bmaps. bmaps is a virtual address and is also used as
- * the lock.
+ * unsigned int hw_walk_phys(pp, preop, op, postop, parm, opmod)
+ * walks all mapping for a physical page and performs
+ * specified operations on each.
*
- * Note that this function clears a single block that contains
- * any address within the range sva to eva (inclusive). To entirely
- * clear any range, hw_rem_blk must be called repeatedly until it
- * returns a 0.
+ * pp is unlocked physent
+ * preop is operation to perform on physent before walk. This would be
+ * used to set cache attribute or protection
+ * op is the operation to perform on each mapping during walk
+ * postop is operation to perform in the phsyent after walk. this would be
+ * used to set or reset the RC bits.
+ * opmod modifies the action taken on any connected PTEs visited during
+ * the mapping walk.
*
- * The block is removed from the list and all hash table entries
- * corresponding to the mapped block are invalidated and the TLB
- * entries are purged. If the block is large, this could take
- * quite a while. We need to hash every possible address in the
- * range and lock down the PCA.
+ * We return the RC bits from before postop is run.
*
- * If we attempt to remove a permanent entry, we will not do it.
- * The block address will be ored with 1 and returned.
+ * Note that this is designed to be called from 32-bit mode with a stack.
*
+ * We disable translation and all interruptions here. This keeps is
+ * from having to worry about a deadlock due to having anything locked
+ * and needing it to process a fault.
*
+ * We lock the physent, execute preop, and then walk each mapping in turn.
+ * If there is a PTE, it is invalidated and the RC merged into the physent.
+ * Then we call the op function.
+ * Then we revalidate the PTE.
+ * Once all all mappings are finished, we save the physent RC and call the
+ * postop routine. Then we unlock the physent and return the RC.
+ *
+ *
*/
.align 5
- .globl EXT(hw_rem_blk)
-
-LEXT(hw_rem_blk)
-
- mfsprg r9,2 ; Get feature flags
- lwz r6,PMAP_PMAPVR(r3) ; Get the v to r translation
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- xor r3,r3,r6 ; Get real address of bmap anchor
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
- la r3,PMAP_BMAPS(r3) ; Point to the bmap chain head
+ .globl EXT(hw_walk_phys)
+
+LEXT(hw_walk_phys)
+ stwu r1,-(FM_ALIGN((31-24+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r24,FM_ARG0+0x00(r1) ; Save a register
+ stw r25,FM_ARG0+0x04(r1) ; Save a register
+ stw r26,FM_ARG0+0x08(r1) ; Save a register
+ stw r27,FM_ARG0+0x0C(r1) ; Save a register
+ mr r24,r8 ; Save the parm
+ mr r25,r7 ; Save the parm
+ stw r28,FM_ARG0+0x10(r1) ; Save a register
+ stw r29,FM_ARG0+0x14(r1) ; Save a register
+ stw r30,FM_ARG0+0x18(r1) ; Save a register
+ stw r31,FM_ARG0+0x1C(r1) ; Save a register
+ stw r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
+
+ mfsprg r26,0 ; (INSTRUMENTATION)
+ lwz r27,hwWalkPhys(r26) ; (INSTRUMENTATION)
+ addi r27,r27,1 ; (INSTRUMENTATION)
+ stw r27,hwWalkPhys(r26) ; (INSTRUMENTATION)
+ la r26,hwWalkFull(r26) ; (INSTRUMENTATION)
+ slwi r12,r24,2 ; (INSTRUMENTATION)
+ lwzx r27,r26,r12 ; (INSTRUMENTATION)
+ addi r27,r27,1 ; (INSTRUMENTATION)
+ stwx r27,r26,r12 ; (INSTRUMENTATION)
+
+ mr r26,r11 ; Save the old MSR
+ lis r27,hi16(hwpOpBase) ; Get high order of op base
+ slwi r4,r4,7 ; Convert preop to displacement
+ ori r27,r27,lo16(hwpOpBase) ; Get low order of op base
+ slwi r5,r5,7 ; Convert op to displacement
+ add r12,r4,r27 ; Point to the preop routine
+ slwi r28,r6,7 ; Convert postop to displacement
+ mtctr r12 ; Set preop routine
+ add r28,r28,r27 ; Get the address of the postop routine
+ add r27,r5,r27 ; Get the address of the op routine
- bt pfNoMSRirb,hrbNoMSR ; No MSR...
+ bl mapPhysLock ; Lock the physent
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hrbNoMSRx
+ mr r29,r3 ; Save the physent address
-hrbNoMSR:
- mr r9,r0
- mr r8,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r8
- mr r0,r9
-hrbNoMSRx:
- li r7,0
- cmp cr5,r0,r7 ; Request to invalidate the ptes
- b rbLck
-
-rbunlink:
- lwz r4,bmstart(r10) ; Get start of current mapping
- lwz r5,bmend(r10) ; Get end of current mapping
- cmp cr5,r3,r3 ; Request to unlink the mapping
+ bt++ pf64Bitb,hwp64 ; skip if 64-bit (only they take the hint)
+
+ bctrl ; Call preop routine
+ bne- hwpEarly32 ; preop says to bail now...
-rbLck: lwarx r9,0,r3 ; Get the block map anchor and lock
- rlwinm. r8,r9,0,31,31 ; Is it locked?
- ori r8,r9,1 ; Set the lock
- bne- rbLckw ; Yeah...
- stwcx. r8,0,r3 ; Lock the bmap list
- bne- rbLck ; Someone else was trying, try again...
- b rbSXg ; All done...
+ cmplwi r24,hwpMergePTE ; Classify operation modifier
+ mtctr r27 ; Set up the op function address
+ lwz r31,ppLink+4(r3) ; Grab the pointer to the first mapping
+ blt hwpSrc32 ; Do TLB invalidate/purge/merge/reload for each mapping
+ beq hwpMSrc32 ; Do TLB merge for each mapping
+
+hwpQSrc32: rlwinm. r31,r31,0,~ppFlags ; Clean and test mapping address
+ beq hwpNone32 ; Did not find one...
- .align 4
+ bctrl ; Call the op function
+
+ bne- hwpEarly32 ; op says to bail now...
+ lwz r31,mpAlias+4(r31) ; Chain on to the next
+ b hwpQSrc32 ; Check it out...
-rbLckw: rlwinm. r11,r9,0,31,31 ; Check if it is still held
- beq+ rbLck ; Not no more...
- lwz r9,0(r3) ; Get lock word again...
- b rbLckw ; Check it out...
+ .align 5
+hwpMSrc32: rlwinm. r31,r31,0,~ppFlags ; Clean and test mapping address
+ beq hwpNone32 ; Did not find one...
- .align 5
+ bl mapMergeRC32 ; Merge reference and change into mapping and physent
+ bctrl ; Call the op function
- nop ; Force ISYNC to last instruction in IFETCH
- nop
+ bne- hwpEarly32 ; op says to bail now...
+ lwz r31,mpAlias+4(r31) ; Chain on to the next
+ b hwpMSrc32 ; Check it out...
-rbSXg: rlwinm. r2,r9,0,0,26 ; Clear out flags and lock
- mr r10,r3 ; Keep anchor as previous pointer
- isync ; Make sure we have not used anything yet
+ .align 5
+hwpSrc32: rlwinm. r31,r31,0,~ppFlags ; Clean and test mapping address
+ beq hwpNone32 ; Did not find one...
+
+;
+; Note: mapInvPte32 returns the PTE in R3 (or 0 if none), PTE high in R4,
+; PTE low in R5. The PCA address is in R7. The PTEG come back locked.
+; If there is no PTE, PTE low is obtained from mapping
+;
+ bl mapInvPte32 ; Invalidate and lock PTE, also merge into physent
+
+ bctrl ; Call the op function
+
+ crmove cr1_eq,cr0_eq ; Save the return code
+
+ mr. r3,r3 ; Was there a previously valid PTE?
+ beq- hwpNxt32 ; Nope...
- beq- rbMT ; There is nothing in the list
+ stw r5,4(r3) ; Store second half of PTE
+ eieio ; Make sure we do not reorder
+ stw r4,0(r3) ; Revalidate the PTE
-rbChk: mr r12,r10 ; Save the previous
- mr. r10,r2 ; End of chain?
- beq rbMT ; Yes, nothing to do...
- lwz r11,bmstart(r10) ; Get start of current area
- lwz r6,bmend(r10) ; Get end of current area
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock the PCA
- cmplw cr0,r5,r11 ; Is the end of range before the start of the area?
- cmplw cr1,r4,r6 ; Is the start of range after the end of the area?
- cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in range
- lwz r2,bmnext(r10) ; Get the next one
- beq+ cr1,rbChk ; Not this one, check the next...
+hwpNxt32: bne- cr1,hwpEarly32 ; op says to bail now...
+ lwz r31,mpAlias+4(r31) ; Chain on to the next
+ b hwpSrc32 ; Check it out...
- cmplw cr1,r12,r3 ; Is the current mapping the first one?
-
- bne cr5,rbblkRem ; Do we have to unchain the mapping
+ .align 5
- bne cr1,rbnFirst ; Yes, is this the first mapping?
- rlwimi r9,r2,0,0,26 ; Yes, Change the lock value
- ori r2,r9,1 ; Turn on the lock bit
-rbnFirst:
- stw r2,bmnext(r12) ; Unchain us
- sync
- b rbDone
+hwpNone32: mtctr r28 ; Get the post routine address
+
+ lwz r30,ppLink+4(r29) ; Save the old RC
+ mr r3,r29 ; Get the physent address
+ bctrl ; Call post routine
-rbblkRem:
-
- lwz r8,blkFlags(r10) ; Get the flags
+ bl mapPhysUnlock ; Unlock the physent
- rlwinm. r7,r8,0,blkPermbit,blkPermbit ; is this a permanent block?
+ mtmsr r26 ; Restore translation/mode/etc.
+ isync
- bne- rbPerm ; This is permanent, do not remove...
+ b hwpReturn ; Go restore registers and return...
- rlwinm. r7,r8,0,blkRembit,blkRembit ; is this mapping partially removed
+ .align 5
- beq rbblkRemcont ; If not, check the max size
- lwz r11,bmcurrent(r10) ; If yes, resume for the current page
+hwpEarly32: lwz r30,ppLink+4(r29) ; Save the old RC
+ mr r3,r29 ; Get the physent address
+ bl mapPhysUnlock ; Unlock the physent
+
+ mtmsr r26 ; Restore translation/mode/etc.
+ isync
+
+ b hwpReturn ; Go restore registers and return...
- cmp cr5,r11,r6 ; No partial remove left
- beq cr5, rbpendret ; But there is a pending remove
+ .align 5
+
+hwp64: bctrl ; Call preop routine
+ bne-- hwpEarly64 ; preop says to bail now...
+
+ cmplwi r24,hwpMergePTE ; Classify operation modifier
+ mtctr r27 ; Set up the op function address
+
+ li r24,ppLFAmask
+ ld r31,ppLink(r3) ; Get the pointer to the first mapping
+ rotrdi r24,r24,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ blt hwpSrc64 ; Do TLB invalidate/purge/merge/reload for each mapping
+ beq hwpMSrc64 ; Do TLB merge for each mapping
+
+hwpQSrc64: andc. r31,r31,r24 ; Clean and test mapping address
+ beq hwpNone64 ; Did not find one...
-rbblkRemcont:
- bne rbblkRemcont1 ; Is it the first remove
+ bctrl ; Call the op function
- oris r8,r8,hi16(blkRem) ; Yes
- stw r8,blkFlags(r10) ; set the blkRem bit in blkFlags
+ bne-- hwpEarly64 ; op says to bail now...
+ ld r31,mpAlias(r31) ; Chain on to the next
+ b hwpQSrc64 ; Check it out...
-rbblkRemcont1:
- lis r5,hi16(BLKREMMAX*4096) ; Load maximun size tear down
- ori r5,r5,lo16(BLKREMMAX*4096) ; Load maximun size tear down
- sub r7,r6,r11 ; Get the remaining size to tear down
- cmp cr5,r7,r5 ; Compare against the maximun size
- ble cr5,rbfullblk ; If less or equal, go remove the mapping
+ .align 5
+hwpMSrc64: andc. r31,r31,r24 ; Clean and test mapping address
+ beq hwpNone64 ; Did not find one...
- add r7,r11,r5 ; Add the max size tear down to the current page
- stw r7,bmcurrent(r10) ; Update the current page
- subi r6,r7,1 ; Set the current end of the partial tear down
- b rbcont
+ bl mapMergeRC64 ; Merge reference and change into mapping and physent
+ bctrl ; Call the op function
-rbfullblk:
- stw r6,bmcurrent(r10) ; Update the current page
+ bne-- hwpEarly64 ; op says to bail now...
+ ld r31,mpAlias(r31) ; Chain on to the next
+ b hwpMSrc64 ; Check it out...
-rbcont:
- lwz r8,bmspace(r10) ; Get the VSID
- sync
- stw r9,0(r3) ; Unlock and chain the new first one
-
- eqv r4,r4,r4 ; Fill the bottom with foxes
- mfspr r12,sdr1 ; Get hash table base and size
- rlwinm r8,r8,6,0,25 ; Align VSID to PTEG
- rlwimi r4,r12,16,0,15 ; Make table size - 1 out of mask
- andc r12,r12,r4 ; Clean up address of hash table
- rlwinm r5,r11,26,6,25 ; Rotate virtual start address into PTEG units
- add r12,r12,r4 ; Point to PCA - 1
- rlwinm r6,r6,26,6,25 ; Rotate virtual end address into PTEG units
- addi r12,r12,1 ; Point to PCA base
- sub r6,r6,r5 ; Get the total number of PTEGs to clear
- cmplw r6,r4 ; See if this wraps all the way around
- blt rbHash ; Nope, length is right
- subi r6,r4,32+31 ; Back down to correct length
-
-rbHash: rlwinm r5,r5,0,10,25 ; Keep only the page index
- xor r2,r8,r5 ; Hash into table
- and r2,r2,r4 ; Wrap into the table
- add r2,r2,r12 ; Point right at the PCA
-
-rbLcka: lwarx r7,0,r2 ; Get the PTEG lock
- mr. r7,r7 ; Is it locked?
- bne- rbLckwa ; Yeah...
- li r7,1 ; Get the locked value
- stwcx. r7,0,r2 ; Take it
- bne- rbLcka ; Someone else was trying, try again...
- b rbSXga ; All done...
-
-rbLckwa: mr. r7,r7 ; Check if it is already held
- beq+ rbLcka ; It is clear...
- lwz r7,0(r2) ; Get lock word again...
- b rbLckwa ; Wait...
-
-rbSXga: isync ; Make sure nothing used yet
- lwz r7,PCAallo(r2) ; Get the allocation word
- rlwinm. r11,r7,8,0,7 ; Isolate the autogenerated PTEs
- or r7,r7,r11 ; Release the autogen slots
- beq+ rbAintNone ; There are not any here
- mtcrf 0xC0,r11 ; Set the branch masks for autogens
- sub r11,r2,r4 ; Move back to the hash table + 1
- rlwinm r7,r7,0,16,7 ; Clear the autogen field
- subi r11,r11,1 ; Point to the PTEG
- stw r7,PCAallo(r2) ; Update the flags
- li r7,0 ; Get an invalid PTE value
-
- bf 0,rbSlot1 ; No autogen here
- stw r7,0x00(r11) ; Invalidate PTE
-rbSlot1: bf 1,rbSlot2 ; No autogen here
- stw r7,0x08(r11) ; Invalidate PTE
-rbSlot2: bf 2,rbSlot3 ; No autogen here
- stw r7,0x10(r11) ; Invalidate PTE
-rbSlot3: bf 3,rbSlot4 ; No autogen here
- stw r7,0x18(r11) ; Invalidate PTE
-rbSlot4: bf 4,rbSlot5 ; No autogen here
- stw r7,0x20(r11) ; Invalidate PTE
-rbSlot5: bf 5,rbSlot6 ; No autogen here
- stw r7,0x28(r11) ; Invalidate PTE
-rbSlot6: bf 6,rbSlot7 ; No autogen here
- stw r7,0x30(r11) ; Invalidate PTE
-rbSlot7: bf 7,rbSlotx ; No autogen here
- stw r7,0x38(r11) ; Invalidate PTE
-rbSlotx:
-
-rbAintNone: li r7,0 ; Clear this out
- sync ; To make SMP happy
- addic. r6,r6,-64 ; Decrement the count
- stw r7,PCAlock(r2) ; Release the PTEG lock
- addi r5,r5,64 ; Move up by adjusted page number
- bge+ rbHash ; Not done...
-
- sync ; Make sure the memory is quiet
-
+ .align 5
+hwpSrc64: andc. r31,r31,r24 ; Clean and test mapping address
+ beq hwpNone64 ; Did not find one...
;
-; Here we take the easy way out and just purge the entire TLB. This is
-; certainly faster and definitly easier than blasting just the correct ones
-; in the range, we only need one lock and one TLBSYNC. We would hope
-; that most blocks are more than 64 pages (256K) and on every machine
-; up to Book E, 64 TLBIEs will invalidate the entire table.
+; Note: mapInvPte64 returns the PTE in R3 (or 0 if none), PTE high in R4,
+; PTE low in R5. PTEG comes back locked if there is one
;
+ bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
- li r5,64 ; Get number of TLB entries to purge
- lis r12,HIGH_ADDR(EXT(tlb_system_lock)) ; Get the TLBIE lock
- li r6,0 ; Start at 0
- ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) ; Grab up the bottom part
-
-rbTlbL: lwarx r2,0,r12 ; Get the TLBIE lock
- mr. r2,r2 ; Is it locked?
- li r2,1 ; Get our lock value
- bne- rbTlbL ; It is locked, go wait...
- stwcx. r2,0,r12 ; Try to get it
- bne- rbTlbL ; We was beat...
-
-rbTlbN: addic. r5,r5,-1 ; See if we did them all
- tlbie r6 ; Invalidate it everywhere
- addi r6,r6,0x1000 ; Up to the next page
- bgt+ rbTlbN ; Make sure we have done it all...
+ bctrl ; Call the op function
+
+ crmove cr1_eq,cr0_eq ; Save the return code
- mfspr r5,pvr ; Find out what kind of machine we are
- li r2,0 ; Lock clear value
+ mr. r3,r3 ; Was there a previously valid PTE?
+ beq-- hwpNxt64 ; Nope...
- rlwinm r5,r5,16,16,31 ; Isolate CPU type
- cmplwi r5,3 ; Is this a 603?
- sync ; Make sure all is quiet
- beq- rbits603a ; It is a 603, skip the tlbsync...
+ std r5,8(r3) ; Save bottom of PTE
+ eieio ; Make sure we do not reorder
+ std r4,0(r3) ; Revalidate the PTE
- eieio ; Make sure that the tlbie happens first
- tlbsync ; wait for everyone to catch up
- isync
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock the PCA
-rbits603a: sync ; Wait for quiet again
- stw r2,0(r12) ; Unlock invalidates
+hwpNxt64: bne-- cr1,hwpEarly64 ; op says to bail now...
+ ld r31,mpAlias(r31) ; Chain on to the next
+ b hwpSrc64 ; Check it out...
+
+ .align 5
- sync ; Make sure that is done
+hwpNone64: mtctr r28 ; Get the post routine address
- ble cr5,rbunlink ; If all ptes are flush, go unlink the mapping
- mtmsr r0 ; Restore xlation and rupts
- mr r3,r10 ; Pass back the removed block in progress
- ori r3,r3,2 ; Indicate that the block remove isn't completed yet
- isync
- blr ; Return...
+ lwz r30,ppLink+4(r29) ; Save the old RC
+ mr r3,r29 ; Get the physent address
+ bctrl ; Call post routine
-rbpendret:
- stw r9,0(r3) ; Unlock
- mtmsr r0 ; Restore xlation and rupts
- mr r3,r10 ; Pass back the removed block in progress
- ori r3,r3,2 ; Indicate that the block remove isn't completed yet
+ bl mapPhysUnlock ; Unlock the physent
+
+ mtmsrd r26 ; Restore translation/mode/etc.
isync
- blr ; Return...
+ b hwpReturn ; Go restore registers and return...
+ .align 5
-rbMT: stw r9,0(r3) ; Unlock
- mtmsr r0 ; Restore xlation and rupts
- li r3,0 ; Say we did not find one
- isync
- blr ; Return...
-
-rbPerm: stw r9,0(r3) ; Unlock
- mtmsr r0 ; Restore xlation and rupts
- ori r3,r10,1 ; Say we did not remove it
- isync
- blr ; Return...
+hwpEarly64: lwz r30,ppLink+4(r29) ; Save the old RC
+ mr r3,r29 ; Get the physent address
+ bl mapPhysUnlock ; Unlock the physent
+
+ mtmsrd r26 ; Restore translation/mode/etc.
+ isync
+
+hwpReturn: lwz r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
+ lwz r24,FM_ARG0+0x00(r1) ; Restore a register
+ lwz r25,FM_ARG0+0x04(r1) ; Restore a register
+ lwz r26,FM_ARG0+0x08(r1) ; Restore a register
+ mr r3,r30 ; Pass back the RC
+ lwz r27,FM_ARG0+0x0C(r1) ; Restore a register
+ lwz r28,FM_ARG0+0x10(r1) ; Restore a register
+ mtlr r0 ; Restore the return
+ lwz r29,FM_ARG0+0x14(r1) ; Restore a register
+ lwz r30,FM_ARG0+0x18(r1) ; Restore a register
+ lwz r31,FM_ARG0+0x1C(r1) ; Restore a register
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
-rbDone: stw r9,0(r3) ; Unlock
- mtmsr r0 ; Restore xlation and rupts
- mr r3,r10 ; Pass back the removed block
- isync
- blr ; Return...
-/*
- * hw_select_mappings(struct mappingflush *mappingflush)
- *
- * Input: PCA addr
- * Ouput: up to 8 user mappings
- *
- * hw_select_mappings() scans every PCA mapping hash lists and select
- * the last user mapping if it exists.
- *
- */
+;
+; The preop/op/postop function table.
+; Each function must be 64-byte aligned and be no more than
+; 16 instructions. If more than 16, we must fix address calculations
+; at the start of hwpOpBase
+;
+; The routine must set CR0_EQ in order to continue scan.
+; If CR0_EQ is not set, an early return from the function is made.
+;
- .align 5
- .globl EXT(hw_select_mappings)
+ .align 7
+
+hwpOpBase:
-LEXT(hw_select_mappings)
- mr r5,r3 ; Get the mapping flush addr
- mfmsr r12 ; Get the MSR
- rlwinm r12,r12,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r12,r12,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- mfsprg r9,2 ; Get feature flags
- andi. r0,r12,0x7FCF ; Disable translation and interruptions
- mtcrf 0x04,r9 ; Set the features
- bt pfNoMSRirb,hvmNoMSR ; No MSR...
- mtmsr r0
- isync
- b hvmNoMSRx
-hvmNoMSR:
- mr r3,r0 ; Get the new MSR
- li r0,loadMSR ; Get the MSR setter SC
- sc
-hvmNoMSRx:
- mr r0,r12
- li r11,1 ; Get the locked value
-
-hvmptegLckx:
- lwz r3,MFpcaptr(r5) ; Get the PCA pointer
- lwarx r10,0,r3 ; Get the PTEG lock
- mr. r10,r10 ; Is it locked?
- bne- hvmptegLckwx ; Yeah...
- stwcx. r11,0,r3 ; Take take it
- bne- hvmptegLckx ; Someone else was trying, try again...
- b hvmptegSXgx ; All done...
-
- .align 4
-
-hvmptegLckwx:
- mr. r10,r10 ; Check if it is already held
- beq+ hvmptegLckx ; It's clear...
- lwz r10,0(r3) ; Get lock word again...
- b hvmptegLckwx ; Wait...
-
- .align 4
-
-hvmptegSXgx:
- isync ; Make sure we haven't used anything yet
-
- li r11,8 ; set count to 8
-
- lwz r6,PCAhash(r3) ; load the first mapping hash list
- la r12,PCAhash(r3) ; Point to the mapping hash area
- la r4,MFmapping(r5) ; Point to the mapping flush mapping area
- li r7,0 ; Load zero
- stw r7,MFmappingcnt(r5) ; Set the current count to 0
-hvmnexthash:
- li r10,0 ; Mapping test
-
-hvmfindmap:
- mr. r6,r6 ; Test if the hash list current pointer is zero
- beq hvmfindmapret ; Did we hit the end of the hash list
- lwz r7,mmPTEv(r6) ; Pick up our virtual ID
- rlwinm r8,r7,5,0,19 ; Pick VSID 20 lower bits
- mr. r8,r8
- beq hvmfindmapnext ; Skip Kernel VSIDs
- rlwinm r8,r7,1,0,3 ; Extract the Segment index
- rlwinm r9,r7,22,4,9 ; Extract API 6 upper bits
- or r8,r8,r9 ; Add to the virtual address
- rlwinm r9,r7,31,6,25 ; Pick VSID 19 lower bits
- xor r9,r9,r3 ; Exclusive or with the PCA address
- rlwinm r9,r9,6,10,19 ; Extract API 10 lower bits
- or r8,r8,r9 ; Add to the virtual address
-
- stw r8,4(r4) ; Store the virtual address
- lwz r8,mmpmap(r6) ; Get the pmap
- stw r8,0(r4) ; Store the pmap
- li r10,1 ; Found one
-
-hvmfindmapnext:
- lwz r6,mmhashnext(r6) ; Pick up next mapping block
- b hvmfindmap ; Scan the next mapping
-hvmfindmapret:
- mr. r10,r10 ; Found mapping
- beq hvmnexthashprep ; If not, do not update the mappingflush array
- lwz r7,MFmappingcnt(r5) ; Get the current count
- addi r7,r7,1 ; Increment the current count
- stw r7,MFmappingcnt(r5) ; Store the current count
- addi r4,r4,MFmappingSize ; Point to the next mapping flush entry
-hvmnexthashprep:
- addi r12,r12,4 ; Load the next hash list
- lwz r6,0(r12) ; Load the next hash list entry
- subi r11,r11,1 ; Decrement hash list index
- mr. r11,r11 ; Test for a remaining hash list
- bne hvmnexthash ; Loop to scan the next hash list
-
- li r10,0
- stw r10,0(r3) ; Unlock the hash list
- mtmsr r0 ; Restore translation and interruptions
- isync
- blr
+; Function 0 - No operation
-/*
- * vm_offset_t hw_cvp_blk(pmap_t pmap, vm_offset_t va)
- *
- * This is used to translate a virtual address within a block mapping entry
- * to a physical address. If not found, 0 is returned.
- *
- */
+hwpNoop: cmplw r0,r0 ; Make sure CR0_EQ is set
+ blr ; Just return...
.align 5
- .globl EXT(hw_cvp_blk)
-
-LEXT(hw_cvp_blk)
- mfsprg r9,2 ; Get feature flags
- lwz r6,PMAP_PMAPVR(r3) ; Get the v to r translation
- mfmsr r0 /* Save the MSR */
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
- mtcrf 0x04,r9 ; Set the features
- xor r3,r3,r6 ; Get real address of bmap anchor
- rlwinm r12,r12,0,28,25 /* Clear IR and DR */
- la r3,PMAP_BMAPS(r3) ; Point to chain header
+; This is the continuation of function 4 - Set attributes in mapping
- bt pfNoMSRirb,hcbNoMSR ; No MSR...
+; We changed the attributes of a mapped page. Make sure there are no cache paradoxes.
+; NOTE: Do we have to deal with i-cache here?
- mtmsr r12 ; Translation and all off
- isync ; Toss prefetch
- b hcbNoMSRx
-
-hcbNoMSR:
- mr r9,r0
- mr r8,r3
- li r0,loadMSR ; Get the MSR setter SC
- mr r3,r12 ; Get new MSR
- sc ; Set it
- mr r3,r8
- mr r0,r9
-hcbNoMSRx:
-
-cbLck: lwarx r9,0,r3 ; Get the block map anchor and lock
- rlwinm. r8,r9,0,31,31 ; Is it locked?
- ori r8,r9,1 ; Set the lock
- bne- cbLckw ; Yeah...
- stwcx. r8,0,r3 ; Lock the bmap list
- bne- cbLck ; Someone else was trying, try again...
- b cbSXg ; All done...
-
- .align 4
-
-cbLckw: rlwinm. r5,r9,0,31,31 ; Check if it is still held
- beq+ cbLck ; Not no more...
- lwz r9,0(r3) ; Get lock word again...
- b cbLckw ; Check it out...
-
- .align 5
-
- nop ; Force ISYNC to last instruction in IFETCH
- nop
- nop
- nop
- nop
-
-cbSXg: rlwinm. r11,r9,0,0,26 ; Clear out flags and lock
- li r2,0 ; Assume we do not find anything
- isync ; Make sure we have not used anything yet
-
-cbChk: mr. r11,r11 ; Is there more?
- beq- cbDone ; No more...
- lwz r5,bmstart(r11) ; Get the bottom of range
- lwz r12,bmend(r11) ; Get the top of range
- cmplw cr0,r4,r5 ; Are we before the entry?
- cmplw cr1,r4,r12 ; Are we after of the entry?
- cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in range
- beq- cr1,cbNo ; We are not in the range...
-
- lwz r2,bmPTEr(r11) ; Get the real part of the PTE
- sub r5,r4,r5 ; Get offset into area
- rlwinm r2,r2,0,0,19 ; Clean out everything but the page
- add r2,r2,r5 ; Adjust the real address
-
-cbDone: stw r9,0(r3) ; Unlock it, we are done with it (no sync needed)
- mtmsr r0 ; Restore translation and interrupts...
- isync ; Make sure it is on
- mr r3,r2 ; Set return physical address
- blr ; Leave...
+hwpSAM: li r11,4096 ; Get page size
- .align 5
+hwpSAMinvd: sub. r11,r11,r9 ; Back off a line
+ dcbf r11,r5 ; Flush the line in the data cache
+ bgt++ hwpSAMinvd ; Go do the rest of it...
-cbNo: lwz r11,bmnext(r11) ; Link next
- b cbChk ; Check it out...
+ sync ; Make sure it is done
+
+ li r11,4096 ; Get page size
+hwpSAMinvi: sub. r11,r11,r9 ; Back off a line
+ icbi r11,r5 ; Flush the line in the icache
+ bgt++ hwpSAMinvi ; Go do the rest of it...
-/*
- * hw_set_user_space(pmap)
- * hw_set_user_space_dis(pmap)
- *
- * Indicate whether memory space needs to be switched.
- * We really need to turn off interrupts here, because we need to be non-preemptable
- *
- * hw_set_user_space_dis is used when interruptions are already disabled. Mind the
- * register usage here. The VMM switch code in vmachmon.s that calls this
- * know what registers are in use. Check that if these change.
- */
+ sync ; Make sure it is done
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ blr ; Return...
-
- .align 5
- .globl EXT(hw_set_user_space)
-
-LEXT(hw_set_user_space)
- mfmsr r10 /* Get the current MSR */
- rlwinm r10,r10,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r10,r10,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r9,r10,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Turn off 'rupts */
- mtmsr r9 /* Disable 'em */
- lwz r7,PMAP_PMAPVR(r3) ; Get the v to r translation
- lwz r4,PMAP_SPACE(r3) ; Get the space
- mfsprg r6,0 /* Get the per_proc_info address */
- xor r3,r3,r7 ; Get real address of bmap anchor
- stw r4,PP_USERSPACE(r6) /* Show our new address space */
- stw r3,PP_USERPMAP(r6) ; Show our real pmap address
- mtmsr r10 /* Restore interruptions */
- blr /* Return... */
-
- .align 5
- .globl EXT(hw_set_user_space_dis)
+; Function 1 - Set protection in physent (obsolete)
-LEXT(hw_set_user_space_dis)
+ .set .,hwpOpBase+(1*128) ; Generate error if previous function too long
- lwz r7,PMAP_PMAPVR(r3) ; Get the v to r translation
- lwz r4,PMAP_SPACE(r3) ; Get the space
- mfsprg r6,0 ; Get the per_proc_info address
- xor r3,r3,r7 ; Get real address of bmap anchor
- stw r4,PP_USERSPACE(r6) ; Show our new address space
- stw r3,PP_USERPMAP(r6) ; Show our real pmap address
+hwpSPrtPhy: cmplw r0,r0 ; Make sure we return CR0_EQ
blr ; Return...
-
+
-/* struct mapping *hw_cpv(struct mapping *mp) - Converts a physcial mapping CB address to virtual
- *
- */
+; Function 2 - Set protection in mapping
- .align 5
- .globl EXT(hw_cpv)
+; NOTE: Changes to no-execute permission are ignored
-LEXT(hw_cpv)
-
- rlwinm. r4,r3,0,0,19 ; Round back to the mapping block allocation control block
- mfmsr r10 ; Get the current MSR
- beq- hcpvret ; Skip if we are passed a 0...
- rlwinm r10,r10,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r10,r10,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- andi. r9,r10,0x7FEF ; Turn off interrupts and data translation
- mtmsr r9 ; Disable DR and EE
- isync
-
- lwz r4,mbvrswap(r4) ; Get the conversion value
- mtmsr r10 ; Interrupts and DR back on
- isync
- xor r3,r3,r4 ; Convert to physical
+ .set .,hwpOpBase+(2*128) ; Generate error if previous function too long
-hcpvret: rlwinm r3,r3,0,0,26 ; Clean out any flags
- blr
+hwpSPrtMap: lwz r9,mpFlags(r31) ; Get the mapping flags
+ lwz r8,mpVAddr+4(r31) ; Get the protection part of mapping
+ rlwinm. r9,r9,0,mpPermb,mpPermb ; Is the mapping permanent?
+ li r0,lo16(mpPP) ; Get protection bits
+ crnot cr0_eq,cr0_eq ; Change CR0_EQ to true if mapping is permanent
+ rlwinm r2,r25,0,mpPP ; Isolate new protection bits
+ beqlr-- ; Leave if permanent mapping (before we trash R5)...
+ andc r5,r5,r0 ; Clear the old prot bits
+ or r5,r5,r2 ; Move in the new prot bits
+ rlwimi r8,r5,0,20,31 ; Copy into the mapping copy
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ blr ; Leave...
+
+; Function 3 - Set attributes in physent
+ .set .,hwpOpBase+(3*128) ; Generate error if previous function too long
-/* struct mapping *hw_cvp(struct mapping *mp) - Converts a virtual mapping CB address to physcial
- *
- * Translation must be on for this
- *
- */
+hwpSAtrPhy: li r5,ppLink ; Get offset for flag part of physent
- .align 5
- .globl EXT(hw_cvp)
+hwpSAtrPhX: lwarx r4,r5,r29 ; Get the old flags
+ rlwimi r4,r25,0,ppIb,ppGb ; Stick in the new attributes
+ stwcx. r4,r5,r29 ; Try to stuff it
+ bne-- hwpSAtrPhX ; Try again...
+; Note: CR0_EQ is set because of stwcx.
+ blr ; Return...
+
+; Function 4 - Set attributes in mapping
+
+ .set .,hwpOpBase+(4*128) ; Generate error if previous function too long
+
+hwpSAtrMap: lwz r9,mpFlags(r31) ; Get the mapping flags
+ lwz r8,mpVAddr+4(r31) ; Get the attribute part of mapping
+ li r2,mpM ; Force on coherent
+ rlwinm. r9,r9,0,mpPermb,mpPermb ; Is the mapping permanent?
+ li r0,lo16(mpWIMG) ; Get wimg mask
+ crnot cr0_eq,cr0_eq ; Change CR0_EQ to true if mapping is permanent
+ rlwimi r2,r25,32-(mpIb-32-ppIb),mpIb-32,mpIb-32
+ ; Copy in the cache inhibited bit
+ beqlr-- ; Leave if permanent mapping (before we trash R5)...
+ andc r5,r5,r0 ; Clear the old wimg
+ rlwimi r2,r25,32-(mpGb-32-ppGb),mpGb-32,mpGb-32
+ ; Copy in the guarded bit
+ mfsprg r9,2 ; Feature flags
+ or r5,r5,r2 ; Move in the new wimg
+ rlwimi r8,r5,0,20,31 ; Copy into the mapping copy
+ lwz r2,mpPAddr(r31) ; Get the physical address
+ li r0,0xFFF ; Start a mask
+ andi. r9,r9,pf32Byte+pf128Byte ; Get cache line size
+ rlwinm r5,r0,0,1,0 ; Copy to top half
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ rlwinm r2,r2,12,1,0 ; Copy to top and rotate to make physical address with junk left
+ and r5,r5,r2 ; Clean stuff in top 32 bits
+ andc r2,r2,r0 ; Clean bottom too
+ rlwimi r5,r2,0,0,31 ; Insert low 23 to make full physical address
+ b hwpSAM ; Join common
+
+; NOTE: we moved the remainder of the code out of here because it
+; did not fit in the 128 bytes allotted. It got stuck into the free space
+; at the end of the no-op function.
+
+
+
+
+; Function 5 - Clear reference in physent
+
+ .set .,hwpOpBase+(5*128) ; Generate error if previous function too long
+
+hwpCRefPhy: li r5,ppLink+4 ; Get offset for flag part of physent
+
+hwpCRefPhX: lwarx r4,r5,r29 ; Get the old flags
+ rlwinm r4,r4,0,ppRb+1-32,ppRb-1-32 ; Clear R
+ stwcx. r4,r5,r29 ; Try to stuff it
+ bne-- hwpCRefPhX ; Try again...
+; Note: CR0_EQ is set because of stwcx.
+ blr ; Return...
-LEXT(hw_cvp)
- rlwinm r4,r3,0,0,19 ; Round back to the mapping block allocation control block
- rlwinm r3,r3,0,0,26 ; Clean out any flags
- lwz r4,mbvrswap(r4) ; Get the conversion value
- xor r3,r3,r4 ; Convert to virtual
- blr
+; Function 6 - Clear reference in mapping
+ .set .,hwpOpBase+(6*128) ; Generate error if previous function too long
-/* int mapalc(struct mappingblok *mb) - Finds, allocates, and checks a free mapping entry in a block
- *
- * Lock must already be held on mapping block list
+hwpCRefMap: li r0,lo16(mpR) ; Get reference bit
+ lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
+ andc r5,r5,r0 ; Clear in PTE copy
+ andc r8,r8,r0 ; and in the mapping
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ blr ; Return...
+
+
+; Function 7 - Clear change in physent
+
+ .set .,hwpOpBase+(7*128) ; Generate error if previous function too long
+
+hwpCCngPhy: li r5,ppLink+4 ; Get offset for flag part of physent
+
+hwpCCngPhX: lwarx r4,r5,r29 ; Get the old flags
+ rlwinm r4,r4,0,ppCb+1-32,ppCb-1-32 ; Clear C
+ stwcx. r4,r5,r29 ; Try to stuff it
+ bne-- hwpCCngPhX ; Try again...
+; Note: CR0_EQ is set because of stwcx.
+ blr ; Return...
+
+
+; Function 8 - Clear change in mapping
+
+ .set .,hwpOpBase+(8*128) ; Generate error if previous function too long
+
+hwpCCngMap: li r0,lo16(mpC) ; Get change bit
+ lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
+ andc r5,r5,r0 ; Clear in PTE copy
+ andc r8,r8,r0 ; and in the mapping
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ blr ; Return...
+
+
+; Function 9 - Set reference in physent
+
+ .set .,hwpOpBase+(9*128) ; Generate error if previous function too long
+
+hwpSRefPhy: li r5,ppLink+4 ; Get offset for flag part of physent
+
+hwpSRefPhX: lwarx r4,r5,r29 ; Get the old flags
+ ori r4,r4,lo16(ppR) ; Set the reference
+ stwcx. r4,r5,r29 ; Try to stuff it
+ bne-- hwpSRefPhX ; Try again...
+; Note: CR0_EQ is set because of stwcx.
+ blr ; Return...
+
+
+; Function 10 - Set reference in mapping
+
+ .set .,hwpOpBase+(10*128) ; Generate error if previous function too long
+
+hwpSRefMap: lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
+ ori r8,r8,lo16(mpR) ; Set reference in mapping
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ blr ; Return...
+
+; Function 11 - Set change in physent
+
+ .set .,hwpOpBase+(11*128) ; Generate error if previous function too long
+
+hwpSCngPhy: li r5,ppLink+4 ; Get offset for flag part of physent
+
+hwpSCngPhX: lwarx r4,r5,r29 ; Get the old flags
+ ori r4,r4,lo16(ppC) ; Set the change bit
+ stwcx. r4,r5,r29 ; Try to stuff it
+ bne-- hwpSCngPhX ; Try again...
+; Note: CR0_EQ is set because of stwcx.
+ blr ; Return...
+
+; Function 12 - Set change in mapping
+
+ .set .,hwpOpBase+(12*128) ; Generate error if previous function too long
+
+hwpSCngMap: lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
+ ori r8,r8,lo16(mpC) ; Set chage in mapping
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ blr ; Return...
+
+; Function 13 - Test reference in physent
+
+ .set .,hwpOpBase+(13*128) ; Generate error if previous function too long
+
+hwpTRefPhy: lwz r0,ppLink+4(r29) ; Get the flags from physent
+ rlwinm. r0,r0,0,ppRb-32,ppRb-32 ; Isolate reference bit and see if 0
+ blr ; Return (CR0_EQ set to continue if reference is off)...
+
+
+; Function 14 - Test reference in mapping
+
+ .set .,hwpOpBase+(14*128) ; Generate error if previous function too long
+
+hwpTRefMap: rlwinm. r0,r5,0,mpRb-32,mpRb-32 ; Isolate reference bit and see if 0
+ blr ; Return (CR0_EQ set to continue if reference is off)...
+
+
+; Function 15 - Test change in physent
+
+ .set .,hwpOpBase+(15*128) ; Generate error if previous function too long
+
+hwpTCngPhy: lwz r0,ppLink+4(r29) ; Get the flags from physent
+ rlwinm. r0,r0,0,ppCb-32,ppCb-32 ; Isolate change bit and see if 0
+ blr ; Return (CR0_EQ set to continue if change is off)...
+
+
+; Function 16 - Test change in mapping
+
+ .set .,hwpOpBase+(16*128) ; Generate error if previous function too long
+
+hwpTCngMap: rlwinm. r0,r5,0,mpCb-32,mpCb-32 ; Isolate change bit and see if 0
+ blr ; Return (CR0_EQ set to continue if change is off)...
+
+
+; Function 17 - Test reference and change in physent
+
+ .set .,hwpOpBase+(17*128) ; Generate error if previous function too long
+
+hwpTRefCngPhy:
+ lwz r0,ppLink+4(r29) ; Get the flags from physent
+ rlwinm r0,r0,0,ppRb-32,ppCb-32 ; Isolate reference and change bits
+ cmplwi r0,lo16(ppR|ppC) ; cr0_eq <- ((R == 1) && (C == 1))
+ crnot cr0_eq,cr0_eq ; cr0_eq <- ((R == 0) || (C == 0))
+ blr ; Return (CR0_EQ set to continue if either R or C is off)...
+
+
+; Function 18 - Test reference and change in mapping
+
+ .set .,hwpOpBase+(18*128) ; Generate error if previous function too long
+hwpTRefCngMap:
+ rlwinm r0,r5,0,mpRb-32,mpCb-32 ; Isolate reference and change bits from mapping
+ cmplwi r0,lo16(mpR|mpC) ; cr0_eq <- ((R == 1) && (C == 1))
+ crnot cr0_eq,cr0_eq ; cr0_eq <- ((R == 0) || (C == 0))
+ blr ; Return (CR0_EQ set to continue if either R or C is off)...
+
+
+; Function 19 - Clear reference and change in physent
+
+ .set .,hwpOpBase+(19*128) ; Generate error if previous function too long
+hwpCRefCngPhy:
+ li r5,ppLink+4 ; Get offset for flag part of physent
+
+hwpCRefCngPhX:
+ lwarx r4,r5,r29 ; Get the old flags
+ andc r4,r4,r25 ; Clear R and C as specified by mask
+ stwcx. r4,r5,r29 ; Try to stuff it
+ bne-- hwpCRefCngPhX ; Try again...
+; Note: CR0_EQ is set because of stwcx.
+ blr ; Return...
+
+
+; Function 20 - Clear reference and change in mapping
+
+ .set .,hwpOpBase+(20*128) ; Generate error if previous function too long
+hwpCRefCngMap:
+ srwi r0,r25,(ppRb - mpRb) ; Align reference/change clear mask (phys->map)
+ lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
+ andc r5,r5,r0 ; Clear in PTE copy
+ andc r8,r8,r0 ; and in the mapping
+ cmpw r0,r0 ; Make sure we return CR0_EQ
+ stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
+ blr ; Return...
+
+
+ .set .,hwpOpBase+(21*128) ; Generate error if previous function too long
+
+;
+; unsigned int hw_protect(pmap, va, prot, *nextva) - Changes protection on a specific mapping.
+;
+; Returns:
+; mapRtOK - if all is ok
+; mapRtBadLk - if mapping lock fails
+; mapRtPerm - if mapping is permanent
+; mapRtNotFnd - if mapping is not found
+; mapRtBlock - if mapping is a block
+;
+ .align 5
+ .globl EXT(hw_protect)
+
+LEXT(hw_protect)
+ stwu r1,-(FM_ALIGN((31-24+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r24,FM_ARG0+0x00(r1) ; Save a register
+ stw r25,FM_ARG0+0x04(r1) ; Save a register
+ mr r25,r7 ; Remember address of next va
+ stw r26,FM_ARG0+0x08(r1) ; Save a register
+ stw r27,FM_ARG0+0x0C(r1) ; Save a register
+ stw r28,FM_ARG0+0x10(r1) ; Save a register
+ mr r24,r6 ; Save the new protection flags
+ stw r29,FM_ARG0+0x14(r1) ; Save a register
+ stw r30,FM_ARG0+0x18(r1) ; Save a register
+ stw r31,FM_ARG0+0x1C(r1) ; Save a register
+ stw r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r3) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne hpPanic ; Call not valid for guest shadow assist pmap
+#endif
+
+ lwz r6,pmapvr(r3) ; Get the first part of the VR translation for pmap
+ lwz r7,pmapvr+4(r3) ; Get the second part
+
+
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
+
+ mr r27,r11 ; Remember the old MSR
+ mr r26,r12 ; Remember the feature bits
+
+ xor r28,r3,r7 ; Change the common 32- and 64-bit half
+
+ bf-- pf64Bitb,hpSF1 ; skip if 32-bit...
+
+ rldimi r28,r6,32,0 ; Shift the fixed upper part of the physical over and cram in top
+
+hpSF1: mr r29,r4 ; Save top half of vaddr
+ mr r30,r5 ; Save the bottom half
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne-- hpBadLock ; Nope...
+
+ mr r3,r28 ; get the pmap address
+ mr r4,r29 ; Get bits 0:31 to look for
+ mr r5,r30 ; Get bits 32:64
+
+ bl EXT(mapSearch) ; Go see if we can find it (note: R7 comes back with mpFlags)
+
+ rlwinm. r0,r7,0,mpType ; Is this a normal mapping?
+ crmove cr1_eq,cr0_eq ; cr1_eq <- this is a normal mapping
+ andi. r0,r7,mpPerm|mpRIP ; Is it permanent or being removed?
+ cror cr1_eq,cr0_eq,cr1_eq ; cr1_eq <- normal mapping and not permanent and not being removed
+ mr. r31,r3 ; Save the mapping if we found it
+ mr r29,r4 ; Save next va high half
+ mr r30,r5 ; Save next va low half
+
+ beq-- hpNotFound ; Not found...
+
+ bf-- cr1_eq,hpNotAllowed ; Something special is happening...
+
+ bt++ pf64Bitb,hpDo64 ; Split for 64 bit
+
+ bl mapInvPte32 ; Invalidate and lock PTEG, also merge into physent
+
+ rlwimi r5,r24,0,mpPPb-32,mpPPe-32 ; Stick in the new pp (note that we ignore no-execute for 32-bit)
+ mr. r3,r3 ; Was there a previously valid PTE?
+
+ stb r5,mpVAddr+7(r31) ; Set the new pp field (do not muck with the rest)
+
+ beq-- hpNoOld32 ; Nope...
+
+ stw r5,4(r3) ; Store second half of PTE
+ eieio ; Make sure we do not reorder
+ stw r4,0(r3) ; Revalidate the PTE
+
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock PCA
+
+hpNoOld32: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ li r3,mapRtOK ; Set normal return
+ b hpR32 ; Join common...
+
+ .align 5
+
+
+hpDo64: bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
+
+ rldimi r5,r24,0,mpNb ; Stick in the new no-exectue and pp bits
+ mr. r3,r3 ; Was there a previously valid PTE?
+
+ stb r5,mpVAddr+7(r31) ; Set the new pp field (do not muck with the rest)
+
+ beq-- hpNoOld64 ; Nope...
+
+ std r5,8(r3) ; Store second half of PTE
+ eieio ; Make sure we do not reorder
+ std r4,0(r3) ; Revalidate the PTE
+
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock PCA
+
+hpNoOld64: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ li r3,mapRtOK ; Set normal return
+ b hpR64 ; Join common...
+
+ .align 5
+
+hpReturn: bt++ pf64Bitb,hpR64 ; Yes...
+
+hpR32: mtmsr r27 ; Restore enables/translation/etc.
+ isync
+ b hpReturnC ; Join common...
+
+hpR64: mtmsrd r27 ; Restore enables/translation/etc.
+ isync
+
+hpReturnC: stw r29,0(r25) ; Save the top of the next va
+ stw r30,4(r25) ; Save the bottom of the next va
+ lwz r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+ lwz r24,FM_ARG0+0x00(r1) ; Save a register
+ lwz r25,FM_ARG0+0x04(r1) ; Save a register
+ lwz r26,FM_ARG0+0x08(r1) ; Save a register
+ mtlr r0 ; Restore the return
+ lwz r27,FM_ARG0+0x0C(r1) ; Save a register
+ lwz r28,FM_ARG0+0x10(r1) ; Save a register
+ lwz r29,FM_ARG0+0x14(r1) ; Save a register
+ lwz r30,FM_ARG0+0x18(r1) ; Save a register
+ lwz r31,FM_ARG0+0x1C(r1) ; Save a register
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
+
+ .align 5
+
+hpBadLock: li r3,mapRtBadLk ; Set lock time out error code
+ b hpReturn ; Leave....
+
+hpNotFound: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ li r3,mapRtNotFnd ; Set that we did not find the requested page
+ b hpReturn ; Leave....
+
+hpNotAllowed:
+ rlwinm. r0,r7,0,mpRIPb,mpRIPb ; Is it actually being removed?
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bne-- hpNotFound ; Yeah...
+ bl sxlkUnlock ; Unlock the search list
+
+ li r3,mapRtBlock ; Assume it was a block
+ rlwinm r0,r7,0,mpType ; Isolate mapping type
+ cmplwi r0,mpBlock ; Is this a block mapping?
+ beq++ hpReturn ; Yes, leave...
+
+ li r3,mapRtPerm ; Set that we hit a permanent page
+ b hpReturn ; Leave....
+
+hpPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
+
+
+;
+; int hw_test_rc(pmap, va, reset) - tests RC on a specific va
+;
+; Returns following code ORed with RC from mapping
+; mapRtOK - if all is ok
+; mapRtBadLk - if mapping lock fails
+; mapRtNotFnd - if mapping is not found
+;
+ .align 5
+ .globl EXT(hw_test_rc)
+
+LEXT(hw_test_rc)
+ stwu r1,-(FM_ALIGN((31-24+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
+ mflr r0 ; Save the link register
+ stw r24,FM_ARG0+0x00(r1) ; Save a register
+ stw r25,FM_ARG0+0x04(r1) ; Save a register
+ stw r26,FM_ARG0+0x08(r1) ; Save a register
+ stw r27,FM_ARG0+0x0C(r1) ; Save a register
+ stw r28,FM_ARG0+0x10(r1) ; Save a register
+ mr r24,r6 ; Save the reset request
+ stw r29,FM_ARG0+0x14(r1) ; Save a register
+ stw r30,FM_ARG0+0x18(r1) ; Save a register
+ stw r31,FM_ARG0+0x1C(r1) ; Save a register
+ stw r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+
+#if DEBUG
+ lwz r11,pmapFlags(r3) ; Get pmaps flags
+ rlwinm. r11,r11,0,pmapVMgsaa ; Is guest shadow assist active?
+ bne htrPanic ; Call not valid for guest shadow assist pmap
+#endif
+
+ lwz r6,pmapvr(r3) ; Get the first part of the VR translation for pmap
+ lwz r7,pmapvr+4(r3) ; Get the second part
+
+
+ bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
+
+ mr r27,r11 ; Remember the old MSR
+ mr r26,r12 ; Remember the feature bits
+
+ xor r28,r3,r7 ; Change the common 32- and 64-bit half
+
+ bf-- pf64Bitb,htrSF1 ; skip if 32-bit...
+
+ rldimi r28,r6,32,0 ; Shift the fixed upper part of the physical over and cram in top
+
+htrSF1: mr r29,r4 ; Save top half of vaddr
+ mr r30,r5 ; Save the bottom half
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ li r25,0 ; Clear RC
+ bne-- htrBadLock ; Nope...
+
+ mr r3,r28 ; get the pmap address
+ mr r4,r29 ; Get bits 0:31 to look for
+ mr r5,r30 ; Get bits 32:64
+
+ bl EXT(mapSearch) ; Go see if we can find it (R7 comes back with mpFlags)
+
+ rlwinm. r0,r7,0,mpType ; Is this a normal mapping?
+ crmove cr1_eq,cr0_eq ; cr1_eq <- this is a normal mapping
+ andi. r0,r7,mpPerm|mpRIP ; Is it permanent or being removed?
+ crand cr1_eq,cr0_eq,cr1_eq ; cr1_eq <- normal mapping and not permanent and not being removed
+ mr. r31,r3 ; Save the mapping if we found it
+ crandc cr1_eq,cr1_eq,cr0_eq ; cr1_eq <- found & normal & not permanent & not being removed
+
+ bf-- cr1_eq,htrNotFound ; Not found, something special, or being removed...
+
+ bt++ pf64Bitb,htrDo64 ; Split for 64 bit
+
+ bl mapInvPte32 ; Invalidate and lock PTEG, also merge into physent
+
+ cmplwi cr1,r24,0 ; Do we want to clear RC?
+ lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
+ mr. r3,r3 ; Was there a previously valid PTE?
+ li r0,lo16(mpR|mpC) ; Get bits to clear
+
+ and r25,r5,r0 ; Save the RC bits
+ beq++ cr1,htrNoClr32 ; Nope...
+
+ andc r12,r12,r0 ; Clear mapping copy of RC
+ andc r5,r5,r0 ; Clear PTE copy of RC
+ sth r12,mpVAddr+6(r31) ; Set the new RC
+
+htrNoClr32: beq-- htrNoOld32 ; No previously valid PTE...
+
+ sth r5,6(r3) ; Store updated RC
+ eieio ; Make sure we do not reorder
+ stw r4,0(r3) ; Revalidate the PTE
+
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock PCA
+
+htrNoOld32: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+ li r3,mapRtOK ; Set normal return
+ b htrR32 ; Join common...
+
+ .align 5
+
+
+htrDo64: bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
+
+ cmplwi cr1,r24,0 ; Do we want to clear RC?
+ lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
+ mr. r3,r3 ; Was there a previously valid PTE?
+ li r0,lo16(mpR|mpC) ; Get bits to clear
+
+ and r25,r5,r0 ; Save the RC bits
+ beq++ cr1,htrNoClr64 ; Nope...
+
+ andc r12,r12,r0 ; Clear mapping copy of RC
+ andc r5,r5,r0 ; Clear PTE copy of RC
+ sth r12,mpVAddr+6(r31) ; Set the new RC
+
+htrNoClr64: beq-- htrNoOld64 ; Nope, no pevious pte...
+
+ sth r5,14(r3) ; Store updated RC
+ eieio ; Make sure we do not reorder
+ std r4,0(r3) ; Revalidate the PTE
+
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock PCA
+
+htrNoOld64: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+ li r3,mapRtOK ; Set normal return
+ b htrR64 ; Join common...
+
+ .align 5
+
+htrReturn: bt++ pf64Bitb,htrR64 ; Yes...
+
+htrR32: mtmsr r27 ; Restore enables/translation/etc.
+ isync
+ b htrReturnC ; Join common...
+
+htrR64: mtmsrd r27 ; Restore enables/translation/etc.
+ isync
+
+htrReturnC: lwz r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
+ or r3,r3,r25 ; Send the RC bits back
+ lwz r24,FM_ARG0+0x00(r1) ; Save a register
+ lwz r25,FM_ARG0+0x04(r1) ; Save a register
+ lwz r26,FM_ARG0+0x08(r1) ; Save a register
+ mtlr r0 ; Restore the return
+ lwz r27,FM_ARG0+0x0C(r1) ; Save a register
+ lwz r28,FM_ARG0+0x10(r1) ; Save a register
+ lwz r29,FM_ARG0+0x14(r1) ; Save a register
+ lwz r30,FM_ARG0+0x18(r1) ; Save a register
+ lwz r31,FM_ARG0+0x1C(r1) ; Save a register
+ lwz r1,0(r1) ; Pop the stack
+ blr ; Leave...
+
+ .align 5
+
+htrBadLock: li r3,mapRtBadLk ; Set lock time out error code
+ b htrReturn ; Leave....
+
+htrNotFound:
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ li r3,mapRtNotFnd ; Set that we did not find the requested page
+ b htrReturn ; Leave....
+
+htrPanic: lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show that we failed some kind of mapping thing
+ sc
+
+
+;
+;
+; mapFindLockPN - find and lock physent for a given page number
+;
+;
+ .align 5
+mapFindLockPN:
+ lis r9,hi16(EXT(pmap_mem_regions)) ; Point to the start of the region table
+ mr r2,r3 ; Save our target
+ ori r9,r9,lo16(EXT(pmap_mem_regions)) ; Point to the start of the region table
+
+mapFLPNitr: lwz r3,mrPhysTab(r9) ; Get the actual table address
+ lwz r5,mrStart(r9) ; Get start of table entry
+ lwz r0,mrEnd(r9) ; Get end of table entry
+ addi r9,r9,mrSize ; Point to the next slot
+ cmplwi cr7,r3,0 ; Are we at the end of the table?
+ cmplw r2,r5 ; See if we are in this table
+ cmplw cr1,r2,r0 ; Check end also
+ sub r4,r2,r5 ; Calculate index to physical entry
+ beq-- cr7,mapFLPNmiss ; Leave if we did not find an entry...
+ cror cr0_lt,cr0_lt,cr1_gt ; Set CR0_LT if it is NOT this entry
+ slwi r4,r4,3 ; Get offset to physical entry
+
+ blt-- mapFLPNitr ; Did not find it...
+
+ add r3,r3,r4 ; Point right to the slot
+ b mapPhysLock ; Join common lock code
+
+mapFLPNmiss:
+ li r3,0 ; Show that we did not find it
+ blr ; Leave...
+
+
+;
+; mapPhysFindLock - find physent list and lock it
+; R31 points to mapping
+;
+ .align 5
+
+mapPhysFindLock:
+ lbz r4,mpFlags+1(r31) ; Get the index into the physent bank table
+ lis r3,ha16(EXT(pmap_mem_regions)) ; Get high order of physent table (note use of ha16 to get value appropriate for an addi of low part)
+ rlwinm r4,r4,2,24,29 ; Mask index bits and convert to byte offset
+ addi r4,r4,lo16(EXT(pmap_mem_regions)) ; Get low part of address of entry
+ add r3,r3,r4 ; Point to table entry
+ lwz r5,mpPAddr(r31) ; Get physical page number
+ lwz r7,mrStart(r3) ; Get the start of range
+ lwz r3,mrPhysTab(r3) ; Get the start of the entries for this bank
+ sub r6,r5,r7 ; Get index to physent
+ rlwinm r6,r6,3,0,28 ; Get offset to physent
+ add r3,r3,r6 ; Point right to the physent
+ b mapPhysLock ; Join in the lock...
+
+;
+; mapPhysLock - lock a physent list
+; R3 contains list header
+;
+ .align 5
+
+mapPhysLockS:
+ li r2,lgKillResv ; Get a spot to kill reservation
+ stwcx. r2,0,r2 ; Kill it...
+
+mapPhysLockT:
+ lwz r2,ppLink(r3) ; Get physent chain header
+ rlwinm. r2,r2,0,0,0 ; Is lock clear?
+ bne-- mapPhysLockT ; Nope, still locked...
+
+mapPhysLock:
+ lwarx r2,0,r3 ; Get the lock
+ rlwinm. r0,r2,0,0,0 ; Is it locked?
+ oris r0,r2,0x8000 ; Set the lock bit
+ bne-- mapPhysLockS ; It is locked, spin on it...
+ stwcx. r0,0,r3 ; Try to stuff it back...
+ bne-- mapPhysLock ; Collision, try again...
+ isync ; Clear any speculations
+ blr ; Leave...
+
+
+;
+; mapPhysUnlock - unlock a physent list
+; R3 contains list header
+;
+ .align 5
+
+mapPhysUnlock:
+ lwz r0,ppLink(r3) ; Get physent chain header
+ rlwinm r0,r0,0,1,31 ; Clear the lock bit
+ eieio ; Make sure unlock comes last
+ stw r0,ppLink(r3) ; Unlock the list
+ blr
+
+;
+; mapPhysMerge - merge the RC bits into the master copy
+; R3 points to the physent
+; R4 contains the RC bits
+;
+; Note: we just return if RC is 0
+;
+ .align 5
+
+mapPhysMerge:
+ rlwinm. r4,r4,PTE1_REFERENCED_BIT+(64-ppRb),ppRb-32,ppCb-32 ; Isolate RC bits
+ la r5,ppLink+4(r3) ; Point to the RC field
+ beqlr-- ; Leave if RC is 0...
+
+mapPhysMergeT:
+ lwarx r6,0,r5 ; Get the RC part
+ or r6,r6,r4 ; Merge in the RC
+ stwcx. r6,0,r5 ; Try to stuff it back...
+ bne-- mapPhysMergeT ; Collision, try again...
+ blr ; Leave...
+
+;
+; Sets the physent link pointer and preserves all flags
+; The list is locked
+; R3 points to physent
+; R4 has link to set
+;
+
+ .align 5
+
+mapPhyCSet32:
+ la r5,ppLink+4(r3) ; Point to the link word
+
+mapPhyCSetR:
+ lwarx r2,0,r5 ; Get the link and flags
+ rlwimi r4,r2,0,ppFlags ; Insert the flags
+ stwcx. r4,0,r5 ; Stick them back
+ bne-- mapPhyCSetR ; Someone else did something, try again...
+ blr ; Return...
+
+ .align 5
+
+mapPhyCSet64:
+ li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+
+mapPhyCSet64x:
+ ldarx r2,0,r3 ; Get the link and flags
+ and r5,r2,r0 ; Isolate the flags
+ or r6,r4,r5 ; Add them to the link
+ stdcx. r6,0,r3 ; Stick them back
+ bne-- mapPhyCSet64x ; Someone else did something, try again...
+ blr ; Return...
+
+;
+; mapBumpBusy - increment the busy count on a mapping
+; R3 points to mapping
+;
+
+ .align 5
+
+mapBumpBusy:
+ lwarx r4,0,r3 ; Get mpBusy
+ addis r4,r4,0x0100 ; Bump the busy count
+ stwcx. r4,0,r3 ; Save it back
+ bne-- mapBumpBusy ; This did not work, try again...
+ blr ; Leave...
+
+;
+; mapDropBusy - increment the busy count on a mapping
+; R3 points to mapping
+;
+
+ .globl EXT(mapping_drop_busy)
+ .align 5
+
+LEXT(mapping_drop_busy)
+mapDropBusy:
+ lwarx r4,0,r3 ; Get mpBusy
+ addis r4,r4,0xFF00 ; Drop the busy count
+ stwcx. r4,0,r3 ; Save it back
+ bne-- mapDropBusy ; This did not work, try again...
+ blr ; Leave...
+
+;
+; mapDrainBusy - drain the busy count on a mapping
+; R3 points to mapping
+; Note: we already have a busy for ourselves. Only one
+; busy per processor is allowed, so we just spin here
+; waiting for the count to drop to 1.
+; Also, the mapping can not be on any lists when we do this
+; so all we are doing is waiting until it can be released.
+;
+
+ .align 5
+
+mapDrainBusy:
+ lwz r4,mpFlags(r3) ; Get mpBusy
+ rlwinm r4,r4,8,24,31 ; Clean it up
+ cmplwi r4,1 ; Is is just our busy?
+ beqlr++ ; Yeah, it is clear...
+ b mapDrainBusy ; Try again...
+
+
+
+;
+; handleDSeg - handle a data segment fault
+; handleISeg - handle an instruction segment fault
+;
+; All that we do here is to map these to DSI or ISI and insure
+; that the hash bit is not set. This forces the fault code
+; to also handle the missing segment.
+;
+; At entry R2 contains per_proc, R13 contains savarea pointer,
+; and R11 is the exception code.
+;
+
+ .align 5
+ .globl EXT(handleDSeg)
+
+LEXT(handleDSeg)
+
+ li r11,T_DATA_ACCESS ; Change fault to DSI
+ stw r11,saveexception(r13) ; Change the exception code from seg fault to PTE miss
+ b EXT(handlePF) ; Join common...
+
+ .align 5
+ .globl EXT(handleISeg)
+
+LEXT(handleISeg)
+
+ li r11,T_INSTRUCTION_ACCESS ; Change fault to ISI
+ stw r11,saveexception(r13) ; Change the exception code from seg fault to PTE miss
+ b EXT(handlePF) ; Join common...
+
+
+/*
+ * handlePF - handle a page fault interruption
+ *
+ * At entry R2 contains per_proc, R13 contains savarea pointer,
+ * and R11 is the exception code.
+ *
+ * This first part does a quick check to see if we can handle the fault.
+ * We canot handle any kind of protection exceptions here, so we pass
+ * them up to the next level.
+ *
+ * NOTE: In order for a page-fault redrive to work, the translation miss
+ * bit must be set in the DSISR (or SRR1 for IFETCH). That must occur
+ * before we come here.
+ */
+
+ .align 5
+ .globl EXT(handlePF)
+
+LEXT(handlePF)
+
+ mfsprg r12,2 ; Get feature flags
+ cmplwi r11,T_INSTRUCTION_ACCESS ; See if this is for the instruction
+ lwz r8,savesrr1+4(r13) ; Get the MSR to determine mode
+ mtcrf 0x02,r12 ; move pf64Bit to cr6
+ lis r0,hi16(dsiNoEx|dsiProt|dsiInvMode|dsiAC) ; Get the types that we cannot handle here
+ lwz r18,SAVflags(r13) ; Get the flags
+
+ beq-- gotIfetch ; We have an IFETCH here...
+
+ lwz r27,savedsisr(r13) ; Get the DSISR
+ lwz r29,savedar(r13) ; Get the first half of the DAR
+ lwz r30,savedar+4(r13) ; And second half
+
+ b ckIfProt ; Go check if this is a protection fault...
+
+gotIfetch: andis. r27,r8,hi16(dsiValid) ; Clean this up to construct a DSISR value
+ lwz r29,savesrr0(r13) ; Get the first half of the instruction address
+ lwz r30,savesrr0+4(r13) ; And second half
+ stw r27,savedsisr(r13) ; Save the "constructed" DSISR
+
+ckIfProt: and. r4,r27,r0 ; Is this a non-handlable exception?
+ li r20,64 ; Set a limit of 64 nests for sanity check
+ bne-- hpfExit ; Yes... (probably not though)
+
+;
+; Note: if the RI is on, we are accessing user space from the kernel, therefore we
+; should be loading the user pmap here.
+;
+
+ andi. r0,r8,lo16(MASK(MSR_PR)|MASK(MSR_RI)) ; Are we addressing user or kernel space?
+ lis r8,hi16(EXT(kernel_pmap_phys)) ; Assume kernel
+ mr r19,r2 ; Remember the per_proc
+ ori r8,r8,lo16(EXT(kernel_pmap_phys)) ; Assume kernel (bottom of address)
+ mr r23,r30 ; Save the low part of faulting address
+ beq-- hpfInKern ; Skip if we are in the kernel
+ la r8,ppUserPmap(r19) ; Point to the current user pmap
+
+hpfInKern: mr r22,r29 ; Save the high part of faulting address
+
+ bt-- pf64Bitb,hpf64a ; If 64-bit, skip the next bit...
+
+;
+; On 32-bit machines we emulate a segment exception by loading unused SRs with a
+; predefined value that corresponds to no address space. When we see that value
+; we turn off the PTE miss bit in the DSISR to drive the code later on that will
+; cause the proper SR to be loaded.
+;
+
+ lwz r28,4(r8) ; Pick up the pmap
+ rlwinm. r18,r18,0,SAVredriveb,SAVredriveb ; Was this a redrive?
+ mr r25,r28 ; Save the original pmap (in case we nest)
+ lwz r0,pmapFlags(r28) ; Get pmap's flags
+ bne hpfGVtest ; Segs are not ours if so...
+ mfsrin r4,r30 ; Get the SR that was used for translation
+ cmplwi r4,invalSpace ; Is this a simulated segment fault?
+ bne++ hpfGVtest ; No...
+
+ rlwinm r27,r27,0,dsiMissb+1,dsiMissb-1 ; Clear the PTE miss bit in DSISR
+ b hpfGVtest ; Join on up...
+
+ .align 5
+
+ nop ; Push hpfNest to a 32-byte boundary
+ nop ; Push hpfNest to a 32-byte boundary
+ nop ; Push hpfNest to a 32-byte boundary
+
+hpf64a: ld r28,0(r8) ; Get the pmap pointer (64-bit)
+ mr r25,r28 ; Save the original pmap (in case we nest)
+ lwz r0,pmapFlags(r28) ; Get pmap's flags
+
+hpfGVtest: rlwinm. r0,r0,0,pmapVMgsaa ; Using guest shadow mapping assist?
+ bne hpfGVxlate ; Yup, do accelerated shadow stuff
+
+;
+; This is where we loop descending nested pmaps
+;
+
+hpfNest: la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ addi r20,r20,-1 ; Count nest try
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne-- hpfBadLock ; Nope...
+
+ mr r3,r28 ; Get the pmap pointer
+ mr r4,r22 ; Get top of faulting vaddr
+ mr r5,r23 ; Get bottom of faulting vaddr
+ bl EXT(mapSearch) ; Go see if we can find it (R7 gets mpFlags)
+
+ rlwinm r0,r7,0,mpRIPb,mpRIPb ; Are we removing this one?
+ mr. r31,r3 ; Save the mapping if we found it
+ cmplwi cr1,r0,0 ; Check for removal
+ crorc cr0_eq,cr0_eq,cr1_eq ; Merge not found and removing
+
+ bt-- cr0_eq,hpfNotFound ; Not found or removing...
+
+ rlwinm r0,r7,0,mpType ; Isolate mapping type
+ cmplwi r0,mpNest ; Are we again nested?
+ cmplwi cr1,r0,mpLinkage ; Are we a linkage type?
+ cror cr0_eq,cr1_eq,cr0_eq ; cr0_eq <- nested or linkage type?
+ mr r26,r7 ; Get the flags for this mapping (passed back from search call)
+
+ lhz r21,mpSpace(r31) ; Get the space
+
+ bne++ hpfFoundIt ; No, we found our guy...
+
+
+#if pmapTransSize != 12
+#error pmapTrans entry size is not 12 bytes!!!!!!!!!!!! It is pmapTransSize
+#endif
+ cmplwi r0,mpLinkage ; Linkage mapping?
+ cmplwi cr1,r20,0 ; Too many nestings?
+ beq-- hpfSpclNest ; Do we need to do special handling?
+
+hpfCSrch: lhz r21,mpSpace(r31) ; Get the space
+ lwz r8,mpNestReloc(r31) ; Get the vaddr relocation
+ lwz r9,mpNestReloc+4(r31) ; Get the vaddr relocation bottom half
+ la r3,pmapSXlk(r28) ; Point to the old pmap search lock
+ lis r0,0x8000 ; Get 0xFFFFFFFF80000000
+ lis r10,hi16(EXT(pmapTrans)) ; Get the translate table
+ add r0,r0,r0 ; Get 0xFFFFFFFF00000000 for 64-bit or 0 for 32-bit
+ blt-- cr1,hpfNestTooMuch ; Too many nestings, must be a loop...
+ or r23,r23,r0 ; Make sure a carry will propagate all the way in 64-bit
+ slwi r11,r21,3 ; Multiply space by 8
+ ori r10,r10,lo16(EXT(pmapTrans)) ; Get the translate table low part
+ addc r23,r23,r9 ; Relocate bottom half of vaddr
+ lwz r10,0(r10) ; Get the actual translation map
+ slwi r12,r21,2 ; Multiply space by 4
+ add r10,r10,r11 ; Add in the higher part of the index
+ rlwinm r23,r23,0,0,31 ; Clean up the relocated address (does nothing in 32-bit)
+ adde r22,r22,r8 ; Relocate the top half of the vaddr
+ add r12,r12,r10 ; Now we are pointing at the space to pmap translation entry
+ bl sxlkUnlock ; Unlock the search list
+
+ bt++ pf64Bitb,hpfGetPmap64 ; Separate handling for 64-bit machines
+ lwz r28,pmapPAddr+4(r12) ; Get the physical address of the new pmap
+ cmplwi r28,0 ; Is the pmap paddr valid?
+ bne+ hpfNest ; Nest into new pmap...
+ b hpfBadPmap ; Handle bad pmap
+
+hpfGetPmap64:
+ ld r28,pmapPAddr(r12) ; Get the physical address of the new pmap
+ cmpldi r28,0 ; Is the pmap paddr valid?
+ bne++ hpfNest ; Nest into new pmap...
+ b hpfBadPmap ; Handle bad pmap
+
+
+;
+; Error condition. We only allow 64 nestings. This keeps us from having to
+; check for recusive nests when we install them.
+;
+
+ .align 5
+
+hpfNestTooMuch:
+ lwz r20,savedsisr(r13) ; Get the DSISR
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list (R3 good from above)
+ ori r20,r20,1 ; Indicate that there was a nesting problem
+ stw r20,savedsisr(r13) ; Stash it
+ lwz r11,saveexception(r13) ; Restore the exception code
+ b EXT(PFSExit) ; Yes... (probably not though)
+
+;
+; Error condition - lock failed - this is fatal
+;
+
+ .align 5
+
+hpfBadLock:
+ lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failMapping ; Show mapping failure
+ sc
+
+;
+; Error condition - space id selected an invalid pmap - fatal
+;
+
+ .align 5
+
+hpfBadPmap:
+ lis r0,hi16(Choke) ; System abend
+ ori r0,r0,lo16(Choke) ; System abend
+ li r3,failPmap ; Show invalid pmap
+ sc
+
+;
+; Did not find any kind of mapping
+;
+
+ .align 5
+
+hpfNotFound:
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock it
+ lwz r11,saveexception(r13) ; Restore the exception code
+
+hpfExit: ; We need this because we can not do a relative branch
+ b EXT(PFSExit) ; Yes... (probably not though)
+
+
+;
+; Here is where we handle special mappings. So far, the only use is to load a
+; processor specific segment register for copy in/out handling.
+;
+; The only (so far implemented) special map is used for copyin/copyout.
+; We keep a mapping of a "linkage" mapping in the per_proc.
+; The linkage mapping is basically a nested pmap that is switched in
+; as part of context switch. It relocates the appropriate user address
+; space slice into the right place in the kernel.
+;
+
+ .align 5
+
+hpfSpclNest:
+ la r31,ppUMWmp(r19) ; Just point to the mapping
+ oris r27,r27,hi16(dsiLinkage) ; Show that we had a linkage mapping here
+ b hpfCSrch ; Go continue search...
+
+
+;
+; We have now found a mapping for the address we faulted on.
+;
+
+;
+; Here we go about calculating what the VSID should be. We concatanate
+; the space ID (14 bits wide) 3 times. We then slide the vaddr over
+; so that bits 0:35 are in 14:49 (leaves a hole for one copy of the space ID).
+; Then we XOR and expanded space ID and the shifted vaddr. This gives us
+; the VSID.
+;
+; This is used both for segment handling and PTE handling
+;
+
+
+#if maxAdrSpb != 14
+#error maxAdrSpb (address space id size) is not 14 bits!!!!!!!!!!!!
+#endif
+
+; Important non-volatile registers at this point ('home' means the final pmap/mapping found
+; when a multi-level mapping has been successfully searched):
+; r21: home space id number
+; r22: relocated high-order 32 bits of vaddr
+; r23: relocated low-order 32 bits of vaddr
+; r25: pmap physical address
+; r27: dsisr
+; r28: home pmap physical address
+; r29: high-order 32 bits of faulting vaddr
+; r30: low-order 32 bits of faulting vaddr
+; r31: mapping's physical address
+
+ .align 5
+
+hpfFoundIt: lwz r12,pmapFlags(r28) ; Get the pmap flags so we can find the keys for this segment
+hpfGVfound: rlwinm. r0,r27,0,dsiMissb,dsiMissb ; Did we actually miss the segment?
+ rlwinm r15,r23,18,14,17 ; Shift 32:35 (0:3) of vaddr just above space ID
+ rlwinm r20,r21,28,22,31 ; Shift upper 10 bits of space into high order
+ rlwinm r14,r22,18,14,31 ; Shift 0:17 of vaddr over
+ rlwinm r0,r27,0,dsiLinkageb,dsiLinkageb ; Isolate linkage mapping flag
+ rlwimi r21,r21,14,4,17 ; Make a second copy of space above first
+ cmplwi cr5,r0,0 ; Did we just do a special nesting?
+ rlwimi r15,r22,18,0,13 ; Shift 18:31 of vaddr just above shifted 32:35
+ crorc cr0_eq,cr0_eq,cr5_eq ; Force outselves through the seg load code if special nest
+ rlwimi r21,r21,28,0,3 ; Get low order of 3rd copy of space at top of register
+ xor r14,r14,r20 ; Calculate the top half of VSID
+ xor r15,r15,r21 ; Calculate the bottom half of the VSID
+ rlwinm r14,r14,12,15,19 ; Slide the top of the VSID over to correct position (trim for 65 bit addressing)
+ rlwinm r12,r12,9,20,22 ; Isolate and position key for cache entry
+ rlwimi r14,r15,12,20,31 ; Slide top of bottom of VSID over into the top
+ rlwinm r15,r15,12,0,19 ; Slide the last nybble into the low order segment position
+ or r12,r12,r15 ; Add key into the bottom of VSID
+;
+; Note: ESID is in R22:R23 pair; VSID is in R14:R15; cache form VSID is R14:R12
+
+ bne++ hpfPteMiss ; Nope, normal PTE miss...
+
+;
+; Here is the only place that we make an entry in the pmap segment cache.
+;
+; Note that we do not make an entry in the segment cache for special
+; nested mappings. This makes the copy in/out segment get refreshed
+; when switching threads.
+;
+; The first thing that we do is to look up the ESID we are going to load
+; into a segment in the pmap cache. If it is already there, this is
+; a segment that appeared since the last time we switched address spaces.
+; If all is correct, then it was another processors that made the cache
+; entry. If not, well, it is an error that we should die on, but I have
+; not figured a good way to trap it yet.
+;
+; If we get a hit, we just bail, otherwise, lock the pmap cache, select
+; an entry based on the generation number, update the cache entry, and
+; also update the pmap sub-tag as well. The sub-tag is a table of 4 bit
+; entries that correspond to the last 4 bits (32:35 for 64-bit and
+; 0:3 for 32-bit) of the ESID.
+;
+; Then we unlock and bail.
+;
+; First lock it. Then select a free slot or steal one based on the generation
+; number. Then store it, update the allocation flags, and unlock.
+;
+; The cache entry contains an image of the ESID/VSID pair we would load for
+; 64-bit architecture. For 32-bit, it is a simple transform to an SR image.
+;
+; Remember, this cache entry goes in the ORIGINAL pmap (saved in R25), not
+; the current one, which may have changed because we nested.
+;
+; Also remember that we do not store the valid bit in the ESID. If we
+; od, this will break some other stuff.
+;
+
+ bne-- cr5,hpfNoCacheEnt2 ; Skip the cache entry if this is a "special nest" fault....
+
+ mr r3,r25 ; Point to the pmap
+ mr r4,r29 ; ESID high half
+ mr r5,r30 ; ESID low half
+ bl pmapCacheLookup ; Go see if this is in the cache already
+
+ mr. r3,r3 ; Did we find it?
+ mr r4,r11 ; Copy this to a different register
+
+ bne-- hpfNoCacheEnt ; Yes, we found it, no need to make another entry...
+
+ lwz r10,pmapSCSubTag(r25) ; Get the first part of the sub-tag lookup table
+ lwz r11,pmapSCSubTag+4(r25) ; Get the second part of the sub-tag lookup table
+
+ cntlzw r7,r4 ; Find a free slot
+
+ subi r6,r7,pmapSegCacheUse ; We end up with a negative if we find one
+ rlwinm r30,r30,0,0,3 ; Clean up the ESID
+ srawi r6,r6,31 ; Get 0xFFFFFFFF if we have one, 0 if not
+ addi r5,r4,1 ; Bump the generation number
+ and r7,r7,r6 ; Clear bit number if none empty
+ andc r8,r4,r6 ; Clear generation count if we found an empty
+ rlwimi r4,r5,0,17,31 ; Insert the new generation number into the control word
+ or r7,r7,r8 ; Select a slot number
+ li r8,0 ; Clear
+ andi. r7,r7,pmapSegCacheUse-1 ; Wrap into the number we are using
+ oris r8,r8,0x8000 ; Get the high bit on
+ la r9,pmapSegCache(r25) ; Point to the segment cache
+ slwi r6,r7,4 ; Get index into the segment cache
+ slwi r2,r7,2 ; Get index into the segment cache sub-tag index
+ srw r8,r8,r7 ; Get the mask
+ cmplwi r2,32 ; See if we are in the first or second half of sub-tag
+ li r0,0 ; Clear
+ rlwinm r2,r2,0,27,31 ; Wrap shift so we do not shift cache entries 8-F out
+ oris r0,r0,0xF000 ; Get the sub-tag mask
+ add r9,r9,r6 ; Point to the cache slot
+ srw r0,r0,r2 ; Slide sub-tag mask to right slot (shift work for either half)
+ srw r5,r30,r2 ; Slide sub-tag to right slot (shift work for either half)
+
+ stw r29,sgcESID(r9) ; Save the top of the ESID
+ andc r10,r10,r0 ; Clear sub-tag slot in case we are in top
+ andc r11,r11,r0 ; Clear sub-tag slot in case we are in bottom
+ stw r30,sgcESID+4(r9) ; Save the bottom of the ESID
+ or r10,r10,r5 ; Stick in subtag in case top half
+ or r11,r11,r5 ; Stick in subtag in case bottom half
+ stw r14,sgcVSID(r9) ; Save the top of the VSID
+ andc r4,r4,r8 ; Clear the invalid bit for the slot we just allocated
+ stw r12,sgcVSID+4(r9) ; Save the bottom of the VSID and the key
+ bge hpfSCSTbottom ; Go save the bottom part of sub-tag
+
+ stw r10,pmapSCSubTag(r25) ; Save the top of the sub-tag
+ b hpfNoCacheEnt ; Go finish up...
+
+hpfSCSTbottom:
+ stw r11,pmapSCSubTag+4(r25) ; Save the bottom of the sub-tag
+
+
+hpfNoCacheEnt:
+ eieio ; Make sure cache is updated before lock
+ stw r4,pmapCCtl(r25) ; Unlock, allocate, and bump generation number
+
+
+hpfNoCacheEnt2:
+ lwz r4,ppMapFlags(r19) ; Get the protection key modifier
+ bt++ pf64Bitb,hpfLoadSeg64 ; If 64-bit, go load the segment...
+
+;
+; Make and enter 32-bit segment register
+;
+
+ lwz r16,validSegs(r19) ; Get the valid SR flags
+ xor r12,r12,r4 ; Alter the storage key before loading segment register
+ rlwinm r2,r30,4,28,31 ; Isolate the segment we are setting
+ rlwinm r6,r12,19,1,3 ; Insert the keys and N bit
+ lis r0,0x8000 ; Set bit 0
+ rlwimi r6,r12,20,12,31 ; Insert 4:23 the VSID
+ srw r0,r0,r2 ; Get bit corresponding to SR
+ rlwimi r6,r14,20,8,11 ; Get the last nybble of the SR contents
+ or r16,r16,r0 ; Show that SR is valid
+
+ mtsrin r6,r30 ; Set the actual SR
+
+ stw r16,validSegs(r19) ; Set the valid SR flags
+
+ b hpfPteMiss ; SR loaded, go do a PTE...
+
+;
+; Make and enter 64-bit segment look-aside buffer entry.
+; Note that the cache entry is the right format except for valid bit.
+; We also need to convert from long long to 64-bit register values.
+;
+
+
+ .align 5
+
+hpfLoadSeg64:
+ ld r16,validSegs(r19) ; Get the valid SLB entry flags
+ sldi r8,r29,32 ; Move high order address over
+ sldi r10,r14,32 ; Move high part of VSID over
+
+ not r3,r16 ; Make valids be 0s
+ li r0,1 ; Prepare to set bit 0
+
+ cntlzd r17,r3 ; Find a free SLB
+ xor r12,r12,r4 ; Alter the storage key before loading segment table entry
+ or r9,r8,r30 ; Form full 64-bit address
+ cmplwi r17,63 ; Did we find a free SLB entry?
+ sldi r0,r0,63 ; Get bit 0 set
+ or r10,r10,r12 ; Move in low part and keys
+ addi r17,r17,1 ; Skip SLB 0 always
+ blt++ hpfFreeSeg ; Yes, go load it...
+
+;
+; No free SLB entries, select one that is in use and invalidate it
+;
+ lwz r4,ppSegSteal(r19) ; Get the next slot to steal
+ addi r17,r4,pmapSegCacheUse+1 ; Select stealee from non-cached slots only
+ addi r4,r4,1 ; Set next slot to steal
+ slbmfee r7,r17 ; Get the entry that is in the selected spot
+ subi r2,r4,63-pmapSegCacheUse ; Force steal to wrap
+ rldicr r7,r7,0,35 ; Clear the valid bit and the rest
+ srawi r2,r2,31 ; Get -1 if steal index still in range
+ slbie r7 ; Invalidate the in-use SLB entry
+ and r4,r4,r2 ; Reset steal index when it should wrap
+ isync ;
+
+ stw r4,ppSegSteal(r19) ; Set the next slot to steal
+;
+; We are now ready to stick the SLB entry in the SLB and mark it in use
+;
+
+hpfFreeSeg:
+ subi r4,r17,1 ; Adjust shift to account for skipping slb 0
+ mr r7,r9 ; Get a copy of the ESID with bits 36:63 clear
+ srd r0,r0,r4 ; Set bit mask for allocation
+ oris r9,r9,0x0800 ; Turn on the valid bit
+ or r16,r16,r0 ; Turn on the allocation flag
+ rldimi r9,r17,0,58 ; Copy in the SLB entry selector
+
+ beq++ cr5,hpfNoBlow ; Skip blowing away the SLBE if this is not a special nest...
+ slbie r7 ; Blow away a potential duplicate
+
+hpfNoBlow: slbmte r10,r9 ; Make that SLB entry
+
+ std r16,validSegs(r19) ; Mark as valid
+ b hpfPteMiss ; STE loaded, go do a PTE...
+
+;
+; The segment has been set up and loaded if need be. Now we are ready to build the
+; PTE and get it into the hash table.
+;
+; Note that there is actually a race here. If we start fault processing on
+; a different pmap, i.e., we have descended into a nested pmap, it is possible
+; that the nest could have been removed from the original pmap. We would
+; succeed with this translation anyway. I do not think we need to worry
+; about this (famous last words) because nobody should be unnesting anything
+; if there are still people activily using them. It should be up to the
+; higher level VM system to put the kibosh on this.
+;
+; There is also another race here: if we fault on the same mapping on more than
+; one processor at the same time, we could end up with multiple PTEs for the same
+; mapping. This is not a good thing.... We really only need one of the
+; fault handlers to finish, so what we do is to set a "fault in progress" flag in
+; the mapping. If we see that set, we just abandon the handler and hope that by
+; the time we restore context and restart the interrupted code, the fault has
+; been resolved by the other guy. If not, we will take another fault.
+;
+
+;
+; NOTE: IMPORTANT - CR7 contains a flag indicating if we have a block mapping or not.
+; It is required to stay there until after we call mapSelSlot!!!!
+;
+
+ .align 5
+
+hpfPteMiss: lwarx r0,0,r31 ; Load the mapping flag field
+ lwz r12,mpPte(r31) ; Get the quick pointer to PTE
+ li r3,mpHValid ; Get the PTE valid bit
+ andi. r2,r0,lo16(mpFIP) ; Are we handling a fault on the other side?
+ ori r2,r0,lo16(mpFIP) ; Set the fault in progress flag
+ crnot cr1_eq,cr0_eq ; Remember if FIP was on
+ and. r12,r12,r3 ; Isolate the valid bit
+ crorc cr0_eq,cr1_eq,cr0_eq ; Bail if FIP is on. Then, if already have PTE, bail...
+ beq-- hpfAbandon ; Yes, other processor is or already has handled this...
+ rlwinm r0,r2,0,mpType ; Isolate mapping type
+ cmplwi r0,mpBlock ; Is this a block mapping?
+ crnot cr7_eq,cr0_eq ; Remember if we have a block mapping
+ stwcx. r2,0,r31 ; Store the flags
+ bne-- hpfPteMiss ; Collision, try again...
+
+ bt++ pf64Bitb,hpfBldPTE64 ; Skip down to the 64 bit stuff...
+
+;
+; At this point we are about to do the 32-bit PTE generation.
+;
+; The following is the R14:R15 pair that contains the "shifted" VSID:
+;
+; 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVV////|////////|
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+;
+; The 24 bits of the 32-bit architecture VSID is in the following:
+;
+; 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; |////////|////////|////////|////VVVV|VVVVVVVV|VVVVVVVV|VVVV////|////////|
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+;
+
+
+hpfBldPTE32:
+ lwz r25,mpVAddr+4(r31) ; Grab the base virtual address for the mapping (32-bit portion)
+ lwz r24,mpPAddr(r31) ; Grab the base physical page number for the mapping
+
+ mfsdr1 r27 ; Get the hash table base address
+
+ rlwinm r0,r23,0,4,19 ; Isolate just the page index
+ rlwinm r18,r23,10,26,31 ; Extract the API
+ xor r19,r15,r0 ; Calculate hash << 12
+ mr r2,r25 ; Save the flag part of the mapping
+ rlwimi r18,r14,27,1,4 ; Move bits 28:31 of the "shifted" VSID into the PTE image
+ rlwinm r16,r27,16,7,15 ; Extract the hash table size
+ rlwinm r25,r25,0,0,19 ; Clear out the flags
+ slwi r24,r24,12 ; Change ppnum to physical address (note: 36-bit addressing no supported)
+ sub r25,r23,r25 ; Get offset in mapping to page (0 unless block map)
+ ori r16,r16,lo16(0xFFC0) ; Slap in the bottom of the mask
+ rlwinm r27,r27,0,0,15 ; Extract the hash table base
+ rlwinm r19,r19,26,6,25 ; Shift hash over to make offset into hash table
+ add r24,r24,r25 ; Adjust to true physical address
+ rlwimi r18,r15,27,5,24 ; Move bits 32:31 of the "shifted" VSID into the PTE image
+ rlwimi r24,r2,0,20,31 ; Slap in the WIMG and prot
+ and r19,r19,r16 ; Wrap hash table offset into the hash table
+ ori r24,r24,lo16(mpR) ; Turn on the reference bit right now
+ rlwinm r20,r19,28,10,29 ; Shift hash over to make offset into PCA
+ add r19,r19,r27 ; Point to the PTEG
+ subfic r20,r20,-4 ; Get negative offset to PCA
+ oris r18,r18,lo16(0x8000) ; Make sure the valid bit is on
+ add r20,r20,r27 ; Point to the PCA slot
+
+;
+; We now have a valid PTE pair in R18/R24. R18 is PTE upper and R24 is PTE lower.
+; R19 contains the offset of the PTEG in the hash table. R20 has offset into the PCA.
+;
+; We need to check PTE pointer (mpPte) again after we lock the PTEG. It is possible
+; that some other processor beat us and stuck in a PTE or that
+; all we had was a simple segment exception and the PTE was there the whole time.
+; If we find one a pointer, we are done.
+;
+
+ mr r7,r20 ; Copy the PCA pointer
+ bl mapLockPteg ; Lock the PTEG
+
+ lwz r12,mpPte(r31) ; Get the offset to the PTE
+ mr r17,r6 ; Remember the PCA image
+ mr r16,r6 ; Prime the post-select PCA image
+ andi. r0,r12,mpHValid ; Is there a PTE here already?
+ li r21,8 ; Get the number of slots
+
+ bne- cr7,hpfNoPte32 ; Skip this for a block mapping...
+
+ bne- hpfBailOut ; Someone already did this for us...
+
+;
+; The mapSelSlot function selects a PTEG slot to use. As input, it uses R6 as a
+; pointer to the PCA. When it returns, R3 contains 0 if an unoccupied slot was
+; selected, 1 if it stole a non-block PTE, or 2 if it stole a block mapped PTE.
+; R4 returns the slot index.
+;
+; REMEMBER: CR7 indicates that we are building a block mapping.
+;
+
+hpfNoPte32: subic. r21,r21,1 ; See if we have tried all slots
+ mr r6,r17 ; Get back the original PCA
+ rlwimi r6,r16,0,8,15 ; Insert the updated steal slot
+ blt- hpfBailOut ; Holy Cow, all slots are locked...
+
+ bl mapSelSlot ; Go select a slot (note that the PCA image is already set up)
+
+ cmplwi cr5,r3,1 ; Did we steal a slot?
+ rlwimi r19,r4,3,26,28 ; Insert PTE index into PTEG address yielding PTE address
+ mr r16,r6 ; Remember the PCA image after selection
+ blt+ cr5,hpfInser32 ; Nope, no steal...
+
+ lwz r6,0(r19) ; Get the old PTE
+ lwz r7,4(r19) ; Get the real part of the stealee
+ rlwinm r6,r6,0,1,31 ; Clear the valid bit
+ bgt cr5,hpfNipBM ; Do not try to lock a non-existant physent for a block mapping...
+ srwi r3,r7,12 ; Change phys address to a ppnum
+ bl mapFindPhyTry ; Go find and try to lock physent (note: if R3 is 0, there is no physent for this page)
+ cmplwi cr1,r3,0 ; Check if this is in RAM
+ bne- hpfNoPte32 ; Could not get it, try for another...
+
+ crmove cr5_gt,cr1_eq ; If we did not find a physent, pretend that this is a block map
+
+hpfNipBM: stw r6,0(r19) ; Set the invalid PTE
+
+ sync ; Make sure the invalid is stored
+ li r9,tlbieLock ; Get the TLBIE lock
+ rlwinm r10,r6,21,0,3 ; Shift last 4 bits of space to segment part
+
+hpfTLBIE32: lwarx r0,0,r9 ; Get the TLBIE lock
+ mfsprg r4,0 ; Get the per_proc
+ rlwinm r8,r6,25,18,31 ; Extract the space ID
+ rlwinm r11,r6,25,18,31 ; Extract the space ID
+ lwz r7,hwSteals(r4) ; Get the steal count
+ srwi r2,r6,7 ; Align segment number with hash
+ rlwimi r11,r11,14,4,17 ; Get copy above ourselves
+ mr. r0,r0 ; Is it locked?
+ srwi r0,r19,6 ; Align PTEG offset for back hash
+ xor r2,r2,r11 ; Get the segment number (plus a whole bunch of extra bits)
+ xor r11,r11,r0 ; Hash backwards to partial vaddr
+ rlwinm r12,r2,14,0,3 ; Shift segment up
+ mfsprg r2,2 ; Get feature flags
+ li r0,1 ; Get our lock word
+ rlwimi r12,r6,22,4,9 ; Move up the API
+ bne- hpfTLBIE32 ; It is locked, go wait...
+ rlwimi r12,r11,12,10,19 ; Move in the rest of the vaddr
+
+ stwcx. r0,0,r9 ; Try to get it
+ bne- hpfTLBIE32 ; We was beat...
+ addi r7,r7,1 ; Bump the steal count
+
+ rlwinm. r0,r2,0,pfSMPcapb,pfSMPcapb ; Can this be an MP box?
+ li r0,0 ; Lock clear value
+
+ tlbie r12 ; Invalidate it everywhere
+
+
+ beq- hpfNoTS32 ; Can not have MP on this machine...
+
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; Wait for everyone to catch up
+ sync ; Make sure of it all
+
+hpfNoTS32: stw r0,tlbieLock(0) ; Clear the tlbie lock
+
+ stw r7,hwSteals(r4) ; Save the steal count
+ bgt cr5,hpfInser32 ; We just stole a block mapping...
+
+ lwz r4,4(r19) ; Get the RC of the just invalidated PTE
+
+ la r11,ppLink+4(r3) ; Point to the master RC copy
+ lwz r7,ppLink+4(r3) ; Grab the pointer to the first mapping
+ rlwinm r2,r4,27,ppRb-32,ppCb-32 ; Position the new RC
+
+hpfMrgRC32: lwarx r0,0,r11 ; Get the master RC
+ or r0,r0,r2 ; Merge in the new RC
+ stwcx. r0,0,r11 ; Try to stick it back
+ bne- hpfMrgRC32 ; Try again if we collided...
+
+
+hpfFPnch: rlwinm. r7,r7,0,~ppFlags ; Clean and test mapping address
+ beq- hpfLostPhys ; We could not find our mapping. Kick the bucket...
+
+ lhz r10,mpSpace(r7) ; Get the space
+ lwz r9,mpVAddr+4(r7) ; And the vaddr
+ cmplw cr1,r10,r8 ; Is this one of ours?
+ xor r9,r12,r9 ; Compare virtual address
+ cmplwi r9,0x1000 ; See if we really match
+ crand cr0_eq,cr1_eq,cr0_lt ; See if both space and vaddr match
+ beq+ hpfFPnch2 ; Yes, found ours...
+
+ lwz r7,mpAlias+4(r7) ; Chain on to the next
+ b hpfFPnch ; Check it out...
+
+hpfFPnch2: sub r0,r19,r27 ; Get offset to the PTEG
+ stw r0,mpPte(r7) ; Invalidate the quick pointer (keep quick pointer pointing to PTEG)
+ bl mapPhysUnlock ; Unlock the physent now
+
+hpfInser32: oris r18,r18,lo16(0x8000) ; Make sure the valid bit is on
+
+ stw r24,4(r19) ; Stuff in the real part of the PTE
+ eieio ; Make sure this gets there first
+
+ stw r18,0(r19) ; Stuff the virtual part of the PTE and make it valid
+ mr r17,r16 ; Get the PCA image to save
+ b hpfFinish ; Go join the common exit code...
+
+
+;
+; At this point we are about to do the 64-bit PTE generation.
+;
+; The following is the R14:R15 pair that contains the "shifted" VSID:
+;
+; 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVV////|////////|
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+;
+;
+
+ .align 5
+
+hpfBldPTE64:
+ ld r10,mpVAddr(r31) ; Grab the base virtual address for the mapping
+ lwz r24,mpPAddr(r31) ; Grab the base physical page number for the mapping
+
+ mfsdr1 r27 ; Get the hash table base address
+
+ sldi r11,r22,32 ; Slide top of adjusted EA over
+ sldi r14,r14,32 ; Slide top of VSID over
+ rlwinm r5,r27,0,27,31 ; Isolate the size
+ eqv r16,r16,r16 ; Get all foxes here
+ rlwimi r15,r23,16,20,24 ; Stick in EA[36:40] to make AVPN
+ mr r2,r10 ; Save the flag part of the mapping
+ or r11,r11,r23 ; Stick in bottom of adjusted EA for full 64-bit value
+ rldicr r27,r27,0,45 ; Clean up the hash table base
+ or r15,r15,r14 ; Stick in bottom of AVPN for full 64-bit value
+ rlwinm r0,r11,0,4,19 ; Clear out everything but the page
+ subfic r5,r5,46 ; Get number of leading zeros
+ xor r19,r0,r15 ; Calculate hash
+ ori r15,r15,1 ; Turn on valid bit in AVPN to make top of PTE
+ srd r16,r16,r5 ; Shift over to get length of table
+ srdi r19,r19,5 ; Convert page offset to hash table offset
+ rldicr r16,r16,0,56 ; Clean up lower bits in hash table size
+ rldicr r10,r10,0,51 ; Clear out flags
+ sldi r24,r24,12 ; Change ppnum to physical address
+ sub r11,r11,r10 ; Get the offset from the base mapping
+ and r19,r19,r16 ; Wrap into hash table
+ add r24,r24,r11 ; Get actual physical address of this page
+ srdi r20,r19,5 ; Convert PTEG offset to PCA offset
+ rldimi r24,r2,0,52 ; Insert the keys, WIMG, RC, etc.
+ subfic r20,r20,-4 ; Get negative offset to PCA
+ ori r24,r24,lo16(mpR) ; Force on the reference bit
+ add r20,r20,r27 ; Point to the PCA slot
+ add r19,r19,r27 ; Point to the PTEG
+
+;
+; We now have a valid PTE pair in R15/R24. R15 is PTE upper and R24 is PTE lower.
+; R19 contains the offset of the PTEG in the hash table. R20 has offset into the PCA.
+;
+; We need to check PTE pointer (mpPte) again after we lock the PTEG. It is possible
+; that some other processor beat us and stuck in a PTE or that
+; all we had was a simple segment exception and the PTE was there the whole time.
+; If we find one a pointer, we are done.
+;
+
+ mr r7,r20 ; Copy the PCA pointer
+ bl mapLockPteg ; Lock the PTEG
+
+ lwz r12,mpPte(r31) ; Get the offset to the PTE
+ mr r17,r6 ; Remember the PCA image
+ mr r18,r6 ; Prime post-selection PCA image
+ andi. r0,r12,mpHValid ; See if we have a PTE now
+ li r21,8 ; Get the number of slots
+
+ bne-- cr7,hpfNoPte64 ; Skip this for a block mapping...
+
+ bne-- hpfBailOut ; Someone already did this for us...
+
+;
+; The mapSelSlot function selects a PTEG slot to use. As input, it uses R3 as a
+; pointer to the PCA. When it returns, R3 contains 0 if an unoccupied slot was
+; selected, 1 if it stole a non-block PTE, or 2 if it stole a block mapped PTE.
+; R4 returns the slot index.
+;
+; REMEMBER: CR7 indicates that we are building a block mapping.
+;
+
+hpfNoPte64: subic. r21,r21,1 ; See if we have tried all slots
+ mr r6,r17 ; Restore original state of PCA
+ rlwimi r6,r18,0,8,15 ; Insert the updated steal slot
+ blt- hpfBailOut ; Holy Cow, all slots are locked...
+
+ bl mapSelSlot ; Go select a slot
+
+ cmplwi cr5,r3,1 ; Did we steal a slot?
+ mr r18,r6 ; Remember the PCA image after selection
+ insrdi r19,r4,3,57 ; Insert slot index into PTEG address bits 57:59, forming the PTE address
+ lwz r10,hwSteals(r2) ; Get the steal count
+ blt++ cr5,hpfInser64 ; Nope, no steal...
+
+ ld r6,0(r19) ; Get the old PTE
+ ld r7,8(r19) ; Get the real part of the stealee
+ rldicr r6,r6,0,62 ; Clear the valid bit
+ bgt cr5,hpfNipBMx ; Do not try to lock a non-existant physent for a block mapping...
+ srdi r3,r7,12 ; Change page address to a page address
+ bl mapFindPhyTry ; Go find and try to lock physent (note: if R3 is 0, there is no physent for this page)
+ cmplwi cr1,r3,0 ; Check if this is in RAM
+ bne-- hpfNoPte64 ; Could not get it, try for another...
+
+ crmove cr5_gt,cr1_eq ; If we did not find a physent, pretend that this is a block map
+
+hpfNipBMx: std r6,0(r19) ; Set the invalid PTE
+ li r9,tlbieLock ; Get the TLBIE lock
+
+ srdi r11,r6,5 ; Shift VSID over for back hash
+ mfsprg r4,0 ; Get the per_proc
+ xor r11,r11,r19 ; Hash backwards to get low bits of VPN
+ sync ; Make sure the invalid is stored
+
+ sldi r12,r6,16 ; Move AVPN to EA position
+ sldi r11,r11,5 ; Move this to the page position
+
+hpfTLBIE64: lwarx r0,0,r9 ; Get the TLBIE lock
+ mr. r0,r0 ; Is it locked?
+ li r0,1 ; Get our lock word
+ bne-- hpfTLBIE65 ; It is locked, go wait...
+
+ stwcx. r0,0,r9 ; Try to get it
+ rldimi r12,r11,0,41 ; Stick the low part of the page number into the AVPN
+ rldicl r8,r6,52,50 ; Isolate the address space ID
+ bne-- hpfTLBIE64 ; We was beat...
+ addi r10,r10,1 ; Bump the steal count
+
+ rldicl r11,r12,0,16 ; Clear cause the book says so
+ li r0,0 ; Lock clear value
+
+ tlbie r11 ; Invalidate it everywhere
+
+ mr r7,r8 ; Get a copy of the space ID
+ eieio ; Make sure that the tlbie happens first
+ rldimi r7,r7,14,36 ; Copy address space to make hash value
+ tlbsync ; Wait for everyone to catch up
+ rldimi r7,r7,28,22 ; Add in a 3rd copy of the hash up top
+ srdi r2,r6,26 ; Shift original segment down to bottom
+
+ ptesync ; Make sure of it all
+ xor r7,r7,r2 ; Compute original segment
+ stw r0,tlbieLock(0) ; Clear the tlbie lock
+
+ stw r10,hwSteals(r4) ; Save the steal count
+ bgt cr5,hpfInser64 ; We just stole a block mapping...
+
+ rldimi r12,r7,28,0 ; Insert decoded segment
+ rldicl r4,r12,0,13 ; Trim to max supported address
+
+ ld r12,8(r19) ; Get the RC of the just invalidated PTE
+
+ la r11,ppLink+4(r3) ; Point to the master RC copy
+ ld r7,ppLink(r3) ; Grab the pointer to the first mapping
+ rlwinm r2,r12,27,ppRb-32,ppCb-32 ; Position the new RC
+
+hpfMrgRC64: lwarx r0,0,r11 ; Get the master RC
+ li r12,ppLFAmask ; Get mask to clean up alias pointer
+ or r0,r0,r2 ; Merge in the new RC
+ rotrdi r12,r12,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ stwcx. r0,0,r11 ; Try to stick it back
+ bne-- hpfMrgRC64 ; Try again if we collided...
+
+hpfFPnchx: andc. r7,r7,r12 ; Clean and test mapping address
+ beq-- hpfLostPhys ; We could not find our mapping. Kick the bucket...
+
+ lhz r10,mpSpace(r7) ; Get the space
+ ld r9,mpVAddr(r7) ; And the vaddr
+ cmplw cr1,r10,r8 ; Is this one of ours?
+ xor r9,r4,r9 ; Compare virtual address
+ cmpldi r9,0x1000 ; See if we really match
+ crand cr0_eq,cr1_eq,cr0_lt ; See if both space and vaddr match
+ beq++ hpfFPnch2x ; Yes, found ours...
+
+ ld r7,mpAlias(r7) ; Chain on to the next
+ b hpfFPnchx ; Check it out...
+
+ .align 5
+
+hpfTLBIE65: li r7,lgKillResv ; Point to the reservatio kill area
+ stwcx. r7,0,r7 ; Kill reservation
+
+hpfTLBIE63: lwz r0,0(r9) ; Get the TLBIE lock
+ mr. r0,r0 ; Is it locked?
+ beq++ hpfTLBIE64 ; Yup, wait for it...
+ b hpfTLBIE63 ; Nope, try again..
+
+
+
+hpfFPnch2x: sub r0,r19,r27 ; Get offset to PTEG
+ stw r0,mpPte(r7) ; Invalidate the quick pointer (keep pointing at PTEG though)
+ bl mapPhysUnlock ; Unlock the physent now
+
+
+hpfInser64: std r24,8(r19) ; Stuff in the real part of the PTE
+ eieio ; Make sure this gets there first
+ std r15,0(r19) ; Stuff the virtual part of the PTE and make it valid
+ mr r17,r18 ; Get the PCA image to set
+ b hpfFinish ; Go join the common exit code...
+
+hpfLostPhys:
+ lis r0,hi16(Choke) ; System abend - we must find the stolen mapping or we are dead
+ ori r0,r0,lo16(Choke) ; System abend
+ sc
+
+;
+; This is the common code we execute when we are finished setting up the PTE.
+;
+
+ .align 5
+
+hpfFinish: sub r4,r19,r27 ; Get offset of PTE
+ ori r4,r4,lo16(mpHValid) ; Add valid bit to PTE offset
+ bne cr7,hpfBailOut ; Do not set the PTE pointer for a block map
+ stw r4,mpPte(r31) ; Remember our PTE
+
+hpfBailOut: eieio ; Make sure all updates come first
+ stw r17,0(r20) ; Unlock and set the final PCA
+
+;
+; This is where we go if we have started processing the fault, but find that someone
+; else has taken care of it.
+;
+
+hpfIgnore: lwz r2,mpFlags(r31) ; Get the mapping flags
+ rlwinm r2,r2,0,mpFIPb+1,mpFIPb-1 ; Clear the "fault in progress" flag
+ sth r2,mpFlags+2(r31) ; Set it
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ li r11,T_IN_VAIN ; Say that it was handled
+ b EXT(PFSExit) ; Leave...
+
+;
+; This is where we go when we find that someone else
+; is in the process of handling the fault.
+;
+
+hpfAbandon: li r3,lgKillResv ; Kill off any reservation
+ stwcx. r3,0,r3 ; Do it
+
+ la r3,pmapSXlk(r28) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+
+ li r11,T_IN_VAIN ; Say that it was handled
+ b EXT(PFSExit) ; Leave...
+
+;
+; Guest shadow assist -- page fault handler
+;
+; Here we handle a fault in a guest pmap that has the guest shadow mapping
+; assist active. We locate the VMM pmap extension block, which contains an
+; index over the discontiguous multi-page shadow hash table. The index
+; corresponding to our vaddr is selected, and the selected group within
+; that page is searched for a valid and active entry that contains
+; our vaddr and space id. The search is pipelined, so that we may fetch
+; the next slot while examining the current slot for a hit. The final
+; search iteration is unrolled so that we don't fetch beyond the end of
+; our group, which could have dire consequences depending upon where the
+; physical hash page is located.
+;
+; The VMM pmap extension block occupies a page. Begining at offset 0, we
+; have the pmap_vmm_ext proper. Aligned at the first 128-byte boundary
+; after the pmap_vmm_ext is the hash table physical address index, a
+; linear list of 64-bit physical addresses of the pages that comprise
+; the hash table.
+;
+; In the event that we succesfully locate a guest mapping, we re-join
+; the page fault path at hpfGVfound with the mapping's address in r31;
+; otherwise, we re-join at hpfNotFound. In either case, we re-join holding
+; a share of the pmap search lock for the host pmap with the host pmap's
+; address in r28, the guest pmap's space id in r21, and the guest pmap's
+; flags in r12.
+;
+
+ .align 5
+hpfGVxlate:
+ bt pf64Bitb,hpfGV64 ; Take 64-bit path for 64-bit machine
+
+ lwz r11,pmapVmmExtPhys+4(r28) ; r11 <- VMM pmap extension block paddr
+ lwz r12,pmapFlags(r28) ; r12 <- guest pmap's flags
+ lwz r21,pmapSpace(r28) ; r21 <- guest space ID number
+ lwz r28,vmxHostPmapPhys+4(r11) ; r28 <- host pmap's paddr
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ rlwinm r10,r30,0,0xFFFFF000 ; r10 <- page-aligned guest vaddr
+ lwz r6,vxsGpf(r11) ; Get guest fault count
+
+ srwi r3,r10,12 ; Form shadow hash:
+ xor r3,r3,r21 ; spaceID ^ (vaddr >> 12)
+ rlwinm r4,r3,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r4 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r3,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+
+ la r3,pmapSXlk(r28) ; Point to the host pmap's search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne- hpfBadLock ; Nope...
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ addi r6,r6,1 ; Increment guest fault count
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ stw r6,vxsGpf(r11) ; Update guest fault count
+ b hpfGVlp32
+
+ .align 5
+hpfGVlp32:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r6,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r21 ; Compare space ID
+ or r0,r6,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r10 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq hpfGVfound ; Join common patch on hit (r31 points to mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz hpfGVlp32 ; Iterate
+
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r3,r3,mpgFree+mpgDormant ; Isolate guest free and dormant flag
+ xor r4,r4,r21 ; Compare space ID
+ or r0,r3,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r10 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq hpfGVfound ; Join common patch on hit (r31 points to mapping)
+
+ b hpfGVmiss
+
+ .align 5
+hpfGV64:
+ ld r11,pmapVmmExtPhys(r28) ; r11 <- VMM pmap extension block paddr
+ lwz r12,pmapFlags(r28) ; r12 <- guest pmap's flags
+ lwz r21,pmapSpace(r28) ; r21 <- guest space ID number
+ ld r28,vmxHostPmapPhys(r11) ; r28 <- host pmap's paddr
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ rlwinm r10,r30,0,0xFFFFF000 ; Form 64-bit guest vaddr
+ rldimi r10,r29,32,0 ; cleaning up low-order 12 bits
+ lwz r6,vxsGpf(r11) ; Get guest fault count
+
+ srwi r3,r10,12 ; Form shadow hash:
+ xor r3,r3,r21 ; spaceID ^ (vaddr >> 12)
+ rlwinm r4,r3,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r4 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r3,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+ la r3,pmapSXlk(r28) ; Point to the host pmap's search lock
+ bl sxlkShared ; Go get a shared lock on the mapping lists
+ mr. r3,r3 ; Did we get the lock?
+ bne-- hpfBadLock ; Nope...
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ addi r6,r6,1 ; Increment guest fault count
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ stw r6,vxsGpf(r11) ; Update guest fault count
+ b hpfGVlp64
+
+ .align 5
+hpfGVlp64:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r6,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flag
+ xor r7,r7,r21 ; Compare space ID
+ or r0,r6,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r10 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq hpfGVfound ; Join common path on hit (r31 points to mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz hpfGVlp64 ; Iterate
+
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r3,r3,mpgFree+mpgDormant ; Isolate guest free and dormant flag
+ xor r4,r4,r21 ; Compare space ID
+ or r0,r3,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r10 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq hpfGVfound ; Join common path on hit (r31 points to mapping)
+
+hpfGVmiss:
+ lwz r6,vxsGpfMiss(r11) ; Guest guest fault miss count
+ addi r6,r6,1 ; Increment miss count
+ stw r6,vxsGpfMiss(r11) ; Update guest fault miss count
+ b hpfNotFound
+
+/*
+ * hw_set_user_space(pmap)
+ * hw_set_user_space_dis(pmap)
+ *
+ * Indicate whether memory space needs to be switched.
+ * We really need to turn off interrupts here, because we need to be non-preemptable
+ *
+ * hw_set_user_space_dis is used when interruptions are already disabled. Mind the
+ * register usage here. The VMM switch code in vmachmon.s that calls this
+ * know what registers are in use. Check that if these change.
+ */
+
+
+
+ .align 5
+ .globl EXT(hw_set_user_space)
+
+LEXT(hw_set_user_space)
+
+ lis r8,hi16(MASK(MSR_VEC)) ; Get the vector enable
+ mfmsr r10 ; Get the current MSR
+ ori r8,r8,lo16(MASK(MSR_FP)) ; Add in FP
+ ori r9,r8,lo16(MASK(MSR_EE)) ; Add in the EE
+ andc r10,r10,r8 ; Turn off VEC, FP for good
+ andc r9,r10,r9 ; Turn off EE also
+ mtmsr r9 ; Disable them
+ isync ; Make sure FP and vec are off
+ mfsprg r6,1 ; Get the current activation
+ lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
+ lwz r2,ppUserPmapVirt(r6) ; Get our virtual pmap address
+ mfsprg r4,2 ; The the feature flags
+ lwz r7,pmapvr(r3) ; Get the v to r translation
+ lwz r8,pmapvr+4(r3) ; Get the v to r translation
+ mtcrf 0x80,r4 ; Get the Altivec flag
+ xor r4,r3,r8 ; Get bottom of the real address of bmap anchor
+ cmplw cr1,r3,r2 ; Same address space as before?
+ stw r7,ppUserPmap(r6) ; Show our real pmap address
+ crorc cr1_eq,cr1_eq,pfAltivecb ; See if same address space or not altivec machine
+ stw r4,ppUserPmap+4(r6) ; Show our real pmap address
+ stw r3,ppUserPmapVirt(r6) ; Show our virtual pmap address
+ mtmsr r10 ; Restore interruptions
+ beqlr-- cr1 ; Leave if the same address space or not Altivec
+
+ dssall ; Need to kill all data streams if adrsp changed
+ sync
+ blr ; Return...
+
+ .align 5
+ .globl EXT(hw_set_user_space_dis)
+
+LEXT(hw_set_user_space_dis)
+
+ lwz r7,pmapvr(r3) ; Get the v to r translation
+ mfsprg r4,2 ; The the feature flags
+ lwz r8,pmapvr+4(r3) ; Get the v to r translation
+ mfsprg r6,1 ; Get the current activation
+ lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
+ lwz r2,ppUserPmapVirt(r6) ; Get our virtual pmap address
+ mtcrf 0x80,r4 ; Get the Altivec flag
+ xor r4,r3,r8 ; Get bottom of the real address of bmap anchor
+ cmplw cr1,r3,r2 ; Same address space as before?
+ stw r7,ppUserPmap(r6) ; Show our real pmap address
+ crorc cr1_eq,cr1_eq,pfAltivecb ; See if same address space or not altivec machine
+ stw r4,ppUserPmap+4(r6) ; Show our real pmap address
+ stw r3,ppUserPmapVirt(r6) ; Show our virtual pmap address
+ beqlr-- cr1 ; Leave if the same
+
+ dssall ; Need to kill all data streams if adrsp changed
+ sync
+ blr ; Return...
+
+/* int mapalc1(struct mappingblok *mb) - Finds, allocates, and zeros a free 1-bit mapping entry
+ *
+ * Lock must already be held on mapping block list
* returns 0 if all slots filled.
* returns n if a slot is found and it is not the last
- * returns -n if a slot os found and it is the last
+ * returns -n if a slot is found and it is the last
* when n and -n are returned, the corresponding bit is cleared
+ * the mapping is zeroed out before return
+ *
+ */
+
+ .align 5
+ .globl EXT(mapalc1)
+
+LEXT(mapalc1)
+ lwz r4,mbfree(r3) ; Get the 1st mask
+ lis r0,0x8000 ; Get the mask to clear the first free bit
+ lwz r5,mbfree+4(r3) ; Get the 2nd mask
+ mr r12,r3 ; Save the block ptr
+ cntlzw r3,r4 ; Get first 1-bit in 1st word
+ srw. r9,r0,r3 ; Get bit corresponding to first free one
+ cntlzw r10,r5 ; Get first free field in second word
+ andc r4,r4,r9 ; Turn 1-bit off in 1st word
+ bne mapalc1f ; Found one in 1st word
+
+ srw. r9,r0,r10 ; Get bit corresponding to first free one in 2nd word
+ li r3,0 ; assume failure return
+ andc r5,r5,r9 ; Turn it off
+ beqlr-- ; There are no 1 bits left...
+ addi r3,r10,32 ; set the correct number
+
+mapalc1f:
+ or. r0,r4,r5 ; any more bits set?
+ stw r4,mbfree(r12) ; update bitmasks
+ stw r5,mbfree+4(r12)
+
+ slwi r6,r3,6 ; get (n * mpBasicSize), ie offset of mapping in block
+ addi r7,r6,32
+ dcbz r6,r12 ; clear the 64-byte mapping
+ dcbz r7,r12
+
+ bnelr++ ; return if another bit remains set
+
+ neg r3,r3 ; indicate we just returned the last bit
+ blr
+
+
+/* int mapalc2(struct mappingblok *mb) - Finds, allocates, and zero's a free 2-bit mapping entry
*
+ * Lock must already be held on mapping block list
+ * returns 0 if all slots filled.
+ * returns n if a slot is found and it is not the last
+ * returns -n if a slot is found and it is the last
+ * when n and -n are returned, the corresponding bits are cleared
+ * We find runs of 2 consecutive 1 bits by cntlzw(n & (n<<1)).
+ * the mapping is zero'd out before return
*/
.align 5
- .globl EXT(mapalc)
+ .globl EXT(mapalc2)
+LEXT(mapalc2)
+ lwz r4,mbfree(r3) ; Get the first mask
+ lis r0,0x8000 ; Get the mask to clear the first free bit
+ lwz r5,mbfree+4(r3) ; Get the second mask
+ mr r12,r3 ; Save the block ptr
+ slwi r6,r4,1 ; shift first word over
+ and r6,r4,r6 ; lite start of double bit runs in 1st word
+ slwi r7,r5,1 ; shift 2nd word over
+ cntlzw r3,r6 ; Get first free 2-bit run in 1st word
+ and r7,r5,r7 ; lite start of double bit runs in 2nd word
+ srw. r9,r0,r3 ; Get bit corresponding to first run in 1st word
+ cntlzw r10,r7 ; Get first free field in second word
+ srwi r11,r9,1 ; shift over for 2nd bit in 1st word
+ andc r4,r4,r9 ; Turn off 1st bit in 1st word
+ andc r4,r4,r11 ; turn off 2nd bit in 1st word
+ bne mapalc2a ; Found two consecutive free bits in 1st word
+
+ srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
+ li r3,0 ; assume failure
+ srwi r11,r9,1 ; get mask for 2nd bit
+ andc r5,r5,r9 ; Turn off 1st bit in 2nd word
+ andc r5,r5,r11 ; turn off 2nd bit in 2nd word
+ beq-- mapalc2c ; There are no runs of 2 bits in 2nd word either
+ addi r3,r10,32 ; set the correct number
+
+mapalc2a:
+ or. r0,r4,r5 ; any more bits set?
+ stw r4,mbfree(r12) ; update bitmasks
+ stw r5,mbfree+4(r12)
+ slwi r6,r3,6 ; get (n * mpBasicSize), ie offset of mapping in block
+ addi r7,r6,32
+ addi r8,r6,64
+ addi r9,r6,96
+ dcbz r6,r12 ; zero out the 128-byte mapping
+ dcbz r7,r12 ; we use the slow 32-byte dcbz even on 64-bit machines
+ dcbz r8,r12 ; because the mapping may not be 128-byte aligned
+ dcbz r9,r12
+
+ bnelr++ ; return if another bit remains set
+
+ neg r3,r3 ; indicate we just returned the last bit
+ blr
+
+mapalc2c:
+ rlwinm r7,r5,1,31,31 ; move bit 0 of 2nd word to bit 31
+ and. r0,r4,r7 ; is the 2-bit field that spans the 2 words free?
+ beqlr ; no, we failed
+ rlwinm r4,r4,0,0,30 ; yes, turn off bit 31 of 1st word
+ rlwinm r5,r5,0,1,31 ; turn off bit 0 of 2nd word
+ li r3,31 ; get index of this field
+ b mapalc2a
+
+
+;
+; This routine initialzes the hash table and PCA.
+; It is done here because we may need to be 64-bit to do it.
+;
+
+ .align 5
+ .globl EXT(hw_hash_init)
+
+LEXT(hw_hash_init)
+
+ mfsprg r10,2 ; Get feature flags
+ lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
+ mtcrf 0x02,r10 ; move pf64Bit to cr6
+ lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
+ lis r4,0xFF01 ; Set all slots free and start steal at end
+ ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
+ ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
+
+ lwz r12,0(r12) ; Get hash table size
+ li r3,0 ; Get start
+ bt++ pf64Bitb,hhiSF ; skip if 64-bit (only they take the hint)
+
+ lwz r11,4(r11) ; Get hash table base
+
+hhiNext32: cmplw r3,r12 ; Have we reached the end?
+ bge- hhiCPCA32 ; Yes...
+ dcbz r3,r11 ; Clear the line
+ addi r3,r3,32 ; Next one...
+ b hhiNext32 ; Go on...
+
+hhiCPCA32: rlwinm r12,r12,28,4,29 ; Get number of slots * 4
+ li r3,-4 ; Displacement to first PCA entry
+ neg r12,r12 ; Get negative end of PCA
+
+hhiNPCA32: stwx r4,r3,r11 ; Initialize the PCA entry
+ subi r3,r3,4 ; Next slot
+ cmpw r3,r12 ; Have we finished?
+ bge+ hhiNPCA32 ; Not yet...
+ blr ; Leave...
+
+hhiSF: mfmsr r9 ; Save the MSR
+ li r8,1 ; Get a 1
+ mr r0,r9 ; Get a copy of the MSR
+ ld r11,0(r11) ; Get hash table base
+ rldimi r0,r8,63,MSR_SF_BIT ; Set SF bit (bit 0)
+ mtmsrd r0 ; Turn on SF
+ isync
+
+
+hhiNext64: cmpld r3,r12 ; Have we reached the end?
+ bge-- hhiCPCA64 ; Yes...
+ dcbz128 r3,r11 ; Clear the line
+ addi r3,r3,128 ; Next one...
+ b hhiNext64 ; Go on...
+
+hhiCPCA64: rlwinm r12,r12,27,5,29 ; Get number of slots * 4
+ li r3,-4 ; Displacement to first PCA entry
+ neg r12,r12 ; Get negative end of PCA
+
+hhiNPCA64: stwx r4,r3,r11 ; Initialize the PCA entry
+ subi r3,r3,4 ; Next slot
+ cmpd r3,r12 ; Have we finished?
+ bge++ hhiNPCA64 ; Not yet...
+
+ mtmsrd r9 ; Turn off SF if it was off
+ isync
+ blr ; Leave...
+
+
+;
+; This routine sets up the hardware to start translation.
+; Note that we do NOT start translation.
+;
+
+ .align 5
+ .globl EXT(hw_setup_trans)
+
+LEXT(hw_setup_trans)
+
+ mfsprg r11,0 ; Get the per_proc block
+ mfsprg r12,2 ; Get feature flags
+ li r0,0 ; Get a 0
+ li r2,1 ; And a 1
+ mtcrf 0x02,r12 ; Move pf64Bit to cr6
+ stw r0,validSegs(r11) ; Make sure we think all SR/STEs are invalid
+ stw r0,validSegs+4(r11) ; Make sure we think all SR/STEs are invalid, part deux
+ sth r2,ppInvSeg(r11) ; Force a reload of the SRs
+ sth r0,ppCurSeg(r11) ; Set that we are starting out in kernel
+
+ bt++ pf64Bitb,hstSF ; skip if 64-bit (only they take the hint)
+
+ li r9,0 ; Clear out a register
+ sync
+ isync
+ mtdbatu 0,r9 ; Invalidate maps
+ mtdbatl 0,r9 ; Invalidate maps
+ mtdbatu 1,r9 ; Invalidate maps
+ mtdbatl 1,r9 ; Invalidate maps
+ mtdbatu 2,r9 ; Invalidate maps
+ mtdbatl 2,r9 ; Invalidate maps
+ mtdbatu 3,r9 ; Invalidate maps
+ mtdbatl 3,r9 ; Invalidate maps
+
+ mtibatu 0,r9 ; Invalidate maps
+ mtibatl 0,r9 ; Invalidate maps
+ mtibatu 1,r9 ; Invalidate maps
+ mtibatl 1,r9 ; Invalidate maps
+ mtibatu 2,r9 ; Invalidate maps
+ mtibatl 2,r9 ; Invalidate maps
+ mtibatu 3,r9 ; Invalidate maps
+ mtibatl 3,r9 ; Invalidate maps
+
+ lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
+ lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
+ ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
+ ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
+ lwz r11,4(r11) ; Get hash table base
+ lwz r12,0(r12) ; Get hash table size
+ subi r12,r12,1 ; Back off by 1
+ rlwimi r11,r12,16,23,31 ; Stick the size into the sdr1 image
+
+ mtsdr1 r11 ; Ok, we now have the hash table set up
+ sync
+
+ li r12,invalSpace ; Get the invalid segment value
+ li r10,0 ; Start low
+
+hstsetsr: mtsrin r12,r10 ; Set the SR
+ addis r10,r10,0x1000 ; Bump the segment
+ mr. r10,r10 ; Are we finished?
+ bne+ hstsetsr ; Nope...
+ sync
+ blr ; Return...
+
+;
+; 64-bit version
+;
+
+hstSF: lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
+ lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
+ ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
+ ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
+ ld r11,0(r11) ; Get hash table base
+ lwz r12,0(r12) ; Get hash table size
+ cntlzw r10,r12 ; Get the number of bits
+ subfic r10,r10,13 ; Get the extra bits we need
+ or r11,r11,r10 ; Add the size field to SDR1
+
+ mtsdr1 r11 ; Ok, we now have the hash table set up
+ sync
+
+ li r0,0 ; Set an SLB slot index of 0
+ slbia ; Trash all SLB entries (except for entry 0 that is)
+ slbmfee r7,r0 ; Get the entry that is in SLB index 0
+ rldicr r7,r7,0,35 ; Clear the valid bit and the rest
+ slbie r7 ; Invalidate it
+
+ blr ; Return...
+
+
+;
+; This routine turns on translation for the first time on a processor
+;
+
+ .align 5
+ .globl EXT(hw_start_trans)
+
+LEXT(hw_start_trans)
+
+
+ mfmsr r10 ; Get the msr
+ ori r10,r10,lo16(MASK(MSR_IR) | MASK(MSR_DR)) ; Turn on translation
+
+ mtmsr r10 ; Everything falls apart here
+ isync
+
+ blr ; Back to it.
+
+
+
+;
+; This routine validates a segment register.
+; hw_map_seg(pmap_t pmap, addr64_t seg, addr64_t va)
+;
+; r3 = virtual pmap
+; r4 = segment[0:31]
+; r5 = segment[32:63]
+; r6 = va[0:31]
+; r7 = va[32:63]
+;
+; Note that we transform the addr64_t (long long) parameters into single 64-bit values.
+; Note that there is no reason to apply the key modifier here because this is only
+; used for kernel accesses.
+;
+
+ .align 5
+ .globl EXT(hw_map_seg)
+
+LEXT(hw_map_seg)
+
+ lwz r0,pmapSpace(r3) ; Get the space, we will need it soon
+ lwz r9,pmapFlags(r3) ; Get the flags for the keys now
+ mfsprg r10,2 ; Get feature flags
+
+;
+; Note: the following code would problably be easier to follow if I split it,
+; but I just wanted to see if I could write this to work on both 32- and 64-bit
+; machines combined.
+;
+
+;
+; Here we enter with va[0:31] in r6[0:31] (or r6[32:63] on 64-bit machines)
+; and va[32:63] in r7[0:31] (or r7[32:63] on 64-bit machines)
+
+ rlwinm r4,r4,0,1,0 ; Copy seg[0:31] into r4[0;31] - no-op for 32-bit
+ rlwinm r7,r7,18,14,17 ; Slide va[32:35] east to just west of space ID
+ mtcrf 0x02,r10 ; Move pf64Bit and pfNoMSRirb to cr5 and 6
+ srwi r8,r6,14 ; Slide va[0:17] east to just west of the rest
+ rlwimi r7,r6,18,0,13 ; Slide va[18:31] east to just west of slid va[32:25]
+ rlwimi r0,r0,14,4,17 ; Dup address space ID above itself
+ rlwinm r8,r8,0,1,0 ; Dup low part into high (does nothing on 32-bit machines)
+ rlwinm r2,r0,28,0,31 ; Rotate rotate low nybble to top of low half
+ rlwimi r2,r2,0,1,0 ; Replicate bottom 32 into top 32
+ rlwimi r8,r7,0,0,31 ; Join va[0:17] with va[18:35] (just like mr on 32-bit machines)
+
+ rlwimi r2,r0,0,4,31 ; We should now have 4 copies of the space
+ ; concatenated together. There is garbage
+ ; at the top for 64-bit but we will clean
+ ; that out later.
+ rlwimi r4,r5,0,0,31 ; Copy seg[32:63] into r4[32:63] - just like mr for 32-bit
+
+
+;
+; Here we exit with va[0:35] shifted into r8[14:51], zeros elsewhere, or
+; va[18:35] shifted into r8[0:17], zeros elsewhere on 32-bit machines
+;
+
+;
+; What we have now is:
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; r2 = |xxxx0000|AAAAAAAA|AAAAAABB|BBBBBBBB|BBBBCCCC|CCCCCCCC|CCDDDDDD|DDDDDDDD| - hash value
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; 0 0 1 2 3 - for 32-bit machines
+; 0 8 6 4 1
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; r8 = |00000000|000000SS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SS000000|00000000| - shifted and cleaned EA
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; 0 0 1 2 3 - for 32-bit machines
+; 0 8 6 4 1
+;
+; 0 0 1 2 3 4 4 5 6
+; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; r4 = |SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSS0000|00000000|00000000|00000000| - Segment
+; +--------+--------+--------+--------+--------+--------+--------+--------+
+; 0 0 1 2 3 - for 32-bit machines
+; 0 8 6 4 1
+
+
+ xor r8,r8,r2 ; Calculate VSID
+
+ bf-- pf64Bitb,hms32bit ; Skip out if 32-bit...
+ mfsprg r12,0 ; Get the per_proc
+ li r0,1 ; Prepare to set bit 0 (also to clear EE)
+ mfmsr r6 ; Get current MSR
+ li r2,MASK(MSR_IR)|MASK(MSR_DR) ; Get the translation bits
+ mtmsrd r0,1 ; Set only the EE bit to 0
+ rlwinm r6,r6,0,MSR_EE_BIT,MSR_EE_BIT ; See if EE bit is on
+ mfmsr r11 ; Get the MSR right now, after disabling EE
+ andc r2,r11,r2 ; Turn off translation now
+ rldimi r2,r0,63,0 ; Get bit 64-bit turned on
+ or r11,r11,r6 ; Turn on the EE bit if it was on
+ mtmsrd r2 ; Make sure translation and EE are off and 64-bit is on
+ isync ; Hang out a bit
+
+ ld r6,validSegs(r12) ; Get the valid SLB entry flags
+ sldi r9,r9,9 ; Position the key and noex bit
+
+ rldimi r5,r8,12,0 ; Form the VSID/key
+
+ not r3,r6 ; Make valids be 0s
+
+ cntlzd r7,r3 ; Find a free SLB
+ cmplwi r7,63 ; Did we find a free SLB entry?
+
+ slbie r4 ; Since this ESID may still be in an SLBE, kill it
+
+ oris r4,r4,0x0800 ; Turn on the valid bit in ESID
+ addi r7,r7,1 ; Make sure we skip slb 0
+ blt++ hmsFreeSeg ; Yes, go load it...
+
+;
+; No free SLB entries, select one that is in use and invalidate it
+;
+ lwz r2,ppSegSteal(r12) ; Get the next slot to steal
+ addi r7,r2,pmapSegCacheUse+1 ; Select stealee from non-cached slots only
+ addi r2,r2,1 ; Set next slot to steal
+ slbmfee r3,r7 ; Get the entry that is in the selected spot
+ subi r8,r2,64-(pmapSegCacheUse+1) ; Force steal to wrap
+ rldicr r3,r3,0,35 ; Clear the valid bit and the rest
+ srawi r8,r8,31 ; Get -1 if steal index still in range
+ slbie r3 ; Invalidate the in-use SLB entry
+ and r2,r2,r8 ; Reset steal index when it should wrap
+ isync ;
+
+ stw r2,ppSegSteal(r12) ; Set the next slot to steal
+;
+; We are now ready to stick the SLB entry in the SLB and mark it in use
+;
+
+hmsFreeSeg: subi r2,r7,1 ; Adjust for skipped slb 0
+ rldimi r4,r7,0,58 ; Copy in the SLB entry selector
+ srd r0,r0,r2 ; Set bit mask for allocation
+ rldicl r5,r5,0,15 ; Clean out the unsupported bits
+ or r6,r6,r0 ; Turn on the allocation flag
+
+ slbmte r5,r4 ; Make that SLB entry
+
+ std r6,validSegs(r12) ; Mark as valid
+ mtmsrd r11 ; Restore the MSR
+ isync
+ blr ; Back to it...
+
+ .align 5
+
+hms32bit:
+ mfsprg r12,1 ; Get the current activation
+ lwz r12,ACT_PER_PROC(r12) ; Get the per_proc block
+ rlwinm r8,r8,0,8,31 ; Clean up the VSID
+ rlwinm r2,r4,4,28,31 ; Isolate the segment we are setting
+ lis r0,0x8000 ; Set bit 0
+ rlwimi r8,r9,28,1,3 ; Insert the keys and N bit
+ srw r0,r0,r2 ; Get bit corresponding to SR
+ addi r7,r12,validSegs ; Point to the valid segment flags directly
+
+ mtsrin r8,r4 ; Set the actual SR
+ isync ; Need to make sure this is done
+
+hmsrupt: lwarx r6,0,r7 ; Get and reserve the valid segment flags
+ or r6,r6,r0 ; Show that SR is valid
+ stwcx. r6,0,r7 ; Set the valid SR flags
+ bne-- hmsrupt ; Had an interrupt, need to get flags again...
+
+ blr ; Back to it...
+
+
+;
+; This routine invalidates a segment register.
+;
+
+ .align 5
+ .globl EXT(hw_blow_seg)
+
+LEXT(hw_blow_seg)
+
+ mfsprg r10,2 ; Get feature flags
+ mtcrf 0x02,r10 ; move pf64Bit and pfNoMSRirb to cr5 and 6
+
+ rlwinm r9,r4,0,0,3 ; Save low segment address and make sure it is clean
+
+ bf-- pf64Bitb,hbs32bit ; Skip out if 32-bit...
+
+ li r0,1 ; Prepare to set bit 0 (also to clear EE)
+ mfmsr r6 ; Get current MSR
+ li r2,MASK(MSR_IR)|MASK(MSR_DR) ; Get the translation bits
+ mtmsrd r0,1 ; Set only the EE bit to 0
+ rlwinm r6,r6,0,MSR_EE_BIT,MSR_EE_BIT ; See if EE bit is on
+ mfmsr r11 ; Get the MSR right now, after disabling EE
+ andc r2,r11,r2 ; Turn off translation now
+ rldimi r2,r0,63,0 ; Get bit 64-bit turned on
+ or r11,r11,r6 ; Turn on the EE bit if it was on
+ mtmsrd r2 ; Make sure translation and EE are off and 64-bit is on
+ isync ; Hang out a bit
+
+ rldimi r9,r3,32,0 ; Insert the top part of the ESID
+
+ slbie r9 ; Invalidate the associated SLB entry
+
+ mtmsrd r11 ; Restore the MSR
+ isync
+ blr ; Back to it.
+
+ .align 5
+
+hbs32bit:
+ mfsprg r12,1 ; Get the current activation
+ lwz r12,ACT_PER_PROC(r12) ; Get the per_proc block
+ addi r7,r12,validSegs ; Point to the valid segment flags directly
+ lwarx r4,0,r7 ; Get and reserve the valid segment flags
+ rlwinm r6,r9,4,28,31 ; Convert segment to number
+ lis r2,0x8000 ; Set up a mask
+ srw r2,r2,r6 ; Make a mask
+ and. r0,r4,r2 ; See if this is even valid
+ li r5,invalSpace ; Set the invalid address space VSID
+ beqlr ; Leave if already invalid...
+
+ mtsrin r5,r9 ; Slam the segment register
+ isync ; Need to make sure this is done
+
+hbsrupt: andc r4,r4,r2 ; Clear the valid bit for this segment
+ stwcx. r4,0,r7 ; Set the valid SR flags
+ beqlr++ ; Stored ok, no interrupt, time to leave...
+
+ lwarx r4,0,r7 ; Get and reserve the valid segment flags again
+ b hbsrupt ; Try again...
+
+;
+; This routine invadates the entire pmap segment cache
+;
+; Translation is on, interrupts may or may not be enabled.
+;
+
+ .align 5
+ .globl EXT(invalidateSegs)
+
+LEXT(invalidateSegs)
+
+ la r10,pmapCCtl(r3) ; Point to the segment cache control
+ eqv r2,r2,r2 ; Get all foxes
+
+isInv: lwarx r4,0,r10 ; Get the segment cache control value
+ rlwimi r4,r2,0,0,15 ; Slam in all invalid bits
+ rlwinm. r0,r4,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
+ bne-- isInv0 ; Yes, try again...
+
+ stwcx. r4,0,r10 ; Try to invalidate it
+ bne-- isInv ; Someone else just stuffed it...
+ blr ; Leave...
+
+
+isInv0: li r4,lgKillResv ; Get reservation kill zone
+ stwcx. r4,0,r4 ; Kill reservation
+
+isInv1: lwz r4,pmapCCtl(r3) ; Get the segment cache control
+ rlwinm. r0,r4,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
+ bne-- isInv ; Nope...
+ b isInv1 ; Still locked do it again...
+
+;
+; This routine switches segment registers between kernel and user.
+; We have some assumptions and rules:
+; We are in the exception vectors
+; pf64Bitb is set up
+; R3 contains the MSR we going to
+; We can not use R4, R13, R20, R21, R25, R26, R29
+; R13 is the savearea
+; R29 has the per_proc
+;
+; We return R3 as 0 if we did not switch between kernel and user
+; We also maintain and apply the user state key modifier used by VMM support;
+; If we go to the kernel it is set to 0, otherwise it follows the bit
+; in spcFlags.
+;
+
+ .align 5
+ .globl EXT(switchSegs)
+
+LEXT(switchSegs)
+
+ lwz r22,ppInvSeg(r29) ; Get the ppInvSeg (force invalidate) and ppCurSeg (user or kernel segments indicator)
+ lwz r9,spcFlags(r29) ; Pick up the special user state flags
+ rlwinm r2,r3,MSR_PR_BIT+1,31,31 ; Isolate the problem mode bit
+ rlwinm r3,r3,MSR_RI_BIT+1,31,31 ; Isolate the recoverable interrupt bit
+ lis r8,hi16(EXT(kernel_pmap_phys)) ; Assume kernel
+ or r2,r2,r3 ; This will 1 if we will be using user segments
+ li r3,0 ; Get a selection mask
+ cmplw r2,r22 ; This will be EQ if same state and not ppInvSeg
+ ori r8,r8,lo16(EXT(kernel_pmap_phys)) ; Assume kernel (bottom of address)
+ sub r3,r3,r2 ; Form select mask - 0 if kernel, -1 if user
+ la r19,ppUserPmap(r29) ; Point to the current user pmap
+
+; The following line is an exercise of a generally unreadable but recompile-friendly programing practice
+ rlwinm r30,r9,userProtKeybit+1+(63-sgcVSKeyUsr),sgcVSKeyUsr-32,sgcVSKeyUsr-32 ; Isolate the user state protection key
+
+ andc r8,r8,r3 ; Zero kernel pmap ptr if user, untouched otherwise
+ and r19,r19,r3 ; Zero user pmap ptr if kernel, untouched otherwise
+ and r30,r30,r3 ; Clear key modifier if kernel, leave otherwise
+ or r8,r8,r19 ; Get the pointer to the pmap we are using
+
+ beqlr ; We are staying in the same mode, do not touch segs...
+
+ lwz r28,0(r8) ; Get top half of pmap address
+ lwz r10,4(r8) ; Get bottom half
+
+ stw r2,ppInvSeg(r29) ; Clear request for invalidate and save ppCurSeg
+ rlwinm r28,r28,0,1,0 ; Copy top to top
+ stw r30,ppMapFlags(r29) ; Set the key modifier
+ rlwimi r28,r10,0,0,31 ; Insert bottom
+
+ la r10,pmapCCtl(r28) ; Point to the segment cache control
+ la r9,pmapSegCache(r28) ; Point to the segment cache
+
+ssgLock: lwarx r15,0,r10 ; Get and reserve the segment cache control
+ rlwinm. r0,r15,0,pmapCCtlLckb,pmapCCtlLckb ; Someone have the lock?
+ ori r16,r15,lo16(pmapCCtlLck) ; Set lock bit
+ bne-- ssgLock0 ; Yup, this is in use...
+
+ stwcx. r16,0,r10 ; Try to set the lock
+ bne-- ssgLock ; Did we get contention?
+
+ not r11,r15 ; Invert the invalids to valids
+ li r17,0 ; Set a mask for the SRs we are loading
+ isync ; Make sure we are all caught up
+
+ bf-- pf64Bitb,ssg32Enter ; If 32-bit, jump into it...
+
+ li r0,0 ; Clear
+ slbia ; Trash all SLB entries (except for entry 0 that is)
+ li r17,1 ; Get SLB index to load (skip slb 0)
+ oris r0,r0,0x8000 ; Get set for a mask
+ b ssg64Enter ; Start on a cache line...
+
+ .align 5
+
+ssgLock0: li r15,lgKillResv ; Killing field
+ stwcx. r15,0,r15 ; Kill reservation
+
+ssgLock1: lwz r15,pmapCCtl(r28) ; Get the segment cache controls
+ rlwinm. r15,r15,0,pmapCCtlLckb,pmapCCtlLckb ; Someone have the lock?
+ beq++ ssgLock ; Yup, this is in use...
+ b ssgLock1 ; Nope, try again...
+;
+; This is the 32-bit address space switch code.
+; We take a reservation on the segment cache and walk through.
+; For each entry, we load the specified entries and remember which
+; we did with a mask. Then, we figure out which segments should be
+; invalid and then see which actually are. Then we load those with the
+; defined invalid VSID.
+; Afterwards, we unlock the segment cache.
+;
+
+ .align 5
+
+ssg32Enter: cntlzw r12,r11 ; Find the next slot in use
+ cmplwi r12,pmapSegCacheUse ; See if we are done
+ slwi r14,r12,4 ; Index to the cache slot
+ lis r0,0x8000 ; Get set for a mask
+ add r14,r14,r9 ; Point to the entry
+
+ bge- ssg32Done ; All done...
+
+ lwz r5,sgcESID+4(r14) ; Get the ESID part
+ srw r2,r0,r12 ; Form a mask for the one we are loading
+ lwz r7,sgcVSID+4(r14) ; And get the VSID bottom
+
+ andc r11,r11,r2 ; Clear the bit
+ lwz r6,sgcVSID(r14) ; And get the VSID top
+
+ rlwinm r2,r5,4,28,31 ; Change the segment number to a number
+
+ xor r7,r7,r30 ; Modify the key before we actually set it
+ srw r0,r0,r2 ; Get a mask for the SR we are loading
+ rlwinm r8,r7,19,1,3 ; Insert the keys and N bit
+ or r17,r17,r0 ; Remember the segment
+ rlwimi r8,r7,20,12,31 ; Insert 4:23 the VSID
+ rlwimi r8,r6,20,8,11 ; Get the last nybble of the SR contents
+
+ mtsrin r8,r5 ; Load the segment
+ b ssg32Enter ; Go enter the next...
+
+ .align 5
+
+ssg32Done: lwz r16,validSegs(r29) ; Get the valid SRs flags
+ stw r15,pmapCCtl(r28) ; Unlock the segment cache controls
+
+ lis r0,0x8000 ; Get set for a mask
+ li r2,invalSpace ; Set the invalid address space VSID
+
+ nop ; Align loop
+ nop ; Align loop
+ andc r16,r16,r17 ; Get list of SRs that were valid before but not now
+ nop ; Align loop
+
+ssg32Inval: cntlzw r18,r16 ; Get the first one to invalidate
+ cmplwi r18,16 ; Have we finished?
+ srw r22,r0,r18 ; Get the mask bit
+ rlwinm r23,r18,28,0,3 ; Get the segment register we need
+ andc r16,r16,r22 ; Get rid of the guy we just did
+ bge ssg32Really ; Yes, we are really done now...
+
+ mtsrin r2,r23 ; Invalidate the SR
+ b ssg32Inval ; Do the next...
+
+ .align 5
+
+ssg32Really:
+ stw r17,validSegs(r29) ; Set the valid SR flags
+ li r3,1 ; Set kernel/user transition
+ blr
+
+;
+; This is the 64-bit address space switch code.
+; First we blow away all of the SLB entries.
+; Walk through,
+; loading the SLB. Afterwards, we release the cache lock
+;
+; Note that because we have to treat SLBE 0 specially, we do not ever use it...
+; Its a performance thing...
+;
+
+ .align 5
+
+ssg64Enter: cntlzw r12,r11 ; Find the next slot in use
+ cmplwi r12,pmapSegCacheUse ; See if we are done
+ slwi r14,r12,4 ; Index to the cache slot
+ srw r16,r0,r12 ; Form a mask for the one we are loading
+ add r14,r14,r9 ; Point to the entry
+ andc r11,r11,r16 ; Clear the bit
+ bge-- ssg64Done ; All done...
+
+ ld r5,sgcESID(r14) ; Get the ESID part
+ ld r6,sgcVSID(r14) ; And get the VSID part
+ oris r5,r5,0x0800 ; Turn on the valid bit
+ or r5,r5,r17 ; Insert the SLB slot
+ xor r6,r6,r30 ; Modify the key before we actually set it
+ addi r17,r17,1 ; Bump to the next slot
+ slbmte r6,r5 ; Make that SLB entry
+ b ssg64Enter ; Go enter the next...
+
+ .align 5
+
+ssg64Done: stw r15,pmapCCtl(r28) ; Unlock the segment cache controls
+
+ eqv r16,r16,r16 ; Load up with all foxes
+ subfic r17,r17,64 ; Get the number of 1 bits we need
+
+ sld r16,r16,r17 ; Get a mask for the used SLB entries
+ li r3,1 ; Set kernel/user transition
+ std r16,validSegs(r29) ; Set the valid SR flags
+ blr
+
+;
+; mapSetUp - this function sets initial state for all mapping functions.
+; We turn off all translations (physical), disable interruptions, and
+; enter 64-bit mode if applicable.
+;
+; We also return the original MSR in r11, the feature flags in R12,
+; and CR6 set up so we can do easy branches for 64-bit
+; hw_clear_maps assumes r10, r9 will not be trashed.
+;
+
+ .align 5
+ .globl EXT(mapSetUp)
+
+LEXT(mapSetUp)
+
+ lis r0,hi16(MASK(MSR_VEC)) ; Get the vector mask
+ mfsprg r12,2 ; Get feature flags
+ ori r0,r0,lo16(MASK(MSR_FP)) ; Get the FP as well
+ mtcrf 0x04,r12 ; move pf64Bit and pfNoMSRirb to cr5 and 6
+ mfmsr r11 ; Save the MSR
+ mtcrf 0x02,r12 ; move pf64Bit and pfNoMSRirb to cr5 and 6
+ andc r11,r11,r0 ; Clear VEC and FP for good
+ ori r0,r0,lo16(MASK(MSR_EE)|MASK(MSR_DR)|MASK(MSR_IR)) ; Get rid of EE, IR, and DR
+ li r2,1 ; Prepare for 64 bit
+ andc r0,r11,r0 ; Clear the rest
+ bt pfNoMSRirb,msuNoMSR ; No MSR...
+ bt++ pf64Bitb,msuSF ; skip if 64-bit (only they take the hint)
+
+ mtmsr r0 ; Translation and all off
+ isync ; Toss prefetch
+ blr ; Return...
+
+ .align 5
+
+msuSF: rldimi r0,r2,63,MSR_SF_BIT ; set SF bit (bit 0)
+ mtmsrd r0 ; set 64-bit mode, turn off EE, DR, and IR
+ isync ; synchronize
+ blr ; Return...
+
+ .align 5
+
+msuNoMSR: mr r2,r3 ; Save R3 across call
+ mr r3,r0 ; Get the new MSR value
+ li r0,loadMSR ; Get the MSR setter SC
+ sc ; Set it
+ mr r3,r2 ; Restore R3
+ blr ; Go back all set up...
+
+
+;
+; Guest shadow assist -- remove all guest mappings
+;
+; Remove all mappings for a guest pmap from the shadow hash table.
+;
+; Parameters:
+; r3 : address of pmap, 32-bit kernel virtual address
+;
+; Non-volatile register usage:
+; r24 : host pmap's physical address
+; r25 : VMM extension block's physical address
+; r26 : physent address
+; r27 : guest pmap's space ID number
+; r28 : current hash table page index
+; r29 : guest pmap's physical address
+; r30 : saved msr image
+; r31 : current mapping
+;
+ .align 5
+ .globl EXT(hw_rem_all_gv)
+
+LEXT(hw_rem_all_gv)
+
+#define graStackSize ((31-24+1)*4)+4
+ stwu r1,-(FM_ALIGN(graStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(graStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+
+ bt++ pf64Bitb,gra64Salt ; Test for 64-bit machine
+ lwz r25,pmapVmmExtPhys+4(r3) ; r25 <- VMM pmap extension block paddr
+ lwz r9,pmapvr+4(r3) ; Get 32-bit virt<->real conversion salt
+ lwz r24,vmxHostPmapPhys+4(r11) ; r24 <- host pmap's paddr
+ b graStart ; Get to it
+gra64Salt: ld r25,pmapVmmExtPhys(r3) ; r25 <- VMM pmap extension block paddr
+ ld r9,pmapvr(r3) ; Get 64-bit virt<->real conversion salt
+ ld r24,vmxHostPmapPhys(r11) ; r24 <- host pmap's paddr
+graStart: bl EXT(mapSetUp) ; Disable 'rupts, translation, enter 64-bit mode
+ xor r29,r3,r9 ; Convert pmap_t virt->real
+ mr r30,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkExclusive ; Get lock exclusive
+
+ lwz r3,vxsGra(r25) ; Get remove all count
+ addi r3,r3,1 ; Increment remove all count
+ stw r3,vxsGra(r25) ; Update remove all count
+
+ li r28,0 ; r28 <- first hash page table index to search
+ lwz r27,pmapSpace(r29) ; r27 <- guest pmap's space ID number
+graPgLoop:
+ la r31,VMX_HPIDX_OFFSET(r25) ; Get base of hash page physical index
+ rlwinm r11,r28,GV_PGIDX_SZ_LG2,GV_HPAGE_MASK
+ ; Convert page index into page physical index offset
+ add r31,r31,r11 ; Calculate page physical index entry address
+ bt++ pf64Bitb,gra64Page ; Separate handling for 64-bit
+ lwz r31,4(r31) ; r31 <- first slot in hash table page to examine
+ b graLoop ; Examine all slots in this page
+gra64Page: ld r31,0(r31) ; r31 <- first slot in hash table page to examine
+ b graLoop ; Examine all slots in this page
+
+ .align 5
+graLoop: lwz r3,mpFlags(r31) ; Get mapping's flags
+ lhz r4,mpSpace(r31) ; Get mapping's space ID number
+ rlwinm r6,r3,0,mpgFree ; Isolate guest free mapping flag
+ xor r4,r4,r27 ; Compare space ID number
+ or. r0,r6,r4 ; cr0_eq <- !free && space id match
+ bne graMiss ; Not one of ours, skip it
+
+ lwz r11,vxsGraHits(r25) ; Get remove hit count
+ addi r11,r11,1 ; Increment remove hit count
+ stw r11,vxsGraHits(r25) ; Update remove hit count
+
+ rlwinm. r0,r3,0,mpgDormant ; Is this entry dormant?
+ bne graRemPhys ; Yes, nothing to disconnect
+
+ lwz r11,vxsGraActive(r25) ; Get remove active count
+ addi r11,r11,1 ; Increment remove hit count
+ stw r11,vxsGraActive(r25) ; Update remove hit count
+
+ bt++ pf64Bitb,graDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b graFreePTE ; Join 64-bit path to release the PTE
+graDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+graFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- graRemPhys ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx doesn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+graRemPhys:
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r26,r3 ; Got lock on our physent?
+ beq-- graBadPLock ; No, time to bail out
+
+ crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,graRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r26) ; Point to chain anchor
+ lwz r9,ppLink+4(r26) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+
+graRemLoop: beq- graRemoveMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne graRemNext ; No, chain onward
+ bt cr1_eq,graRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b graRemoved ; Exit loop
+graRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- graRemRetry ; Lost reservation, retry
+ b graRemoved ; Good work, let's get outta here
+
+graRemNext: la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b graRemLoop ; Carry on
+
+graRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r26) ; Point to chain anchor
+ ld r9,ppLink(r26) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+graRem64Lp: beq-- graRemoveMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne graRem64Nxt ; Not mapping to remove, chain on, dude
+ bt cr1_eq,graRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b graRemoved ; Exit loop
+graRem64Rt: ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- graRem64Rt ; Lost reservation, retry
+ b graRemoved ; Good work, let's go home
+
+graRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b graRem64Lp ; Carry on
+
+graRemoved:
+ mr r3,r26 ; r3 <- physent's address
+ bl mapPhysUnlock ; Unlock the physent (and its chain of mappings)
+
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ rlwinm r3,r3,0,~mpgFlags ; Clear all guest flags
+ ori r3,r3,mpgFree ; Mark mapping free
+ stw r3,mpFlags(r31) ; Update flags
+
+graMiss: addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping
+ rlwinm. r0,r31,0,GV_PAGE_MASK ; End of hash table page?
+ bne graLoop ; No, examine next slot
+ addi r28,r28,1 ; Increment hash table page index
+ cmplwi r28,GV_HPAGES ; End of hash table?
+ bne graPgLoop ; Examine next hash table page
+
+ la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkUnlock ; Release host pmap's search lock
+
+ bt++ pf64Bitb,graRtn64 ; Handle 64-bit separately
+ mtmsr r30 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b graPopFrame ; Nothing to do now but pop a frame and return
+graRtn64: mtmsrd r30 ; Restore 'rupts, translation, 32-bit mode
+graPopFrame:
+ lwz r0,(FM_ALIGN(graStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+graBadPLock:
+graRemoveMiss:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- remove local guest mappings
+;
+; Remove local mappings for a guest pmap from the shadow hash table.
+;
+; Parameters:
+; r3 : address of guest pmap, 32-bit kernel virtual address
+;
+; Non-volatile register usage:
+; r20 : current active map word's physical address
+; r21 : current hash table page address
+; r22 : updated active map word in process
+; r23 : active map word in process
+; r24 : host pmap's physical address
+; r25 : VMM extension block's physical address
+; r26 : physent address
+; r27 : guest pmap's space ID number
+; r28 : current active map index
+; r29 : guest pmap's physical address
+; r30 : saved msr image
+; r31 : current mapping
+;
+ .align 5
+ .globl EXT(hw_rem_local_gv)
+
+LEXT(hw_rem_local_gv)
+
+#define grlStackSize ((31-20+1)*4)+4
+ stwu r1,-(FM_ALIGN(grlStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(grlStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+ stw r23,FM_ARG0+0x20(r1) ; Save non-volatile r23
+ stw r22,FM_ARG0+0x24(r1) ; Save non-volatile r22
+ stw r21,FM_ARG0+0x28(r1) ; Save non-volatile r21
+ stw r20,FM_ARG0+0x2C(r1) ; Save non-volatile r20
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+
+ bt++ pf64Bitb,grl64Salt ; Test for 64-bit machine
+ lwz r25,pmapVmmExtPhys+4(r3) ; r25 <- VMM pmap extension block paddr
+ lwz r9,pmapvr+4(r3) ; Get 32-bit virt<->real conversion salt
+ lwz r24,vmxHostPmapPhys+4(r11) ; r24 <- host pmap's paddr
+ b grlStart ; Get to it
+grl64Salt: ld r25,pmapVmmExtPhys(r3) ; r25 <- VMM pmap extension block paddr
+ ld r9,pmapvr(r3) ; Get 64-bit virt<->real conversion salt
+ ld r24,vmxHostPmapPhys(r11) ; r24 <- host pmap's paddr
+
+grlStart: bl EXT(mapSetUp) ; Disable 'rupts, translation, enter 64-bit mode
+ xor r29,r3,r9 ; Convert pmap_t virt->real
+ mr r30,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r28,0 ; r28 <- index of first active map word to search
+ lwz r27,pmapSpace(r29) ; r27 <- guest pmap's space ID number
+ b grlMap1st ; Examine first map word
+
+ .align 5
+grlNextMap: stw r22,0(r21) ; Save updated map word
+ addi r28,r28,1 ; Increment map word index
+ cmplwi r28,GV_MAP_WORDS ; See if we're done
+ beq grlDone ; Yup, let's get outta here
+
+grlMap1st: la r20,VMX_ACTMAP_OFFSET(r25) ; Get base of active map word array
+ rlwinm r11,r28,GV_MAPWD_SZ_LG2,GV_MAP_MASK
+ ; Convert map index into map index offset
+ add r20,r20,r11 ; Calculate map array element address
+ lwz r22,0(r20) ; Get active map word at index
+ mr. r23,r22 ; Any active mappings indicated?
+ beq grlNextMap ; Nope, check next word
+
+ la r21,VMX_HPIDX_OFFSET(r25) ; Get base of hash page physical index
+ rlwinm r11,r28,GV_MAP_SHIFT,GV_HPAGE_MASK
+ ; Extract page index from map word index and convert
+ ; into page physical index offset
+ add r21,r21,r11 ; Calculate page physical index entry address
+ bt++ pf64Bitb,grl64Page ; Separate handling for 64-bit
+ lwz r21,4(r21) ; Get selected hash table page's address
+ b grlLoop ; Examine all slots in this page
+grl64Page: ld r21,0(r21) ; Get selected hash table page's address
+ b grlLoop ; Examine all slots in this page
+
+ .align 5
+grlLoop: cntlzw r11,r23 ; Get next active bit lit in map word
+ cmplwi r11,32 ; Any active mappings left in this word?
+ lis r12,0x8000 ; Prepare mask to reset bit
+ srw r12,r12,r11 ; Position mask bit
+ andc r23,r23,r12 ; Reset lit bit
+ beq grlNextMap ; No bits lit, examine next map word
+
+ slwi r31,r11,GV_SLOT_SZ_LG2 ; Get slot offset in slot band from lit bit number
+ rlwinm r31,r28,GV_BAND_SHIFT,GV_BAND_MASK
+ ; Extract slot band number from index and insert
+ add r31,r31,r21 ; Add hash page address yielding mapping slot address
+
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ lhz r4,mpSpace(r31) ; Get mapping's space ID number
+ rlwinm r5,r3,0,mpgGlobal ; Extract global bit
+ xor r4,r4,r27 ; Compare space ID number
+ or. r4,r4,r5 ; (space id miss || global)
+ bne grlLoop ; Not one of ours, skip it
+ andc r22,r22,r12 ; Reset active bit corresponding to this mapping
+ ori r3,r3,mpgDormant ; Mark entry dormant
+ stw r3,mpFlags(r31) ; Update mapping's flags
+
+ bt++ pf64Bitb,grlDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b grlFreePTE ; Join 64-bit path to release the PTE
+grlDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+grlFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- grlLoop ; No valid PTE, we're done with this mapping
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx doesn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+ b grlLoop ; On to next active mapping in this map word
+
+grlDone: la r3,pmapSXlk(r24) ; r3 <- host pmap's search lock
+ bl sxlkUnlock ; Release host pmap's search lock
+
+ bt++ pf64Bitb,grlRtn64 ; Handle 64-bit separately
+ mtmsr r30 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b grlPopFrame ; Nothing to do now but pop a frame and return
+grlRtn64: mtmsrd r30 ; Restore 'rupts, translation, 32-bit mode
+grlPopFrame:
+ lwz r0,(FM_ALIGN(grlStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r23,FM_ARG0+0x20(r1) ; Restore non-volatile r23
+ lwz r22,FM_ARG0+0x24(r1) ; Restore non-volatile r22
+ lwz r21,FM_ARG0+0x28(r1) ; Restore non-volatile r21
+ lwz r20,FM_ARG0+0x2C(r1) ; Restore non-volatile r20
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+
+;
+; Guest shadow assist -- resume a guest mapping
+;
+; Locates the specified dormant mapping, and if it exists validates it and makes it
+; active.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : host virtual address, high-order 32 bits
+; r6 : host virtual address, low-order 32 bits
+; r7 : guest virtual address, high-order 32 bits
+; r8 : guest virtual address, low-order 32 bits
+; r9 : guest mapping protection code
+;
+; Non-volatile register usage:
+; r23 : VMM extension block's physical address
+; r24 : physent physical address
+; r25 : caller's msr image from mapSetUp
+; r26 : guest mapping protection code
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : host virtual address
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+ .align 5
+ .globl EXT(hw_res_map_gv)
+
+LEXT(hw_res_map_gv)
+
+#define grsStackSize ((31-23+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(grsStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(grsStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+ stw r23,FM_ARG0+0x20(r1) ; Save non-volatile r23
+
+ rlwinm r29,r6,0,0xFFFFF000 ; Clean up low-order 32 bits of host vaddr
+ rlwinm r30,r8,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+ mr r26,r9 ; Copy guest mapping protection code
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+ bt++ pf64Bitb,grs64Salt ; Handle 64-bit machine separately
+ lwz r23,pmapVmmExtPhys+4(r3) ; r23 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b grsStart ; Get to it
+
+grs64Salt: rldimi r29,r5,32,0 ; Insert high-order 32 bits of 64-bit host vaddr
+ rldimi r30,r7,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r23,pmapVmmExtPhys(r3) ; r23 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+grsStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r25,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,grs64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b grs32SrchLp ; Let the search begin!
+
+ .align 5
+grs32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq grsSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz grs32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ beq grsSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b grsSrchMiss ; No joy in our hash group
+
+grs64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b grs64SrchLp ; Let the search begin!
+
+ .align 5
+grs64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq grsSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz grs64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ bne grsSrchMiss ; No joy in our hash group
+
+grsSrchHit:
+ rlwinm. r0,r6,0,mpgDormant ; Is the mapping dormant?
+ bne grsFindHost ; Yes, nothing to disconnect
+
+ bt++ pf64Bitb,grsDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b grsFreePTE ; Join 64-bit path to release the PTE
+grsDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+grsFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- grsFindHost ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+grsFindHost:
+
+// We now have a dormant guest mapping that matches our space id and virtual address. Our next
+// step is to locate the host mapping that completes the guest mapping's connection to a physical
+// frame. The guest and host mappings must connect to the same physical frame, so they must both
+// be chained on the same physent. We search the physent chain for a host mapping matching our
+// host's space id and the host virtual address. If we succeed, we know that the entire chain
+// of mappings (guest virtual->host virtual->physical) is valid, so the dormant mapping can be
+// resumed. If we fail to find the specified host virtual->physical mapping, it is because the
+// host virtual or physical address has changed since the guest mapping was suspended, so it
+// is no longer valid and cannot be resumed -- we therefore delete the guest mappping and tell
+// our caller that it will have to take its long path, translating the host virtual address
+// through the host's skiplist and installing a new guest mapping.
+
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r24,r3 ; Got lock on our physent?
+ beq-- grsBadPLock ; No, time to bail out
+
+ bt++ pf64Bitb,grsPFnd64 ; 64-bit version of physent chain search
+
+ lwz r9,ppLink+4(r24) ; Get first mapping on physent
+ lwz r6,pmapSpace(r27) ; Get host pmap's space id number
+ rlwinm r9,r9,0,~ppFlags ; Be-gone, unsightly flags
+grsPELoop: mr. r12,r9 ; Got a mapping to look at?
+ beq- grsPEMiss ; Nope, we've missed hva->phys mapping
+ lwz r7,mpFlags(r12) ; Get mapping's flags
+ lhz r4,mpSpace(r12) ; Get mapping's space id number
+ lwz r5,mpVAddr+4(r12) ; Get mapping's virtual address
+ lwz r9,mpAlias+4(r12) ; Next mapping in physent alias chain
+
+ rlwinm r0,r7,0,mpType ; Isolate mapping's type
+ rlwinm r5,r5,0,~mpHWFlags ; Bye-bye unsightly flags
+ xori r0,r0,mpNormal ; Normal mapping?
+ xor r4,r4,r6 ; Compare w/ host space id number
+ xor r5,r5,r29 ; Compare w/ host virtual address
+ or r0,r0,r4 ; r0 <- (wrong type || !space id)
+ or. r0,r0,r5 ; cr0_eq <- (right type && space id hit && hva hit)
+ beq grsPEHit ; Hit
+ b grsPELoop ; Iterate
+
+grsPFnd64: li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ ld r9,ppLink(r24) ; Get first mapping on physent
+ lwz r6,pmapSpace(r27) ; Get pmap's space id number
+ andc r9,r9,r0 ; Cleanup mapping pointer
+grsPELp64: mr. r12,r9 ; Got a mapping to look at?
+ beq-- grsPEMiss ; Nope, we've missed hva->phys mapping
+ lwz r7,mpFlags(r12) ; Get mapping's flags
+ lhz r4,mpSpace(r12) ; Get mapping's space id number
+ ld r5,mpVAddr(r12) ; Get mapping's virtual address
+ ld r9,mpAlias(r12) ; Next mapping physent alias chain
+ rlwinm r0,r7,0,mpType ; Isolate mapping's type
+ rldicr r5,r5,0,mpHWFlagsb-1 ; Bye-bye unsightly flags
+ xori r0,r0,mpNormal ; Normal mapping?
+ xor r4,r4,r6 ; Compare w/ host space id number
+ xor r5,r5,r29 ; Compare w/ host virtual address
+ or r0,r0,r4 ; r0 <- (wrong type || !space id)
+ or. r0,r0,r5 ; cr0_eq <- (right type && space id hit && hva hit)
+ beq grsPEHit ; Hit
+ b grsPELp64 ; Iterate
+
+grsPEHit: lwz r0,mpVAddr+4(r31) ; Get va byte containing protection bits
+ rlwimi r0,r26,0,mpPP ; Insert new protection bits
+ stw r0,mpVAddr+4(r31) ; Write 'em back
+
+ eieio ; Ensure previous mapping updates are visible
+ lwz r0,mpFlags(r31) ; Get flags
+ rlwinm r0,r0,0,~mpgDormant ; Turn off dormant flag
+ stw r0,mpFlags(r31) ; Set updated flags, entry is now valid
+
+ li r31,mapRtOK ; Indicate success
+ b grsRelPhy ; Exit through physent lock release
+
+grsPEMiss: crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,grsRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r24) ; Point to chain anchor
+ lwz r9,ppLink+4(r24) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+grsRemLoop: beq- grsPEMissMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne grsRemNext ; No, chain onward
+ bt cr1_eq,grsRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b grsDelete ; Finish deleting mapping
+grsRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- grsRemRetry ; Lost reservation, retry
+ b grsDelete ; Finish deleting mapping
+
+ .align 5
+grsRemNext: la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b grsRemLoop ; Carry on
+
+grsRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r24) ; Point to chain anchor
+ ld r9,ppLink(r24) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+grsRem64Lp: beq-- grsPEMissMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne grsRem64Nxt ; Not mapping to remove, chain on, dude
+ bt cr1_eq,grsRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b grsDelete ; Finish deleting mapping
+grsRem64Rt: ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- grsRem64Rt ; Lost reservation, retry
+ b grsDelete ; Finish deleting mapping
+
+ .align 5
+grsRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b grsRem64Lp ; Carry on
+
+grsDelete:
+ lwz r3,mpFlags(r31) ; Get mapping's flags
+ rlwinm r3,r3,0,~mpgFlags ; Clear all guest flags
+ ori r3,r3,mpgFree ; Mark mapping free
+ stw r3,mpFlags(r31) ; Update flags
+
+ li r31,mapRtNotFnd ; Didn't succeed
+
+grsRelPhy: mr r3,r24 ; r3 <- physent addr
+ bl mapPhysUnlock ; Unlock physent chain
+
+grsRelPmap: la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+grsRtn: mr r3,r31 ; r3 <- result code
+ bt++ pf64Bitb,grsRtn64 ; Handle 64-bit separately
+ mtmsr r25 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b grsPopFrame ; Nothing to do now but pop a frame and return
+grsRtn64: mtmsrd r25 ; Restore 'rupts, translation, 32-bit mode
+grsPopFrame:
+ lwz r0,(FM_ALIGN(grsStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r23,FM_ARG0+0x20(r1) ; Restore non-volatile r23
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+ .align 5
+grsSrchMiss:
+ li r31,mapRtNotFnd ; Could not locate requested mapping
+ b grsRelPmap ; Exit through host pmap search lock release
+
+grsBadPLock:
+grsPEMissMiss:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- add a guest mapping
+;
+; Adds a guest mapping.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : guest virtual address, high-order 32 bits
+; r6 : guest virtual address, low-order 32 bits (with mpHWFlags)
+; r7 : new mapping's flags
+; r8 : physical address, 32-bit page number
+;
+; Non-volatile register usage:
+; r22 : hash group's physical address
+; r23 : VMM extension block's physical address
+; r24 : mapping's flags
+; r25 : caller's msr image from mapSetUp
+; r26 : physent physical address
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : physical address, 32-bit 4k-page number
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_add_map_gv)
+
+
+LEXT(hw_add_map_gv)
+
+#define gadStackSize ((31-22+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gadStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gadStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+ stw r23,FM_ARG0+0x20(r1) ; Save non-volatile r23
+ stw r22,FM_ARG0+0x24(r1) ; Save non-volatile r22
+
+ rlwinm r30,r5,0,1,0 ; Get high-order 32 bits of guest vaddr
+ rlwimi r30,r6,0,0,31 ; Get low-order 32 bits of guest vaddr
+ mr r24,r7 ; Copy guest mapping's flags
+ mr r29,r8 ; Copy target frame's physical address
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+ bt++ pf64Bitb,gad64Salt ; Test for 64-bit machine
+ lwz r23,pmapVmmExtPhys+4(r3) ; r23 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r22,VMX_HPIDX_OFFSET(r11) ; r22 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r22,r22,r10 ; r22 <- hash page index entry
+ lwz r22,4(r22) ; r22 <- hash page paddr
+ rlwimi r22,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r22 <- hash group paddr
+ b gadStart ; Get to it
+
+gad64Salt: ld r23,pmapVmmExtPhys(r3) ; r23 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r22,VMX_HPIDX_OFFSET(r11) ; r22 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r22,r22,r10 ; r22 <- hash page index entry
+ ld r22,0(r22) ; r22 <- hash page paddr
+ insrdi r22,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r22 <- hash group paddr
+
+gadStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r25,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exlusive
+
+ mr r31,r22 ; Prepare to search this group
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gad64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ clrrwi r12,r30,12 ; r12 <- virtual address we're searching for
+ b gad32SrchLp ; Let the search begin!
+
+ .align 5
+gad32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r12 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq gadRelPmap ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gad32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && && space match)
+ xor r5,r5,r12 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- free && space match && virtual addr match
+ beq gadRelPmap ; Join common path on hit (r31 points to guest mapping)
+ b gadScan ; No joy in our hash group
+
+gad64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ clrrdi r12,r30,12 ; r12 <- virtual address we're searching for
+ b gad64SrchLp ; Let the search begin!
+
+ .align 5
+gad64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && space match)
+ xor r8,r8,r12 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && space match && virtual addr match
+ beq gadRelPmap ; Hit, let upper-level redrive sort it out
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gad64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ rlwinm r11,r6,0,mpgFree ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && && space match)
+ xor r5,r5,r12 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && space match && virtual addr match
+ bne gadScan ; No joy in our hash group
+ b gadRelPmap ; Hit, let upper-level redrive sort it out
+
+gadScan: lbz r12,mpgCursor(r22) ; Get group's cursor
+ rlwinm r12,r12,GV_SLOT_SZ_LG2,(GV_SLOT_MASK << GV_SLOT_SZ_LG2)
+ ; Prepare to address slot at cursor
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ or r2,r22,r12 ; r2 <- 1st mapping to search
+ lwz r3,mpFlags(r2) ; r3 <- 1st mapping slot's flags
+ li r11,0 ; No dormant entries found yet
+ b gadScanLoop ; Let the search begin!
+
+ .align 5
+gadScanLoop:
+ addi r12,r12,GV_SLOT_SZ ; Calculate next slot number to search
+ rlwinm r12,r12,0,(GV_SLOT_MASK << GV_SLOT_SZ_LG2)
+ ; Trim off any carry, wrapping into slot number range
+ mr r31,r2 ; r31 <- current mapping's address
+ or r2,r22,r12 ; r2 <- next mapping to search
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags(r2) ; r3 <- next mapping slot's flags
+ rlwinm. r0,r6,0,mpgFree ; Test free flag
+ bne gadFillMap ; Join common path on hit (r31 points to free mapping)
+ rlwinm r0,r6,0,mpgDormant ; Dormant entry?
+ xori r0,r0,mpgDormant ; Invert dormant flag
+ or. r0,r0,r11 ; Skip all but the first dormant entry we see
+ bne gadNotDorm ; Not dormant or we've already seen one
+ mr r11,r31 ; We'll use this dormant entry if we don't find a free one first
+gadNotDorm: bdnz gadScanLoop ; Iterate
+
+ mr r31,r2 ; r31 <- final mapping's address
+ rlwinm. r0,r6,0,mpgFree ; Test free flag in final mapping
+ bne gadFillMap ; Join common path on hit (r31 points to dormant mapping)
+ rlwinm r0,r6,0,mpgDormant ; Dormant entry?
+ xori r0,r0,mpgDormant ; Invert dormant flag
+ or. r0,r0,r11 ; Skip all but the first dormant entry we see
+ bne gadCkDormant ; Not dormant or we've already seen one
+ mr r11,r31 ; We'll use this dormant entry if we don't find a free one first
+
+gadCkDormant:
+ mr. r31,r11 ; Get dormant mapping, if any, and test
+ bne gadUpCursor ; Go update the cursor, we'll take the dormant entry
+
+gadSteal:
+ lbz r12,mpgCursor(r22) ; Get group's cursor
+ rlwinm r12,r12,GV_SLOT_SZ_LG2,(GV_SLOT_MASK << GV_SLOT_SZ_LG2)
+ ; Prepare to address slot at cursor
+ or r31,r22,r12 ; r31 <- address of mapping to steal
+
+ bt++ pf64Bitb,gadDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b gadFreePTE ; Join 64-bit path to release the PTE
+gadDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+gadFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- gadUpCursor ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+gadUpCursor:
+ rlwinm r12,r31,(32-GV_SLOT_SZ_LG2),GV_SLOT_MASK
+ ; Recover slot number from stolen mapping's address
+ addi r12,r12,1 ; Increment slot number
+ rlwinm r12,r12,0,GV_SLOT_MASK ; Clip to slot number range
+ stb r12,mpgCursor(r22) ; Update group's cursor
+
+ lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r26,r3 ; Got lock on our physent?
+ beq-- gadBadPLock ; No, time to bail out
+
+ crset cr1_eq ; cr1_eq <- previous link is the anchor
+ bt++ pf64Bitb,gadRemove64 ; Use 64-bit version on 64-bit machine
+ la r11,ppLink+4(r26) ; Point to chain anchor
+ lwz r9,ppLink+4(r26) ; Get chain anchor
+ rlwinm. r9,r9,0,~ppFlags ; Remove flags, yielding 32-bit physical chain pointer
+gadRemLoop: beq- gadPEMissMiss ; End of chain, this is not good
+ cmplw r9,r31 ; Is this the mapping to remove?
+ lwz r8,mpAlias+4(r9) ; Get forward chain pointer
+ bne gadRemNext ; No, chain onward
+ bt cr1_eq,gadRemRetry ; Mapping to remove is chained from anchor
+ stw r8,0(r11) ; Unchain gpv->phys mapping
+ b gadDelDone ; Finish deleting mapping
+gadRemRetry:
+ lwarx r0,0,r11 ; Get previous link
+ rlwimi r0,r8,0,~ppFlags ; Insert new forward pointer whilst preserving flags
+ stwcx. r0,0,r11 ; Update previous link
+ bne- gadRemRetry ; Lost reservation, retry
+ b gadDelDone ; Finish deleting mapping
+
+gadRemNext: la r11,mpAlias+4(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b gadRemLoop ; Carry on
+
+gadRemove64:
+ li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ la r11,ppLink(r26) ; Point to chain anchor
+ ld r9,ppLink(r26) ; Get chain anchor
+ andc. r9,r9,r7 ; Remove flags, yielding 64-bit physical chain pointer
+gadRem64Lp: beq-- gadPEMissMiss ; End of chain, this is not good
+ cmpld r9,r31 ; Is this the mapping to remove?
+ ld r8,mpAlias(r9) ; Get forward chain pinter
+ bne gadRem64Nxt ; Not mapping to remove, chain on, dude
+ bt cr1_eq,gadRem64Rt ; Mapping to remove is chained from anchor
+ std r8,0(r11) ; Unchain gpv->phys mapping
+ b gadDelDone ; Finish deleting mapping
+gadRem64Rt: ldarx r0,0,r11 ; Get previous link
+ and r0,r0,r7 ; Get flags
+ or r0,r0,r8 ; Insert new forward pointer
+ stdcx. r0,0,r11 ; Slam it back in
+ bne-- gadRem64Rt ; Lost reservation, retry
+ b gadDelDone ; Finish deleting mapping
+
+ .align 5
+gadRem64Nxt:
+ la r11,mpAlias(r9) ; Point to (soon to be) previous link
+ crclr cr1_eq ; ~cr1_eq <- Previous link is not the anchor
+ mr. r9,r8 ; Does next entry exist?
+ b gadRem64Lp ; Carry on
+
+gadDelDone:
+ mr r3,r26 ; Get physent address
+ bl mapPhysUnlock ; Unlock physent chain
+
+gadFillMap:
+ lwz r12,pmapSpace(r28) ; Get guest space id number
+ li r2,0 ; Get a zero
+ stw r24,mpFlags(r31) ; Set mapping's flags
+ sth r12,mpSpace(r31) ; Set mapping's space id number
+ stw r2,mpPte(r31) ; Set mapping's pte pointer invalid
+ stw r29,mpPAddr(r31) ; Set mapping's physical address
+ bt++ pf64Bitb,gadVA64 ; Use 64-bit version on 64-bit machine
+ stw r30,mpVAddr+4(r31) ; Set mapping's virtual address (w/flags)
+ b gadChain ; Continue with chaining mapping to physent
+gadVA64: std r30,mpVAddr(r31) ; Set mapping's virtual address (w/flags)
+
+gadChain: mr r3,r29 ; r3 <- physical frame address
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r26,r3 ; Got lock on our physent?
+ beq-- gadBadPLock ; No, time to bail out
+
+ bt++ pf64Bitb,gadChain64 ; Use 64-bit version on 64-bit machine
+ lwz r12,ppLink+4(r26) ; Get forward chain
+ rlwinm r11,r12,0,~ppFlags ; Get physent's forward pointer sans flags
+ rlwimi r12,r31,0,~ppFlags ; Insert new mapping, preserve physent flags
+ stw r11,mpAlias+4(r31) ; New mapping will head chain
+ stw r12,ppLink+4(r26) ; Point physent to new mapping
+ b gadFinish ; All over now...
+
+gadChain64: li r7,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r7,r7,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ ld r12,ppLink(r26) ; Get forward chain
+ andc r11,r12,r7 ; Get physent's forward chain pointer sans flags
+ and r12,r12,r7 ; Isolate pointer's flags
+ or r12,r12,r31 ; Insert new mapping's address forming pointer
+ std r11,mpAlias(r31) ; New mapping will head chain
+ std r12,ppLink(r26) ; Point physent to new mapping
+
+gadFinish: eieio ; Ensure new mapping is completely visible
+
+gadRelPhy: mr r3,r26 ; r3 <- physent addr
+ bl mapPhysUnlock ; Unlock physent chain
+
+gadRelPmap: la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+ bt++ pf64Bitb,gadRtn64 ; Handle 64-bit separately
+ mtmsr r25 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b gadPopFrame ; Nothing to do now but pop a frame and return
+gadRtn64: mtmsrd r25 ; Restore 'rupts, translation, 32-bit mode
+gadPopFrame:
+ lwz r0,(FM_ALIGN(gadStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r23,FM_ARG0+0x20(r1) ; Restore non-volatile r23
+ lwz r22,FM_ARG0+0x24(r1) ; Restore non-volatile r22
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+gadPEMissMiss:
+gadBadPLock:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- supend a guest mapping
+;
+; Suspends a guest mapping.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : guest virtual address, high-order 32 bits
+; r6 : guest virtual address, low-order 32 bits
+;
+; Non-volatile register usage:
+; r26 : VMM extension block's physical address
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : caller's msr image from mapSetUp
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_susp_map_gv)
+
+LEXT(hw_susp_map_gv)
+
+#define gsuStackSize ((31-26+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gsuStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gsuStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+
+ rlwinm r30,r6,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+ bt++ pf64Bitb,gsu64Salt ; Test for 64-bit machine
+
+ lwz r26,pmapVmmExtPhys+4(r3) ; r26 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gsuStart ; Get to it
+gsu64Salt: rldimi r30,r5,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r26,pmapVmmExtPhys(r3) ; r26 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gsuStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r29,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gsu64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gsu32SrchLp ; Let the search begin!
+
+ .align 5
+gsu32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gsuSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gsu32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gsuSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gsuSrchMiss ; No joy in our hash group
+
+gsu64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gsu64SrchLp ; Let the search begin!
+
+ .align 5
+gsu64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gsuSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gsu64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gsuSrchMiss ; No joy in our hash group
+
+gsuSrchHit:
+ bt++ pf64Bitb,gsuDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b gsuFreePTE ; Join 64-bit path to release the PTE
+gsuDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+gsuFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- gsuNoPTE ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+gsuNoPTE: lwz r3,mpFlags(r31) ; Get mapping's flags
+ ori r3,r3,mpgDormant ; Mark entry dormant
+ stw r3,mpFlags(r31) ; Save updated flags
+ eieio ; Ensure update is visible when we unlock
+
+gsuSrchMiss:
+ la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+ bt++ pf64Bitb,gsuRtn64 ; Handle 64-bit separately
+ mtmsr r29 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b gsuPopFrame ; Nothing to do now but pop a frame and return
+gsuRtn64: mtmsrd r29 ; Restore 'rupts, translation, 32-bit mode
+gsuPopFrame:
+ lwz r0,(FM_ALIGN(gsuStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+;
+; Guest shadow assist -- test guest mapping reference and change bits
+;
+; Locates the specified guest mapping, and if it exists gathers its reference
+; and change bit, optionallyÊresetting them.
+;
+; Parameters:
+; r3 : address of host pmap, 32-bit kernel virtual address
+; r4 : address of guest pmap, 32-bit kernel virtual address
+; r5 : guest virtual address, high-order 32 bits
+; r6 : guest virtual address, low-order 32 bits
+; r7 : reset boolean
+;
+; Non-volatile register usage:
+; r24 : VMM extension block's physical address
+; r25 : return code (w/reference and change bits)
+; r26 : reset boolean
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : caller's msr image from mapSetUp
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_test_rc_gv)
+
+LEXT(hw_test_rc_gv)
+
+#define gtdStackSize ((31-24+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gtdStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gtdStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+
+ rlwinm r30,r6,0,0xFFFFF000 ; Clean up low-order 20 bits of guest vaddr
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r4) ; r9 <- guest space ID number
+
+ bt++ pf64Bitb,gtd64Salt ; Test for 64-bit machine
+
+ lwz r24,pmapVmmExtPhys+4(r3) ; r24 <- VMM pmap extension block paddr
+ lwz r27,pmapvr+4(r3) ; Get 32-bit virt<->real host pmap conversion salt
+ lwz r28,pmapvr+4(r4) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gtdStart ; Get to it
+
+gtd64Salt: rldimi r30,r5,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r24,pmapVmmExtPhys(r3) ; r24 <- VMM pmap extension block paddr
+ ld r27,pmapvr(r3) ; Get 64-bit virt<->real host pmap conversion salt
+ ld r28,pmapvr(r4) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gtdStart: xor r27,r3,r27 ; Convert host pmap_t virt->real
+ xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ mr r26,r7 ; Save reset boolean
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r29,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gtd64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gtd32SrchLp ; Let the search begin!
+
+ .align 5
+gtd32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gtdSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gtd32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gtdSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gtdSrchMiss ; No joy in our hash group
+
+gtd64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gtd64SrchLp ; Let the search begin!
+
+ .align 5
+gtd64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gtdSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gtd64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gtdSrchMiss ; No joy in our hash group
+
+gtdSrchHit:
+ bt++ pf64Bitb,gtdDo64 ; Split for 64 bit
+
+ bl mapInvPte32 ; Invalidate and lock PTEG, also merge into physent
+
+ cmplwi cr1,r26,0 ; Do we want to clear RC?
+ lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
+ mr. r3,r3 ; Was there a previously valid PTE?
+ li r0,lo16(mpR|mpC) ; Get bits to clear
-LEXT(mapalc)
+ and r25,r5,r0 ; Copy RC bits into result
+ beq++ cr1,gtdNoClr32 ; Nope...
- lwz r4,mbfree(r3) ; Get the first mask
- lis r0,0x8000 ; Get the mask to clear the first free bit
- lwz r5,mbfree+4(r3) ; Get the second mask
- mr r12,r3 ; Save the return
- cntlzw r8,r4 ; Get first free field
- lwz r6,mbfree+8(r3) ; Get the third mask
- srw. r9,r0,r8 ; Get bit corresponding to first free one
- lwz r7,mbfree+12(r3) ; Get the fourth mask
- cntlzw r10,r5 ; Get first free field in second word
- andc r4,r4,r9 ; Turn it off
- bne malcfnd0 ; Found one...
+ andc r12,r12,r0 ; Clear mapping copy of RC
+ andc r5,r5,r0 ; Clear PTE copy of RC
+ sth r12,mpVAddr+6(r31) ; Set the new RC in mapping
+
+gtdNoClr32: beq-- gtdNoOld32 ; No previously valid PTE...
- srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
- cntlzw r11,r6 ; Get first free field in third word
- andc r5,r5,r9 ; Turn it off
- bne malcfnd1 ; Found one...
+ sth r5,6(r3) ; Store updated RC in PTE
+ eieio ; Make sure we do not reorder
+ stw r4,0(r3) ; Revalidate the PTE
+
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock PCA
+
+gtdNoOld32: la r3,pmapSXlk(r27) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+ b gtdR32 ; Join common...
+
+ .align 5
- srw. r9,r0,r11 ; Get bit corresponding to first free one in third word
- cntlzw r10,r7 ; Get first free field in fourth word
- andc r6,r6,r9 ; Turn it off
- bne malcfnd2 ; Found one...
- srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
- li r3,0 ; Assume abject failure
- andc r7,r7,r9 ; Turn it off
- beqlr ; There are none any left...
+gtdDo64: bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
+
+ cmplwi cr1,r26,0 ; Do we want to clear RC?
+ lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
+ mr. r3,r3 ; Was there a previously valid PTE?
+ li r0,lo16(mpR|mpC) ; Get bits to clear
+
+ and r25,r5,r0 ; Copy RC bits into result
+ beq++ cr1,gtdNoClr64 ; Nope...
- addi r3,r10,96 ; Set the correct bit number
- stw r7,mbfree+12(r12) ; Actually allocate the slot
+ andc r12,r12,r0 ; Clear mapping copy of RC
+ andc r5,r5,r0 ; Clear PTE copy of RC
+ sth r12,mpVAddr+6(r31) ; Set the new RC
+
+gtdNoClr64: beq-- gtdNoOld64 ; Nope, no pevious pte...
-mapafin: or r4,r4,r5 ; Merge the first two allocation maps
- or r6,r6,r7 ; Then the last two
- or. r4,r4,r6 ; Merge both halves
- bnelr+ ; Return if some left for next time...
+ sth r5,14(r3) ; Store updated RC
+ eieio ; Make sure we do not reorder
+ std r4,0(r3) ; Revalidate the PTE
+
+ eieio ; Make sure all updates come first
+ stw r6,0(r7) ; Unlock PCA
+
+gtdNoOld64: la r3,pmapSXlk(r27) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+ b gtdR64 ; Join common...
+
+gtdSrchMiss:
+ la r3,pmapSXlk(r27) ; Point to the pmap search lock
+ bl sxlkUnlock ; Unlock the search list
+ li r25,mapRtNotFnd ; Get ready to return not found
+ bt++ pf64Bitb,gtdR64 ; Test for 64-bit machine
- neg r3,r3 ; Indicate we just allocated the last one
- blr ; Leave...
+gtdR32: mtmsr r29 ; Restore caller's msr image
+ isync
+ b gtdEpilog
+
+gtdR64: mtmsrd r29 ; Restore caller's msr image
+
+gtdEpilog: lwz r0,(FM_ALIGN(gtdStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ mr r3,r25 ; Get return code
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+;
+; Guest shadow assist -- convert guest to host virtual address
+;
+; Locates the specified guest mapping, and if it exists locates the
+; first mapping belonging to its host on the physical chain and returns
+; its virtual address.
+;
+; Note that if there are multiple mappings belonging to this host
+; chained to the physent to which the guest mapping is chained, then
+; host virtual aliases exist for this physical address. If host aliases
+; exist, then we select the first on the physent chain, making it
+; unpredictable which of the two or more possible host virtual addresses
+; will be returned.
+;
+; Parameters:
+; r3 : address of guest pmap, 32-bit kernel virtual address
+; r4 : guest virtual address, high-order 32 bits
+; r5 : guest virtual address, low-order 32 bits
+;
+; Non-volatile register usage:
+; r24 : physent physical address
+; r25 : VMM extension block's physical address
+; r26 : host virtual address
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : caller's msr image from mapSetUp
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_gva_to_hva)
+
+LEXT(hw_gva_to_hva)
+
+#define gthStackSize ((31-24+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gtdStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gtdStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+ stw r24,FM_ARG0+0x1C(r1) ; Save non-volatile r24
+
+ rlwinm r30,r5,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r3) ; r9 <- guest space ID number
+
+ bt++ pf64Bitb,gth64Salt ; Test for 64-bit machine
+
+ lwz r25,pmapVmmExtPhys+4(r3) ; r25 <- VMM pmap extension block paddr
+ lwz r28,pmapvr+4(r3) ; Get 32-bit virt<->real guest pmap conversion salt
+ lwz r27,vmxHostPmapPhys+4(r11) ; Get host pmap physical address
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gthStart ; Get to it
+
+gth64Salt: rldimi r30,r4,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r25,pmapVmmExtPhys(r3) ; r24 <- VMM pmap extension block paddr
+ ld r28,pmapvr(r3) ; Get 64-bit virt<->real guest pmap conversion salt
+ ld r27,vmxHostPmapPhys(r11) ; Get host pmap physical address
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gthStart: xor r28,r3,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r29,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gth64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gth32SrchLp ; Let the search begin!
+
+ .align 5
+gth32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gthSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gth32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gthSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gthSrchMiss ; No joy in our hash group
+
+gth64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gth64SrchLp ; Let the search begin!
-malcfnd0: stw r4,mbfree(r12) ; Actually allocate the slot
- mr r3,r8 ; Set the correct bit number
- b mapafin ; Exit now...
+ .align 5
+gth64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gthSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gth64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gthSrchMiss ; No joy in our hash group
+
+gthSrchHit: lwz r3,mpPAddr(r31) ; r3 <- physical 4K-page number
+ bl mapFindLockPN ; Find 'n' lock this page's physent
+ mr. r24,r3 ; Got lock on our physent?
+ beq-- gthBadPLock ; No, time to bail out
+
+ bt++ pf64Bitb,gthPFnd64 ; 64-bit version of physent chain search
+
+ lwz r9,ppLink+4(r24) ; Get first mapping on physent
+ lwz r6,pmapSpace(r27) ; Get host pmap's space id number
+ rlwinm r9,r9,0,~ppFlags ; Be-gone, unsightly flags
+gthPELoop: mr. r12,r9 ; Got a mapping to look at?
+ beq- gthPEMiss ; Nope, we've missed hva->phys mapping
+ lwz r7,mpFlags(r12) ; Get mapping's flags
+ lhz r4,mpSpace(r12) ; Get mapping's space id number
+ lwz r26,mpVAddr+4(r12) ; Get mapping's virtual address
+ lwz r9,mpAlias+4(r12) ; Next mapping in physent alias chain
+
+ rlwinm r0,r7,0,mpType ; Isolate mapping's type
+ rlwinm r26,r26,0,~mpHWFlags ; Bye-bye unsightly flags
+ xori r0,r0,mpNormal ; Normal mapping?
+ xor r4,r4,r6 ; Compare w/ host space id number
+ or. r0,r0,r4 ; cr0_eq <- (normal && space id hit)
+ beq gthPEHit ; Hit
+ b gthPELoop ; Iterate
+
+gthPFnd64: li r0,ppLFAmask ; Get mask to clean up mapping pointer
+ rotrdi r0,r0,ppLFArrot ; Rotate clean up mask to get 0xF0000000000000000F
+ ld r9,ppLink(r24) ; Get first mapping on physent
+ lwz r6,pmapSpace(r27) ; Get host pmap's space id number
+ andc r9,r9,r0 ; Cleanup mapping pointer
+gthPELp64: mr. r12,r9 ; Got a mapping to look at?
+ beq-- gthPEMiss ; Nope, we've missed hva->phys mapping
+ lwz r7,mpFlags(r12) ; Get mapping's flags
+ lhz r4,mpSpace(r12) ; Get mapping's space id number
+ ld r26,mpVAddr(r12) ; Get mapping's virtual address
+ ld r9,mpAlias(r12) ; Next mapping physent alias chain
+ rlwinm r0,r7,0,mpType ; Isolate mapping's type
+ rldicr r26,r26,0,mpHWFlagsb-1 ; Bye-bye unsightly flags
+ xori r0,r0,mpNormal ; Normal mapping?
+ xor r4,r4,r6 ; Compare w/ host space id number
+ or. r0,r0,r4 ; cr0_eq <- (normal && space id hit)
+ beq gthPEHit ; Hit
+ b gthPELp64 ; Iterate
+
+ .align 5
+gthPEMiss: mr r3,r24 ; Get physent's address
+ bl mapPhysUnlock ; Unlock physent chain
+gthSrchMiss:
+ la r3,pmapSXlk(r27) ; Get host pmap search lock address
+ bl sxlkUnlock ; Release host pmap search lock
+ li r3,-1 ; Return 64-bit -1
+ li r4,-1
+ bt++ pf64Bitb,gthEpi64 ; Take 64-bit exit
+ b gthEpi32 ; Take 32-bit exit
+
+ .align 5
+gthPEHit: mr r3,r24 ; Get physent's address
+ bl mapPhysUnlock ; Unlock physent chain
+ la r3,pmapSXlk(r27) ; Get host pmap search lock address
+ bl sxlkUnlock ; Release host pmap search lock
+
+ bt++ pf64Bitb,gthR64 ; Test for 64-bit machine
-malcfnd1: stw r5,mbfree+4(r12) ; Actually allocate the slot
- addi r3,r10,32 ; Set the correct bit number
- b mapafin ; Exit now...
+gthR32: li r3,0 ; High-order 32 bits host virtual address
+ mr r4,r26 ; Low-order 32 bits host virtual address
+gthEpi32: mtmsr r29 ; Restore caller's msr image
+ isync
+ b gthEpilog
+
+ .align 5
+gthR64: srdi r3,r26,32 ; High-order 32 bits host virtual address
+ clrldi r4,r26,32 ; Low-order 32 bits host virtual address
+gthEpi64: mtmsrd r29 ; Restore caller's msr image
+
+gthEpilog: lwz r0,(FM_ALIGN(gthStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r24,FM_ARG0+0x1C(r1) ; Restore non-volatile r24
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+gthBadPLock:
+ lis r0,hi16(Choke) ; Dmitri, you know how we've always talked about the
+ ori r0,r0,lo16(Choke) ; possibility of something going wrong with the bomb?
+ li r3,failMapping ; The BOMB, Dmitri.
+ sc ; The hydrogen bomb.
+
+
+;
+; Guest shadow assist -- find a guest mapping
+;
+; Locates the specified guest mapping, and if it exists returns a copy
+; of it.
+;
+; Parameters:
+; r3 : address of guest pmap, 32-bit kernel virtual address
+; r4 : guest virtual address, high-order 32 bits
+; r5 : guest virtual address, low-order 32 bits
+; r6 : 32 byte copy area, 32-bit kernel virtual address
+;
+; Non-volatile register usage:
+; r25 : VMM extension block's physical address
+; r26 : copy area virtual address
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : caller's msr image from mapSetUp
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+
+ .align 5
+ .globl EXT(hw_find_map_gv)
+
+LEXT(hw_find_map_gv)
+
+#define gfmStackSize ((31-25+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gfmStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gfmStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+
+ rlwinm r30,r5,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+ mr r26,r6 ; Copy copy buffer vaddr
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r3) ; r9 <- guest space ID number
+
+ bt++ pf64Bitb,gfm64Salt ; Test for 64-bit machine
+
+ lwz r25,pmapVmmExtPhys+4(r3) ; r25 <- VMM pmap extension block paddr
+ lwz r28,pmapvr+4(r3) ; Get 32-bit virt<->real guest pmap conversion salt
+ lwz r27,vmxHostPmapPhys+4(r11) ; Get host pmap physical address
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gfmStart ; Get to it
+
+gfm64Salt: rldimi r30,r4,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r25,pmapVmmExtPhys(r3) ; r24 <- VMM pmap extension block paddr
+ ld r28,pmapvr(r3) ; Get 64-bit virt<->real guest pmap conversion salt
+ ld r27,vmxHostPmapPhys(r11) ; Get host pmap physical address
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gfmStart: xor r28,r3,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r29,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gfm64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gfm32SrchLp ; Let the search begin!
-malcfnd2: stw r6,mbfree+8(r12) ; Actually allocate the slot
- addi r3,r11,64 ; Set the correct bit number
- b mapafin ; Exit now...
+ .align 5
+gfm32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gfmSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gfm32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gfmSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gfmSrchMiss ; No joy in our hash group
+
+gfm64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gfm64SrchLp ; Let the search begin!
+ .align 5
+gfm64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- !(!free && !dormant && space match)
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gfmSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gfm64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free and dormant flags
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- !(!free && !dormant && space match)
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gfmSrchMiss ; No joy in our hash group
+
+gfmSrchHit: lwz r5,0(r31) ; Fetch 32 bytes of mapping from physical
+ lwz r6,4(r31) ; +4
+ lwz r7,8(r31) ; +8
+ lwz r8,12(r31) ; +12
+ lwz r9,16(r31) ; +16
+ lwz r10,20(r31) ; +20
+ lwz r11,24(r31) ; +24
+ lwz r12,28(r31) ; +28
+
+ li r31,mapRtOK ; Return found mapping
+
+ la r3,pmapSXlk(r27) ; Get host pmap search lock address
+ bl sxlkUnlock ; Release host pmap search lock
+
+ bt++ pf64Bitb,gfmEpi64 ; Test for 64-bit machine
+
+gfmEpi32: mtmsr r29 ; Restore caller's msr image
+ isync ; A small wrench
+ b gfmEpilog ; and a larger bubble
+
+ .align 5
+gfmEpi64: mtmsrd r29 ; Restore caller's msr image
+
+gfmEpilog: mr. r3,r31 ; Copy/test mapping address
+ beq gfmNotFound ; Skip copy if no mapping found
+
+ stw r5,0(r26) ; Store 32 bytes of mapping into virtual
+ stw r6,4(r26) ; +4
+ stw r7,8(r26) ; +8
+ stw r8,12(r26) ; +12
+ stw r9,16(r26) ; +16
+ stw r10,20(r26) ; +20
+ stw r11,24(r26) ; +24
+ stw r12,28(r26) ; +28
+
+gfmNotFound:
+ lwz r0,(FM_ALIGN(gfmStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
+
+ .align 5
+gfmSrchMiss:
+ li r31,mapRtNotFnd ; Indicate mapping not found
+ la r3,pmapSXlk(r27) ; Get host pmap search lock address
+ bl sxlkUnlock ; Release host pmap search lock
+ bt++ pf64Bitb,gfmEpi64 ; Take 64-bit exit
+ b gfmEpi32 ; Take 32-bit exit
-/*
- * Log out all memory usage
- */
+
+;
+; Guest shadow assist -- change guest page protection
+;
+; Locates the specified dormant mapping, and if it is active, changes its
+; protection.
+;
+; Parameters:
+; r3 : address of guest pmap, 32-bit kernel virtual address
+; r4 : guest virtual address, high-order 32 bits
+; r5 : guest virtual address, low-order 32 bits
+; r6 : guest mapping protection code
+;
+; Non-volatile register usage:
+; r25 : caller's msr image from mapSetUp
+; r26 : guest mapping protection code
+; r27 : host pmap physical address
+; r28 : guest pmap physical address
+; r29 : VMM extension block's physical address
+; r30 : guest virtual address
+; r31 : gva->phys mapping's physical address
+;
+ .align 5
+ .globl EXT(hw_protect_gv)
+
+LEXT(hw_protect_gv)
+
+#define gcpStackSize ((31-24+1)*4)+4
+
+ stwu r1,-(FM_ALIGN(gcpStackSize)+FM_SIZE)(r1)
+ ; Mint a new stack frame
+ mflr r0 ; Get caller's return address
+ mfsprg r11,2 ; Get feature flags
+ mtcrf 0x02,r11 ; Insert feature flags into cr6
+ stw r0,(FM_ALIGN(gcpStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Save caller's return address
+ stw r31,FM_ARG0+0x00(r1) ; Save non-volatile r31
+ stw r30,FM_ARG0+0x04(r1) ; Save non-volatile r30
+ stw r29,FM_ARG0+0x08(r1) ; Save non-volatile r29
+ stw r28,FM_ARG0+0x0C(r1) ; Save non-volatile r28
+ stw r27,FM_ARG0+0x10(r1) ; Save non-volatile r27
+ stw r26,FM_ARG0+0x14(r1) ; Save non-volatile r26
+ stw r25,FM_ARG0+0x18(r1) ; Save non-volatile r25
+
+ rlwinm r30,r5,0,0xFFFFF000 ; Clean up low-order 32 bits of guest vaddr
+ mr r26,r6 ; Copy guest mapping protection code
+
+ lwz r11,pmapVmmExt(r3) ; r11 <- VMM pmap extension block vaddr
+ lwz r9,pmapSpace(r3) ; r9 <- guest space ID number
+ bt++ pf64Bitb,gcp64Salt ; Handle 64-bit machine separately
+ lwz r29,pmapVmmExtPhys+4(r3) ; r29 <- VMM pmap extension block paddr
+ lwz r27,vmxHostPmapPhys+4(r11) ; r27 <- host pmap paddr
+ lwz r28,pmapvr+4(r3) ; Get 32-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ lwz r31,4(r31) ; r31 <- hash page paddr
+ rlwimi r31,r11,GV_HGRP_SHIFT,GV_HGRP_MASK
+ ; r31 <- hash group paddr
+ b gcpStart ; Get to it
+
+gcp64Salt: rldimi r30,r4,32,0 ; Insert high-order 32 bits of 64-bit guest vaddr
+ ld r29,pmapVmmExtPhys(r3) ; r29 <- VMM pmap extension block paddr
+ ld r27,vmxHostPmapPhys(r11) ; r27 <- host pmap paddr
+ ld r28,pmapvr(r3) ; Get 64-bit virt<->real guest pmap conversion salt
+ la r31,VMX_HPIDX_OFFSET(r11) ; r31 <- base of hash page physical index
+ srwi r11,r30,12 ; Form shadow hash:
+ xor r11,r11,r9 ; spaceID ^ (vaddr >> 12)
+ rlwinm r10,r11,GV_HPAGE_SHIFT,GV_HPAGE_MASK
+ ; Form index offset from hash page number
+ add r31,r31,r10 ; r31 <- hash page index entry
+ ld r31,0(r31) ; r31 <- hash page paddr
+ insrdi r31,r11,GV_GRPS_PPG_LG2,64-(GV_HGRP_SHIFT+GV_GRPS_PPG_LG2)
+ ; r31 <- hash group paddr
+
+gcpStart: xor r28,r4,r28 ; Convert guest pmap_t virt->real
+ bl EXT(mapSetUp) ; Disable 'rupts, translation, maybe enter 64-bit mode
+ mr r25,r11 ; Save caller's msr image
+
+ la r3,pmapSXlk(r27) ; r3 <- host pmap's search lock address
+ bl sxlkExclusive ; Get lock exclusive
+
+ li r0,(GV_SLOTS - 1) ; Prepare to iterate over mapping slots
+ mtctr r0 ; in this group
+ bt++ pf64Bitb,gcp64Search ; Test for 64-bit machine
+
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ lwz r5,mpVAddr+4(r31) ; r5 <- 1st mapping slot's virtual address
+ b gcp32SrchLp ; Let the search begin!
+
+ .align 5
+gcp32SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrwi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ lwz r5,mpVAddr+4+GV_SLOT_SZ(r31); r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- free || dormant || !space match
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gcpSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gcp32SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrwi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- free || dormant || !space match
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gcpSrchHit ; Join common path on hit (r31 points to guest mapping)
+ b gcpSrchMiss ; No joy in our hash group
+
+gcp64Search:
+ lwz r3,mpFlags(r31) ; r3 <- 1st mapping slot's flags
+ lhz r4,mpSpace(r31) ; r4 <- 1st mapping slot's space ID
+ ld r5,mpVAddr(r31) ; r5 <- 1st mapping slot's virtual address
+ b gcp64SrchLp ; Let the search begin!
+
+ .align 5
+gcp64SrchLp:
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ lwz r3,mpFlags+GV_SLOT_SZ(r31) ; r3 <- next mapping slot's flags
+ mr r7,r4 ; r7 <- current mapping slot's space ID
+ lhz r4,mpSpace+GV_SLOT_SZ(r31) ; r4 <- next mapping slot's space ID
+ clrrdi r8,r5,12 ; r8 <- current mapping slot's virtual addr w/o flags
+ ld r5,mpVAddr+GV_SLOT_SZ(r31) ; r5 <- next mapping slot's virtual addr
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free flag
+ xor r7,r7,r9 ; Compare space ID
+ or r0,r11,r7 ; r0 <- free || dormant || !space match
+ xor r8,r8,r30 ; Compare virtual address
+ or. r0,r0,r8 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ beq gcpSrchHit ; Join common path on hit (r31 points to guest mapping)
+
+ addi r31,r31,GV_SLOT_SZ ; r31 <- next mapping slot
+ bdnz gcp64SrchLp ; Iterate
+
+ mr r6,r3 ; r6 <- current mapping slot's flags
+ clrrdi r5,r5,12 ; Remove flags from virtual address
+ andi. r11,r6,mpgFree+mpgDormant ; Isolate guest free flag
+ xor r4,r4,r9 ; Compare space ID
+ or r0,r11,r4 ; r0 <- free || dormant || !space match
+ xor r5,r5,r30 ; Compare virtual address
+ or. r0,r0,r5 ; cr0_eq <- !free && !dormant && space match && virtual addr match
+ bne gcpSrchMiss ; No joy in our hash group
+
+gcpSrchHit:
+ bt++ pf64Bitb,gcpDscon64 ; Handle 64-bit disconnect separately
+ bl mapInvPte32 ; Disconnect PTE, invalidate, gather ref and change
+ ; r31 <- mapping's physical address
+ ; r3 -> PTE slot physical address
+ ; r4 -> High-order 32 bits of PTE
+ ; r5 -> Low-order 32 bits of PTE
+ ; r6 -> PCA
+ ; r7 -> PCA physical address
+ rlwinm r2,r3,29,29,31 ; Get PTE's slot number in the PTEG (8-byte PTEs)
+ b gcpFreePTE ; Join 64-bit path to release the PTE
+gcpDscon64: bl mapInvPte64 ; Disconnect PTE, invalidate, gather ref and change
+ rlwinm r2,r3,28,29,31 ; Get PTE's slot number in the PTEG (16-byte PTEs)
+gcpFreePTE: mr. r3,r3 ; Was there a valid PTE?
+ beq- gcpSetKey ; No valid PTE, we're almost done
+ lis r0,0x8000 ; Prepare free bit for this slot
+ srw r0,r0,r2 ; Position free bit
+ or r6,r6,r0 ; Set it in our PCA image
+ lwz r8,mpPte(r31) ; Get PTE pointer
+ rlwinm r8,r8,0,~mpHValid ; Make the pointer invalid
+ stw r8,mpPte(r31) ; Save invalidated PTE pointer
+ eieio ; Synchronize all previous updates (mapInvPtexx didn't)
+ stw r6,0(r7) ; Update PCA and unlock the PTEG
+
+gcpSetKey: lwz r0,mpVAddr+4(r31) ; Get va word containing protection bits
+ rlwimi r0,r26,0,mpPP ; Insert new protection bits
+ stw r0,mpVAddr+4(r31) ; Write 'em back
+ eieio ; Ensure previous mapping updates are visible
+ li r31,mapRtOK ; I'm a success
+
+gcpRelPmap: la r3,pmapSXlk(r27) ; r3 <- host pmap search lock phys addr
+ bl sxlkUnlock ; Release host pmap search lock
+
+ mr r3,r31 ; r3 <- result code
+ bt++ pf64Bitb,gcpRtn64 ; Handle 64-bit separately
+ mtmsr r25 ; Restore 'rupts, translation
+ isync ; Throw a small wrench into the pipeline
+ b gcpPopFrame ; Nothing to do now but pop a frame and return
+gcpRtn64: mtmsrd r25 ; Restore 'rupts, translation, 32-bit mode
+gcpPopFrame:
+ lwz r0,(FM_ALIGN(gcpStackSize)+FM_SIZE+FM_LR_SAVE)(r1)
+ ; Get caller's return address
+ lwz r31,FM_ARG0+0x00(r1) ; Restore non-volatile r31
+ lwz r30,FM_ARG0+0x04(r1) ; Restore non-volatile r30
+ lwz r29,FM_ARG0+0x08(r1) ; Restore non-volatile r29
+ lwz r28,FM_ARG0+0x0C(r1) ; Restore non-volatile r28
+ mtlr r0 ; Prepare return address
+ lwz r27,FM_ARG0+0x10(r1) ; Restore non-volatile r27
+ lwz r26,FM_ARG0+0x14(r1) ; Restore non-volatile r26
+ lwz r25,FM_ARG0+0x18(r1) ; Restore non-volatile r25
+ lwz r1,0(r1) ; Pop stack frame
+ blr ; Return to caller
.align 5
- .globl EXT(logmem)
+gcpSrchMiss:
+ li r31,mapRtNotFnd ; Could not locate requested mapping
+ b gcpRelPmap ; Exit through host pmap search lock release
-LEXT(logmem)
- mfmsr r2 ; Get the MSR
- lis r10,hi16(EXT(DebugWork)) ; High part of area
- rlwinm r2,r2,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- lis r12,hi16(EXT(mem_actual)) ; High part of actual
- rlwinm r2,r2,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- andi. r0,r2,0x7FCF ; Interrupts and translation off
- ori r10,r10,lo16(EXT(DebugWork)) ; Get the entry
- mtmsr r0 ; Turn stuff off
- ori r12,r12,lo16(EXT(mem_actual)) ; Get the actual
- li r0,1 ; Get a one
+;
+; Find the physent based on a physical page and try to lock it (but not too hard)
+; Note that this table always has an entry that with a 0 table pointer at the end
+;
+; R3 contains ppnum on entry
+; R3 is 0 if no entry was found
+; R3 is physent if found
+; cr0_eq is true if lock was obtained or there was no entry to lock
+; cr0_eq is false of there was an entry and it was locked
+;
+
+ .align 5
+
+mapFindPhyTry:
+ lis r9,hi16(EXT(pmap_mem_regions)) ; Point to the start of the region table
+ mr r2,r3 ; Save our target
+ ori r9,r9,lo16(EXT(pmap_mem_regions)) ; Point to the start of the region table
+
+mapFindPhz: lwz r3,mrPhysTab(r9) ; Get the actual table address
+ lwz r5,mrStart(r9) ; Get start of table entry
+ lwz r0,mrEnd(r9) ; Get end of table entry
+ addi r9,r9,mrSize ; Point to the next slot
+ cmplwi cr2,r3,0 ; Are we at the end of the table?
+ cmplw r2,r5 ; See if we are in this table
+ cmplw cr1,r2,r0 ; Check end also
+ sub r4,r2,r5 ; Calculate index to physical entry
+ beq-- cr2,mapFindNo ; Leave if we did not find an entry...
+ cror cr0_lt,cr0_lt,cr1_gt ; Set CR0_LT if it is NOT this entry
+ slwi r4,r4,3 ; Get offset to physical entry
+
+ blt-- mapFindPhz ; Did not find it...
+
+ add r3,r3,r4 ; Point right to the slot
+mapFindOv: lwz r2,0(r3) ; Get the lock contents right now
+ rlwinm. r0,r2,0,0,0 ; Is it locked?
+ bnelr-- ; Yes it is...
+
+ lwarx r2,0,r3 ; Get the lock
+ rlwinm. r0,r2,0,0,0 ; Is it locked?
+ oris r0,r2,0x8000 ; Set the lock bit
+ bne-- mapFindKl ; It is locked, go get rid of reservation and leave...
+ stwcx. r0,0,r3 ; Try to stuff it back...
+ bne-- mapFindOv ; Collision, try again...
+ isync ; Clear any speculations
+ blr ; Leave...
+
+mapFindKl: li r2,lgKillResv ; Killing field
+ stwcx. r2,0,r2 ; Trash reservation...
+ crclr cr0_eq ; Make sure we do not think we got the lock
+ blr ; Leave...
+
+mapFindNo: crset cr0_eq ; Make sure that we set this
+ li r3,0 ; Show that we did not find it
+ blr ; Leave...
+;
+; pmapCacheLookup - This function will look up an entry in the pmap segment cache.
+;
+; How the pmap cache lookup works:
+;
+; We use a combination of three things: a mask of valid entries, a sub-tag, and the
+; ESID (aka the "tag"). The mask indicates which of the cache slots actually contain
+; an entry. The sub-tag is a 16 entry 4 bit array that contains the low order 4 bits
+; of the ESID, bits 32:36 of the effective for 64-bit and 0:3 for 32-bit. The cache
+; entry contains the full 36 bit ESID.
+;
+; The purpose of the sub-tag is to limit the number of searches necessary when looking
+; for an existing cache entry. Because there are 16 slots in the cache, we could end up
+; searching all 16 if an match is not found.
+;
+; Essentially, we will search only the slots that have a valid entry and whose sub-tag
+; matches. More than likely, we will eliminate almost all of the searches.
+;
+; Inputs:
+; R3 = pmap
+; R4 = ESID high half
+; R5 = ESID low half
+;
+; Outputs:
+; R3 = pmap cache slot if found, 0 if not
+; R10 = pmapCCtl address
+; R11 = pmapCCtl image
+; pmapCCtl locked on exit
+;
+
+ .align 5
+
+pmapCacheLookup:
+ la r10,pmapCCtl(r3) ; Point to the segment cache control
+
+pmapCacheLookuq:
+ lwarx r11,0,r10 ; Get the segment cache control value
+ rlwinm. r0,r11,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
+ ori r0,r11,lo16(pmapCCtlLck) ; Turn on the lock bit
+ bne-- pmapCacheLookur ; Nope...
+ stwcx. r0,0,r10 ; Try to take the lock
+ bne-- pmapCacheLookuq ; Someone else just stuffed it, try again...
+
+ isync ; Make sure we get reservation first
+ lwz r9,pmapSCSubTag(r3) ; Get the high part of the sub-tag
+ rlwimi r5,r5,28,4,7 ; Copy sub-tag just to right of itself (XX------)
+ lwz r10,pmapSCSubTag+4(r3) ; And the bottom half
+ rlwimi r5,r5,24,8,15 ; Copy doubled sub-tag to right of itself (XXXX----)
+ lis r8,0x8888 ; Get some eights
+ rlwimi r5,r5,16,16,31 ; Copy quadrupled sub-tags to the right
+ ori r8,r8,0x8888 ; Fill the rest with eights
+
+ eqv r10,r10,r5 ; Get 0xF where we hit in bottom half
+ eqv r9,r9,r5 ; Get 0xF where we hit in top half
+
+ rlwinm r2,r10,1,0,30 ; Shift over 1
+ rlwinm r0,r9,1,0,30 ; Shift over 1
+ and r2,r2,r10 ; AND the even/odd pair into the even
+ and r0,r0,r9 ; AND the even/odd pair into the even
+ rlwinm r10,r2,2,0,28 ; Shift over 2
+ rlwinm r9,r0,2,0,28 ; Shift over 2
+ and r10,r2,r10 ; AND the even of the ANDed pairs giving the AND of all 4 bits in 0, 4, ...
+ and r9,r0,r9 ; AND the even of the ANDed pairs giving the AND of all 4 bits in 0, 4, ...
+
+ and r10,r10,r8 ; Clear out extras
+ and r9,r9,r8 ; Clear out extras
+
+ rlwinm r0,r10,3,1,28 ; Slide adjacent next to each other
+ rlwinm r2,r9,3,1,28 ; Slide adjacent next to each other
+ or r10,r0,r10 ; Merge them
+ or r9,r2,r9 ; Merge them
+ rlwinm r0,r10,6,2,26 ; Slide adjacent pairs next to each other
+ rlwinm r2,r9,6,2,26 ; Slide adjacent pairs next to each other
+ or r10,r0,r10 ; Merge them
+ or r9,r2,r9 ; Merge them
+ rlwimi r10,r10,12,4,7 ; Stick in the low-order adjacent quad
+ rlwimi r9,r9,12,4,7 ; Stick in the low-order adjacent quad
+ not r6,r11 ; Turn invalid into valid
+ rlwimi r9,r10,24,8,15 ; Merge in the adjacent octs giving a hit mask
+
+ la r10,pmapSegCache(r3) ; Point at the cache slots
+ and. r6,r9,r6 ; Get mask of valid and hit
+ li r0,0 ; Clear
+ li r3,0 ; Assume not found
+ oris r0,r0,0x8000 ; Start a mask
+ beqlr++ ; Leave, should usually be no hits...
+
+pclNextEnt: cntlzw r5,r6 ; Find an in use one
+ cmplwi cr1,r5,pmapSegCacheUse ; Did we find one?
+ rlwinm r7,r5,4,0,27 ; Index to the cache entry
+ srw r2,r0,r5 ; Get validity mask bit
+ add r7,r7,r10 ; Point to the cache slot
+ andc r6,r6,r2 ; Clear the validity bit we just tried
+ bgelr-- cr1 ; Leave if there are no more to check...
+
+ lwz r5,sgcESID(r7) ; Get the top half
+
+ cmplw r5,r4 ; Only need to check top because sub-tag is the entire other half
+
+ bne++ pclNextEnt ; Nope, try again...
+
+ mr r3,r7 ; Point to the slot
+ blr ; Leave....
+
+ .align 5
+
+pmapCacheLookur:
+ li r11,lgKillResv ; The killing spot
+ stwcx. r11,0,r11 ; Kill the reservation
+
+pmapCacheLookus:
+ lwz r11,pmapCCtl(r3) ; Get the segment cache control
+ rlwinm. r0,r11,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
+ beq++ pmapCacheLookup ; Nope...
+ b pmapCacheLookus ; Yup, keep waiting...
+
+
+;
+; mapMergeRC -- Given a physical mapping address in R31, locate its
+; connected PTE (if any) and merge the PTE referenced and changed bits
+; into the mapping and physent.
+;
+
+ .align 5
+
+mapMergeRC32:
+ lwz r0,mpPte(r31) ; Grab the PTE offset
+ mfsdr1 r7 ; Get the pointer to the hash table
+ lwz r5,mpVAddr+4(r31) ; Grab the virtual address
+ rlwinm r10,r7,0,0,15 ; Clean up the hash table base
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ srwi r7,r0,4 ; Convert to PCA units
+ rlwinm r7,r7,0,0,29 ; Clean up PCA offset
+ mflr r2 ; Save the return
+ subfic r7,r7,-4 ; Convert to -4 based negative index
+ add r7,r10,r7 ; Point to the PCA directly
+ beqlr-- ; There was no PTE to start with...
+
+ bl mapLockPteg ; Lock the PTEG
+
+ lwz r0,mpPte(r31) ; Grab the PTE offset
+ mtlr r2 ; Restore the LR
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ beq- mMPUnlock ; There is no PTE, someone took it so just unlock and leave...
+
+ rlwinm r3,r0,0,0,30 ; Clear the valid bit
+ add r3,r3,r10 ; Point to actual PTE
+ lwz r5,4(r3) ; Get the real part of the PTE
+ srwi r10,r5,12 ; Change physical address to a ppnum
+
+mMNmerge: lbz r11,mpFlags+1(r31) ; Get the offset to the physical entry table
+ lwz r0,mpVAddr+4(r31) ; Get the flags part of the field
+ lis r8,hi16(EXT(pmap_mem_regions)) ; Get the top of the region table
+ ori r8,r8,lo16(EXT(pmap_mem_regions)) ; Get the bottom of the region table
+ rlwinm r11,r11,2,24,29 ; Mask index bits and convert to byte offset
+ add r11,r11,r8 ; Point to the bank table
+ lwz r2,mrPhysTab(r11) ; Get the physical table bank pointer
+ lwz r11,mrStart(r11) ; Get the start of bank
+ rlwimi r0,r5,0,mpRb-32,mpCb-32 ; Copy in the RC
+ addi r2,r2,4 ; Offset to last half of field
+ stw r0,mpVAddr+4(r31) ; Set the new RC into the field
+ sub r11,r10,r11 ; Get the index into the table
+ rlwinm r11,r11,3,0,28 ; Get offset to the physent
+
+mMmrgRC: lwarx r10,r11,r2 ; Get the master RC
+ rlwinm r0,r5,27,ppRb-32,ppCb-32 ; Position the new RC
+ or r0,r0,r10 ; Merge in the new RC
+ stwcx. r0,r11,r2 ; Try to stick it back
+ bne-- mMmrgRC ; Try again if we collided...
+ eieio ; Commit all updates
+
+mMPUnlock:
+ stw r6,0(r7) ; Unlock PTEG
+ blr ; Return
+
+;
+; 64-bit version of mapMergeRC
+;
+ .align 5
+
+mapMergeRC64:
+ lwz r0,mpPte(r31) ; Grab the PTE offset
+ ld r5,mpVAddr(r31) ; Grab the virtual address
+ mfsdr1 r7 ; Get the pointer to the hash table
+ rldicr r10,r7,0,45 ; Clean up the hash table base
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ srdi r7,r0,5 ; Convert to PCA units
+ rldicr r7,r7,0,61 ; Clean up PCA
+ subfic r7,r7,-4 ; Convert to -4 based negative index
+ mflr r2 ; Save the return
+ add r7,r10,r7 ; Point to the PCA directly
+ beqlr-- ; There was no PTE to start with...
+
+ bl mapLockPteg ; Lock the PTEG
+
+ lwz r0,mpPte(r31) ; Grab the PTE offset again
+ mtlr r2 ; Restore the LR
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ beq-- mMPUnlock ; There is no PTE, someone took it so just unlock and leave...
+
+ rlwinm r3,r0,0,0,30 ; Clear the valid bit
+ add r3,r3,r10 ; Point to the actual PTE
+ ld r5,8(r3) ; Get the real part
+ srdi r10,r5,12 ; Change physical address to a ppnum
+ b mMNmerge ; Join the common 32-64-bit code...
+
+
+;
+; This routine, given a mapping, will find and lock the PTEG
+; If mpPte does not point to a PTE (checked before and after lock), it will unlock the
+; PTEG and return. In this case we will have undefined in R4
+; and the low 12 bits of mpVAddr valid in R5. R3 will contain 0.
+;
+; If the mapping is still valid, we will invalidate the PTE and merge
+; the RC bits into the physent and also save them into the mapping.
+;
+; We then return with R3 pointing to the PTE slot, R4 is the
+; top of the PTE and R5 is the bottom. R6 contains the PCA.
+; R7 points to the PCA entry.
+;
+; Note that we should NEVER be called on a block or special mapping.
+; We could do many bad things.
+;
+
+ .align 5
+
+mapInvPte32:
+ lwz r0,mpPte(r31) ; Grab the PTE offset
+ mfsdr1 r7 ; Get the pointer to the hash table
+ lwz r5,mpVAddr+4(r31) ; Grab the virtual address
+ rlwinm r10,r7,0,0,15 ; Clean up the hash table base
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ srwi r7,r0,4 ; Convert to PCA units
+ rlwinm r7,r7,0,0,29 ; Clean up PCA offset
+ mflr r2 ; Save the return
+ subfic r7,r7,-4 ; Convert to -4 based negative index
+ add r7,r10,r7 ; Point to the PCA directly
+ beqlr-- ; There was no PTE to start with...
+
+ bl mapLockPteg ; Lock the PTEG
+
+ lwz r0,mpPte(r31) ; Grab the PTE offset
+ mtlr r2 ; Restore the LR
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ beq- mIPUnlock ; There is no PTE, someone took it so just unlock and leave...
+
+ rlwinm r3,r0,0,0,30 ; Clear the valid bit
+ add r3,r3,r10 ; Point to actual PTE
+ lwz r4,0(r3) ; Get the top of the PTE
+
+ li r8,tlbieLock ; Get the TLBIE lock
+ rlwinm r0,r4,0,1,31 ; Clear the valid bit
+ stw r0,0(r3) ; Invalidate the PTE
+
+ sync ; Make sure everyone sees the invalidate
+
+mITLBIE32: lwarx r0,0,r8 ; Get the TLBIE lock
+ mfsprg r2,2 ; Get feature flags
+ mr. r0,r0 ; Is it locked?
+ li r0,1 ; Get our lock word
+ bne- mITLBIE32 ; It is locked, go wait...
+
+ stwcx. r0,0,r8 ; Try to get it
+ bne- mITLBIE32 ; We was beat...
+
+ rlwinm. r0,r2,0,pfSMPcapb,pfSMPcapb ; Can this be an MP box?
+ li r0,0 ; Lock clear value
+
+ tlbie r5 ; Invalidate it everywhere
+
+ beq- mINoTS32 ; Can not have MP on this machine...
+
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; Wait for everyone to catch up
+ sync ; Make sure of it all
+
+mINoTS32: stw r0,tlbieLock(0) ; Clear the tlbie lock
+ lwz r5,4(r3) ; Get the real part
+ srwi r10,r5,12 ; Change physical address to a ppnum
+
+mINmerge: lbz r11,mpFlags+1(r31) ; Get the offset to the physical entry table
+ lwz r0,mpVAddr+4(r31) ; Get the flags part of the field
+ lis r8,hi16(EXT(pmap_mem_regions)) ; Get the top of the region table
+ ori r8,r8,lo16(EXT(pmap_mem_regions)) ; Get the bottom of the region table
+ rlwinm r11,r11,2,24,29 ; Mask index bits and convert to byte offset
+ add r11,r11,r8 ; Point to the bank table
+ lwz r2,mrPhysTab(r11) ; Get the physical table bank pointer
+ lwz r11,mrStart(r11) ; Get the start of bank
+ rlwimi r0,r5,0,mpRb-32,mpCb-32 ; Copy in the RC
+ addi r2,r2,4 ; Offset to last half of field
+ stw r0,mpVAddr+4(r31) ; Set the new RC into the field
+ sub r11,r10,r11 ; Get the index into the table
+ rlwinm r11,r11,3,0,28 ; Get offset to the physent
+
+
+mImrgRC: lwarx r10,r11,r2 ; Get the master RC
+ rlwinm r0,r5,27,ppRb-32,ppCb-32 ; Position the new RC
+ or r0,r0,r10 ; Merge in the new RC
+ stwcx. r0,r11,r2 ; Try to stick it back
+ bne-- mImrgRC ; Try again if we collided...
+
+ blr ; Leave with the PCA still locked up...
+
+mIPUnlock: eieio ; Make sure all updates come first
+
+ stw r6,0(r7) ; Unlock
+ blr
+
+;
+; 64-bit version
+;
+ .align 5
+
+mapInvPte64:
+ lwz r0,mpPte(r31) ; Grab the PTE offset
+ ld r5,mpVAddr(r31) ; Grab the virtual address
+ mfsdr1 r7 ; Get the pointer to the hash table
+ rldicr r10,r7,0,45 ; Clean up the hash table base
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ srdi r7,r0,5 ; Convert to PCA units
+ rldicr r7,r7,0,61 ; Clean up PCA
+ subfic r7,r7,-4 ; Convert to -4 based negative index
+ mflr r2 ; Save the return
+ add r7,r10,r7 ; Point to the PCA directly
+ beqlr-- ; There was no PTE to start with...
+
+ bl mapLockPteg ; Lock the PTEG
+
+ lwz r0,mpPte(r31) ; Grab the PTE offset again
+ mtlr r2 ; Restore the LR
+ andi. r3,r0,mpHValid ; Is there a possible PTE?
+ beq-- mIPUnlock ; There is no PTE, someone took it so just unlock and leave...
+
+ rlwinm r3,r0,0,0,30 ; Clear the valid bit
+ add r3,r3,r10 ; Point to the actual PTE
+ ld r4,0(r3) ; Get the top of the PTE
+
+ li r8,tlbieLock ; Get the TLBIE lock
+ rldicr r0,r4,0,62 ; Clear the valid bit
+ std r0,0(r3) ; Invalidate the PTE
+
+ rldicr r2,r4,16,35 ; Shift the AVPN over to match VPN
+ sync ; Make sure everyone sees the invalidate
+ rldimi r2,r5,0,36 ; Cram in the page portion of the EA
+
+mITLBIE64: lwarx r0,0,r8 ; Get the TLBIE lock
+ mr. r0,r0 ; Is it locked?
+ li r0,1 ; Get our lock word
+ bne-- mITLBIE64a ; It is locked, toss reservation and wait...
+
+ stwcx. r0,0,r8 ; Try to get it
+ bne-- mITLBIE64 ; We was beat...
+
+ rldicl r2,r2,0,16 ; Clear bits 0:15 because we are under orders
+
+ li r0,0 ; Lock clear value
+
+ tlbie r2 ; Invalidate it everywhere
+
+ eieio ; Make sure that the tlbie happens first
+ tlbsync ; Wait for everyone to catch up
+ ptesync ; Wait for quiet again
+
+ stw r0,tlbieLock(0) ; Clear the tlbie lock
+
+ ld r5,8(r3) ; Get the real part
+ srdi r10,r5,12 ; Change physical address to a ppnum
+ b mINmerge ; Join the common 32-64-bit code...
+
+mITLBIE64a: li r5,lgKillResv ; Killing field
+ stwcx. r5,0,r5 ; Kill reservation
+
+mITLBIE64b: lwz r0,0(r8) ; Get the TLBIE lock
+ mr. r0,r0 ; Is it locked?
+ beq++ mITLBIE64 ; Nope, try again...
+ b mITLBIE64b ; Yup, wait for it...
+
+;
+; mapLockPteg - Locks a PTEG
+; R7 points to PCA entry
+; R6 contains PCA on return
+;
+;
+
+ .align 5
+
+mapLockPteg:
+ lwarx r6,0,r7 ; Pick up the PCA
+ rlwinm. r0,r6,0,PCAlockb,PCAlockb ; Is the PTEG locked?
+ ori r0,r6,PCAlock ; Set the lock bit
+ bne-- mLSkill ; It is locked...
+
+ stwcx. r0,0,r7 ; Try to lock the PTEG
+ bne-- mapLockPteg ; We collided...
+
+ isync ; Nostradamus lied
+ blr ; Leave...
+
+mLSkill: li r6,lgKillResv ; Get killing field
+ stwcx. r6,0,r6 ; Kill it
+
+mapLockPteh:
+ lwz r6,0(r7) ; Pick up the PCA
+ rlwinm. r0,r6,0,PCAlockb,PCAlockb ; Is the PTEG locked?
+ beq++ mapLockPteg ; Nope, try again...
+ b mapLockPteh ; Yes, wait for it...
+
+
+;
+; The mapSelSlot function selects a PTEG slot to use. As input, it expects R6
+; to contain the PCA. When it returns, R3 contains 0 if an unoccupied slot was
+; selected, 1 if it stole a non-block PTE, or 2 if it stole a block mapped PTE.
+; R4 returns the slot index.
+;
+; CR7 also indicates that we have a block mapping
+;
+; The PTEG allocation controls are a bit map of the state of the PTEG.
+; PCAfree indicates that the PTE slot is empty.
+; PCAauto means that it comes from an autogen area. These
+; guys do not keep track of reference and change and are actually "wired".
+; They are easy to maintain. PCAsteal
+; is a sliding position mask used to "randomize" PTE slot stealing. All 4 of these
+; fields fit in a single word and are loaded and stored under control of the
+; PTEG control area lock (PCAlock).
+;
+; Note that PCAauto does not contribute to the steal calculations at all. Originally
+; it did, autogens were second in priority. This can result in a pathalogical
+; case where an instruction can not make forward progress, or one PTE slot
+; thrashes.
+;
+; Note that the PCA must be locked when we get here.
+;
+; Physically, the fields are arranged:
+; 0: PCAfree
+; 1: PCAsteal
+; 2: PCAauto
+; 3: PCAmisc
+;
+;
+; At entry, R6 contains new unlocked PCA image (real PCA is locked and untouched)
+;
+; At exit:
+;
+; R3 = 0 - no steal
+; R3 = 1 - steal regular
+; R3 = 2 - steal autogen
+; R4 contains slot number
+; R6 contains updated PCA image
+;
+
+ .align 5
+
+mapSelSlot: lis r10,0 ; Clear autogen mask
+ li r9,0 ; Start a mask
+ beq cr7,mSSnotblk ; Skip if this is not a block mapping
+ ori r10,r10,lo16(0xFFFF) ; Make sure we mark a block mapping (autogen)
+
+mSSnotblk: rlwinm r11,r6,16,24,31 ; Isolate just the steal mask
+ oris r9,r9,0x8000 ; Get a mask
+ cntlzw r4,r6 ; Find a slot or steal one
+ ori r9,r9,lo16(0x8000) ; Insure that we have 0x80008000
+ rlwinm r4,r4,0,29,31 ; Isolate bit position
+ rlwimi r11,r11,8,16,23 ; Get set to march a 1 back into top of 8 bit rotate
+ srw r2,r9,r4 ; Get mask to isolate selected inuse and autogen flags
+ srwi r11,r11,1 ; Slide steal mask right
+ and r8,r6,r2 ; Isolate the old in use and autogen bits
+ andc r6,r6,r2 ; Allocate the slot and also clear autogen flag
+ addi r0,r8,0x7F00 ; Push autogen flag to bit 16
+ and r2,r2,r10 ; Keep the autogen part if autogen
+ addis r8,r8,0xFF00 ; Push in use to bit 0 and invert
+ or r6,r6,r2 ; Add in the new autogen bit
+ rlwinm r0,r0,17,31,31 ; Get a 1 if the old was autogenned (always 0 if not in use)
+ rlwinm r8,r8,1,31,31 ; Isolate old in use
+ rlwimi r6,r11,16,8,15 ; Stick the new steal slot in
+
+ add r3,r0,r8 ; Get 0 if no steal, 1 if steal normal, 2 if steal autogen
+ blr ; Leave...
+
+;
+; Shared/Exclusive locks
+;
+; A shared/exclusive lock allows multiple shares of a lock to be taken
+; but only one exclusive. A shared lock can be "promoted" to exclusive
+; when it is the only share. If there are multiple sharers, the lock
+; must be "converted". A promotion drops the share and gains exclusive as
+; an atomic operation. If anyone else has a share, the operation fails.
+; A conversion first drops the share and then takes an exclusive lock.
+;
+; We will want to add a timeout to this eventually.
+;
+; R3 is set to 0 for success, non-zero for failure
+;
+
+;
+; Convert a share into an exclusive
+;
+
+ .align 5
+
+sxlkConvert:
+
+ lis r0,0x8000 ; Get the locked lock image
+#if 0
+ mflr r0 ; (TEST/DEBUG)
+ oris r0,r0,0x8000 ; (TEST/DEBUG)
+#endif
+
+sxlkCTry: lwarx r2,0,r3 ; Get the lock word
+ cmplwi r2,1 ; Does it just have our share?
+ subi r2,r2,1 ; Drop our share in case we do not get it
+ bne-- sxlkCnotfree ; No, we need to unlock...
+ stwcx. r0,0,r3 ; Try to take it exclusively
+ bne-- sxlkCTry ; Collision, try again...
+
+ isync
+ li r3,0 ; Set RC
+ blr ; Leave...
+
+sxlkCnotfree:
+ stwcx. r2,0,r3 ; Try to drop our share...
+ bne-- sxlkCTry ; Try again if we collided...
+ b sxlkExclusive ; Go take it exclusively...
+
+;
+; Promote shared to exclusive
+;
+
+ .align 5
+
+sxlkPromote:
+ lis r0,0x8000 ; Get the locked lock image
+#if 0
+ mflr r0 ; (TEST/DEBUG)
+ oris r0,r0,0x8000 ; (TEST/DEBUG)
+#endif
+
+sxlkPTry: lwarx r2,0,r3 ; Get the lock word
+ cmplwi r2,1 ; Does it just have our share?
+ bne-- sxlkPkill ; No, just fail (R3 is non-zero)...
+ stwcx. r0,0,r3 ; Try to take it exclusively
+ bne-- sxlkPTry ; Collision, try again...
+
+ isync
+ li r3,0 ; Set RC
+ blr ; Leave...
+
+sxlkPkill: li r2,lgKillResv ; Point to killing field
+ stwcx. r2,0,r2 ; Kill reservation
+ blr ; Leave
+
+
+
+;
+; Take lock exclusivily
+;
+
+ .align 5
+
+sxlkExclusive:
+ lis r0,0x8000 ; Get the locked lock image
+#if 0
+ mflr r0 ; (TEST/DEBUG)
+ oris r0,r0,0x8000 ; (TEST/DEBUG)
+#endif
+
+sxlkXTry: lwarx r2,0,r3 ; Get the lock word
+ mr. r2,r2 ; Is it locked?
+ bne-- sxlkXWait ; Yes...
+ stwcx. r0,0,r3 ; Try to take it
+ bne-- sxlkXTry ; Collision, try again...
+
+ isync ; Toss anything younger than us
+ li r3,0 ; Set RC
+ blr ; Leave...
+
+ .align 5
+
+sxlkXWait: li r2,lgKillResv ; Point to killing field
+ stwcx. r2,0,r2 ; Kill reservation
+
+sxlkXWaiu: lwz r2,0(r3) ; Get the lock again
+ mr. r2,r2 ; Is it free yet?
+ beq++ sxlkXTry ; Yup...
+ b sxlkXWaiu ; Hang around a bit more...
+
+;
+; Take a share of the lock
+;
+
+ .align 5
+
+sxlkShared: lwarx r2,0,r3 ; Get the lock word
+ rlwinm. r0,r2,0,0,0 ; Is it locked exclusively?
+ addi r2,r2,1 ; Up the share count
+ bne-- sxlkSWait ; Yes...
+ stwcx. r2,0,r3 ; Try to take it
+ bne-- sxlkShared ; Collision, try again...
+
+ isync ; Toss anything younger than us
+ li r3,0 ; Set RC
+ blr ; Leave...
+
+ .align 5
+
+sxlkSWait: li r2,lgKillResv ; Point to killing field
+ stwcx. r2,0,r2 ; Kill reservation
+
+sxlkSWaiu: lwz r2,0(r3) ; Get the lock again
+ rlwinm. r0,r2,0,0,0 ; Is it locked exclusively?
+ beq++ sxlkShared ; Nope...
+ b sxlkSWaiu ; Hang around a bit more...
+
+;
+; Unlock either exclusive or shared.
+;
+
+ .align 5
+
+sxlkUnlock: eieio ; Make sure we order our stores out
+
+sxlkUnTry: lwarx r2,0,r3 ; Get the lock
+ rlwinm. r0,r2,0,0,0 ; Do we hold it exclusively?
+ subi r2,r2,1 ; Remove our share if we have one
+ li r0,0 ; Clear this
+ bne-- sxlkUExclu ; We hold exclusive...
+
+ stwcx. r2,0,r3 ; Try to lose our share
+ bne-- sxlkUnTry ; Collision...
+ blr ; Leave...
+
+sxlkUExclu: stwcx. r0,0,r3 ; Unlock and release reservation
+ beqlr++ ; Leave if ok...
+ b sxlkUnTry ; Could not store, try over...
+
+
+ .align 5
+ .globl EXT(fillPage)
+
+LEXT(fillPage)
+
+ mfsprg r0,2 ; Get feature flags
+ mtcrf 0x02,r0 ; move pf64Bit to cr
+
+ rlwinm r4,r4,0,1,0 ; Copy fill to top of 64-bit register
+ lis r2,0x0200 ; Get vec
+ mr r6,r4 ; Copy
+ ori r2,r2,0x2000 ; Get FP
+ mr r7,r4 ; Copy
+ mfmsr r5 ; Get MSR
+ mr r8,r4 ; Copy
+ andc r5,r5,r2 ; Clear out permanent turn-offs
+ mr r9,r4 ; Copy
+ ori r2,r2,0x8030 ; Clear IR, DR and EE
+ mr r10,r4 ; Copy
+ andc r0,r5,r2 ; Kill them
+ mr r11,r4 ; Copy
+ mr r12,r4 ; Copy
+ bt++ pf64Bitb,fpSF1 ; skip if 64-bit (only they take the hint)
+
+ slwi r3,r3,12 ; Make into a physical address
+ mtmsr r2 ; Interrupts and translation off
+ isync
+
+ li r2,4096/32 ; Get number of cache lines
+
+fp32again: dcbz 0,r3 ; Clear
+ addic. r2,r2,-1 ; Count down
+ stw r4,0(r3) ; Fill
+ stw r6,4(r3) ; Fill
+ stw r7,8(r3) ; Fill
+ stw r8,12(r3) ; Fill
+ stw r9,16(r3) ; Fill
+ stw r10,20(r3) ; Fill
+ stw r11,24(r3) ; Fill
+ stw r12,28(r3) ; Fill
+ addi r3,r3,32 ; Point next
+ bgt+ fp32again ; Keep going
+
+ mtmsr r5 ; Restore all
isync
+ blr ; Return...
+
+ .align 5
+
+fpSF1: li r2,1
+ sldi r2,r2,63 ; Get 64-bit bit
+ or r0,r0,r2 ; Turn on 64-bit
+ sldi r3,r3,12 ; Make into a physical address
- stw r0,4(r10) ; Force logging off
- lwz r0,0(r12) ; Get the end of memory
-
- lis r12,hi16(EXT(mem_size)) ; High part of defined memory
- ori r12,r12,lo16(EXT(mem_size)) ; Low part of defined memory
- lwz r12,0(r12) ; Make it end of defined
-
- cmplw r0,r12 ; Is there room for the data?
- ble- logmemexit ; No, do not even try...
-
- stw r12,0(r12) ; Set defined memory size
- stw r0,4(r12) ; Set the actual amount of memory
-
- lis r3,hi16(EXT(hash_table_base)) ; Hash table address
- lis r4,hi16(EXT(hash_table_size)) ; Hash table size
- lis r5,hi16(EXT(pmap_mem_regions)) ; Memory regions
- lis r6,hi16(EXT(mapCtl)) ; Mappings
- ori r3,r3,lo16(EXT(hash_table_base))
- ori r4,r4,lo16(EXT(hash_table_size))
- ori r5,r5,lo16(EXT(pmap_mem_regions))
- ori r6,r6,lo16(EXT(mapCtl))
- lwz r3,0(r3)
- lwz r4,0(r4)
- lwz r5,4(r5) ; Get the pointer to the phys_ent table
- lwz r6,0(r6) ; Get the pointer to the current mapping block
- stw r3,8(r12) ; Save the hash table address
- stw r4,12(r12) ; Save the hash table size
- stw r5,16(r12) ; Save the physent pointer
- stw r6,20(r12) ; Save the mappings
-
- addi r11,r12,0x1000 ; Point to area to move hash table and PCA
-
- add r4,r4,r4 ; Double size for both
-
-copyhash: lwz r7,0(r3) ; Copy both of them
- lwz r8,4(r3)
- lwz r9,8(r3)
- lwz r10,12(r3)
- subic. r4,r4,0x10
- addi r3,r3,0x10
- stw r7,0(r11)
- stw r8,4(r11)
- stw r9,8(r11)
- stw r10,12(r11)
- addi r11,r11,0x10
- bgt+ copyhash
-
- rlwinm r4,r12,20,12,31 ; Get number of phys_ents
-
-copyphys: lwz r7,0(r5) ; Copy physents
- lwz r8,4(r5)
- subic. r4,r4,1
- addi r5,r5,8
- stw r7,0(r11)
- stw r8,4(r11)
- addi r11,r11,8
- bgt+ copyphys
-
- addi r11,r11,4095 ; Round up to next page
- rlwinm r11,r11,0,0,19
-
- lwz r4,4(r6) ; Get the size of the mapping area
-
-copymaps: lwz r7,0(r6) ; Copy the mappings
- lwz r8,4(r6)
- lwz r9,8(r6)
- lwz r10,12(r6)
- subic. r4,r4,0x10
- addi r6,r6,0x10
- stw r7,0(r11)
- stw r8,4(r11)
- stw r9,8(r11)
- stw r10,12(r11)
- addi r11,r11,0x10
- bgt+ copymaps
-
- sub r11,r11,r12 ; Get the total length we saved
- stw r11,24(r12) ; Save the size
-
-logmemexit: mtmsr r2 ; Back to normal
- li r3,0
+ mtmsrd r0 ; Interrupts and translation off
isync
+
+ li r2,4096/128 ; Get number of cache lines
+
+fp64again: dcbz128 0,r3 ; Clear
+ addic. r2,r2,-1 ; Count down
+ std r4,0(r3) ; Fill
+ std r6,8(r3) ; Fill
+ std r7,16(r3) ; Fill
+ std r8,24(r3) ; Fill
+ std r9,32(r3) ; Fill
+ std r10,40(r3) ; Fill
+ std r11,48(r3) ; Fill
+ std r12,56(r3) ; Fill
+ std r4,64+0(r3) ; Fill
+ std r6,64+8(r3) ; Fill
+ std r7,64+16(r3) ; Fill
+ std r8,64+24(r3) ; Fill
+ std r9,64+32(r3) ; Fill
+ std r10,64+40(r3) ; Fill
+ std r11,64+48(r3) ; Fill
+ std r12,64+56(r3) ; Fill
+ addi r3,r3,128 ; Point next
+ bgt+ fp64again ; Keep going
+
+ mtmsrd r5 ; Restore all
+ isync
+ blr ; Return...
+
+ .align 5
+ .globl EXT(mapLog)
+
+LEXT(mapLog)
+
+ mfmsr r12
+ lis r11,hi16(EXT(mapdebug))
+ ori r11,r11,lo16(EXT(mapdebug))
+ lwz r10,0(r11)
+ mr. r10,r10
+ bne++ mLxx
+ mr r10,r3
+mLxx: rlwinm r0,r12,0,MSR_DR_BIT+1,MSR_DR_BIT-1
+ mtmsr r0
+ isync
+ stw r4,0(r10)
+ stw r4,4(r10)
+ stw r5,8(r10)
+ stw r6,12(r10)
+ mtmsr r12
+ isync
+ addi r10,r10,16
+ stw r10,0(r11)
blr
+
+#if 1
+ .align 5
+ .globl EXT(checkBogus)
+
+LEXT(checkBogus)
+
+ BREAKPOINT_TRAP
+ blr ; No-op normally
+
+#endif
+
+
projects / apple / xnu.git / blobdiff