2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
25 #include <db_machine_commands.h>
28 #include <mach_debug.h>
30 #include <ppc/proc_reg.h>
31 #include <ppc/exception.h>
32 #include <ppc/Performance.h>
33 #include <ppc/exception.h>
34 #include <ppc/pmap_internals.h>
35 #include <mach/ppc/vm_param.h>
42 * Random notes and musings...
44 * Access to mappings via the PTEG hash must be done with the list locked.
45 * Access via the physical entries is controlled by the physent lock.
46 * Access to mappings is controlled by the PTEG lock once they are queued.
47 * If they are not on the list, they don't really exist, so
48 * only one processor at a time can find them, so no access control is needed.
50 * The second half of the PTE is kept in the physical entry. It is done this
51 * way, because there may be multiple mappings that refer to the same physical
52 * page (i.e., address aliases or synonymns). We must do it this way, because
53 * maintenance of the reference and change bits becomes nightmarish if each mapping
54 * has its own. One side effect of this, and not necessarily a bad one, is that
55 * all mappings for a single page can have a single WIMG, protection state, and RC bits.
56 * The only "bad" thing, is the reference bit. With a single copy, we can not get
57 * a completely accurate working set calculation, i.e., we can't tell which mapping was
58 * used to reference the page, all we can tell is that the physical page was
61 * The master copys of the reference and change bits are kept in the phys_entry.
62 * Other than the reference and change bits, changes to the phys_entry are not
63 * allowed if it has any mappings. The master reference and change bits must be
64 * changed via atomic update.
66 * Invalidating a PTE merges the RC bits into the phys_entry.
68 * Before checking the reference and/or bits, ALL mappings to the physical page are
71 * PTEs are never explicitly validated, they are always faulted in. They are also
72 * not visible outside of the hw_vm modules. Complete seperation of church and state.
74 * Removal of a mapping is invalidates its PTE.
76 * So, how do we deal with mappings to I/O space? We don't have a physent for it.
77 * Within the mapping is a copy of the second half of the PTE. This is used
78 * ONLY when there is no physical entry. It is swapped into the PTE whenever
79 * it is built. There is no need to swap it back out, because RC is not
80 * maintained for these mappings.
82 * So, I'm starting to get concerned about the number of lwarx/stcwx loops in
83 * this. Satisfying a mapped address with no stealing requires one lock. If we
84 * steal an entry, there's two locks and an atomic update. Invalidation of an entry
85 * takes one lock and, if there is a PTE, another lock and an atomic update. Other
86 * operations are multiples (per mapping) of the above. Maybe we should look for
87 * an alternative. So far, I haven't found one, but I haven't looked hard.
91 /* hw_add_map(struct mapping *mp, space_t space, vm_offset_t va) - Adds a mapping
93 * Adds a mapping to the PTEG hash list.
95 * Interrupts must be disabled before calling.
97 * Using the space and the virtual address, we hash into the hash table
98 * and get a lock on the PTEG hash chain. Then we chain the
99 * mapping to the front of the list.
104 .globl EXT(hw_add_map)
108 #if PERFTIMES && DEBUG
112 bl EXT(dbgLog2) ; Start of hw_add_map
117 mfmsr r0 /* Get the MSR */
118 eqv r6,r6,r6 /* Fill the bottom with foxes */
119 rlwinm r11,r4,6,6,25 /* Position the space for the VSID */
120 mfspr r10,sdr1 /* Get hash table base and size */
121 rlwimi r11,r5,30,2,5 /* Insert the segment no. to make a VSID */
122 mfsprg r12,2 ; Get feature flags
123 rlwimi r6,r10,16,0,15 /* Make table size -1 out of mask */
124 rlwinm r7,r5,26,10,25 /* Isolate the page index */
125 or r8,r10,r6 /* Point to the last byte in table */
126 rlwinm r9,r5,4,0,3 ; Move nybble 1 up to 0
127 xor r7,r7,r11 /* Get primary hash */
128 mtcrf 0x04,r12 ; Set the features
129 andi. r12,r0,0x7FCF /* Disable translation and interruptions */
130 rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
131 addi r8,r8,1 /* Point to the PTEG Control Area */
132 xor r9,r9,r5 ; Splooch vaddr nybble 0 and 1 together
133 and r7,r7,r6 /* Wrap the hash */
134 rlwimi r11,r5,10,26,31 /* Move API into pte ID */
135 rlwinm r9,r9,6,27,29 ; Get splooched bits in place
136 add r8,r8,r7 /* Point to our PCA entry */
137 rlwinm r10,r4,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
139 bt pfNoMSRirb,hamNoMSR ; No MSR...
141 mtmsr r12 ; Translation and all off
142 isync ; Toss prefetch
145 hamNoMSR: mr r4,r0 ; Save R0
147 li r0,loadMSR ; Get the MSR setter SC
148 mr r3,r12 ; Get new MSR
154 la r4,PCAhash(r8) /* Point to the mapping hash area */
155 xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
156 isync /* Get rid of anything prefetched before we ref storage */
158 * We've now got the address of our PCA, the hash chain anchor, our API subhash,
159 * and word 0 of the PTE (the virtual part).
161 * Now, we just lock the PCA.
164 li r12,1 /* Get the locked value */
165 dcbt 0,r4 /* We'll need the hash area in a sec, so get it */
166 add r4,r4,r9 /* Point to the right mapping hash slot */
170 ptegLckx: lwarx r10,0,r8 /* Get the PTEG lock */
171 mr. r10,r10 /* Is it locked? */
172 bne- ptegLckwx /* Yeah... */
173 stwcx. r12,0,r8 /* Take take it */
174 bne- ptegLckx /* Someone else was trying, try again... */
175 b ptegSXgx /* All done... */
179 ptegLckwx: mr. r10,r10 /* Check if it's already held */
180 beq+ ptegLckx /* It's clear... */
181 lwz r10,0(r8) /* Get lock word again... */
182 b ptegLckwx /* Wait... */
186 ptegSXgx: isync /* Make sure we haven't used anything yet */
188 lwz r7,0(r4) /* Pick up the anchor of hash list */
189 stw r3,0(r4) /* Save the new head */
190 stw r7,mmhashnext(r3) /* Chain in the old head */
192 stw r4,mmPTEhash(r3) /* Point to the head of the hash list */
194 sync /* Make sure the chain is updated */
195 stw r10,0(r8) /* Unlock the hash list */
196 mtmsr r0 /* Restore translation and interruptions */
197 isync /* Toss anything done with DAT off */
198 #if PERFTIMES && DEBUG
202 bl EXT(dbgLog2) ; end of hw_add_map
209 /* mp=hw_lock_phys_vir(space, va) - Finds and locks a physical entry by vaddr.
211 * Returns the mapping with the associated physent locked if found, or a
212 * zero and no lock if not. It we timed out trying to get a the lock on
213 * the physical entry, we retun a 1. A physical entry can never be on an
214 * odd boundary, so we can distinguish between a mapping and a timeout code.
216 * Interrupts must be disabled before calling.
218 * Using the space and the virtual address, we hash into the hash table
219 * and get a lock on the PTEG hash chain. Then we search the chain for the
220 * mapping for our virtual address. From there, we extract the pointer to
221 * the physical entry.
223 * Next comes a bit of monkey business. we need to get a lock on the physical
224 * entry. But, according to our rules, we can't get it after we've gotten the
225 * PTEG hash lock, we could deadlock if we do. So, we need to release the
226 * hash lock. The problem is, though, that as soon as we release it, some
227 * other yahoo may remove our mapping between the time that we release the
228 * hash lock and obtain the phys entry lock. So, we can't count on the
229 * mapping once we release the lock. Instead, after we lock the phys entry,
230 * we search the mapping list (phys_link) for our translation. If we don't find it,
231 * we unlock the phys entry, bail out, and return a 0 for the mapping address. If we
232 * did find it, we keep the lock and return the address of the mapping block.
234 * What happens when a mapping is found, but there is no physical entry?
235 * This is what happens when there is I/O area mapped. It one of these mappings
236 * is found, the mapping is returned, as is usual for this call, but we don't
237 * try to lock anything. There could possibly be some problems here if another
238 * processor releases the mapping while we still alre using it. Hope this
239 * ain't gonna happen.
241 * Taaa-dahhh! Easy as pie, huh?
243 * So, we have a few hacks hacks for running translate off in here.
244 * First, when we call the lock routine, we have carnel knowlege of the registers is uses.
245 * That way, we don't need a stack frame, which we can't have 'cause the stack is in
246 * virtual storage. But wait, as if that's not enough... We need one more register. So,
247 * we cram the LR into the CTR and return from there.
251 .globl EXT(hw_lock_phys_vir)
253 LEXT(hw_lock_phys_vir)
255 #if PERFTIMES && DEBUG
259 bl EXT(dbgLog2) ; Start of hw_add_map
263 mfmsr r12 /* Get the MSR */
264 eqv r6,r6,r6 /* Fill the bottom with foxes */
265 mfsprg r9,2 ; Get feature flags
266 rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
267 mfspr r5,sdr1 /* Get hash table base and size */
268 rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
269 mtcrf 0x04,r9 ; Set the features
270 rlwimi r6,r5,16,0,15 /* Make table size -1 out of mask */
271 andi. r0,r12,0x7FCF /* Disable translation and interruptions */
272 rlwinm r9,r4,4,0,3 ; Move nybble 1 up to 0
273 rlwinm r7,r4,26,10,25 /* Isolate the page index */
274 or r8,r5,r6 /* Point to the last byte in table */
275 xor r7,r7,r11 /* Get primary hash */
276 rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
277 addi r8,r8,1 /* Point to the PTEG Control Area */
278 xor r9,r9,r4 ; Splooch vaddr nybble 0 and 1 together
279 and r7,r7,r6 /* Wrap the hash */
280 rlwimi r11,r4,10,26,31 /* Move API into pte ID */
281 rlwinm r9,r9,6,27,29 ; Get splooched bits in place
282 add r8,r8,r7 /* Point to our PCA entry */
283 rlwinm r10,r3,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
285 bt pfNoMSRirb,hlpNoMSR ; No MSR...
287 mtmsr r0 ; Translation and all off
288 isync ; Toss prefetch
291 hlpNoMSR: mr r3,r0 ; Get the new MSR
292 li r0,loadMSR ; Get the MSR setter SC
296 la r3,PCAhash(r8) /* Point to the mapping hash area */
297 xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
298 isync /* Make sure translation is off before we ref storage */
301 * We've now got the address of our PCA, the hash chain anchor, our API subhash,
302 * and word 0 of the PTE (the virtual part).
304 * Now, we just lock the PCA and find our mapping, if it exists.
307 dcbt 0,r3 /* We'll need the hash area in a sec, so get it */
308 add r3,r3,r9 /* Point to the right mapping hash slot */
312 ptegLcka: lwarx r10,0,r8 /* Get the PTEG lock */
313 li r5,1 /* Get the locked value */
314 mr. r10,r10 /* Is it locked? */
315 bne- ptegLckwa /* Yeah... */
316 stwcx. r5,0,r8 /* Take take it */
317 bne- ptegLcka /* Someone else was trying, try again... */
318 b ptegSXga /* All done... */
322 ptegLckwa: mr. r10,r10 /* Check if it's already held */
323 beq+ ptegLcka /* It's clear... */
324 lwz r10,0(r8) /* Get lock word again... */
325 b ptegLckwa /* Wait... */
329 ptegSXga: isync /* Make sure we haven't used anything yet */
331 mflr r0 /* Get the LR */
332 lwz r9,0(r3) /* Pick up the first mapping block */
333 mtctr r0 /* Stuff it into the CTR */
337 mr. r3,r9 /* Did we hit the end? */
338 bne+ chkmapa /* Nope... */
340 stw r3,0(r8) /* Unlock the PTEG lock
341 Note: we never saved anything while we
342 had the lock, so we don't need a sync
343 before we unlock it */
345 vbail: mtmsr r12 /* Restore translation and interruptions */
346 isync /* Make sure translation is cool */
347 #if PERFTIMES && DEBUG
351 bl EXT(dbgLog2) ; Start of hw_add_map
355 bctr /* Return in abject failure... */
359 chkmapa: lwz r10,mmPTEv(r3) /* Pick up our virtual ID */
360 lwz r9,mmhashnext(r3) /* Pick up next mapping block */
361 cmplw r10,r11 /* Have we found ourself? */
362 bne- findmapa /* Nope, still wandering... */
364 lwz r9,mmphysent(r3) /* Get our physical entry pointer */
365 li r5,0 /* Clear this out */
366 mr. r9,r9 /* Is there, like, a physical entry? */
367 stw r5,0(r8) /* Unlock the PTEG lock
368 Note: we never saved anything while we
369 had the lock, so we don't need a sync
370 before we unlock it */
372 beq- vbail /* If there is no physical entry, it's time
375 /* Here we want to call hw_lock_bit. We don't want to use the stack, 'cause it's
376 * in virtual storage, and we're in real. So, we've carefully looked at the code
377 * in hw_lock_bit (and unlock) and cleverly don't use any of the registers that it uses.
378 * Be very, very aware of how you change this code. By the way, it uses:
379 * R0, R6, R7, R8, and R9. R3, R4, and R5 contain parameters
380 * Unfortunatly, we need to stash R9 still. So... Since we know we will not be interrupted
381 * ('cause we turned off interruptions and translation is off) we will use SPRG3...
384 lwz r10,mmPTEhash(r3) /* Save the head of the hash-alike chain. We need it to find ourselves later */
385 lis r5,HIGH_ADDR(EXT(LockTimeOut)) /* Get address of timeout value */
386 la r3,pephyslink(r9) /* Point to the lock word */
387 ori r5,r5,LOW_ADDR(EXT(LockTimeOut)) /* Get second half of address */
388 li r4,PHYS_LOCK /* Get the lock bit value */
389 lwz r5,0(r5) /* Pick up the timeout value */
390 mtsprg 3,r9 /* Save R9 in SPRG3 */
392 bl EXT(hw_lock_bit) /* Go do the lock */
394 mfsprg r9,3 /* Restore pointer to the phys_entry */
395 mr. r3,r3 /* Did we timeout? */
396 lwz r4,pephyslink(r9) /* Pick up first mapping block */
397 beq- penterr /* Bad deal, we timed out... */
399 rlwinm r4,r4,0,0,26 ; Clear out the flags from first link
401 findmapb: mr. r3,r4 /* Did we hit the end? */
402 bne+ chkmapb /* Nope... */
404 la r3,pephyslink(r9) /* Point to where the lock is */
405 li r4,PHYS_LOCK /* Get the lock bit value */
406 bl EXT(hw_unlock_bit) /* Go unlock the physentry */
408 li r3,0 /* Say we failed */
409 b vbail /* Return in abject failure... */
411 penterr: li r3,1 /* Set timeout */
412 b vbail /* Return in abject failure... */
416 chkmapb: lwz r6,mmPTEv(r3) /* Pick up our virtual ID */
417 lwz r4,mmnext(r3) /* Pick up next mapping block */
418 cmplw r6,r11 /* Have we found ourself? */
419 lwz r5,mmPTEhash(r3) /* Get the start of our hash chain */
420 bne- findmapb /* Nope, still wandering... */
421 cmplw r5,r10 /* On the same hash chain? */
422 bne- findmapb /* Nope, keep looking... */
424 b vbail /* Return in glorious triumph... */
428 * hw_rem_map(mapping) - remove a mapping from the system.
430 * Upon entry, R3 contains a pointer to a mapping block and the associated
431 * physical entry is locked if there is one.
433 * If the mapping entry indicates that there is a PTE entry, we invalidate
434 * if and merge the reference and change information into the phys_entry.
436 * Next, we remove the mapping from the phys_ent and the PTEG hash list.
438 * Unlock any locks that are left, and exit.
440 * Note that this must be done with both interruptions off and VM off
442 * Note that this code depends upon the VSID being of the format 00SXXXXX
443 * where S is the segment number.
449 .globl EXT(hw_rem_map)
452 #if PERFTIMES && DEBUG
456 bl EXT(dbgLog2) ; Start of hw_add_map
460 mfsprg r9,2 ; Get feature flags
461 mfmsr r0 /* Save the MSR */
462 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
463 mtcrf 0x04,r9 ; Set the features
464 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
466 bt pfNoMSRirb,lmvNoMSR ; No MSR...
468 mtmsr r12 ; Translation and all off
469 isync ; Toss prefetch
475 li r0,loadMSR ; Get the MSR setter SC
476 mr r3,r12 ; Get new MSR
484 lwz r6,mmPTEhash(r3) /* Get pointer to hash list anchor */
485 lwz r5,mmPTEv(r3) /* Get the VSID */
486 dcbt 0,r6 /* We'll need that chain in a bit */
488 rlwinm r7,r6,0,0,25 /* Round hash list down to PCA boundary */
489 li r12,1 /* Get the locked value */
490 subi r6,r6,mmhashnext /* Make the anchor look like an entry */
494 ptegLck1: lwarx r10,0,r7 /* Get the PTEG lock */
495 mr. r10,r10 /* Is it locked? */
496 bne- ptegLckw1 /* Yeah... */
497 stwcx. r12,0,r7 /* Try to take it */
498 bne- ptegLck1 /* Someone else was trying, try again... */
499 b ptegSXg1 /* All done... */
503 ptegLckw1: mr. r10,r10 /* Check if it's already held */
504 beq+ ptegLck1 /* It's clear... */
505 lwz r10,0(r7) /* Get lock word again... */
506 b ptegLckw1 /* Wait... */
510 ptegSXg1: isync /* Make sure we haven't used anything yet */
512 lwz r12,mmhashnext(r3) /* Prime with our forward pointer */
513 lwz r4,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
515 srchmaps: mr. r10,r6 /* Save the previous entry */
516 bne+ mapok /* No error... */
518 lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
519 ori r0,r0,LOW_ADDR(Choke)
520 sc /* Firmware Heimlich manuever */
524 mapok: lwz r6,mmhashnext(r6) /* Look at the next one */
525 cmplwi cr5,r4,0 /* Is there a PTE? */
526 cmplw r6,r3 /* Have we found ourselves? */
527 bne+ srchmaps /* Nope, get your head together... */
529 stw r12,mmhashnext(r10) /* Remove us from the queue */
530 rlwinm r9,r5,1,0,3 /* Move in the segment */
531 rlwinm r8,r4,6,4,19 /* Line PTEG disp up to a page */
532 rlwinm r11,r5,5,4,19 /* Line up the VSID */
533 lwz r10,mmphysent(r3) /* Point to the physical entry */
535 beq+ cr5,nopte /* There's no PTE to invalidate... */
537 xor r8,r8,r11 /* Back hash to virt index */
538 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
539 rlwimi r9,r5,22,4,9 /* Move in the API */
540 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
541 mfspr r11,pvr /* Find out what kind of machine we are */
542 rlwimi r9,r8,0,10,19 /* Create the virtual address */
543 rlwinm r11,r11,16,16,31 /* Isolate CPU type */
545 stw r5,0(r4) /* Make the PTE invalid */
547 cmplwi cr1,r11,3 /* Is this a 603? */
548 sync /* Make sure the invalid is stored */
552 tlbhang1: lwarx r5,0,r12 /* Get the TLBIE lock */
553 rlwinm r11,r4,29,29,31 /* Get the bit position of entry */
554 mr. r5,r5 /* Is it locked? */
555 lis r6,0x8000 /* Start up a bit mask */
556 li r5,1 /* Get our lock word */
557 bne- tlbhang1 /* It's locked, go wait... */
558 stwcx. r5,0,r12 /* Try to get it */
559 bne- tlbhang1 /* We was beat... */
561 srw r6,r6,r11 /* Make a "free slot" mask */
562 lwz r5,PCAallo(r7) /* Get the allocation control bits */
563 rlwinm r11,r6,24,8,15 /* Make the autogen bit to turn off */
564 or r5,r5,r6 /* turn on the free bit */
565 rlwimi r11,r11,24,16,23 /* Get lock bit mask to turn it off */
567 andc r5,r5,r11 /* Turn off the lock and autogen bits in allocation flags */
568 li r11,0 /* Lock clear value */
570 tlbie r9 /* Invalidate it everywhere */
573 beq- cr1,its603a /* It's a 603, skip the tlbsync... */
575 eieio /* Make sure that the tlbie happens first */
576 tlbsync /* wait for everyone to catch up */
578 its603a: sync /* Make sure of it all */
579 stw r11,0(r12) /* Clear the tlbie lock */
580 eieio /* Make sure those RC bit are loaded */
581 stw r5,PCAallo(r7) /* Show that the slot is free */
582 stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
584 nopte: mr. r10,r10 /* See if there is a physical entry */
585 la r9,pephyslink(r10) /* Point to the physical mapping chain */
586 beq- nophys /* No physical entry, we're done... */
587 beq- cr5,nadamrg /* No PTE to merge... */
589 lwz r6,4(r4) /* Get the latest reference and change bits */
590 la r12,pepte1(r10) /* Point right at the master copy */
591 rlwinm r6,r6,0,23,24 /* Extract just the RC bits */
595 mrgrc: lwarx r8,0,r12 /* Get the master copy */
596 or r8,r8,r6 /* Merge in latest RC */
597 stwcx. r8,0,r12 /* Save it back */
598 bne- mrgrc /* If it changed, try again... */
600 nadamrg: li r11,0 /* Clear this out */
601 lwz r12,mmnext(r3) /* Prime with our next */
602 stw r11,0(r7) /* Unlock the hash chain now so we don't
603 lock out another processor during the
604 our next little search */
607 srchpmap: mr. r10,r9 /* Save the previous entry */
608 bne+ mapok1 /* No error... */
610 lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
611 ori r0,r0,LOW_ADDR(Choke)
612 sc /* Firmware Heimlich maneuver */
616 mapok1: lwz r9,mmnext(r9) /* Look at the next one */
617 rlwinm r8,r9,0,27,31 ; Save the flags (including the lock)
618 rlwinm r9,r9,0,0,26 ; Clear out the flags from first link
619 cmplw r9,r3 /* Have we found ourselves? */
620 bne+ srchpmap /* Nope, get your head together... */
622 rlwimi r12,r8,0,27,31 ; Insert the lock and flags */
623 stw r12,mmnext(r10) /* Remove us from the queue */
625 mtmsr r0 /* Interrupts and translation back on */
627 #if PERFTIMES && DEBUG
630 bl EXT(dbgLog2) ; Start of hw_add_map
637 nophys: li r4,0 /* Make sure this is 0 */
638 sync /* Make sure that chain is updated */
639 stw r4,0(r7) /* Unlock the hash chain */
640 mtmsr r0 /* Interrupts and translation back on */
642 #if PERFTIMES && DEBUG
645 bl EXT(dbgLog2) ; Start of hw_add_map
652 * hw_prot(physent, prot) - Change the protection of a physical page
654 * Upon entry, R3 contains a pointer to a physical entry which is locked.
655 * R4 contains the PPC protection bits.
657 * The first thing we do is to slam the new protection into the phys entry.
658 * Then we scan the mappings and process each one.
660 * Acquire the lock on the PTEG hash list for the mapping being processed.
662 * If the current mapping has a PTE entry, we invalidate
663 * it and merge the reference and change information into the phys_entry.
665 * Next, slam the protection bits into the entry and unlock the hash list.
667 * Note that this must be done with both interruptions off and VM off
676 #if PERFTIMES && DEBUG
682 bl EXT(dbgLog2) ; Start of hw_add_map
686 mfsprg r9,2 ; Get feature flags
687 mfmsr r0 /* Save the MSR */
688 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
689 li r5,pepte1 /* Get displacement to the second word of master pte */
690 mtcrf 0x04,r9 ; Set the features
691 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
693 bt pfNoMSRirb,hpNoMSR ; No MSR...
695 mtmsr r12 ; Translation and all off
696 isync ; Toss prefetch
702 li r0,loadMSR ; Get the MSR setter SC
703 mr r3,r12 ; Get new MSR
711 lwz r10,pephyslink(r3) /* Get the first mapping block */
712 rlwinm r10,r10,0,0,26 ; Clear out the flags from first link
715 * Note that we need to to do the interlocked update here because another processor
716 * can be updating the reference and change bits even though the physical entry
717 * is locked. All modifications to the PTE portion of the physical entry must be
718 * done via interlocked update.
723 protcng: lwarx r8,r5,r3 /* Get the master copy */
724 rlwimi r8,r4,0,30,31 /* Move in the protection bits */
725 stwcx. r8,r5,r3 /* Save it back */
726 bne- protcng /* If it changed, try again... */
730 protnext: mr. r10,r10 /* Are there any more mappings? */
731 beq- protdone /* Naw... */
733 lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
734 lwz r5,mmPTEv(r10) /* Get the virtual address */
735 rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
737 li r12,1 /* Get the locked value */
741 protLck1: lwarx r11,0,r7 /* Get the PTEG lock */
742 mr. r11,r11 /* Is it locked? */
743 bne- protLckw1 /* Yeah... */
744 stwcx. r12,0,r7 /* Try to take it */
745 bne- protLck1 /* Someone else was trying, try again... */
746 b protSXg1 /* All done... */
750 protLckw1: mr. r11,r11 /* Check if it's already held */
751 beq+ protLck1 /* It's clear... */
752 lwz r11,0(r7) /* Get lock word again... */
753 b protLckw1 /* Wait... */
757 protSXg1: isync /* Make sure we haven't used anything yet */
759 lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
761 rlwinm r9,r5,1,0,3 /* Move in the segment */
762 lwz r2,mmPTEr(r10) ; Get the mapping copy of the PTE
763 mr. r6,r6 /* See if there is a PTE here */
764 rlwinm r8,r5,31,2,25 /* Line it up */
765 rlwimi r2,r4,0,30,31 ; Move protection bits into the mapping copy
767 beq+ protul /* There's no PTE to invalidate... */
769 xor r8,r8,r6 /* Back hash to virt index */
770 rlwimi r9,r5,22,4,9 /* Move in the API */
771 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
772 rlwinm r5,r5,0,1,31 /* Clear the valid bit */
773 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
774 mfspr r11,pvr /* Find out what kind of machine we are */
775 rlwimi r9,r8,6,10,19 /* Create the virtual address */
776 rlwinm r11,r11,16,16,31 /* Isolate CPU type */
778 stw r5,0(r6) /* Make the PTE invalid */
779 cmplwi cr1,r11,3 /* Is this a 603? */
780 sync /* Make sure the invalid is stored */
784 tlbhangp: lwarx r11,0,r12 /* Get the TLBIE lock */
785 rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
786 mr. r11,r11 /* Is it locked? */
787 lis r5,0x8000 /* Start up a bit mask */
788 li r11,1 /* Get our lock word */
789 bne- tlbhangp /* It's locked, go wait... */
790 stwcx. r11,0,r12 /* Try to get it */
791 bne- tlbhangp /* We was beat... */
793 li r11,0 /* Lock clear value */
795 tlbie r9 /* Invalidate it everywhere */
797 beq- cr1,its603p /* It's a 603, skip the tlbsync... */
799 eieio /* Make sure that the tlbie happens first */
800 tlbsync /* wait for everyone to catch up */
802 its603p: stw r11,0(r12) /* Clear the lock */
803 srw r5,r5,r8 /* Make a "free slot" mask */
804 sync /* Make sure of it all */
806 lwz r6,4(r6) /* Get the latest reference and change bits */
807 stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
808 rlwinm r6,r6,0,23,24 /* Extract the RC bits */
809 lwz r9,PCAallo(r7) /* Get the allocation control bits */
810 rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
811 rlwimi r2,r6,0,23,24 ; Put the latest RC bit in mapping copy
812 or r9,r9,r5 /* Set the slot free */
813 rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
814 andc r9,r9,r8 /* Clear the auto and lock bits */
815 li r5,pepte1 /* Get displacement to the second word of master pte */
816 stw r9,PCAallo(r7) /* Store the allocation controls */
819 protmod: lwarx r11,r5,r3 /* Get the master copy */
820 or r11,r11,r6 /* Merge in latest RC */
821 stwcx. r11,r5,r3 /* Save it back */
822 bne- protmod /* If it changed, try again... */
824 sync /* Make sure that chain is updated */
826 protul: li r4,0 /* Get a 0 */
827 stw r2,mmPTEr(r10) ; Save the updated mapping PTE
828 lwz r10,mmnext(r10) /* Get the next */
829 stw r4,0(r7) /* Unlock the hash chain */
830 b protnext /* Go get the next one */
834 protdone: mtmsr r0 /* Interrupts and translation back on */
836 #if PERFTIMES && DEBUG
839 bl EXT(dbgLog2) ; Start of hw_add_map
846 * hw_prot_virt(mapping, prot) - Change the protection of single page
848 * Upon entry, R3 contains a pointer (real) to a mapping.
849 * R4 contains the PPC protection bits.
851 * Acquire the lock on the PTEG hash list for the mapping being processed.
853 * If the current mapping has a PTE entry, we invalidate
854 * it and merge the reference and change information into the phys_entry.
856 * Next, slam the protection bits into the entry, merge the RC bits,
857 * and unlock the hash list.
859 * Note that this must be done with both interruptions off and VM off
865 .globl EXT(hw_prot_virt)
868 #if PERFTIMES && DEBUG
874 bl EXT(dbgLog2) ; Start of hw_add_map
878 mfsprg r9,2 ; Get feature flags
879 mfmsr r0 /* Save the MSR */
880 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
881 mtcrf 0x04,r9 ; Set the features
882 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
884 bt pfNoMSRirb,hpvNoMSR ; No MSR...
886 mtmsr r12 ; Translation and all off
887 isync ; Toss prefetch
893 li r0,loadMSR ; Get the MSR setter SC
894 mr r3,r12 ; Get new MSR
903 * Note that we need to to do the interlocked update here because another processor
904 * can be updating the reference and change bits even though the physical entry
905 * is locked. All modifications to the PTE portion of the physical entry must be
906 * done via interlocked update.
909 lwz r7,mmPTEhash(r3) /* Get pointer to hash list anchor */
910 lwz r5,mmPTEv(r3) /* Get the virtual address */
911 rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
913 li r12,1 /* Get the locked value */
917 protvLck1: lwarx r11,0,r7 /* Get the PTEG lock */
918 mr. r11,r11 /* Is it locked? */
919 bne- protvLckw1 /* Yeah... */
920 stwcx. r12,0,r7 /* Try to take it */
921 bne- protvLck1 /* Someone else was trying, try again... */
922 b protvSXg1 /* All done... */
926 protvLckw1: mr. r11,r11 /* Check if it's already held */
927 beq+ protvLck1 /* It's clear... */
928 lwz r11,0(r7) /* Get lock word again... */
929 b protvLckw1 /* Wait... */
933 protvSXg1: isync /* Make sure we haven't used anything yet */
935 lwz r6,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
936 lwz r2,mmPTEr(r3) ; Get the mapping copy if the real part
938 rlwinm r9,r5,1,0,3 /* Move in the segment */
939 cmplwi cr7,r6,0 ; Any PTE to invalidate?
940 rlwimi r2,r4,0,30,31 ; Move in the new protection bits
941 rlwinm r8,r5,31,2,25 /* Line it up */
943 beq+ cr7,pvnophys /* There's no PTE to invalidate... */
945 xor r8,r8,r6 /* Back hash to virt index */
946 rlwimi r9,r5,22,4,9 /* Move in the API */
947 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
948 rlwinm r5,r5,0,1,31 /* Clear the valid bit */
949 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
950 mfspr r11,pvr /* Find out what kind of machine we are */
951 rlwimi r9,r8,6,10,19 /* Create the virtual address */
952 rlwinm r11,r11,16,16,31 /* Isolate CPU type */
954 stw r5,0(r6) /* Make the PTE invalid */
955 cmplwi cr1,r11,3 /* Is this a 603? */
956 sync /* Make sure the invalid is stored */
960 tlbhangpv: lwarx r11,0,r12 /* Get the TLBIE lock */
961 rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
962 mr. r11,r11 /* Is it locked? */
963 lis r5,0x8000 /* Start up a bit mask */
964 li r11,1 /* Get our lock word */
965 bne- tlbhangpv /* It's locked, go wait... */
966 stwcx. r11,0,r12 /* Try to get it */
967 bne- tlbhangpv /* We was beat... */
969 li r11,0 /* Lock clear value */
971 tlbie r9 /* Invalidate it everywhere */
973 beq- cr1,its603pv /* It's a 603, skip the tlbsync... */
975 eieio /* Make sure that the tlbie happens first */
976 tlbsync /* wait for everyone to catch up */
978 its603pv: stw r11,0(r12) /* Clear the lock */
979 srw r5,r5,r8 /* Make a "free slot" mask */
980 sync /* Make sure of it all */
982 lwz r6,4(r6) /* Get the latest reference and change bits */
983 stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
984 rlwinm r6,r6,0,23,24 /* Extract the RC bits */
985 lwz r9,PCAallo(r7) /* Get the allocation control bits */
986 rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
987 lwz r10,mmphysent(r3) ; Get any physical entry
988 or r9,r9,r5 /* Set the slot free */
989 rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
990 andc r9,r9,r8 /* Clear the auto and lock bits */
991 mr. r10,r10 ; Is there a physical entry?
992 li r5,pepte1 /* Get displacement to the second word of master pte */
993 stw r9,PCAallo(r7) /* Store the allocation controls */
994 rlwimi r2,r6,0,23,24 ; Stick in RC bits
995 beq- pvnophys ; No physical entry...
1000 protvmod: lwarx r11,r5,r10 /* Get the master copy */
1001 or r11,r11,r6 /* Merge in latest RC */
1002 stwcx. r11,r5,r10 /* Save it back */
1003 bne- protvmod /* If it changed, try again... */
1005 sync /* Make sure that chain is updated */
1007 pvnophys: li r4,0 /* Get a 0 */
1008 stw r2,mmPTEr(r3) ; Set the real part of the PTE
1009 stw r4,0(r7) /* Unlock the hash chain */
1010 mtmsr r0 ; Restore interrupts and translation
1013 #if PERFTIMES && DEBUG
1023 * hw_attr_virt(mapping, attr) - Change the attributes of single page
1025 * Upon entry, R3 contains a pointer (real) to a mapping.
1026 * R4 contains the WIMG bits.
1028 * Acquire the lock on the PTEG hash list for the mapping being processed.
1030 * If the current mapping has a PTE entry, we invalidate
1031 * it and merge the reference and change information into the phys_entry.
1033 * Next, slam the WIMG bits into the entry, merge the RC bits,
1034 * and unlock the hash list.
1036 * Note that this must be done with both interruptions off and VM off
1042 .globl EXT(hw_attr_virt)
1045 #if PERFTIMES && DEBUG
1051 bl EXT(dbgLog2) ; Start of hw_add_map
1055 mfsprg r9,2 ; Get feature flags
1056 mfmsr r0 /* Save the MSR */
1057 mtcrf 0x04,r9 ; Set the features
1058 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1059 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
1061 bt pfNoMSRirb,havNoMSR ; No MSR...
1063 mtmsr r12 ; Translation and all off
1064 isync ; Toss prefetch
1070 li r0,loadMSR ; Get the MSR setter SC
1071 mr r3,r12 ; Get new MSR
1078 * Note that we need to to do the interlocked update here because another processor
1079 * can be updating the reference and change bits even though the physical entry
1080 * is locked. All modifications to the PTE portion of the physical entry must be
1081 * done via interlocked update.
1084 lwz r7,mmPTEhash(r3) /* Get pointer to hash list anchor */
1085 lwz r5,mmPTEv(r3) /* Get the virtual address */
1086 rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
1088 li r12,1 /* Get the locked value */
1092 attrvLck1: lwarx r11,0,r7 /* Get the PTEG lock */
1093 mr. r11,r11 /* Is it locked? */
1094 bne- attrvLckw1 /* Yeah... */
1095 stwcx. r12,0,r7 /* Try to take it */
1096 bne- attrvLck1 /* Someone else was trying, try again... */
1097 b attrvSXg1 /* All done... */
1101 attrvLckw1: mr. r11,r11 /* Check if it's already held */
1102 beq+ attrvLck1 /* It's clear... */
1103 lwz r11,0(r7) /* Get lock word again... */
1104 b attrvLckw1 /* Wait... */
1108 attrvSXg1: isync /* Make sure we haven't used anything yet */
1110 lwz r6,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
1111 lwz r2,mmPTEr(r3) ; Get the mapping copy if the real part
1113 rlwinm r9,r5,1,0,3 /* Move in the segment */
1114 mr. r6,r6 /* See if there is a PTE here */
1115 rlwimi r2,r4,0,25,28 ; Move in the new attribute bits
1116 rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
1118 beq+ avnophys /* There's no PTE to invalidate... */
1120 xor r8,r8,r6 /* Back hash to virt index */
1121 rlwimi r9,r5,22,4,9 /* Move in the API */
1122 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
1123 rlwinm r5,r5,0,1,31 /* Clear the valid bit */
1124 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
1125 mfspr r11,pvr /* Find out what kind of machine we are */
1126 rlwimi r9,r8,6,10,19 /* Create the virtual address */
1127 rlwinm r11,r11,16,16,31 /* Isolate CPU type */
1128 stw r5,0(r6) /* Make the PTE invalid */
1129 cmplwi cr1,r11,3 /* Is this a 603? */
1130 sync /* Make sure the invalid is stored */
1134 tlbhangav: lwarx r11,0,r12 /* Get the TLBIE lock */
1135 rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
1136 mr. r11,r11 /* Is it locked? */
1137 lis r5,0x8000 /* Start up a bit mask */
1138 li r11,1 /* Get our lock word */
1139 bne- tlbhangav /* It's locked, go wait... */
1140 stwcx. r11,0,r12 /* Try to get it */
1141 bne- tlbhangav /* We was beat... */
1143 li r11,0 /* Lock clear value */
1145 tlbie r9 /* Invalidate it everywhere */
1147 beq- cr1,its603av /* It's a 603, skip the tlbsync... */
1149 eieio /* Make sure that the tlbie happens first */
1150 tlbsync /* wait for everyone to catch up */
1152 its603av: stw r11,0(r12) /* Clear the lock */
1153 srw r5,r5,r8 /* Make a "free slot" mask */
1154 sync /* Make sure of it all */
1156 lwz r6,4(r6) /* Get the latest reference and change bits */
1157 stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
1158 rlwinm r6,r6,0,23,24 /* Extract the RC bits */
1159 lwz r9,PCAallo(r7) /* Get the allocation control bits */
1160 rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
1161 lwz r10,mmphysent(r3) ; Get any physical entry
1162 or r9,r9,r5 /* Set the slot free */
1163 rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
1164 andc r9,r9,r8 /* Clear the auto and lock bits */
1165 mr. r10,r10 ; Is there a physical entry?
1166 li r5,pepte1 /* Get displacement to the second word of master pte */
1167 stw r9,PCAallo(r7) /* Store the allocation controls */
1168 rlwimi r2,r6,0,23,24 ; Stick in RC bits
1169 beq- avnophys ; No physical entry...
1171 lwarx r11,r5,r10 ; ?
1173 attrvmod: lwarx r11,r5,r10 /* Get the master copy */
1174 or r11,r11,r6 /* Merge in latest RC */
1175 stwcx. r11,r5,r10 /* Save it back */
1176 bne- attrvmod /* If it changed, try again... */
1178 sync /* Make sure that chain is updated */
1180 avnophys: li r4,0 /* Get a 0 */
1181 stw r2,mmPTEr(r3) ; Set the real part of the PTE
1182 stw r4,0(r7) /* Unlock the hash chain */
1184 rlwinm r2,r2,0,0,19 ; Clear back to page boundary
1186 attrflsh: cmplwi r4,(4096-32) ; Are we about to do the last line on page?
1187 dcbst r2,r4 ; Flush cache because we changed attributes
1188 addi r4,r4,32 ; Bump up cache
1189 blt+ attrflsh ; Do the whole page...
1193 attrimvl: cmplwi r4,(4096-32) ; Are we about to do the last line on page?
1194 dcbi r2,r4 ; Invalidate dcache because we changed attributes
1195 icbi r2,r4 ; Invalidate icache because we changed attributes
1196 icbi r2,r4 ; Invalidate icache because we changed attributes
1197 addi r4,r4,32 ; Bump up cache
1198 blt+ attrimvl ; Do the whole page...
1201 mtmsr r0 ; Restore interrupts and translation
1204 #if PERFTIMES && DEBUG
1214 * hw_pte_comm(physent) - Do something to the PTE pointing to a physical page
1216 * Upon entry, R3 contains a pointer to a physical entry which is locked.
1217 * Note that this must be done with both interruptions off and VM off
1219 * First, we set up CRs 5 and 7 to indicate which of the 7 calls this is.
1221 * Now we scan the mappings to invalidate any with an active PTE.
1223 * Acquire the lock on the PTEG hash list for the mapping being processed.
1225 * If the current mapping has a PTE entry, we invalidate
1226 * it and merge the reference and change information into the phys_entry.
1228 * Next, unlock the hash list and go on to the next mapping.
1235 .globl EXT(hw_inv_all)
1239 li r9,0x800 /* Indicate invalidate all */
1240 li r2,0 ; No inadvertant modifications please
1241 b hw_pte_comm /* Join in the fun... */
1245 .globl EXT(hw_tst_mod)
1249 lwz r8,pepte1(r3) ; Get the saved PTE image
1250 li r9,0x400 /* Indicate test modify */
1251 li r2,0 ; No inadvertant modifications please
1252 rlwinm. r8,r8,25,31,31 ; Make change bit into return code
1253 beq+ hw_pte_comm ; Assume we do not know if it is set...
1254 mr r3,r8 ; Set the return code
1255 blr ; Return quickly...
1258 .globl EXT(hw_tst_ref)
1261 lwz r8,pepte1(r3) ; Get the saved PTE image
1262 li r9,0x200 /* Indicate test reference bit */
1263 li r2,0 ; No inadvertant modifications please
1264 rlwinm. r8,r8,24,31,31 ; Make reference bit into return code
1265 beq+ hw_pte_comm ; Assume we do not know if it is set...
1266 mr r3,r8 ; Set the return code
1267 blr ; Return quickly...
1270 * Note that the following are all in one CR for ease of use later
1273 .globl EXT(hw_set_mod)
1277 li r9,0x008 /* Indicate set modify bit */
1278 li r2,0x4 ; Set set C, clear none
1279 b hw_pte_comm /* Join in the fun... */
1283 .globl EXT(hw_clr_mod)
1287 li r9,0x004 /* Indicate clear modify bit */
1288 li r2,0x1 ; Set set none, clear C
1289 b hw_pte_comm /* Join in the fun... */
1293 .globl EXT(hw_set_ref)
1297 li r9,0x002 /* Indicate set reference */
1298 li r2,0x8 ; Set set R, clear none
1299 b hw_pte_comm /* Join in the fun... */
1302 .globl EXT(hw_clr_ref)
1306 li r9,0x001 /* Indicate clear reference bit */
1307 li r2,0x2 ; Set set none, clear R
1308 b hw_pte_comm /* Join in the fun... */
1312 * This is the common stuff.
1317 hw_pte_comm: /* Common routine for pte tests and manips */
1319 #if PERFTIMES && DEBUG
1325 bl EXT(dbgLog2) ; Start of hw_add_map
1329 mfsprg r8,2 ; Get feature flags
1330 lwz r10,pephyslink(r3) /* Get the first mapping block */
1331 mfmsr r0 /* Save the MSR */
1332 rlwinm. r10,r10,0,0,26 ; Clear out the flags from first link and see if we are mapped
1333 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1334 mtcrf 0x04,r8 ; Set the features
1335 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
1336 beq- comnmap ; No mapping
1337 dcbt br0,r10 ; Touch the first mapping in before the isync
1341 bt pfNoMSRirb,hpcNoMSR ; No MSR...
1343 mtmsr r12 ; Translation and all off
1344 isync ; Toss prefetch
1350 li r0,loadMSR ; Get the MSR setter SC
1351 mr r3,r12 ; Get new MSR
1357 mtcrf 0x05,r9 /* Set the call type flags into cr5 and 7 */
1359 beq- commdone ; Nothing us mapped to this page...
1360 b commnext ; Jump to first pass (jump here so we can align loop)
1364 commnext: lwz r11,mmnext(r10) ; Get the pointer to the next mapping (if any)
1365 lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
1366 lwz r5,mmPTEv(r10) /* Get the virtual address */
1367 mr. r11,r11 ; More mappings to go?
1368 rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
1369 beq- commnxtch ; No more mappings...
1370 dcbt br0,r11 ; Touch the next mapping
1372 commnxtch: li r12,1 /* Get the locked value */
1376 commLck1: lwarx r11,0,r7 /* Get the PTEG lock */
1377 mr. r11,r11 /* Is it locked? */
1378 bne- commLckw1 /* Yeah... */
1379 stwcx. r12,0,r7 /* Try to take it */
1380 bne- commLck1 /* Someone else was trying, try again... */
1381 b commSXg1 /* All done... */
1385 commLckw1: mr. r11,r11 /* Check if it's already held */
1386 beq+ commLck1 /* It's clear... */
1387 lwz r11,0(r7) /* Get lock word again... */
1388 b commLckw1 /* Wait... */
1392 commSXg1: isync /* Make sure we haven't used anything yet */
1394 lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
1396 rlwinm r9,r5,1,0,3 /* Move in the segment */
1397 mr. r6,r6 /* See if there is a PTE entry here */
1398 rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
1400 beq+ commul /* There's no PTE to invalidate... */
1402 xor r8,r8,r6 /* Back hash to virt index */
1403 rlwimi r9,r5,22,4,9 /* Move in the API */
1404 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
1405 rlwinm r5,r5,0,1,31 /* Clear the valid bit */
1406 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
1407 rlwimi r9,r8,6,10,19 /* Create the virtual address */
1409 stw r5,0(r6) /* Make the PTE invalid */
1410 mfspr r4,pvr /* Find out what kind of machine we are */
1411 sync /* Make sure the invalid is stored */
1415 tlbhangco: lwarx r11,0,r12 /* Get the TLBIE lock */
1416 rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
1417 mr. r11,r11 /* Is it locked? */
1418 lis r5,0x8000 /* Start up a bit mask */
1419 li r11,1 /* Get our lock word */
1420 bne- tlbhangco /* It's locked, go wait... */
1421 stwcx. r11,0,r12 /* Try to get it */
1422 bne- tlbhangco /* We was beat... */
1424 rlwinm r4,r4,16,16,31 /* Isolate CPU type */
1425 li r11,0 /* Lock clear value */
1426 cmplwi r4,3 /* Is this a 603? */
1428 tlbie r9 /* Invalidate it everywhere */
1430 beq- its603co /* It's a 603, skip the tlbsync... */
1432 eieio /* Make sure that the tlbie happens first */
1433 tlbsync /* wait for everyone to catch up */
1435 its603co: stw r11,0(r12) /* Clear the lock */
1436 srw r5,r5,r8 /* Make a "free slot" mask */
1437 sync /* Make sure of it all */
1439 lwz r6,4(r6) /* Get the latest reference and change bits */
1440 lwz r9,PCAallo(r7) /* Get the allocation control bits */
1441 stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
1442 rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
1443 or r9,r9,r5 /* Set the slot free */
1444 rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
1445 rlwinm r4,r6,0,23,24 /* Extract the RC bits */
1446 andc r9,r9,r8 /* Clear the auto and lock bits */
1447 li r5,pepte1 /* Get displacement to the second word of master pte */
1448 stw r9,PCAallo(r7) /* Store the allocation controls */
1451 commmod: lwarx r11,r5,r3 /* Get the master copy */
1452 or r11,r11,r4 /* Merge in latest RC */
1453 stwcx. r11,r5,r3 /* Save it back */
1454 bne- commmod /* If it changed, try again... */
1456 sync /* Make sure that chain is updated */
1457 b commulnl ; Skip loading the old real part...
1459 commul: lwz r6,mmPTEr(r10) ; Get the real part
1461 commulnl: rlwinm r12,r2,5,23,24 ; Get the "set" bits
1462 rlwinm r11,r2,7,23,24 ; Get the "clear" bits
1464 or r6,r6,r12 ; Set the bits to come on
1465 andc r6,r6,r11 ; Clear those to come off
1467 stw r6,mmPTEr(r10) ; Set the new RC
1469 lwz r10,mmnext(r10) /* Get the next */
1470 li r4,0 /* Make sure this is 0 */
1471 mr. r10,r10 ; Is there another mapping?
1472 stw r4,0(r7) /* Unlock the hash chain */
1473 bne+ commnext ; Go get the next if there is one...
1476 * Now that all PTEs have been invalidated and the master RC bits are updated,
1477 * we go ahead and figure out what the original call was and do that. Note that
1478 * another processor could be messing around and may have entered one of the
1479 * PTEs we just removed into the hash table. Too bad... You takes yer chances.
1480 * If there's a problem with that, it's because some higher level was trying to
1481 * do something with a mapping that it shouldn't. So, the problem's really
1482 * there, nyaaa, nyaaa, nyaaa... nyaaa, nyaaa... nyaaa! So there!
1485 commdone: li r5,pepte1 /* Get displacement to the second word of master pte */
1486 blt cr5,commfini /* We're finished, it was invalidate all... */
1487 bgt cr5,commtst /* It was a test modified... */
1488 beq cr5,commtst /* It was a test reference... */
1491 * Note that we need to to do the interlocked update here because another processor
1492 * can be updating the reference and change bits even though the physical entry
1493 * is locked. All modifications to the PTE portion of the physical entry must be
1494 * done via interlocked update.
1497 rlwinm r12,r2,5,23,24 ; Get the "set" bits
1498 rlwinm r11,r2,7,23,24 ; Get the "clear" bits
1502 commcng: lwarx r8,r5,r3 /* Get the master copy */
1503 or r8,r8,r12 ; Set the bits to come on
1504 andc r8,r8,r11 ; Clear those to come off
1505 stwcx. r8,r5,r3 /* Save it back */
1506 bne- commcng /* If it changed, try again... */
1508 mtmsr r0 /* Interrupts and translation back on */
1510 #if PERFTIMES && DEBUG
1514 bl EXT(dbgLog2) ; Start of hw_add_map
1522 commtst: lwz r8,pepte1(r3) /* Get the PTE */
1523 bne- cr5,commtcb ; This is for the change bit...
1524 mtmsr r0 ; Interrupts and translation back on
1525 rlwinm r3,r8,24,31,31 ; Copy reference bit to bit 31
1526 isync ; Toss prefetching
1527 #if PERFTIMES && DEBUG
1531 bl EXT(dbgLog2) ; Start of hw_add_map
1539 commtcb: rlwinm r3,r8,25,31,31 ; Copy change bit to bit 31
1541 commfini: mtmsr r0 ; Interrupts and translation back on
1542 isync ; Toss prefetching
1544 #if PERFTIMES && DEBUG
1548 bl EXT(dbgLog2) ; Start of hw_add_map
1555 * unsigned int hw_test_rc(mapping *mp, boolean_t reset);
1557 * Test the RC bits for a specific mapping. If reset is non-zero, clear them.
1558 * We return the RC value in the mapping if there is no PTE or if C is set.
1559 * (Note: R is always set with C.) Otherwise we invalidate the PTE and
1560 * collect the RC bits from there, also merging them into the global copy.
1562 * For now, we release the PTE slot and leave it invalid. In the future, we
1563 * may consider re-validating and not releasing the slot. It would be faster,
1564 * but our current implementation says that we will have not PTEs valid
1565 * without the reference bit set.
1567 * We will special case C==1 && not reset to just return the RC.
1569 * Probable state is worst performance state: C bit is off and there is a PTE.
1575 .globl EXT(hw_test_rc)
1579 mfsprg r9,2 ; Get feature flags
1580 mfmsr r0 ; Save the MSR
1581 mr. r4,r4 ; See if we have a reset to do later
1582 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Clear interruption mask
1583 crnot htrReset,cr0_eq ; Remember reset
1584 mtcrf 0x04,r9 ; Set the features
1585 rlwinm r12,r12,0,28,25 ; Clear IR and DR
1587 bt pfNoMSRirb,htrNoMSR ; No MSR...
1589 mtmsr r12 ; Translation and all off
1590 isync ; Toss prefetch
1596 li r0,loadMSR ; Get the MSR setter SC
1597 mr r3,r12 ; Get new MSR
1603 lwz r2,mmPTEr(r3) ; Get the real part
1604 lwz r7,mmPTEhash(r3) ; Get pointer to hash list anchor
1605 rlwinm. r12,r2,0,24,24 ; Is the change bit on?
1606 lwz r5,mmPTEv(r3) ; Get the virtual address
1607 crnor cr0_eq,cr0_eq,htrReset ; Set if C=1 && not reset
1608 rlwinm r7,r7,0,0,25 ; Round hash list down to PCA boundary
1609 bt cr0_eq,htrcset ; Special case changed but no reset case...
1611 li r12,1 ; Get the locked value
1613 htrLck1: lwarx r11,0,r7 ; Get the PTEG lock
1614 mr. r11,r11 ; Is it locked?
1615 bne- htrLckw1 ; Yeah...
1616 stwcx. r12,0,r7 ; Try to take it
1617 bne- htrLck1 ; Someone else was trying, try again...
1618 b htrSXg1 ; All done...
1622 htrLckw1: mr. r11,r11 ; Check if it is already held
1623 beq+ htrLck1 ; It is clear...
1624 lwz r11,0(r7) ; Get lock word again...
1625 b htrLckw1 ; Wait...
1629 htrSXg1: isync ; Make sure we have not used anything yet
1631 lwz r6,mmPTEent(r3) ; Get the pointer to the PTE now that the lock is set
1632 lwz r2,mmPTEr(r3) ; Get the mapping copy of the real part
1634 rlwinm r9,r5,1,0,3 ; Move in the segment
1635 mr. r6,r6 ; Any PTE to invalidate?
1636 rlwinm r8,r5,31,2,25 ; Line it up
1638 beq+ htrnopte ; There is no PTE to invalidate...
1640 xor r8,r8,r6 ; Back hash to virt index
1641 rlwimi r9,r5,22,4,9 ; Move in the API
1642 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) ; Get the TLBIE lock
1643 rlwinm r5,r5,0,1,31 ; Clear the valid bit
1644 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) ; Grab up the bottom part
1645 mfspr r11,pvr ; Find out what kind of machine we are
1646 rlwimi r9,r8,6,10,19 ; Create the virtual address
1647 rlwinm r11,r11,16,16,31 ; Isolate CPU type
1649 stw r5,0(r6) ; Make the PTE invalid
1650 cmplwi cr1,r11,3 ; Is this a 603?
1651 sync ; Make sure the invalid is stored
1653 htrtlbhang: lwarx r11,0,r12 ; Get the TLBIE lock
1654 rlwinm r8,r6,29,29,31 ; Get the bit position of entry
1655 mr. r11,r11 ; Is it locked?
1656 lis r5,0x8000 ; Start up a bit mask
1657 li r11,1 ; Get our lock word
1658 bne- htrtlbhang ; It is locked, go wait...
1659 stwcx. r11,0,r12 ; Try to get it
1660 bne- htrtlbhang ; We was beat...
1662 li r11,0 ; Lock clear value
1664 tlbie r9 ;Invalidate it everywhere
1666 beq- cr1,htr603 ; It is a 603, skip the tlbsync...
1668 eieio ; Make sure that the tlbie happens first
1669 tlbsync ; wait for everyone to catch up
1671 htr603: stw r11,0(r12) ; Clear the lock
1672 srw r5,r5,r8 ; Make a "free slot" mask
1673 sync ; Make sure of it all
1675 lwz r6,4(r6) ; Get the latest reference and change bits
1676 stw r11,mmPTEent(r3) ; Clear the pointer to the PTE
1677 rlwinm r6,r6,0,23,24 ; Extract the RC bits
1678 lwz r9,PCAallo(r7) ; Get the allocation control bits
1679 rlwinm r8,r5,24,8,15 ; Make the autogen bit to turn off
1680 lwz r10,mmphysent(r3) ; Get any physical entry
1681 or r9,r9,r5 ; Set the slot free
1682 rlwimi r8,r8,24,16,23 ; Get lock bit mask to turn it off
1683 andc r9,r9,r8 ; Clear the auto and lock bits
1684 mr. r10,r10 ; Is there a physical entry?
1685 li r5,pepte1 ; Get displacement to the second word of master pte
1686 stw r9,PCAallo(r7) ; Store the allocation controls
1687 rlwimi r2,r6,0,23,24 ; Stick in RC bits
1688 beq- htrnopte ; No physical entry...
1690 htrmrc: lwarx r11,r5,r10 ; Get the master copy
1691 or r11,r11,r6 ; Merge in latest RC
1692 stwcx. r11,r5,r10 ; Save it back
1693 bne- htrmrc ; If it changed, try again...
1695 sync ; Make sure that chain update is stored
1697 htrnopte: rlwinm r3,r2,25,30,31 ; Position RC and mask off
1698 bf htrReset,htrnorst ; No reset to do...
1699 rlwinm r2,r2,0,25,22 ; Clear the RC if requested
1701 htrnorst: li r4,0 ; Get a 0
1702 stw r2,mmPTEr(r3) ; Set the real part of the PTE
1703 stw r4,0(r7) ; Unlock the hash chain
1705 mtmsr r0 ; Restore interrupts and translation
1711 htrcset: rlwinm r3,r2,25,30,31 ; Position RC and mask off
1712 mtmsr r0 ; Restore interrupts and translation
1718 * hw_phys_attr(struct phys_entry *pp, vm_prot_t prot, unsigned int wimg) - Sets the default physical page attributes
1720 * Note that this must be done with both interruptions off and VM off
1721 * Move the passed in attributes into the pte image in the phys entry
1727 .globl EXT(hw_phys_attr)
1731 #if PERFTIMES && DEBUG
1739 bl EXT(dbgLog2) ; Start of hw_add_map
1745 mfsprg r9,2 ; Get feature flags
1746 mfmsr r0 /* Save the MSR */
1747 andi. r5,r5,0x0078 /* Clean up the WIMG */
1748 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1749 mtcrf 0x04,r9 ; Set the features
1750 rlwimi r5,r4,0,30,31 /* Move the protection into the wimg register */
1751 la r6,pepte1(r3) /* Point to the default pte */
1752 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
1754 bt pfNoMSRirb,hpaNoMSR ; No MSR...
1756 mtmsr r12 ; Translation and all off
1757 isync ; Toss prefetch
1763 li r0,loadMSR ; Get the MSR setter SC
1764 mr r3,r12 ; Get new MSR
1770 atmattr: lwarx r10,0,r6 /* Get the pte */
1771 rlwimi r10,r5,0,25,31 /* Move in the new attributes */
1772 stwcx. r10,0,r6 /* Try it on for size */
1773 bne- atmattr /* Someone else was trying, try again... */
1775 mtmsr r0 /* Interrupts and translation back on */
1777 #if PERFTIMES && DEBUG
1781 bl EXT(dbgLog2) ; Start of hw_add_map
1784 blr /* All done... */
1789 * handlePF - handle a page fault interruption
1791 * If the fault can be handled, this routine will RFI directly,
1792 * otherwise it will return with all registers as in entry.
1794 * Upon entry, state and all registers have been saved in savearea.
1795 * This is pointed to by R13.
1796 * IR and DR are off, interrupts are masked,
1797 * Floating point be disabled.
1798 * R3 is the interrupt code.
1800 * If we bail, we must restore cr5, and all registers except 6 and
1806 .globl EXT(handlePF)
1811 * This first part does a quick check to see if we can handle the fault.
1812 * We can't handle any kind of protection exceptions here, so we pass
1813 * them up to the next level.
1815 * The mapping lists are kept in MRS (most recently stolen)
1816 * order on queues anchored within from the
1817 * PTEG to which the virtual address hashes. This is further segregated by
1818 * the low-order 3 bits of the VSID XORed with the segment number and XORed
1819 * with bits 4-7 of the vaddr in an attempt to keep the searches
1822 * MRS is handled by moving the entry to the head of its list when stolen in the
1823 * assumption that it will be revalidated soon. Entries are created on the head
1824 * of the list because they will be used again almost immediately.
1826 * We need R13 set to the savearea, R3 set to the interrupt code, and R2
1827 * set to the per_proc.
1829 * NOTE: In order for a page-fault redrive to work, the translation miss
1830 * bit must be set in the DSISR (or SRR1 for IFETCH). That must occur
1831 * before we come here.
1834 cmplwi r3,T_INSTRUCTION_ACCESS /* See if this is for the instruction */
1835 lwz r8,savesrr1(r13) ; Get the MSR to determine mode
1836 beq- gotIfetch ; We have an IFETCH here...
1838 lwz r7,savedsisr(r13) /* Get the DSISR */
1839 lwz r6,savedar(r13) /* Get the fault address */
1840 b ckIfProt ; Go check if this is a protection fault...
1842 gotIfetch: mr r7,r8 ; IFETCH info is in SRR1
1843 lwz r6,savesrr0(r13) /* Get the instruction address */
1845 ckIfProt: rlwinm. r7,r7,0,1,1 ; Is this a protection exception?
1846 beqlr- ; Yes... (probably not though)
1849 * We will need to restore registers if we bail after this point.
1850 * Note that at this point several SRs have been changed to the kernel versions.
1851 * Therefore, for these we must build these values.
1854 #if PERFTIMES && DEBUG
1859 bl EXT(dbgLog2) ; Start of hw_add_map
1864 lwz r3,PP_USERPMAP(r2) ; Get the user pmap (not needed if kernel access, but optimize for user??)
1865 rlwinm. r8,r8,0,MSR_PR_BIT,MSR_PR_BIT ; Supervisor state access?
1866 rlwinm r5,r6,6,26,29 ; Get index to the segment slot
1867 eqv r1,r1,r1 ; Fill the bottom with foxes
1868 bne+ notsuper ; Go do the user mode interrupt stuff...
1870 cmplwi cr1,r5,SR_COPYIN_NUM*4 ; See if this is the copyin/copyout segment
1871 rlwinm r3,r6,24,8,11 ; Make the kernel VSID
1872 bne+ cr1,havevsid ; We are done if we do not want the copyin/out guy...
1874 mfsr r3,SR_COPYIN ; Get the copy vsid
1875 b havevsid ; Join up...
1879 notsuper: addi r5,r5,PMAP_SEGS ; Get offset to table
1880 lwzx r3,r3,r5 ; Get the VSID
1882 havevsid: mfspr r5,sdr1 /* Get hash table base and size */
1883 cror cr1_eq,cr0_eq,cr0_eq ; Remember if kernel fault for later
1884 rlwinm r9,r6,2,2,5 ; Move nybble 1 up to 0 (keep aligned with VSID)
1885 rlwimi r1,r5,16,0,15 /* Make table size -1 out of mask */
1886 rlwinm r3,r3,6,2,25 /* Position the space for the VSID */
1887 rlwinm r7,r6,26,10,25 /* Isolate the page index */
1888 xor r9,r9,r3 ; Splooch vaddr nybble 0 (from VSID) and 1 together
1889 or r8,r5,r1 /* Point to the last byte in table */
1890 xor r7,r7,r3 /* Get primary hash */
1891 rlwinm r3,r3,1,1,24 /* Position VSID for pte ID */
1892 addi r8,r8,1 /* Point to the PTEG Control Area */
1893 rlwinm r9,r9,8,27,29 ; Get splooched bits in place
1894 and r7,r7,r1 /* Wrap the hash */
1895 rlwimi r3,r6,10,26,31 /* Move API into pte ID */
1896 add r8,r8,r7 /* Point to our PCA entry */
1897 rlwinm r12,r3,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
1898 la r11,PCAhash(r8) /* Point to the mapping hash area */
1899 xor r9,r9,r12 ; Finish splooching nybble 0, 1, and the low bits of the VSID
1903 * We have about as much as we need to start searching the autogen (aka block maps)
1904 * and mappings. From here on, any kind of failure will bail, and
1905 * contention will either bail or restart from here.
1910 li r12,1 /* Get the locked value */
1911 dcbt 0,r11 /* We'll need the hash area in a sec, so get it */
1912 add r11,r11,r9 /* Point to the right mapping hash slot */
1915 ptegLck: lwarx r10,0,r8 /* Get the PTEG lock */
1916 mr. r10,r10 /* Is it locked? */
1917 bne- ptegLckw /* Yeah... */
1918 stwcx. r12,0,r8 /* Take take it */
1919 bne- ptegLck /* Someone else was trying, try again... */
1920 b ptegSXg /* All done... */
1924 ptegLckw: mr. r10,r10 /* Check if it's already held */
1925 beq+ ptegLck /* It's clear... */
1926 lwz r10,0(r8) /* Get lock word again... */
1927 b ptegLckw /* Wait... */
1931 nop ; Force ISYNC to last instruction in IFETCH
1935 ptegSXg: isync /* Make sure we haven't used anything yet */
1937 lwz r9,0(r11) /* Pick up first mapping block */
1938 mr r5,r11 /* Get the address of the anchor */
1939 mr r7,r9 /* Save the first in line */
1940 b findmap ; Take space and force loop to cache line
1942 findmap: mr. r12,r9 /* Are there more? */
1943 beq- tryAuto /* Nope, nothing in mapping list for us... */
1945 lwz r10,mmPTEv(r12) /* Get unique PTE identification */
1946 lwz r9,mmhashnext(r12) /* Get the chain, just in case */
1947 cmplw r10,r3 /* Did we hit our PTE? */
1948 lwz r0,mmPTEent(r12) /* Get the pointer to the hash table entry */
1949 mr r5,r12 /* Save the current as previous */
1950 bne- findmap ; Nothing here, try the next...
1952 ; Cache line boundary here
1954 cmplwi cr1,r0,0 /* Is there actually a PTE entry in the hash? */
1955 lwz r2,mmphysent(r12) /* Get the physical entry */
1956 bne- cr1,MustBeOK /* There's an entry in the hash table, so, this must
1957 have been taken care of already... */
1958 lis r4,0x8000 ; Tell PTE inserter that this was not an auto
1959 cmplwi cr2,r2,0 /* Is there a physical entry? */
1960 li r0,0x0100 /* Force on the reference bit whenever we make a PTE valid */
1961 bne+ cr2,gotphys /* Skip down if we have a physical entry */
1962 li r0,0x0180 /* When there is no physical entry, force on
1963 both R and C bits to keep hardware from
1964 updating the PTE to set them. We don't
1965 keep track of RC for I/O areas, so this is ok */
1967 gotphys: lwz r2,mmPTEr(r12) ; Get the second part of the PTE
1968 b insert /* Go insert into the PTEG... */
1970 MustBeOK: li r10,0 /* Get lock clear value */
1971 li r3,T_IN_VAIN /* Say that we handled it */
1972 stw r10,PCAlock(r8) /* Clear the PTEG lock */
1974 #if PERFTIMES && DEBUG
1978 bl EXT(dbgLog2) ; Start of hw_add_map
1982 blr /* Blow back and handle exception */
1987 * We couldn't find it in the mapping list. As a last try, we will
1988 * see if we can autogen it from the block mapped list.
1990 * A block mapped area is defined as a contiguous virtual area that is mapped to
1991 * a contiguous physical area. The olde-tyme IBM VM/XA Interpretive Execution
1992 * architecture referred to this as a V=F, or Virtual = Fixed area.
1994 * We consider a V=F area to be a single entity, adjacent areas can not be merged
1995 * or overlapped. The protection and memory attributes are the same and reference
1996 * and change indications are not kept. The areas are not considered part of the
1997 * physical RAM of the machine and do not have any associated physical table
1998 * entries. Their primary use is intended for mapped I/O areas (e.g., framebuffers)
1999 * although certain areas of RAM, such as the kernel V=R memory, can be mapped.
2001 * We also have a problem in the case of copyin/out: that access is done
2002 * within the kernel for a user address. Unfortunately, the user isn't
2003 * necessarily the current guy. That means that we don't have access to the
2004 * right autogen list. We can't support this kind of access. So, we need to do
2005 * a quick check here and cause a fault if an attempt to copyin or out to
2006 * any autogenned area.
2008 * The lists must be kept short.
2010 * NOTE: kernel_pmap_store must be in V=R storage!!!!!!!!!!!!!!
2015 tryAuto: rlwinm. r11,r3,0,5,24 ; Check if this is a kernel VSID
2016 lis r10,HIGH_ADDR(EXT(kernel_pmap_store)+PMAP_BMAPS) ; Get the top part of kernel block map anchor
2017 crandc cr0_eq,cr1_eq,cr0_eq ; Set if kernel access and non-zero VSID (copyin or copyout)
2018 mfsprg r11,0 ; Get the per_proc area
2019 beq- cr0,realFault ; Can not autogen for copyin/copyout...
2020 ori r10,r10,LOW_ADDR(EXT(kernel_pmap_store)+PMAP_BMAPS) ; Get the bottom part
2021 beq- cr1,bmInKernel ; We are in kernel... (cr1 set way back at entry)
2023 lwz r10,PP_USERPMAP(r11) ; Get the user pmap
2024 la r10,PMAP_BMAPS(r10) ; Point to the chain anchor
2025 b bmInKernel ; Jump over alignment gap...
2033 #ifndef CHIP_ERRATA_MAX_V1
2035 #endif /* CHIP_ERRATA_MAX_V1 */
2037 bmapLck: lwarx r9,0,r10 ; Get the block map anchor and lock
2038 rlwinm. r5,r9,0,31,31 ; Is it locked?
2039 ori r5,r5,1 ; Set the lock
2040 bne- bmapLckw ; Yeah...
2041 stwcx. r5,0,r10 ; Lock the bmap list
2042 bne- bmapLck ; Someone else was trying, try again...
2043 b bmapSXg ; All done...
2047 bmapLckw: rlwinm. r5,r9,0,31,31 ; Check if it is still held
2048 beq+ bmapLck ; Not no more...
2049 lwz r9,0(r10) ; Get lock word again...
2050 b bmapLckw ; Check it out...
2054 nop ; Force ISYNC to last instruction in IFETCH
2058 bmapSXg: rlwinm. r4,r9,0,0,26 ; Clear out flags and lock
2059 isync ; Make sure we have not used anything yet
2060 bne+ findAuto ; We have something, let us go...
2062 bmapNone: stw r9,0(r10) ; Unlock it, we have nothing here
2063 ; No sync here because we have not changed anything
2066 * When we come here, we know that we can't handle this. Restore whatever
2067 * state that we trashed and go back to continue handling the interrupt.
2070 realFault: li r10,0 /* Get lock clear value */
2071 lwz r3,saveexception(r13) /* Figure out the exception code again */
2072 stw r10,PCAlock(r8) /* Clear the PTEG lock */
2073 #if PERFTIMES && DEBUG
2077 bl EXT(dbgLog2) ; Start of hw_add_map
2081 blr /* Blow back and handle exception */
2085 findAuto: mr. r4,r4 ; Is there more?
2086 beq- bmapNone ; No more...
2087 lwz r5,bmstart(r4) ; Get the bottom of range
2088 lwz r11,bmend(r4) ; Get the top of range
2089 cmplw cr0,r6,r5 ; Are we before the entry?
2090 cmplw cr1,r6,r11 ; Are we after the entry?
2091 cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in range
2092 bne+ cr1,faGot ; Found it...
2094 lwz r4,bmnext(r4) ; Get the next one
2095 b findAuto ; Check it out...
2097 faGot: rlwinm r6,r6,0,0,19 ; Round to page
2098 lwz r2,bmPTEr(r4) ; Get the real part of the PTE
2099 sub r5,r6,r5 ; Get offset into area
2100 stw r9,0(r10) ; Unlock it, we are done with it (no sync needed)
2101 add r2,r2,r5 ; Adjust the real address
2103 lis r4,0x8080 /* Indicate that this was autogened */
2104 li r0,0x0180 /* Autogenned areas always set RC bits.
2105 This keeps the hardware from having
2106 to do two storage writes */
2109 * Here where we insert the PTE into the hash. The PTE image is in R3, R2.
2110 * The PTEG allocation controls are a bit map of the state of the PTEG. The
2111 * PCAlock bits are a temporary lock for the specified PTE. PCAfree indicates that
2112 * the PTE slot is empty. PCAauto means that it comes from an autogen area. These
2113 * guys do not keep track of reference and change and are actually "wired".
2114 * They're easy to maintain. PCAsteal
2115 * is a sliding position mask used to "randomize" PTE slot stealing. All 4 of these
2116 * fields fit in a single word and are loaded and stored under control of the
2117 * PTEG control area lock (PCAlock).
2119 * Note that PCAauto does not contribute to the steal calculations at all. Originally
2120 * it did, autogens were second in priority. This can result in a pathalogical
2121 * case where an instruction can not make forward progress, or one PTE slot
2124 * Physically, the fields are arranged:
2131 insert: lwz r10,PCAallo(r8) /* Get the PTEG controls */
2132 eqv r6,r6,r6 /* Get all ones */
2133 mr r11,r10 /* Make a copy */
2134 rlwimi r6,r10,8,16,23 /* Insert sliding steal position */
2135 rlwimi r11,r11,24,24,31 /* Duplicate the locked field */
2136 addi r6,r6,-256 /* Form mask */
2137 rlwimi r11,r11,16,0,15 /* This gives us a quadrupled lock mask */
2138 rlwinm r5,r10,31,24,0 /* Slide over the mask for next time */
2139 mr r9,r10 /* Make a copy to test */
2140 not r11,r11 /* Invert the quadrupled lock */
2141 or r2,r2,r0 /* Force on R, and maybe C bit */
2142 and r9,r9,r11 /* Remove the locked guys */
2143 rlwimi r5,r5,8,24,24 /* Wrap bottom bit to top in mask */
2144 rlwimi r9,r11,0,16,31 /* Put two copies of the unlocked entries at the end */
2145 rlwinm r6,r6,0,16,7 ; Remove the autogens from the priority calculations
2146 rlwimi r10,r5,0,24,31 /* Move steal map back in */
2147 and r9,r9,r6 /* Set the starting point for stealing */
2149 /* So, now we have in R9:
2150 byte 0 = ~locked & free
2152 byte 2 = ~locked & (PCAsteal - 1)
2155 Each bit position represents (modulo 8) a PTE. If it is 1, it is available for
2156 allocation at its priority level, left to right.
2158 Additionally, the PCA steal field in R10 has been rotated right one bit.
2162 rlwinm r21,r10,8,0,7 ; Isolate just the old autogen bits
2163 cntlzw r6,r9 /* Allocate a slot */
2164 mr r14,r12 /* Save our mapping for later */
2165 cmplwi r6,32 ; Was there anything available?
2166 rlwinm r7,r6,29,30,31 /* Get the priority slot we got this from */
2167 rlwinm r6,r6,0,29,31 ; Isolate bit position
2168 srw r11,r4,r6 /* Position the PTEG control bits */
2169 slw r21,r21,r6 ; Move corresponding old autogen flag to bit 0
2170 mr r22,r11 ; Get another copy of the selected slot
2172 beq- realFault /* Arghh, no slots! Take the long way 'round... */
2174 /* Remember, we've already set up the mask pattern
2175 depending upon how we got here:
2176 if got here from simple mapping, R4=0x80000000,
2177 if we got here from autogen it is 0x80800000. */
2179 rlwinm r6,r6,3,26,28 /* Start calculating actual PTE address */
2180 rlwimi r22,r22,24,8,15 ; Duplicate selected slot in second byte
2181 rlwinm. r11,r11,0,8,15 /* Isolate just the auto bit (remember about it too) */
2182 andc r10,r10,r22 /* Turn off the free and auto bits */
2183 add r6,r8,r6 /* Get position into PTEG control area */
2184 cmplwi cr1,r7,1 /* Set the condition based upon the old PTE type */
2185 sub r6,r6,r1 /* Switch it to the hash table */
2186 or r10,r10,r11 /* Turn auto on if it is (PTEG control all set up now) */
2187 subi r6,r6,1 /* Point right */
2188 stw r10,PCAallo(r8) /* Allocate our slot */
2189 dcbt br0,r6 ; Touch in the PTE
2190 bne wasauto /* This was autogenned... */
2192 stw r6,mmPTEent(r14) /* Link the mapping to the PTE slot */
2195 * So, now we're here and what exactly do we have? We've got:
2196 * 1) a full PTE entry, both top and bottom words in R3 and R2
2197 * 2) an allocated slot in the PTEG.
2198 * 3) R8 still points to the PTEG Control Area (PCA)
2199 * 4) R6 points to the PTE entry.
2200 * 5) R1 contains length of the hash table-1. We use this to back-translate
2201 * a PTE to a virtual address so we can invalidate TLBs.
2202 * 6) R11 has a copy of the PCA controls we set.
2203 * 7a) R7 indicates what the PTE slot was before we got to it. 0 shows
2204 * that it was empty and 2 or 3, that it was
2205 * a we've stolen a live one. CR1 is set to LT for empty and GT
2207 * 7b) Bit 0 of R21 is 1 if the stolen PTE was autogenned
2208 * 8) So far as our selected PTE, it should be valid if it was stolen
2209 * and invalid if not. We could put some kind of assert here to
2210 * check, but I think that I'd rather leave it in as a mysterious,
2211 * non-reproducable bug.
2212 * 9) The new PTE's mapping has been moved to the front of its PTEG hash list
2213 * so that it's kept in some semblance of a MRU list.
2214 * 10) R14 points to the mapping we're adding.
2216 * So, what do we have to do yet?
2217 * 1) If we stole a slot, we need to invalidate the PTE completely.
2218 * 2) If we stole one AND it was not an autogen,
2219 * copy the entire old PTE (including R and C bits) to its mapping.
2220 * 3) Set the new PTE in the PTEG and make sure it is valid.
2221 * 4) Unlock the PTEG control area.
2222 * 5) Go back to the interrupt handler, changing the interrupt
2223 * code to "in vain" which will restore the registers and bail out.
2226 wasauto: oris r3,r3,0x8000 /* Turn on the valid bit */
2227 blt+ cr1,slamit /* It was empty, go slam it on in... */
2229 lwz r10,0(r6) /* Grab the top part of the PTE */
2230 rlwinm r12,r6,6,4,19 /* Match up the hash to a page boundary */
2231 rlwinm r5,r10,5,4,19 /* Extract the VSID to a page boundary */
2232 rlwinm r10,r10,0,1,31 /* Make it invalid */
2233 xor r12,r5,r12 /* Calculate vaddr */
2234 stw r10,0(r6) /* Invalidate the PTE */
2235 rlwinm r5,r10,7,27,29 ; Move nybble 0 up to subhash position
2236 rlwimi r12,r10,1,0,3 /* Move in the segment portion */
2237 lis r9,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
2238 xor r5,r5,r10 ; Splooch nybble 0 and 1
2239 rlwimi r12,r10,22,4,9 /* Move in the API */
2240 ori r9,r9,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
2241 rlwinm r4,r10,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
2243 sync /* Make sure the invalid is stored */
2245 xor r4,r4,r5 ; Finish splooching nybble 0, 1, and the low bits of the VSID
2249 tlbhang: lwarx r5,0,r9 /* Get the TLBIE lock */
2251 rlwinm r4,r4,0,27,29 ; Clean up splooched hash value
2253 mr. r5,r5 /* Is it locked? */
2254 add r4,r4,r8 /* Point to the offset into the PCA area */
2255 li r5,1 /* Get our lock word */
2256 bne- tlbhang /* It's locked, go wait... */
2258 la r4,PCAhash(r4) /* Point to the start of the hash chain for the PTE we're replacing */
2260 stwcx. r5,0,r9 /* Try to get it */
2261 bne- tlbhang /* We was beat... */
2263 mfspr r7,pvr /* Find out what kind of machine we are */
2264 li r5,0 /* Lock clear value */
2265 rlwinm r7,r7,16,16,31 /* Isolate CPU type */
2267 tlbie r12 /* Invalidate it everywhere */
2269 cmplwi r7,3 /* Is this a 603? */
2270 stw r5,0(r9) /* Clear the lock */
2272 beq- its603 /* It's a 603, skip the tlbsync... */
2274 eieio /* Make sure that the tlbie happens first */
2275 tlbsync /* wait for everyone to catch up */
2277 its603: rlwinm. r21,r21,0,0,0 ; See if we just stole an autogenned entry
2278 sync /* Make sure of it all */
2280 bne slamit ; The old was an autogen, time to slam the new in...
2282 lwz r9,4(r6) /* Get the real portion of old PTE */
2283 lwz r7,0(r4) /* Get the first element. We can't get to here
2284 if we aren't working with a mapping... */
2285 mr r0,r7 ; Save pointer to first element
2287 findold: mr r1,r11 ; Save the previous guy
2288 mr. r11,r7 /* Copy and test the chain */
2289 beq- bebad /* Assume it's not zero... */
2291 lwz r5,mmPTEv(r11) /* See if this is the old active one */
2292 cmplw cr2,r11,r14 /* Check if this is actually the new one */
2293 cmplw r5,r10 /* Is this us? (Note: valid bit kept off in mappings) */
2294 lwz r7,mmhashnext(r11) /* Get the next one in line */
2295 beq- cr2,findold /* Don't count the new one... */
2296 cmplw cr2,r11,r0 ; Check if we are first on the list
2297 bne+ findold /* Not it (and assume the worst)... */
2299 lwz r12,mmphysent(r11) /* Get the pointer to the physical entry */
2300 beq- cr2,nomove ; We are first, no need to requeue...
2302 stw r11,0(r4) ; Chain us to the head
2303 stw r0,mmhashnext(r11) ; Chain the old head to us
2304 stw r7,mmhashnext(r1) ; Unlink us
2306 nomove: li r5,0 /* Clear this on out */
2308 mr. r12,r12 /* Is there a physical entry? */
2309 stw r5,mmPTEent(r11) ; Clear the PTE entry pointer
2310 li r5,pepte1 /* Point to the PTE last half */
2311 stw r9,mmPTEr(r11) ; Squirrel away the whole thing (RC bits are in here)
2313 beq- mrgmrcx ; No physical entry for this one...
2315 rlwinm r11,r9,0,23,24 /* Keep only the RC bits */
2319 mrgmrcx: lwarx r9,r5,r12 /* Get the master copy */
2320 or r9,r9,r11 /* Merge in latest RC */
2321 stwcx. r9,r5,r12 /* Save it back */
2322 bne- mrgmrcx /* If it changed, try again... */
2325 * Here's where we finish up. We save the real part of the PTE, eieio it, to make sure it's
2326 * out there before the top half (with the valid bit set).
2329 slamit: stw r2,4(r6) /* Stash the real part */
2330 li r4,0 /* Get a lock clear value */
2331 eieio /* Erect a barricade */
2332 stw r3,0(r6) /* Stash the virtual part and set valid on */
2334 stw r4,PCAlock(r8) /* Clear the PCA lock */
2336 li r3,T_IN_VAIN /* Say that we handled it */
2337 sync /* Go no further until the stores complete */
2338 #if PERFTIMES && DEBUG
2342 bl EXT(dbgLog2) ; Start of hw_add_map
2346 blr /* Back to the fold... */
2348 bebad: lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
2349 ori r0,r0,LOW_ADDR(Choke)
2350 sc /* Firmware Heimlich maneuver */
2353 * This walks the hash table or DBATs to locate the physical address of a virtual one.
2354 * The space is provided. If it is the kernel space, the DBATs are searched first. Failing
2355 * that, the hash table is accessed. Zero is returned for failure, so it must be special cased.
2356 * This is usually used for debugging, so we try not to rely
2357 * on anything that we don't have to.
2360 ENTRY(LRA, TAG_NO_FRAME_USED)
2362 mfsprg r8,2 ; Get feature flags
2363 mfmsr r10 /* Save the current MSR */
2364 mtcrf 0x04,r8 ; Set the features
2365 xoris r5,r3,HIGH_ADDR(PPC_SID_KERNEL) /* Clear the top half if equal */
2366 andi. r9,r10,0x7FCF /* Turn off interrupts and translation */
2367 eqv r12,r12,r12 /* Fill the bottom with foxes */
2369 bt pfNoMSRirb,lraNoMSR ; No MSR...
2371 mtmsr r9 ; Translation and all off
2372 isync ; Toss prefetch
2377 li r0,loadMSR ; Get the MSR setter SC
2378 mr r3,r9 ; Get new MSR
2383 cmplwi r5,LOW_ADDR(PPC_SID_KERNEL) /* See if this is kernel space */
2384 rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
2385 isync /* Purge pipe */
2386 bne- notkernsp /* This is not for the kernel... */
2388 mfspr r5,dbat0u /* Get the virtual address and length */
2389 eqv r8,r8,r8 /* Get all foxes */
2390 rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
2391 rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
2392 beq- ckbat1 /* not valid, skip this one... */
2393 sub r7,r4,r7 /* Subtract out the base */
2394 rlwimi r8,r5,15,0,14 /* Get area length - 1 */
2395 mfspr r6,dbat0l /* Get the real part */
2396 cmplw r7,r8 /* Check if it is in the range */
2397 bng+ fndbat /* Yup, she's a good un... */
2399 ckbat1: mfspr r5,dbat1u /* Get the virtual address and length */
2400 eqv r8,r8,r8 /* Get all foxes */
2401 rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
2402 rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
2403 beq- ckbat2 /* not valid, skip this one... */
2404 sub r7,r4,r7 /* Subtract out the base */
2405 rlwimi r8,r5,15,0,14 /* Get area length - 1 */
2406 mfspr r6,dbat1l /* Get the real part */
2407 cmplw r7,r8 /* Check if it is in the range */
2408 bng+ fndbat /* Yup, she's a good un... */
2410 ckbat2: mfspr r5,dbat2u /* Get the virtual address and length */
2411 eqv r8,r8,r8 /* Get all foxes */
2412 rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
2413 rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
2414 beq- ckbat3 /* not valid, skip this one... */
2415 sub r7,r4,r7 /* Subtract out the base */
2416 rlwimi r8,r5,15,0,14 /* Get area length - 1 */
2417 mfspr r6,dbat2l /* Get the real part */
2418 cmplw r7,r8 /* Check if it is in the range */
2419 bng- fndbat /* Yup, she's a good un... */
2421 ckbat3: mfspr r5,dbat3u /* Get the virtual address and length */
2422 eqv r8,r8,r8 /* Get all foxes */
2423 rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
2424 rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
2425 beq- notkernsp /* not valid, skip this one... */
2426 sub r7,r4,r7 /* Subtract out the base */
2427 rlwimi r8,r5,15,0,14 /* Get area length - 1 */
2428 mfspr r6,dbat3l /* Get the real part */
2429 cmplw r7,r8 /* Check if it is in the range */
2430 bgt+ notkernsp /* No good... */
2432 fndbat: rlwinm r6,r6,0,0,14 /* Clean up the real address */
2433 mtmsr r10 /* Restore state */
2434 add r3,r7,r6 /* Relocate the offset to real */
2435 isync /* Purge pipe */
2436 blr /* Bye, bye... */
2437 notkernsp: mfspr r5,sdr1 /* Get hash table base and size */
2438 rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
2439 rlwimi r12,r5,16,0,15 /* Make table size -1 out of mask */
2440 rlwinm r7,r4,26,10,25 /* Isolate the page index */
2441 andc r5,r5,r12 /* Clean up the hash table */
2442 xor r7,r7,r11 /* Get primary hash */
2443 rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
2444 and r7,r7,r12 /* Wrap the hash */
2445 rlwimi r11,r4,10,26,31 /* Move API into pte ID */
2446 add r5,r7,r5 /* Point to the PTEG */
2447 oris r11,r11,0x8000 /* Slam on valid bit so's we don't match an invalid one */
2449 li r9,8 /* Get the number of PTEs to check */
2450 lwz r6,0(r5) /* Preload the virtual half */
2452 fndpte: subi r9,r9,1 /* Count the pte */
2453 lwz r3,4(r5) /* Get the real half */
2454 cmplw cr1,r6,r11 /* Is this what we want? */
2455 lwz r6,8(r5) /* Start to get the next virtual half */
2456 mr. r9,r9 /* Any more to try? */
2457 addi r5,r5,8 /* Bump to next slot */
2458 beq cr1,gotxlate /* We found what we were looking for... */
2459 bne+ fndpte /* Go try the next PTE... */
2461 mtmsr r10 /* Restore state */
2462 li r3,0 /* Show failure */
2463 isync /* Purge pipe */
2466 gotxlate: mtmsr r10 /* Restore state */
2467 rlwimi r3,r4,0,20,31 /* Cram in the page displacement */
2468 isync /* Purge pipe */
2474 * struct blokmap *hw_add_blk(pmap_t pmap, struct blokmap *bmr)
2476 * This is used to add a block mapping entry to the MRU list whose top
2477 * node is anchored at bmaps. This is a real address and is also used as
2480 * Overlapping areas are not allowed. If we find one, we return it's address and
2481 * expect the upper layers to panic. We only check this for a debug build...
2486 .globl EXT(hw_add_blk)
2490 mfsprg r9,2 ; Get feature flags
2491 lwz r6,PMAP_PMAPVR(r3) ; Get the v to r translation
2492 mfmsr r0 /* Save the MSR */
2493 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
2494 mtcrf 0x04,r9 ; Set the features
2495 xor r3,r3,r6 ; Get real address of bmap anchor
2496 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
2497 la r3,PMAP_BMAPS(r3) ; Point to bmap header
2499 bt pfNoMSRirb,habNoMSR ; No MSR...
2501 mtmsr r12 ; Translation and all off
2502 isync ; Toss prefetch
2508 li r0,loadMSR ; Get the MSR setter SC
2509 mr r3,r12 ; Get new MSR
2515 abLck: lwarx r9,0,r3 ; Get the block map anchor and lock
2516 rlwinm. r8,r9,0,31,31 ; Is it locked?
2517 ori r8,r9,1 ; Set the lock
2518 bne- abLckw ; Yeah...
2519 stwcx. r8,0,r3 ; Lock the bmap list
2520 bne- abLck ; Someone else was trying, try again...
2521 b abSXg ; All done...
2525 abLckw: rlwinm. r5,r9,0,31,31 ; Check if it is still held
2526 beq+ abLck ; Not no more...
2527 lwz r9,0(r3) ; Get lock word again...
2528 b abLckw ; Check it out...
2532 nop ; Force ISYNC to last instruction in IFETCH
2535 abSXg: rlwinm r11,r9,0,0,26 ; Clear out flags and lock
2536 isync ; Make sure we have not used anything yet
2542 lwz r7,bmstart(r4) ; Get start
2543 lwz r8,bmend(r4) ; Get end
2544 mr r2,r11 ; Get chain
2546 abChk: mr. r10,r2 ; End of chain?
2547 beq abChkD ; Yes, chain is ok...
2548 lwz r5,bmstart(r10) ; Get start of current area
2549 lwz r6,bmend(r10) ; Get end of current area
2551 cmplw cr0,r8,r5 ; Is the end of the new before the old?
2552 cmplw cr1,r8,r6 ; Is the end of the new after the old?
2553 cmplw cr6,r6,r7 ; Is the end of the old before the new?
2554 cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in old
2555 cmplw cr7,r6,r8 ; Is the end of the old after the new?
2556 lwz r2,bmnext(r10) ; Get pointer to the next
2557 cror cr6_eq,cr6_lt,cr7_gt ; Set cr2_eq if old not in new
2558 crand cr1_eq,cr1_eq,cr6_eq ; Set cr1_eq if no overlap
2559 beq+ cr1,abChk ; Ok check the next...
2561 stw r9,0(r3) ; Unlock
2562 mtmsr r0 ; Restore xlation and rupts
2563 mr r3,r10 ; Pass back the overlap
2567 abChkD: stw r11,bmnext(r4) ; Chain this on in
2568 rlwimi r4,r9,0,27,31 ; Copy in locks and flags
2569 sync ; Make sure that is done
2571 stw r4,0(r3) ; Unlock and chain the new first one
2572 mtmsr r0 ; Restore xlation and rupts
2573 li r3,0 ; Pass back a no failure return code
2579 * struct blokmap *hw_rem_blk(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
2581 * This is used to remove a block mapping entry from the list that
2582 * is anchored at bmaps. bmaps is a virtual address and is also used as
2585 * Note that this function clears a single block that contains
2586 * any address within the range sva to eva (inclusive). To entirely
2587 * clear any range, hw_rem_blk must be called repeatedly until it
2590 * The block is removed from the list and all hash table entries
2591 * corresponding to the mapped block are invalidated and the TLB
2592 * entries are purged. If the block is large, this could take
2593 * quite a while. We need to hash every possible address in the
2594 * range and lock down the PCA.
2596 * If we attempt to remove a permanent entry, we will not do it.
2597 * The block address will be ored with 1 and returned.
2603 .globl EXT(hw_rem_blk)
2607 mfsprg r9,2 ; Get feature flags
2608 lwz r6,PMAP_PMAPVR(r3) ; Get the v to r translation
2609 mfmsr r0 /* Save the MSR */
2610 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
2611 mtcrf 0x04,r9 ; Set the features
2612 xor r3,r3,r6 ; Get real address of bmap anchor
2613 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
2614 la r3,PMAP_BMAPS(r3) ; Point to the bmap chain head
2616 bt pfNoMSRirb,hrbNoMSR ; No MSR...
2618 mtmsr r12 ; Translation and all off
2619 isync ; Toss prefetch
2625 li r0,loadMSR ; Get the MSR setter SC
2626 mr r3,r12 ; Get new MSR
2632 rbLck: lwarx r9,0,r3 ; Get the block map anchor and lock
2633 rlwinm. r8,r9,0,31,31 ; Is it locked?
2634 ori r8,r9,1 ; Set the lock
2635 bne- rbLckw ; Yeah...
2636 stwcx. r8,0,r3 ; Lock the bmap list
2637 bne- rbLck ; Someone else was trying, try again...
2638 b rbSXg ; All done...
2642 rbLckw: rlwinm. r11,r9,0,31,31 ; Check if it is still held
2643 beq+ rbLck ; Not no more...
2644 lwz r9,0(r3) ; Get lock word again...
2645 b rbLckw ; Check it out...
2649 nop ; Force ISYNC to last instruction in IFETCH
2652 rbSXg: rlwinm. r2,r9,0,0,26 ; Clear out flags and lock
2653 mr r10,r3 ; Keep anchor as previous pointer
2654 isync ; Make sure we have not used anything yet
2656 beq- rbMT ; There is nothing in the list
2658 rbChk: mr r12,r10 ; Save the previous
2659 mr. r10,r2 ; End of chain?
2660 beq rbMT ; Yes, nothing to do...
2661 lwz r11,bmstart(r10) ; Get start of current area
2662 lwz r6,bmend(r10) ; Get end of current area
2664 cmplw cr0,r5,r11 ; Is the end of range before the start of the area?
2665 cmplw cr1,r4,r6 ; Is the start of range after the end of the area?
2666 cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in range
2667 lwz r2,bmnext(r10) ; Get the next one
2668 beq+ cr1,rbChk ; Not this one, check the next...
2670 lwz r8,blkFlags(r10) ; Get the flags
2672 cmplw cr1,r12,r3 ; Did we delete the first one?
2673 rlwinm. r8,r8,0,blkPermbit,blkPermbit ; is this a permanent block?
2674 bne cr1,rbnFirst ; Nope...
2675 rlwimi r9,r2,0,0,26 ; Change the lock value
2676 ori r2,r9,1 ; Turn on the lock bit
2678 rbnFirst: bne- rbPerm ; This is permanent, do not remove...
2679 lwz r8,bmspace(r10) ; Get the VSID
2680 stw r2,bmnext(r12) ; Unchain us
2682 eqv r4,r4,r4 ; Fill the bottom with foxes
2683 mfspr r12,sdr1 ; Get hash table base and size
2684 rlwinm r8,r8,6,0,25 ; Align VSID to PTEG
2685 rlwimi r4,r12,16,0,15 ; Make table size - 1 out of mask
2686 andc r12,r12,r4 ; Clean up address of hash table
2687 rlwinm r5,r11,26,6,25 ; Rotate virtual start address into PTEG units
2688 add r12,r12,r4 ; Point to PCA - 1
2689 rlwinm r6,r6,26,6,25 ; Rotate virtual end address into PTEG units
2690 addi r12,r12,1 ; Point to PCA base
2691 sub r6,r6,r5 ; Get the total number of PTEGs to clear
2692 cmplw r6,r4 ; See if this wraps all the way around
2693 blt rbHash ; Nope, length is right
2694 subi r6,r4,32+31 ; Back down to correct length
2696 rbHash: xor r2,r8,r5 ; Hash into table
2697 and r2,r2,r4 ; Wrap into the table
2698 add r2,r2,r12 ; Point right at the PCA
2700 rbLcka: lwarx r7,0,r2 ; Get the PTEG lock
2701 mr. r7,r7 ; Is it locked?
2702 bne- rbLckwa ; Yeah...
2703 li r7,1 ; Get the locked value
2704 stwcx. r7,0,r2 ; Take it
2705 bne- rbLcka ; Someone else was trying, try again...
2706 b rbSXga ; All done...
2708 rbLckwa: mr. r7,r7 ; Check if it is already held
2709 beq+ rbLcka ; It is clear...
2710 lwz r7,0(r2) ; Get lock word again...
2713 rbSXga: isync ; Make sure nothing used yet
2714 lwz r7,PCAallo(r2) ; Get the allocation word
2715 rlwinm. r11,r7,8,0,7 ; Isolate the autogenerated PTEs
2716 or r7,r7,r11 ; Release the autogen slots
2717 beq+ rbAintNone ; There are not any here
2718 mtcrf 0xC0,r11 ; Set the branch masks for autogens
2719 sub r11,r2,r4 ; Move back to the hash table + 1
2720 rlwinm r7,r7,0,16,7 ; Clear the autogen field
2721 subi r11,r11,1 ; Point to the PTEG
2722 stw r7,PCAallo(r2) ; Update the flags
2723 li r7,0 ; Get an invalid PTE value
2725 bf 0,rbSlot1 ; No autogen here
2726 stw r7,0x00(r11) ; Invalidate PTE
2727 rbSlot1: bf 1,rbSlot2 ; No autogen here
2728 stw r7,0x08(r11) ; Invalidate PTE
2729 rbSlot2: bf 2,rbSlot3 ; No autogen here
2730 stw r7,0x10(r11) ; Invalidate PTE
2731 rbSlot3: bf 3,rbSlot4 ; No autogen here
2732 stw r7,0x18(r11) ; Invalidate PTE
2733 rbSlot4: bf 4,rbSlot5 ; No autogen here
2734 stw r7,0x20(r11) ; Invalidate PTE
2735 rbSlot5: bf 5,rbSlot6 ; No autogen here
2736 stw r7,0x28(r11) ; Invalidate PTE
2737 rbSlot6: bf 6,rbSlot7 ; No autogen here
2738 stw r7,0x30(r11) ; Invalidate PTE
2739 rbSlot7: bf 7,rbSlotx ; No autogen here
2740 stw r7,0x38(r11) ; Invalidate PTE
2743 rbAintNone: li r7,0 ; Clear this out
2744 sync ; To make SMP happy
2745 addic. r6,r6,-64 ; Decrement the count
2746 stw r7,PCAlock(r2) ; Release the PTEG lock
2747 addi r5,r5,64 ; Move up by adjusted page number
2748 bge+ rbHash ; Not done...
2750 sync ; Make sure the memory is quiet
2753 ; Here we take the easy way out and just purge the entire TLB. This is
2754 ; certainly faster and definitly easier than blasting just the correct ones
2755 ; in the range, we only need one lock and one TLBSYNC. We would hope
2756 ; that most blocks are more than 64 pages (256K) and on every machine
2757 ; up to Book E, 64 TLBIEs will invalidate the entire table.
2760 li r5,64 ; Get number of TLB entries to purge
2761 lis r12,HIGH_ADDR(EXT(tlb_system_lock)) ; Get the TLBIE lock
2762 li r6,0 ; Start at 0
2763 ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) ; Grab up the bottom part
2765 rbTlbL: lwarx r2,0,r12 ; Get the TLBIE lock
2766 mr. r2,r2 ; Is it locked?
2767 li r2,1 ; Get our lock value
2768 bne- rbTlbL ; It is locked, go wait...
2769 stwcx. r2,0,r12 ; Try to get it
2770 bne- rbTlbL ; We was beat...
2772 rbTlbN: addic. r5,r5,-1 ; See if we did them all
2773 tlbie r6 ; Invalidate it everywhere
2774 addi r6,r6,0x1000 ; Up to the next page
2775 bgt+ rbTlbN ; Make sure we have done it all...
2777 mfspr r5,pvr ; Find out what kind of machine we are
2778 li r2,0 ; Lock clear value
2780 rlwinm r5,r5,16,16,31 ; Isolate CPU type
2781 cmplwi r5,3 ; Is this a 603?
2782 sync ; Make sure all is quiet
2783 beq- rbits603a ; It is a 603, skip the tlbsync...
2785 eieio ; Make sure that the tlbie happens first
2786 tlbsync ; wait for everyone to catch up
2788 rbits603a: sync ; Wait for quiet again
2789 stw r2,0(r12) ; Unlock invalidates
2791 sync ; Make sure that is done
2793 stw r9,0(r3) ; Unlock and chain the new first one
2794 mtmsr r0 ; Restore xlation and rupts
2795 mr r3,r10 ; Pass back the removed block
2799 rbMT: stw r9,0(r3) ; Unlock
2800 mtmsr r0 ; Restore xlation and rupts
2801 li r3,0 ; Say we did not find one
2805 rbPerm: stw r9,0(r3) ; Unlock
2806 mtmsr r0 ; Restore xlation and rupts
2807 ori r3,r10,1 ; Say we did not remove it
2813 * vm_offset_t hw_cvp_blk(pmap_t pmap, vm_offset_t va)
2815 * This is used to translate a virtual address within a block mapping entry
2816 * to a physical address. If not found, 0 is returned.
2821 .globl EXT(hw_cvp_blk)
2825 mfsprg r9,2 ; Get feature flags
2826 lwz r6,PMAP_PMAPVR(r3) ; Get the v to r translation
2827 mfmsr r0 /* Save the MSR */
2828 rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
2829 mtcrf 0x04,r9 ; Set the features
2830 xor r3,r3,r6 ; Get real address of bmap anchor
2831 rlwinm r12,r12,0,28,25 /* Clear IR and DR */
2832 la r3,PMAP_BMAPS(r3) ; Point to chain header
2834 bt pfNoMSRirb,hcbNoMSR ; No MSR...
2836 mtmsr r12 ; Translation and all off
2837 isync ; Toss prefetch
2843 li r0,loadMSR ; Get the MSR setter SC
2844 mr r3,r12 ; Get new MSR
2850 cbLck: lwarx r9,0,r3 ; Get the block map anchor and lock
2851 rlwinm. r8,r9,0,31,31 ; Is it locked?
2852 ori r8,r9,1 ; Set the lock
2853 bne- cbLckw ; Yeah...
2854 stwcx. r8,0,r3 ; Lock the bmap list
2855 bne- cbLck ; Someone else was trying, try again...
2856 b cbSXg ; All done...
2860 cbLckw: rlwinm. r5,r9,0,31,31 ; Check if it is still held
2861 beq+ cbLck ; Not no more...
2862 lwz r9,0(r3) ; Get lock word again...
2863 b cbLckw ; Check it out...
2867 nop ; Force ISYNC to last instruction in IFETCH
2873 cbSXg: rlwinm. r11,r9,0,0,26 ; Clear out flags and lock
2874 li r2,0 ; Assume we do not find anything
2875 isync ; Make sure we have not used anything yet
2877 cbChk: mr. r11,r11 ; Is there more?
2878 beq- cbDone ; No more...
2879 lwz r5,bmstart(r11) ; Get the bottom of range
2880 lwz r12,bmend(r11) ; Get the top of range
2881 cmplw cr0,r4,r5 ; Are we before the entry?
2882 cmplw cr1,r4,r12 ; Are we after of the entry?
2883 cror cr1_eq,cr0_lt,cr1_gt ; Set cr1_eq if new not in range
2884 beq- cr1,cbNo ; We are not in the range...
2886 lwz r2,bmPTEr(r11) ; Get the real part of the PTE
2887 sub r5,r4,r5 ; Get offset into area
2888 rlwinm r2,r2,0,0,19 ; Clean out everything but the page
2889 add r2,r2,r5 ; Adjust the real address
2891 cbDone: stw r9,0(r3) ; Unlock it, we are done with it (no sync needed)
2892 mtmsr r0 ; Restore translation and interrupts...
2893 isync ; Make sure it is on
2894 mr r3,r2 ; Set return physical address
2899 cbNo: lwz r11,bmnext(r11) ; Link next
2900 b cbChk ; Check it out...
2904 * hw_set_user_space(pmap)
2905 * hw_set_user_space_dis(pmap)
2907 * Indicate whether memory space needs to be switched.
2908 * We really need to turn off interrupts here, because we need to be non-preemptable
2910 * hw_set_user_space_dis is used when interruptions are already disabled. Mind the
2911 * register usage here. The VMM switch code in vmachmon.s that calls this
2912 * know what registers are in use. Check that if these change.
2918 .globl EXT(hw_set_user_space)
2920 LEXT(hw_set_user_space)
2922 mfmsr r10 /* Get the current MSR */
2923 rlwinm r9,r10,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Turn off 'rupts */
2924 mtmsr r9 /* Disable 'em */
2925 lwz r7,PMAP_PMAPVR(r3) ; Get the v to r translation
2926 lwz r4,PMAP_SPACE(r3) ; Get the space
2927 mfsprg r6,0 /* Get the per_proc_info address */
2928 xor r3,r3,r7 ; Get real address of bmap anchor
2929 stw r4,PP_USERSPACE(r6) /* Show our new address space */
2930 stw r3,PP_USERPMAP(r6) ; Show our real pmap address
2931 mtmsr r10 /* Restore interruptions */
2935 .globl EXT(hw_set_user_space_dis)
2937 LEXT(hw_set_user_space_dis)
2939 lwz r7,PMAP_PMAPVR(r3) ; Get the v to r translation
2940 lwz r4,PMAP_SPACE(r3) ; Get the space
2941 mfsprg r6,0 ; Get the per_proc_info address
2942 xor r3,r3,r7 ; Get real address of bmap anchor
2943 stw r4,PP_USERSPACE(r6) ; Show our new address space
2944 stw r3,PP_USERPMAP(r6) ; Show our real pmap address
2948 /* struct mapping *hw_cpv(struct mapping *mp) - Converts a physcial mapping CB address to virtual
2957 rlwinm. r4,r3,0,0,19 ; Round back to the mapping block allocation control block
2958 mfmsr r10 ; Get the current MSR
2959 beq- hcpvret ; Skip if we are passed a 0...
2960 andi. r9,r10,0x7FEF ; Turn off interrupts and data translation
2961 mtmsr r9 ; Disable DR and EE
2964 lwz r4,mbvrswap(r4) ; Get the conversion value
2965 mtmsr r10 ; Interrupts and DR back on
2967 xor r3,r3,r4 ; Convert to physical
2969 hcpvret: rlwinm r3,r3,0,0,26 ; Clean out any flags
2973 /* struct mapping *hw_cvp(struct mapping *mp) - Converts a virtual mapping CB address to physcial
2975 * Translation must be on for this
2984 rlwinm r4,r3,0,0,19 ; Round back to the mapping block allocation control block
2985 rlwinm r3,r3,0,0,26 ; Clean out any flags
2986 lwz r4,mbvrswap(r4) ; Get the conversion value
2987 xor r3,r3,r4 ; Convert to virtual
2991 /* int mapalc(struct mappingblok *mb) - Finds, allocates, and checks a free mapping entry in a block
2993 * Lock must already be held on mapping block list
2994 * returns 0 if all slots filled.
2995 * returns n if a slot is found and it is not the last
2996 * returns -n if a slot os found and it is the last
2997 * when n and -n are returned, the corresponding bit is cleared
3006 lwz r4,mbfree(r3) ; Get the first mask
3007 lis r0,0x8000 ; Get the mask to clear the first free bit
3008 lwz r5,mbfree+4(r3) ; Get the second mask
3009 mr r12,r3 ; Save the return
3010 cntlzw r8,r4 ; Get first free field
3011 lwz r6,mbfree+8(r3) ; Get the third mask
3012 srw. r9,r0,r8 ; Get bit corresponding to first free one
3013 lwz r7,mbfree+12(r3) ; Get the fourth mask
3014 cntlzw r10,r5 ; Get first free field in second word
3015 andc r4,r4,r9 ; Turn it off
3016 bne malcfnd0 ; Found one...
3018 srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
3019 cntlzw r11,r6 ; Get first free field in third word
3020 andc r5,r5,r9 ; Turn it off
3021 bne malcfnd1 ; Found one...
3023 srw. r9,r0,r11 ; Get bit corresponding to first free one in third word
3024 cntlzw r10,r7 ; Get first free field in fourth word
3025 andc r6,r6,r9 ; Turn it off
3026 bne malcfnd2 ; Found one...
3028 srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
3029 li r3,0 ; Assume abject failure
3030 andc r7,r7,r9 ; Turn it off
3031 beqlr ; There are none any left...
3033 addi r3,r10,96 ; Set the correct bit number
3034 stw r7,mbfree+12(r12) ; Actually allocate the slot
3036 mapafin: or r4,r4,r5 ; Merge the first two allocation maps
3037 or r6,r6,r7 ; Then the last two
3038 or. r4,r4,r6 ; Merge both halves
3039 bnelr+ ; Return if some left for next time...
3041 neg r3,r3 ; Indicate we just allocated the last one
3044 malcfnd0: stw r4,mbfree(r12) ; Actually allocate the slot
3045 mr r3,r8 ; Set the correct bit number
3046 b mapafin ; Exit now...
3048 malcfnd1: stw r5,mbfree+4(r12) ; Actually allocate the slot
3049 addi r3,r10,32 ; Set the correct bit number
3050 b mapafin ; Exit now...
3052 malcfnd2: stw r6,mbfree+8(r12) ; Actually allocate the slot
3053 addi r3,r11,64 ; Set the correct bit number
3054 b mapafin ; Exit now...
3058 * Log out all memory usage
3066 mfmsr r2 ; Get the MSR
3067 lis r10,hi16(EXT(DebugWork)) ; High part of area
3068 lis r12,hi16(EXT(mem_actual)) ; High part of actual
3069 andi. r0,r10,0x7FCF ; Interrupts and translation off
3070 ori r10,r10,lo16(EXT(DebugWork)) ; Get the entry
3071 mtmsr r0 ; Turn stuff off
3072 ori r12,r12,lo16(EXT(mem_actual)) ; Get the actual
3077 stw r0,4(r10) ; Force logging off
3078 lwz r0,0(r12) ; Get the end of memory
3080 lis r12,hi16(EXT(mem_size)) ; High part of defined memory
3081 ori r12,r12,lo16(EXT(mem_size)) ; Low part of defined memory
3082 lwz r12,0(r12) ; Make it end of defined
3084 cmplw r0,r12 ; Is there room for the data?
3085 ble- logmemexit ; No, do not even try...
3087 stw r12,0(r12) ; Set defined memory size
3088 stw r0,4(r12) ; Set the actual amount of memory
3090 lis r3,hi16(EXT(hash_table_base)) ; Hash table address
3091 lis r4,hi16(EXT(hash_table_size)) ; Hash table size
3092 lis r5,hi16(EXT(pmap_mem_regions)) ; Memory regions
3093 lis r6,hi16(EXT(mapCtl)) ; Mappings
3094 ori r3,r3,lo16(EXT(hash_table_base))
3095 ori r4,r4,lo16(EXT(hash_table_size))
3096 ori r5,r5,lo16(EXT(pmap_mem_regions))
3097 ori r6,r6,lo16(EXT(mapCtl))
3100 lwz r5,4(r5) ; Get the pointer to the phys_ent table
3101 lwz r6,0(r6) ; Get the pointer to the current mapping block
3102 stw r3,8(r12) ; Save the hash table address
3103 stw r4,12(r12) ; Save the hash table size
3104 stw r5,16(r12) ; Save the physent pointer
3105 stw r6,20(r12) ; Save the mappings
3107 addi r11,r12,0x1000 ; Point to area to move hash table and PCA
3109 add r4,r4,r4 ; Double size for both
3111 copyhash: lwz r7,0(r3) ; Copy both of them
3124 rlwinm r4,r12,20,12,31 ; Get number of phys_ents
3126 copyphys: lwz r7,0(r5) ; Copy physents
3135 addi r11,r11,4095 ; Round up to next page
3136 rlwinm r11,r11,0,0,19
3138 lwz r4,4(r6) ; Get the size of the mapping area
3140 copymaps: lwz r7,0(r6) ; Copy the mappings
3153 sub r11,r11,r12 ; Get the total length we saved
3154 stw r11,24(r12) ; Save the size
3156 logmemexit: mtmsr r2 ; Back to normal
projects / apple / xnu.git / blob