2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
23 * @APPLE_LICENSE_HEADER_END@
28 #include <db_machine_commands.h>
31 #include <mach_debug.h>
33 #include <ppc/proc_reg.h>
34 #include <ppc/exception.h>
35 #include <ppc/Performance.h>
36 #include <ppc/exception.h>
37 #include <mach/ppc/vm_param.h>
46 ; +--------+--------+--------+--------+--------+--------+--------+--------+
47 ; |00000000|00000SSS|SSSSSSSS|SSSSSSSS|SSSSPPPP|PPPPPPPP|PPPPxxxx|xxxxxxxx| - EA
48 ; +--------+--------+--------+--------+--------+--------+--------+--------+
52 ; +--------+--------+--------+
53 ; |//////BB|BBBBBBBB|BBBB////| - SID - base
54 ; +--------+--------+--------+
58 ; +--------+--------+--------+
59 ; |////////|11111111|111111//| - SID - copy 1
60 ; +--------+--------+--------+
64 ; +--------+--------+--------+
65 ; |////////|//222222|22222222| - SID - copy 2
66 ; +--------+--------+--------+
70 ; +--------+--------+--------+
71 ; |//////33|33333333|33//////| - SID - copy 3 - not needed
72 ; +--------+--------+--------+ for 65 bit VPN
76 ; +--------+--------+--------+--------+--------+--------+--------+
77 ; |00000000|00000002|22222222|11111111|111111BB|BBBBBBBB|BBBB////| - SID Hash - this is all
78 ; +--------+--------+--------+--------+--------+--------+--------+ SID copies ORed
81 ; +--------+--------+--------+--------+--------+--------+--------+
82 ; |00000000|0000000S|SSSSSSSS|SSSSSSSS|SSSSSS00|00000000|0000////| - Shifted high order EA
83 ; +--------+--------+--------+--------+--------+--------+--------+ left shifted "segment"
90 ; +--------+--------+--------+--------+--------+--------+--------+
91 ; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVV////| - VSID - SID Hash XORed
92 ; +--------+--------+--------+--------+--------+--------+--------+ with shifted EA
94 ; 0 0 1 2 3 4 4 5 6 7 7
95 ; 0 8 6 4 2 0 8 6 4 2 9
96 ; +--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+
97 ; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVPPPP|PPPPPPPP|PPPPxxxx|xxxxxxxx| - VPN
98 ; +--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+
102 /* addr64_t hw_add_map(struct pmap *pmap, struct mapping *mp) - Adds a mapping
104 * Maps a page or block into a pmap
106 * Returns 0 if add worked or the vaddr of the first overlap if not
108 * Make mapping - not block or I/O - note: this is low-level, upper should remove duplicates
110 * 1) bump mapping busy count
112 * 3) find mapping full path - finds all possible list previous elements
113 * 4) upgrade pmap to exclusive
114 * 5) add mapping to search list
120 * 11) drop mapping busy count
123 * Make mapping - block or I/O - note: this is low-level, upper should remove duplicates
125 * 1) bump mapping busy count
127 * 3) find mapping full path - finds all possible list previous elements
128 * 4) upgrade pmap to exclusive
129 * 5) add mapping to search list
131 * 7) drop mapping busy count
136 .globl EXT(hw_add_map)
140 stwu r1,-(FM_ALIGN((31-17+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
141 mflr r0 ; Save the link register
142 stw r17,FM_ARG0+0x00(r1) ; Save a register
143 stw r18,FM_ARG0+0x04(r1) ; Save a register
144 stw r19,FM_ARG0+0x08(r1) ; Save a register
145 mfsprg r19,2 ; Get feature flags
146 stw r20,FM_ARG0+0x0C(r1) ; Save a register
147 stw r21,FM_ARG0+0x10(r1) ; Save a register
148 mtcrf 0x02,r19 ; move pf64Bit cr6
149 stw r22,FM_ARG0+0x14(r1) ; Save a register
150 stw r23,FM_ARG0+0x18(r1) ; Save a register
151 stw r24,FM_ARG0+0x1C(r1) ; Save a register
152 stw r25,FM_ARG0+0x20(r1) ; Save a register
153 stw r26,FM_ARG0+0x24(r1) ; Save a register
154 stw r27,FM_ARG0+0x28(r1) ; Save a register
155 stw r28,FM_ARG0+0x2C(r1) ; Save a register
156 stw r29,FM_ARG0+0x30(r1) ; Save a register
157 stw r30,FM_ARG0+0x34(r1) ; Save a register
158 stw r31,FM_ARG0+0x38(r1) ; Save a register
159 stw r0,(FM_ALIGN((31-17+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
161 rlwinm r11,r4,0,0,19 ; Round down to get mapping block address
162 mr r28,r3 ; Save the pmap
163 mr r31,r4 ; Save the mapping
164 bt++ pf64Bitb,hamSF1 ; skip if 64-bit (only they take the hint)
165 lwz r20,pmapvr+4(r3) ; Get conversion mask for pmap
166 lwz r21,mbvrswap+4(r11) ; Get conversion mask for mapping
170 hamSF1: ld r20,pmapvr(r3) ; Get conversion mask for pmap
171 ld r21,mbvrswap(r11) ; Get conversion mask for mapping
173 hamSF1x: bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
175 mr r17,r11 ; Save the MSR
176 xor r28,r28,r20 ; Convert the pmap to physical addressing
177 xor r31,r31,r21 ; Convert the mapping to physical addressing
179 la r3,pmapSXlk(r28) ; Point to the pmap search lock
180 bl sxlkShared ; Go get a shared lock on the mapping lists
181 mr. r3,r3 ; Did we get the lock?
182 lwz r24,mpFlags(r31) ; Pick up the flags
183 bne-- hamBadLock ; Nope...
185 li r21,0 ; Remember that we have the shared lock
188 ; Note that we do a full search (i.e., no shortcut level skips, etc.)
189 ; here so that we will know the previous elements so we can dequeue them
193 hamRescan: lwz r4,mpVAddr(r31) ; Get the new vaddr top half
194 lwz r5,mpVAddr+4(r31) ; Get the new vaddr bottom half
195 mr r3,r28 ; Pass in pmap to search
196 lhz r23,mpBSize(r31) ; Get the block size for later
197 mr r29,r4 ; Save top half of vaddr for later
198 mr r30,r5 ; Save bottom half of vaddr for later
201 mfspr r0,pmc1 ; INSTRUMENT - saveinstr[16] - Take stamp before mapSearchFull
202 stw r0,0x6100+(16*16)+0x0(0) ; INSTRUMENT - Save it
203 mfspr r0,pmc2 ; INSTRUMENT - Get stamp
204 stw r0,0x6100+(16*16)+0x4(0) ; INSTRUMENT - Save it
205 mfspr r0,pmc3 ; INSTRUMENT - Get stamp
206 stw r0,0x6100+(16*16)+0x8(0) ; INSTRUMENT - Save it
207 mfspr r0,pmc4 ; INSTRUMENT - Get stamp
208 stw r0,0x6100+(16*16)+0xC(0) ; INSTRUMENT - Save it
211 bl EXT(mapSearchFull) ; Go see if we can find it
214 mfspr r0,pmc1 ; INSTRUMENT - saveinstr[14] - Take stamp after mapSearchFull
215 stw r0,0x6100+(17*16)+0x0(0) ; INSTRUMENT - Save it
216 mfspr r0,pmc2 ; INSTRUMENT - Get stamp
217 stw r0,0x6100+(17*16)+0x4(0) ; INSTRUMENT - Save it
218 mfspr r0,pmc3 ; INSTRUMENT - Get stamp
219 stw r0,0x6100+(17*16)+0x8(0) ; INSTRUMENT - Save it
220 mfspr r0,pmc4 ; INSTRUMENT - Get stamp
221 stw r0,0x6100+(17*16)+0xC(0) ; INSTRUMENT - Save it
224 andi. r0,r24,mpNest ; See if we are a nest
225 rlwinm r23,r23,12,0,19 ; Convert standard block size to bytes
226 lis r0,0x8000 ; Get 0xFFFFFFFF80000000
227 li r22,0 ; Assume high part of size is 0
228 beq++ hamNoNest ; This is not a nest...
230 rlwinm r22,r23,16,16,31 ; Convert partially converted size to segments
231 rlwinm r23,r23,16,0,3 ; Finish shift
233 hamNoNest: add r0,r0,r0 ; Get 0xFFFFFFFF00000000 for 64-bit or 0 for 32-bit
234 mr. r3,r3 ; Did we find a mapping here?
235 or r0,r0,r30 ; Make sure a carry will propagate all the way in 64-bit
236 crmove cr5_eq,cr0_eq ; Remember that if we found the mapping
237 addc r9,r0,r23 ; Add size to get last page in new range
238 or. r0,r4,r5 ; Are we beyond the end?
239 adde r8,r29,r22 ; Add the rest of the length on
240 bne-- cr5,hamOverlay ; Yeah, this is no good, can not double map...
241 rlwinm r9,r9,0,0,31 ; Clean top half of sum
242 beq++ hamFits ; We are at the end...
244 cmplw cr1,r9,r5 ; Is the bottom part of our end less?
245 cmplw r8,r4 ; Is our end before the next (top part)
246 crand cr0_eq,cr0_eq,cr1_lt ; Is the second half less and the first half equal?
247 cror cr0_eq,cr0_eq,cr0_lt ; Or is the top half less
249 bf-- cr0_eq,hamOverlay ; No, we do fit, there is an overlay...
252 ; Here we try to convert to an exclusive lock. This will fail if someone else
255 hamFits: mr. r21,r21 ; Do we already have the exclusive lock?
256 la r3,pmapSXlk(r28) ; Point to the pmap search lock
258 bne-- hamGotX ; We already have the exclusive...
260 bl sxlkPromote ; Try to promote shared to exclusive
261 mr. r3,r3 ; Could we?
262 beq++ hamGotX ; Yeah...
265 ; Since we could not promote our lock, we need to convert to it.
266 ; That means that we drop the shared lock and wait to get it
267 ; exclusive. Since we release the lock, we need to do the look up
271 la r3,pmapSXlk(r28) ; Point to the pmap search lock
272 bl sxlkConvert ; Convert shared to exclusive
273 mr. r3,r3 ; Could we?
274 bne-- hamBadLock ; Nope, we must have timed out...
276 li r21,1 ; Remember that we have the exclusive lock
277 b hamRescan ; Go look again...
283 mfspr r3,pmc1 ; INSTRUMENT - saveinstr[18] - Take stamp before mapSearchFull
284 stw r3,0x6100+(18*16)+0x0(0) ; INSTRUMENT - Save it
285 mfspr r3,pmc2 ; INSTRUMENT - Get stamp
286 stw r3,0x6100+(18*16)+0x4(0) ; INSTRUMENT - Save it
287 mfspr r3,pmc3 ; INSTRUMENT - Get stamp
288 stw r3,0x6100+(18*16)+0x8(0) ; INSTRUMENT - Save it
289 mfspr r3,pmc4 ; INSTRUMENT - Get stamp
290 stw r4,0x6100+(18*16)+0xC(0) ; INSTRUMENT - Save it
292 mr r3,r28 ; Get the pmap to insert into
293 mr r4,r31 ; Point to the mapping
294 bl EXT(mapInsert) ; Insert the mapping into the list
297 mfspr r4,pmc1 ; INSTRUMENT - saveinstr[19] - Take stamp before mapSearchFull
298 stw r4,0x6100+(19*16)+0x0(0) ; INSTRUMENT - Save it
299 mfspr r4,pmc2 ; INSTRUMENT - Get stamp
300 stw r4,0x6100+(19*16)+0x4(0) ; INSTRUMENT - Save it
301 mfspr r4,pmc3 ; INSTRUMENT - Get stamp
302 stw r4,0x6100+(19*16)+0x8(0) ; INSTRUMENT - Save it
303 mfspr r4,pmc4 ; INSTRUMENT - Get stamp
304 stw r4,0x6100+(19*16)+0xC(0) ; INSTRUMENT - Save it
307 lhz r8,mpSpace(r31) ; Get the address space
308 mfsdr1 r7 ; Get the hash table base/bounds
309 lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
310 andi. r0,r24,mpNest|mpBlock ; Is this a nest or block?
312 rlwimi r8,r8,14,4,17 ; Double address space
313 rlwinm r9,r30,20,16,31 ; Isolate the page number
314 rlwinm r10,r30,18,14,17 ; Shift EA[32:35] down to correct spot in VSID (actually shift up 14)
315 rlwimi r8,r8,28,0,3 ; Get the last nybble of the hash
316 rlwimi r10,r29,18,0,13 ; Shift EA[18:31] down to VSID (31-bit math works because of max hash table size)
317 rlwinm r7,r7,0,16,31 ; Isolate length mask (or count)
318 addi r4,r4,1 ; Bump up the mapped page count
319 xor r10,r10,r8 ; Calculate the low 32 bits of the VSID
320 stw r4,pmapResidentCnt(r28) ; Set the mapped page count
321 xor r9,r9,r10 ; Get the hash to the PTEG
323 bne-- hamDoneNP ; This is a block or nest, therefore, no physent...
325 bl mapPhysFindLock ; Go find and lock the physent
327 bt++ pf64Bitb,ham64 ; This is 64-bit...
329 lwz r11,ppLink+4(r3) ; Get the alias chain pointer
330 rlwinm r7,r7,16,0,15 ; Get the PTEG wrap size
331 slwi r9,r9,6 ; Make PTEG offset
332 ori r7,r7,0xFFC0 ; Stick in the bottom part
333 rlwinm r12,r11,0,0,25 ; Clean it up
334 and r9,r9,r7 ; Wrap offset into table
335 mr r4,r31 ; Set the link to install
336 stw r9,mpPte(r31) ; Point the mapping at the PTEG (exact offset is invalid)
337 stw r12,mpAlias+4(r31) ; Move to the mapping
338 bl mapPhyCSet32 ; Install the link
339 b hamDone ; Go finish up...
343 ham64: li r0,0xFF ; Get mask to clean up alias pointer
344 subfic r7,r7,46 ; Get number of leading zeros
345 eqv r4,r4,r4 ; Get all ones
346 ld r11,ppLink(r3) ; Get the alias chain pointer
347 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
348 srd r4,r4,r7 ; Get the wrap mask
349 sldi r9,r9,7 ; Change hash to PTEG offset
350 andc r11,r11,r0 ; Clean out the lock and flags
351 and r9,r9,r4 ; Wrap to PTEG
353 stw r9,mpPte(r31) ; Point the mapping at the PTEG (exact offset is invalid)
354 std r11,mpAlias(r31) ; Set the alias pointer in the mapping
356 bl mapPhyCSet64 ; Install the link
358 hamDone: bl mapPhysUnlock ; Unlock the physent chain
360 hamDoneNP: la r3,pmapSXlk(r28) ; Point to the pmap search lock
361 bl sxlkUnlock ; Unlock the search list
363 mr r3,r31 ; Get the mapping pointer
364 bl mapDropBusy ; Drop the busy count
366 li r3,0 ; Set successful return
367 li r4,0 ; Set successful return
369 hamReturn: bt++ pf64Bitb,hamR64 ; Yes...
371 mtmsr r17 ; Restore enables/translation/etc.
373 b hamReturnC ; Join common...
375 hamR64: mtmsrd r17 ; Restore enables/translation/etc.
380 mfspr r0,pmc1 ; INSTRUMENT - saveinstr[20] - Take stamp before mapSearchFull
381 stw r0,0x6100+(20*16)+0x0(0) ; INSTRUMENT - Save it
382 mfspr r0,pmc2 ; INSTRUMENT - Get stamp
383 stw r0,0x6100+(20*16)+0x4(0) ; INSTRUMENT - Save it
384 mfspr r0,pmc3 ; INSTRUMENT - Get stamp
385 stw r0,0x6100+(20*16)+0x8(0) ; INSTRUMENT - Save it
386 mfspr r0,pmc4 ; INSTRUMENT - Get stamp
387 stw r0,0x6100+(20*16)+0xC(0) ; INSTRUMENT - Save it
389 lwz r0,(FM_ALIGN((31-17+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Get the return
390 lwz r17,FM_ARG0+0x00(r1) ; Save a register
391 lwz r18,FM_ARG0+0x04(r1) ; Save a register
392 lwz r19,FM_ARG0+0x08(r1) ; Save a register
393 lwz r20,FM_ARG0+0x0C(r1) ; Save a register
394 mtlr r0 ; Restore the return
395 lwz r21,FM_ARG0+0x10(r1) ; Save a register
396 lwz r22,FM_ARG0+0x14(r1) ; Save a register
397 lwz r23,FM_ARG0+0x18(r1) ; Save a register
398 lwz r24,FM_ARG0+0x1C(r1) ; Save a register
399 lwz r25,FM_ARG0+0x20(r1) ; Save a register
400 lwz r26,FM_ARG0+0x24(r1) ; Save a register
401 lwz r27,FM_ARG0+0x28(r1) ; Save a register
402 lwz r28,FM_ARG0+0x2C(r1) ; Save a register
403 lwz r29,FM_ARG0+0x30(r1) ; Save a register
404 lwz r30,FM_ARG0+0x34(r1) ; Save a register
405 lwz r31,FM_ARG0+0x38(r1) ; Save a register
406 lwz r1,0(r1) ; Pop the stack
413 hamOverlay: lwz r22,mpFlags(r3) ; Get the overlay flags
414 li r0,mpC|mpR ; Get a mask to turn off RC bits
415 lwz r23,mpFlags(r31) ; Get the requested flags
416 lwz r20,mpVAddr(r3) ; Get the overlay address
417 lwz r8,mpVAddr(r31) ; Get the requested address
418 lwz r21,mpVAddr+4(r3) ; Get the overlay address
419 lwz r9,mpVAddr+4(r31) ; Get the requested address
420 lhz r10,mpBSize(r3) ; Get the overlay length
421 lhz r11,mpBSize(r31) ; Get the requested length
422 lwz r24,mpPAddr(r3) ; Get the overlay physical address
423 lwz r25,mpPAddr(r31) ; Get the requested physical address
424 andc r21,r21,r0 ; Clear RC bits
425 andc r9,r9,r0 ; Clear RC bits
427 la r3,pmapSXlk(r28) ; Point to the pmap search lock
428 bl sxlkUnlock ; Unlock the search list
430 rlwinm. r0,r22,0,mpRIPb,mpRIPb ; Are we in the process of removing this one?
431 mr r3,r20 ; Save the top of the colliding address
432 rlwinm r4,r21,0,0,19 ; Save the bottom of the colliding address
434 bne++ hamRemv ; Removing, go say so so we help...
436 cmplw r20,r8 ; High part of vaddr the same?
437 cmplw cr1,r21,r9 ; Low part?
438 crand cr5_eq,cr0_eq,cr1_eq ; Remember if same
440 cmplw r10,r11 ; Size the same?
441 cmplw cr1,r24,r25 ; Physical address?
442 crand cr5_eq,cr5_eq,cr0_eq ; Remember
443 crand cr5_eq,cr5_eq,cr1_eq ; Remember if same
445 xor r23,r23,r22 ; Check for differences in flags
446 ori r23,r23,mpFIP ; "Fault in Progress" is ok to be different
447 xori r23,r23,mpFIP ; Force mpFIP off
448 rlwinm. r0,r23,0,mpSpecialb,mpListsb-1 ; See if any important flags are different
449 crand cr5_eq,cr5_eq,cr0_eq ; Merge in final check
450 bf-- cr5_eq,hamReturn ; This is not the same, so we just return a collision...
452 ori r4,r4,mapRtMapDup ; Set duplicate
453 b hamReturn ; And leave...
455 hamRemv: ori r4,r4,mapRtRemove ; We are in the process of removing the collision
456 b hamReturn ; Come back yall...
460 hamBadLock: li r3,0 ; Set lock time out error code
461 li r4,mapRtBadLk ; Set lock time out error code
462 b hamReturn ; Leave....
469 * mapping *hw_rem_map(pmap, vaddr, addr64_t *next) - remove a mapping from the system.
471 * Upon entry, R3 contains a pointer to a pmap. Since vaddr is
472 * a 64-bit quantity, it is a long long so it is in R4 and R5.
474 * We return the virtual address of the removed mapping as a
477 * Note that this is designed to be called from 32-bit mode with a stack.
479 * We disable translation and all interruptions here. This keeps is
480 * from having to worry about a deadlock due to having anything locked
481 * and needing it to process a fault.
483 * Note that this must be done with both interruptions off and VM off
485 * Remove mapping via pmap, regular page, no pte
488 * 2) find mapping full path - finds all possible list previous elements
489 * 4) upgrade pmap to exclusive
490 * 3) bump mapping busy count
491 * 5) remove mapping from search list
494 * 8) remove from physent
496 * 10) drop mapping busy count
497 * 11) drain mapping busy count
500 * Remove mapping via pmap, regular page, with pte
503 * 2) find mapping full path - finds all possible list previous elements
504 * 3) upgrade lock to exclusive
505 * 4) bump mapping busy count
507 * 6) invalidate pte and tlbie
508 * 7) atomic merge rc into physent
510 * 9) remove mapping from search list
513 * 12) remove from physent
515 * 14) drop mapping busy count
516 * 15) drain mapping busy count
519 * Remove mapping via pmap, I/O or block
522 * 2) find mapping full path - finds all possible list previous elements
523 * 3) upgrade lock to exclusive
524 * 4) bump mapping busy count
525 * 5) mark remove-in-progress
526 * 6) check and bump remove chunk cursor if needed
528 * 8) if something to invalidate, go to step 11
531 * 10) return with mapRtRemove to force higher level to call again
534 * 12) invalidate ptes, no tlbie
536 * 14) repeat 11 - 13 for all pages in chunk
537 * 15) if not final chunk, go to step 9
538 * 16) invalidate tlb entries for the whole block map but no more than the full tlb
539 * 17) lock pmap share
540 * 18) find mapping full path - finds all possible list previous elements
541 * 19) upgrade lock to exclusive
542 * 20) remove mapping from search list
543 * 21) drop mapping busy count
544 * 22) drain mapping busy count
549 .globl EXT(hw_rem_map)
554 ; NOTE NOTE NOTE - IF WE CHANGE THIS STACK FRAME STUFF WE NEED TO CHANGE
555 ; THE HW_PURGE_* ROUTINES ALSO
558 #define hrmStackSize ((31-15+1)*4)+4
559 stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
560 mflr r0 ; Save the link register
561 stw r15,FM_ARG0+0x00(r1) ; Save a register
562 stw r16,FM_ARG0+0x04(r1) ; Save a register
563 stw r17,FM_ARG0+0x08(r1) ; Save a register
564 stw r18,FM_ARG0+0x0C(r1) ; Save a register
565 stw r19,FM_ARG0+0x10(r1) ; Save a register
566 mfsprg r19,2 ; Get feature flags
567 stw r20,FM_ARG0+0x14(r1) ; Save a register
568 stw r21,FM_ARG0+0x18(r1) ; Save a register
569 mtcrf 0x02,r19 ; move pf64Bit cr6
570 stw r22,FM_ARG0+0x1C(r1) ; Save a register
571 stw r23,FM_ARG0+0x20(r1) ; Save a register
572 stw r24,FM_ARG0+0x24(r1) ; Save a register
573 stw r25,FM_ARG0+0x28(r1) ; Save a register
574 stw r26,FM_ARG0+0x2C(r1) ; Save a register
575 stw r27,FM_ARG0+0x30(r1) ; Save a register
576 stw r28,FM_ARG0+0x34(r1) ; Save a register
577 stw r29,FM_ARG0+0x38(r1) ; Save a register
578 stw r30,FM_ARG0+0x3C(r1) ; Save a register
579 stw r31,FM_ARG0+0x40(r1) ; Save a register
580 stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
581 stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
583 bt++ pf64Bitb,hrmSF1 ; skip if 64-bit (only they take the hint)
584 lwz r9,pmapvr+4(r3) ; Get conversion mask
587 hrmSF1: ld r9,pmapvr(r3) ; Get conversion mask
590 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
592 xor r28,r3,r9 ; Convert the pmap to physical addressing
595 ; Here is where we join in from the hw_purge_* routines
598 hrmJoin: mfsprg r19,2 ; Get feature flags again (for alternate entries)
600 mr r17,r11 ; Save the MSR
601 mr r29,r4 ; Top half of vaddr
602 mr r30,r5 ; Bottom half of vaddr
604 la r3,pmapSXlk(r28) ; Point to the pmap search lock
605 bl sxlkShared ; Go get a shared lock on the mapping lists
606 mr. r3,r3 ; Did we get the lock?
607 bne-- hrmBadLock ; Nope...
610 ; Note that we do a full search (i.e., no shortcut level skips, etc.)
611 ; here so that we will know the previous elements so we can dequeue them
612 ; later. Note: we get back mpFlags in R7.
615 mr r3,r28 ; Pass in pmap to search
616 mr r4,r29 ; High order of address
617 mr r5,r30 ; Low order of address
618 bl EXT(mapSearchFull) ; Go see if we can find it
620 andi. r0,r7,lo16(mpPerm|mpSpecial|mpNest) ; Is this nested, special, or a perm mapping?
621 mr r20,r7 ; Remember mpFlags
622 rlwinm r0,r7,0,mpRemovableb,mpRemovableb ; Are we allowed to remove it?
623 crmove cr5_eq,cr0_eq ; Remember if we should remove this
624 mr. r31,r3 ; Did we? (And remember mapping address for later)
625 cmplwi cr1,r0,0 ; Are we allowed to remove?
626 mr r15,r4 ; Save top of next vaddr
627 crorc cr5_eq,cr5_eq,cr1_eq ; cr5_eq is true if this is not removable
628 mr r16,r5 ; Save bottom of next vaddr
629 beq hrmNotFound ; Nope, not found...
631 bf-- cr5_eq,hrmPerm ; This one can't be removed...
633 ; Here we try to promote to an exclusive lock. This will fail if someone else
637 la r3,pmapSXlk(r28) ; Point to the pmap search lock
638 bl sxlkPromote ; Try to promote shared to exclusive
639 mr. r3,r3 ; Could we?
640 beq++ hrmGotX ; Yeah...
643 ; Since we could not promote our lock, we need to convert to it.
644 ; That means that we drop the shared lock and wait to get it
645 ; exclusive. Since we release the lock, we need to do the look up
649 la r3,pmapSXlk(r28) ; Point to the pmap search lock
650 bl sxlkConvert ; Convert shared to exclusive
651 mr. r3,r3 ; Could we?
652 bne-- hrmBadLock ; Nope, we must have timed out...
654 mr r3,r28 ; Pass in pmap to search
655 mr r4,r29 ; High order of address
656 mr r5,r30 ; Low order of address
657 bl EXT(mapSearchFull) ; Rescan the list
659 andi. r0,r7,lo16(mpPerm|mpSpecial|mpNest) ; Is this nested, special, or a perm mapping?
660 rlwinm r0,r7,0,mpRemovableb,mpRemovableb ; Are we allowed to remove it?
661 crmove cr5_eq,cr0_eq ; Remember if we should remove this
662 mr. r31,r3 ; Did we lose it when we converted?
663 cmplwi cr1,r0,0 ; Are we allowed to remove?
664 mr r20,r7 ; Remember mpFlags
665 crorc cr5_eq,cr5_eq,cr1_eq ; cr5_eq is true if this is not removable
666 mr r15,r4 ; Save top of next vaddr
667 mr r16,r5 ; Save bottom of next vaddr
668 beq-- hrmNotFound ; Yeah, we did, someone tossed it for us...
670 bf-- cr5_eq,hrmPerm ; This one can't be removed...
673 ; We have an exclusive lock on the mapping chain. And we
674 ; also have the busy count bumped in the mapping so it can
678 hrmGotX: mr r3,r31 ; Get the mapping
679 bl mapBumpBusy ; Bump up the busy count
682 ; Invalidate any PTEs associated with this
683 ; mapping (more than one if a block) and accumulate the reference
686 ; Here is also where we need to split 32- and 64-bit processing
689 lwz r21,mpPte(r31) ; Grab the offset to the PTE
690 rlwinm r23,r29,0,1,0 ; Copy high order vaddr to high if 64-bit machine
691 mfsdr1 r29 ; Get the hash table base and size
692 rlwinm r0,r20,0,mpBlockb,mpBlockb ; Is this a block mapping?
693 andi. r2,r20,lo16(mpSpecial|mpNest) ; Is this nest or special mapping?
694 cmplwi cr5,r0,0 ; Remember if this is a block mapping
695 rlwinm r0,r21,0,mpHValidb,mpHValidb ; See if we actually have a PTE
696 ori r2,r2,0xFFFF ; Get mask to clean out hash table base (works for both 32- and 64-bit)
697 cmpwi cr1,r0,0 ; Have we made a PTE for this yet?
698 rlwinm r21,r21,0,0,30 ; Clear out valid bit
699 crorc cr0_eq,cr1_eq,cr0_eq ; No need to look at PTE if none or a special mapping
700 rlwimi r23,r30,0,0,31 ; Insert low under high part of address
701 andc r29,r29,r2 ; Clean up hash table base
702 li r22,0 ; Clear this on out (also sets RC to 0 if we bail)
703 mr r30,r23 ; Move the now merged vaddr to the correct register
704 add r26,r29,r21 ; Point to the PTEG slot
706 bt++ pf64Bitb,hrmSplit64 ; Go do 64-bit version...
708 rlwinm r9,r21,28,4,29 ; Convert PTEG to PCA entry
709 bne- cr5,hrmBlock32 ; Go treat block specially...
710 subfic r9,r9,-4 ; Get the PCA entry offset
711 bt- cr0_eq,hrmPysDQ32 ; Skip next if no possible PTE...
712 add r7,r9,r29 ; Point to the PCA slot
715 bl mapLockPteg ; Go lock up the PTEG (Note: we need to save R6 to set PCA)
717 lwz r21,mpPte(r31) ; Get the quick pointer again
718 lwz r5,0(r26) ; Get the top of PTE
720 rlwinm. r0,r21,0,mpHValidb,mpHValidb ; See if we actually have a PTE
721 rlwinm r21,r21,0,0,30 ; Clear out valid bit
722 rlwinm r5,r5,0,1,31 ; Turn off valid bit in PTE
723 stw r21,mpPte(r31) ; Make sure we invalidate mpPte, still pointing to PTEG (keep walk_page from making a mistake)
724 beq- hrmUlckPCA32 ; Pte is gone, no need to invalidate...
726 stw r5,0(r26) ; Invalidate the PTE
728 li r9,tlbieLock ; Get the TLBIE lock
730 sync ; Make sure the invalid PTE is actually in memory
732 hrmPtlb32: lwarx r5,0,r9 ; Get the TLBIE lock
733 mr. r5,r5 ; Is it locked?
734 li r5,1 ; Get locked indicator
735 bne- hrmPtlb32 ; It is locked, go spin...
736 stwcx. r5,0,r9 ; Try to get it
737 bne- hrmPtlb32 ; We was beat...
739 rlwinm. r0,r19,0,pfSMPcapb,pfSMPcapb ; Can this processor do SMP?
741 tlbie r30 ; Invalidate it all corresponding TLB entries
743 beq- hrmNTlbs ; Jump if we can not do a TLBSYNC....
745 eieio ; Make sure that the tlbie happens first
746 tlbsync ; Wait for everyone to catch up
747 sync ; Make sure of it all
749 hrmNTlbs: li r0,0 ; Clear this
750 rlwinm r2,r21,29,29,31 ; Get slot number (8 byte entries)
751 stw r0,tlbieLock(0) ; Clear the tlbie lock
752 lis r0,0x8000 ; Get bit for slot 0
753 eieio ; Make sure those RC bit have been stashed in PTE
755 srw r0,r0,r2 ; Get the allocation hash mask
756 lwz r22,4(r26) ; Get the latest reference and change bits
757 or r6,r6,r0 ; Show that this slot is free
760 eieio ; Make sure all updates come first
761 stw r6,0(r7) ; Unlock the PTEG
764 ; Now, it is time to remove the mapping and unlock the chain.
765 ; But first, we need to make sure no one else is using this
766 ; mapping so we drain the busy now
769 hrmPysDQ32: mr r3,r31 ; Point to the mapping
770 bl mapDrainBusy ; Go wait until mapping is unused
772 mr r3,r28 ; Get the pmap to remove from
773 mr r4,r31 ; Point to the mapping
774 bl EXT(mapRemove) ; Remove the mapping from the list
777 lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
778 andi. r0,r20,lo16(mpSpecial|mpNest) ; Is this nest or special mapping?
779 cmplwi cr1,r0,0 ; Special thingie?
780 la r3,pmapSXlk(r28) ; Point to the pmap search lock
781 subi r4,r4,1 ; Drop down the mapped page count
782 stw r4,pmapResidentCnt(r28) ; Set the mapped page count
783 bl sxlkUnlock ; Unlock the search list
785 bne-- cr1,hrmRetn32 ; This one has no real memory associated with it so we are done...
787 bl mapPhysFindLock ; Go find and lock the physent
789 lwz r9,ppLink+4(r3) ; Get first mapping
791 mr r4,r22 ; Get the RC bits we just got
792 bl mapPhysMerge ; Go merge the RC bits
794 rlwinm r9,r9,0,0,25 ; Clear the flags from the mapping pointer
796 cmplw r9,r31 ; Are we the first on the list?
797 bne- hrmNot1st ; Nope...
800 lwz r4,mpAlias+4(r31) ; Get our new forward pointer
801 stw r9,mpAlias+4(r31) ; Make sure we are off the chain
802 bl mapPhyCSet32 ; Go set the physent link and preserve flags
804 b hrmPhyDQd ; Join up and unlock it all...
808 hrmPerm: li r8,-4096 ; Get the value we need to round down to a page
809 and r8,r8,r31 ; Get back to a page
810 lwz r8,mbvrswap+4(r8) ; Get last half of virtual to real swap
812 la r3,pmapSXlk(r28) ; Point to the pmap search lock
813 bl sxlkUnlock ; Unlock the search list
815 xor r3,r31,r8 ; Flip mapping address to virtual
816 ori r3,r3,mapRtPerm ; Set permanent mapping error
819 hrmBadLock: li r3,mapRtBadLk ; Set bad lock
823 la r3,pmapSXlk(r28) ; Point to the pmap search lock
824 bl sxlkUnlock ; Unlock the search list
827 mr r3,r31 ; Point to the mapping
828 bl mapDropBusy ; Drop the busy here since we need to come back
829 li r3,mapRtRemove ; Say we are still removing this
835 la r3,pmapSXlk(r28) ; Point to the pmap search lock
836 bl sxlkUnlock ; Unlock the search list
837 li r3,0 ; Make sure we know we did not find it
839 hrmErRtn: bt++ pf64Bitb,hrmSF1z ; skip if 64-bit (only they take the hint)
841 mtmsr r17 ; Restore enables/translation/etc.
843 b hrmRetnCmn ; Join the common return code...
845 hrmSF1z: mtmsrd r17 ; Restore enables/translation/etc.
847 b hrmRetnCmn ; Join the common return code...
851 hrmNot1st: mr. r8,r9 ; Remember and test current node
852 beq- hrmPhyDQd ; Could not find our node, someone must have unmapped us...
853 lwz r9,mpAlias+4(r9) ; Chain to the next
854 cmplw r9,r31 ; Is this us?
855 bne- hrmNot1st ; Not us...
857 lwz r9,mpAlias+4(r9) ; Get our forward pointer
858 stw r9,mpAlias+4(r8) ; Unchain us
862 hrmPhyDQd: bl mapPhysUnlock ; Unlock the physent chain
864 hrmRetn32: rlwinm r8,r31,0,0,19 ; Find start of page
865 mr r3,r31 ; Copy the pointer to the mapping
866 lwz r8,mbvrswap+4(r8) ; Get last half of virtual to real swap
867 bl mapDrainBusy ; Go wait until mapping is unused
869 xor r3,r31,r8 ; Flip mapping address to virtual
871 mtmsr r17 ; Restore enables/translation/etc.
874 hrmRetnCmn: lwz r6,FM_ARG0+0x44(r1) ; Get address to save next mapped vaddr
875 lwz r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
876 lwz r17,FM_ARG0+0x08(r1) ; Restore a register
877 lwz r18,FM_ARG0+0x0C(r1) ; Restore a register
878 mr. r6,r6 ; Should we pass back the "next" vaddr?
879 lwz r19,FM_ARG0+0x10(r1) ; Restore a register
880 lwz r20,FM_ARG0+0x14(r1) ; Restore a register
881 mtlr r0 ; Restore the return
883 rlwinm r16,r16,0,0,19 ; Clean to a page boundary
884 beq hrmNoNextAdr ; Do not pass back the next vaddr...
885 stw r15,0(r6) ; Pass back the top of the next vaddr
886 stw r16,4(r6) ; Pass back the bottom of the next vaddr
889 lwz r15,FM_ARG0+0x00(r1) ; Restore a register
890 lwz r16,FM_ARG0+0x04(r1) ; Restore a register
891 lwz r21,FM_ARG0+0x18(r1) ; Restore a register
892 rlwinm r3,r3,0,0,31 ; Clear top of register if 64-bit
893 lwz r22,FM_ARG0+0x1C(r1) ; Restore a register
894 lwz r23,FM_ARG0+0x20(r1) ; Restore a register
895 lwz r24,FM_ARG0+0x24(r1) ; Restore a register
896 lwz r25,FM_ARG0+0x28(r1) ; Restore a register
897 lwz r26,FM_ARG0+0x2C(r1) ; Restore a register
898 lwz r27,FM_ARG0+0x30(r1) ; Restore a register
899 lwz r28,FM_ARG0+0x34(r1) ; Restore a register
900 lwz r29,FM_ARG0+0x38(r1) ; Restore a register
901 lwz r30,FM_ARG0+0x3C(r1) ; Restore a register
902 lwz r31,FM_ARG0+0x40(r1) ; Restore a register
903 lwz r1,0(r1) ; Pop the stack
907 ; Here is where we come when all is lost. Somehow, we failed a mapping function
908 ; that must work... All hope is gone. Alas, we die.......
911 hrmPanic: lis r0,hi16(Choke) ; System abend
912 ori r0,r0,lo16(Choke) ; System abend
913 li r3,failMapping ; Show that we failed some kind of mapping thing
918 ; Invalidate block mappings by invalidating a chunk of autogen PTEs in PTEGs hashed
919 ; in the range. Then, if we did not finish, return a code indicating that we need to
920 ; be called again. Eventually, we will finish and then, we will do a TLBIE for each
921 ; PTEG up to the point where we have cleared it all (64 for 32-bit architecture)
923 ; A potential speed up is that we stop the invalidate loop once we have walked through
924 ; the hash table once. This really is not worth the trouble because we need to have
925 ; mapped 1/2 of physical RAM in an individual block. Way unlikely.
927 ; We should rethink this and see if we think it will be faster to check PTE and
928 ; only invalidate the specific PTE rather than all block map PTEs in the PTEG.
934 lhz r23,mpSpace(r31) ; Get the address space hash
935 lhz r25,mpBSize(r31) ; Get the number of pages in block
936 lwz r9,mpBlkRemCur(r31) ; Get our current remove position
937 ori r0,r20,mpRIP ; Turn on the remove in progress flag
938 mfsdr1 r29 ; Get the hash table base and size
939 rlwinm r24,r23,maxAdrSpb,32-maxAdrSpb-maxAdrSpb,31-maxAdrSpb ; Get high order of hash
940 lwz r27,mpVAddr+4(r31) ; Get the base vaddr
941 sub r4,r25,r9 ; Get number of pages left
942 cmplw cr1,r9,r25 ; Have we already hit the end?
943 addi r10,r9,mapRemChunk ; Point to the start of the next chunk
944 addi r2,r4,-mapRemChunk ; See if mapRemChunk or more
945 rlwinm r26,r29,16,7,15 ; Get the hash table size
946 srawi r2,r2,31 ; We have -1 if less than mapRemChunk or 0 if equal or more
947 stb r0,mpFlags+3(r31) ; Save the flags with the mpRIP bit on
948 subi r4,r4,mapRemChunk-1 ; Back off for a running start (will be negative for more than mapRemChunk)
949 cmpwi cr7,r2,0 ; Remember if we have finished
950 slwi r0,r9,12 ; Make cursor into page offset
951 or r24,r24,r23 ; Get full hash
952 and r4,r4,r2 ; If more than a chunk, bring this back to 0
953 rlwinm r29,r29,0,0,15 ; Isolate the hash table base
954 add r27,r27,r0 ; Adjust vaddr to start of current chunk
955 addi r4,r4,mapRemChunk-1 ; Add mapRemChunk-1 to get max(num left, chunksize)
957 bgt- cr1,hrmEndInSight ; Someone is already doing the last hunk...
959 la r3,pmapSXlk(r28) ; Point to the pmap search lock
960 stw r10,mpBlkRemCur(r31) ; Set next chunk to do (note: this may indicate after end)
961 bl sxlkUnlock ; Unlock the search list while we are invalidating
963 rlwinm r8,r27,4+maxAdrSpb,31-maxAdrSpb-3,31-maxAdrSpb ; Isolate the segment
964 rlwinm r30,r27,26,6,25 ; Shift vaddr to PTEG offset (and remember VADDR in R27)
965 xor r24,r24,r8 ; Get the proper VSID
966 rlwinm r21,r27,26,10,25 ; Shift page index to PTEG offset (and remember VADDR in R27)
967 ori r26,r26,lo16(0xFFC0) ; Stick in the rest of the length
968 rlwinm r22,r4,6,10,25 ; Shift size to PTEG offset
969 rlwinm r24,r24,6,0,25 ; Shift hash to PTEG units
970 add r22,r22,r30 ; Get end address (in PTEG units)
972 hrmBInv32: rlwinm r23,r30,0,10,25 ; Isolate just the page index
973 xor r23,r23,r24 ; Hash it
974 and r23,r23,r26 ; Wrap it into the table
975 rlwinm r3,r23,28,4,29 ; Change to PCA offset
976 subfic r3,r3,-4 ; Get the PCA entry offset
977 add r7,r3,r29 ; Point to the PCA slot
978 cmplw cr5,r30,r22 ; Check if we reached the end of the range
979 addi r30,r30,64 ; bump to the next vaddr
981 bl mapLockPteg ; Lock the PTEG
983 rlwinm. r4,r6,16,0,7 ; Position, save, and test block mappings in PCA
984 add r5,r23,r29 ; Point to the PTEG
985 li r0,0 ; Set an invalid PTE value
986 beq+ hrmBNone32 ; No block map PTEs in this PTEG...
987 mtcrf 0x80,r4 ; Set CRs to select PTE slots
988 mtcrf 0x40,r4 ; Set CRs to select PTE slots
990 bf 0,hrmSlot0 ; No autogen here
991 stw r0,0x00(r5) ; Invalidate PTE
993 hrmSlot0: bf 1,hrmSlot1 ; No autogen here
994 stw r0,0x08(r5) ; Invalidate PTE
996 hrmSlot1: bf 2,hrmSlot2 ; No autogen here
997 stw r0,0x10(r5) ; Invalidate PTE
999 hrmSlot2: bf 3,hrmSlot3 ; No autogen here
1000 stw r0,0x18(r5) ; Invalidate PTE
1002 hrmSlot3: bf 4,hrmSlot4 ; No autogen here
1003 stw r0,0x20(r5) ; Invalidate PTE
1005 hrmSlot4: bf 5,hrmSlot5 ; No autogen here
1006 stw r0,0x28(r5) ; Invalidate PTE
1008 hrmSlot5: bf 6,hrmSlot6 ; No autogen here
1009 stw r0,0x30(r5) ; Invalidate PTE
1011 hrmSlot6: bf 7,hrmSlot7 ; No autogen here
1012 stw r0,0x38(r5) ; Invalidate PTE
1014 hrmSlot7: rlwinm r0,r4,16,16,23 ; Move in use to autogen
1015 or r6,r6,r4 ; Flip on the free bits that corrospond to the autogens we cleared
1016 andc r6,r6,r0 ; Turn off all the old autogen bits
1018 hrmBNone32: eieio ; Make sure all updates come first
1020 stw r6,0(r7) ; Unlock and set the PCA
1022 bne+ cr5,hrmBInv32 ; Go invalidate the next...
1024 bge+ cr7,hrmDoneChunk ; We have not as yet done the last chunk, go tell our caller to call again...
1026 mr r3,r31 ; Copy the pointer to the mapping
1027 bl mapDrainBusy ; Go wait until we are sure all other removers are done with this one
1029 sync ; Make sure memory is consistent
1031 subi r5,r25,63 ; Subtract TLB size from page count (note we are 0 based here)
1032 li r6,63 ; Assume full invalidate for now
1033 srawi r5,r5,31 ; Make 0 if we need a full purge, -1 otherwise
1034 andc r6,r6,r5 ; Clear max if we have less to do
1035 and r5,r25,r5 ; Clear count if we have more than max
1036 lwz r27,mpVAddr+4(r31) ; Get the base vaddr again
1037 li r7,tlbieLock ; Get the TLBIE lock
1038 or r5,r5,r6 ; Get number of TLBIEs needed
1040 hrmBTLBlck: lwarx r2,0,r7 ; Get the TLBIE lock
1041 mr. r2,r2 ; Is it locked?
1042 li r2,1 ; Get our lock value
1043 bne- hrmBTLBlck ; It is locked, go wait...
1044 stwcx. r2,0,r7 ; Try to get it
1045 bne- hrmBTLBlck ; We was beat...
1047 hrmBTLBi: addic. r5,r5,-1 ; See if we did them all
1048 tlbie r27 ; Invalidate it everywhere
1049 addi r27,r27,0x1000 ; Up to the next page
1050 bge+ hrmBTLBi ; Make sure we have done it all...
1052 rlwinm. r0,r19,0,pfSMPcapb,pfSMPcapb ; Can this processor do SMP?
1053 li r2,0 ; Lock clear value
1055 sync ; Make sure all is quiet
1056 beq- hrmBNTlbs ; Jump if we can not do a TLBSYNC....
1058 eieio ; Make sure that the tlbie happens first
1059 tlbsync ; Wait for everyone to catch up
1060 sync ; Wait for quiet again
1062 hrmBNTlbs: stw r2,tlbieLock(0) ; Clear the tlbie lock
1064 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1065 bl sxlkShared ; Go get a shared lock on the mapping lists
1066 mr. r3,r3 ; Did we get the lock?
1067 bne- hrmPanic ; Nope...
1069 lwz r4,mpVAddr(r31) ; High order of address
1070 lwz r5,mpVAddr+4(r31) ; Low order of address
1071 mr r3,r28 ; Pass in pmap to search
1072 mr r29,r4 ; Save this in case we need it (only promote fails)
1073 mr r30,r5 ; Save this in case we need it (only promote fails)
1074 bl EXT(mapSearchFull) ; Go see if we can find it
1076 mr. r3,r3 ; Did we? (And remember mapping address for later)
1077 mr r15,r4 ; Save top of next vaddr
1078 mr r16,r5 ; Save bottom of next vaddr
1079 beq- hrmPanic ; Nope, not found...
1081 cmplw r3,r31 ; Same mapping?
1082 bne- hrmPanic ; Not good...
1084 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1085 bl sxlkPromote ; Try to promote shared to exclusive
1086 mr. r3,r3 ; Could we?
1087 mr r3,r31 ; Restore the mapping pointer
1088 beq+ hrmBDone1 ; Yeah...
1090 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1091 bl sxlkConvert ; Convert shared to exclusive
1092 mr. r3,r3 ; Could we?
1093 bne-- hrmPanic ; Nope, we must have timed out...
1095 mr r3,r28 ; Pass in pmap to search
1096 mr r4,r29 ; High order of address
1097 mr r5,r30 ; Low order of address
1098 bl EXT(mapSearchFull) ; Rescan the list
1100 mr. r3,r3 ; Did we lose it when we converted?
1101 mr r15,r4 ; Save top of next vaddr
1102 mr r16,r5 ; Save bottom of next vaddr
1103 beq-- hrmPanic ; Yeah, we did, someone tossed it for us...
1105 hrmBDone1: bl mapDrainBusy ; Go wait until mapping is unused
1107 mr r3,r28 ; Get the pmap to remove from
1108 mr r4,r31 ; Point to the mapping
1109 bl EXT(mapRemove) ; Remove the mapping from the list
1111 lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
1112 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1113 subi r4,r4,1 ; Drop down the mapped page count
1114 stw r4,pmapResidentCnt(r28) ; Set the mapped page count
1115 bl sxlkUnlock ; Unlock the search list
1117 b hrmRetn32 ; We are all done, get out...
1120 ; Here we handle the 64-bit version of hw_rem_map
1125 hrmSplit64: rlwinm r9,r21,27,5,29 ; Convert PTEG to PCA entry
1126 bne-- cr5,hrmBlock64 ; Go treat block specially...
1127 subfic r9,r9,-4 ; Get the PCA entry offset
1128 bt-- cr0_eq,hrmPysDQ64 ; Skip next if no possible PTE...
1129 add r7,r9,r29 ; Point to the PCA slot
1131 bl mapLockPteg ; Go lock up the PTEG
1133 lwz r21,mpPte(r31) ; Get the quick pointer again
1134 ld r5,0(r26) ; Get the top of PTE
1136 rlwinm. r0,r21,0,mpHValidb,mpHValidb ; See if we actually have a PTE
1137 rlwinm r21,r21,0,0,30 ; Clear out valid bit
1138 sldi r23,r5,16 ; Shift AVPN up to EA format
1139 rldicr r5,r5,0,62 ; Clear the valid bit
1140 rldimi r23,r30,0,36 ; Insert the page portion of the VPN
1141 stw r21,mpPte(r31) ; Make sure we invalidate mpPte but keep pointing to PTEG (keep walk_page from making a mistake)
1142 beq-- hrmUlckPCA64 ; Pte is gone, no need to invalidate...
1144 std r5,0(r26) ; Invalidate the PTE
1146 li r9,tlbieLock ; Get the TLBIE lock
1148 sync ; Make sure the invalid PTE is actually in memory
1150 hrmPtlb64: lwarx r5,0,r9 ; Get the TLBIE lock
1151 rldicl r23,r23,0,16 ; Clear bits 0:15 cause they say to
1152 mr. r5,r5 ; Is it locked?
1153 li r5,1 ; Get locked indicator
1154 bne-- hrmPtlb64w ; It is locked, go spin...
1155 stwcx. r5,0,r9 ; Try to get it
1156 bne-- hrmPtlb64 ; We was beat...
1158 tlbie r23 ; Invalidate it all corresponding TLB entries
1160 eieio ; Make sure that the tlbie happens first
1161 tlbsync ; Wait for everyone to catch up
1164 ptesync ; Make sure of it all
1165 li r0,0 ; Clear this
1166 rlwinm r2,r21,28,29,31 ; Get slot number (16 byte entries)
1167 stw r0,tlbieLock(0) ; Clear the tlbie lock
1168 oris r0,r0,0x8000 ; Assume slot 0
1169 eieio ; Make sure those RC bit have been stashed in PTE
1170 srw r0,r0,r2 ; Get slot mask to deallocate
1172 lwz r22,12(r26) ; Get the latest reference and change bits
1173 or r6,r6,r0 ; Make the guy we killed free
1176 eieio ; Make sure all updates come first
1178 stw r6,0(r7) ; Unlock and change the PCA
1180 hrmPysDQ64: mr r3,r31 ; Point to the mapping
1181 bl mapDrainBusy ; Go wait until mapping is unused
1183 mr r3,r28 ; Get the pmap to insert into
1184 mr r4,r31 ; Point to the mapping
1185 bl EXT(mapRemove) ; Remove the mapping from the list
1187 andi. r0,r20,lo16(mpSpecial|mpNest) ; Is this nest or special mapping?
1188 lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
1189 cmplwi cr1,r0,0 ; Special thingie?
1190 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1191 subi r4,r4,1 ; Drop down the mapped page count
1192 stw r4,pmapResidentCnt(r28) ; Set the mapped page count
1193 bl sxlkUnlock ; Unlock the search list
1195 bne-- cr1,hrmRetn64 ; This one has no real memory associated with it so we are done...
1197 bl mapPhysFindLock ; Go find and lock the physent
1199 li r0,0xFF ; Get mask to clean up mapping pointer
1200 ld r9,ppLink(r3) ; Get first mapping
1201 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
1202 mr r4,r22 ; Get the RC bits we just got
1204 bl mapPhysMerge ; Go merge the RC bits
1206 andc r9,r9,r0 ; Clean up the mapping pointer
1208 cmpld r9,r31 ; Are we the first on the list?
1209 bne- hrmNot1st64 ; Nope...
1212 ld r4,mpAlias(r31) ; Get our forward pointer
1214 std r9,mpAlias(r31) ; Make sure we are off the chain
1215 bl mapPhyCSet64 ; Go set the physent link and preserve flags
1217 b hrmPhyDQd64 ; Join up and unlock it all...
1219 hrmPtlb64w: li r5,lgKillResv ; Point to some spare memory
1220 stwcx. r5,0,r5 ; Clear the pending reservation
1223 hrmPtlb64x: lwz r5,0(r9) ; Do a regular load to avoid taking reservation
1224 mr. r5,r5 ; is it locked?
1225 beq++ hrmPtlb64 ; Nope...
1226 b hrmPtlb64x ; Sniff some more...
1231 mr. r8,r9 ; Remember and test current node
1232 beq- hrmNotFound ; Could not find our node...
1233 ld r9,mpAlias(r9) ; Chain to the next
1234 cmpld r9,r31 ; Is this us?
1235 bne- hrmNot1st64 ; Not us...
1237 ld r9,mpAlias(r9) ; Get our forward pointer
1238 std r9,mpAlias(r8) ; Unchain us
1243 bl mapPhysUnlock ; Unlock the physent chain
1245 hrmRetn64: rldicr r8,r31,0,51 ; Find start of page
1246 mr r3,r31 ; Copy the pointer to the mapping
1247 lwz r8,mbvrswap+4(r8) ; Get last half of virtual to real swap
1248 bl mapDrainBusy ; Go wait until mapping is unused
1250 xor r3,r31,r8 ; Flip mapping address to virtual
1252 mtmsrd r17 ; Restore enables/translation/etc.
1255 b hrmRetnCmn ; Join the common return path...
1259 ; Check hrmBlock32 for comments.
1265 lhz r24,mpSpace(r31) ; Get the address space hash
1266 lhz r25,mpBSize(r31) ; Get the number of pages in block
1267 lwz r9,mpBlkRemCur(r31) ; Get our current remove position
1268 ori r0,r20,mpRIP ; Turn on the remove in progress flag
1269 mfsdr1 r29 ; Get the hash table base and size
1270 ld r27,mpVAddr(r31) ; Get the base vaddr
1271 rlwinm r5,r29,0,27,31 ; Isolate the size
1272 sub r4,r25,r9 ; Get number of pages left
1273 cmplw cr1,r9,r25 ; Have we already hit the end?
1274 addi r10,r9,mapRemChunk ; Point to the start of the next chunk
1275 addi r2,r4,-mapRemChunk ; See if mapRemChunk or more
1276 stb r0,mpFlags+3(r31) ; Save the flags with the mpRIP bit on
1277 srawi r2,r2,31 ; We have -1 if less than mapRemChunk or 0 if equal or more
1278 subi r4,r4,mapRemChunk-1 ; Back off for a running start (will be negative for more than mapRemChunk)
1279 cmpwi cr7,r2,0 ; Remember if we are doing the last chunk
1280 and r4,r4,r2 ; If more than a chunk, bring this back to 0
1281 srdi r27,r27,12 ; Change address into page index
1282 addi r4,r4,mapRemChunk-1 ; Add mapRemChunk-1 to get max(num left, chunksize)
1283 add r27,r27,r9 ; Adjust vaddr to start of current chunk
1285 bgt-- cr1,hrmEndInSight ; Someone is already doing the last hunk...
1287 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1288 stw r10,mpBlkRemCur(r31) ; Set next chunk to do (note: this may indicate after end)
1289 bl sxlkUnlock ; Unlock the search list while we are invalidating
1291 rlwimi r24,r24,14,4,17 ; Insert a copy of space hash
1292 eqv r26,r26,r26 ; Get all foxes here
1293 rldimi r24,r24,28,8 ; Make a couple copies up higher
1294 rldicr r29,r29,0,47 ; Isolate just the hash table base
1295 subfic r5,r5,46 ; Get number of leading zeros
1296 srd r26,r26,r5 ; Shift the size bits over
1297 mr r30,r27 ; Get start of chunk to invalidate
1298 rldicr r26,r26,0,56 ; Make length in PTEG units
1299 add r22,r4,r30 ; Get end page number
1301 hrmBInv64: srdi r0,r30,2 ; Shift page index over to form ESID
1302 rldicr r0,r0,0,49 ; Clean all but segment portion
1303 rlwinm r2,r30,0,16,31 ; Get the current page index
1304 xor r0,r0,r24 ; Form VSID
1305 xor r8,r2,r0 ; Hash the vaddr
1306 sldi r8,r8,7 ; Make into PTEG offset
1307 and r23,r8,r26 ; Wrap into the hash table
1308 rlwinm r3,r23,27,5,29 ; Change to PCA offset (table is always 2GB or less so 32-bit instructions work here)
1309 subfic r3,r3,-4 ; Get the PCA entry offset
1310 add r7,r3,r29 ; Point to the PCA slot
1312 cmplw cr5,r30,r22 ; Have we reached the end of the range?
1314 bl mapLockPteg ; Lock the PTEG
1316 rlwinm. r4,r6,16,0,7 ; Extract the block mappings in this here PTEG and see if there are any
1317 add r5,r23,r29 ; Point to the PTEG
1318 li r0,0 ; Set an invalid PTE value
1319 beq++ hrmBNone64 ; No block map PTEs in this PTEG...
1320 mtcrf 0x80,r4 ; Set CRs to select PTE slots
1321 mtcrf 0x40,r4 ; Set CRs to select PTE slots
1324 bf 0,hrmSlot0s ; No autogen here
1325 std r0,0x00(r5) ; Invalidate PTE
1327 hrmSlot0s: bf 1,hrmSlot1s ; No autogen here
1328 std r0,0x10(r5) ; Invalidate PTE
1330 hrmSlot1s: bf 2,hrmSlot2s ; No autogen here
1331 std r0,0x20(r5) ; Invalidate PTE
1333 hrmSlot2s: bf 3,hrmSlot3s ; No autogen here
1334 std r0,0x30(r5) ; Invalidate PTE
1336 hrmSlot3s: bf 4,hrmSlot4s ; No autogen here
1337 std r0,0x40(r5) ; Invalidate PTE
1339 hrmSlot4s: bf 5,hrmSlot5s ; No autogen here
1340 std r0,0x50(r5) ; Invalidate PTE
1342 hrmSlot5s: bf 6,hrmSlot6s ; No autogen here
1343 std r0,0x60(r5) ; Invalidate PTE
1345 hrmSlot6s: bf 7,hrmSlot7s ; No autogen here
1346 std r0,0x70(r5) ; Invalidate PTE
1348 hrmSlot7s: rlwinm r0,r4,16,16,23 ; Move in use to autogen
1349 or r6,r6,r4 ; Flip on the free bits that corrospond to the autogens we cleared
1350 andc r6,r6,r0 ; Turn off all the old autogen bits
1352 hrmBNone64: eieio ; Make sure all updates come first
1353 stw r6,0(r7) ; Unlock and set the PCA
1355 addi r30,r30,1 ; bump to the next PTEG
1356 bne++ cr5,hrmBInv64 ; Go invalidate the next...
1358 bge+ cr7,hrmDoneChunk ; We have not as yet done the last chunk, go tell our caller to call again...
1360 mr r3,r31 ; Copy the pointer to the mapping
1361 bl mapDrainBusy ; Go wait until we are sure all other removers are done with this one
1363 sync ; Make sure memory is consistent
1365 subi r5,r25,255 ; Subtract TLB size from page count (note we are 0 based here)
1366 li r6,255 ; Assume full invalidate for now
1367 srawi r5,r5,31 ; Make 0 if we need a full purge, -1 otherwise
1368 andc r6,r6,r5 ; Clear max if we have less to do
1369 and r5,r25,r5 ; Clear count if we have more than max
1370 sldi r24,r24,28 ; Get the full XOR value over to segment position
1371 ld r27,mpVAddr(r31) ; Get the base vaddr
1372 li r7,tlbieLock ; Get the TLBIE lock
1373 or r5,r5,r6 ; Get number of TLBIEs needed
1375 hrmBTLBlcl: lwarx r2,0,r7 ; Get the TLBIE lock
1376 mr. r2,r2 ; Is it locked?
1377 li r2,1 ; Get our lock value
1378 bne-- hrmBTLBlcm ; It is locked, go wait...
1379 stwcx. r2,0,r7 ; Try to get it
1380 bne-- hrmBTLBlcl ; We was beat...
1382 hrmBTLBj: sldi r2,r27,maxAdrSpb ; Move to make room for address space ID
1383 rldicr r2,r2,0,35-maxAdrSpb ; Clear out the extra
1384 addic. r5,r5,-1 ; See if we did them all
1385 xor r2,r2,r24 ; Make the VSID
1386 rldimi r2,r27,0,36 ; Insert the page portion of the VPN
1387 rldicl r2,r2,0,16 ; Clear bits 0:15 cause they say we gotta
1389 tlbie r2 ; Invalidate it everywhere
1390 addi r27,r27,0x1000 ; Up to the next page
1391 bge++ hrmBTLBj ; Make sure we have done it all...
1393 sync ; Make sure all is quiet
1395 eieio ; Make sure that the tlbie happens first
1396 tlbsync ; wait for everyone to catch up
1399 li r2,0 ; Lock clear value
1401 ptesync ; Wait for quiet again
1402 sync ; Make sure that is done
1404 stw r2,tlbieLock(0) ; Clear the tlbie lock
1406 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1407 bl sxlkShared ; Go get a shared lock on the mapping lists
1408 mr. r3,r3 ; Did we get the lock?
1409 bne- hrmPanic ; Nope...
1411 lwz r4,mpVAddr(r31) ; High order of address
1412 lwz r5,mpVAddr+4(r31) ; Low order of address
1413 mr r3,r28 ; Pass in pmap to search
1414 mr r29,r4 ; Save this in case we need it (only promote fails)
1415 mr r30,r5 ; Save this in case we need it (only promote fails)
1416 bl EXT(mapSearchFull) ; Go see if we can find it
1418 mr. r3,r3 ; Did we? (And remember mapping address for later)
1419 mr r15,r4 ; Save top of next vaddr
1420 mr r16,r5 ; Save bottom of next vaddr
1421 beq- hrmPanic ; Nope, not found...
1423 cmpld r3,r31 ; Same mapping?
1424 bne- hrmPanic ; Not good...
1426 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1427 bl sxlkPromote ; Try to promote shared to exclusive
1428 mr. r3,r3 ; Could we?
1429 mr r3,r31 ; Restore the mapping pointer
1430 beq+ hrmBDone2 ; Yeah...
1432 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1433 bl sxlkConvert ; Convert shared to exclusive
1434 mr. r3,r3 ; Could we?
1435 bne-- hrmPanic ; Nope, we must have timed out...
1437 mr r3,r28 ; Pass in pmap to search
1438 mr r4,r29 ; High order of address
1439 mr r5,r30 ; Low order of address
1440 bl EXT(mapSearchFull) ; Rescan the list
1442 mr. r3,r3 ; Did we lose it when we converted?
1443 mr r15,r4 ; Save top of next vaddr
1444 mr r16,r5 ; Save bottom of next vaddr
1445 beq-- hrmPanic ; Yeah, we did, someone tossed it for us...
1447 hrmBDone2: bl mapDrainBusy ; Go wait until mapping is unused
1449 mr r3,r28 ; Get the pmap to remove from
1450 mr r4,r31 ; Point to the mapping
1451 bl EXT(mapRemove) ; Remove the mapping from the list
1453 lwz r4,pmapResidentCnt(r28) ; Get the mapped page count
1454 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1455 subi r4,r4,1 ; Drop down the mapped page count
1456 stw r4,pmapResidentCnt(r28) ; Set the mapped page count
1457 bl sxlkUnlock ; Unlock the search list
1459 b hrmRetn64 ; We are all done, get out...
1461 hrmBTLBlcm: li r2,lgKillResv ; Get space unreserve line
1462 stwcx. r2,0,r2 ; Unreserve it
1464 hrmBTLBlcn: lwz r2,0(r7) ; Get the TLBIE lock
1465 mr. r2,r2 ; Is it held?
1466 beq++ hrmBTLBlcl ; Nope...
1467 b hrmBTLBlcn ; Yeah...
1472 * mapping *hw_purge_phys(physent) - remove a mapping from the system
1474 * Upon entry, R3 contains a pointer to a physent.
1476 * This function removes the first mapping from a physical entry
1477 * alias list. It locks the list, extracts the vaddr and pmap from
1478 * the first entry. It then jumps into the hw_rem_map function.
1479 * NOTE: since we jump into rem_map, we need to set up the stack
1480 * identically. Also, we set the next parm to 0 so we do not
1481 * try to save a next vaddr.
1483 * We return the virtual address of the removed mapping as a
1486 * Note that this is designed to be called from 32-bit mode with a stack.
1488 * We disable translation and all interruptions here. This keeps is
1489 * from having to worry about a deadlock due to having anything locked
1490 * and needing it to process a fault.
1492 * Note that this must be done with both interruptions off and VM off
1495 * Remove mapping via physical page (mapping_purge)
1498 * 2) extract vaddr and pmap
1500 * 4) do "remove mapping via pmap"
1506 .globl EXT(hw_purge_phys)
1509 stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
1510 mflr r0 ; Save the link register
1511 stw r15,FM_ARG0+0x00(r1) ; Save a register
1512 stw r16,FM_ARG0+0x04(r1) ; Save a register
1513 stw r17,FM_ARG0+0x08(r1) ; Save a register
1514 stw r18,FM_ARG0+0x0C(r1) ; Save a register
1515 stw r19,FM_ARG0+0x10(r1) ; Save a register
1516 stw r20,FM_ARG0+0x14(r1) ; Save a register
1517 stw r21,FM_ARG0+0x18(r1) ; Save a register
1518 stw r22,FM_ARG0+0x1C(r1) ; Save a register
1519 stw r23,FM_ARG0+0x20(r1) ; Save a register
1520 stw r24,FM_ARG0+0x24(r1) ; Save a register
1521 stw r25,FM_ARG0+0x28(r1) ; Save a register
1522 li r6,0 ; Set no next address return
1523 stw r26,FM_ARG0+0x2C(r1) ; Save a register
1524 stw r27,FM_ARG0+0x30(r1) ; Save a register
1525 stw r28,FM_ARG0+0x34(r1) ; Save a register
1526 stw r29,FM_ARG0+0x38(r1) ; Save a register
1527 stw r30,FM_ARG0+0x3C(r1) ; Save a register
1528 stw r31,FM_ARG0+0x40(r1) ; Save a register
1529 stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
1530 stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
1532 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
1534 bl mapPhysLock ; Lock the physent
1536 bt++ pf64Bitb,hppSF ; skip if 64-bit (only they take the hint)
1538 lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
1539 li r0,0x3F ; Set the bottom stuff to clear
1540 b hppJoin ; Join the common...
1543 ld r12,ppLink(r3) ; Get the pointer to the first mapping
1544 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
1546 hppJoin: andc. r12,r12,r0 ; Clean and test link
1547 beq-- hppNone ; There are no more mappings on physical page
1549 lis r28,hi16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
1550 lhz r7,mpSpace(r12) ; Get the address space hash
1551 ori r28,r28,lo16(EXT(pmapTrans)) ; Get the top of the start of the pmap hash to pmap translate table
1552 slwi r0,r7,2 ; Multiply space by 4
1553 lwz r4,mpVAddr(r12) ; Get the top of the vaddr
1554 slwi r7,r7,3 ; Multiply space by 8
1555 lwz r5,mpVAddr+4(r12) ; and the bottom
1556 add r7,r7,r0 ; Get correct displacement into translate table
1557 lwz r28,0(r28) ; Get the actual translation map
1559 add r28,r28,r7 ; Point to the pmap translation
1561 bl mapPhysUnlock ; Time to unlock the physical entry
1563 bt++ pf64Bitb,hppSF2 ; skip if 64-bit (only they take the hint)
1565 lwz r28,pmapPAddr+4(r28) ; Get the physical address of the pmap
1566 b hrmJoin ; Go remove the mapping...
1568 hppSF2: ld r28,pmapPAddr(r28) ; Get the physical address of the pmap
1569 b hrmJoin ; Go remove the mapping...
1573 hppNone: bl mapPhysUnlock ; Time to unlock the physical entry
1575 bt++ pf64Bitb,hppSF3 ; skip if 64-bit (only they take the hint)...
1577 mtmsr r11 ; Restore enables/translation/etc.
1579 b hppRetnCmn ; Join the common return code...
1581 hppSF3: mtmsrd r11 ; Restore enables/translation/etc.
1585 ; NOTE: we have not used any registers other than the volatiles to this point
1588 hppRetnCmn: lwz r12,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
1590 li r3,0 ; Clear high order mapping address because we are 32-bit
1591 mtlr r12 ; Restore the return
1592 lwz r1,0(r1) ; Pop the stack
1596 * mapping *hw_purge_map(pmap, vaddr, addr64_t *next) - remove a mapping from the system.
1598 * Upon entry, R3 contains a pointer to a pmap. Since vaddr is
1599 * a 64-bit quantity, it is a long long so it is in R4 and R5.
1601 * We return the virtual address of the removed mapping as a
1604 * Note that this is designed to be called from 32-bit mode with a stack.
1606 * We disable translation and all interruptions here. This keeps is
1607 * from having to worry about a deadlock due to having anything locked
1608 * and needing it to process a fault.
1610 * Note that this must be done with both interruptions off and VM off
1612 * Remove a mapping which can be reestablished by VM
1617 .globl EXT(hw_purge_map)
1620 stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
1621 mflr r0 ; Save the link register
1622 stw r15,FM_ARG0+0x00(r1) ; Save a register
1623 stw r16,FM_ARG0+0x04(r1) ; Save a register
1624 stw r17,FM_ARG0+0x08(r1) ; Save a register
1625 stw r18,FM_ARG0+0x0C(r1) ; Save a register
1626 stw r19,FM_ARG0+0x10(r1) ; Save a register
1627 mfsprg r19,2 ; Get feature flags
1628 stw r20,FM_ARG0+0x14(r1) ; Save a register
1629 stw r21,FM_ARG0+0x18(r1) ; Save a register
1630 mtcrf 0x02,r19 ; move pf64Bit cr6
1631 stw r22,FM_ARG0+0x1C(r1) ; Save a register
1632 stw r23,FM_ARG0+0x20(r1) ; Save a register
1633 stw r24,FM_ARG0+0x24(r1) ; Save a register
1634 stw r25,FM_ARG0+0x28(r1) ; Save a register
1635 stw r26,FM_ARG0+0x2C(r1) ; Save a register
1636 stw r27,FM_ARG0+0x30(r1) ; Save a register
1637 stw r28,FM_ARG0+0x34(r1) ; Save a register
1638 stw r29,FM_ARG0+0x38(r1) ; Save a register
1639 stw r30,FM_ARG0+0x3C(r1) ; Save a register
1640 stw r31,FM_ARG0+0x40(r1) ; Save a register
1641 stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
1642 stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
1644 bt++ pf64Bitb,hpmSF1 ; skip if 64-bit (only they take the hint)
1645 lwz r9,pmapvr+4(r3) ; Get conversion mask
1648 hpmSF1: ld r9,pmapvr(r3) ; Get conversion mask
1651 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
1653 xor r28,r3,r9 ; Convert the pmap to physical addressing
1655 mr r17,r11 ; Save the MSR
1657 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1658 bl sxlkExclusive ; Go get an exclusive lock on the mapping lists
1659 mr. r3,r3 ; Did we get the lock?
1660 bne-- hrmBadLock ; Nope...
1662 ; Note that we do a full search (i.e., no shortcut level skips, etc.)
1663 ; here so that we will know the previous elements so we can dequeue them
1667 mr r3,r28 ; Pass in pmap to search
1668 mr r29,r4 ; Top half of vaddr
1669 mr r30,r5 ; Bottom half of vaddr
1670 bl EXT(mapSearchFull) ; Rescan the list
1671 mr. r31,r3 ; Did we? (And remember mapping address for later)
1672 or r0,r4,r5 ; Are we beyond the end?
1673 mr r15,r4 ; Save top of next vaddr
1674 cmplwi cr1,r0,0 ; See if there is another
1675 mr r16,r5 ; Save bottom of next vaddr
1676 bne-- hpmGotOne ; We found one, go check it out...
1678 hpmCNext: bne++ cr1,hpmSearch ; There is another to check...
1679 b hrmNotFound ; No more in pmap to check...
1681 hpmGotOne: lwz r20,mpFlags(r3) ; Get the flags
1682 andi. r9,r20,lo16(mpSpecial|mpNest|mpPerm|mpBlock) ; Are we allowed to remove it?
1683 beq++ hrmGotX ; Found, branch to remove the mapping...
1684 b hpmCNext ; Nope...
1687 * mapping *hw_purge_space(physent, pmap) - remove a mapping from the system based upon address space
1689 * Upon entry, R3 contains a pointer to a pmap.
1690 * pa is a pointer to the physent
1692 * This function removes the first mapping for a specific pmap from a physical entry
1693 * alias list. It locks the list, extracts the vaddr and pmap from
1694 * the first apporpriate entry. It then jumps into the hw_rem_map function.
1695 * NOTE: since we jump into rem_map, we need to set up the stack
1696 * identically. Also, we set the next parm to 0 so we do not
1697 * try to save a next vaddr.
1699 * We return the virtual address of the removed mapping as a
1702 * Note that this is designed to be called from 32-bit mode with a stack.
1704 * We disable translation and all interruptions here. This keeps is
1705 * from having to worry about a deadlock due to having anything locked
1706 * and needing it to process a fault.
1708 * Note that this must be done with both interruptions off and VM off
1711 * Remove mapping via physical page (mapping_purge)
1714 * 2) extract vaddr and pmap
1716 * 4) do "remove mapping via pmap"
1722 .globl EXT(hw_purge_space)
1724 LEXT(hw_purge_space)
1725 stwu r1,-(FM_ALIGN(hrmStackSize)+FM_SIZE)(r1) ; Make some space on the stack
1726 mflr r0 ; Save the link register
1727 stw r15,FM_ARG0+0x00(r1) ; Save a register
1728 stw r16,FM_ARG0+0x04(r1) ; Save a register
1729 stw r17,FM_ARG0+0x08(r1) ; Save a register
1730 mfsprg r2,2 ; Get feature flags
1731 stw r18,FM_ARG0+0x0C(r1) ; Save a register
1732 stw r19,FM_ARG0+0x10(r1) ; Save a register
1733 stw r20,FM_ARG0+0x14(r1) ; Save a register
1734 stw r21,FM_ARG0+0x18(r1) ; Save a register
1735 stw r22,FM_ARG0+0x1C(r1) ; Save a register
1736 mtcrf 0x02,r2 ; move pf64Bit cr6
1737 stw r23,FM_ARG0+0x20(r1) ; Save a register
1738 stw r24,FM_ARG0+0x24(r1) ; Save a register
1739 stw r25,FM_ARG0+0x28(r1) ; Save a register
1740 stw r26,FM_ARG0+0x2C(r1) ; Save a register
1741 stw r27,FM_ARG0+0x30(r1) ; Save a register
1742 li r6,0 ; Set no next address return
1743 stw r28,FM_ARG0+0x34(r1) ; Save a register
1744 stw r29,FM_ARG0+0x38(r1) ; Save a register
1745 stw r30,FM_ARG0+0x3C(r1) ; Save a register
1746 stw r31,FM_ARG0+0x40(r1) ; Save a register
1747 stw r6,FM_ARG0+0x44(r1) ; Save address to save next mapped vaddr
1748 stw r0,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
1750 bt++ pf64Bitb,hpsSF1 ; skip if 64-bit (only they take the hint)
1752 lwz r9,pmapvr+4(r4) ; Get conversion mask for pmap
1756 hpsSF1: ld r9,pmapvr(r4) ; Get conversion mask for pmap
1758 hpsSF1x: bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
1760 xor r4,r4,r9 ; Convert the pmap to physical addressing
1762 bl mapPhysLock ; Lock the physent
1764 lwz r8,pmapSpace(r4) ; Get the space hash
1766 bt++ pf64Bitb,hpsSF ; skip if 64-bit (only they take the hint)
1768 lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
1770 hpsSrc32: rlwinm. r12,r12,0,0,25 ; Clean and test mapping address
1771 beq hpsNone ; Did not find one...
1773 lhz r10,mpSpace(r12) ; Get the space
1775 cmplw r10,r8 ; Is this one of ours?
1778 lwz r12,mpAlias+4(r12) ; Chain on to the next
1779 b hpsSrc32 ; Check it out...
1784 ld r12,ppLink(r3) ; Get the pointer to the first mapping
1785 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
1787 hpsSrc64: andc. r12,r12,r0 ; Clean and test mapping address
1788 beq hpsNone ; Did not find one...
1790 lhz r10,mpSpace(r12) ; Get the space
1792 cmplw r10,r8 ; Is this one of ours?
1795 ld r12,mpAlias(r12) ; Chain on to the next
1796 b hpsSrc64 ; Check it out...
1800 hpsFnd: mr r28,r4 ; Set the pmap physical address
1801 lwz r4,mpVAddr(r12) ; Get the top of the vaddr
1802 lwz r5,mpVAddr+4(r12) ; and the bottom
1804 bl mapPhysUnlock ; Time to unlock the physical entry
1805 b hrmJoin ; Go remove the mapping...
1809 hpsNone: bl mapPhysUnlock ; Time to unlock the physical entry
1811 bt++ pf64Bitb,hpsSF3 ; skip if 64-bit (only they take the hint)...
1813 mtmsr r11 ; Restore enables/translation/etc.
1815 b hpsRetnCmn ; Join the common return code...
1817 hpsSF3: mtmsrd r11 ; Restore enables/translation/etc.
1821 ; NOTE: we have not used any registers other than the volatiles to this point
1824 hpsRetnCmn: lwz r12,(FM_ALIGN(hrmStackSize)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
1826 li r3,0 ; Set return code
1827 mtlr r12 ; Restore the return
1828 lwz r1,0(r1) ; Pop the stack
1833 * mapping *hw_find_space(physent, space) - finds the first mapping on physent for specified space
1835 * Upon entry, R3 contains a pointer to a physent.
1836 * space is the space ID from the pmap in question
1838 * We return the virtual address of the found mapping in
1839 * R3. Note that the mapping busy is bumped.
1841 * Note that this is designed to be called from 32-bit mode with a stack.
1843 * We disable translation and all interruptions here. This keeps is
1844 * from having to worry about a deadlock due to having anything locked
1845 * and needing it to process a fault.
1850 .globl EXT(hw_find_space)
1853 stwu r1,-(FM_SIZE)(r1) ; Make some space on the stack
1854 mflr r0 ; Save the link register
1855 mr r8,r4 ; Remember the space
1856 stw r0,(FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
1858 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
1860 bl mapPhysLock ; Lock the physent
1862 bt++ pf64Bitb,hfsSF ; skip if 64-bit (only they take the hint)
1864 lwz r12,ppLink+4(r3) ; Grab the pointer to the first mapping
1866 hfsSrc32: rlwinm. r12,r12,0,0,25 ; Clean and test mapping address
1867 beq hfsNone ; Did not find one...
1869 lhz r10,mpSpace(r12) ; Get the space
1871 cmplw r10,r8 ; Is this one of ours?
1874 lwz r12,mpAlias+4(r12) ; Chain on to the next
1875 b hfsSrc32 ; Check it out...
1880 ld r12,ppLink(r3) ; Get the pointer to the first mapping
1881 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
1883 hfsSrc64: andc. r12,r12,r0 ; Clean and test mapping address
1884 beq hfsNone ; Did not find one...
1886 lhz r10,mpSpace(r12) ; Get the space
1888 cmplw r10,r8 ; Is this one of ours?
1891 ld r12,mpAlias(r12) ; Chain on to the next
1892 b hfsSrc64 ; Check it out...
1896 hfsFnd: mr r8,r3 ; Save the physent
1897 mr r3,r12 ; Point to the mapping
1898 bl mapBumpBusy ; If we found it, bump up the busy count so the mapping does not disapear
1900 mr r3,r8 ; Get back the physical entry
1901 li r7,0xFFF ; Get a page size mask
1902 bl mapPhysUnlock ; Time to unlock the physical entry
1904 andc r3,r12,r7 ; Move the mapping back down to a page
1905 lwz r3,mbvrswap+4(r3) ; Get last half of virtual to real swap
1906 xor r12,r3,r12 ; Convert to virtual
1907 b hfsRet ; Time to return
1911 hfsNone: bl mapPhysUnlock ; Time to unlock the physical entry
1913 hfsRet: bt++ pf64Bitb,hfsSF3 ; skip if 64-bit (only they take the hint)...
1915 mtmsr r11 ; Restore enables/translation/etc.
1917 b hfsRetnCmn ; Join the common return code...
1919 hfsSF3: mtmsrd r11 ; Restore enables/translation/etc.
1923 ; NOTE: we have not used any registers other than the volatiles to this point
1926 hfsRetnCmn: mr r3,r12 ; Get the mapping or a 0 if we failed
1927 lwz r12,(FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
1929 mtlr r12 ; Restore the return
1930 lwz r1,0(r1) ; Pop the stack
1935 ; mapping *hw_find_map(pmap, va, *nextva) - Looks up a vaddr in a pmap
1936 ; Returns 0 if not found or the virtual address of the mapping if
1937 ; if is. Also, the mapping has the busy count bumped.
1940 .globl EXT(hw_find_map)
1943 stwu r1,-(FM_ALIGN((31-25+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
1944 mflr r0 ; Save the link register
1945 stw r25,FM_ARG0+0x00(r1) ; Save a register
1946 stw r26,FM_ARG0+0x04(r1) ; Save a register
1947 mr r25,r6 ; Remember address of next va
1948 stw r27,FM_ARG0+0x08(r1) ; Save a register
1949 stw r28,FM_ARG0+0x0C(r1) ; Save a register
1950 stw r29,FM_ARG0+0x10(r1) ; Save a register
1951 stw r30,FM_ARG0+0x14(r1) ; Save a register
1952 stw r31,FM_ARG0+0x18(r1) ; Save a register
1953 stw r0,(FM_ALIGN((31-26+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
1955 lwz r6,pmapvr(r3) ; Get the first part of the VR translation for pmap
1956 lwz r7,pmapvr+4(r3) ; Get the second part
1959 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
1961 mr r27,r11 ; Remember the old MSR
1962 mr r26,r12 ; Remember the feature bits
1964 xor r28,r3,r7 ; Change the common 32- and 64-bit half
1966 bf-- pf64Bitb,hfmSF1 ; skip if 32-bit...
1968 rldimi r28,r6,32,0 ; Shift the fixed upper part of the physical over and cram in top
1970 hfmSF1: mr r29,r4 ; Save top half of vaddr
1971 mr r30,r5 ; Save the bottom half
1973 la r3,pmapSXlk(r28) ; Point to the pmap search lock
1974 bl sxlkShared ; Go get a shared lock on the mapping lists
1975 mr. r3,r3 ; Did we get the lock?
1976 bne-- hfmBadLock ; Nope...
1978 mr r3,r28 ; get the pmap address
1979 mr r4,r29 ; Get bits 0:31 to look for
1980 mr r5,r30 ; Get bits 32:64
1982 bl EXT(mapSearch) ; Go see if we can find it (note: R7 comes back with mpFlags)
1984 rlwinm r0,r7,0,mpRIPb,mpRIPb ; Find remove in progress bit
1985 mr. r31,r3 ; Save the mapping if we found it
1986 cmplwi cr1,r0,0 ; Are we removing?
1987 mr r29,r4 ; Save next va high half
1988 crorc cr0_eq,cr0_eq,cr1_eq ; Not found or removing
1989 mr r30,r5 ; Save next va low half
1990 li r6,0 ; Assume we did not find it
1991 li r26,0xFFF ; Get a mask to relocate to start of mapping page
1993 bt-- cr0_eq,hfmNotFnd ; We did not find it...
1995 bl mapBumpBusy ; If we found it, bump up the busy count so the mapping does not disapear
1997 andc r4,r31,r26 ; Get back to the mapping page start
1999 ; Note: we can treat 32- and 64-bit the same here. Because we are going from
2000 ; physical to virtual and we only do 32-bit virtual, we only need the low order
2003 lwz r4,mbvrswap+4(r4) ; Get last half of virtual to real swap
2004 li r6,-1 ; Indicate we found it and it is not being removed
2005 xor r31,r31,r4 ; Flip to virtual
2007 hfmNotFnd: la r3,pmapSXlk(r28) ; Point to the pmap search lock
2008 bl sxlkUnlock ; Unlock the search list
2010 rlwinm r3,r31,0,0,31 ; Move mapping to return register and clear top of register if 64-bit
2011 and r3,r3,r6 ; Clear if not found or removing
2013 hfmReturn: bt++ pf64Bitb,hfmR64 ; Yes...
2015 mtmsr r27 ; Restore enables/translation/etc.
2017 b hfmReturnC ; Join common...
2019 hfmR64: mtmsrd r27 ; Restore enables/translation/etc.
2022 hfmReturnC: stw r29,0(r25) ; Save the top of the next va
2023 stw r30,4(r25) ; Save the bottom of the next va
2024 lwz r0,(FM_ALIGN((31-25+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2025 lwz r25,FM_ARG0+0x00(r1) ; Restore a register
2026 lwz r26,FM_ARG0+0x04(r1) ; Restore a register
2027 and r3,r3,r6 ; Clear return if the mapping is being removed
2028 lwz r27,FM_ARG0+0x08(r1) ; Restore a register
2029 mtlr r0 ; Restore the return
2030 lwz r28,FM_ARG0+0x0C(r1) ; Restore a register
2031 lwz r29,FM_ARG0+0x10(r1) ; Restore a register
2032 lwz r30,FM_ARG0+0x14(r1) ; Restore a register
2033 lwz r31,FM_ARG0+0x18(r1) ; Restore a register
2034 lwz r1,0(r1) ; Pop the stack
2039 hfmBadLock: li r3,1 ; Set lock time out error code
2040 b hfmReturn ; Leave....
2044 * unsigned int hw_walk_phys(pp, preop, op, postop, parm)
2045 * walks all mapping for a physical page and performs
2046 * specified operations on each.
2048 * pp is unlocked physent
2049 * preop is operation to perform on physent before walk. This would be
2050 * used to set cache attribute or protection
2051 * op is the operation to perform on each mapping during walk
2052 * postop is operation to perform in the phsyent after walk. this would be
2053 * used to set or reset the RC bits.
2055 * We return the RC bits from before postop is run.
2057 * Note that this is designed to be called from 32-bit mode with a stack.
2059 * We disable translation and all interruptions here. This keeps is
2060 * from having to worry about a deadlock due to having anything locked
2061 * and needing it to process a fault.
2063 * We lock the physent, execute preop, and then walk each mapping in turn.
2064 * If there is a PTE, it is invalidated and the RC merged into the physent.
2065 * Then we call the op function.
2066 * Then we revalidate the PTE.
2067 * Once all all mappings are finished, we save the physent RC and call the
2068 * postop routine. Then we unlock the physent and return the RC.
2074 .globl EXT(hw_walk_phys)
2077 stwu r1,-(FM_ALIGN((31-25+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
2078 mflr r0 ; Save the link register
2079 stw r25,FM_ARG0+0x00(r1) ; Save a register
2080 stw r26,FM_ARG0+0x04(r1) ; Save a register
2081 stw r27,FM_ARG0+0x08(r1) ; Save a register
2082 stw r28,FM_ARG0+0x0C(r1) ; Save a register
2083 mr r25,r7 ; Save the parm
2084 stw r29,FM_ARG0+0x10(r1) ; Save a register
2085 stw r30,FM_ARG0+0x14(r1) ; Save a register
2086 stw r31,FM_ARG0+0x18(r1) ; Save a register
2087 stw r0,(FM_ALIGN((31-25+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2089 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
2091 mr r26,r11 ; Save the old MSR
2092 lis r27,hi16(hwpOpBase) ; Get high order of op base
2093 slwi r4,r4,7 ; Convert preop to displacement
2094 ori r27,r27,lo16(hwpOpBase) ; Get low order of op base
2095 slwi r5,r5,7 ; Convert op to displacement
2096 add r12,r4,r27 ; Point to the preop routine
2097 slwi r28,r6,7 ; Convert postop to displacement
2098 mtctr r12 ; Set preop routine
2099 add r28,r28,r27 ; Get the address of the postop routine
2100 add r27,r5,r27 ; Get the address of the op routine
2102 bl mapPhysLock ; Lock the physent
2104 mr r29,r3 ; Save the physent address
2106 bt++ pf64Bitb,hwp64 ; skip if 64-bit (only they take the hint)
2108 bctrl ; Call preop routine
2109 bne- hwpEarly32 ; preop says to bail now...
2111 mtctr r27 ; Set up the op function address
2112 lwz r31,ppLink+4(r3) ; Grab the pointer to the first mapping
2114 hwpSrc32: rlwinm. r31,r31,0,0,25 ; Clean and test mapping address
2115 beq hwpNone32 ; Did not find one...
2118 ; Note: mapInvPte32 returns the PTE in R3 (or 0 if none), PTE high in R4,
2119 ; PTE low in R5. The PCA address is in R7. The PTEG come back locked.
2120 ; If there is no PTE, PTE low is obtained from mapping
2122 bl mapInvPte32 ; Invalidate and lock PTE, also merge into physent
2124 bctrl ; Call the op function
2126 crmove cr1_eq,cr0_eq ; Save the return code
2128 mr. r3,r3 ; Was there a previously valid PTE?
2129 beq- hwpNxt32 ; Nope...
2131 stw r5,4(r3) ; Store second half of PTE
2132 eieio ; Make sure we do not reorder
2133 stw r4,0(r3) ; Revalidate the PTE
2135 eieio ; Make sure all updates come first
2136 stw r6,0(r7) ; Unlock the PCA
2138 hwpNxt32: bne- cr1,hwpEarly32 ; op says to bail now...
2139 lwz r31,mpAlias+4(r31) ; Chain on to the next
2140 b hwpSrc32 ; Check it out...
2144 hwpNone32: mtctr r28 ; Get the post routine address
2146 lwz r30,ppLink+4(r29) ; Save the old RC
2147 mr r3,r29 ; Get the physent address
2148 bctrl ; Call post routine
2150 bl mapPhysUnlock ; Unlock the physent
2152 mtmsr r26 ; Restore translation/mode/etc.
2155 b hwpReturn ; Go restore registers and return...
2159 hwpEarly32: lwz r30,ppLink+4(r29) ; Save the old RC
2160 mr r3,r29 ; Get the physent address
2161 bl mapPhysUnlock ; Unlock the physent
2163 mtmsr r26 ; Restore translation/mode/etc.
2166 b hwpReturn ; Go restore registers and return...
2170 hwp64: bctrl ; Call preop routine
2171 bne-- hwpEarly64 ; preop says to bail now...
2173 mtctr r27 ; Set up the op function address
2176 ld r31,ppLink(r3) ; Get the pointer to the first mapping
2177 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
2179 hwpSrc64: andc. r31,r31,r0 ; Clean and test mapping address
2180 beq hwpNone64 ; Did not find one...
2182 ; Note: mapInvPte64 returns the PTE in R3 (or 0 if none), PTE high in R4,
2183 ; PTE low in R5. PTEG comes back locked if there is one
2185 bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
2187 bctrl ; Call the op function
2189 crmove cr1_eq,cr0_eq ; Save the return code
2191 mr. r3,r3 ; Was there a previously valid PTE?
2192 beq-- hwpNxt64 ; Nope...
2194 std r5,8(r3) ; Save bottom of PTE
2195 eieio ; Make sure we do not reorder
2196 std r4,0(r3) ; Revalidate the PTE
2198 eieio ; Make sure all updates come first
2199 stw r6,0(r7) ; Unlock the PCA
2201 hwpNxt64: bne-- cr1,hwpEarly64 ; op says to bail now...
2202 ld r31,mpAlias(r31) ; Chain on to the next
2204 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
2205 b hwpSrc64 ; Check it out...
2209 hwpNone64: mtctr r28 ; Get the post routine address
2211 lwz r30,ppLink+4(r29) ; Save the old RC
2212 mr r3,r29 ; Get the physent address
2213 bctrl ; Call post routine
2215 bl mapPhysUnlock ; Unlock the physent
2217 mtmsrd r26 ; Restore translation/mode/etc.
2219 b hwpReturn ; Go restore registers and return...
2223 hwpEarly64: lwz r30,ppLink+4(r29) ; Save the old RC
2224 mr r3,r29 ; Get the physent address
2225 bl mapPhysUnlock ; Unlock the physent
2227 mtmsrd r26 ; Restore translation/mode/etc.
2230 hwpReturn: lwz r0,(FM_ALIGN((31-25+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Restore the return
2231 lwz r25,FM_ARG0+0x00(r1) ; Restore a register
2232 lwz r26,FM_ARG0+0x04(r1) ; Restore a register
2233 mr r3,r30 ; Pass back the RC
2234 lwz r27,FM_ARG0+0x08(r1) ; Restore a register
2235 lwz r28,FM_ARG0+0x0C(r1) ; Restore a register
2236 mtlr r0 ; Restore the return
2237 lwz r29,FM_ARG0+0x10(r1) ; Restore a register
2238 lwz r30,FM_ARG0+0x14(r1) ; Restore a register
2239 lwz r31,FM_ARG0+0x18(r1) ; Restore a register
2240 lwz r1,0(r1) ; Pop the stack
2245 ; The preop/op/postop function table.
2246 ; Each function must be 64-byte aligned and be no more than
2247 ; 16 instructions. If more than 16, we must fix address calculations
2248 ; at the start of hwpOpBase
2250 ; The routine must set CR0_EQ in order to continue scan.
2251 ; If CR0_EQ is not set, an early return from the function is made.
2258 ; Function 0 - No operation
2260 hwpNoop: cmplw r0,r0 ; Make sure CR0_EQ is set
2261 blr ; Just return...
2265 ; This is the continuation of function 4 - Set attributes in mapping
2267 ; We changed the attributes of a mapped page. Make sure there are no cache paradoxes.
2268 ; NOTE: Do we have to deal with i-cache here?
2270 hwpSAM: li r11,4096 ; Get page size
2272 hwpSAMinvd: sub. r11,r11,r9 ; Back off a line
2273 dcbf r11,r5 ; Flush the line in the data cache
2274 bgt++ hwpSAMinvd ; Go do the rest of it...
2276 sync ; Make sure it is done
2278 li r11,4096 ; Get page size
2280 hwpSAMinvi: sub. r11,r11,r9 ; Back off a line
2281 icbi r11,r5 ; Flush the line in the icache
2282 bgt++ hwpSAMinvi ; Go do the rest of it...
2284 sync ; Make sure it is done
2286 cmpw r0,r0 ; Make sure we return CR0_EQ
2290 ; Function 1 - Set protection in physent
2292 .set .,hwpOpBase+(1*128) ; Generate error if previous function too long
2294 hwpSPrtPhy: li r5,ppLink+4 ; Get offset for flag part of physent
2296 hwpSPrtPhX: lwarx r4,r5,r29 ; Get the old flags
2297 rlwimi r4,r25,0,ppPPb-32,ppPPe-32 ; Stick in the new protection
2298 stwcx. r4,r5,r29 ; Try to stuff it
2299 bne-- hwpSPrtPhX ; Try again...
2300 ; Note: CR0_EQ is set because of stwcx.
2304 ; Function 2 - Set protection in mapping
2306 .set .,hwpOpBase+(2*128) ; Generate error if previous function too long
2308 hwpSPrtMap: lwz r9,mpFlags(r31) ; Get the mapping flags
2309 lwz r8,mpVAddr+4(r31) ; Get the protection part of mapping
2310 rlwinm. r9,r9,0,mpPermb,mpPermb ; Is the mapping permanent?
2311 li r0,lo16(mpPP) ; Get protection bits
2312 crnot cr0_eq,cr0_eq ; Change CR0_EQ to true if mapping is permanent
2313 rlwinm r2,r25,0,mpPPb-32,mpPPb-32+2 ; Position new protection
2314 beqlr-- ; Leave if permanent mapping (before we trash R5)...
2315 andc r5,r5,r0 ; Clear the old prot bits
2316 or r5,r5,r2 ; Move in the prot bits
2317 rlwimi r8,r5,0,20,31 ; Copy into the mapping copy
2318 cmpw r0,r0 ; Make sure we return CR0_EQ
2319 stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
2322 ; Function 3 - Set attributes in physent
2324 .set .,hwpOpBase+(3*128) ; Generate error if previous function too long
2326 hwpSAtrPhy: li r5,ppLink+4 ; Get offset for flag part of physent
2328 hwpSAtrPhX: lwarx r4,r5,r29 ; Get the old flags
2329 rlwimi r4,r25,0,ppIb-32,ppGb-32 ; Stick in the new attributes
2330 stwcx. r4,r5,r29 ; Try to stuff it
2331 bne-- hwpSAtrPhX ; Try again...
2332 ; Note: CR0_EQ is set because of stwcx.
2335 ; Function 4 - Set attributes in mapping
2337 .set .,hwpOpBase+(4*128) ; Generate error if previous function too long
2339 hwpSAtrMap: lwz r9,mpFlags(r31) ; Get the mapping flags
2340 lwz r8,mpVAddr+4(r31) ; Get the attribute part of mapping
2341 li r2,0x10 ; Force on coherent
2342 rlwinm. r9,r9,0,mpPermb,mpPermb ; Is the mapping permanent?
2343 li r0,lo16(mpWIMG) ; Get wimg mask
2344 crnot cr0_eq,cr0_eq ; Change CR0_EQ to true if mapping is permanent
2345 rlwimi r2,r2,mpIb-ppIb,mpIb-32,mpIb-32 ; Copy in the cache inhibited bit
2346 beqlr-- ; Leave if permanent mapping (before we trash R5)...
2347 andc r5,r5,r0 ; Clear the old wimg
2348 rlwimi r2,r2,32-(mpGb-ppGb),mpGb-32,mpGb-32 ; Copy in the guarded bit
2349 mfsprg r9,2 ; Feature flags
2350 or r5,r5,r2 ; Move in the new wimg
2351 rlwimi r8,r5,0,20,31 ; Copy into the mapping copy
2352 lwz r2,mpPAddr(r31) ; Get the physical address
2353 li r0,0xFFF ; Start a mask
2354 andi. r9,r9,pf32Byte+pf128Byte ; Get cache line size
2355 rlwinm r5,r0,0,1,0 ; Copy to top half
2356 stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
2357 rlwinm r2,r2,12,1,0 ; Copy to top and rotate to make physical address with junk left
2358 and r5,r5,r2 ; Clean stuff in top 32 bits
2359 andc r2,r2,r0 ; Clean bottom too
2360 rlwimi r5,r2,0,0,31 ; Insert low 23 to make full physical address
2361 b hwpSAM ; Join common
2363 ; NOTE: we moved the remainder of the code out of here because it
2364 ; did not fit in the 128 bytes allotted. It got stuck into the free space
2365 ; at the end of the no-op function.
2370 ; Function 5 - Clear reference in physent
2372 .set .,hwpOpBase+(5*128) ; Generate error if previous function too long
2374 hwpCRefPhy: li r5,ppLink+4 ; Get offset for flag part of physent
2376 hwpCRefPhX: lwarx r4,r5,r29 ; Get the old flags
2377 rlwinm r4,r4,0,ppRb+1-32,ppRb-1-32 ; Clear R
2378 stwcx. r4,r5,r29 ; Try to stuff it
2379 bne-- hwpCRefPhX ; Try again...
2380 ; Note: CR0_EQ is set because of stwcx.
2384 ; Function 6 - Clear reference in mapping
2386 .set .,hwpOpBase+(6*128) ; Generate error if previous function too long
2388 hwpCRefMap: li r0,lo16(mpR) ; Get reference bit
2389 lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
2390 andc r5,r5,r0 ; Clear in PTE copy
2391 andc r8,r8,r0 ; and in the mapping
2392 cmpw r0,r0 ; Make sure we return CR0_EQ
2393 stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
2397 ; Function 7 - Clear change in physent
2399 .set .,hwpOpBase+(7*128) ; Generate error if previous function too long
2401 hwpCCngPhy: li r5,ppLink+4 ; Get offset for flag part of physent
2403 hwpCCngPhX: lwarx r4,r5,r29 ; Get the old flags
2404 rlwinm r4,r4,0,ppCb+1-32,ppCb-1-32 ; Clear C
2405 stwcx. r4,r5,r29 ; Try to stuff it
2406 bne-- hwpCCngPhX ; Try again...
2407 ; Note: CR0_EQ is set because of stwcx.
2411 ; Function 8 - Clear change in mapping
2413 .set .,hwpOpBase+(8*128) ; Generate error if previous function too long
2415 hwpCCngMap: li r0,lo16(mpC) ; Get change bit
2416 lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
2417 andc r5,r5,r0 ; Clear in PTE copy
2418 andc r8,r8,r0 ; and in the mapping
2419 cmpw r0,r0 ; Make sure we return CR0_EQ
2420 stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
2424 ; Function 9 - Set reference in physent
2426 .set .,hwpOpBase+(9*128) ; Generate error if previous function too long
2428 hwpSRefPhy: li r5,ppLink+4 ; Get offset for flag part of physent
2430 hwpSRefPhX: lwarx r4,r5,r29 ; Get the old flags
2431 ori r4,r4,lo16(ppR) ; Set the reference
2432 stwcx. r4,r5,r29 ; Try to stuff it
2433 bne-- hwpSRefPhX ; Try again...
2434 ; Note: CR0_EQ is set because of stwcx.
2438 ; Function 10 - Set reference in mapping
2440 .set .,hwpOpBase+(10*128) ; Generate error if previous function too long
2442 hwpSRefMap: lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
2443 ori r5,r5,lo16(mpR) ; Set reference in PTE low
2444 ori r8,r8,lo16(mpR) ; Set reference in mapping
2445 cmpw r0,r0 ; Make sure we return CR0_EQ
2446 stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
2449 ; Function 11 - Set change in physent
2451 .set .,hwpOpBase+(11*128) ; Generate error if previous function too long
2453 hwpSCngPhy: li r5,ppLink+4 ; Get offset for flag part of physent
2455 hwpSCngPhX: lwarx r4,r5,r29 ; Get the old flags
2456 ori r4,r4,lo16(ppC) ; Set the change bit
2457 stwcx. r4,r5,r29 ; Try to stuff it
2458 bne-- hwpSCngPhX ; Try again...
2459 ; Note: CR0_EQ is set because of stwcx.
2462 ; Function 12 - Set change in mapping
2464 .set .,hwpOpBase+(12*128) ; Generate error if previous function too long
2466 hwpSCngMap: lwz r8,mpVAddr+4(r31) ; Get the flag part of mapping
2467 ori r5,r5,lo16(mpC) ; Set change in PTE low
2468 ori r8,r8,lo16(mpC) ; Set chage in mapping
2469 cmpw r0,r0 ; Make sure we return CR0_EQ
2470 stw r8,mpVAddr+4(r31) ; Set the flag part of mapping
2473 ; Function 13 - Test reference in physent
2475 .set .,hwpOpBase+(13*128) ; Generate error if previous function too long
2477 hwpTRefPhy: lwz r0,ppLink+4(r29) ; Get the flags from physent
2478 rlwinm. r0,r0,0,ppRb-32,ppRb-32 ; Isolate reference bit and see if 0
2479 blr ; Return (CR0_EQ set to continue if reference is off)...
2482 ; Function 14 - Test reference in mapping
2484 .set .,hwpOpBase+(14*128) ; Generate error if previous function too long
2486 hwpTRefMap: rlwinm. r0,r5,0,mpRb-32,mpRb-32 ; Isolate reference bit and see if 0
2487 blr ; Return (CR0_EQ set to continue if reference is off)...
2489 ; Function 15 - Test change in physent
2491 .set .,hwpOpBase+(15*128) ; Generate error if previous function too long
2493 hwpTCngPhy: lwz r0,ppLink+4(r29) ; Get the flags from physent
2494 rlwinm. r0,r0,0,ppCb-32,ppCb-32 ; Isolate change bit and see if 0
2495 blr ; Return (CR0_EQ set to continue if reference is off)...
2498 ; Function 16 - Test change in mapping
2500 .set .,hwpOpBase+(16*128) ; Generate error if previous function too long
2502 hwpTCngMap: rlwinm. r0,r5,0,mpCb-32,mpCb-32 ; Isolate change bit and see if 0
2503 blr ; Return (CR0_EQ set to continue if reference is off)...
2505 .set .,hwpOpBase+(17*128) ; Generate error if previous function too long
2510 ; int hw_protect(pmap, va, prot, *nextva) - Changes protection on a specific mapping.
2513 ; mapRtOK - if all is ok
2514 ; mapRtBadLk - if mapping lock fails
2515 ; mapRtPerm - if mapping is permanent
2516 ; mapRtNotFnd - if mapping is not found
2517 ; mapRtBlock - if mapping is a block
2520 .globl EXT(hw_protect)
2523 stwu r1,-(FM_ALIGN((31-24+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
2524 mflr r0 ; Save the link register
2525 stw r24,FM_ARG0+0x00(r1) ; Save a register
2526 stw r25,FM_ARG0+0x04(r1) ; Save a register
2527 mr r25,r7 ; Remember address of next va
2528 stw r26,FM_ARG0+0x08(r1) ; Save a register
2529 stw r27,FM_ARG0+0x0C(r1) ; Save a register
2530 stw r28,FM_ARG0+0x10(r1) ; Save a register
2531 mr r24,r6 ; Save the new protection flags
2532 stw r29,FM_ARG0+0x14(r1) ; Save a register
2533 stw r30,FM_ARG0+0x18(r1) ; Save a register
2534 stw r31,FM_ARG0+0x1C(r1) ; Save a register
2535 stw r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2537 lwz r6,pmapvr(r3) ; Get the first part of the VR translation for pmap
2538 lwz r7,pmapvr+4(r3) ; Get the second part
2541 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
2543 mr r27,r11 ; Remember the old MSR
2544 mr r26,r12 ; Remember the feature bits
2546 xor r28,r3,r7 ; Change the common 32- and 64-bit half
2548 bf-- pf64Bitb,hpSF1 ; skip if 32-bit...
2550 rldimi r28,r6,32,0 ; Shift the fixed upper part of the physical over and cram in top
2552 hpSF1: mr r29,r4 ; Save top half of vaddr
2553 mr r30,r5 ; Save the bottom half
2555 la r3,pmapSXlk(r28) ; Point to the pmap search lock
2556 bl sxlkShared ; Go get a shared lock on the mapping lists
2557 mr. r3,r3 ; Did we get the lock?
2558 bne-- hpBadLock ; Nope...
2560 mr r3,r28 ; get the pmap address
2561 mr r4,r29 ; Get bits 0:31 to look for
2562 mr r5,r30 ; Get bits 32:64
2564 bl EXT(mapSearch) ; Go see if we can find it (note: R7 comes back with mpFlags)
2566 andi. r7,r7,lo16(mpSpecial|mpNest|mpPerm|mpBlock|mpRIP) ; Are we allowed to change it or is it being removed?
2567 mr. r31,r3 ; Save the mapping if we found it
2568 cmplwi cr1,r7,0 ; Anything special going on?
2569 mr r29,r4 ; Save next va high half
2570 mr r30,r5 ; Save next va low half
2572 beq-- hpNotFound ; Not found...
2574 bne-- cr1,hpNotAllowed ; Something special is happening...
2576 bt++ pf64Bitb,hpDo64 ; Split for 64 bit
2578 bl mapInvPte32 ; Invalidate and lock PTEG, also merge into physent
2580 rlwimi r5,r24,0,mpPPb-32,mpPPb-32+2 ; Stick in the new pp
2581 mr. r3,r3 ; Was there a previously valid PTE?
2583 stb r5,mpVAddr+7(r31) ; Set the new pp field (do not muck with the rest)
2585 beq-- hpNoOld32 ; Nope...
2587 stw r5,4(r3) ; Store second half of PTE
2588 eieio ; Make sure we do not reorder
2589 stw r4,0(r3) ; Revalidate the PTE
2591 eieio ; Make sure all updates come first
2592 stw r6,0(r7) ; Unlock PCA
2594 hpNoOld32: la r3,pmapSXlk(r28) ; Point to the pmap search lock
2595 bl sxlkUnlock ; Unlock the search list
2597 li r3,mapRtOK ; Set normal return
2598 b hpR32 ; Join common...
2603 hpDo64: bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
2605 rldimi r5,r24,0,mpPPb ; Stick in the new pp
2606 mr. r3,r3 ; Was there a previously valid PTE?
2608 stb r5,mpVAddr+7(r31) ; Set the new pp field (do not muck with the rest)
2610 beq-- hpNoOld64 ; Nope...
2612 std r5,8(r3) ; Store second half of PTE
2613 eieio ; Make sure we do not reorder
2614 std r4,0(r3) ; Revalidate the PTE
2616 eieio ; Make sure all updates come first
2617 stw r6,0(r7) ; Unlock PCA
2619 hpNoOld64: la r3,pmapSXlk(r28) ; Point to the pmap search lock
2620 bl sxlkUnlock ; Unlock the search list
2622 li r3,mapRtOK ; Set normal return
2623 b hpR64 ; Join common...
2627 hpReturn: bt++ pf64Bitb,hpR64 ; Yes...
2629 hpR32: mtmsr r27 ; Restore enables/translation/etc.
2631 b hpReturnC ; Join common...
2633 hpR64: mtmsrd r27 ; Restore enables/translation/etc.
2636 hpReturnC: stw r29,0(r25) ; Save the top of the next va
2637 stw r30,4(r25) ; Save the bottom of the next va
2638 lwz r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2639 lwz r24,FM_ARG0+0x00(r1) ; Save a register
2640 lwz r25,FM_ARG0+0x04(r1) ; Save a register
2641 lwz r26,FM_ARG0+0x08(r1) ; Save a register
2642 mtlr r0 ; Restore the return
2643 lwz r27,FM_ARG0+0x0C(r1) ; Save a register
2644 lwz r28,FM_ARG0+0x10(r1) ; Save a register
2645 lwz r29,FM_ARG0+0x14(r1) ; Save a register
2646 lwz r30,FM_ARG0+0x18(r1) ; Save a register
2647 lwz r31,FM_ARG0+0x1C(r1) ; Save a register
2648 lwz r1,0(r1) ; Pop the stack
2653 hpBadLock: li r3,mapRtBadLk ; Set lock time out error code
2654 b hpReturn ; Leave....
2656 hpNotFound: la r3,pmapSXlk(r28) ; Point to the pmap search lock
2657 bl sxlkUnlock ; Unlock the search list
2659 li r3,mapRtNotFnd ; Set that we did not find the requested page
2660 b hpReturn ; Leave....
2663 rlwinm. r0,r7,0,mpRIPb,mpRIPb ; Is it actually being removed?
2664 la r3,pmapSXlk(r28) ; Point to the pmap search lock
2665 bne-- hpNotFound ; Yeah...
2666 bl sxlkUnlock ; Unlock the search list
2668 li r3,mapRtBlock ; Assume it was a block
2669 andi. r7,r7,lo16(mpBlock) ; Is this a block?
2670 bne++ hpReturn ; Yes, leave...
2672 li r3,mapRtPerm ; Set that we hit a permanent page
2673 b hpReturn ; Leave....
2677 ; int hw_test_rc(pmap, va, reset) - tests RC on a specific va
2679 ; Returns following code ORed with RC from mapping
2680 ; mapRtOK - if all is ok
2681 ; mapRtBadLk - if mapping lock fails
2682 ; mapRtNotFnd - if mapping is not found
2685 .globl EXT(hw_test_rc)
2688 stwu r1,-(FM_ALIGN((31-24+1)*4)+FM_SIZE)(r1) ; Make some space on the stack
2689 mflr r0 ; Save the link register
2690 stw r24,FM_ARG0+0x00(r1) ; Save a register
2691 stw r25,FM_ARG0+0x04(r1) ; Save a register
2692 stw r26,FM_ARG0+0x08(r1) ; Save a register
2693 stw r27,FM_ARG0+0x0C(r1) ; Save a register
2694 stw r28,FM_ARG0+0x10(r1) ; Save a register
2695 mr r24,r6 ; Save the reset request
2696 stw r29,FM_ARG0+0x14(r1) ; Save a register
2697 stw r30,FM_ARG0+0x18(r1) ; Save a register
2698 stw r31,FM_ARG0+0x1C(r1) ; Save a register
2699 stw r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2701 lwz r6,pmapvr(r3) ; Get the first part of the VR translation for pmap
2702 lwz r7,pmapvr+4(r3) ; Get the second part
2705 bl EXT(mapSetUp) ; Turn off interrupts, translation, and possibly enter 64-bit
2707 mr r27,r11 ; Remember the old MSR
2708 mr r26,r12 ; Remember the feature bits
2710 xor r28,r3,r7 ; Change the common 32- and 64-bit half
2712 bf-- pf64Bitb,htrSF1 ; skip if 32-bit...
2714 rldimi r28,r6,32,0 ; Shift the fixed upper part of the physical over and cram in top
2716 htrSF1: mr r29,r4 ; Save top half of vaddr
2717 mr r30,r5 ; Save the bottom half
2719 la r3,pmapSXlk(r28) ; Point to the pmap search lock
2720 bl sxlkShared ; Go get a shared lock on the mapping lists
2721 mr. r3,r3 ; Did we get the lock?
2723 bne-- htrBadLock ; Nope...
2725 mr r3,r28 ; get the pmap address
2726 mr r4,r29 ; Get bits 0:31 to look for
2727 mr r5,r30 ; Get bits 32:64
2729 bl EXT(mapSearch) ; Go see if we can find it (R7 comes back with mpFlags)
2731 andi. r0,r7,lo16(mpSpecial|mpNest|mpPerm|mpBlock|mpRIP) ; Are we allowed to change it or is it being removed?
2732 mr. r31,r3 ; Save the mapping if we found it
2733 cmplwi cr1,r0,0 ; Are we removing it?
2734 crorc cr0_eq,cr0_eq,cr1_eq ; Did we not find it or is it being removed?
2736 bt-- cr0_eq,htrNotFound ; Not found, something special, or being removed...
2738 bt++ pf64Bitb,htrDo64 ; Split for 64 bit
2740 bl mapInvPte32 ; Invalidate and lock PTEG, also merge into physent
2742 cmplwi cr1,r24,0 ; Do we want to clear RC?
2743 lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
2744 mr. r3,r3 ; Was there a previously valid PTE?
2745 li r0,lo16(mpR|mpC) ; Get bits to clear
2747 and r25,r5,r0 ; Save the RC bits
2748 beq++ cr1,htrNoClr32 ; Nope...
2750 andc r12,r12,r0 ; Clear mapping copy of RC
2751 andc r5,r5,r0 ; Clear PTE copy of RC
2752 sth r12,mpVAddr+6(r31) ; Set the new RC
2754 htrNoClr32: beq-- htrNoOld32 ; No previously valid PTE...
2756 sth r5,6(r3) ; Store updated RC
2757 eieio ; Make sure we do not reorder
2758 stw r4,0(r3) ; Revalidate the PTE
2760 eieio ; Make sure all updates come first
2761 stw r6,0(r7) ; Unlock PCA
2763 htrNoOld32: la r3,pmapSXlk(r28) ; Point to the pmap search lock
2764 bl sxlkUnlock ; Unlock the search list
2765 li r3,mapRtOK ; Set normal return
2766 b htrR32 ; Join common...
2771 htrDo64: bl mapInvPte64 ; Invalidate and lock PTEG, also merge into physent
2773 cmplwi cr1,r24,0 ; Do we want to clear RC?
2774 lwz r12,mpVAddr+4(r31) ; Get the bottom of the mapping vaddr field
2775 mr. r3,r3 ; Was there a previously valid PTE?
2776 li r0,lo16(mpR|mpC) ; Get bits to clear
2778 and r25,r5,r0 ; Save the RC bits
2779 beq++ cr1,htrNoClr64 ; Nope...
2781 andc r12,r12,r0 ; Clear mapping copy of RC
2782 andc r5,r5,r0 ; Clear PTE copy of RC
2783 sth r12,mpVAddr+6(r31) ; Set the new RC
2785 htrNoClr64: beq-- htrNoOld64 ; Nope, no pevious pte...
2787 sth r5,14(r3) ; Store updated RC
2788 eieio ; Make sure we do not reorder
2789 std r4,0(r3) ; Revalidate the PTE
2791 eieio ; Make sure all updates come first
2792 stw r6,0(r7) ; Unlock PCA
2794 htrNoOld64: la r3,pmapSXlk(r28) ; Point to the pmap search lock
2795 bl sxlkUnlock ; Unlock the search list
2796 li r3,mapRtOK ; Set normal return
2797 b htrR64 ; Join common...
2801 htrReturn: bt++ pf64Bitb,htrR64 ; Yes...
2803 htrR32: mtmsr r27 ; Restore enables/translation/etc.
2805 b htrReturnC ; Join common...
2807 htrR64: mtmsrd r27 ; Restore enables/translation/etc.
2810 htrReturnC: lwz r0,(FM_ALIGN((31-24+1)*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2811 or r3,r3,r25 ; Send the RC bits back
2812 lwz r24,FM_ARG0+0x00(r1) ; Save a register
2813 lwz r25,FM_ARG0+0x04(r1) ; Save a register
2814 lwz r26,FM_ARG0+0x08(r1) ; Save a register
2815 mtlr r0 ; Restore the return
2816 lwz r27,FM_ARG0+0x0C(r1) ; Save a register
2817 lwz r28,FM_ARG0+0x10(r1) ; Save a register
2818 lwz r29,FM_ARG0+0x14(r1) ; Save a register
2819 lwz r30,FM_ARG0+0x18(r1) ; Save a register
2820 lwz r31,FM_ARG0+0x1C(r1) ; Save a register
2821 lwz r1,0(r1) ; Pop the stack
2826 htrBadLock: li r3,mapRtBadLk ; Set lock time out error code
2827 b htrReturn ; Leave....
2830 la r3,pmapSXlk(r28) ; Point to the pmap search lock
2831 bl sxlkUnlock ; Unlock the search list
2833 li r3,mapRtNotFnd ; Set that we did not find the requested page
2834 b htrReturn ; Leave....
2839 ; mapPhysFindLock - find physent list and lock it
2840 ; R31 points to mapping
2845 lbz r4,mpFlags+1(r31) ; Get the index into the physent bank table
2846 lis r3,ha16(EXT(pmap_mem_regions)) ; Get high order of physent table (note use of ha16 to get value appropriate for an addi of low part)
2847 rlwinm r4,r4,2,0,29 ; Change index into byte offset
2848 addi r4,r4,lo16(EXT(pmap_mem_regions)) ; Get low part of address of entry
2849 add r3,r3,r4 ; Point to table entry
2850 lwz r5,mpPAddr(r31) ; Get physical page number
2851 lwz r7,mrStart(r3) ; Get the start of range
2852 lwz r3,mrPhysTab(r3) ; Get the start of the entries for this bank
2853 sub r6,r5,r7 ; Get index to physent
2854 rlwinm r6,r6,3,0,28 ; Get offset to physent
2855 add r3,r3,r6 ; Point right to the physent
2856 b mapPhysLock ; Join in the lock...
2859 ; mapPhysLock - lock a physent list
2860 ; R3 contains list header
2865 li r2,lgKillResv ; Get a spot to kill reservation
2866 stwcx. r2,0,r2 ; Kill it...
2869 lwz r2,ppLink(r3) ; Get physent chain header
2870 rlwinm. r2,r2,0,0,0 ; Is lock clear?
2871 bne-- mapPhysLockT ; Nope, still locked...
2874 lwarx r2,0,r3 ; Get the lock
2875 rlwinm. r0,r2,0,0,0 ; Is it locked?
2876 oris r0,r2,0x8000 ; Set the lock bit
2877 bne-- mapPhysLockS ; It is locked, spin on it...
2878 stwcx. r0,0,r3 ; Try to stuff it back...
2879 bne-- mapPhysLock ; Collision, try again...
2880 isync ; Clear any speculations
2885 ; mapPhysUnlock - unlock a physent list
2886 ; R3 contains list header
2891 lwz r0,ppLink(r3) ; Get physent chain header
2892 rlwinm r0,r0,0,1,31 ; Clear the lock bit
2893 eieio ; Make sure unlock comes last
2894 stw r0,ppLink(r3) ; Unlock the list
2898 ; mapPhysMerge - merge the RC bits into the master copy
2899 ; R3 points to the physent
2900 ; R4 contains the RC bits
2902 ; Note: we just return if RC is 0
2907 rlwinm. r4,r4,PTE1_REFERENCED_BIT+(64-ppRb),ppRb-32,ppCb-32 ; Isolate RC bits
2908 la r5,ppLink+4(r3) ; Point to the RC field
2909 beqlr-- ; Leave if RC is 0...
2912 lwarx r6,0,r5 ; Get the RC part
2913 or r6,r6,r4 ; Merge in the RC
2914 stwcx. r6,0,r5 ; Try to stuff it back...
2915 bne-- mapPhysMergeT ; Collision, try again...
2919 ; Sets the physent link pointer and preserves all flags
2920 ; The list is locked
2921 ; R3 points to physent
2922 ; R4 has link to set
2928 la r5,ppLink+4(r3) ; Point to the link word
2931 lwarx r2,0,r5 ; Get the link and flags
2932 rlwimi r4,r2,0,26,31 ; Insert the flags
2933 stwcx. r4,0,r5 ; Stick them back
2934 bne-- mapPhyCSetR ; Someone else did something, try again...
2940 li r0,0xFF ; Get mask to clean up mapping pointer
2941 rldicl r0,r0,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
2944 ldarx r2,0,r3 ; Get the link and flags
2945 and r5,r2,r0 ; Isolate the flags
2946 or r6,r4,r5 ; Add them to the link
2947 stdcx. r6,0,r3 ; Stick them back
2948 bne-- mapPhyCSet64x ; Someone else did something, try again...
2952 ; mapBumpBusy - increment the busy count on a mapping
2953 ; R3 points to mapping
2959 lwarx r4,0,r3 ; Get mpBusy
2960 addis r4,r4,0x0100 ; Bump the busy count
2961 stwcx. r4,0,r3 ; Save it back
2962 bne-- mapBumpBusy ; This did not work, try again...
2966 ; mapDropBusy - increment the busy count on a mapping
2967 ; R3 points to mapping
2970 .globl EXT(mapping_drop_busy)
2973 LEXT(mapping_drop_busy)
2975 lwarx r4,0,r3 ; Get mpBusy
2976 addis r4,r4,0xFF00 ; Drop the busy count
2977 stwcx. r4,0,r3 ; Save it back
2978 bne-- mapDropBusy ; This did not work, try again...
2982 ; mapDrainBusy - drain the busy count on a mapping
2983 ; R3 points to mapping
2984 ; Note: we already have a busy for ourselves. Only one
2985 ; busy per processor is allowed, so we just spin here
2986 ; waiting for the count to drop to 1.
2987 ; Also, the mapping can not be on any lists when we do this
2988 ; so all we are doing is waiting until it can be released.
2994 lwz r4,mpFlags(r3) ; Get mpBusy
2995 rlwinm r4,r4,8,24,31 ; Clean it up
2996 cmplwi r4,1 ; Is is just our busy?
2997 beqlr++ ; Yeah, it is clear...
2998 b mapDrainBusy ; Try again...
3003 ; handleDSeg - handle a data segment fault
3004 ; handleISeg - handle an instruction segment fault
3006 ; All that we do here is to map these to DSI or ISI and insure
3007 ; that the hash bit is not set. This forces the fault code
3008 ; to also handle the missing segment.
3010 ; At entry R2 contains per_proc, R13 contains savarea pointer,
3011 ; and R11 is the exception code.
3015 .globl EXT(handleDSeg)
3019 li r11,T_DATA_ACCESS ; Change fault to DSI
3020 stw r11,saveexception(r13) ; Change the exception code from seg fault to PTE miss
3021 b EXT(handlePF) ; Join common...
3024 .globl EXT(handleISeg)
3028 li r11,T_INSTRUCTION_ACCESS ; Change fault to ISI
3029 stw r11,saveexception(r13) ; Change the exception code from seg fault to PTE miss
3030 b EXT(handlePF) ; Join common...
3034 * handlePF - handle a page fault interruption
3036 * At entry R2 contains per_proc, R13 contains savarea pointer,
3037 * and R11 is the exception code.
3039 * This first part does a quick check to see if we can handle the fault.
3040 * We canot handle any kind of protection exceptions here, so we pass
3041 * them up to the next level.
3043 * NOTE: In order for a page-fault redrive to work, the translation miss
3044 * bit must be set in the DSISR (or SRR1 for IFETCH). That must occur
3045 * before we come here.
3049 .globl EXT(handlePF)
3053 mfsprg r12,2 ; Get feature flags
3054 cmplwi r11,T_INSTRUCTION_ACCESS ; See if this is for the instruction
3055 lwz r8,savesrr1+4(r13) ; Get the MSR to determine mode
3056 mtcrf 0x02,r12 ; move pf64Bit to cr6
3057 lis r0,hi16(dsiNoEx|dsiProt|dsiInvMode|dsiAC) ; Get the types that we cannot handle here
3058 lwz r18,SAVflags(r13) ; Get the flags
3060 beq-- gotIfetch ; We have an IFETCH here...
3062 lwz r27,savedsisr(r13) ; Get the DSISR
3063 lwz r29,savedar(r13) ; Get the first half of the DAR
3064 lwz r30,savedar+4(r13) ; And second half
3066 b ckIfProt ; Go check if this is a protection fault...
3068 gotIfetch: andis. r27,r8,hi16(dsiValid) ; Clean this up to construct a DSISR value
3069 lwz r29,savesrr0(r13) ; Get the first half of the instruction address
3070 lwz r30,savesrr0+4(r13) ; And second half
3071 stw r27,savedsisr(r13) ; Save the "constructed" DSISR
3073 ckIfProt: and. r4,r27,r0 ; Is this a non-handlable exception?
3074 li r20,64 ; Set a limit of 64 nests for sanity check
3075 bne-- hpfExit ; Yes... (probably not though)
3078 ; Note: if the RI is on, we are accessing user space from the kernel, therefore we
3079 ; should be loading the user pmap here.
3082 andi. r0,r8,lo16(MASK(MSR_PR)|MASK(MSR_RI)) ; Are we addressing user or kernel space?
3083 lis r8,hi16(EXT(kernel_pmap_phys)) ; Assume kernel
3084 mr r19,r2 ; Remember the per_proc
3085 ori r8,r8,lo16(EXT(kernel_pmap_phys)) ; Assume kernel (bottom of address)
3086 mr r23,r30 ; Save the low part of faulting address
3087 beq-- hpfInKern ; Skip if we are in the kernel
3088 la r8,ppUserPmap(r19) ; Point to the current user pmap
3090 hpfInKern: mr r22,r29 ; Save the high part of faulting address
3092 bt-- pf64Bitb,hpf64a ; If 64-bit, skip the next bit...
3095 ; On 32-bit machines we emulate a segment exception by loading unused SRs with a
3096 ; predefined value that corresponds to no address space. When we see that value
3097 ; we turn off the PTE miss bit in the DSISR to drive the code later on that will
3098 ; cause the proper SR to be loaded.
3101 lwz r28,4(r8) ; Pick up the pmap
3102 rlwinm. r18,r18,0,SAVredriveb,SAVredriveb ; Was this a redrive?
3103 mr r25,r28 ; Save the original pmap (in case we nest)
3104 bne hpfNest ; Segs are not ours if so...
3105 mfsrin r4,r30 ; Get the SR that was used for translation
3106 cmplwi r4,invalSpace ; Is this a simulated segment fault?
3107 bne++ hpfNest ; No...
3109 rlwinm r27,r27,0,dsiMissb+1,dsiMissb-1 ; Clear the PTE miss bit in DSISR
3110 b hpfNest ; Join on up...
3114 nop ; Push hpfNest to a 32-byte boundary
3115 nop ; Push hpfNest to a 32-byte boundary
3116 nop ; Push hpfNest to a 32-byte boundary
3117 nop ; Push hpfNest to a 32-byte boundary
3118 nop ; Push hpfNest to a 32-byte boundary
3119 nop ; Push hpfNest to a 32-byte boundary
3121 hpf64a: ld r28,0(r8) ; Get the pmap pointer (64-bit)
3122 mr r25,r28 ; Save the original pmap (in case we nest)
3125 ; This is where we loop descending nested pmaps
3128 hpfNest: la r3,pmapSXlk(r28) ; Point to the pmap search lock
3129 addi r20,r20,-1 ; Count nest try
3130 bl sxlkShared ; Go get a shared lock on the mapping lists
3131 mr. r3,r3 ; Did we get the lock?
3132 bne-- hpfBadLock ; Nope...
3134 mr r3,r28 ; Get the pmap pointer
3135 mr r4,r22 ; Get top of faulting vaddr
3136 mr r5,r23 ; Get bottom of faulting vaddr
3137 bl EXT(mapSearch) ; Go see if we can find it (R7 gets mpFlags)
3139 rlwinm r0,r7,0,mpRIPb,mpRIPb ; Are we removing this one?
3140 mr. r31,r3 ; Save the mapping if we found it
3141 cmplwi cr1,r0,0 ; Check for removal
3142 crorc cr0_eq,cr0_eq,cr1_eq ; Merge not found and removing
3144 bt-- cr0_eq,hpfNotFound ; Not found or removing...
3146 rlwinm. r0,r7,0,mpNestb,mpNestb ; Are we nested?
3147 mr r26,r7 ; Get the flags for this mapping (passed back from search call)
3149 lhz r21,mpSpace(r31) ; Get the space
3151 beq++ hpfFoundIt ; No, we found our guy...
3154 #if pmapTransSize != 12
3155 #error pmapTrans entry size is not 12 bytes!!!!!!!!!!!! It is pmapTransSize
3157 rlwinm. r0,r26,0,mpSpecialb,mpSpecialb ; Special handling?
3158 cmplwi cr1,r20,0 ; Too many nestings?
3159 bne-- hpfSpclNest ; Do we need to do special handling?
3161 hpfCSrch: lhz r21,mpSpace(r31) ; Get the space
3162 lwz r8,mpNestReloc(r31) ; Get the vaddr relocation
3163 lwz r9,mpNestReloc+4(r31) ; Get the vaddr relocation bottom half
3164 la r3,pmapSXlk(r28) ; Point to the old pmap search lock
3165 lis r0,0x8000 ; Get 0xFFFFFFFF80000000
3166 lis r10,hi16(EXT(pmapTrans)) ; Get the translate table
3167 add r0,r0,r0 ; Get 0xFFFFFFFF00000000 for 64-bit or 0 for 32-bit
3168 blt-- cr1,hpfNestTooMuch ; Too many nestings, must be a loop...
3169 or r23,r23,r0 ; Make sure a carry will propagate all the way in 64-bit
3170 slwi r11,r21,3 ; Multiply space by 8
3171 ori r10,r10,lo16(EXT(pmapTrans)) ; Get the translate table low part
3172 addc r23,r23,r9 ; Relocate bottom half of vaddr
3173 lwz r10,0(r10) ; Get the actual translation map
3174 slwi r12,r21,2 ; Multiply space by 4
3175 add r10,r10,r11 ; Add in the higher part of the index
3176 rlwinm r23,r23,0,0,31 ; Clean up the relocated address (does nothing in 32-bit)
3177 adde r22,r22,r8 ; Relocate the top half of the vaddr
3178 add r12,r12,r10 ; Now we are pointing at the space to pmap translation entry
3179 bl sxlkUnlock ; Unlock the search list
3181 lwz r28,pmapPAddr+4(r12) ; Get the physical address of the new pmap
3182 bf-- pf64Bitb,hpfNest ; Done if 32-bit...
3184 ld r28,pmapPAddr(r12) ; Get the physical address of the new pmap
3185 b hpfNest ; Go try the new pmap...
3188 ; Error condition. We only allow 64 nestings. This keeps us from having to
3189 ; check for recusive nests when we install them.
3195 lwz r20,savedsisr(r13) ; Get the DSISR
3196 la r3,pmapSXlk(r28) ; Point to the pmap search lock
3197 bl sxlkUnlock ; Unlock the search list (R3 good from above)
3198 ori r20,r20,1 ; Indicate that there was a nesting problem
3199 stw r20,savedsisr(r13) ; Stash it
3200 lwz r11,saveexception(r13) ; Restore the exception code
3201 b EXT(PFSExit) ; Yes... (probably not though)
3204 ; Error condition - lock failed - this is fatal
3210 lis r0,hi16(Choke) ; System abend
3211 ori r0,r0,lo16(Choke) ; System abend
3212 li r3,failMapping ; Show mapping failure
3215 ; Did not find any kind of mapping
3221 la r3,pmapSXlk(r28) ; Point to the pmap search lock
3222 bl sxlkUnlock ; Unlock it
3223 lwz r11,saveexception(r13) ; Restore the exception code
3225 hpfExit: ; We need this because we can not do a relative branch
3226 b EXT(PFSExit) ; Yes... (probably not though)
3230 ; Here is where we handle special mappings. So far, the only use is to load a
3231 ; processor specific segment register for copy in/out handling.
3233 ; The only (so far implemented) special map is used for copyin/copyout.
3234 ; We keep a mapping of a "linkage" mapping in the per_proc.
3235 ; The linkage mapping is basically a nested pmap that is switched in
3236 ; as part of context switch. It relocates the appropriate user address
3237 ; space slice into the right place in the kernel.
3243 la r31,ppCIOmp(r19) ; Just point to the mapping
3244 oris r27,r27,hi16(dsiSpcNest) ; Show that we had a special nesting here
3245 b hpfCSrch ; Go continue search...
3249 ; We have now found a mapping for the address we faulted on.
3253 ; Here we go about calculating what the VSID should be. We concatanate
3254 ; the space ID (14 bits wide) 3 times. We then slide the vaddr over
3255 ; so that bits 0:35 are in 14:49 (leaves a hole for one copy of the space ID).
3256 ; Then we XOR and expanded space ID and the shifted vaddr. This gives us
3259 ; This is used both for segment handling and PTE handling
3264 #error maxAdrSpb (address space id size) is not 14 bits!!!!!!!!!!!!
3269 hpfFoundIt: lwz r12,pmapFlags(r28) ; Get the pmap flags so we can find the keys for this segment
3270 rlwinm. r0,r27,0,dsiMissb,dsiMissb ; Did we actually miss the segment?
3271 rlwinm r15,r23,18,14,17 ; Shift 32:35 (0:3) of vaddr just above space ID
3272 rlwinm r20,r21,28,22,31 ; Shift upper 10 bits of space into high order
3273 rlwinm r14,r22,18,14,31 ; Shift 0:17 of vaddr over
3274 rlwinm r0,r27,0,dsiSpcNestb,dsiSpcNestb ; Isolate special nest flag
3275 rlwimi r21,r21,14,4,17 ; Make a second copy of space above first
3276 cmplwi cr5,r0,0 ; Did we just do a special nesting?
3277 rlwimi r15,r22,18,0,13 ; Shift 18:31 of vaddr just above shifted 32:35
3278 crorc cr0_eq,cr0_eq,cr5_eq ; Force outselves through the seg load code if special nest
3279 rlwimi r21,r21,28,0,3 ; Get low order of 3rd copy of space at top of register
3280 xor r14,r14,r20 ; Calculate the top half of VSID
3281 xor r15,r15,r21 ; Calculate the bottom half of the VSID
3282 rlwinm r14,r14,12,15,19 ; Slide the top of the VSID over to correct position (trim for 65 bit addressing)
3283 rlwinm r12,r12,9,20,22 ; Isolate and position key for cache entry
3284 rlwimi r14,r15,12,20,31 ; Slide top of bottom of VSID over into the top
3285 rlwinm r15,r15,12,0,19 ; Slide the last nybble into the low order segment position
3286 or r12,r12,r15 ; Add key into the bottom of VSID
3288 ; Note: ESID is in R22:R23 pair; VSID is in R14:R15; cache form VSID is R14:R12
3290 bne++ hpfPteMiss ; Nope, normal PTE miss...
3293 ; Here is the only place that we make an entry in the pmap segment cache.
3295 ; Note that we do not make an entry in the segment cache for special
3296 ; nested mappings. This makes the copy in/out segment get refreshed
3297 ; when switching threads.
3299 ; The first thing that we do is to look up the ESID we are going to load
3300 ; into a segment in the pmap cache. If it is already there, this is
3301 ; a segment that appeared since the last time we switched address spaces.
3302 ; If all is correct, then it was another processors that made the cache
3303 ; entry. If not, well, it is an error that we should die on, but I have
3304 ; not figured a good way to trap it yet.
3306 ; If we get a hit, we just bail, otherwise, lock the pmap cache, select
3307 ; an entry based on the generation number, update the cache entry, and
3308 ; also update the pmap sub-tag as well. The sub-tag is a table of 4 bit
3309 ; entries that correspond to the last 4 bits (32:35 for 64-bit and
3310 ; 0:3 for 32-bit) of the ESID.
3312 ; Then we unlock and bail.
3314 ; First lock it. Then select a free slot or steal one based on the generation
3315 ; number. Then store it, update the allocation flags, and unlock.
3317 ; The cache entry contains an image of the ESID/VSID pair we would load for
3318 ; 64-bit architecture. For 32-bit, it is a simple transform to an SR image.
3320 ; Remember, this cache entry goes in the ORIGINAL pmap (saved in R25), not
3321 ; the current one, which may have changed because we nested.
3323 ; Also remember that we do not store the valid bit in the ESID. If we
3324 ; od, this will break some other stuff.
3327 bne-- cr5,hpfNoCacheEnt2 ; Skip the cache entry if this is a "special nest" fault....
3329 mr r3,r25 ; Point to the pmap
3330 mr r4,r22 ; ESID high half
3331 mr r5,r23 ; ESID low half
3332 bl pmapCacheLookup ; Go see if this is in the cache already
3334 mr. r3,r3 ; Did we find it?
3335 mr r4,r11 ; Copy this to a different register
3337 bne-- hpfNoCacheEnt ; Yes, we found it, no need to make another entry...
3339 lwz r10,pmapSCSubTag(r25) ; Get the first part of the sub-tag lookup table
3340 lwz r11,pmapSCSubTag+4(r25) ; Get the second part of the sub-tag lookup table
3342 cntlzw r7,r4 ; Find a free slot
3344 subi r6,r7,pmapSegCacheUse ; We end up with a negative if we find one
3345 rlwinm r30,r30,0,0,3 ; Clean up the ESID
3346 srawi r6,r6,31 ; Get 0xFFFFFFFF if we have one, 0 if not
3347 addi r5,r4,1 ; Bump the generation number
3348 and r7,r7,r6 ; Clear bit number if none empty
3349 andc r8,r4,r6 ; Clear generation count if we found an empty
3350 rlwimi r4,r5,0,17,31 ; Insert the new generation number into the control word
3351 or r7,r7,r8 ; Select a slot number
3353 andi. r7,r7,pmapSegCacheUse-1 ; Wrap into the number we are using
3354 oris r8,r8,0x8000 ; Get the high bit on
3355 la r9,pmapSegCache(r25) ; Point to the segment cache
3356 slwi r6,r7,4 ; Get index into the segment cache
3357 slwi r2,r7,2 ; Get index into the segment cache sub-tag index
3358 srw r8,r8,r7 ; Get the mask
3359 cmplwi r2,32 ; See if we are in the first or second half of sub-tag
3361 rlwinm r2,r2,0,27,31 ; Wrap shift so we do not shift cache entries 8-F out
3362 oris r0,r0,0xF000 ; Get the sub-tag mask
3363 add r9,r9,r6 ; Point to the cache slot
3364 srw r0,r0,r2 ; Slide sub-tag mask to right slot (shift work for either half)
3365 srw r5,r30,r2 ; Slide sub-tag to right slot (shift work for either half)
3367 stw r29,sgcESID(r9) ; Save the top of the ESID
3368 andc r10,r10,r0 ; Clear sub-tag slot in case we are in top
3369 andc r11,r11,r0 ; Clear sub-tag slot in case we are in bottom
3370 stw r30,sgcESID+4(r9) ; Save the bottom of the ESID
3371 or r10,r10,r5 ; Stick in subtag in case top half
3372 or r11,r11,r5 ; Stick in subtag in case bottom half
3373 stw r14,sgcVSID(r9) ; Save the top of the VSID
3374 andc r4,r4,r8 ; Clear the invalid bit for the slot we just allocated
3375 stw r12,sgcVSID+4(r9) ; Save the bottom of the VSID and the key
3376 bge hpfSCSTbottom ; Go save the bottom part of sub-tag
3378 stw r10,pmapSCSubTag(r25) ; Save the top of the sub-tag
3379 b hpfNoCacheEnt ; Go finish up...
3382 stw r11,pmapSCSubTag+4(r25) ; Save the bottom of the sub-tag
3386 eieio ; Make sure cache is updated before lock
3387 stw r4,pmapCCtl(r25) ; Unlock, allocate, and bump generation number
3391 lwz r4,ppMapFlags(r19) ; Get the protection key modifier
3392 bt++ pf64Bitb,hpfLoadSeg64 ; If 64-bit, go load the segment...
3395 ; Make and enter 32-bit segment register
3398 lwz r16,validSegs(r19) ; Get the valid SR flags
3399 xor r12,r12,r4 ; Alter the storage key before loading segment register
3400 rlwinm r2,r30,4,28,31 ; Isolate the segment we are setting
3401 rlwinm r6,r12,19,1,3 ; Insert the keys and N bit
3402 lis r0,0x8000 ; Set bit 0
3403 rlwimi r6,r12,20,12,31 ; Insert 4:23 the VSID
3404 srw r0,r0,r2 ; Get bit corresponding to SR
3405 rlwimi r6,r14,20,8,11 ; Get the last nybble of the SR contents
3406 or r16,r16,r0 ; Show that SR is valid
3408 mtsrin r6,r30 ; Set the actual SR
3410 stw r16,validSegs(r19) ; Set the valid SR flags
3412 b hpfPteMiss ; SR loaded, go do a PTE...
3415 ; Make and enter 64-bit segment look-aside buffer entry.
3416 ; Note that the cache entry is the right format except for valid bit.
3417 ; We also need to convert from long long to 64-bit register values.
3424 ld r16,validSegs(r19) ; Get the valid SLB entry flags
3425 sldi r8,r29,32 ; Move high order address over
3426 sldi r10,r14,32 ; Move high part of VSID over
3428 not r3,r16 ; Make valids be 0s
3429 li r0,1 ; Prepare to set bit 0
3431 cntlzd r17,r3 ; Find a free SLB
3432 xor r12,r12,r4 ; Alter the storage key before loading segment table entry
3433 or r9,r8,r30 ; Form full 64-bit address
3434 cmplwi r17,63 ; Did we find a free SLB entry?
3435 sldi r0,r0,63 ; Get bit 0 set
3436 or r10,r10,r12 ; Move in low part and keys
3437 addi r17,r17,1 ; Skip SLB 0 always
3438 blt++ hpfFreeSeg ; Yes, go load it...
3441 ; No free SLB entries, select one that is in use and invalidate it
3443 lwz r4,ppSegSteal(r19) ; Get the next slot to steal
3444 addi r17,r4,pmapSegCacheUse+1 ; Select stealee from non-cached slots only
3445 addi r4,r4,1 ; Set next slot to steal
3446 slbmfee r7,r17 ; Get the entry that is in the selected spot
3447 subi r2,r4,63-pmapSegCacheUse ; Force steal to wrap
3448 rldicr r7,r7,0,35 ; Clear the valid bit and the rest
3449 srawi r2,r2,31 ; Get -1 if steal index still in range
3450 slbie r7 ; Invalidate the in-use SLB entry
3451 and r4,r4,r2 ; Reset steal index when it should wrap
3454 stw r4,ppSegSteal(r19) ; Set the next slot to steal
3456 ; We are now ready to stick the SLB entry in the SLB and mark it in use
3460 subi r4,r17,1 ; Adjust shift to account for skipping slb 0
3461 mr r7,r9 ; Get a copy of the ESID with bits 36:63 clear
3462 srd r0,r0,r4 ; Set bit mask for allocation
3463 oris r9,r9,0x0800 ; Turn on the valid bit
3464 or r16,r16,r0 ; Turn on the allocation flag
3465 rldimi r9,r17,0,58 ; Copy in the SLB entry selector
3467 beq++ cr5,hpfNoBlow ; Skip blowing away the SLBE if this is not a special nest...
3468 slbie r7 ; Blow away a potential duplicate
3470 hpfNoBlow: slbmte r10,r9 ; Make that SLB entry
3472 std r16,validSegs(r19) ; Mark as valid
3473 b hpfPteMiss ; STE loaded, go do a PTE...
3476 ; The segment has been set up and loaded if need be. Now we are ready to build the
3477 ; PTE and get it into the hash table.
3479 ; Note that there is actually a race here. If we start fault processing on
3480 ; a different pmap, i.e., we have descended into a nested pmap, it is possible
3481 ; that the nest could have been removed from the original pmap. We would
3482 ; succeed with this translation anyway. I do not think we need to worry
3483 ; about this (famous last words) because nobody should be unnesting anything
3484 ; if there are still people activily using them. It should be up to the
3485 ; higher level VM system to put the kibosh on this.
3487 ; There is also another race here: if we fault on the same mapping on more than
3488 ; one processor at the same time, we could end up with multiple PTEs for the same
3489 ; mapping. This is not a good thing.... We really only need one of the
3490 ; fault handlers to finish, so what we do is to set a "fault in progress" flag in
3491 ; the mapping. If we see that set, we just abandon the handler and hope that by
3492 ; the time we restore context and restart the interrupted code, the fault has
3493 ; been resolved by the other guy. If not, we will take another fault.
3497 ; NOTE: IMPORTANT - CR7 contains a flag indicating if we have a block mapping or not.
3498 ; It is required to stay there until after we call mapSelSlot!!!!
3503 hpfPteMiss: lwarx r0,0,r31 ; Load the mapping flag field
3504 lwz r12,mpPte(r31) ; Get the quick pointer to PTE
3505 li r3,mpHValid ; Get the PTE valid bit
3506 andi. r2,r0,lo16(mpFIP) ; Are we handling a fault on the other side?
3507 ori r2,r0,lo16(mpFIP) ; Set the fault in progress flag
3508 crnot cr1_eq,cr0_eq ; Remember if FIP was on
3509 and. r12,r12,r3 ; Isolate the valid bit
3510 crorc cr0_eq,cr1_eq,cr0_eq ; Bail if FIP is on. Then, if already have PTE, bail...
3511 beq-- hpfAbandon ; Yes, other processor is or already has handled this...
3512 andi. r0,r2,mpBlock ; Is this a block mapping?
3513 crmove cr7_eq,cr0_eq ; Remember if we have a block mapping
3514 stwcx. r2,0,r31 ; Store the flags
3515 bne-- hpfPteMiss ; Collision, try again...
3517 bt++ pf64Bitb,hpfBldPTE64 ; Skip down to the 64 bit stuff...
3520 ; At this point we are about to do the 32-bit PTE generation.
3522 ; The following is the R14:R15 pair that contains the "shifted" VSID:
3526 ; +--------+--------+--------+--------+--------+--------+--------+--------+
3527 ; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVV////|////////|
3528 ; +--------+--------+--------+--------+--------+--------+--------+--------+
3530 ; The 24 bits of the 32-bit architecture VSID is in the following:
3534 ; +--------+--------+--------+--------+--------+--------+--------+--------+
3535 ; |////////|////////|////////|////VVVV|VVVVVVVV|VVVVVVVV|VVVV////|////////|
3536 ; +--------+--------+--------+--------+--------+--------+--------+--------+
3541 lwz r25,mpVAddr+4(r31) ; Grab the base virtual address for the mapping (32-bit portion)
3542 lwz r24,mpPAddr(r31) ; Grab the base physical page number for the mapping
3544 mfsdr1 r27 ; Get the hash table base address
3546 rlwinm r0,r23,0,4,19 ; Isolate just the page index
3547 rlwinm r18,r23,10,26,31 ; Extract the API
3548 xor r19,r15,r0 ; Calculate hash << 12
3549 mr r2,r25 ; Save the flag part of the mapping
3550 rlwimi r18,r14,27,1,4 ; Move bits 28:31 of the "shifted" VSID into the PTE image
3551 rlwinm r16,r27,16,7,15 ; Extract the hash table size
3552 rlwinm r25,r25,0,0,19 ; Clear out the flags
3553 slwi r24,r24,12 ; Change ppnum to physical address (note: 36-bit addressing no supported)
3554 sub r25,r23,r25 ; Get offset in mapping to page (0 unless block map)
3555 ori r16,r16,lo16(0xFFC0) ; Slap in the bottom of the mask
3556 rlwinm r27,r27,0,0,15 ; Extract the hash table base
3557 rlwinm r19,r19,26,6,25 ; Shift hash over to make offset into hash table
3558 add r24,r24,r25 ; Adjust to true physical address
3559 rlwimi r18,r15,27,5,24 ; Move bits 32:31 of the "shifted" VSID into the PTE image
3560 rlwimi r24,r2,0,20,31 ; Slap in the WIMG and prot
3561 and r19,r19,r16 ; Wrap hash table offset into the hash table
3562 ori r24,r24,lo16(mpR) ; Turn on the reference bit right now
3563 rlwinm r20,r19,28,10,29 ; Shift hash over to make offset into PCA
3564 add r19,r19,r27 ; Point to the PTEG
3565 subfic r20,r20,-4 ; Get negative offset to PCA
3566 oris r18,r18,lo16(0x8000) ; Make sure the valid bit is on
3567 add r20,r20,r27 ; Point to the PCA slot
3570 ; We now have a valid PTE pair in R18/R24. R18 is PTE upper and R24 is PTE lower.
3571 ; R19 contains the offset of the PTEG in the hash table. R20 has offset into the PCA.
3573 ; We need to check PTE pointer (mpPte) again after we lock the PTEG. It is possible
3574 ; that some other processor beat us and stuck in a PTE or that
3575 ; all we had was a simple segment exception and the PTE was there the whole time.
3576 ; If we find one a pointer, we are done.
3579 mr r7,r20 ; Copy the PCA pointer
3580 bl mapLockPteg ; Lock the PTEG
3582 lwz r12,mpPte(r31) ; Get the offset to the PTE
3583 mr r17,r6 ; Remember the PCA image
3584 mr r16,r6 ; Prime the post-select PCA image
3585 andi. r0,r12,mpHValid ; Is there a PTE here already?
3586 li r21,8 ; Get the number of slots
3588 bne- cr7,hpfNoPte32 ; Skip this for a block mapping...
3590 bne- hpfBailOut ; Someone already did this for us...
3593 ; The mapSelSlot function selects a PTEG slot to use. As input, it uses R3 as a
3594 ; pointer to the PCA. When it returns, R3 contains 0 if an unoccupied slot was
3595 ; selected, 1 if it stole a non-block PTE, or 2 if it stole a block mapped PTE.
3596 ; R4 returns the slot index.
3598 ; REMEMBER: CR7 indicates that we are building a block mapping.
3601 hpfNoPte32: subic. r21,r21,1 ; See if we have tried all slots
3602 mr r6,r17 ; Get back the original PCA
3603 rlwimi r6,r16,0,8,15 ; Insert the updated steal slot
3604 blt- hpfBailOut ; Holy Cow, all slots are locked...
3606 bl mapSelSlot ; Go select a slot (note that the PCA image is already set up)
3608 cmplwi cr5,r3,1 ; Did we steal a slot?
3609 rlwinm r5,r4,3,26,28 ; Convert index to slot offset
3610 add r19,r19,r5 ; Point directly to the PTE
3611 mr r16,r6 ; Remember the PCA image after selection
3612 blt+ cr5,hpfInser32 ; Nope, no steal...
3614 lwz r6,0(r19) ; Get the old PTE
3615 lwz r7,4(r19) ; Get the real part of the stealee
3616 rlwinm r6,r6,0,1,31 ; Clear the valid bit
3617 bgt cr5,hpfNipBM ; Do not try to lock a non-existant physent for a block mapping...
3618 srwi r3,r7,12 ; Change phys address to a ppnum
3619 bl mapFindPhyTry ; Go find and try to lock physent (note: if R3 is 0, there is no physent for this page)
3620 cmplwi cr1,r3,0 ; Check if this is in RAM
3621 bne- hpfNoPte32 ; Could not get it, try for another...
3623 crmove cr5_gt,cr1_eq ; If we did not find a physent, pretend that this is a block map
3625 hpfNipBM: stw r6,0(r19) ; Set the invalid PTE
3627 sync ; Make sure the invalid is stored
3628 li r9,tlbieLock ; Get the TLBIE lock
3629 rlwinm r10,r6,21,0,3 ; Shift last 4 bits of space to segment part
3631 hpfTLBIE32: lwarx r0,0,r9 ; Get the TLBIE lock
3632 mfsprg r4,0 ; Get the per_proc
3633 rlwinm r8,r6,25,18,31 ; Extract the space ID
3634 rlwinm r11,r6,25,18,31 ; Extract the space ID
3635 lwz r7,hwSteals(r4) ; Get the steal count
3636 srwi r2,r6,7 ; Align segment number with hash
3637 rlwimi r11,r11,14,4,17 ; Get copy above ourselves
3638 mr. r0,r0 ; Is it locked?
3639 srwi r0,r19,6 ; Align PTEG offset for back hash
3640 xor r2,r2,r11 ; Get the segment number (plus a whole bunch of extra bits)
3641 xor r11,r11,r0 ; Hash backwards to partial vaddr
3642 rlwinm r12,r2,14,0,3 ; Shift segment up
3643 mfsprg r2,2 ; Get feature flags
3644 li r0,1 ; Get our lock word
3645 rlwimi r12,r6,22,4,9 ; Move up the API
3646 bne- hpfTLBIE32 ; It is locked, go wait...
3647 rlwimi r12,r11,12,10,19 ; Move in the rest of the vaddr
3649 stwcx. r0,0,r9 ; Try to get it
3650 bne- hpfTLBIE32 ; We was beat...
3651 addi r7,r7,1 ; Bump the steal count
3653 rlwinm. r0,r2,0,pfSMPcapb,pfSMPcapb ; Can this be an MP box?
3654 li r0,0 ; Lock clear value
3656 tlbie r12 ; Invalidate it everywhere
3658 stw r0,tlbieLock(0) ; Clear the tlbie lock
3660 beq- hpfNoTS32 ; Can not have MP on this machine...
3662 eieio ; Make sure that the tlbie happens first
3663 tlbsync ; Wait for everyone to catch up
3664 sync ; Make sure of it all
3666 hpfNoTS32: stw r7,hwSteals(r4) ; Save the steal count
3667 bgt cr5,hpfInser32 ; We just stole a block mapping...
3669 lwz r4,4(r19) ; Get the RC of the just invalidated PTE
3671 la r11,ppLink+4(r3) ; Point to the master RC copy
3672 lwz r7,ppLink+4(r3) ; Grab the pointer to the first mapping
3673 rlwinm r2,r4,27,ppRb-32,ppCb-32 ; Position the new RC
3675 hpfMrgRC32: lwarx r0,0,r11 ; Get the master RC
3676 or r0,r0,r2 ; Merge in the new RC
3677 stwcx. r0,0,r11 ; Try to stick it back
3678 bne- hpfMrgRC32 ; Try again if we collided...
3681 hpfFPnch: rlwinm. r7,r7,0,0,25 ; Clean and test mapping address
3682 beq- hpfLostPhys ; We could not find our mapping. Kick the bucket...
3684 lhz r10,mpSpace(r7) ; Get the space
3685 lwz r9,mpVAddr+4(r7) ; And the vaddr
3686 cmplw cr1,r10,r8 ; Is this one of ours?
3687 xor r9,r12,r9 ; Compare virtual address
3688 cmplwi r9,0x1000 ; See if we really match
3689 crand cr0_eq,cr1_eq,cr0_lt ; See if both space and vaddr match
3690 beq+ hpfFPnch2 ; Yes, found ours...
3692 lwz r7,mpAlias+4(r7) ; Chain on to the next
3693 b hpfFPnch ; Check it out...
3695 hpfFPnch2: sub r0,r19,r27 ; Get offset to the PTEG
3696 stw r0,mpPte(r7) ; Invalidate the quick pointer (keep quick pointer pointing to PTEG)
3697 bl mapPhysUnlock ; Unlock the physent now
3699 hpfInser32: oris r18,r18,lo16(0x8000) ; Make sure the valid bit is on
3701 stw r24,4(r19) ; Stuff in the real part of the PTE
3702 eieio ; Make sure this gets there first
3704 stw r18,0(r19) ; Stuff the virtual part of the PTE and make it valid
3705 mr r17,r16 ; Get the PCA image to save
3706 b hpfFinish ; Go join the common exit code...
3710 ; At this point we are about to do the 64-bit PTE generation.
3712 ; The following is the R14:R15 pair that contains the "shifted" VSID:
3716 ; +--------+--------+--------+--------+--------+--------+--------+--------+
3717 ; |00000000|0000000V|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVVVVVV|VVVV////|////////|
3718 ; +--------+--------+--------+--------+--------+--------+--------+--------+
3725 ld r10,mpVAddr(r31) ; Grab the base virtual address for the mapping
3726 lwz r24,mpPAddr(r31) ; Grab the base physical page number for the mapping
3728 mfsdr1 r27 ; Get the hash table base address
3730 sldi r11,r22,32 ; Slide top of adjusted EA over
3731 sldi r14,r14,32 ; Slide top of VSID over
3732 rlwinm r5,r27,0,27,31 ; Isolate the size
3733 eqv r16,r16,r16 ; Get all foxes here
3734 rlwimi r15,r23,16,20,24 ; Stick in EA[36:40] to make AVPN
3735 mr r2,r10 ; Save the flag part of the mapping
3736 or r11,r11,r23 ; Stick in bottom of adjusted EA for full 64-bit value
3737 rldicr r27,r27,0,45 ; Clean up the hash table base
3738 or r15,r15,r14 ; Stick in bottom of AVPN for full 64-bit value
3739 rlwinm r0,r11,0,4,19 ; Clear out everything but the page
3740 subfic r5,r5,46 ; Get number of leading zeros
3741 xor r19,r0,r15 ; Calculate hash
3742 ori r15,r15,1 ; Turn on valid bit in AVPN to make top of PTE
3743 srd r16,r16,r5 ; Shift over to get length of table
3744 srdi r19,r19,5 ; Convert page offset to hash table offset
3745 rldicr r16,r16,0,56 ; Clean up lower bits in hash table size
3746 rldicr r10,r10,0,51 ; Clear out flags
3747 sldi r24,r24,12 ; Change ppnum to physical address
3748 sub r11,r11,r10 ; Get the offset from the base mapping
3749 and r19,r19,r16 ; Wrap into hash table
3750 add r24,r24,r11 ; Get actual physical address of this page
3751 srdi r20,r19,5 ; Convert PTEG offset to PCA offset
3752 rldimi r24,r2,0,52 ; Insert the keys, WIMG, RC, etc.
3753 subfic r20,r20,-4 ; Get negative offset to PCA
3754 ori r24,r24,lo16(mpR) ; Force on the reference bit
3755 add r20,r20,r27 ; Point to the PCA slot
3756 add r19,r19,r27 ; Point to the PTEG
3759 ; We now have a valid PTE pair in R15/R24. R15 is PTE upper and R24 is PTE lower.
3760 ; R19 contains the offset of the PTEG in the hash table. R20 has offset into the PCA.
3762 ; We need to check PTE pointer (mpPte) again after we lock the PTEG. It is possible
3763 ; that some other processor beat us and stuck in a PTE or that
3764 ; all we had was a simple segment exception and the PTE was there the whole time.
3765 ; If we find one a pointer, we are done.
3768 mr r7,r20 ; Copy the PCA pointer
3769 bl mapLockPteg ; Lock the PTEG
3771 lwz r12,mpPte(r31) ; Get the offset to the PTE
3772 mr r17,r6 ; Remember the PCA image
3773 mr r18,r6 ; Prime post-selection PCA image
3774 andi. r0,r12,mpHValid ; See if we have a PTE now
3775 li r21,8 ; Get the number of slots
3777 bne-- cr7,hpfNoPte64 ; Skip this for a block mapping...
3779 bne-- hpfBailOut ; Someone already did this for us...
3782 ; The mapSelSlot function selects a PTEG slot to use. As input, it uses R3 as a
3783 ; pointer to the PCA. When it returns, R3 contains 0 if an unoccupied slot was
3784 ; selected, 1 if it stole a non-block PTE, or 2 if it stole a block mapped PTE.
3785 ; R4 returns the slot index.
3787 ; REMEMBER: CR7 indicates that we are building a block mapping.
3790 hpfNoPte64: subic. r21,r21,1 ; See if we have tried all slots
3791 mr r6,r17 ; Restore original state of PCA
3792 rlwimi r6,r18,0,8,15 ; Insert the updated steal slot
3793 blt- hpfBailOut ; Holy Cow, all slots are locked...
3795 bl mapSelSlot ; Go select a slot
3797 cmplwi cr5,r3,1 ; Did we steal a slot?
3798 rlwinm r5,r4,4,25,27 ; Convert index to slot offset
3799 mr r18,r6 ; Remember the PCA image after selection
3800 add r19,r19,r5 ; Point directly to the PTE
3801 lwz r10,hwSteals(r2) ; Get the steal count
3802 blt++ cr5,hpfInser64 ; Nope, no steal...
3804 ld r6,0(r19) ; Get the old PTE
3805 ld r7,8(r19) ; Get the real part of the stealee
3806 rldicr r6,r6,0,62 ; Clear the valid bit
3807 bgt cr5,hpfNipBMx ; Do not try to lock a non-existant physent for a block mapping...
3808 srdi r3,r7,12 ; Change page address to a page address
3809 bl mapFindPhyTry ; Go find and try to lock physent (note: if R3 is 0, there is no physent for this page)
3810 cmplwi cr1,r3,0 ; Check if this is in RAM
3811 bne-- hpfNoPte64 ; Could not get it, try for another...
3813 crmove cr5_gt,cr1_eq ; If we did not find a physent, pretend that this is a block map
3815 hpfNipBMx: std r6,0(r19) ; Set the invalid PTE
3816 li r9,tlbieLock ; Get the TLBIE lock
3818 srdi r11,r6,5 ; Shift VSID over for back hash
3819 mfsprg r4,0 ; Get the per_proc
3820 xor r11,r11,r19 ; Hash backwards to get low bits of VPN
3821 sync ; Make sure the invalid is stored
3823 sldi r12,r6,16 ; Move AVPN to EA position
3824 sldi r11,r11,5 ; Move this to the page position
3826 hpfTLBIE64: lwarx r0,0,r9 ; Get the TLBIE lock
3827 mr. r0,r0 ; Is it locked?
3828 li r0,1 ; Get our lock word
3829 bne-- hpfTLBIE65 ; It is locked, go wait...
3831 stwcx. r0,0,r9 ; Try to get it
3832 rldimi r12,r11,0,41 ; Stick the low part of the page number into the AVPN
3833 rldicl r8,r6,52,50 ; Isolate the address space ID
3834 bne-- hpfTLBIE64 ; We was beat...
3835 addi r10,r10,1 ; Bump the steal count
3837 rldicl r11,r12,0,16 ; Clear cause the book says so
3838 li r0,0 ; Lock clear value
3840 tlbie r11 ; Invalidate it everywhere
3842 stw r0,tlbieLock(0) ; Clear the tlbie lock
3844 mr r7,r8 ; Get a copy of the space ID
3845 eieio ; Make sure that the tlbie happens first
3846 rldimi r7,r7,14,36 ; Copy address space to make hash value
3847 tlbsync ; Wait for everyone to catch up
3848 rldimi r7,r7,28,22 ; Add in a 3rd copy of the hash up top
3850 srdi r2,r6,26 ; Shift original segment down to bottom
3852 ptesync ; Make sure of it all
3853 xor r7,r7,r2 ; Compute original segment
3855 stw r10,hwSteals(r4) ; Save the steal count
3856 bgt cr5,hpfInser64 ; We just stole a block mapping...
3858 rldimi r12,r7,28,0 ; Insert decoded segment
3859 rldicl r4,r12,0,13 ; Trim to max supported address
3861 ld r12,8(r19) ; Get the RC of the just invalidated PTE
3863 la r11,ppLink+4(r3) ; Point to the master RC copy
3864 ld r7,ppLink(r3) ; Grab the pointer to the first mapping
3865 rlwinm r2,r12,27,ppRb-32,ppCb-32 ; Position the new RC
3867 hpfMrgRC64: lwarx r0,0,r11 ; Get the master RC
3868 li r12,0xFF ; Get mask to clean up alias pointer
3869 or r0,r0,r2 ; Merge in the new RC
3870 rldicl r12,r12,62,0 ; Rotate clean up mask to get 0xC0000000000000003F
3871 stwcx. r0,0,r11 ; Try to stick it back
3872 bne-- hpfMrgRC64 ; Try again if we collided...
3874 hpfFPnchx: andc. r7,r7,r12 ; Clean and test mapping address
3875 beq-- hpfLostPhys ; We could not find our mapping. Kick the bucket...
3877 lhz r10,mpSpace(r7) ; Get the space
3878 ld r9,mpVAddr(r7) ; And the vaddr
3879 cmplw cr1,r10,r8 ; Is this one of ours?
3880 xor r9,r4,r9 ; Compare virtual address
3881 cmpldi r9,0x1000 ; See if we really match
3882 crand cr0_eq,cr1_eq,cr0_lt ; See if both space and vaddr match
3883 beq++ hpfFPnch2x ; Yes, found ours...
3885 ld r7,mpAlias(r7) ; Chain on to the next
3886 b hpfFPnchx ; Check it out...
3890 hpfTLBIE65: li r7,lgKillResv ; Point to the reservatio kill area
3891 stwcx. r7,0,r7 ; Kill reservation
3893 hpfTLBIE63: lwz r0,0(r9) ; Get the TLBIE lock
3894 mr. r0,r0 ; Is it locked?
3895 beq++ hpfTLBIE64 ; Yup, wait for it...
3896 b hpfTLBIE63 ; Nope, try again..
3900 hpfFPnch2x: sub r0,r19,r27 ; Get offset to PTEG
3901 stw r0,mpPte(r7) ; Invalidate the quick pointer (keep pointing at PTEG though)
3902 bl mapPhysUnlock ; Unlock the physent now
3905 hpfInser64: std r24,8(r19) ; Stuff in the real part of the PTE
3906 eieio ; Make sure this gets there first
3907 std r15,0(r19) ; Stuff the virtual part of the PTE and make it valid
3908 mr r17,r18 ; Get the PCA image to set
3909 b hpfFinish ; Go join the common exit code...
3912 lis r0,hi16(Choke) ; System abend - we must find the stolen mapping or we are dead
3913 ori r0,r0,lo16(Choke) ; System abend
3917 ; This is the common code we execute when we are finished setting up the PTE.
3922 hpfFinish: sub r4,r19,r27 ; Get offset of PTE
3923 ori r4,r4,lo16(mpHValid) ; Add valid bit to PTE offset
3924 bne cr7,hpfBailOut ; Do not set the PTE pointer for a block map
3925 stw r4,mpPte(r31) ; Remember our PTE
3927 hpfBailOut: eieio ; Make sure all updates come first
3928 stw r17,0(r20) ; Unlock and set the final PCA
3931 ; This is where we go if we have started processing the fault, but find that someone
3932 ; else has taken care of it.
3935 hpfIgnore: lwz r2,mpFlags(r31) ; Get the mapping flags
3936 rlwinm r2,r2,0,mpFIPb+1,mpFIPb-1 ; Clear the "fault in progress" flag
3937 sth r2,mpFlags+2(r31) ; Set it
3939 la r3,pmapSXlk(r28) ; Point to the pmap search lock
3940 bl sxlkUnlock ; Unlock the search list
3942 li r11,T_IN_VAIN ; Say that it was handled
3943 b EXT(PFSExit) ; Leave...
3946 ; This is where we go when we find that someone else
3947 ; is in the process of handling the fault.
3950 hpfAbandon: li r3,lgKillResv ; Kill off any reservation
3951 stwcx. r3,0,r3 ; Do it
3953 la r3,pmapSXlk(r28) ; Point to the pmap search lock
3954 bl sxlkUnlock ; Unlock the search list
3956 li r11,T_IN_VAIN ; Say that it was handled
3957 b EXT(PFSExit) ; Leave...
3962 * hw_set_user_space(pmap)
3963 * hw_set_user_space_dis(pmap)
3965 * Indicate whether memory space needs to be switched.
3966 * We really need to turn off interrupts here, because we need to be non-preemptable
3968 * hw_set_user_space_dis is used when interruptions are already disabled. Mind the
3969 * register usage here. The VMM switch code in vmachmon.s that calls this
3970 * know what registers are in use. Check that if these change.
3976 .globl EXT(hw_set_user_space)
3978 LEXT(hw_set_user_space)
3980 lis r8,hi16(MASK(MSR_VEC)) ; Get the vector enable
3981 mfmsr r10 ; Get the current MSR
3982 ori r8,r8,lo16(MASK(MSR_FP)) ; Add in FP
3983 ori r9,r8,lo16(MASK(MSR_EE)) ; Add in the EE
3984 andc r10,r10,r8 ; Turn off VEC, FP for good
3985 andc r9,r10,r9 ; Turn off EE also
3986 mtmsr r9 ; Disable them
3987 isync ; Make sure FP and vec are off
3988 mfsprg r6,0 ; Get the per_proc_info address
3989 lwz r2,ppUserPmapVirt(r6) ; Get our virtual pmap address
3990 mfsprg r4,2 ; The the feature flags
3991 lwz r7,pmapvr(r3) ; Get the v to r translation
3992 lwz r8,pmapvr+4(r3) ; Get the v to r translation
3993 mtcrf 0x80,r4 ; Get the Altivec flag
3994 xor r4,r3,r8 ; Get bottom of the real address of bmap anchor
3995 cmplw cr1,r3,r2 ; Same address space as before?
3996 stw r7,ppUserPmap(r6) ; Show our real pmap address
3997 crorc cr1_eq,cr1_eq,pfAltivecb ; See if same address space or not altivec machine
3998 stw r4,ppUserPmap+4(r6) ; Show our real pmap address
3999 stw r3,ppUserPmapVirt(r6) ; Show our virtual pmap address
4000 mtmsr r10 ; Restore interruptions
4001 beqlr-- cr1 ; Leave if the same address space or not Altivec
4003 dssall ; Need to kill all data streams if adrsp changed
4008 .globl EXT(hw_set_user_space_dis)
4010 LEXT(hw_set_user_space_dis)
4012 lwz r7,pmapvr(r3) ; Get the v to r translation
4013 mfsprg r4,2 ; The the feature flags
4014 lwz r8,pmapvr+4(r3) ; Get the v to r translation
4015 mfsprg r6,0 ; Get the per_proc_info address
4016 lwz r2,ppUserPmapVirt(r6) ; Get our virtual pmap address
4017 mtcrf 0x80,r4 ; Get the Altivec flag
4018 xor r4,r3,r8 ; Get bottom of the real address of bmap anchor
4019 cmplw cr1,r3,r2 ; Same address space as before?
4020 stw r7,ppUserPmap(r6) ; Show our real pmap address
4021 crorc cr1_eq,cr1_eq,pfAltivecb ; See if same address space or not altivec machine
4022 stw r4,ppUserPmap+4(r6) ; Show our real pmap address
4023 stw r3,ppUserPmapVirt(r6) ; Show our virtual pmap address
4024 beqlr-- cr1 ; Leave if the same
4026 dssall ; Need to kill all data streams if adrsp changed
4030 /* int mapalc1(struct mappingblok *mb) - Finds, allocates, and zeros a free 1-bit mapping entry
4032 * Lock must already be held on mapping block list
4033 * returns 0 if all slots filled.
4034 * returns n if a slot is found and it is not the last
4035 * returns -n if a slot is found and it is the last
4036 * when n and -n are returned, the corresponding bit is cleared
4037 * the mapping is zeroed out before return
4045 lwz r4,mbfree(r3) ; Get the 1st mask
4046 lis r0,0x8000 ; Get the mask to clear the first free bit
4047 lwz r5,mbfree+4(r3) ; Get the 2nd mask
4048 mr r12,r3 ; Save the block ptr
4049 cntlzw r3,r4 ; Get first 1-bit in 1st word
4050 srw. r9,r0,r3 ; Get bit corresponding to first free one
4051 cntlzw r10,r5 ; Get first free field in second word
4052 andc r4,r4,r9 ; Turn 1-bit off in 1st word
4053 bne mapalc1f ; Found one in 1st word
4055 srw. r9,r0,r10 ; Get bit corresponding to first free one in 2nd word
4056 li r3,0 ; assume failure return
4057 andc r5,r5,r9 ; Turn it off
4058 beqlr-- ; There are no 1 bits left...
4059 addi r3,r10,32 ; set the correct number
4062 or. r0,r4,r5 ; any more bits set?
4063 stw r4,mbfree(r12) ; update bitmasks
4064 stw r5,mbfree+4(r12)
4066 slwi r6,r3,6 ; get (n * mpBasicSize), ie offset of mapping in block
4068 dcbz r6,r12 ; clear the 64-byte mapping
4071 bnelr++ ; return if another bit remains set
4073 neg r3,r3 ; indicate we just returned the last bit
4077 /* int mapalc2(struct mappingblok *mb) - Finds, allocates, and zero's a free 2-bit mapping entry
4079 * Lock must already be held on mapping block list
4080 * returns 0 if all slots filled.
4081 * returns n if a slot is found and it is not the last
4082 * returns -n if a slot is found and it is the last
4083 * when n and -n are returned, the corresponding bits are cleared
4084 * We find runs of 2 consecutive 1 bits by cntlzw(n & (n<<1)).
4085 * the mapping is zero'd out before return
4091 lwz r4,mbfree(r3) ; Get the first mask
4092 lis r0,0x8000 ; Get the mask to clear the first free bit
4093 lwz r5,mbfree+4(r3) ; Get the second mask
4094 mr r12,r3 ; Save the block ptr
4095 slwi r6,r4,1 ; shift first word over
4096 and r6,r4,r6 ; lite start of double bit runs in 1st word
4097 slwi r7,r5,1 ; shift 2nd word over
4098 cntlzw r3,r6 ; Get first free 2-bit run in 1st word
4099 and r7,r5,r7 ; lite start of double bit runs in 2nd word
4100 srw. r9,r0,r3 ; Get bit corresponding to first run in 1st word
4101 cntlzw r10,r7 ; Get first free field in second word
4102 srwi r11,r9,1 ; shift over for 2nd bit in 1st word
4103 andc r4,r4,r9 ; Turn off 1st bit in 1st word
4104 andc r4,r4,r11 ; turn off 2nd bit in 1st word
4105 bne mapalc2a ; Found two consecutive free bits in 1st word
4107 srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
4108 li r3,0 ; assume failure
4109 srwi r11,r9,1 ; get mask for 2nd bit
4110 andc r5,r5,r9 ; Turn off 1st bit in 2nd word
4111 andc r5,r5,r11 ; turn off 2nd bit in 2nd word
4112 beq-- mapalc2c ; There are no runs of 2 bits in 2nd word either
4113 addi r3,r10,32 ; set the correct number
4116 or. r0,r4,r5 ; any more bits set?
4117 stw r4,mbfree(r12) ; update bitmasks
4118 stw r5,mbfree+4(r12)
4119 slwi r6,r3,6 ; get (n * mpBasicSize), ie offset of mapping in block
4123 dcbz r6,r12 ; zero out the 128-byte mapping
4124 dcbz r7,r12 ; we use the slow 32-byte dcbz even on 64-bit machines
4125 dcbz r8,r12 ; because the mapping may not be 128-byte aligned
4128 bnelr++ ; return if another bit remains set
4130 neg r3,r3 ; indicate we just returned the last bit
4134 rlwinm r7,r5,1,31,31 ; move bit 0 of 2nd word to bit 31
4135 and. r0,r4,r7 ; is the 2-bit field that spans the 2 words free?
4136 beqlr ; no, we failed
4137 rlwinm r4,r4,0,0,30 ; yes, turn off bit 31 of 1st word
4138 rlwinm r5,r5,0,1,31 ; turn off bit 0 of 2nd word
4139 li r3,31 ; get index of this field
4144 ; This routine initialzes the hash table and PCA.
4145 ; It is done here because we may need to be 64-bit to do it.
4149 .globl EXT(hw_hash_init)
4153 mfsprg r10,2 ; Get feature flags
4154 lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
4155 mtcrf 0x02,r10 ; move pf64Bit to cr6
4156 lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
4157 lis r4,0xFF01 ; Set all slots free and start steal at end
4158 ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
4159 ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
4161 lwz r12,0(r12) ; Get hash table size
4163 bt++ pf64Bitb,hhiSF ; skip if 64-bit (only they take the hint)
4165 lwz r11,4(r11) ; Get hash table base
4167 hhiNext32: cmplw r3,r12 ; Have we reached the end?
4168 bge- hhiCPCA32 ; Yes...
4169 dcbz r3,r11 ; Clear the line
4170 addi r3,r3,32 ; Next one...
4171 b hhiNext32 ; Go on...
4173 hhiCPCA32: rlwinm r12,r12,28,4,29 ; Get number of slots * 4
4174 li r3,-4 ; Displacement to first PCA entry
4175 neg r12,r12 ; Get negative end of PCA
4177 hhiNPCA32: stwx r4,r3,r11 ; Initialize the PCA entry
4178 subi r3,r3,4 ; Next slot
4179 cmpw r3,r12 ; Have we finished?
4180 bge+ hhiNPCA32 ; Not yet...
4183 hhiSF: mfmsr r9 ; Save the MSR
4185 mr r0,r9 ; Get a copy of the MSR
4186 ld r11,0(r11) ; Get hash table base
4187 rldimi r0,r8,63,MSR_SF_BIT ; Set SF bit (bit 0)
4188 mtmsrd r0 ; Turn on SF
4192 hhiNext64: cmpld r3,r12 ; Have we reached the end?
4193 bge-- hhiCPCA64 ; Yes...
4194 dcbz128 r3,r11 ; Clear the line
4195 addi r3,r3,128 ; Next one...
4196 b hhiNext64 ; Go on...
4198 hhiCPCA64: rlwinm r12,r12,27,5,29 ; Get number of slots * 4
4199 li r3,-4 ; Displacement to first PCA entry
4200 neg r12,r12 ; Get negative end of PCA
4202 hhiNPCA64: stwx r4,r3,r11 ; Initialize the PCA entry
4203 subi r3,r3,4 ; Next slot
4204 cmpd r3,r12 ; Have we finished?
4205 bge++ hhiNPCA64 ; Not yet...
4207 mtmsrd r9 ; Turn off SF if it was off
4213 ; This routine sets up the hardware to start translation.
4214 ; Note that we do NOT start translation.
4218 .globl EXT(hw_setup_trans)
4220 LEXT(hw_setup_trans)
4222 mfsprg r11,0 ; Get the per_proc block
4223 mfsprg r12,2 ; Get feature flags
4226 mtcrf 0x02,r12 ; Move pf64Bit to cr6
4227 stw r0,validSegs(r11) ; Make sure we think all SR/STEs are invalid
4228 stw r0,validSegs+4(r11) ; Make sure we think all SR/STEs are invalid, part deux
4229 sth r2,ppInvSeg(r11) ; Force a reload of the SRs
4230 sth r0,ppCurSeg(r11) ; Set that we are starting out in kernel
4232 bt++ pf64Bitb,hstSF ; skip if 64-bit (only they take the hint)
4234 li r9,0 ; Clear out a register
4237 mtdbatu 0,r9 ; Invalidate maps
4238 mtdbatl 0,r9 ; Invalidate maps
4239 mtdbatu 1,r9 ; Invalidate maps
4240 mtdbatl 1,r9 ; Invalidate maps
4241 mtdbatu 2,r9 ; Invalidate maps
4242 mtdbatl 2,r9 ; Invalidate maps
4243 mtdbatu 3,r9 ; Invalidate maps
4244 mtdbatl 3,r9 ; Invalidate maps
4246 mtibatu 0,r9 ; Invalidate maps
4247 mtibatl 0,r9 ; Invalidate maps
4248 mtibatu 1,r9 ; Invalidate maps
4249 mtibatl 1,r9 ; Invalidate maps
4250 mtibatu 2,r9 ; Invalidate maps
4251 mtibatl 2,r9 ; Invalidate maps
4252 mtibatu 3,r9 ; Invalidate maps
4253 mtibatl 3,r9 ; Invalidate maps
4255 lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
4256 lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
4257 ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
4258 ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
4259 lwz r11,4(r11) ; Get hash table base
4260 lwz r12,0(r12) ; Get hash table size
4261 subi r12,r12,1 ; Back off by 1
4262 rlwimi r11,r12,16,23,31 ; Stick the size into the sdr1 image
4264 mtsdr1 r11 ; Ok, we now have the hash table set up
4267 li r12,invalSpace ; Get the invalid segment value
4268 li r10,0 ; Start low
4270 hstsetsr: mtsrin r12,r10 ; Set the SR
4271 addis r10,r10,0x1000 ; Bump the segment
4272 mr. r10,r10 ; Are we finished?
4273 bne+ hstsetsr ; Nope...
4281 hstSF: lis r11,hi16(EXT(hash_table_base)) ; Get hash table base address
4282 lis r12,hi16(EXT(hash_table_size)) ; Get hash table size address
4283 ori r11,r11,lo16(EXT(hash_table_base)) ; Get hash table base address
4284 ori r12,r12,lo16(EXT(hash_table_size)) ; Get hash table size address
4285 ld r11,0(r11) ; Get hash table base
4286 lwz r12,0(r12) ; Get hash table size
4287 cntlzw r10,r12 ; Get the number of bits
4288 subfic r10,r10,13 ; Get the extra bits we need
4289 or r11,r11,r10 ; Add the size field to SDR1
4291 mtsdr1 r11 ; Ok, we now have the hash table set up
4294 li r0,0 ; Set an SLB slot index of 0
4295 slbia ; Trash all SLB entries (except for entry 0 that is)
4296 slbmfee r7,r0 ; Get the entry that is in SLB index 0
4297 rldicr r7,r7,0,35 ; Clear the valid bit and the rest
4298 slbie r7 ; Invalidate it
4304 ; This routine turns on translation for the first time on a processor
4308 .globl EXT(hw_start_trans)
4310 LEXT(hw_start_trans)
4313 mfmsr r10 ; Get the msr
4314 ori r10,r10,lo16(MASK(MSR_IR) | MASK(MSR_DR)) ; Turn on translation
4316 mtmsr r10 ; Everything falls apart here
4324 ; This routine validates a segment register.
4325 ; hw_map_seg(pmap_t pmap, addr64_t seg, addr64_t va)
4328 ; r4 = segment[0:31]
4329 ; r5 = segment[32:63]
4333 ; Note that we transform the addr64_t (long long) parameters into single 64-bit values.
4334 ; Note that there is no reason to apply the key modifier here because this is only
4335 ; used for kernel accesses.
4339 .globl EXT(hw_map_seg)
4343 lwz r0,pmapSpace(r3) ; Get the space, we will need it soon
4344 lwz r9,pmapFlags(r3) ; Get the flags for the keys now
4345 mfsprg r10,2 ; Get feature flags
4346 mfsprg r12,0 ; Get the per_proc
4349 ; Note: the following code would problably be easier to follow if I split it,
4350 ; but I just wanted to see if I could write this to work on both 32- and 64-bit
4351 ; machines combined.
4355 ; Here we enter with va[0:31] in r6[0:31] (or r6[32:63] on 64-bit machines)
4356 ; and va[32:63] in r7[0:31] (or r7[32:63] on 64-bit machines)
4358 rlwinm r4,r4,0,1,0 ; Copy seg[0:31] into r4[0;31] - no-op for 32-bit
4359 rlwinm r7,r7,18,14,17 ; Slide va[32:35] east to just west of space ID
4360 mtcrf 0x02,r10 ; Move pf64Bit and pfNoMSRirb to cr5 and 6
4361 srwi r8,r6,14 ; Slide va[0:17] east to just west of the rest
4362 rlwimi r7,r6,18,0,13 ; Slide va[18:31] east to just west of slid va[32:25]
4363 rlwimi r0,r0,14,4,17 ; Dup address space ID above itself
4364 rlwinm r8,r8,0,1,0 ; Dup low part into high (does nothing on 32-bit machines)
4365 rlwinm r2,r0,28,0,31 ; Rotate rotate low nybble to top of low half
4366 rlwimi r2,r2,0,1,0 ; Replicate bottom 32 into top 32
4367 rlwimi r8,r7,0,0,31 ; Join va[0:17] with va[18:35] (just like mr on 32-bit machines)
4369 rlwimi r2,r0,0,4,31 ; We should now have 4 copies of the space
4370 ; concatenated together. There is garbage
4371 ; at the top for 64-bit but we will clean
4373 rlwimi r4,r5,0,0,31 ; Copy seg[32:63] into r4[32:63] - just like mr for 32-bit
4377 ; Here we exit with va[0:35] shifted into r8[14:51], zeros elsewhere, or
4378 ; va[18:35] shifted into r8[0:17], zeros elsewhere on 32-bit machines
4382 ; What we have now is:
4385 ; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
4386 ; +--------+--------+--------+--------+--------+--------+--------+--------+
4387 ; r2 = |xxxx0000|AAAAAAAA|AAAAAABB|BBBBBBBB|BBBBCCCC|CCCCCCCC|CCDDDDDD|DDDDDDDD| - hash value
4388 ; +--------+--------+--------+--------+--------+--------+--------+--------+
4389 ; 0 0 1 2 3 - for 32-bit machines
4393 ; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
4394 ; +--------+--------+--------+--------+--------+--------+--------+--------+
4395 ; r8 = |00000000|000000SS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SS000000|00000000| - shifted and cleaned EA
4396 ; +--------+--------+--------+--------+--------+--------+--------+--------+
4397 ; 0 0 1 2 3 - for 32-bit machines
4401 ; 0 8 6 4 2 0 8 6 3 - for 64-bit machines
4402 ; +--------+--------+--------+--------+--------+--------+--------+--------+
4403 ; r4 = |SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSSSSSS|SSSS0000|00000000|00000000|00000000| - Segment
4404 ; +--------+--------+--------+--------+--------+--------+--------+--------+
4405 ; 0 0 1 2 3 - for 32-bit machines
4409 xor r8,r8,r2 ; Calculate VSID
4411 bf-- pf64Bitb,hms32bit ; Skip out if 32-bit...
4413 li r0,1 ; Prepare to set bit 0 (also to clear EE)
4414 mfmsr r6 ; Get current MSR
4415 li r2,MASK(MSR_IR)|MASK(MSR_DR) ; Get the translation bits
4416 mtmsrd r0,1 ; Set only the EE bit to 0
4417 rlwinm r6,r6,0,MSR_EE_BIT,MSR_EE_BIT ; See if EE bit is on
4418 mfmsr r11 ; Get the MSR right now, after disabling EE
4419 andc r2,r11,r2 ; Turn off translation now
4420 rldimi r2,r0,63,0 ; Get bit 64-bit turned on
4421 or r11,r11,r6 ; Turn on the EE bit if it was on
4422 mtmsrd r2 ; Make sure translation and EE are off and 64-bit is on
4423 isync ; Hang out a bit
4425 ld r6,validSegs(r12) ; Get the valid SLB entry flags
4426 sldi r9,r9,9 ; Position the key and noex bit
4428 rldimi r5,r8,12,0 ; Form the VSID/key
4430 not r3,r6 ; Make valids be 0s
4432 cntlzd r7,r3 ; Find a free SLB
4433 cmplwi r7,63 ; Did we find a free SLB entry?
4435 slbie r4 ; Since this ESID may still be in an SLBE, kill it
4437 oris r4,r4,0x0800 ; Turn on the valid bit in ESID
4438 addi r7,r7,1 ; Make sure we skip slb 0
4439 blt++ hmsFreeSeg ; Yes, go load it...
4442 ; No free SLB entries, select one that is in use and invalidate it
4444 lwz r2,ppSegSteal(r12) ; Get the next slot to steal
4445 addi r7,r2,pmapSegCacheUse+1 ; Select stealee from non-cached slots only
4446 addi r2,r2,1 ; Set next slot to steal
4447 slbmfee r3,r7 ; Get the entry that is in the selected spot
4448 subi r8,r2,64-(pmapSegCacheUse+1) ; Force steal to wrap
4449 rldicr r3,r3,0,35 ; Clear the valid bit and the rest
4450 srawi r8,r8,31 ; Get -1 if steal index still in range
4451 slbie r3 ; Invalidate the in-use SLB entry
4452 and r2,r2,r8 ; Reset steal index when it should wrap
4455 stw r2,ppSegSteal(r12) ; Set the next slot to steal
4457 ; We are now ready to stick the SLB entry in the SLB and mark it in use
4460 hmsFreeSeg: subi r2,r7,1 ; Adjust for skipped slb 0
4461 rldimi r4,r7,0,58 ; Copy in the SLB entry selector
4462 srd r0,r0,r2 ; Set bit mask for allocation
4463 rldicl r5,r5,0,15 ; Clean out the unsupported bits
4464 or r6,r6,r0 ; Turn on the allocation flag
4466 slbmte r5,r4 ; Make that SLB entry
4468 std r6,validSegs(r12) ; Mark as valid
4469 mtmsrd r11 ; Restore the MSR
4475 hms32bit: rlwinm r8,r8,0,8,31 ; Clean up the VSID
4476 rlwinm r2,r4,4,28,31 ; Isolate the segment we are setting
4477 lis r0,0x8000 ; Set bit 0
4478 rlwimi r8,r9,28,1,3 ; Insert the keys and N bit
4479 srw r0,r0,r2 ; Get bit corresponding to SR
4480 addi r7,r12,validSegs ; Point to the valid segment flags directly
4482 mtsrin r8,r4 ; Set the actual SR
4483 isync ; Need to make sure this is done
4485 hmsrupt: lwarx r6,0,r7 ; Get and reserve the valid segment flags
4486 or r6,r6,r0 ; Show that SR is valid
4487 stwcx. r6,0,r7 ; Set the valid SR flags
4488 bne-- hmsrupt ; Had an interrupt, need to get flags again...
4494 ; This routine invalidates a segment register.
4498 .globl EXT(hw_blow_seg)
4502 mfsprg r10,2 ; Get feature flags
4503 mfsprg r12,0 ; Get the per_proc
4504 mtcrf 0x02,r10 ; move pf64Bit and pfNoMSRirb to cr5 and 6
4506 addi r7,r12,validSegs ; Point to the valid segment flags directly
4507 rlwinm r9,r4,0,0,3 ; Save low segment address and make sure it is clean
4509 bf-- pf64Bitb,hbs32bit ; Skip out if 32-bit...
4511 li r0,1 ; Prepare to set bit 0 (also to clear EE)
4512 mfmsr r6 ; Get current MSR
4513 li r2,MASK(MSR_IR)|MASK(MSR_DR) ; Get the translation bits
4514 mtmsrd r0,1 ; Set only the EE bit to 0
4515 rlwinm r6,r6,0,MSR_EE_BIT,MSR_EE_BIT ; See if EE bit is on
4516 mfmsr r11 ; Get the MSR right now, after disabling EE
4517 andc r2,r11,r2 ; Turn off translation now
4518 rldimi r2,r0,63,0 ; Get bit 64-bit turned on
4519 or r11,r11,r6 ; Turn on the EE bit if it was on
4520 mtmsrd r2 ; Make sure translation and EE are off and 64-bit is on
4521 isync ; Hang out a bit
4523 rldimi r9,r3,32,0 ; Insert the top part of the ESID
4525 slbie r9 ; Invalidate the associated SLB entry
4527 mtmsrd r11 ; Restore the MSR
4533 hbs32bit: lwarx r4,0,r7 ; Get and reserve the valid segment flags
4534 rlwinm r6,r9,4,28,31 ; Convert segment to number
4535 lis r2,0x8000 ; Set up a mask
4536 srw r2,r2,r6 ; Make a mask
4537 and. r0,r4,r2 ; See if this is even valid
4538 li r5,invalSpace ; Set the invalid address space VSID
4539 beqlr ; Leave if already invalid...
4541 mtsrin r5,r9 ; Slam the segment register
4542 isync ; Need to make sure this is done
4544 hbsrupt: andc r4,r4,r2 ; Clear the valid bit for this segment
4545 stwcx. r4,0,r7 ; Set the valid SR flags
4546 beqlr++ ; Stored ok, no interrupt, time to leave...
4548 lwarx r4,0,r7 ; Get and reserve the valid segment flags again
4549 b hbsrupt ; Try again...
4552 ; This routine invadates the entire pmap segment cache
4554 ; Translation is on, interrupts may or may not be enabled.
4558 .globl EXT(invalidateSegs)
4560 LEXT(invalidateSegs)
4562 la r10,pmapCCtl(r3) ; Point to the segment cache control
4563 eqv r2,r2,r2 ; Get all foxes
4565 isInv: lwarx r4,0,r10 ; Get the segment cache control value
4566 rlwimi r4,r2,0,0,15 ; Slam in all invalid bits
4567 rlwinm. r0,r4,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
4568 bne-- isInv0 ; Yes, try again...
4570 stwcx. r4,0,r10 ; Try to invalidate it
4571 bne-- isInv ; Someone else just stuffed it...
4575 isInv0: li r4,lgKillResv ; Get reservation kill zone
4576 stwcx. r4,0,r4 ; Kill reservation
4578 isInv1: lwz r4,pmapCCtl(r3) ; Get the segment cache control
4579 rlwinm. r0,r4,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
4580 bne-- isInv ; Nope...
4581 b isInv1 ; Still locked do it again...
4584 ; This routine switches segment registers between kernel and user.
4585 ; We have some assumptions and rules:
4586 ; We are in the exception vectors
4587 ; pf64Bitb is set up
4588 ; R3 contains the MSR we going to
4589 ; We can not use R4, R13, R20, R21, R29
4590 ; R13 is the savearea
4591 ; R29 has the per_proc
4593 ; We return R3 as 0 if we did not switch between kernel and user
4594 ; We also maintain and apply the user state key modifier used by VMM support;
4595 ; If we go to the kernel it is set to 0, otherwise it follows the bit
4600 .globl EXT(switchSegs)
4604 lwz r22,ppInvSeg(r29) ; Get the ppInvSeg (force invalidate) and ppCurSeg (user or kernel segments indicator)
4605 lwz r9,spcFlags(r29) ; Pick up the special user state flags
4606 rlwinm r2,r3,MSR_PR_BIT+1,31,31 ; Isolate the problem mode bit
4607 rlwinm r3,r3,MSR_RI_BIT+1,31,31 ; Isolate the recoverable interrupt bit
4608 lis r8,hi16(EXT(kernel_pmap_phys)) ; Assume kernel
4609 or r2,r2,r3 ; This will 1 if we will be using user segments
4610 li r3,0 ; Get a selection mask
4611 cmplw r2,r22 ; This will be EQ if same state and not ppInvSeg
4612 ori r8,r8,lo16(EXT(kernel_pmap_phys)) ; Assume kernel (bottom of address)
4613 sub r3,r3,r2 ; Form select mask - 0 if kernel, -1 if user
4614 la r19,ppUserPmap(r29) ; Point to the current user pmap
4616 ; The following line is an exercise of a generally unreadable but recompile-friendly programing practice
4617 rlwinm r30,r9,userProtKeybit+1+(63-sgcVSKeyUsr),sgcVSKeyUsr-32,sgcVSKeyUsr-32 ; Isolate the user state protection key
4619 andc r8,r8,r3 ; Zero kernel pmap ptr if user, untouched otherwise
4620 and r19,r19,r3 ; Zero user pmap ptr if kernel, untouched otherwise
4621 and r30,r30,r3 ; Clear key modifier if kernel, leave otherwise
4622 or r8,r8,r19 ; Get the pointer to the pmap we are using
4624 beqlr ; We are staying in the same mode, do not touch segs...
4626 lwz r28,0(r8) ; Get top half of pmap address
4627 lwz r10,4(r8) ; Get bottom half
4629 stw r2,ppInvSeg(r29) ; Clear request for invalidate and save ppCurSeg
4630 rlwinm r28,r28,0,1,0 ; Copy top to top
4631 stw r30,ppMapFlags(r29) ; Set the key modifier
4632 rlwimi r28,r10,0,0,31 ; Insert bottom
4634 la r10,pmapCCtl(r28) ; Point to the segment cache control
4635 la r9,pmapSegCache(r28) ; Point to the segment cache
4637 ssgLock: lwarx r15,0,r10 ; Get and reserve the segment cache control
4638 rlwinm. r0,r15,0,pmapCCtlLckb,pmapCCtlLckb ; Someone have the lock?
4639 ori r16,r15,lo16(pmapCCtlLck) ; Set lock bit
4640 bne-- ssgLock0 ; Yup, this is in use...
4642 stwcx. r16,0,r10 ; Try to set the lock
4643 bne-- ssgLock ; Did we get contention?
4645 not r11,r15 ; Invert the invalids to valids
4646 li r17,0 ; Set a mask for the SRs we are loading
4647 isync ; Make sure we are all caught up
4649 bf-- pf64Bitb,ssg32Enter ; If 32-bit, jump into it...
4652 slbia ; Trash all SLB entries (except for entry 0 that is)
4653 li r17,1 ; Get SLB index to load (skip slb 0)
4654 oris r0,r0,0x8000 ; Get set for a mask
4655 b ssg64Enter ; Start on a cache line...
4659 ssgLock0: li r15,lgKillResv ; Killing field
4660 stwcx. r15,0,r15 ; Kill reservation
4662 ssgLock1: lwz r15,pmapCCtl(r28) ; Get the segment cache controls
4663 rlwinm. r15,r15,0,pmapCCtlLckb,pmapCCtlLckb ; Someone have the lock?
4664 beq++ ssgLock ; Yup, this is in use...
4665 b ssgLock1 ; Nope, try again...
4667 ; This is the 32-bit address space switch code.
4668 ; We take a reservation on the segment cache and walk through.
4669 ; For each entry, we load the specified entries and remember which
4670 ; we did with a mask. Then, we figure out which segments should be
4671 ; invalid and then see which actually are. Then we load those with the
4672 ; defined invalid VSID.
4673 ; Afterwards, we unlock the segment cache.
4678 ssg32Enter: cntlzw r12,r11 ; Find the next slot in use
4679 cmplwi r12,pmapSegCacheUse ; See if we are done
4680 slwi r14,r12,4 ; Index to the cache slot
4681 lis r0,0x8000 ; Get set for a mask
4682 add r14,r14,r9 ; Point to the entry
4684 bge- ssg32Done ; All done...
4686 lwz r5,sgcESID+4(r14) ; Get the ESID part
4687 srw r2,r0,r12 ; Form a mask for the one we are loading
4688 lwz r7,sgcVSID+4(r14) ; And get the VSID bottom
4690 andc r11,r11,r2 ; Clear the bit
4691 lwz r6,sgcVSID(r14) ; And get the VSID top
4693 rlwinm r2,r5,4,28,31 ; Change the segment number to a number
4695 xor r7,r7,r30 ; Modify the key before we actually set it
4696 srw r0,r0,r2 ; Get a mask for the SR we are loading
4697 rlwinm r8,r7,19,1,3 ; Insert the keys and N bit
4698 or r17,r17,r0 ; Remember the segment
4699 rlwimi r8,r7,20,12,31 ; Insert 4:23 the VSID
4700 rlwimi r8,r6,20,8,11 ; Get the last nybble of the SR contents
4702 mtsrin r8,r5 ; Load the segment
4703 b ssg32Enter ; Go enter the next...
4707 ssg32Done: lwz r16,validSegs(r29) ; Get the valid SRs flags
4708 stw r15,pmapCCtl(r28) ; Unlock the segment cache controls
4710 lis r0,0x8000 ; Get set for a mask
4711 li r2,invalSpace ; Set the invalid address space VSID
4715 andc r16,r16,r17 ; Get list of SRs that were valid before but not now
4718 ssg32Inval: cntlzw r18,r16 ; Get the first one to invalidate
4719 cmplwi r18,16 ; Have we finished?
4720 srw r22,r0,r18 ; Get the mask bit
4721 rlwinm r23,r18,28,0,3 ; Get the segment register we need
4722 andc r16,r16,r22 ; Get rid of the guy we just did
4723 bge ssg32Really ; Yes, we are really done now...
4725 mtsrin r2,r23 ; Invalidate the SR
4726 b ssg32Inval ; Do the next...
4731 stw r17,validSegs(r29) ; Set the valid SR flags
4732 li r3,1 ; Set kernel/user transition
4736 ; This is the 64-bit address space switch code.
4737 ; First we blow away all of the SLB entries.
4739 ; loading the SLB. Afterwards, we release the cache lock
4741 ; Note that because we have to treat SLBE 0 specially, we do not ever use it...
4742 ; Its a performance thing...
4747 ssg64Enter: cntlzw r12,r11 ; Find the next slot in use
4748 cmplwi r12,pmapSegCacheUse ; See if we are done
4749 slwi r14,r12,4 ; Index to the cache slot
4750 srw r16,r0,r12 ; Form a mask for the one we are loading
4751 add r14,r14,r9 ; Point to the entry
4752 andc r11,r11,r16 ; Clear the bit
4753 bge-- ssg64Done ; All done...
4755 ld r5,sgcESID(r14) ; Get the ESID part
4756 ld r6,sgcVSID(r14) ; And get the VSID part
4757 oris r5,r5,0x0800 ; Turn on the valid bit
4758 or r5,r5,r17 ; Insert the SLB slot
4759 xor r6,r6,r30 ; Modify the key before we actually set it
4760 addi r17,r17,1 ; Bump to the next slot
4761 slbmte r6,r5 ; Make that SLB entry
4762 b ssg64Enter ; Go enter the next...
4766 ssg64Done: stw r15,pmapCCtl(r28) ; Unlock the segment cache controls
4768 eqv r16,r16,r16 ; Load up with all foxes
4769 subfic r17,r17,64 ; Get the number of 1 bits we need
4771 sld r16,r16,r17 ; Get a mask for the used SLB entries
4772 li r3,1 ; Set kernel/user transition
4773 std r16,validSegs(r29) ; Set the valid SR flags
4777 ; mapSetUp - this function sets initial state for all mapping functions.
4778 ; We turn off all translations (physical), disable interruptions, and
4779 ; enter 64-bit mode if applicable.
4781 ; We also return the original MSR in r11, the feature flags in R12,
4782 ; and CR6 set up so we can do easy branches for 64-bit
4786 .globl EXT(mapSetUp)
4790 lis r0,hi16(MASK(MSR_VEC)) ; Get the vector mask
4791 mfsprg r12,2 ; Get feature flags
4792 ori r0,r0,lo16(MASK(MSR_FP)) ; Get the FP as well
4793 mtcrf 0x04,r12 ; move pf64Bit and pfNoMSRirb to cr5 and 6
4794 mfmsr r11 ; Save the MSR
4795 mtcrf 0x02,r12 ; move pf64Bit and pfNoMSRirb to cr5 and 6
4796 andc r11,r11,r0 ; Clear VEC and FP for good
4797 ori r0,r0,lo16(MASK(MSR_EE)|MASK(MSR_DR)|MASK(MSR_IR)) ; Get rid of EE, IR, and DR
4798 li r2,1 ; Prepare for 64 bit
4799 andc r0,r11,r0 ; Clear the rest
4800 bt pfNoMSRirb,msuNoMSR ; No MSR...
4801 bt++ pf64Bitb,msuSF ; skip if 64-bit (only they take the hint)
4803 mtmsr r0 ; Translation and all off
4804 isync ; Toss prefetch
4809 msuSF: rldimi r0,r2,63,MSR_SF_BIT ; set SF bit (bit 0)
4810 mtmsrd r0 ; set 64-bit mode, turn off EE, DR, and IR
4816 msuNoMSR: mr r2,r3 ; Save R3 across call
4817 mr r3,r0 ; Get the new MSR value
4818 li r0,loadMSR ; Get the MSR setter SC
4820 mr r3,r2 ; Restore R3
4821 blr ; Go back all set up...
4825 ; Find the physent based on a physical page and try to lock it (but not too hard)
4826 ; Note that this table always has an entry that with a 0 table pointer at the end
4828 ; R3 contains ppnum on entry
4829 ; R3 is 0 if no entry was found
4830 ; R3 is physent if found
4831 ; cr0_eq is true if lock was obtained or there was no entry to lock
4832 ; cr0_eq is false of there was an entry and it was locked
4838 lis r9,hi16(EXT(pmap_mem_regions)) ; Point to the start of the region table
4839 mr r2,r3 ; Save our target
4840 ori r9,r9,lo16(EXT(pmap_mem_regions)) ; Point to the start of the region table
4842 mapFindPhz: lwz r3,mrPhysTab(r9) ; Get the actual table address
4843 lwz r5,mrStart(r9) ; Get start of table entry
4844 lwz r0,mrEnd(r9) ; Get end of table entry
4845 addi r9,r9,mrSize ; Point to the next slot
4846 cmplwi cr2,r3,0 ; Are we at the end of the table?
4847 cmplw r2,r5 ; See if we are in this table
4848 cmplw cr1,r2,r0 ; Check end also
4849 sub r4,r2,r5 ; Calculate index to physical entry
4850 beq-- cr2,mapFindNo ; Leave if we did not find an entry...
4851 cror cr0_lt,cr0_lt,cr1_gt ; Set CR0_LT if it is NOT this entry
4852 slwi r4,r4,3 ; Get offset to physical entry
4854 blt-- mapFindPhz ; Did not find it...
4856 add r3,r3,r4 ; Point right to the slot
4858 mapFindOv: lwz r2,0(r3) ; Get the lock contents right now
4859 rlwinm. r0,r2,0,0,0 ; Is it locked?
4860 bnelr-- ; Yes it is...
4862 lwarx r2,0,r3 ; Get the lock
4863 rlwinm. r0,r2,0,0,0 ; Is it locked?
4864 oris r0,r2,0x8000 ; Set the lock bit
4865 bne-- mapFindKl ; It is locked, go get rid of reservation and leave...
4866 stwcx. r0,0,r3 ; Try to stuff it back...
4867 bne-- mapFindOv ; Collision, try again...
4868 isync ; Clear any speculations
4871 mapFindKl: li r2,lgKillResv ; Killing field
4872 stwcx. r2,0,r2 ; Trash reservation...
4873 crclr cr0_eq ; Make sure we do not think we got the lock
4876 mapFindNo: crset cr0_eq ; Make sure that we set this
4877 li r3,0 ; Show that we did not find it
4880 ; pmapCacheLookup - This function will look up an entry in the pmap segment cache.
4882 ; How the pmap cache lookup works:
4884 ; We use a combination of three things: a mask of valid entries, a sub-tag, and the
4885 ; ESID (aka the "tag"). The mask indicates which of the cache slots actually contain
4886 ; an entry. The sub-tag is a 16 entry 4 bit array that contains the low order 4 bits
4887 ; of the ESID, bits 32:36 of the effective for 64-bit and 0:3 for 32-bit. The cache
4888 ; entry contains the full 36 bit ESID.
4890 ; The purpose of the sub-tag is to limit the number of searches necessary when looking
4891 ; for an existing cache entry. Because there are 16 slots in the cache, we could end up
4892 ; searching all 16 if an match is not found.
4894 ; Essentially, we will search only the slots that have a valid entry and whose sub-tag
4895 ; matches. More than likely, we will eliminate almost all of the searches.
4899 ; R4 = ESID high half
4900 ; R5 = ESID low half
4903 ; R3 = pmap cache slot if found, 0 if not
4904 ; R10 = pmapCCtl address
4905 ; R11 = pmapCCtl image
4906 ; pmapCCtl locked on exit
4912 la r10,pmapCCtl(r3) ; Point to the segment cache control
4915 lwarx r11,0,r10 ; Get the segment cache control value
4916 rlwinm. r0,r11,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
4917 ori r0,r11,lo16(pmapCCtlLck) ; Turn on the lock bit
4918 bne-- pmapCacheLookur ; Nope...
4919 stwcx. r0,0,r10 ; Try to take the lock
4920 bne-- pmapCacheLookuq ; Someone else just stuffed it, try again...
4922 isync ; Make sure we get reservation first
4923 lwz r9,pmapSCSubTag(r3) ; Get the high part of the sub-tag
4924 rlwimi r5,r5,28,4,7 ; Copy sub-tag just to right of itself (XX------)
4925 lwz r10,pmapSCSubTag+4(r3) ; And the bottom half
4926 rlwimi r5,r5,24,8,15 ; Copy doubled sub-tag to right of itself (XXXX----)
4927 lis r8,0x8888 ; Get some eights
4928 rlwimi r5,r5,16,16,31 ; Copy quadrupled sub-tags to the right
4929 ori r8,r8,0x8888 ; Fill the rest with eights
4931 eqv r10,r10,r5 ; Get 0xF where we hit in bottom half
4932 eqv r9,r9,r5 ; Get 0xF where we hit in top half
4934 rlwinm r2,r10,1,0,30 ; Shift over 1
4935 rlwinm r0,r9,1,0,30 ; Shift over 1
4936 and r2,r2,r10 ; AND the even/odd pair into the even
4937 and r0,r0,r9 ; AND the even/odd pair into the even
4938 rlwinm r10,r2,2,0,28 ; Shift over 2
4939 rlwinm r9,r0,2,0,28 ; Shift over 2
4940 and r10,r2,r10 ; AND the even of the ANDed pairs giving the AND of all 4 bits in 0, 4, ...
4941 and r9,r0,r9 ; AND the even of the ANDed pairs giving the AND of all 4 bits in 0, 4, ...
4943 and r10,r10,r8 ; Clear out extras
4944 and r9,r9,r8 ; Clear out extras
4946 rlwinm r0,r10,3,1,28 ; Slide adjacent next to each other
4947 rlwinm r2,r9,3,1,28 ; Slide adjacent next to each other
4948 or r10,r0,r10 ; Merge them
4949 or r9,r2,r9 ; Merge them
4950 rlwinm r0,r10,6,2,26 ; Slide adjacent pairs next to each other
4951 rlwinm r2,r9,6,2,26 ; Slide adjacent pairs next to each other
4952 or r10,r0,r10 ; Merge them
4953 or r9,r2,r9 ; Merge them
4954 rlwimi r10,r10,12,4,7 ; Stick in the low-order adjacent quad
4955 rlwimi r9,r9,12,4,7 ; Stick in the low-order adjacent quad
4956 not r6,r11 ; Turn invalid into valid
4957 rlwimi r9,r10,24,8,15 ; Merge in the adjacent octs giving a hit mask
4959 la r10,pmapSegCache(r3) ; Point at the cache slots
4960 and. r6,r9,r6 ; Get mask of valid and hit
4962 li r3,0 ; Assume not found
4963 oris r0,r0,0x8000 ; Start a mask
4964 beqlr++ ; Leave, should usually be no hits...
4966 pclNextEnt: cntlzw r5,r6 ; Find an in use one
4967 cmplwi cr1,r5,pmapSegCacheUse ; Did we find one?
4968 rlwinm r7,r5,4,0,27 ; Index to the cache entry
4969 srw r2,r0,r5 ; Get validity mask bit
4970 add r7,r7,r10 ; Point to the cache slot
4971 andc r6,r6,r2 ; Clear the validity bit we just tried
4972 bgelr-- cr1 ; Leave if there are no more to check...
4974 lwz r5,sgcESID(r7) ; Get the top half
4976 cmplw r5,r4 ; Only need to check top because sub-tag is the entire other half
4978 bne++ pclNextEnt ; Nope, try again...
4980 mr r3,r7 ; Point to the slot
4986 li r11,lgKillResv ; The killing spot
4987 stwcx. r11,0,r11 ; Kill the reservation
4990 lwz r11,pmapCCtl(r3) ; Get the segment cache control
4991 rlwinm. r0,r11,0,pmapCCtlLckb,pmapCCtlLckb ; Is it already locked?
4992 beq++ pmapCacheLookup ; Nope...
4993 b pmapCacheLookus ; Yup, keep waiting...
4999 ; This routine, given a mapping, will find and lock the PTEG
5000 ; If mpPte does not point to a PTE (checked before and after lock), it will unlock the
5001 ; PTEG and return. In this case we will have undefined in R4
5002 ; and the low 12 bits of mpVAddr valid in R5. R3 will contain 0.
5004 ; If the mapping is still valid, we will invalidate the PTE and merge
5005 ; the RC bits into the physent and also save them into the mapping.
5007 ; We then return with R3 pointing to the PTE slot, R4 is the
5008 ; top of the PTE and R5 is the bottom. R6 contains the PCA.
5009 ; R7 points to the PCA entry.
5011 ; Note that we should NEVER be called on a block or special mapping.
5012 ; We could do many bad things.
5018 lwz r0,mpPte(r31) ; Grab the PTE offset
5019 mfsdr1 r7 ; Get the pointer to the hash table
5020 lwz r5,mpVAddr+4(r31) ; Grab the virtual address
5021 rlwinm r10,r7,0,0,15 ; Clean up the hash table base
5022 andi. r3,r0,mpHValid ; Is there a possible PTE?
5023 srwi r7,r0,4 ; Convert to PCA units
5024 rlwinm r7,r7,0,0,29 ; Clean up PCA offset
5025 mflr r2 ; Save the return
5026 subfic r7,r7,-4 ; Convert to -4 based negative index
5027 add r7,r10,r7 ; Point to the PCA directly
5028 beqlr-- ; There was no PTE to start with...
5030 bl mapLockPteg ; Lock the PTEG
5032 lwz r0,mpPte(r31) ; Grab the PTE offset
5033 mtlr r2 ; Restore the LR
5034 andi. r3,r0,mpHValid ; Is there a possible PTE?
5035 beq- mIPUnlock ; There is no PTE, someone took it so just unlock and leave...
5037 rlwinm r3,r0,0,0,30 ; Clear the valid bit
5038 add r3,r3,r10 ; Point to actual PTE
5039 lwz r4,0(r3) ; Get the top of the PTE
5041 li r8,tlbieLock ; Get the TLBIE lock
5042 rlwinm r0,r4,0,1,31 ; Clear the valid bit
5043 stw r0,0(r3) ; Invalidate the PTE
5045 sync ; Make sure everyone sees the invalidate
5047 mITLBIE32: lwarx r0,0,r8 ; Get the TLBIE lock
5048 mfsprg r2,2 ; Get feature flags
5049 mr. r0,r0 ; Is it locked?
5050 li r0,1 ; Get our lock word
5051 bne- mITLBIE32 ; It is locked, go wait...
5053 stwcx. r0,0,r8 ; Try to get it
5054 bne- mITLBIE32 ; We was beat...
5056 rlwinm. r0,r2,0,pfSMPcapb,pfSMPcapb ; Can this be an MP box?
5057 li r0,0 ; Lock clear value
5059 tlbie r5 ; Invalidate it everywhere
5061 stw r0,tlbieLock(0) ; Clear the tlbie lock
5063 beq- mINoTS32 ; Can not have MP on this machine...
5065 eieio ; Make sure that the tlbie happens first
5066 tlbsync ; Wait for everyone to catch up
5067 sync ; Make sure of it all
5069 mINoTS32: lwz r5,4(r3) ; Get the real part
5070 srwi r10,r5,12 ; Change physical address to a ppnum
5072 mINmerge: lbz r11,mpFlags+1(r31) ; Get the offset to the physical entry table
5073 lwz r0,mpVAddr+4(r31) ; Get the flags part of the field
5074 lis r8,hi16(EXT(pmap_mem_regions)) ; Get the top of the region table
5075 ori r8,r8,lo16(EXT(pmap_mem_regions)) ; Get the bottom of the region table
5076 rlwinm r11,r11,2,0,29 ; Change index into byte offset
5077 add r11,r11,r8 ; Point to the bank table
5078 lwz r2,mrPhysTab(r11) ; Get the physical table bank pointer
5079 lwz r11,mrStart(r11) ; Get the start of bank
5080 rlwimi r0,r5,0,mpRb-32,mpCb-32 ; Copy in the RC
5081 addi r2,r2,4 ; Offset to last half of field
5082 stw r0,mpVAddr+4(r31) ; Set the new RC into the field
5083 sub r11,r10,r11 ; Get the index into the table
5084 rlwinm r11,r11,3,0,28 ; Get offset to the physent
5087 mImrgRC: lwarx r10,r11,r2 ; Get the master RC
5088 rlwinm r0,r5,27,ppRb-32,ppCb-32 ; Position the new RC
5089 or r0,r0,r10 ; Merge in the new RC
5090 stwcx. r0,r11,r2 ; Try to stick it back
5091 bne-- mImrgRC ; Try again if we collided...
5093 blr ; Leave with the PCA still locked up...
5095 mIPUnlock: eieio ; Make sure all updates come first
5097 stw r6,0(r7) ; Unlock
5106 lwz r0,mpPte(r31) ; Grab the PTE offset
5107 ld r5,mpVAddr(r31) ; Grab the virtual address
5108 mfsdr1 r7 ; Get the pointer to the hash table
5109 rldicr r10,r7,0,45 ; Clean up the hash table base
5110 andi. r3,r0,mpHValid ; Is there a possible PTE?
5111 srdi r7,r0,5 ; Convert to PCA units
5112 rldicr r7,r7,0,61 ; Clean up PCA
5113 subfic r7,r7,-4 ; Convert to -4 based negative index
5114 mflr r2 ; Save the return
5115 add r7,r10,r7 ; Point to the PCA directly
5116 beqlr-- ; There was no PTE to start with...
5118 bl mapLockPteg ; Lock the PTEG
5120 lwz r0,mpPte(r31) ; Grab the PTE offset again
5121 mtlr r2 ; Restore the LR
5122 andi. r3,r0,mpHValid ; Is there a possible PTE?
5123 beq-- mIPUnlock ; There is no PTE, someone took it so just unlock and leave...
5125 rlwinm r3,r0,0,0,30 ; Clear the valid bit
5126 add r3,r3,r10 ; Point to the actual PTE
5127 ld r4,0(r3) ; Get the top of the PTE
5129 li r8,tlbieLock ; Get the TLBIE lock
5130 rldicr r0,r4,0,62 ; Clear the valid bit
5131 std r0,0(r3) ; Invalidate the PTE
5133 rldicr r2,r4,16,35 ; Shift the AVPN over to match VPN
5134 sync ; Make sure everyone sees the invalidate
5135 rldimi r2,r5,0,36 ; Cram in the page portion of the EA
5137 mITLBIE64: lwarx r0,0,r8 ; Get the TLBIE lock
5138 mr. r0,r0 ; Is it locked?
5139 li r0,1 ; Get our lock word
5140 bne-- mITLBIE64a ; It is locked, toss reservation and wait...
5142 stwcx. r0,0,r8 ; Try to get it
5143 bne-- mITLBIE64 ; We was beat...
5145 rldicl r2,r2,0,16 ; Clear bits 0:15 because we are under orders
5147 li r0,0 ; Lock clear value
5149 tlbie r2 ; Invalidate it everywhere
5151 stw r0,tlbieLock(0) ; Clear the tlbie lock
5153 eieio ; Make sure that the tlbie happens first
5154 tlbsync ; Wait for everyone to catch up
5156 ptesync ; Wait for quiet again
5158 mINoTS64: sync ; Make sure of it all
5160 ld r5,8(r3) ; Get the real part
5161 srdi r10,r5,12 ; Change physical address to a ppnum
5162 b mINmerge ; Join the common 32-64-bit code...
5164 mITLBIE64a: li r5,lgKillResv ; Killing field
5165 stwcx. r5,0,r5 ; Kill reservation
5167 mITLBIE64b: lwz r0,0(r8) ; Get the TLBIE lock
5168 mr. r0,r0 ; Is it locked?
5169 beq++ mITLBIE64 ; Nope, try again...
5170 b mITLBIE64b ; Yup, wait for it...
5173 ; mapLockPteg - Locks a PTEG
5174 ; R7 points to PCA entry
5175 ; R6 contains PCA on return
5182 lwarx r6,0,r7 ; Pick up the PCA
5183 rlwinm. r0,r6,0,PCAlockb,PCAlockb ; Is the PTEG locked?
5184 ori r0,r6,PCAlock ; Set the lock bit
5185 bne-- mLSkill ; It is locked...
5187 stwcx. r0,0,r7 ; Try to lock the PTEG
5188 bne-- mapLockPteg ; We collided...
5190 isync ; Nostradamus lied
5193 mLSkill: li r6,lgKillResv ; Get killing field
5194 stwcx. r6,0,r6 ; Kill it
5197 lwz r6,0(r7) ; Pick up the PCA
5198 rlwinm. r0,r6,0,PCAlockb,PCAlockb ; Is the PTEG locked?
5199 beq++ mapLockPteg ; Nope, try again...
5200 b mapLockPteh ; Yes, wait for it...
5204 ; The mapSelSlot function selects a PTEG slot to use. As input, it expects R6
5205 ; to contain the PCA. When it returns, R3 contains 0 if an unoccupied slot was
5206 ; selected, 1 if it stole a non-block PTE, or 2 if it stole a block mapped PTE.
5207 ; R4 returns the slot index.
5209 ; CR7 also indicates that we have a block mapping
5211 ; The PTEG allocation controls are a bit map of the state of the PTEG.
5212 ; PCAfree indicates that the PTE slot is empty.
5213 ; PCAauto means that it comes from an autogen area. These
5214 ; guys do not keep track of reference and change and are actually "wired".
5215 ; They are easy to maintain. PCAsteal
5216 ; is a sliding position mask used to "randomize" PTE slot stealing. All 4 of these
5217 ; fields fit in a single word and are loaded and stored under control of the
5218 ; PTEG control area lock (PCAlock).
5220 ; Note that PCAauto does not contribute to the steal calculations at all. Originally
5221 ; it did, autogens were second in priority. This can result in a pathalogical
5222 ; case where an instruction can not make forward progress, or one PTE slot
5225 ; Note that the PCA must be locked when we get here.
5227 ; Physically, the fields are arranged:
5234 ; At entry, R6 contains new unlocked PCA image (real PCA is locked and untouched)
5239 ; R3 = 1 - steal regular
5240 ; R3 = 2 - steal autogen
5241 ; R4 contains slot number
5242 ; R6 contains updated PCA image
5247 mapSelSlot: lis r10,0 ; Clear autogen mask
5248 li r9,0 ; Start a mask
5249 beq cr7,mSSnotblk ; Skip if this is not a block mapping
5250 ori r10,r10,lo16(0xFFFF) ; Make sure we mark a block mapping (autogen)
5252 mSSnotblk: rlwinm r11,r6,16,24,31 ; Isolate just the steal mask
5253 oris r9,r9,0x8000 ; Get a mask
5254 cntlzw r4,r6 ; Find a slot or steal one
5255 ori r9,r9,lo16(0x8000) ; Insure that we have 0x80008000
5256 rlwinm r4,r4,0,29,31 ; Isolate bit position
5257 rlwimi r11,r11,8,16,23 ; Get set to march a 1 back into top of 8 bit rotate
5258 srw r2,r9,r4 ; Get mask to isolate selected inuse and autogen flags
5259 srwi r11,r11,1 ; Slide steal mask right
5260 and r8,r6,r2 ; Isolate the old in use and autogen bits
5261 andc r6,r6,r2 ; Allocate the slot and also clear autogen flag
5262 addi r0,r8,0x7F00 ; Push autogen flag to bit 16
5263 and r2,r2,r10 ; Keep the autogen part if autogen
5264 addis r8,r8,0xFF00 ; Push in use to bit 0 and invert
5265 or r6,r6,r2 ; Add in the new autogen bit
5266 rlwinm r0,r0,17,31,31 ; Get a 1 if the old was autogenned (always 0 if not in use)
5267 rlwinm r8,r8,1,31,31 ; Isolate old in use
5268 rlwimi r6,r11,16,8,15 ; Stick the new steal slot in
5270 add r3,r0,r8 ; Get 0 if no steal, 1 if steal normal, 2 if steal autogen
5274 ; Shared/Exclusive locks
5276 ; A shared/exclusive lock allows multiple shares of a lock to be taken
5277 ; but only one exclusive. A shared lock can be "promoted" to exclusive
5278 ; when it is the only share. If there are multiple sharers, the lock
5279 ; must be "converted". A promotion drops the share and gains exclusive as
5280 ; an atomic operation. If anyone else has a share, the operation fails.
5281 ; A conversion first drops the share and then takes an exclusive lock.
5283 ; We will want to add a timeout to this eventually.
5285 ; R3 is set to 0 for success, non-zero for failure
5289 ; Convert a share into an exclusive
5296 lis r0,0x8000 ; Get the locked lock image
5298 mflr r0 ; (TEST/DEBUG)
5299 oris r0,r0,0x8000 ; (TEST/DEBUG)
5302 sxlkCTry: lwarx r2,0,r3 ; Get the lock word
5303 cmplwi r2,1 ; Does it just have our share?
5304 subi r2,r2,1 ; Drop our share in case we do not get it
5305 bne-- sxlkCnotfree ; No, we need to unlock...
5306 stwcx. r0,0,r3 ; Try to take it exclusively
5307 bne-- sxlkCTry ; Collision, try again...
5314 stwcx. r2,0,r3 ; Try to drop our share...
5315 bne-- sxlkCTry ; Try again if we collided...
5316 b sxlkExclusive ; Go take it exclusively...
5319 ; Promote shared to exclusive
5325 lis r0,0x8000 ; Get the locked lock image
5327 mflr r0 ; (TEST/DEBUG)
5328 oris r0,r0,0x8000 ; (TEST/DEBUG)
5331 sxlkPTry: lwarx r2,0,r3 ; Get the lock word
5332 cmplwi r2,1 ; Does it just have our share?
5333 bne-- sxlkPkill ; No, just fail (R3 is non-zero)...
5334 stwcx. r0,0,r3 ; Try to take it exclusively
5335 bne-- sxlkPTry ; Collision, try again...
5341 sxlkPkill: li r2,lgKillResv ; Point to killing field
5342 stwcx. r2,0,r2 ; Kill reservation
5348 ; Take lock exclusivily
5354 lis r0,0x8000 ; Get the locked lock image
5356 mflr r0 ; (TEST/DEBUG)
5357 oris r0,r0,0x8000 ; (TEST/DEBUG)
5360 sxlkXTry: lwarx r2,0,r3 ; Get the lock word
5361 mr. r2,r2 ; Is it locked?
5362 bne-- sxlkXWait ; Yes...
5363 stwcx. r0,0,r3 ; Try to take it
5364 bne-- sxlkXTry ; Collision, try again...
5366 isync ; Toss anything younger than us
5372 sxlkXWait: li r2,lgKillResv ; Point to killing field
5373 stwcx. r2,0,r2 ; Kill reservation
5375 sxlkXWaiu: lwz r2,0(r3) ; Get the lock again
5376 mr. r2,r2 ; Is it free yet?
5377 beq++ sxlkXTry ; Yup...
5378 b sxlkXWaiu ; Hang around a bit more...
5381 ; Take a share of the lock
5386 sxlkShared: lwarx r2,0,r3 ; Get the lock word
5387 rlwinm. r0,r2,0,0,0 ; Is it locked exclusively?
5388 addi r2,r2,1 ; Up the share count
5389 bne-- sxlkSWait ; Yes...
5390 stwcx. r2,0,r3 ; Try to take it
5391 bne-- sxlkShared ; Collision, try again...
5393 isync ; Toss anything younger than us
5399 sxlkSWait: li r2,lgKillResv ; Point to killing field
5400 stwcx. r2,0,r2 ; Kill reservation
5402 sxlkSWaiu: lwz r2,0(r3) ; Get the lock again
5403 rlwinm. r0,r2,0,0,0 ; Is it locked exclusively?
5404 beq++ sxlkShared ; Nope...
5405 b sxlkSWaiu ; Hang around a bit more...
5408 ; Unlock either exclusive or shared.
5413 sxlkUnlock: eieio ; Make sure we order our stores out
5415 sxlkUnTry: lwarx r2,0,r3 ; Get the lock
5416 rlwinm. r0,r2,0,0,0 ; Do we hold it exclusively?
5417 subi r2,r2,1 ; Remove our share if we have one
5418 li r0,0 ; Clear this
5419 bne-- sxlkUExclu ; We hold exclusive...
5421 stwcx. r2,0,r3 ; Try to lose our share
5422 bne-- sxlkUnTry ; Collision...
5425 sxlkUExclu: stwcx. r0,0,r3 ; Unlock and release reservation
5426 beqlr++ ; Leave if ok...
5427 b sxlkUnTry ; Could not store, try over...
5431 .globl EXT(fillPage)
5435 mfsprg r0,2 ; Get feature flags
5436 mtcrf 0x02,r0 ; move pf64Bit to cr
5438 rlwinm r4,r4,0,1,0 ; Copy fill to top of 64-bit register
5439 lis r2,0x0200 ; Get vec
5441 ori r2,r2,0x2000 ; Get FP
5445 andc r5,r5,r2 ; Clear out permanent turn-offs
5447 ori r2,r2,0x8030 ; Clear IR, DR and EE
5449 andc r0,r5,r2 ; Kill them
5452 bt++ pf64Bitb,fpSF1 ; skip if 64-bit (only they take the hint)
5454 slwi r3,r3,12 ; Make into a physical address
5455 mtmsr r2 ; Interrupts and translation off
5458 li r2,4096/32 ; Get number of cache lines
5460 fp32again: dcbz 0,r3 ; Clear
5461 addic. r2,r2,-1 ; Count down
5465 stw r8,12(r3) ; Fill
5466 stw r9,16(r3) ; Fill
5467 stw r10,20(r3) ; Fill
5468 stw r11,24(r3) ; Fill
5469 stw r12,28(r3) ; Fill
5470 addi r3,r3,32 ; Point next
5471 bgt+ fp32again ; Keep going
5473 mtmsr r5 ; Restore all
5480 sldi r2,r2,63 ; Get 64-bit bit
5481 or r0,r0,r2 ; Turn on 64-bit
5482 sldi r3,r3,12 ; Make into a physical address
5484 mtmsrd r0 ; Interrupts and translation off
5487 li r2,4096/128 ; Get number of cache lines
5489 fp64again: dcbz128 0,r3 ; Clear
5490 addic. r2,r2,-1 ; Count down
5493 std r7,16(r3) ; Fill
5494 std r8,24(r3) ; Fill
5495 std r9,32(r3) ; Fill
5496 std r10,40(r3) ; Fill
5497 std r11,48(r3) ; Fill
5498 std r12,56(r3) ; Fill
5499 std r4,64+0(r3) ; Fill
5500 std r6,64+8(r3) ; Fill
5501 std r7,64+16(r3) ; Fill
5502 std r8,64+24(r3) ; Fill
5503 std r9,64+32(r3) ; Fill
5504 std r10,64+40(r3) ; Fill
5505 std r11,64+48(r3) ; Fill
5506 std r12,64+56(r3) ; Fill
5507 addi r3,r3,128 ; Point next
5508 bgt+ fp64again ; Keep going
5510 mtmsrd r5 ; Restore all
5520 lis r11,hi16(EXT(mapdebug))
5521 ori r11,r11,lo16(EXT(mapdebug))
5526 mLxx: rlwinm r0,r12,0,MSR_DR_BIT+1,MSR_DR_BIT-1
5541 .globl EXT(checkBogus)
5546 blr ; No-op normally
projects / apple / xnu.git / blob