2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
27 #include <db_machine_commands.h>
30 #include <mach_debug.h>
32 #include <ppc/proc_reg.h>
33 #include <ppc/exception.h>
34 #include <ppc/Performance.h>
35 #include <ppc/exception.h>
36 #include <ppc/savearea.h>
37 #include <mach/ppc/vm_param.h>
41 /* Register usage conventions in this code:
45 * cr6 = feature flags (ie, pf64Bit)
47 * Because much of this code deals with physical addresses,
48 * there are parallel paths for 32- and 64-bit machines.
53 * *****************************
54 * * s a v e _ s n a p s h o t *
55 * *****************************
57 * void save_snapshot();
59 * Link the current free list & processor local list on an independent list.
62 .globl EXT(save_snapshot)
65 mflr r9 ; get return address
66 bl saveSetup ; turn translation off, 64-bit on, load many regs
67 bf-- pf64Bitb,save_snapshot32 ; skip if 32-bit processor
69 ; Handle 64-bit processor.
73 ld r8,next_savearea(r10) ; Start with the current savearea
74 std r8,SVsavefreesnapshot(0) ; Make it the restore list anchor
75 ld r5,SVfree(0) ; Get free save area list anchor
77 save_snapshot64nextfree:
79 std r7,savemisc1(r8) ; Link this one
80 ld r5,SAVprev(r7) ; Get the next
83 bne save_snapshot64nextfree
85 lwz r6,SVinuse(0) ; Get inuse count
86 ld r5,lclfree(r10) ; Get the local savearea list
87 subi r6,r6,1 ; Count the first as free
89 save_snapshot64nextlocalfree:
90 subi r6,r6,1 ; Count as free
92 std r7,savemisc1(r8) ; Link this one
93 ld r5,SAVprev(r7) ; Get the next
96 bne save_snapshot64nextlocalfree
98 std r5,savemisc1(r8) ; End the list
99 stw r6,SVsaveinusesnapshot(0) ; Save the new number of inuse saveareas
101 mtlr r9 ; Restore the return
102 b saveRestore64 ; Restore interrupts and translation
104 ; Handle 32-bit processor.
107 lwz r8,next_savearea+4(r10) ; Start with the current savearea
108 stw r8,SVsavefreesnapshot+4(0) ; Make it the restore list anchor
109 lwz r5,SVfree+4(0) ; Get free save area list anchor
111 save_snapshot32nextfree:
113 stw r7,savemisc1+4(r8) ; Link this one
114 lwz r5,SAVprev+4(r7) ; Get the next
117 bne save_snapshot32nextfree
119 lwz r6,SVinuse(0) ; Get inuse count
120 lwz r5,lclfree+4(r10) ; Get the local savearea list
121 subi r6,r6,1 ; Count the first as free
123 save_snapshot32nextlocalfree:
124 subi r6,r6,1 ; Count as free
126 stw r7,savemisc1+4(r8) ; Link this one
127 lwz r5,SAVprev+4(r7) ; Get the next
130 bne save_snapshot32nextlocalfree
132 stw r5,savemisc1+4(r8) ; End the list
133 stw r6,SVsaveinusesnapshot(0) ; Save the new number of inuse saveareas
135 mtlr r9 ; Restore the return
136 b saveRestore32 ; Restore interrupts and translation
139 * *********************************************
140 * * s a v e _ s n a p s h o t _ r e s t o r e *
141 * *********************************************
143 * void save_snapshot_restore();
145 * Restore the free list from the snapshot list, and reset the processors next savearea.
148 .globl EXT(save_snapshot_restore)
150 LEXT(save_snapshot_restore)
151 mflr r9 ; get return address
152 bl saveSetup ; turn translation off, 64-bit on, load many regs
153 bf-- pf64Bitb,save_snapshot_restore32 ; skip if 32-bit processor
155 ; Handle 64-bit processor.
157 save_snapshot_restore64:
158 lwz r7,SVsaveinusesnapshot(0)
159 stw r7,SVinuse(0) ; Set the new inuse count
162 stw r6,lclfreecnt(r10) ; None local now
163 std r6,lclfree(r10) ; None local now
165 ld r8,SVsavefreesnapshot(0) ; Get the restore list anchor
166 std r8,SVfree(0) ; Make it the free list anchor
167 li r5,SAVempty ; Get marker for free savearea
169 save_snapshot_restore64nextfree:
170 addi r6,r6,1 ; Count as free
171 stb r5,SAVflags+2(r8) ; Mark savearea free
172 ld r7,savemisc1(r8) ; Get the next
173 std r7,SAVprev(r8) ; Set the next in free list
175 bne save_snapshot_restore64nextfree
177 stw r6,SVfreecnt(0) ; Set the new free count
180 std r3,next_savearea(r10) ; Get the next savearea
182 mtlr r9 ; Restore the return
183 b saveRestore64 ; Restore interrupts and translation
185 ; Handle 32-bit processor.
187 save_snapshot_restore32:
188 lwz r7,SVsaveinusesnapshot(0)
189 stw r7,SVinuse(0) ; Set the new inuse count
192 stw r6,lclfreecnt(r10) ; None local now
193 stw r6,lclfree+4(r10) ; None local now
195 lwz r8,SVsavefreesnapshot+4(0) ; Get the restore list anchor
196 stw r8,SVfree+4(0) ; Make it the free list anchor
197 li r5,SAVempty ; Get marker for free savearea
199 save_snapshot_restore32nextfree:
200 addi r6,r6,1 ; Count as free
201 stb r5,SAVflags+2(r8) ; Mark savearea free
202 lwz r7,savemisc1+4(r8) ; Get the next
203 stw r7,SAVprev+4(r8) ; Set the next in free list
205 bne save_snapshot_restore32nextfree
207 stw r6,SVfreecnt(0) ; Set the new free count
210 stw r3,next_savearea+4(r10) ; Get the next savearea
212 mtlr r9 ; Restore the return
213 b saveRestore32 ; Restore interrupts and translation
216 * ***********************
217 * * s a v e _ q u e u e *
218 * ***********************
220 * void save_queue(ppnum_t pagenum);
222 * This routine will add a savearea block to the free list.
223 * We also queue the block to the free pool list. This is a
224 * circular double linked list. Because this block has no free entries,
225 * it gets queued to the end of the list
228 .globl EXT(save_queue)
231 mflr r9 ; get return address
232 mr r8,r3 ; move pagenum out of the way
233 bl saveSetup ; turn translation off, 64-bit on, load many regs
234 bf-- pf64Bitb,saveQueue32 ; skip if 32-bit processor
236 sldi r2,r8,12 ; r2 <-- phys address of page
237 li r8,sac_cnt ; Get the number of saveareas per page
238 mr r4,r2 ; Point to start of chain
239 li r0,SAVempty ; Get empty marker
242 addic. r8,r8,-1 ; Keep track of how many we did
243 stb r0,SAVflags+2(r4) ; Set empty
244 addi r7,r4,SAVsize ; Point to the next slot
245 ble- saveQueue64b ; We are done with the chain
246 std r7,SAVprev(r4) ; Set this chain
247 mr r4,r7 ; Step to the next
248 b saveQueue64a ; Fill the whole block...
251 bl savelock ; Go lock the save anchor
253 ld r7,SVfree(0) ; Get the free save area list anchor
254 lwz r6,SVfreecnt(0) ; Get the number of free saveareas
256 std r2,SVfree(0) ; Queue in the new one
257 addi r6,r6,sac_cnt ; Count the ones we are linking in
258 std r7,SAVprev(r4) ; Queue the old first one off of us
259 stw r6,SVfreecnt(0) ; Save the new count
262 ; Handle 32-bit processor.
265 slwi r2,r8,12 ; r2 <-- phys address of page
266 li r8,sac_cnt ; Get the number of saveareas per page
267 mr r4,r2 ; Point to start of chain
268 li r0,SAVempty ; Get empty marker
271 addic. r8,r8,-1 ; Keep track of how many we did
272 stb r0,SAVflags+2(r4) ; Set empty
273 addi r7,r4,SAVsize ; Point to the next slot
274 ble- saveQueue32b ; We are done with the chain
275 stw r7,SAVprev+4(r4) ; Set this chain
276 mr r4,r7 ; Step to the next
277 b saveQueue32a ; Fill the whole block...
280 bl savelock ; Go lock the save anchor
282 lwz r7,SVfree+4(0) ; Get the free save area list anchor
283 lwz r6,SVfreecnt(0) ; Get the number of free saveareas
285 stw r2,SVfree+4(0) ; Queue in the new one
286 addi r6,r6,sac_cnt ; Count the ones we are linking in
287 stw r7,SAVprev+4(r4) ; Queue the old first one off of us
288 stw r6,SVfreecnt(0) ; Save the new count
290 saveQueueExit: ; join here from 64-bit path
291 bl saveunlock ; Unlock the list and set the adjust count
292 mtlr r9 ; Restore the return
295 mfsprg r2,1 ; (TEST/DEBUG)
296 mr. r2,r2 ; (TEST/DEBUG)
297 beq-- saveRestore ; (TEST/DEBUG)
298 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
299 li r2,0x2201 ; (TEST/DEBUG)
300 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
303 b saveRestore ; Restore interrupts and translation
306 * *****************************
307 * * s a v e _ g e t _ i n i t *
308 * *****************************
310 * addr64_t save_get_init(void);
312 * Note that save_get_init is used in initial processor startup only. It
313 * is used because translation is on, but no tables exist yet and we have
314 * no V=R BAT registers that cover the entire physical memory.
317 .globl EXT(save_get_init)
320 mflr r9 ; get return address
321 bl saveSetup ; turn translation off, 64-bit on, load many regs
322 bfl-- pf64Bitb,saveGet32 ; Get r3 <- savearea, r5 <- page address (with SAC)
323 btl++ pf64Bitb,saveGet64 ; get one on a 64-bit machine
324 bl saveRestore ; restore translation etc
327 ; unpack the physaddr in r3 into a long long in (r3,r4)
329 mr r4,r3 ; copy low word of phys address to r4
330 li r3,0 ; assume upper word was 0
331 bflr-- pf64Bitb ; if 32-bit processor, return
332 srdi r3,r4,32 ; unpack reg64_t to addr64_t on 64-bit machine
338 * *******************
339 * * s a v e _ g e t *
340 * *******************
342 * savearea *save_get(void);
344 * Allocate a savearea, returning a virtual address. NOTE: we must preserve
345 * r0, r2, and r12. Our callers in cswtch.s depend on this.
351 mflr r9 ; get return address
352 mr r5,r0 ; copy regs before saveSetup nails them
353 bl saveSetup ; turn translation off, 64-bit on, load many regs
354 bf-- pf64Bitb,svgt1 ; skip if 32-bit processor
356 std r5,tempr0(r10) ; save r0 in per-proc across call to saveGet64
357 std r2,tempr2(r10) ; and r2
358 std r12,tempr4(r10) ; and r12
359 bl saveGet64 ; get r3 <- savearea, r5 <- page address (with SAC)
360 ld r0,tempr0(r10) ; restore callers regs
365 svgt1: ; handle 32-bit processor
366 stw r5,tempr0+4(r10) ; save r0 in per-proc across call to saveGet32
367 stw r2,tempr2+4(r10) ; and r2
368 stw r12,tempr4+4(r10) ; and r12
369 bl saveGet32 ; get r3 <- savearea, r5 <- page address (with SAC)
370 lwz r0,tempr0+4(r10) ; restore callers regs
372 lwz r12,tempr4+4(r10)
375 lwz r5,SACvrswap+4(r5) ; Get the virtual to real translation (only need low word)
376 mtlr r9 ; restore return address
377 xor r3,r3,r5 ; convert physaddr to virtual
378 rlwinm r3,r3,0,0,31 ; 0 upper word if a 64-bit machine
381 mr r6,r0 ; (TEST/DEBUG)
382 mr r7,r2 ; (TEST/DEBUG)
383 mfsprg r2,1 ; (TEST/DEBUG)
384 mr. r2,r2 ; (TEST/DEBUG)
385 beq-- svgDBBypass ; (TEST/DEBUG)
386 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
387 li r2,0x2203 ; (TEST/DEBUG)
388 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
390 svgDBBypass: ; (TEST/DEBUG)
391 mr r0,r6 ; (TEST/DEBUG)
392 mr r2,r7 ; (TEST/DEBUG)
394 b saveRestore ; restore MSR and return to our caller
398 * ***********************************
399 * * s a v e _ g e t _ p h y s _ 3 2 *
400 * ***********************************
402 * reg64_t save_get_phys(void);
404 * This is the entry normally called from lowmem_vectors.s with
405 * translation and interrupts already off.
409 .globl EXT(save_get_phys_32)
411 LEXT(save_get_phys_32)
412 mfsprg r10,0 ; get the per-proc ptr
413 b saveGet32 ; Get r3 <- savearea, r5 <- page address (with SAC)
417 * ***********************************
418 * * s a v e _ g e t _ p h y s _ 6 4 *
419 * ***********************************
421 * reg64_t save_get_phys_64(void);
423 * This is the entry normally called from lowmem_vectors.s with
424 * translation and interrupts already off, and in 64-bit mode.
428 .globl EXT(save_get_phys_64)
430 LEXT(save_get_phys_64)
431 mfsprg r10,0 ; get the per-proc ptr
432 b saveGet64 ; Get r3 <- savearea, r5 <- page address (with SAC)
436 * *********************
437 * * s a v e G e t 6 4 *
438 * *********************
440 * This is the internal routine to allocate a savearea on a 64-bit processor.
441 * Note that we must not take any exceptions of any kind, including PTE misses, as that
442 * would deadlock trying to reenter this routine. We pass back the 64-bit physical address.
443 * First we try the local list. If that is below a threshold, we try the global free list,
444 * which requires taking a lock, and replenish. If there are no saveareas in either list,
445 * we will install the backpocket and choke. This routine assumes that the caller has
446 * turned translation off, masked interrupts, turned on 64-bit mode, and set up:
450 * r3 = 64-bit physical address of the savearea
451 * r5 = 64-bit physical address of the page the savearea is in, with SAC
460 lwz r8,lclfreecnt(r10) ; Get the count
461 ld r3,lclfree(r10) ; Get the start of local savearea list
462 cmplwi r8,LocalSaveMin ; Are we too low?
463 ble-- saveGet64GetGlobal ; We are too low and need to grow list...
465 ; Get it from the per-processor local list.
468 li r2,0x5555 ; get r2 <-- 0x55555555 55555555, our bugbug constant
469 ld r4,SAVprev(r3) ; Chain to the next one
471 subi r8,r8,1 ; Back down count
474 std r2,SAVprev(r3) ; bug next ptr
475 stw r2,SAVlevel(r3) ; bug context ID
477 std r4,lclfree(r10) ; Unchain first savearea
478 stw r2,SAVact(r3) ; bug activation ptr
479 rldicr r5,r3,0,51 ; r5 <-- page ptr, where SAC is kept
480 stw r8,lclfreecnt(r10) ; Set new count
481 stw r6,SAVflags(r3) ; clear the flags
485 ; Local list was low so replenish from global list.
486 ; r7 = return address to caller of saveGet64
491 mflr r7 ; save return adress
492 subfic r5,r8,LocalSaveTarget ; Get the number of saveareas we need to grab to get to target
493 bl savelock ; Go lock up the anchor
495 lwz r2,SVfreecnt(0) ; Get the number on this list
496 ld r8,SVfree(0) ; Get the head of the save area list
498 sub r3,r2,r5 ; Get number left after we swipe enough for local list
499 sradi r3,r3,63 ; Get 0 if enough or -1 if not
500 andc r4,r5,r3 ; Get number to get if there are enough, 0 otherwise
501 and r5,r2,r3 ; Get 0 if there are enough, number on list otherwise
502 or. r5,r4,r5 ; r5 <- number we will move from global to local list
503 beq-- saveGet64NoFree ; There are none to get...
505 mtctr r5 ; Get loop count
506 mr r6,r8 ; Remember the first in the list
509 bdz saveGet64d ; Count down and branch when we hit 0...
510 ld r8,SAVprev(r8) ; Get the next
511 b saveGet64c ; Keep going...
514 ld r3,SAVprev(r8) ; Get the next one
515 lwz r4,SVinuse(0) ; Get the in use count
516 sub r2,r2,r5 ; Count down what we stole
517 std r3,SVfree(0) ; Set the new first in list
518 add r4,r4,r5 ; Count the ones we just put in the local list as "in use"
519 stw r2,SVfreecnt(0) ; Set the new count
520 stw r4,SVinuse(0) ; Set the new in use count
522 ld r4,lclfree(r10) ; Get the old head of list
523 lwz r3,lclfreecnt(r10) ; Get the old count
524 std r6,lclfree(r10) ; Set the new head of the list
525 add r3,r3,r5 ; Get the new count
526 std r4,SAVprev(r8) ; Point to the old head
527 stw r3,lclfreecnt(r10) ; Set the new count
529 bl saveunlock ; Update the adjust field and unlock
530 mtlr r7 ; restore return address
531 b saveGet64 ; Start over and finally allocate the savearea...
533 ; The local list is below the repopulate threshold and the global list is empty.
534 ; First we check if there are any left in the local list and if so, we allow
535 ; them to be allocated. If not, we release the backpocket list and choke.
536 ; There is nothing more that we can do at this point. Hopefully we stay alive
537 ; long enough to grab some much-needed panic information.
538 ; r7 = return address to caller of saveGet64
542 lwz r8,lclfreecnt(r10) ; Get the count
543 mr. r8,r8 ; Are there any reserve to get?
544 beq-- saveGet64Choke ; No, go choke and die...
545 bl saveunlock ; Update the adjust field and unlock
546 ld r3,lclfree(r10) ; Get the start of local savearea list
547 lwz r8,lclfreecnt(r10) ; Get the count
548 mtlr r7 ; restore return address
549 b saveGet64GetLocal ; We have some left, dip on in...
551 ; We who are about to die salute you. The savearea chain is messed up or
552 ; empty. Add in a few so we have enough to take down the system.
555 lis r9,hi16(EXT(backpocket)) ; Get high order of back pocket
556 ori r9,r9,lo16(EXT(backpocket)) ; and low part
558 lwz r8,SVfreecnt-saveanchor(r9) ; Get the new number of free elements
559 ld r7,SVfree-saveanchor(r9) ; Get the head of the chain
560 lwz r6,SVinuse(0) ; Get total in the old list
562 stw r8,SVfreecnt(0) ; Set the new number of free elements
563 add r6,r6,r8 ; Add in the new ones
564 std r7,SVfree(0) ; Set the new head of the chain
565 stw r6,SVinuse(0) ; Set total in the new list
567 saveGetChokeJoin: ; join in the fun from 32-bit mode
568 lis r0,hi16(Choke) ; Set choke firmware call
569 li r7,0 ; Get a clear register to unlock
570 ori r0,r0,lo16(Choke) ; Set the rest of the choke call
571 li r3,failNoSavearea ; Set failure code
573 eieio ; Make sure all is committed
574 stw r7,SVlock(0) ; Unlock the free list
579 * *********************
580 * * s a v e G e t 3 2 *
581 * *********************
583 * This is the internal routine to allocate a savearea on a 32-bit processor.
584 * Note that we must not take any exceptions of any kind, including PTE misses, as that
585 * would deadlock trying to reenter this routine. We pass back the 32-bit physical address.
586 * First we try the local list. If that is below a threshold, we try the global free list,
587 * which requires taking a lock, and replenish. If there are no saveareas in either list,
588 * we will install the backpocket and choke. This routine assumes that the caller has
589 * turned translation off, masked interrupts, and set up:
593 * r3 = 32-bit physical address of the savearea
594 * r5 = 32-bit physical address of the page the savearea is in, with SAC
601 lwz r8,lclfreecnt(r10) ; Get the count
602 lwz r3,lclfree+4(r10) ; Get the start of local savearea list
603 cmplwi r8,LocalSaveMin ; Are we too low?
604 ble- saveGet32GetGlobal ; We are too low and need to grow list...
606 ; Get savearea from per-processor local list.
609 li r2,0x5555 ; get r2 <-- 0x55555555, our bugbug constant
610 lwz r4,SAVprev+4(r3) ; Chain to the next one
612 subi r8,r8,1 ; Back down count
614 stw r2,SAVprev+4(r3) ; bug next ptr
615 stw r2,SAVlevel(r3) ; bug context ID
617 stw r4,lclfree+4(r10) ; Unchain first savearea
618 stw r2,SAVact(r3) ; bug activation ptr
619 rlwinm r5,r3,0,0,19 ; r5 <-- page ptr, where SAC is kept
620 stw r8,lclfreecnt(r10) ; Set new count
621 stw r6,SAVflags(r3) ; clear the flags
625 ; Local list was low so replenish from global list.
626 ; r7 = return address to caller of saveGet32
631 mflr r7 ; save return adress
632 subfic r5,r8,LocalSaveTarget ; Get the number of saveareas we need to grab to get to target
633 bl savelock ; Go lock up the anchor
635 lwz r2,SVfreecnt(0) ; Get the number on this list
636 lwz r8,SVfree+4(0) ; Get the head of the save area list
638 sub r3,r2,r5 ; Get number left after we swipe enough for local list
639 srawi r3,r3,31 ; Get 0 if enough or -1 if not
640 andc r4,r5,r3 ; Get number to get if there are enough, 0 otherwise
641 and r5,r2,r3 ; Get 0 if there are enough, number on list otherwise
642 or. r5,r4,r5 ; r5 <- number we will move from global to local list
643 beq- saveGet32NoFree ; There are none to get...
645 mtctr r5 ; Get loop count
646 mr r6,r8 ; Remember the first in the list
649 bdz saveGet32d ; Count down and branch when we hit 0...
650 lwz r8,SAVprev+4(r8) ; Get the next
651 b saveGet32c ; Keep going...
654 lwz r3,SAVprev+4(r8) ; Get the next one
655 lwz r4,SVinuse(0) ; Get the in use count
656 sub r2,r2,r5 ; Count down what we stole
657 stw r3,SVfree+4(0) ; Set the new first in list
658 add r4,r4,r5 ; Count the ones we just put in the local list as "in use"
659 stw r2,SVfreecnt(0) ; Set the new count
660 stw r4,SVinuse(0) ; Set the new in use count
662 lwz r4,lclfree+4(r10) ; Get the old head of list
663 lwz r3,lclfreecnt(r10) ; Get the old count
664 stw r6,lclfree+4(r10) ; Set the new head of the list
665 add r3,r3,r5 ; Get the new count
666 stw r4,SAVprev+4(r8) ; Point to the old head
667 stw r3,lclfreecnt(r10) ; Set the new count
669 bl saveunlock ; Update the adjust field and unlock
670 mtlr r7 ; restore return address
671 b saveGet32 ; Start over and finally allocate the savearea...
673 ; The local list is below the repopulate threshold and the global list is empty.
674 ; First we check if there are any left in the local list and if so, we allow
675 ; them to be allocated. If not, we release the backpocket list and choke.
676 ; There is nothing more that we can do at this point. Hopefully we stay alive
677 ; long enough to grab some much-needed panic information.
678 ; r7 = return address to caller of saveGet32
682 lwz r8,lclfreecnt(r10) ; Get the count
683 mr. r8,r8 ; Are there any reserve to get?
684 beq- saveGet32Choke ; No, go choke and die...
685 bl saveunlock ; Update the adjust field and unlock
686 lwz r3,lclfree+4(r10) ; Get the start of local savearea list
687 lwz r8,lclfreecnt(r10) ; Get the count
688 mtlr r7 ; restore return address
689 b saveGet32GetLocal ; We have some left, dip on in...
691 ; We who are about to die salute you. The savearea chain is messed up or
692 ; empty. Add in a few so we have enough to take down the system.
695 lis r9,hi16(EXT(backpocket)) ; Get high order of back pocket
696 ori r9,r9,lo16(EXT(backpocket)) ; and low part
698 lwz r8,SVfreecnt-saveanchor(r9) ; Get the new number of free elements
699 lwz r7,SVfree+4-saveanchor(r9) ; Get the head of the chain
700 lwz r6,SVinuse(0) ; Get total in the old list
702 stw r8,SVfreecnt(0) ; Set the new number of free elements
703 add r6,r6,r8 ; Add in the new ones (why?)
704 stw r7,SVfree+4(0) ; Set the new head of the chain
705 stw r6,SVinuse(0) ; Set total in the new list
711 * *******************
712 * * s a v e _ r e t *
713 * *******************
715 * void save_ret(struct savearea *); // normal call
716 * void save_ret_wMSR(struct savearea *,reg64_t); // passes MSR to restore as 2nd arg
718 * Return a savearea passed by virtual address to the free list.
719 * Note really well: we can take NO exceptions of any kind,
720 * including a PTE miss once the savearea lock is held. That's
721 * a guaranteed deadlock. That means we must disable for interrutions
722 * and turn all translation off.
724 .globl EXT(save_ret_wMSR) ; alternate entry pt w MSR to restore in r4
727 crset 31 ; set flag for save_ret_wMSR
728 b svrt1 ; join common code
734 crclr 31 ; clear flag for save_ret_wMSR
735 svrt1: ; join from save_ret_wMSR
736 mflr r9 ; get return address
737 rlwinm r7,r3,0,0,19 ; get virtual address of SAC area at start of page
738 mr r8,r3 ; save virtual address
739 lwz r5,SACvrswap+0(r7) ; get 64-bit converter from V to R
740 lwz r6,SACvrswap+4(r7) ; both halves, though only bottom used on 32-bit machine
742 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
743 li r2,0x2204 ; (TEST/DEBUG)
744 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
747 bl saveSetup ; turn translation off, 64-bit on, load many regs
748 bf++ 31,svrt3 ; skip if not save_ret_wMSR
749 mr r11,r4 ; was save_ret_wMSR, so overwrite saved MSR
751 bf-- pf64Bitb,svrt4 ; skip if a 32-bit processor
753 ; Handle 64-bit processor.
755 rldimi r6,r5,32,0 ; merge upper and lower halves of SACvrswap together
756 xor r3,r8,r6 ; get r3 <- 64-bit physical address of this savearea
757 bl saveRet64 ; return it
758 mtlr r9 ; restore return address
759 b saveRestore64 ; restore MSR
761 ; Handle 32-bit processor.
764 xor r3,r8,r6 ; get r3 <- 32-bit physical address of this savearea
765 bl saveRet32 ; return it
766 mtlr r9 ; restore return address
767 b saveRestore32 ; restore MSR
771 * *****************************
772 * * s a v e _ r e t _ p h y s *
773 * *****************************
775 * void save_ret_phys(reg64_t);
777 * Called from lowmem vectors to return (ie, free) a savearea by physical address.
778 * Translation and interrupts are already off, and 64-bit mode is set if defined.
779 * We can take _no_ exceptions of any kind in this code, including PTE miss, since
780 * that would result in a deadlock. We expect:
781 * r3 = phys addr of savearea
782 * msr = IR, DR, and EE off, SF on
788 .globl EXT(save_ret_phys)
791 mfsprg r10,0 ; get the per-proc ptr
792 bf-- pf64Bitb,saveRet32 ; handle 32-bit machine
793 b saveRet64 ; handle 64-bit machine
797 * *********************
798 * * s a v e R e t 6 4 *
799 * *********************
801 * This is the internal routine to free a savearea, passed by 64-bit physical
802 * address. We assume that IR, DR, and EE are all off, that SF is on, and:
803 * r3 = phys address of the savearea
810 li r0,SAVempty ; Get marker for free savearea
811 lwz r7,lclfreecnt(r10) ; Get the local count
812 ld r6,lclfree(r10) ; Get the old local header
813 addi r7,r7,1 ; Pop up the free count
814 std r6,SAVprev(r3) ; Plant free chain pointer
815 cmplwi r7,LocalSaveMax ; Has the list gotten too long?
816 stb r0,SAVflags+2(r3) ; Mark savearea free
817 std r3,lclfree(r10) ; Chain us on in
818 stw r7,lclfreecnt(r10) ; Bump up the count
819 bltlr++ ; List not too long, so done
821 /* The local savearea chain has gotten too long. Trim it down to the target.
822 * Here's a tricky bit, and important:
824 * When we trim the list, we NEVER trim the very first one. This is because that is
825 * the very last one released and the exception exit code will release the savearea
826 * BEFORE it is done using it. Wouldn't be too good if another processor started
827 * using it, eh? So for this case, we are safe so long as the savearea stays on
828 * the local list. (Note: the exit routine needs to do this because it is in the
829 * process of restoring all context and it needs to keep it until the last second.)
832 mflr r0 ; save return to caller of saveRet64
833 mr r2,r3 ; r2 <- 1st one on local list, which must not be trimmed
834 ld r3,SAVprev(r3) ; Skip over the first
835 subi r7,r7,LocalSaveTarget ; Figure out how much to trim
836 mr r6,r3 ; r6 <- first one to trim
837 mr r5,r7 ; Save the number we are trimming
840 addic. r7,r7,-1 ; Any left to do?
841 ble-- saveRet64b ; Nope...
842 ld r3,SAVprev(r3) ; Skip to the next one
843 b saveRet64a ; Keep going...
845 saveRet64b: ; r3 <- last one to trim
846 ld r7,SAVprev(r3) ; Point to the first one not to trim
847 li r4,LocalSaveTarget ; Set the target count
848 std r7,SAVprev(r2) ; Trim stuff leaving the one just released as first
849 stw r4,lclfreecnt(r10) ; Set the current count
851 bl savelock ; Lock up the anchor
853 ld r8,SVfree(0) ; Get the old head of the free list
854 lwz r4,SVfreecnt(0) ; Get the number of free ones
855 lwz r7,SVinuse(0) ; Get the number that are in use
856 std r6,SVfree(0) ; Point to the first trimmed savearea
857 add r4,r4,r5 ; Add number trimmed to free count
858 std r8,SAVprev(r3) ; Chain the old head to the tail of the trimmed guys
859 sub r7,r7,r5 ; Remove the trims from the in use count
860 stw r4,SVfreecnt(0) ; Set new free count
861 stw r7,SVinuse(0) ; Set new in use count
863 mtlr r0 ; Restore the return to our caller
864 b saveunlock ; Set adjust count, unlock the saveanchor, and return
868 * *********************
869 * * s a v e R e t 3 2 *
870 * *********************
872 * This is the internal routine to free a savearea, passed by 32-bit physical
873 * address. We assume that IR, DR, and EE are all off, and:
874 * r3 = phys address of the savearea
881 li r0,SAVempty ; Get marker for free savearea
882 lwz r7,lclfreecnt(r10) ; Get the local count
883 lwz r6,lclfree+4(r10) ; Get the old local header
884 addi r7,r7,1 ; Pop up the free count
885 stw r6,SAVprev+4(r3) ; Plant free chain pointer
886 cmplwi r7,LocalSaveMax ; Has the list gotten too long?
887 stb r0,SAVflags+2(r3) ; Mark savearea free
888 stw r3,lclfree+4(r10) ; Chain us on in
889 stw r7,lclfreecnt(r10) ; Bump up the count
890 bltlr+ ; List not too long, so done
892 /* The local savearea chain has gotten too long. Trim it down to the target.
893 * Here's a tricky bit, and important:
895 * When we trim the list, we NEVER trim the very first one. This is because that is
896 * the very last one released and the exception exit code will release the savearea
897 * BEFORE it is done using it. Wouldn't be too good if another processor started
898 * using it, eh? So for this case, we are safe so long as the savearea stays on
899 * the local list. (Note: the exit routine needs to do this because it is in the
900 * process of restoring all context and it needs to keep it until the last second.)
903 mflr r0 ; save return to caller of saveRet32
904 mr r2,r3 ; r2 <- 1st one on local list, which must not be trimmed
905 lwz r3,SAVprev+4(r3) ; Skip over the first
906 subi r7,r7,LocalSaveTarget ; Figure out how much to trim
907 mr r6,r3 ; r6 <- first one to trim
908 mr r5,r7 ; Save the number we are trimming
911 addic. r7,r7,-1 ; Any left to do?
912 ble- saveRet32b ; Nope...
913 lwz r3,SAVprev+4(r3) ; Skip to the next one
914 b saveRet32a ; Keep going...
916 saveRet32b: ; r3 <- last one to trim
917 lwz r7,SAVprev+4(r3) ; Point to the first one not to trim
918 li r4,LocalSaveTarget ; Set the target count
919 stw r7,SAVprev+4(r2) ; Trim stuff leaving the one just released as first
920 stw r4,lclfreecnt(r10) ; Set the current count
922 bl savelock ; Lock up the anchor
924 lwz r8,SVfree+4(0) ; Get the old head of the free list
925 lwz r4,SVfreecnt(0) ; Get the number of free ones
926 lwz r7,SVinuse(0) ; Get the number that are in use
927 stw r6,SVfree+4(0) ; Point to the first trimmed savearea
928 add r4,r4,r5 ; Add number trimmed to free count
929 stw r8,SAVprev+4(r3) ; Chain the old head to the tail of the trimmed guys
930 sub r7,r7,r5 ; Remove the trims from the in use count
931 stw r4,SVfreecnt(0) ; Set new free count
932 stw r7,SVinuse(0) ; Set new in use count
934 mtlr r0 ; Restore the return to our caller
935 b saveunlock ; Set adjust count, unlock the saveanchor, and return
939 * *******************************
940 * * s a v e _ t r i m _ f r e e *
941 * *******************************
943 * struct savearea_comm *save_trim_free(void);
945 * Trim the free list down to the target count, ie by -(SVadjust) save areas.
946 * It trims the list and, if a pool page was fully allocated, puts that page on
947 * the start of the pool list.
949 * If the savearea being released is the last on a pool page (i.e., all entries
950 * are released), the page is dequeued from the pool and queued to any other
951 * found during this scan. Note that this queue is maintained virtually.
953 * When the scan is done, the saveanchor lock is released and the list of
954 * freed pool pages is returned to our caller.
956 * For latency sake we may want to revisit this code. If we are trimming a
957 * large number of saveareas, we could be disabled and holding the savearea lock
958 * for quite a while. It may be that we want to break the trim down into parts.
959 * Possibly trimming the free list, then individually pushing them into the free pool.
961 * This function expects to be called with translation on and a valid stack.
962 * It uses the standard ABI, ie we destroy r2 and r3-r11, and return the ptr in r3.
965 .globl EXT(save_trim_free)
969 subi r1,r1,(FM_ALIGN(16)+FM_SIZE) ; Make space for 4 registers on stack
970 mflr r9 ; save our return address
971 stw r28,FM_SIZE+0(r1) ; Save R28
972 stw r29,FM_SIZE+4(r1) ; Save R29
973 stw r30,FM_SIZE+8(r1) ; Save R30
974 stw r31,FM_SIZE+12(r1) ; Save R31
976 bl saveSetup ; turn off translation and interrupts, load many regs
977 bl savelock ; Go lock up the anchor
979 lwz r8,SVadjust(0) ; How many do we need to clear out?
981 neg. r8,r8 ; Get the actual we need to toss (adjust is neg if too many)
982 ble- save_trim_free1 ; skip if no trimming needed anymore
983 bf-- pf64Bitb,saveTrim32 ; handle 32-bit processors
984 b saveTrim64 ; handle 64-bit processors
986 save_trim_free1: ; by the time we were called, no need to trim anymore
987 stw r3,SVlock(0) ; Quick unlock (no need for sync or to set adjust, nothing changed)
988 mtlr r9 ; Restore return
991 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
992 li r2,0x2206 ; (TEST/DEBUG)
993 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
996 addi r1,r1,(FM_ALIGN(16)+FM_SIZE); Pop stack - have not trashed register so no need to reload
997 b saveRestore ; restore translation and EE, turn SF off, return to our caller
1001 * ***********************
1002 * * s a v e T r i m 3 2 *
1003 * ***********************
1005 * Handle "save_trim_free" on 32-bit processors. At this point, translation and interrupts
1006 * are off, the savearea anchor is locked, and:
1007 * r8 = #pages to trim (>0)
1008 * r9 = return address
1009 * r10 = per-proc ptr
1010 * r11 = MSR at entry
1014 lwz r7,SVfree+4(0) ; Get the first on the free list
1015 mr r6,r7 ; Save the first one
1016 mr r5,r8 ; Save the number we are trimming
1018 sttrimming: addic. r5,r5,-1 ; Any left to do?
1019 ble- sttrimmed ; Nope...
1020 lwz r7,SAVprev+4(r7) ; Skip to the next one
1021 b sttrimming ; Keep going...
1023 sttrimmed: lwz r5,SAVprev+4(r7) ; Get the next one (for new head of free list)
1024 lwz r4,SVfreecnt(0) ; Get the free count
1025 stw r5,SVfree+4(0) ; Set new head
1026 sub r4,r4,r8 ; Calculate the new free count
1027 li r31,0 ; Show we have no free pool blocks yet
1028 crclr cr1_eq ; dont exit loop before 1st iteration
1029 stw r4,SVfreecnt(0) ; Set new free count
1030 lis r30,hi16(sac_empty) ; Get what empty looks like
1032 ; NOTE: The savearea size must be 640 (0x280). We are doing a divide by shifts and stuff
1036 #error Savearea size is not 640!!!!!!!!!!!!
1039 ; Loop over each savearea we are trimming.
1040 ; r6 = next savearea to trim
1041 ; r7 = last savearea to trim
1042 ; r8 = #pages to trim (>0)
1043 ; r9 = return address
1044 ; r10 = per-proc ptr
1045 ; r11 = MSR at entry
1046 ; r30 = what SACalloc looks like when all saveareas are free
1047 ; r31 = free pool block list
1048 ; cr1 = beq set if we just trimmed the last, ie if we are done
1050 sttoss: beq+ cr1,stdone ; All done now...
1052 cmplw cr1,r6,r7 ; Have we finished the loop?
1054 lis r0,0x0044 ; Get top of table
1055 rlwinm r2,r6,0,0,19 ; Back down to the savearea control stuff
1056 ori r0,r0,0x2200 ; Finish shift table
1057 rlwinm r4,r6,25,27,30 ; Get (addr >> 7) & 0x1E (same as twice high nybble)
1058 lwz r5,SACalloc(r2) ; Get the allocation bits
1059 addi r4,r4,1 ; Shift 1 extra
1060 rlwinm r3,r6,25,31,31 ; Get (addr >> 7) & 1
1061 rlwnm r0,r0,r4,29,31 ; Get partial index
1062 lis r4,lo16(0x8000) ; Get the bit mask
1063 add r0,r0,r3 ; Make the real index
1064 srw r4,r4,r0 ; Get the allocation mask
1065 or r5,r5,r4 ; Free this entry
1066 cmplw r5,r4 ; Is this the only free entry?
1067 lwz r6,SAVprev+4(r6) ; Chain to the next trimmed savearea
1068 cmplw cr7,r30,r5 ; Does this look empty?
1069 stw r5,SACalloc(r2) ; Save back the allocation bits
1070 beq- stputpool ; First free entry, go put it into the pool...
1071 bne+ cr7,sttoss ; Not an empty block
1074 ; We have an empty block. Remove it from the pool list.
1077 lwz r29,SACflags(r2) ; Get the flags
1078 cmplwi cr5,r31,0 ; Is this guy on the release list?
1079 lwz r28,SACnext+4(r2) ; Get the forward chain
1081 rlwinm. r0,r29,0,sac_permb,sac_permb ; Is this a permanently allocated area? (also sets 0 needed below)
1082 bne- sttoss ; This is permanent entry, do not try to release...
1084 lwz r29,SACprev+4(r2) ; and the previous
1085 beq- cr5,stnot1st ; Not first
1086 lwz r0,SACvrswap+4(r31) ; Load the previous pool page vr conversion
1088 stnot1st: stw r28,SACnext+4(r29) ; Previous guy points to my next
1089 xor r0,r0,r31 ; Make the last guy virtual
1090 stw r29,SACprev+4(r28) ; Next guy points back to my previous
1091 stw r0,SAVprev+4(r2) ; Store the old top virtual as my back chain
1092 mr r31,r2 ; My physical is now the head of the chain
1093 b sttoss ; Get the next one...
1096 ; A pool block that had no free entries now has one. Stick it on the pool list.
1099 stputpool: lwz r28,SVpoolfwd+4(0) ; Get the first guy on the list
1100 li r0,saveanchor ; Point to the saveanchor
1101 stw r2,SVpoolfwd+4(0) ; Put us on the top of the list
1102 stw r28,SACnext+4(r2) ; We point to the old top
1103 stw r2,SACprev+4(r28) ; Old top guy points back to us
1104 stw r0,SACprev+4(r2) ; Our back points to the anchor
1105 b sttoss ; Go on to the next one...
1109 * ***********************
1110 * * s a v e T r i m 6 4 *
1111 * ***********************
1113 * Handle "save_trim_free" on 64-bit processors. At this point, translation and interrupts
1114 * are off, SF is on, the savearea anchor is locked, and:
1115 * r8 = #pages to trim (>0)
1116 * r9 = return address
1117 * r10 = per-proc ptr
1118 * r11 = MSR at entry
1122 ld r7,SVfree(0) ; Get the first on the free list
1123 mr r6,r7 ; Save the first one
1124 mr r5,r8 ; Save the number we are trimming
1127 addic. r5,r5,-1 ; Any left to do?
1128 ble-- sttrimmed64 ; Nope...
1129 ld r7,SAVprev(r7) ; Skip to the next one
1130 b sttrimming64 ; Keep going...
1133 ld r5,SAVprev(r7) ; Get the next one (for new head of free list)
1134 lwz r4,SVfreecnt(0) ; Get the free count
1135 std r5,SVfree(0) ; Set new head
1136 sub r4,r4,r8 ; Calculate the new free count
1137 li r31,0 ; Show we have no free pool blocks yet
1138 crclr cr1_eq ; dont exit loop before 1st iteration
1139 stw r4,SVfreecnt(0) ; Set new free count
1140 lis r30,hi16(sac_empty) ; Get what empty looks like
1143 ; Loop over each savearea we are trimming.
1144 ; r6 = next savearea to trim
1145 ; r7 = last savearea to trim
1146 ; r8 = #pages to trim (>0)
1147 ; r9 = return address
1148 ; r10 = per-proc ptr
1149 ; r11 = MSR at entry
1150 ; r30 = what SACalloc looks like when all saveareas are free
1151 ; r31 = free pool block list
1152 ; cr1 = beq set if we just trimmed the last, ie if we are done
1154 ; WARNING: as in the 32-bit path, this code is doing a divide by 640 (SAVsize).
1157 beq++ cr1,stdone ; All done now...
1159 cmpld cr1,r6,r7 ; Have we finished the loop?
1161 lis r0,0x0044 ; Get top of table
1162 rldicr r2,r6,0,51 ; r2 <- phys addr of savearea block (with control area)
1163 ori r0,r0,0x2200 ; Finish shift table
1164 rlwinm r4,r6,25,27,30 ; Get (addr >> 7) & 0x1E (same as twice high nybble)
1165 lwz r5,SACalloc(r2) ; Get the allocation bits
1166 addi r4,r4,1 ; Shift 1 extra
1167 rlwinm r3,r6,25,31,31 ; Get (addr >> 7) & 1
1168 rlwnm r0,r0,r4,29,31 ; Get partial index
1169 lis r4,lo16(0x8000) ; Get the bit mask
1170 add r0,r0,r3 ; Make the real index
1171 srw r4,r4,r0 ; Get the allocation mask
1172 or r5,r5,r4 ; Free this entry
1173 cmplw r5,r4 ; Is this the only free entry?
1174 ld r6,SAVprev(r6) ; Chain to the next trimmed savearea
1175 cmplw cr7,r30,r5 ; Does this look empty?
1176 stw r5,SACalloc(r2) ; Save back the allocation bits
1177 beq-- stputpool64 ; First free entry, go put it into the pool...
1178 bne++ cr7,sttoss64 ; Not an empty block
1180 ; We have an empty block. Remove it from the pool list.
1182 lwz r29,SACflags(r2) ; Get the flags
1183 cmpldi cr5,r31,0 ; Is this guy on the release list?
1184 ld r28,SACnext(r2) ; Get the forward chain
1186 rlwinm. r0,r29,0,sac_permb,sac_permb ; Is this a permanently allocated area? (also sets 0 needed below)
1187 bne-- sttoss64 ; This is permanent entry, do not try to release...
1189 ld r29,SACprev(r2) ; and the previous
1190 beq-- cr5,stnot1st64 ; Not first
1191 ld r0,SACvrswap(r31) ; Load the previous pool page vr conversion
1194 std r28,SACnext(r29) ; Previous guy points to my next
1195 xor r0,r0,r31 ; Make the last guy virtual
1196 std r29,SACprev(r28) ; Next guy points back to my previous
1197 std r0,SAVprev(r2) ; Store the old top virtual as my back chain
1198 mr r31,r2 ; My physical is now the head of the chain
1199 b sttoss64 ; Get the next one...
1201 ; A pool block that had no free entries now has one. Stick it on the pool list.
1204 ld r28,SVpoolfwd(0) ; Get the first guy on the list
1205 li r0,saveanchor ; Point to the saveanchor
1206 std r2,SVpoolfwd(0) ; Put us on the top of the list
1207 std r28,SACnext(r2) ; We point to the old top
1208 std r2,SACprev(r28) ; Old top guy points back to us
1209 std r0,SACprev(r2) ; Our back points to the anchor
1210 b sttoss64 ; Go on to the next one...
1213 ; We are all done. Relocate pool release head, restore all, and go. This code
1214 ; is used both by the 32 and 64-bit paths.
1215 ; r9 = return address
1216 ; r10 = per-proc ptr
1217 ; r11 = MSR at entry
1218 ; r31 = free pool block list
1220 stdone: bl saveunlock ; Unlock the saveanchor and set adjust field
1222 mr. r3,r31 ; Move release chain and see if there are any
1223 li r5,0 ; Assume either V=R or no release chain
1224 beq- stnorel ; Nothing to release...
1225 lwz r5,SACvrswap+4(r31) ; Get the vr conversion (only need low half if 64-bit)
1228 bl saveRestore ; restore translation and exceptions, turn off SF
1229 mtlr r9 ; Restore the return
1231 lwz r28,FM_SIZE+0(r1) ; Restore R28
1232 lwz r29,FM_SIZE+4(r1) ; Restore R29
1233 lwz r30,FM_SIZE+8(r1) ; Restore R30
1234 lwz r31,FM_SIZE+12(r1) ; Restore R31
1235 addi r1,r1,(FM_ALIGN(16)+FM_SIZE) ; Pop the stack
1236 xor r3,r3,r5 ; Convert release chain address to virtual
1237 rlwinm r3,r3,0,0,31 ; if 64-bit, clear upper half of virtual address
1240 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1241 li r2,0x2207 ; (TEST/DEBUG)
1242 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1249 * ***************************
1250 * * s a v e _ r e c o v e r *
1251 * ***************************
1253 * int save_recover(void);
1255 * Returns nonzero if we can get enough saveareas to hit the target. We scan the free
1256 * pool. If we empty a pool block, we remove it from the pool list.
1260 .globl EXT(save_recover)
1263 mflr r9 ; save return address
1264 bl saveSetup ; turn translation and interrupts off, SF on, load many regs
1265 bl savelock ; lock the savearea anchor
1267 lwz r8,SVadjust(0) ; How many do we need to clear get?
1269 mr. r8,r8 ; Do we need any?
1270 ble-- save_recover1 ; not any more
1271 bf-- pf64Bitb,saveRecover32 ; handle 32-bit processor
1272 b saveRecover64 ; handle 64-bit processor
1274 save_recover1: ; by the time we locked the anchor, no longer short
1275 mtlr r9 ; Restore return
1276 stw r3,SVlock(0) ; Quick unlock (no need for sync or to set adjust, nothing changed)
1278 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1279 li r2,0x2208 ; (TEST/DEBUG)
1280 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1283 b saveRestore ; turn translation etc back on, return to our caller
1287 * *****************************
1288 * * s a v e R e c o v e r 3 2 *
1289 * *****************************
1291 * Handle "save_recover" on 32-bit processors. At this point, translation and interrupts
1292 * are off, the savearea anchor is locked, and:
1293 * r8 = #pages to recover
1294 * r9 = return address
1295 * r10 = per-proc ptr
1296 * r11 = MSR at entry
1300 li r6,saveanchor ; Start at pool anchor
1301 crclr cr1_eq ; initialize the loop test
1302 lwz r7,SVfreecnt(0) ; Get the current free count
1305 ; Loop over next block in free pool. r6 is the ptr to the last block we looked at.
1307 srcnpool: lwz r6,SACnext+4(r6) ; Point to the next one
1308 cmplwi r6,saveanchor ; Have we wrapped?
1309 beq- srcdone ; Yes, did not have enough...
1311 lwz r5,SACalloc(r6) ; Pick up the allocation for this pool block
1314 ; NOTE: The savearea size must be 640 (0x280). We are doing a multiply by shifts and add.
1315 ; offset = (index << 9) + (index << 7)
1318 #error Savearea size is not 640!!!!!!!!!!!!
1321 ; Loop over free savearea in current block.
1322 ; r5 = bitmap of free saveareas in block at r6 (ie, SACalloc)
1323 ; r6 = ptr to current free pool block
1325 ; r8 = #pages more we still need to recover
1326 ; r9 = return address
1327 ; r10 = per-proc ptr
1328 ; r11 = MSR at entry
1329 ; cr1 = beq if (r8==0)
1331 srcnext: beq- cr1,srcdone ; We have no more to get...
1333 lis r3,0x8000 ; Get the top bit on
1334 cntlzw r4,r5 ; Find a free slot
1335 addi r7,r7,1 ; Bump up the free count
1336 srw r3,r3,r4 ; Make a mask
1337 slwi r0,r4,7 ; First multiply by 128
1338 subi r8,r8,1 ; Decrement the need count
1339 slwi r2,r4,9 ; Then multiply by 512
1340 andc. r5,r5,r3 ; Clear out the "free" bit
1341 add r2,r2,r0 ; Sum to multiply by 640
1343 stw r5,SACalloc(r6) ; Set new allocation bits
1345 add r2,r2,r6 ; Get the actual address of the savearea
1346 lwz r3,SVfree+4(0) ; Get the head of the chain
1347 cmplwi cr1,r8,0 ; Do we actually need any more?
1348 stw r2,SVfree+4(0) ; Push ourselves in the front
1349 stw r3,SAVprev+4(r2) ; Chain the rest of the list behind
1351 bne+ srcnext ; The pool block is not empty yet, try for another...
1353 lwz r2,SACnext+4(r6) ; Get the next pointer
1354 lwz r3,SACprev+4(r6) ; Get the previous pointer
1355 stw r3,SACprev+4(r2) ; The previous of my next points to my previous
1356 stw r2,SACnext+4(r3) ; The next of my previous points to my next
1357 bne+ cr1,srcnpool ; We still have more to do...
1360 ; Join here from 64-bit path when we have recovered all the saveareas we need to.
1362 srcdone: stw r7,SVfreecnt(0) ; Set the new free count
1363 bl saveunlock ; Unlock the save and set adjust field
1365 mtlr r9 ; Restore the return
1367 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1368 li r2,0x2209 ; (TEST/DEBUG)
1369 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1372 b saveRestore ; turn xlate and EE back on, SF off, and return to our caller
1376 * *****************************
1377 * * s a v e R e c o v e r 6 4 *
1378 * *****************************
1380 * Handle "save_recover" on 64-bit processors. At this point, translation and interrupts
1381 * are off, the savearea anchor is locked, and:
1382 * r8 = #pages to recover
1383 * r9 = return address
1384 * r10 = per-proc ptr
1385 * r11 = MSR at entry
1389 li r6,saveanchor ; Start at pool anchor
1390 crclr cr1_eq ; initialize the loop test
1391 lwz r7,SVfreecnt(0) ; Get the current free count
1394 ; Loop over next block in free pool. r6 is the ptr to the last block we looked at.
1397 ld r6,SACnext(r6) ; Point to the next one
1398 cmpldi r6,saveanchor ; Have we wrapped?
1399 beq-- srcdone ; Yes, did not have enough...
1401 lwz r5,SACalloc(r6) ; Pick up the allocation for this pool block
1404 ; Loop over free savearea in current block.
1405 ; r5 = bitmap of free saveareas in block at r6 (ie, SACalloc)
1406 ; r6 = ptr to current free pool block
1408 ; r8 = #pages more we still need to recover
1409 ; r9 = return address
1410 ; r10 = per-proc ptr
1411 ; r11 = MSR at entry
1412 ; cr1 = beq if (r8==0)
1414 ; WARNING: as in the 32-bit path, we depend on (SAVsize==640)
1417 beq-- cr1,srcdone ; We have no more to get...
1419 lis r3,0x8000 ; Get the top bit on
1420 cntlzw r4,r5 ; Find a free slot
1421 addi r7,r7,1 ; Bump up the free count
1422 srw r3,r3,r4 ; Make a mask
1423 slwi r0,r4,7 ; First multiply by 128
1424 subi r8,r8,1 ; Decrement the need count
1425 slwi r2,r4,9 ; Then multiply by 512
1426 andc. r5,r5,r3 ; Clear out the "free" bit
1427 add r2,r2,r0 ; Sum to multiply by 640
1429 stw r5,SACalloc(r6) ; Set new allocation bits
1431 add r2,r2,r6 ; Get the actual address of the savearea
1432 ld r3,SVfree(0) ; Get the head of the chain
1433 cmplwi cr1,r8,0 ; Do we actually need any more?
1434 std r2,SVfree(0) ; Push ourselves in the front
1435 std r3,SAVprev(r2) ; Chain the rest of the list behind
1437 bne++ srcnext64 ; The pool block is not empty yet, try for another...
1439 ld r2,SACnext(r6) ; Get the next pointer
1440 ld r3,SACprev(r6) ; Get the previous pointer
1441 std r3,SACprev(r2) ; The previous of my next points to my previous
1442 std r2,SACnext(r3) ; The next of my previous points to my next
1443 bne++ cr1,srcnpool64 ; We still have more to do...
1449 * *******************
1450 * * s a v e l o c k *
1451 * *******************
1453 * Lock the savearea anchor, so we can manipulate the free list.
1454 * msr = interrupts and translation off
1460 savelock: lwz r8,SVlock(0) ; See if lock is held
1462 li r12,saveanchor ; Point to the saveanchor
1463 bne-- savelock ; loop until lock released...
1465 savelock0: lwarx r8,0,r12 ; Grab the lock value
1467 li r8,1 ; get nonzero to lock it with
1468 bne-- savelock1 ; already locked, wait for it to clear...
1469 stwcx. r8,0,r12 ; Try to seize that there durn lock
1470 isync ; assume we got it
1471 beqlr++ ; reservation not lost, so we have the lock
1472 b savelock0 ; Try again...
1474 savelock1: li r8,lgKillResv ; Point to killing field
1475 stwcx. r8,0,r8 ; Kill reservation
1476 b savelock ; Start over....
1480 * ***********************
1481 * * s a v e u n l o c k *
1482 * ***********************
1485 * This is the common routine that sets the saveadjust field and unlocks the savearea
1487 * msr = interrupts and translation off
1493 lwz r6,SVfreecnt(0) ; and the number on the free list
1494 lwz r5,SVinuse(0) ; Pick up the in use count
1495 subic. r8,r6,FreeListMin ; do we have at least the minimum?
1496 lwz r2,SVtarget(0) ; Get the target
1497 neg r8,r8 ; assuming we are short, get r8 <- shortfall
1498 blt-- saveunlock1 ; skip if fewer than minimum on free list
1500 add r6,r6,r5 ; Get the total number of saveareas
1501 addi r5,r2,-SaveLowHysteresis ; Find low end of acceptible range
1502 sub r5,r6,r5 ; Make everything below hysteresis negative
1503 sub r2,r2,r6 ; Get the distance from the target
1504 addi r5,r5,-(SaveLowHysteresis + SaveHighHysteresis + 1) ; Subtract full hysteresis range
1505 srawi r5,r5,31 ; Get 0xFFFFFFFF if outside range or 0 if inside
1506 and r8,r2,r5 ; r8 <- 0 if in range or distance to target if not
1509 li r5,0 ; Set a clear value
1510 stw r8,SVadjust(0) ; Set the adjustment value
1511 eieio ; Make sure everything is done
1512 stw r5,SVlock(0) ; Unlock the savearea chain
1517 * *******************
1518 * * s a v e _ c p v *
1519 * *******************
1521 * struct savearea *save_cpv(addr64_t saveAreaPhysAddr);
1523 * Converts a physical savearea address to virtual. Called with translation on
1524 * and in 32-bit mode. Note that the argument is passed as a long long in (r3,r4).
1528 .globl EXT(save_cpv)
1531 mflr r9 ; save return address
1532 mr r8,r3 ; save upper half of phys address here
1533 bl saveSetup ; turn off translation and interrupts, turn SF on
1534 rlwinm r5,r4,0,0,19 ; Round back to the start of the physical savearea block
1535 bf-- pf64Bitb,save_cpv1 ; skip if 32-bit processor
1536 rldimi r5,r8,32,0 ; r5 <- 64-bit phys address of block
1538 lwz r6,SACvrswap+4(r5) ; Get the conversion to virtual (only need low half if 64-bit)
1539 mtlr r9 ; restore return address
1540 xor r3,r4,r6 ; convert phys to virtual
1541 rlwinm r3,r3,0,0,31 ; if 64-bit, zero upper half of virtual address
1542 b saveRestore ; turn translation etc back on, SF off, and return r3
1546 * *********************
1547 * * s a v e S e t u p *
1548 * *********************
1550 * This routine is called at the start of all the save-area subroutines.
1551 * It turns off translation, disabled interrupts, turns on 64-bit mode,
1552 * and sets up cr6 with the feature flags (especially pf64Bit).
1554 * Note that most save-area routines cannot take _any_ interrupt (such as a
1555 * PTE miss) once the savearea anchor is locked, since that would result in
1556 * instant deadlock as we need a save-area to process any exception.
1558 * r10 = per-proc ptr
1560 * cr5 = pfNoMSRir feature flag
1561 * cr6 = pf64Bit feature flag
1563 * We use r0, r3, r10, and r11.
1568 mfsprg r3,2 ; get feature flags
1570 mtcrf 0x2,r3 ; copy pf64Bit to cr6
1571 ori r0,r0,lo16(MASK(MSR_IR)+MASK(MSR_DR)+MASK(MSR_EE))
1572 mtcrf 0x4,r3 ; copy pfNoMSRir to cr5
1573 andc r3,r11,r0 ; turn off IR, DR, and EE
1574 li r0,1 ; get a 1 in case its a 64-bit machine
1575 bf-- pf64Bitb,saveSetup1 ; skip if not a 64-bit machine
1576 rldimi r3,r0,63,MSR_SF_BIT ; turn SF (bit 0) on
1577 mtmsrd r3 ; turn translation and interrupts off, 64-bit mode on
1578 isync ; wait for it to happen
1579 mfsprg r10,0 ; get per-proc ptr
1581 saveSetup1: ; here on 32-bit machines
1582 bt- pfNoMSRirb,saveSetup2 ; skip if cannot turn off IR with a mtmsr
1583 mtmsr r3 ; turn translation and interrupts off
1584 isync ; wait for it to happen
1585 mfsprg r10,0 ; get per-proc ptr
1587 saveSetup2: ; here if pfNoMSRir set for this machine
1588 li r0,loadMSR ; we will "mtmsr r3" via system call
1590 mfsprg r10,0 ; get per-proc ptr
1595 * *************************
1596 * * s a v e R e s t o r e *
1597 * *************************
1599 * Undoes the effect of calling "saveSetup", ie it turns relocation and interrupts back on,
1600 * and turns 64-bit mode back off.
1602 * cr6 = pf64Bit feature flag
1606 bt++ pf64Bitb,saveRestore64 ; handle a 64-bit processor
1608 mtmsr r11 ; restore MSR
1609 isync ; wait for translation to start up
1611 saveRestore64: ; 64-bit processor
1612 mtmsrd r11 ; restore MSR
1613 isync ; wait for changes to happen
projects / apple / xnu.git / blob