]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ppc/savearea_asm.s
projects / apple / xnu.git / blob
? search:


22fadc96804aeed4b11bafdc9af5ef7b8130ed66
[apple/xnu.git] / osfmk / ppc / savearea_asm.s
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #define FPVECDBG 0
25
26 #include <assym.s>
27 #include <debug.h>
28 #include <db_machine_commands.h>
29 #include <mach_rt.h>
30
31 #include <mach_debug.h>
32 #include <ppc/asm.h>
33 #include <ppc/proc_reg.h>
34 #include <ppc/exception.h>
35 #include <ppc/Performance.h>
36 #include <ppc/exception.h>
37 #include <ppc/savearea.h>
38 #include <mach/ppc/vm_param.h>
39
40 .text
41
42 /* Register usage conventions in this code:
43 * r9 = return address
44 * r10 = per-proc ptr
45 * r11 = MSR at entry
46 * cr6 = feature flags (ie, pf64Bit)
47 *
48 * Because much of this code deals with physical addresses,
49 * there are parallel paths for 32- and 64-bit machines.
50 */
51
52
53 /*
54 * *****************************
55 * * s a v e _ s n a p s h o t *
56 * *****************************
57 *
58 * void save_snapshot();
59 *
60 * Link the current free list & processor local list on an independent list.
61 */
62 .align 5
63 .globl EXT(save_snapshot)
64
65 LEXT(save_snapshot)
66 mflr r9 ; get return address
67 bl saveSetup ; turn translation off, 64-bit on, load many regs
68 bf-- pf64Bitb,save_snapshot32 ; skip if 32-bit processor
69
70 ; Handle 64-bit processor.
71
72 save_snapshot64:
73
74 ld r8,next_savearea(r10) ; Start with the current savearea
75 std r8,SVsavefreesnapshot(0) ; Make it the restore list anchor
76 ld r5,SVfree(0) ; Get free save area list anchor
77
78 save_snapshot64nextfree:
79 mr r7,r5
80 std r7,savemisc1(r8) ; Link this one
81 ld r5,SAVprev(r7) ; Get the next
82 mr r8,r7
83 mr. r0,r5
84 bne save_snapshot64nextfree
85
86 lwz r6,SVinuse(0) ; Get inuse count
87 ld r5,lclfree(r10) ; Get the local savearea list
88 subi r6,r6,1 ; Count the first as free
89
90 save_snapshot64nextlocalfree:
91 subi r6,r6,1 ; Count as free
92 mr r7,r5
93 std r7,savemisc1(r8) ; Link this one
94 ld r5,SAVprev(r7) ; Get the next
95 mr r8,r7
96 mr. r0,r5
97 bne save_snapshot64nextlocalfree
98
99 std r5,savemisc1(r8) ; End the list
100 stw r6,SVsaveinusesnapshot(0) ; Save the new number of inuse saveareas
101
102 mtlr r9 ; Restore the return
103 b saveRestore64 ; Restore interrupts and translation
104
105 ; Handle 32-bit processor.
106
107 save_snapshot32:
108 lwz r8,next_savearea+4(r10) ; Start with the current savearea
109 stw r8,SVsavefreesnapshot+4(0) ; Make it the restore list anchor
110 lwz r5,SVfree+4(0) ; Get free save area list anchor
111
112 save_snapshot32nextfree:
113 mr r7,r5
114 stw r7,savemisc1+4(r8) ; Link this one
115 lwz r5,SAVprev+4(r7) ; Get the next
116 mr r8,r7
117 mr. r0,r5
118 bne save_snapshot32nextfree
119
120 lwz r6,SVinuse(0) ; Get inuse count
121 lwz r5,lclfree+4(r10) ; Get the local savearea list
122 subi r6,r6,1 ; Count the first as free
123
124 save_snapshot32nextlocalfree:
125 subi r6,r6,1 ; Count as free
126 mr r7,r5
127 stw r7,savemisc1+4(r8) ; Link this one
128 lwz r5,SAVprev+4(r7) ; Get the next
129 mr r8,r7
130 mr. r0,r5
131 bne save_snapshot32nextlocalfree
132
133 stw r5,savemisc1+4(r8) ; End the list
134 stw r6,SVsaveinusesnapshot(0) ; Save the new number of inuse saveareas
135
136 mtlr r9 ; Restore the return
137 b saveRestore32 ; Restore interrupts and translation
138
139 /*
140 * *********************************************
141 * * s a v e _ s n a p s h o t _ r e s t o r e *
142 * *********************************************
143 *
144 * void save_snapshot_restore();
145 *
146 * Restore the free list from the snapshot list, and reset the processors next savearea.
147 */
148 .align 5
149 .globl EXT(save_snapshot_restore)
150
151 LEXT(save_snapshot_restore)
152 mflr r9 ; get return address
153 bl saveSetup ; turn translation off, 64-bit on, load many regs
154 bf-- pf64Bitb,save_snapshot_restore32 ; skip if 32-bit processor
155
156 ; Handle 64-bit processor.
157
158 save_snapshot_restore64:
159 lwz r7,SVsaveinusesnapshot(0)
160 stw r7,SVinuse(0) ; Set the new inuse count
161
162 li r6,0
163 stw r6,lclfreecnt(r10) ; None local now
164 std r6,lclfree(r10) ; None local now
165
166 ld r8,SVsavefreesnapshot(0) ; Get the restore list anchor
167 std r8,SVfree(0) ; Make it the free list anchor
168 li r5,SAVempty ; Get marker for free savearea
169
170 save_snapshot_restore64nextfree:
171 addi r6,r6,1 ; Count as free
172 stb r5,SAVflags+2(r8) ; Mark savearea free
173 ld r7,savemisc1(r8) ; Get the next
174 std r7,SAVprev(r8) ; Set the next in free list
175 mr. r8,r7
176 bne save_snapshot_restore64nextfree
177
178 stw r6,SVfreecnt(0) ; Set the new free count
179
180 bl saveGet64
181 std r3,next_savearea(r10) ; Get the next savearea
182
183 mtlr r9 ; Restore the return
184 b saveRestore64 ; Restore interrupts and translation
185
186 ; Handle 32-bit processor.
187
188 save_snapshot_restore32:
189 lwz r7,SVsaveinusesnapshot(0)
190 stw r7,SVinuse(0) ; Set the new inuse count
191
192 li r6,0
193 stw r6,lclfreecnt(r10) ; None local now
194 stw r6,lclfree+4(r10) ; None local now
195
196 lwz r8,SVsavefreesnapshot+4(0) ; Get the restore list anchor
197 stw r8,SVfree+4(0) ; Make it the free list anchor
198 li r5,SAVempty ; Get marker for free savearea
199
200 save_snapshot_restore32nextfree:
201 addi r6,r6,1 ; Count as free
202 stb r5,SAVflags+2(r8) ; Mark savearea free
203 lwz r7,savemisc1+4(r8) ; Get the next
204 stw r7,SAVprev+4(r8) ; Set the next in free list
205 mr. r8,r7
206 bne save_snapshot_restore32nextfree
207
208 stw r6,SVfreecnt(0) ; Set the new free count
209
210 bl saveGet32
211 stw r3,next_savearea+4(r10) ; Get the next savearea
212
213 mtlr r9 ; Restore the return
214 b saveRestore32 ; Restore interrupts and translation
215
216 /*
217 * ***********************
218 * * s a v e _ q u e u e *
219 * ***********************
220 *
221 * void save_queue(ppnum_t pagenum);
222 *
223 * This routine will add a savearea block to the free list.
224 * We also queue the block to the free pool list. This is a
225 * circular double linked list. Because this block has no free entries,
226 * it gets queued to the end of the list
227 */
228 .align 5
229 .globl EXT(save_queue)
230
231 LEXT(save_queue)
232 mflr r9 ; get return address
233 mr r8,r3 ; move pagenum out of the way
234 bl saveSetup ; turn translation off, 64-bit on, load many regs
235 bf-- pf64Bitb,saveQueue32 ; skip if 32-bit processor
236
237 sldi r2,r8,12 ; r2 <-- phys address of page
238 li r8,sac_cnt ; Get the number of saveareas per page
239 mr r4,r2 ; Point to start of chain
240 li r0,SAVempty ; Get empty marker
241
242 saveQueue64a:
243 addic. r8,r8,-1 ; Keep track of how many we did
244 stb r0,SAVflags+2(r4) ; Set empty
245 addi r7,r4,SAVsize ; Point to the next slot
246 ble- saveQueue64b ; We are done with the chain
247 std r7,SAVprev(r4) ; Set this chain
248 mr r4,r7 ; Step to the next
249 b saveQueue64a ; Fill the whole block...
250
251 saveQueue64b:
252 bl savelock ; Go lock the save anchor
253
254 ld r7,SVfree(0) ; Get the free save area list anchor
255 lwz r6,SVfreecnt(0) ; Get the number of free saveareas
256
257 std r2,SVfree(0) ; Queue in the new one
258 addi r6,r6,sac_cnt ; Count the ones we are linking in
259 std r7,SAVprev(r4) ; Queue the old first one off of us
260 stw r6,SVfreecnt(0) ; Save the new count
261 b saveQueueExit
262
263 ; Handle 32-bit processor.
264
265 saveQueue32:
266 slwi r2,r8,12 ; r2 <-- phys address of page
267 li r8,sac_cnt ; Get the number of saveareas per page
268 mr r4,r2 ; Point to start of chain
269 li r0,SAVempty ; Get empty marker
270
271 saveQueue32a:
272 addic. r8,r8,-1 ; Keep track of how many we did
273 stb r0,SAVflags+2(r4) ; Set empty
274 addi r7,r4,SAVsize ; Point to the next slot
275 ble- saveQueue32b ; We are done with the chain
276 stw r7,SAVprev+4(r4) ; Set this chain
277 mr r4,r7 ; Step to the next
278 b saveQueue32a ; Fill the whole block...
279
280 saveQueue32b:
281 bl savelock ; Go lock the save anchor
282
283 lwz r7,SVfree+4(0) ; Get the free save area list anchor
284 lwz r6,SVfreecnt(0) ; Get the number of free saveareas
285
286 stw r2,SVfree+4(0) ; Queue in the new one
287 addi r6,r6,sac_cnt ; Count the ones we are linking in
288 stw r7,SAVprev+4(r4) ; Queue the old first one off of us
289 stw r6,SVfreecnt(0) ; Save the new count
290
291 saveQueueExit: ; join here from 64-bit path
292 bl saveunlock ; Unlock the list and set the adjust count
293 mtlr r9 ; Restore the return
294
295 #if FPVECDBG
296 mfsprg r2,1 ; (TEST/DEBUG)
297 mr. r2,r2 ; (TEST/DEBUG)
298 beq-- saveRestore ; (TEST/DEBUG)
299 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
300 li r2,0x2201 ; (TEST/DEBUG)
301 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
302 sc ; (TEST/DEBUG)
303 #endif
304 b saveRestore ; Restore interrupts and translation
305
306 /*
307 * *****************************
308 * * s a v e _ g e t _ i n i t *
309 * *****************************
310 *
311 * addr64_t save_get_init(void);
312 *
313 * Note that save_get_init is used in initial processor startup only. It
314 * is used because translation is on, but no tables exist yet and we have
315 * no V=R BAT registers that cover the entire physical memory.
316 */
317 .align 5
318 .globl EXT(save_get_init)
319
320 LEXT(save_get_init)
321 mflr r9 ; get return address
322 bl saveSetup ; turn translation off, 64-bit on, load many regs
323 bfl-- pf64Bitb,saveGet32 ; Get r3 <- savearea, r5 <- page address (with SAC)
324 btl++ pf64Bitb,saveGet64 ; get one on a 64-bit machine
325 bl saveRestore ; restore translation etc
326 mtlr r9
327
328 ; unpack the physaddr in r3 into a long long in (r3,r4)
329
330 mr r4,r3 ; copy low word of phys address to r4
331 li r3,0 ; assume upper word was 0
332 bflr-- pf64Bitb ; if 32-bit processor, return
333 srdi r3,r4,32 ; unpack reg64_t to addr64_t on 64-bit machine
334 rlwinm r4,r4,0,0,31
335 blr
336
337
338 /*
339 * *******************
340 * * s a v e _ g e t *
341 * *******************
342 *
343 * savearea *save_get(void);
344 *
345 * Allocate a savearea, returning a virtual address. NOTE: we must preserve
346 * r0, r2, and r12. Our callers in cswtch.s depend on this.
347 */
348 .align 5
349 .globl EXT(save_get)
350
351 LEXT(save_get)
352 mflr r9 ; get return address
353 mr r5,r0 ; copy regs before saveSetup nails them
354 bl saveSetup ; turn translation off, 64-bit on, load many regs
355 bf-- pf64Bitb,svgt1 ; skip if 32-bit processor
356
357 std r5,tempr0(r10) ; save r0 in per-proc across call to saveGet64
358 std r2,tempr2(r10) ; and r2
359 std r12,tempr4(r10) ; and r12
360 bl saveGet64 ; get r3 <- savearea, r5 <- page address (with SAC)
361 ld r0,tempr0(r10) ; restore callers regs
362 ld r2,tempr2(r10)
363 ld r12,tempr4(r10)
364 b svgt2
365
366 svgt1: ; handle 32-bit processor
367 stw r5,tempr0+4(r10) ; save r0 in per-proc across call to saveGet32
368 stw r2,tempr2+4(r10) ; and r2
369 stw r12,tempr4+4(r10) ; and r12
370 bl saveGet32 ; get r3 <- savearea, r5 <- page address (with SAC)
371 lwz r0,tempr0+4(r10) ; restore callers regs
372 lwz r2,tempr2+4(r10)
373 lwz r12,tempr4+4(r10)
374
375 svgt2:
376 lwz r5,SACvrswap+4(r5) ; Get the virtual to real translation (only need low word)
377 mtlr r9 ; restore return address
378 xor r3,r3,r5 ; convert physaddr to virtual
379 rlwinm r3,r3,0,0,31 ; 0 upper word if a 64-bit machine
380
381 #if FPVECDBG
382 mr r6,r0 ; (TEST/DEBUG)
383 mr r7,r2 ; (TEST/DEBUG)
384 mfsprg r2,1 ; (TEST/DEBUG)
385 mr. r2,r2 ; (TEST/DEBUG)
386 beq-- svgDBBypass ; (TEST/DEBUG)
387 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
388 li r2,0x2203 ; (TEST/DEBUG)
389 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
390 sc ; (TEST/DEBUG)
391 svgDBBypass: ; (TEST/DEBUG)
392 mr r0,r6 ; (TEST/DEBUG)
393 mr r2,r7 ; (TEST/DEBUG)
394 #endif
395 b saveRestore ; restore MSR and return to our caller
396
397
398 /*
399 * ***********************************
400 * * s a v e _ g e t _ p h y s _ 3 2 *
401 * ***********************************
402 *
403 * reg64_t save_get_phys(void);
404 *
405 * This is the entry normally called from lowmem_vectors.s with
406 * translation and interrupts already off.
407 * MUST NOT TOUCH CR7
408 */
409 .align 5
410 .globl EXT(save_get_phys_32)
411
412 LEXT(save_get_phys_32)
413 mfsprg r10,0 ; get the per-proc ptr
414 b saveGet32 ; Get r3 <- savearea, r5 <- page address (with SAC)
415
416
417 /*
418 * ***********************************
419 * * s a v e _ g e t _ p h y s _ 6 4 *
420 * ***********************************
421 *
422 * reg64_t save_get_phys_64(void);
423 *
424 * This is the entry normally called from lowmem_vectors.s with
425 * translation and interrupts already off, and in 64-bit mode.
426 * MUST NOT TOUCH CR7
427 */
428 .align 5
429 .globl EXT(save_get_phys_64)
430
431 LEXT(save_get_phys_64)
432 mfsprg r10,0 ; get the per-proc ptr
433 b saveGet64 ; Get r3 <- savearea, r5 <- page address (with SAC)
434
435
436 /*
437 * *********************
438 * * s a v e G e t 6 4 *
439 * *********************
440 *
441 * This is the internal routine to allocate a savearea on a 64-bit processor.
442 * Note that we must not take any exceptions of any kind, including PTE misses, as that
443 * would deadlock trying to reenter this routine. We pass back the 64-bit physical address.
444 * First we try the local list. If that is below a threshold, we try the global free list,
445 * which requires taking a lock, and replenish. If there are no saveareas in either list,
446 * we will install the backpocket and choke. This routine assumes that the caller has
447 * turned translation off, masked interrupts, turned on 64-bit mode, and set up:
448 * r10 = per-proc ptr
449 *
450 * We return:
451 * r3 = 64-bit physical address of the savearea
452 * r5 = 64-bit physical address of the page the savearea is in, with SAC
453 *
454 * We destroy:
455 * r2-r8.
456 *
457 * MUST NOT TOUCH CR7
458 */
459
460 saveGet64:
461 lwz r8,lclfreecnt(r10) ; Get the count
462 ld r3,lclfree(r10) ; Get the start of local savearea list
463 cmplwi r8,LocalSaveMin ; Are we too low?
464 ble-- saveGet64GetGlobal ; We are too low and need to grow list...
465
466 ; Get it from the per-processor local list.
467
468 saveGet64GetLocal:
469 li r2,0x5555 ; get r2 <-- 0x55555555 55555555, our bugbug constant
470 ld r4,SAVprev(r3) ; Chain to the next one
471 oris r2,r2,0x5555
472 subi r8,r8,1 ; Back down count
473 rldimi r2,r2,32,0
474
475 std r2,SAVprev(r3) ; bug next ptr
476 stw r2,SAVlevel(r3) ; bug context ID
477 li r6,0
478 std r4,lclfree(r10) ; Unchain first savearea
479 stw r2,SAVact(r3) ; bug activation ptr
480 rldicr r5,r3,0,51 ; r5 <-- page ptr, where SAC is kept
481 stw r8,lclfreecnt(r10) ; Set new count
482 stw r6,SAVflags(r3) ; clear the flags
483
484 blr
485
486 ; Local list was low so replenish from global list.
487 ; r7 = return address to caller of saveGet64
488 ; r8 = lclfreecnt
489 ; r10 = per-proc ptr
490
491 saveGet64GetGlobal:
492 mflr r7 ; save return adress
493 subfic r5,r8,LocalSaveTarget ; Get the number of saveareas we need to grab to get to target
494 bl savelock ; Go lock up the anchor
495
496 lwz r2,SVfreecnt(0) ; Get the number on this list
497 ld r8,SVfree(0) ; Get the head of the save area list
498
499 sub r3,r2,r5 ; Get number left after we swipe enough for local list
500 sradi r3,r3,63 ; Get 0 if enough or -1 if not
501 andc r4,r5,r3 ; Get number to get if there are enough, 0 otherwise
502 and r5,r2,r3 ; Get 0 if there are enough, number on list otherwise
503 or. r5,r4,r5 ; r5 <- number we will move from global to local list
504 beq-- saveGet64NoFree ; There are none to get...
505
506 mtctr r5 ; Get loop count
507 mr r6,r8 ; Remember the first in the list
508
509 saveGet64c:
510 bdz saveGet64d ; Count down and branch when we hit 0...
511 ld r8,SAVprev(r8) ; Get the next
512 b saveGet64c ; Keep going...
513
514 saveGet64d:
515 ld r3,SAVprev(r8) ; Get the next one
516 lwz r4,SVinuse(0) ; Get the in use count
517 sub r2,r2,r5 ; Count down what we stole
518 std r3,SVfree(0) ; Set the new first in list
519 add r4,r4,r5 ; Count the ones we just put in the local list as "in use"
520 stw r2,SVfreecnt(0) ; Set the new count
521 stw r4,SVinuse(0) ; Set the new in use count
522
523 ld r4,lclfree(r10) ; Get the old head of list
524 lwz r3,lclfreecnt(r10) ; Get the old count
525 std r6,lclfree(r10) ; Set the new head of the list
526 add r3,r3,r5 ; Get the new count
527 std r4,SAVprev(r8) ; Point to the old head
528 stw r3,lclfreecnt(r10) ; Set the new count
529
530 bl saveunlock ; Update the adjust field and unlock
531 mtlr r7 ; restore return address
532 b saveGet64 ; Start over and finally allocate the savearea...
533
534 ; The local list is below the repopulate threshold and the global list is empty.
535 ; First we check if there are any left in the local list and if so, we allow
536 ; them to be allocated. If not, we release the backpocket list and choke.
537 ; There is nothing more that we can do at this point. Hopefully we stay alive
538 ; long enough to grab some much-needed panic information.
539 ; r7 = return address to caller of saveGet64
540 ; r10 = per-proc ptr
541
542 saveGet64NoFree:
543 lwz r8,lclfreecnt(r10) ; Get the count
544 mr. r8,r8 ; Are there any reserve to get?
545 beq-- saveGet64Choke ; No, go choke and die...
546 bl saveunlock ; Update the adjust field and unlock
547 ld r3,lclfree(r10) ; Get the start of local savearea list
548 lwz r8,lclfreecnt(r10) ; Get the count
549 mtlr r7 ; restore return address
550 b saveGet64GetLocal ; We have some left, dip on in...
551
552 ; We who are about to die salute you. The savearea chain is messed up or
553 ; empty. Add in a few so we have enough to take down the system.
554
555 saveGet64Choke:
556 lis r9,hi16(EXT(backpocket)) ; Get high order of back pocket
557 ori r9,r9,lo16(EXT(backpocket)) ; and low part
558
559 lwz r8,SVfreecnt-saveanchor(r9) ; Get the new number of free elements
560 ld r7,SVfree-saveanchor(r9) ; Get the head of the chain
561 lwz r6,SVinuse(0) ; Get total in the old list
562
563 stw r8,SVfreecnt(0) ; Set the new number of free elements
564 add r6,r6,r8 ; Add in the new ones
565 std r7,SVfree(0) ; Set the new head of the chain
566 stw r6,SVinuse(0) ; Set total in the new list
567
568 saveGetChokeJoin: ; join in the fun from 32-bit mode
569 lis r0,hi16(Choke) ; Set choke firmware call
570 li r7,0 ; Get a clear register to unlock
571 ori r0,r0,lo16(Choke) ; Set the rest of the choke call
572 li r3,failNoSavearea ; Set failure code
573
574 eieio ; Make sure all is committed
575 stw r7,SVlock(0) ; Unlock the free list
576 sc ; System ABEND
577
578
579 /*
580 * *********************
581 * * s a v e G e t 3 2 *
582 * *********************
583 *
584 * This is the internal routine to allocate a savearea on a 32-bit processor.
585 * Note that we must not take any exceptions of any kind, including PTE misses, as that
586 * would deadlock trying to reenter this routine. We pass back the 32-bit physical address.
587 * First we try the local list. If that is below a threshold, we try the global free list,
588 * which requires taking a lock, and replenish. If there are no saveareas in either list,
589 * we will install the backpocket and choke. This routine assumes that the caller has
590 * turned translation off, masked interrupts, and set up:
591 * r10 = per-proc ptr
592 *
593 * We return:
594 * r3 = 32-bit physical address of the savearea
595 * r5 = 32-bit physical address of the page the savearea is in, with SAC
596 *
597 * We destroy:
598 * r2-r8.
599 */
600
601 saveGet32:
602 lwz r8,lclfreecnt(r10) ; Get the count
603 lwz r3,lclfree+4(r10) ; Get the start of local savearea list
604 cmplwi r8,LocalSaveMin ; Are we too low?
605 ble- saveGet32GetGlobal ; We are too low and need to grow list...
606
607 ; Get savearea from per-processor local list.
608
609 saveGet32GetLocal:
610 li r2,0x5555 ; get r2 <-- 0x55555555, our bugbug constant
611 lwz r4,SAVprev+4(r3) ; Chain to the next one
612 oris r2,r2,0x5555
613 subi r8,r8,1 ; Back down count
614
615 stw r2,SAVprev+4(r3) ; bug next ptr
616 stw r2,SAVlevel(r3) ; bug context ID
617 li r6,0
618 stw r4,lclfree+4(r10) ; Unchain first savearea
619 stw r2,SAVact(r3) ; bug activation ptr
620 rlwinm r5,r3,0,0,19 ; r5 <-- page ptr, where SAC is kept
621 stw r8,lclfreecnt(r10) ; Set new count
622 stw r6,SAVflags(r3) ; clear the flags
623
624 blr
625
626 ; Local list was low so replenish from global list.
627 ; r7 = return address to caller of saveGet32
628 ; r8 = lclfreecnt
629 ; r10 = per-proc ptr
630
631 saveGet32GetGlobal:
632 mflr r7 ; save return adress
633 subfic r5,r8,LocalSaveTarget ; Get the number of saveareas we need to grab to get to target
634 bl savelock ; Go lock up the anchor
635
636 lwz r2,SVfreecnt(0) ; Get the number on this list
637 lwz r8,SVfree+4(0) ; Get the head of the save area list
638
639 sub r3,r2,r5 ; Get number left after we swipe enough for local list
640 srawi r3,r3,31 ; Get 0 if enough or -1 if not
641 andc r4,r5,r3 ; Get number to get if there are enough, 0 otherwise
642 and r5,r2,r3 ; Get 0 if there are enough, number on list otherwise
643 or. r5,r4,r5 ; r5 <- number we will move from global to local list
644 beq- saveGet32NoFree ; There are none to get...
645
646 mtctr r5 ; Get loop count
647 mr r6,r8 ; Remember the first in the list
648
649 saveGet32c:
650 bdz saveGet32d ; Count down and branch when we hit 0...
651 lwz r8,SAVprev+4(r8) ; Get the next
652 b saveGet32c ; Keep going...
653
654 saveGet32d:
655 lwz r3,SAVprev+4(r8) ; Get the next one
656 lwz r4,SVinuse(0) ; Get the in use count
657 sub r2,r2,r5 ; Count down what we stole
658 stw r3,SVfree+4(0) ; Set the new first in list
659 add r4,r4,r5 ; Count the ones we just put in the local list as "in use"
660 stw r2,SVfreecnt(0) ; Set the new count
661 stw r4,SVinuse(0) ; Set the new in use count
662
663 lwz r4,lclfree+4(r10) ; Get the old head of list
664 lwz r3,lclfreecnt(r10) ; Get the old count
665 stw r6,lclfree+4(r10) ; Set the new head of the list
666 add r3,r3,r5 ; Get the new count
667 stw r4,SAVprev+4(r8) ; Point to the old head
668 stw r3,lclfreecnt(r10) ; Set the new count
669
670 bl saveunlock ; Update the adjust field and unlock
671 mtlr r7 ; restore return address
672 b saveGet32 ; Start over and finally allocate the savearea...
673
674 ; The local list is below the repopulate threshold and the global list is empty.
675 ; First we check if there are any left in the local list and if so, we allow
676 ; them to be allocated. If not, we release the backpocket list and choke.
677 ; There is nothing more that we can do at this point. Hopefully we stay alive
678 ; long enough to grab some much-needed panic information.
679 ; r7 = return address to caller of saveGet32
680 ; r10 = per-proc ptr
681
682 saveGet32NoFree:
683 lwz r8,lclfreecnt(r10) ; Get the count
684 mr. r8,r8 ; Are there any reserve to get?
685 beq- saveGet32Choke ; No, go choke and die...
686 bl saveunlock ; Update the adjust field and unlock
687 lwz r3,lclfree+4(r10) ; Get the start of local savearea list
688 lwz r8,lclfreecnt(r10) ; Get the count
689 mtlr r7 ; restore return address
690 b saveGet32GetLocal ; We have some left, dip on in...
691
692 ; We who are about to die salute you. The savearea chain is messed up or
693 ; empty. Add in a few so we have enough to take down the system.
694
695 saveGet32Choke:
696 lis r9,hi16(EXT(backpocket)) ; Get high order of back pocket
697 ori r9,r9,lo16(EXT(backpocket)) ; and low part
698
699 lwz r8,SVfreecnt-saveanchor(r9) ; Get the new number of free elements
700 lwz r7,SVfree+4-saveanchor(r9) ; Get the head of the chain
701 lwz r6,SVinuse(0) ; Get total in the old list
702
703 stw r8,SVfreecnt(0) ; Set the new number of free elements
704 add r6,r6,r8 ; Add in the new ones (why?)
705 stw r7,SVfree+4(0) ; Set the new head of the chain
706 stw r6,SVinuse(0) ; Set total in the new list
707
708 b saveGetChokeJoin
709
710
711 /*
712 * *******************
713 * * s a v e _ r e t *
714 * *******************
715 *
716 * void save_ret(struct savearea *); // normal call
717 * void save_ret_wMSR(struct savearea *,reg64_t); // passes MSR to restore as 2nd arg
718 *
719 * Return a savearea passed by virtual address to the free list.
720 * Note really well: we can take NO exceptions of any kind,
721 * including a PTE miss once the savearea lock is held. That's
722 * a guaranteed deadlock. That means we must disable for interrutions
723 * and turn all translation off.
724 */
725 .globl EXT(save_ret_wMSR) ; alternate entry pt w MSR to restore in r4
726
727 LEXT(save_ret_wMSR)
728 crset 31 ; set flag for save_ret_wMSR
729 b svrt1 ; join common code
730
731 .align 5
732 .globl EXT(save_ret)
733
734 LEXT(save_ret)
735 crclr 31 ; clear flag for save_ret_wMSR
736 svrt1: ; join from save_ret_wMSR
737 mflr r9 ; get return address
738 rlwinm r7,r3,0,0,19 ; get virtual address of SAC area at start of page
739 mr r8,r3 ; save virtual address
740 lwz r5,SACvrswap+0(r7) ; get 64-bit converter from V to R
741 lwz r6,SACvrswap+4(r7) ; both halves, though only bottom used on 32-bit machine
742 #if FPVECDBG
743 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
744 li r2,0x2204 ; (TEST/DEBUG)
745 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
746 sc ; (TEST/DEBUG)
747 #endif
748 bl saveSetup ; turn translation off, 64-bit on, load many regs
749 bf++ 31,svrt3 ; skip if not save_ret_wMSR
750 mr r11,r4 ; was save_ret_wMSR, so overwrite saved MSR
751 svrt3:
752 bf-- pf64Bitb,svrt4 ; skip if a 32-bit processor
753
754 ; Handle 64-bit processor.
755
756 rldimi r6,r5,32,0 ; merge upper and lower halves of SACvrswap together
757 xor r3,r8,r6 ; get r3 <- 64-bit physical address of this savearea
758 bl saveRet64 ; return it
759 mtlr r9 ; restore return address
760 b saveRestore64 ; restore MSR
761
762 ; Handle 32-bit processor.
763
764 svrt4:
765 xor r3,r8,r6 ; get r3 <- 32-bit physical address of this savearea
766 bl saveRet32 ; return it
767 mtlr r9 ; restore return address
768 b saveRestore32 ; restore MSR
769
770
771 /*
772 * *****************************
773 * * s a v e _ r e t _ p h y s *
774 * *****************************
775 *
776 * void save_ret_phys(reg64_t);
777 *
778 * Called from lowmem vectors to return (ie, free) a savearea by physical address.
779 * Translation and interrupts are already off, and 64-bit mode is set if defined.
780 * We can take _no_ exceptions of any kind in this code, including PTE miss, since
781 * that would result in a deadlock. We expect:
782 * r3 = phys addr of savearea
783 * msr = IR, DR, and EE off, SF on
784 * cr6 = pf64Bit flag
785 * We destroy:
786 * r0,r2-r10.
787 */
788 .align 5
789 .globl EXT(save_ret_phys)
790
791 LEXT(save_ret_phys)
792 mfsprg r10,0 ; get the per-proc ptr
793 bf-- pf64Bitb,saveRet32 ; handle 32-bit machine
794 b saveRet64 ; handle 64-bit machine
795
796
797 /*
798 * *********************
799 * * s a v e R e t 6 4 *
800 * *********************
801 *
802 * This is the internal routine to free a savearea, passed by 64-bit physical
803 * address. We assume that IR, DR, and EE are all off, that SF is on, and:
804 * r3 = phys address of the savearea
805 * r10 = per-proc ptr
806 * We destroy:
807 * r0,r2-r8.
808 */
809 .align 5
810 saveRet64:
811 li r0,SAVempty ; Get marker for free savearea
812 lwz r7,lclfreecnt(r10) ; Get the local count
813 ld r6,lclfree(r10) ; Get the old local header
814 addi r7,r7,1 ; Pop up the free count
815 std r6,SAVprev(r3) ; Plant free chain pointer
816 cmplwi r7,LocalSaveMax ; Has the list gotten too long?
817 stb r0,SAVflags+2(r3) ; Mark savearea free
818 std r3,lclfree(r10) ; Chain us on in
819 stw r7,lclfreecnt(r10) ; Bump up the count
820 bltlr++ ; List not too long, so done
821
822 /* The local savearea chain has gotten too long. Trim it down to the target.
823 * Here's a tricky bit, and important:
824 *
825 * When we trim the list, we NEVER trim the very first one. This is because that is
826 * the very last one released and the exception exit code will release the savearea
827 * BEFORE it is done using it. Wouldn't be too good if another processor started
828 * using it, eh? So for this case, we are safe so long as the savearea stays on
829 * the local list. (Note: the exit routine needs to do this because it is in the
830 * process of restoring all context and it needs to keep it until the last second.)
831 */
832
833 mflr r0 ; save return to caller of saveRet64
834 mr r2,r3 ; r2 <- 1st one on local list, which must not be trimmed
835 ld r3,SAVprev(r3) ; Skip over the first
836 subi r7,r7,LocalSaveTarget ; Figure out how much to trim
837 mr r6,r3 ; r6 <- first one to trim
838 mr r5,r7 ; Save the number we are trimming
839
840 saveRet64a:
841 addic. r7,r7,-1 ; Any left to do?
842 ble-- saveRet64b ; Nope...
843 ld r3,SAVprev(r3) ; Skip to the next one
844 b saveRet64a ; Keep going...
845
846 saveRet64b: ; r3 <- last one to trim
847 ld r7,SAVprev(r3) ; Point to the first one not to trim
848 li r4,LocalSaveTarget ; Set the target count
849 std r7,SAVprev(r2) ; Trim stuff leaving the one just released as first
850 stw r4,lclfreecnt(r10) ; Set the current count
851
852 bl savelock ; Lock up the anchor
853
854 ld r8,SVfree(0) ; Get the old head of the free list
855 lwz r4,SVfreecnt(0) ; Get the number of free ones
856 lwz r7,SVinuse(0) ; Get the number that are in use
857 std r6,SVfree(0) ; Point to the first trimmed savearea
858 add r4,r4,r5 ; Add number trimmed to free count
859 std r8,SAVprev(r3) ; Chain the old head to the tail of the trimmed guys
860 sub r7,r7,r5 ; Remove the trims from the in use count
861 stw r4,SVfreecnt(0) ; Set new free count
862 stw r7,SVinuse(0) ; Set new in use count
863
864 mtlr r0 ; Restore the return to our caller
865 b saveunlock ; Set adjust count, unlock the saveanchor, and return
866
867
868 /*
869 * *********************
870 * * s a v e R e t 3 2 *
871 * *********************
872 *
873 * This is the internal routine to free a savearea, passed by 32-bit physical
874 * address. We assume that IR, DR, and EE are all off, and:
875 * r3 = phys address of the savearea
876 * r10 = per-proc ptr
877 * We destroy:
878 * r0,r2-r8.
879 */
880 .align 5
881 saveRet32:
882 li r0,SAVempty ; Get marker for free savearea
883 lwz r7,lclfreecnt(r10) ; Get the local count
884 lwz r6,lclfree+4(r10) ; Get the old local header
885 addi r7,r7,1 ; Pop up the free count
886 stw r6,SAVprev+4(r3) ; Plant free chain pointer
887 cmplwi r7,LocalSaveMax ; Has the list gotten too long?
888 stb r0,SAVflags+2(r3) ; Mark savearea free
889 stw r3,lclfree+4(r10) ; Chain us on in
890 stw r7,lclfreecnt(r10) ; Bump up the count
891 bltlr+ ; List not too long, so done
892
893 /* The local savearea chain has gotten too long. Trim it down to the target.
894 * Here's a tricky bit, and important:
895 *
896 * When we trim the list, we NEVER trim the very first one. This is because that is
897 * the very last one released and the exception exit code will release the savearea
898 * BEFORE it is done using it. Wouldn't be too good if another processor started
899 * using it, eh? So for this case, we are safe so long as the savearea stays on
900 * the local list. (Note: the exit routine needs to do this because it is in the
901 * process of restoring all context and it needs to keep it until the last second.)
902 */
903
904 mflr r0 ; save return to caller of saveRet32
905 mr r2,r3 ; r2 <- 1st one on local list, which must not be trimmed
906 lwz r3,SAVprev+4(r3) ; Skip over the first
907 subi r7,r7,LocalSaveTarget ; Figure out how much to trim
908 mr r6,r3 ; r6 <- first one to trim
909 mr r5,r7 ; Save the number we are trimming
910
911 saveRet32a:
912 addic. r7,r7,-1 ; Any left to do?
913 ble- saveRet32b ; Nope...
914 lwz r3,SAVprev+4(r3) ; Skip to the next one
915 b saveRet32a ; Keep going...
916
917 saveRet32b: ; r3 <- last one to trim
918 lwz r7,SAVprev+4(r3) ; Point to the first one not to trim
919 li r4,LocalSaveTarget ; Set the target count
920 stw r7,SAVprev+4(r2) ; Trim stuff leaving the one just released as first
921 stw r4,lclfreecnt(r10) ; Set the current count
922
923 bl savelock ; Lock up the anchor
924
925 lwz r8,SVfree+4(0) ; Get the old head of the free list
926 lwz r4,SVfreecnt(0) ; Get the number of free ones
927 lwz r7,SVinuse(0) ; Get the number that are in use
928 stw r6,SVfree+4(0) ; Point to the first trimmed savearea
929 add r4,r4,r5 ; Add number trimmed to free count
930 stw r8,SAVprev+4(r3) ; Chain the old head to the tail of the trimmed guys
931 sub r7,r7,r5 ; Remove the trims from the in use count
932 stw r4,SVfreecnt(0) ; Set new free count
933 stw r7,SVinuse(0) ; Set new in use count
934
935 mtlr r0 ; Restore the return to our caller
936 b saveunlock ; Set adjust count, unlock the saveanchor, and return
937
938
939 /*
940 * *******************************
941 * * s a v e _ t r i m _ f r e e *
942 * *******************************
943 *
944 * struct savearea_comm *save_trim_free(void);
945 *
946 * Trim the free list down to the target count, ie by -(SVadjust) save areas.
947 * It trims the list and, if a pool page was fully allocated, puts that page on
948 * the start of the pool list.
949 *
950 * If the savearea being released is the last on a pool page (i.e., all entries
951 * are released), the page is dequeued from the pool and queued to any other
952 * found during this scan. Note that this queue is maintained virtually.
953 *
954 * When the scan is done, the saveanchor lock is released and the list of
955 * freed pool pages is returned to our caller.
956 *
957 * For latency sake we may want to revisit this code. If we are trimming a
958 * large number of saveareas, we could be disabled and holding the savearea lock
959 * for quite a while. It may be that we want to break the trim down into parts.
960 * Possibly trimming the free list, then individually pushing them into the free pool.
961 *
962 * This function expects to be called with translation on and a valid stack.
963 * It uses the standard ABI, ie we destroy r2 and r3-r11, and return the ptr in r3.
964 */
965 .align 5
966 .globl EXT(save_trim_free)
967
968 LEXT(save_trim_free)
969
970 subi r1,r1,(FM_ALIGN(16)+FM_SIZE) ; Make space for 4 registers on stack
971 mflr r9 ; save our return address
972 stw r28,FM_SIZE+0(r1) ; Save R28
973 stw r29,FM_SIZE+4(r1) ; Save R29
974 stw r30,FM_SIZE+8(r1) ; Save R30
975 stw r31,FM_SIZE+12(r1) ; Save R31
976
977 bl saveSetup ; turn off translation and interrupts, load many regs
978 bl savelock ; Go lock up the anchor
979
980 lwz r8,SVadjust(0) ; How many do we need to clear out?
981 li r3,0 ; Get a 0
982 neg. r8,r8 ; Get the actual we need to toss (adjust is neg if too many)
983 ble- save_trim_free1 ; skip if no trimming needed anymore
984 bf-- pf64Bitb,saveTrim32 ; handle 32-bit processors
985 b saveTrim64 ; handle 64-bit processors
986
987 save_trim_free1: ; by the time we were called, no need to trim anymore
988 stw r3,SVlock(0) ; Quick unlock (no need for sync or to set adjust, nothing changed)
989 mtlr r9 ; Restore return
990
991 #if FPVECDBG
992 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
993 li r2,0x2206 ; (TEST/DEBUG)
994 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
995 sc ; (TEST/DEBUG)
996 #endif
997 addi r1,r1,(FM_ALIGN(16)+FM_SIZE); Pop stack - have not trashed register so no need to reload
998 b saveRestore ; restore translation and EE, turn SF off, return to our caller
999
1000
1001 /*
1002 * ***********************
1003 * * s a v e T r i m 3 2 *
1004 * ***********************
1005 *
1006 * Handle "save_trim_free" on 32-bit processors. At this point, translation and interrupts
1007 * are off, the savearea anchor is locked, and:
1008 * r8 = #pages to trim (>0)
1009 * r9 = return address
1010 * r10 = per-proc ptr
1011 * r11 = MSR at entry
1012 */
1013
1014 saveTrim32:
1015 lwz r7,SVfree+4(0) ; Get the first on the free list
1016 mr r6,r7 ; Save the first one
1017 mr r5,r8 ; Save the number we are trimming
1018
1019 sttrimming: addic. r5,r5,-1 ; Any left to do?
1020 ble- sttrimmed ; Nope...
1021 lwz r7,SAVprev+4(r7) ; Skip to the next one
1022 b sttrimming ; Keep going...
1023
1024 sttrimmed: lwz r5,SAVprev+4(r7) ; Get the next one (for new head of free list)
1025 lwz r4,SVfreecnt(0) ; Get the free count
1026 stw r5,SVfree+4(0) ; Set new head
1027 sub r4,r4,r8 ; Calculate the new free count
1028 li r31,0 ; Show we have no free pool blocks yet
1029 crclr cr1_eq ; dont exit loop before 1st iteration
1030 stw r4,SVfreecnt(0) ; Set new free count
1031 lis r30,hi16(sac_empty) ; Get what empty looks like
1032
1033 ; NOTE: The savearea size must be 640 (0x280). We are doing a divide by shifts and stuff
1034 ; here.
1035 ;
1036 #if SAVsize != 640
1037 #error Savearea size is not 640!!!!!!!!!!!!
1038 #endif
1039
1040 ; Loop over each savearea we are trimming.
1041 ; r6 = next savearea to trim
1042 ; r7 = last savearea to trim
1043 ; r8 = #pages to trim (>0)
1044 ; r9 = return address
1045 ; r10 = per-proc ptr
1046 ; r11 = MSR at entry
1047 ; r30 = what SACalloc looks like when all saveareas are free
1048 ; r31 = free pool block list
1049 ; cr1 = beq set if we just trimmed the last, ie if we are done
1050
1051 sttoss: beq+ cr1,stdone ; All done now...
1052
1053 cmplw cr1,r6,r7 ; Have we finished the loop?
1054
1055 lis r0,0x0044 ; Get top of table
1056 rlwinm r2,r6,0,0,19 ; Back down to the savearea control stuff
1057 ori r0,r0,0x2200 ; Finish shift table
1058 rlwinm r4,r6,25,27,30 ; Get (addr >> 7) & 0x1E (same as twice high nybble)
1059 lwz r5,SACalloc(r2) ; Get the allocation bits
1060 addi r4,r4,1 ; Shift 1 extra
1061 rlwinm r3,r6,25,31,31 ; Get (addr >> 7) & 1
1062 rlwnm r0,r0,r4,29,31 ; Get partial index
1063 lis r4,lo16(0x8000) ; Get the bit mask
1064 add r0,r0,r3 ; Make the real index
1065 srw r4,r4,r0 ; Get the allocation mask
1066 or r5,r5,r4 ; Free this entry
1067 cmplw r5,r4 ; Is this the only free entry?
1068 lwz r6,SAVprev+4(r6) ; Chain to the next trimmed savearea
1069 cmplw cr7,r30,r5 ; Does this look empty?
1070 stw r5,SACalloc(r2) ; Save back the allocation bits
1071 beq- stputpool ; First free entry, go put it into the pool...
1072 bne+ cr7,sttoss ; Not an empty block
1073
1074 ;
1075 ; We have an empty block. Remove it from the pool list.
1076 ;
1077
1078 lwz r29,SACflags(r2) ; Get the flags
1079 cmplwi cr5,r31,0 ; Is this guy on the release list?
1080 lwz r28,SACnext+4(r2) ; Get the forward chain
1081
1082 rlwinm. r0,r29,0,sac_permb,sac_permb ; Is this a permanently allocated area? (also sets 0 needed below)
1083 bne- sttoss ; This is permanent entry, do not try to release...
1084
1085 lwz r29,SACprev+4(r2) ; and the previous
1086 beq- cr5,stnot1st ; Not first
1087 lwz r0,SACvrswap+4(r31) ; Load the previous pool page vr conversion
1088
1089 stnot1st: stw r28,SACnext+4(r29) ; Previous guy points to my next
1090 xor r0,r0,r31 ; Make the last guy virtual
1091 stw r29,SACprev+4(r28) ; Next guy points back to my previous
1092 stw r0,SAVprev+4(r2) ; Store the old top virtual as my back chain
1093 mr r31,r2 ; My physical is now the head of the chain
1094 b sttoss ; Get the next one...
1095
1096 ;
1097 ; A pool block that had no free entries now has one. Stick it on the pool list.
1098 ;
1099
1100 stputpool: lwz r28,SVpoolfwd+4(0) ; Get the first guy on the list
1101 li r0,saveanchor ; Point to the saveanchor
1102 stw r2,SVpoolfwd+4(0) ; Put us on the top of the list
1103 stw r28,SACnext+4(r2) ; We point to the old top
1104 stw r2,SACprev+4(r28) ; Old top guy points back to us
1105 stw r0,SACprev+4(r2) ; Our back points to the anchor
1106 b sttoss ; Go on to the next one...
1107
1108
1109 /*
1110 * ***********************
1111 * * s a v e T r i m 6 4 *
1112 * ***********************
1113 *
1114 * Handle "save_trim_free" on 64-bit processors. At this point, translation and interrupts
1115 * are off, SF is on, the savearea anchor is locked, and:
1116 * r8 = #pages to trim (>0)
1117 * r9 = return address
1118 * r10 = per-proc ptr
1119 * r11 = MSR at entry
1120 */
1121
1122 saveTrim64:
1123 ld r7,SVfree(0) ; Get the first on the free list
1124 mr r6,r7 ; Save the first one
1125 mr r5,r8 ; Save the number we are trimming
1126
1127 sttrimming64:
1128 addic. r5,r5,-1 ; Any left to do?
1129 ble-- sttrimmed64 ; Nope...
1130 ld r7,SAVprev(r7) ; Skip to the next one
1131 b sttrimming64 ; Keep going...
1132
1133 sttrimmed64:
1134 ld r5,SAVprev(r7) ; Get the next one (for new head of free list)
1135 lwz r4,SVfreecnt(0) ; Get the free count
1136 std r5,SVfree(0) ; Set new head
1137 sub r4,r4,r8 ; Calculate the new free count
1138 li r31,0 ; Show we have no free pool blocks yet
1139 crclr cr1_eq ; dont exit loop before 1st iteration
1140 stw r4,SVfreecnt(0) ; Set new free count
1141 lis r30,hi16(sac_empty) ; Get what empty looks like
1142
1143
1144 ; Loop over each savearea we are trimming.
1145 ; r6 = next savearea to trim
1146 ; r7 = last savearea to trim
1147 ; r8 = #pages to trim (>0)
1148 ; r9 = return address
1149 ; r10 = per-proc ptr
1150 ; r11 = MSR at entry
1151 ; r30 = what SACalloc looks like when all saveareas are free
1152 ; r31 = free pool block list
1153 ; cr1 = beq set if we just trimmed the last, ie if we are done
1154 ;
1155 ; WARNING: as in the 32-bit path, this code is doing a divide by 640 (SAVsize).
1156
1157 sttoss64:
1158 beq++ cr1,stdone ; All done now...
1159
1160 cmpld cr1,r6,r7 ; Have we finished the loop?
1161
1162 lis r0,0x0044 ; Get top of table
1163 rldicr r2,r6,0,51 ; r2 <- phys addr of savearea block (with control area)
1164 ori r0,r0,0x2200 ; Finish shift table
1165 rlwinm r4,r6,25,27,30 ; Get (addr >> 7) & 0x1E (same as twice high nybble)
1166 lwz r5,SACalloc(r2) ; Get the allocation bits
1167 addi r4,r4,1 ; Shift 1 extra
1168 rlwinm r3,r6,25,31,31 ; Get (addr >> 7) & 1
1169 rlwnm r0,r0,r4,29,31 ; Get partial index
1170 lis r4,lo16(0x8000) ; Get the bit mask
1171 add r0,r0,r3 ; Make the real index
1172 srw r4,r4,r0 ; Get the allocation mask
1173 or r5,r5,r4 ; Free this entry
1174 cmplw r5,r4 ; Is this the only free entry?
1175 ld r6,SAVprev(r6) ; Chain to the next trimmed savearea
1176 cmplw cr7,r30,r5 ; Does this look empty?
1177 stw r5,SACalloc(r2) ; Save back the allocation bits
1178 beq-- stputpool64 ; First free entry, go put it into the pool...
1179 bne++ cr7,sttoss64 ; Not an empty block
1180
1181 ; We have an empty block. Remove it from the pool list.
1182
1183 lwz r29,SACflags(r2) ; Get the flags
1184 cmpldi cr5,r31,0 ; Is this guy on the release list?
1185 ld r28,SACnext(r2) ; Get the forward chain
1186
1187 rlwinm. r0,r29,0,sac_permb,sac_permb ; Is this a permanently allocated area? (also sets 0 needed below)
1188 bne-- sttoss64 ; This is permanent entry, do not try to release...
1189
1190 ld r29,SACprev(r2) ; and the previous
1191 beq-- cr5,stnot1st64 ; Not first
1192 ld r0,SACvrswap(r31) ; Load the previous pool page vr conversion
1193
1194 stnot1st64:
1195 std r28,SACnext(r29) ; Previous guy points to my next
1196 xor r0,r0,r31 ; Make the last guy virtual
1197 std r29,SACprev(r28) ; Next guy points back to my previous
1198 std r0,SAVprev(r2) ; Store the old top virtual as my back chain
1199 mr r31,r2 ; My physical is now the head of the chain
1200 b sttoss64 ; Get the next one...
1201
1202 ; A pool block that had no free entries now has one. Stick it on the pool list.
1203
1204 stputpool64:
1205 ld r28,SVpoolfwd(0) ; Get the first guy on the list
1206 li r0,saveanchor ; Point to the saveanchor
1207 std r2,SVpoolfwd(0) ; Put us on the top of the list
1208 std r28,SACnext(r2) ; We point to the old top
1209 std r2,SACprev(r28) ; Old top guy points back to us
1210 std r0,SACprev(r2) ; Our back points to the anchor
1211 b sttoss64 ; Go on to the next one...
1212
1213
1214 ; We are all done. Relocate pool release head, restore all, and go. This code
1215 ; is used both by the 32 and 64-bit paths.
1216 ; r9 = return address
1217 ; r10 = per-proc ptr
1218 ; r11 = MSR at entry
1219 ; r31 = free pool block list
1220
1221 stdone: bl saveunlock ; Unlock the saveanchor and set adjust field
1222
1223 mr. r3,r31 ; Move release chain and see if there are any
1224 li r5,0 ; Assume either V=R or no release chain
1225 beq- stnorel ; Nothing to release...
1226 lwz r5,SACvrswap+4(r31) ; Get the vr conversion (only need low half if 64-bit)
1227
1228 stnorel:
1229 bl saveRestore ; restore translation and exceptions, turn off SF
1230 mtlr r9 ; Restore the return
1231
1232 lwz r28,FM_SIZE+0(r1) ; Restore R28
1233 lwz r29,FM_SIZE+4(r1) ; Restore R29
1234 lwz r30,FM_SIZE+8(r1) ; Restore R30
1235 lwz r31,FM_SIZE+12(r1) ; Restore R31
1236 addi r1,r1,(FM_ALIGN(16)+FM_SIZE) ; Pop the stack
1237 xor r3,r3,r5 ; Convert release chain address to virtual
1238 rlwinm r3,r3,0,0,31 ; if 64-bit, clear upper half of virtual address
1239
1240 #if FPVECDBG
1241 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1242 li r2,0x2207 ; (TEST/DEBUG)
1243 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1244 sc ; (TEST/DEBUG)
1245 #endif
1246 blr ; Return...
1247
1248
1249 /*
1250 * ***************************
1251 * * s a v e _ r e c o v e r *
1252 * ***************************
1253 *
1254 * int save_recover(void);
1255 *
1256 * Returns nonzero if we can get enough saveareas to hit the target. We scan the free
1257 * pool. If we empty a pool block, we remove it from the pool list.
1258 */
1259
1260 .align 5
1261 .globl EXT(save_recover)
1262
1263 LEXT(save_recover)
1264 mflr r9 ; save return address
1265 bl saveSetup ; turn translation and interrupts off, SF on, load many regs
1266 bl savelock ; lock the savearea anchor
1267
1268 lwz r8,SVadjust(0) ; How many do we need to clear get?
1269 li r3,0 ; Get a 0
1270 mr. r8,r8 ; Do we need any?
1271 ble-- save_recover1 ; not any more
1272 bf-- pf64Bitb,saveRecover32 ; handle 32-bit processor
1273 b saveRecover64 ; handle 64-bit processor
1274
1275 save_recover1: ; by the time we locked the anchor, no longer short
1276 mtlr r9 ; Restore return
1277 stw r3,SVlock(0) ; Quick unlock (no need for sync or to set adjust, nothing changed)
1278 #if FPVECDBG
1279 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1280 li r2,0x2208 ; (TEST/DEBUG)
1281 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1282 sc ; (TEST/DEBUG)
1283 #endif
1284 b saveRestore ; turn translation etc back on, return to our caller
1285
1286
1287 /*
1288 * *****************************
1289 * * s a v e R e c o v e r 3 2 *
1290 * *****************************
1291 *
1292 * Handle "save_recover" on 32-bit processors. At this point, translation and interrupts
1293 * are off, the savearea anchor is locked, and:
1294 * r8 = #pages to recover
1295 * r9 = return address
1296 * r10 = per-proc ptr
1297 * r11 = MSR at entry
1298 */
1299
1300 saveRecover32:
1301 li r6,saveanchor ; Start at pool anchor
1302 crclr cr1_eq ; initialize the loop test
1303 lwz r7,SVfreecnt(0) ; Get the current free count
1304
1305
1306 ; Loop over next block in free pool. r6 is the ptr to the last block we looked at.
1307
1308 srcnpool: lwz r6,SACnext+4(r6) ; Point to the next one
1309 cmplwi r6,saveanchor ; Have we wrapped?
1310 beq- srcdone ; Yes, did not have enough...
1311
1312 lwz r5,SACalloc(r6) ; Pick up the allocation for this pool block
1313
1314 ;
1315 ; NOTE: The savearea size must be 640 (0x280). We are doing a multiply by shifts and add.
1316 ; offset = (index << 9) + (index << 7)
1317 ;
1318 #if SAVsize != 640
1319 #error Savearea size is not 640!!!!!!!!!!!!
1320 #endif
1321
1322 ; Loop over free savearea in current block.
1323 ; r5 = bitmap of free saveareas in block at r6 (ie, SACalloc)
1324 ; r6 = ptr to current free pool block
1325 ; r7 = free count
1326 ; r8 = #pages more we still need to recover
1327 ; r9 = return address
1328 ; r10 = per-proc ptr
1329 ; r11 = MSR at entry
1330 ; cr1 = beq if (r8==0)
1331
1332 srcnext: beq- cr1,srcdone ; We have no more to get...
1333
1334 lis r3,0x8000 ; Get the top bit on
1335 cntlzw r4,r5 ; Find a free slot
1336 addi r7,r7,1 ; Bump up the free count
1337 srw r3,r3,r4 ; Make a mask
1338 slwi r0,r4,7 ; First multiply by 128
1339 subi r8,r8,1 ; Decrement the need count
1340 slwi r2,r4,9 ; Then multiply by 512
1341 andc. r5,r5,r3 ; Clear out the "free" bit
1342 add r2,r2,r0 ; Sum to multiply by 640
1343
1344 stw r5,SACalloc(r6) ; Set new allocation bits
1345
1346 add r2,r2,r6 ; Get the actual address of the savearea
1347 lwz r3,SVfree+4(0) ; Get the head of the chain
1348 cmplwi cr1,r8,0 ; Do we actually need any more?
1349 stw r2,SVfree+4(0) ; Push ourselves in the front
1350 stw r3,SAVprev+4(r2) ; Chain the rest of the list behind
1351
1352 bne+ srcnext ; The pool block is not empty yet, try for another...
1353
1354 lwz r2,SACnext+4(r6) ; Get the next pointer
1355 lwz r3,SACprev+4(r6) ; Get the previous pointer
1356 stw r3,SACprev+4(r2) ; The previous of my next points to my previous
1357 stw r2,SACnext+4(r3) ; The next of my previous points to my next
1358 bne+ cr1,srcnpool ; We still have more to do...
1359
1360
1361 ; Join here from 64-bit path when we have recovered all the saveareas we need to.
1362
1363 srcdone: stw r7,SVfreecnt(0) ; Set the new free count
1364 bl saveunlock ; Unlock the save and set adjust field
1365
1366 mtlr r9 ; Restore the return
1367 #if FPVECDBG
1368 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1369 li r2,0x2209 ; (TEST/DEBUG)
1370 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1371 sc ; (TEST/DEBUG)
1372 #endif
1373 b saveRestore ; turn xlate and EE back on, SF off, and return to our caller
1374
1375
1376 /*
1377 * *****************************
1378 * * s a v e R e c o v e r 6 4 *
1379 * *****************************
1380 *
1381 * Handle "save_recover" on 64-bit processors. At this point, translation and interrupts
1382 * are off, the savearea anchor is locked, and:
1383 * r8 = #pages to recover
1384 * r9 = return address
1385 * r10 = per-proc ptr
1386 * r11 = MSR at entry
1387 */
1388
1389 saveRecover64:
1390 li r6,saveanchor ; Start at pool anchor
1391 crclr cr1_eq ; initialize the loop test
1392 lwz r7,SVfreecnt(0) ; Get the current free count
1393
1394
1395 ; Loop over next block in free pool. r6 is the ptr to the last block we looked at.
1396
1397 srcnpool64:
1398 ld r6,SACnext(r6) ; Point to the next one
1399 cmpldi r6,saveanchor ; Have we wrapped?
1400 beq-- srcdone ; Yes, did not have enough...
1401
1402 lwz r5,SACalloc(r6) ; Pick up the allocation for this pool block
1403
1404
1405 ; Loop over free savearea in current block.
1406 ; r5 = bitmap of free saveareas in block at r6 (ie, SACalloc)
1407 ; r6 = ptr to current free pool block
1408 ; r7 = free count
1409 ; r8 = #pages more we still need to recover
1410 ; r9 = return address
1411 ; r10 = per-proc ptr
1412 ; r11 = MSR at entry
1413 ; cr1 = beq if (r8==0)
1414 ;
1415 ; WARNING: as in the 32-bit path, we depend on (SAVsize==640)
1416
1417 srcnext64:
1418 beq-- cr1,srcdone ; We have no more to get...
1419
1420 lis r3,0x8000 ; Get the top bit on
1421 cntlzw r4,r5 ; Find a free slot
1422 addi r7,r7,1 ; Bump up the free count
1423 srw r3,r3,r4 ; Make a mask
1424 slwi r0,r4,7 ; First multiply by 128
1425 subi r8,r8,1 ; Decrement the need count
1426 slwi r2,r4,9 ; Then multiply by 512
1427 andc. r5,r5,r3 ; Clear out the "free" bit
1428 add r2,r2,r0 ; Sum to multiply by 640
1429
1430 stw r5,SACalloc(r6) ; Set new allocation bits
1431
1432 add r2,r2,r6 ; Get the actual address of the savearea
1433 ld r3,SVfree(0) ; Get the head of the chain
1434 cmplwi cr1,r8,0 ; Do we actually need any more?
1435 std r2,SVfree(0) ; Push ourselves in the front
1436 std r3,SAVprev(r2) ; Chain the rest of the list behind
1437
1438 bne++ srcnext64 ; The pool block is not empty yet, try for another...
1439
1440 ld r2,SACnext(r6) ; Get the next pointer
1441 ld r3,SACprev(r6) ; Get the previous pointer
1442 std r3,SACprev(r2) ; The previous of my next points to my previous
1443 std r2,SACnext(r3) ; The next of my previous points to my next
1444 bne++ cr1,srcnpool64 ; We still have more to do...
1445
1446 b srcdone
1447
1448
1449 /*
1450 * *******************
1451 * * s a v e l o c k *
1452 * *******************
1453 *
1454 * Lock the savearea anchor, so we can manipulate the free list.
1455 * msr = interrupts and translation off
1456 * We destroy:
1457 * r8, r3, r12
1458 */
1459 .align 5
1460
1461 savelock: lwz r8,SVlock(0) ; See if lock is held
1462 cmpwi r8,0
1463 li r12,saveanchor ; Point to the saveanchor
1464 bne-- savelock ; loop until lock released...
1465
1466 savelock0: lwarx r8,0,r12 ; Grab the lock value
1467 cmpwi r8,0 ; taken?
1468 li r8,1 ; get nonzero to lock it with
1469 bne-- savelock1 ; already locked, wait for it to clear...
1470 stwcx. r8,0,r12 ; Try to seize that there durn lock
1471 isync ; assume we got it
1472 beqlr++ ; reservation not lost, so we have the lock
1473 b savelock0 ; Try again...
1474
1475 savelock1: li r8,lgKillResv ; Point to killing field
1476 stwcx. r8,0,r8 ; Kill reservation
1477 b savelock ; Start over....
1478
1479
1480 /*
1481 * ***********************
1482 * * s a v e u n l o c k *
1483 * ***********************
1484 *
1485 *
1486 * This is the common routine that sets the saveadjust field and unlocks the savearea
1487 * anchor.
1488 * msr = interrupts and translation off
1489 * We destroy:
1490 * r2, r5, r6, r8.
1491 */
1492 .align 5
1493 saveunlock:
1494 lwz r6,SVfreecnt(0) ; and the number on the free list
1495 lwz r5,SVinuse(0) ; Pick up the in use count
1496 subic. r8,r6,FreeListMin ; do we have at least the minimum?
1497 lwz r2,SVtarget(0) ; Get the target
1498 neg r8,r8 ; assuming we are short, get r8 <- shortfall
1499 blt-- saveunlock1 ; skip if fewer than minimum on free list
1500
1501 add r6,r6,r5 ; Get the total number of saveareas
1502 addi r5,r2,-SaveLowHysteresis ; Find low end of acceptible range
1503 sub r5,r6,r5 ; Make everything below hysteresis negative
1504 sub r2,r2,r6 ; Get the distance from the target
1505 addi r5,r5,-(SaveLowHysteresis + SaveHighHysteresis + 1) ; Subtract full hysteresis range
1506 srawi r5,r5,31 ; Get 0xFFFFFFFF if outside range or 0 if inside
1507 and r8,r2,r5 ; r8 <- 0 if in range or distance to target if not
1508
1509 saveunlock1:
1510 li r5,0 ; Set a clear value
1511 stw r8,SVadjust(0) ; Set the adjustment value
1512 eieio ; Make sure everything is done
1513 stw r5,SVlock(0) ; Unlock the savearea chain
1514 blr
1515
1516
1517 /*
1518 * *******************
1519 * * s a v e _ c p v *
1520 * *******************
1521 *
1522 * struct savearea *save_cpv(addr64_t saveAreaPhysAddr);
1523 *
1524 * Converts a physical savearea address to virtual. Called with translation on
1525 * and in 32-bit mode. Note that the argument is passed as a long long in (r3,r4).
1526 */
1527
1528 .align 5
1529 .globl EXT(save_cpv)
1530
1531 LEXT(save_cpv)
1532 mflr r9 ; save return address
1533 mr r8,r3 ; save upper half of phys address here
1534 bl saveSetup ; turn off translation and interrupts, turn SF on
1535 rlwinm r5,r4,0,0,19 ; Round back to the start of the physical savearea block
1536 bf-- pf64Bitb,save_cpv1 ; skip if 32-bit processor
1537 rldimi r5,r8,32,0 ; r5 <- 64-bit phys address of block
1538 save_cpv1:
1539 lwz r6,SACvrswap+4(r5) ; Get the conversion to virtual (only need low half if 64-bit)
1540 mtlr r9 ; restore return address
1541 xor r3,r4,r6 ; convert phys to virtual
1542 rlwinm r3,r3,0,0,31 ; if 64-bit, zero upper half of virtual address
1543 b saveRestore ; turn translation etc back on, SF off, and return r3
1544
1545
1546 /*
1547 * *********************
1548 * * s a v e S e t u p *
1549 * *********************
1550 *
1551 * This routine is called at the start of all the save-area subroutines.
1552 * It turns off translation, disabled interrupts, turns on 64-bit mode,
1553 * and sets up cr6 with the feature flags (especially pf64Bit).
1554 *
1555 * Note that most save-area routines cannot take _any_ interrupt (such as a
1556 * PTE miss) once the savearea anchor is locked, since that would result in
1557 * instant deadlock as we need a save-area to process any exception.
1558 * We set up:
1559 * r10 = per-proc ptr
1560 * r11 = old MSR
1561 * cr5 = pfNoMSRir feature flag
1562 * cr6 = pf64Bit feature flag
1563 *
1564 * We use r0, r3, r10, and r11.
1565 */
1566
1567 saveSetup:
1568 mfmsr r11 ; get msr
1569 mfsprg r3,2 ; get feature flags
1570 li r0,0
1571 mtcrf 0x2,r3 ; copy pf64Bit to cr6
1572 ori r0,r0,lo16(MASK(MSR_IR)+MASK(MSR_DR)+MASK(MSR_EE))
1573 mtcrf 0x4,r3 ; copy pfNoMSRir to cr5
1574 andc r3,r11,r0 ; turn off IR, DR, and EE
1575 li r0,1 ; get a 1 in case its a 64-bit machine
1576 bf-- pf64Bitb,saveSetup1 ; skip if not a 64-bit machine
1577 rldimi r3,r0,63,MSR_SF_BIT ; turn SF (bit 0) on
1578 mtmsrd r3 ; turn translation and interrupts off, 64-bit mode on
1579 isync ; wait for it to happen
1580 mfsprg r10,0 ; get per-proc ptr
1581 blr
1582 saveSetup1: ; here on 32-bit machines
1583 bt- pfNoMSRirb,saveSetup2 ; skip if cannot turn off IR with a mtmsr
1584 mtmsr r3 ; turn translation and interrupts off
1585 isync ; wait for it to happen
1586 mfsprg r10,0 ; get per-proc ptr
1587 blr
1588 saveSetup2: ; here if pfNoMSRir set for this machine
1589 li r0,loadMSR ; we will "mtmsr r3" via system call
1590 sc
1591 mfsprg r10,0 ; get per-proc ptr
1592 blr
1593
1594
1595 /*
1596 * *************************
1597 * * s a v e R e s t o r e *
1598 * *************************
1599 *
1600 * Undoes the effect of calling "saveSetup", ie it turns relocation and interrupts back on,
1601 * and turns 64-bit mode back off.
1602 * r11 = old MSR
1603 * cr6 = pf64Bit feature flag
1604 */
1605
1606 saveRestore:
1607 bt++ pf64Bitb,saveRestore64 ; handle a 64-bit processor
1608 saveRestore32:
1609 mtmsr r11 ; restore MSR
1610 isync ; wait for translation to start up
1611 blr
1612 saveRestore64: ; 64-bit processor
1613 mtmsrd r11 ; restore MSR
1614 isync ; wait for changes to happen
1615 blr
1616
mirror of XNU