]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ppc/cswtch.s
projects / apple / xnu.git / blob
? search:


8cbf06cfbcee70e9b5c4eee3ea47556c1f031804
[apple/xnu.git] / osfmk / ppc / cswtch.s
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
7 *
8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * file.
14 *
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
22 *
23 * @APPLE_LICENSE_HEADER_END@
24 */
25 /*
26 * @OSF_COPYRIGHT@
27 */
28
29 #include <ppc/asm.h>
30 #include <ppc/proc_reg.h>
31 #include <cpus.h>
32 #include <assym.s>
33 #include <debug.h>
34 #include <mach/ppc/vm_param.h>
35 #include <ppc/exception.h>
36 #include <ppc/savearea.h>
37
38 #define FPVECDBG 0
39 #define GDDBG 0
40
41 .text
42
43 /*
44 * void machine_load_context(thread_t thread)
45 *
46 * Load the context for the first thread to run on a
47 * cpu, and go.
48 */
49
50 .align 5
51 .globl EXT(machine_load_context)
52
53 LEXT(machine_load_context)
54 mfsprg r6,0
55 lwz r0,PP_INTSTACK_TOP_SS(r6)
56 stw r0,PP_ISTACKPTR(r6)
57 lwz r9,THREAD_TOP_ACT(r3) /* Set up the current thread */
58 mtsprg 1,r9
59 li r0,0 /* Clear a register */
60 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
61 mfmsr r5 /* Since we are passing control, get our MSR values */
62 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
63 lwz r1,saver1+4(r3) /* Load new stack pointer */
64 lwz r10,ACT_MACT_SPF(r9) /* Get the special flags */
65 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
66 stw r0,FM_BACKPTR(r1) /* zero backptr */
67 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
68 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
69 oris r10,r10,hi16(OnProc) /* Set OnProc bit */
70 stw r0,ACT_PREEMPT_CNT(r9) /* Enable preemption */
71 stw r10,ACT_MACT_SPF(r9) /* Update the special flags */
72 stw r10,spcFlags(r6) /* Set per_proc copy of the special flags */
73 b EXT(exception_exit) /* Go for it */
74
75 /* thread_t Switch_context(thread_t old,
76 * void (*cont)(void),
77 * thread_t new)
78 *
79 * Switch from one thread to another. If a continuation is supplied, then
80 * we do not need to save callee save registers.
81 *
82 */
83
84 /* void Call_continuation( void (*continuation)(void), vm_offset_t stack_ptr)
85 */
86
87 .align 5
88 .globl EXT(Call_continuation)
89
90 LEXT(Call_continuation)
91
92 mtlr r3
93 mr r1, r4 /* Load new stack pointer */
94 blr /* Jump to the continuation */
95
96 /*
97 * Get the old kernel stack, and store into the thread structure.
98 * See if a continuation is supplied, and skip state save if so.
99 *
100 * Note that interrupts must be disabled before we get here (i.e., splsched)
101 */
102
103 /* Context switches are double jumps. We pass the following to the
104 * context switch firmware call:
105 *
106 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
107 * R4 = old thread
108 * R5 = new SRR0
109 * R6 = new SRR1
110 * R7 = high order physical address of savearea for full switch
111 *
112 * savesrr0 is set to go to switch_in
113 * savesrr1 is set to uninterruptible with translation on
114 */
115
116
117 .align 5
118 .globl EXT(Switch_context)
119
120 LEXT(Switch_context)
121
122 mfsprg r12,0 ; Get the per_proc block
123 #if DEBUG
124 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
125 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
126 bne++ notonintstack ; (DEBUG/TRACE) stack
127 BREAKPOINT_TRAP
128 notonintstack:
129 #endif
130 lwz r5,THREAD_TOP_ACT(r5) ; Get the new activation
131 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
132 lwz r9,cioSpace(r5) ; Get copyin/out address space
133 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
134 lwz r7,CTHREAD_SELF(r5) ; Pick up the user assist word
135 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
136 lwz r6,cioRelo(r5) ; Get copyin/out relocation top
137 mtsprg 1,r5
138 lwz r2,cioRelo+4(r5) ; Get copyin/out relocation bottom
139
140 stw r7,UAW(r12) ; Save the assist word for the "ultra fast path"
141
142 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
143
144 sth r9,ppCIOmp+mpSpace(r12) ; Save the space
145 stw r6,ppCIOmp+mpNestReloc(r12) ; Save top part of physical address
146 stw r2,ppCIOmp+mpNestReloc+4(r12) ; Save bottom part of physical address
147 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
148 lwz r2,traceMask(0) ; Get the enabled traces
149 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
150 lis r0,hi16(CutTrace) ; Trace FW call
151 mr. r2,r2 ; Any tracing going on?
152 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
153 ori r0,r0,lo16(CutTrace) ; Trace FW call
154 beq++ cswNoTrc ; No trace today, dude...
155 mr r10,r3 ; Save across trace
156 lwz r2,THREAD_TOP_ACT(r3) ; Trace old activation
157 mr r3,r11 ; Trace prev savearea
158 sc ; Cut trace entry of context switch
159 mr r3,r10 ; Restore
160
161 cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
162 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
163 lwz r9,VMXowner(r12) ; Grab the owner of the vector
164 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
165 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
166 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
167 li r0,1 ; Get set to hold off quickfret
168
169 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
170 cmplw r10,r2 ; Do we have the live float context?
171 lwz r10,FPUlevel(r2) ; Get the live level
172 mr r4,r3 ; Save our old thread to pass back
173 cmplw cr5,r9,r2 ; Do we have the live vector context?
174 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
175 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
176 bne++ cswnofloat ; Float is not ours...
177
178 cmplw r10,r11 ; Is the level the same?
179 lwz r5,FPUcpu(r2) ; Get the owning cpu
180 bne++ cswnofloat ; Level not the same, this is not live...
181
182 cmplw r5,r0 ; Still owned by this cpu?
183 lwz r10,FPUsave(r2) ; Get the level
184 bne++ cswnofloat ; CPU claimed by someone else...
185
186 mr. r10,r10 ; Is there a savearea here?
187 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
188
189 beq-- cswnofloat ; No savearea to check...
190
191 lwz r3,SAVlevel(r10) ; Get the level
192 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
193 cmplw r3,r11 ; Is it for the current level?
194
195 bne++ cswnofloat ; Nope...
196
197 stw r5,FPUsave(r2) ; Pop off this savearea
198
199 rlwinm r3,r10,0,0,19 ; Move back to start of page
200
201 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
202 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
203 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
204 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
205 stw r5,SAVprev(r10) ; Link the old in (top)
206 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
207 xor r3,r10,r3 ; Convert to physical
208 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
209 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
210
211 #if FPVECDBG
212 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
213 mr r7,r2 ; (TEST/DEBUG)
214 li r2,0x4401 ; (TEST/DEBUG)
215 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
216 sc ; (TEST/DEBUG)
217 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
218 mr r2,r7 ; (TEST/DEBUG)
219 #endif
220
221 cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
222
223 lwz r10,VMXlevel(r2) ; Get the live level
224
225 cmplw r10,r11 ; Is the level the same?
226 lwz r5,VMXcpu(r2) ; Get the owning cpu
227 bne++ cswnovect ; Level not the same, this is not live...
228
229 cmplw r5,r0 ; Still owned by this cpu?
230 lwz r10,VMXsave(r2) ; Get the level
231 bne++ cswnovect ; CPU claimed by someone else...
232
233 mr. r10,r10 ; Is there a savearea here?
234 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
235
236 beq-- cswnovect ; No savearea to check...
237
238 lwz r3,SAVlevel(r10) ; Get the level
239 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
240 cmplw r3,r11 ; Is it for the current level?
241
242 bne++ cswnovect ; Nope...
243
244 stw r5,VMXsave(r2) ; Pop off this savearea
245 rlwinm r3,r10,0,0,19 ; Move back to start of page
246
247 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
248 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
249 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
250 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
251 stw r5,SAVprev(r10) ; Link the old in (top)
252 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
253 xor r3,r10,r3 ; Convert to physical
254 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
255 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
256
257 #if FPVECDBG
258 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
259 li r2,0x4501 ; (TEST/DEBUG)
260 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
261 sc ; (TEST/DEBUG)
262 #endif
263
264 cswnovect: li r0,0 ; Get set to release quickfret holdoff
265 rlwinm r11,r8,0,0,19 ; Switch to savearea base
266 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
267 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
268 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
269 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
270 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
271 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
272 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
273
274 lwz r9,SAVflags(r8) /* Get the flags */
275 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
276 li r10,MSR_SUPERVISOR_INT_OFF /* Get the switcher's MSR */
277 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
278 stw r10,savesrr1+4(r8) /* Set up for switch in */
279 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
280 xor r3,r11,r8 /* Get the physical address of the new context save area */
281 stw r9,SAVflags(r8) /* Set the flags */
282
283 bne cr1,swtchtocont ; Switch to the continuation
284 sc /* Switch to the new context */
285
286 /* We come back here in the new thread context
287 * R4 was set to hold the old thread pointer, but switch_in will put it into
288 * R3 where it belongs.
289 */
290 blr /* Jump into the new thread */
291
292 ;
293 ; This is where we go when a continuation is set. We are actually
294 ; killing off the old context of the new guy so we need to pop off
295 ; any float or vector states for the ditched level.
296 ;
297 ; Note that we do the same kind of thing a chkfac in hw_exceptions.s
298 ;
299
300
301 swtchtocont:
302
303 stw r5,savesrr0+4(r8) ; Set the pc
304 stw r6,savesrr1+4(r8) ; Set the next MSR to use
305 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
306 mr r3,r8 ; Pass in the virtual address of savearea
307
308 b EXT(exception_exit) ; Blocking on continuation, toss old context...
309
310
311
312 /*
313 * All switched to threads come here first to clean up the old thread.
314 * We need to do the following contortions because we need to keep
315 * the LR clean. And because we need to manipulate the savearea chain
316 * with translation on. If we could, this should be done in lowmem_vectors
317 * before translation is turned on. But we can't, dang it!
318 *
319 * R3 = switcher's savearea (32-bit virtual)
320 * saver4 = old thread in switcher's save
321 * saver5 = new SRR0 in switcher's save
322 * saver6 = new SRR1 in switcher's save
323
324
325 */
326
327
328 .align 5
329 .globl EXT(switch_in)
330
331 LEXT(switch_in)
332
333 lwz r4,saver4+4(r3) ; Get the old thread
334 lwz r5,saver5+4(r3) ; Get the srr0 value
335
336 mfsprg r0,2 ; Get feature flags
337 lwz r9,THREAD_TOP_ACT(r4) ; Get the switched from ACT
338 lwz r6,saver6+4(r3) ; Get the srr1 value
339 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
340 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
341
342 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
343 stw r10,SAVprev+4(r3) ; Chain on the old one
344
345 mr r3,r4 ; Pass back the old thread
346
347 mtsrr0 r5 ; Set return point
348 mtsrr1 r6 ; Set return MSR
349
350 bne++ siSixtyFour ; Go do 64-bit...
351
352 rfi ; Jam...
353
354 siSixtyFour:
355 rfid ; Jam...
356
357 /*
358 * void fpu_save(facility_context ctx)
359 *
360 * Note that there are some oddities here when we save a context we are using.
361 * It is really not too cool to do this, but what the hey... Anyway,
362 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
363 * savearea containing the context just saved will go away. So, bottom line is
364 * that don't use fpus until after you are done with the saved context.
365 */
366 .align 5
367 .globl EXT(fpu_save)
368
369 LEXT(fpu_save)
370
371 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
372 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
373 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
374
375 mfmsr r0 ; Get the MSR
376 andc r0,r0,r2 ; Clear FP, VEC
377 andc r2,r0,r12 ; Clear EE
378 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
379 mtmsr r2 ; Set the MSR
380 isync
381
382 mfsprg r6,0 ; Get the per_processor block
383 lwz r12,FPUowner(r6) ; Get the context ID for owner
384
385 #if FPVECDBG
386 mr r7,r0 ; (TEST/DEBUG)
387 li r4,0 ; (TEST/DEBUG)
388 mr r10,r3 ; (TEST/DEBUG)
389 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
390 mr. r3,r12 ; (TEST/DEBUG)
391 li r2,0x6F00 ; (TEST/DEBUG)
392 li r5,0 ; (TEST/DEBUG)
393 beq-- noowneryet ; (TEST/DEBUG)
394 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
395 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
396
397 noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
398 sc ; (TEST/DEBUG)
399 mr r0,r7 ; (TEST/DEBUG)
400 mr r3,r10 ; (TEST/DEBUG)
401 #endif
402 mflr r2 ; Save the return address
403
404 fsretry: mr. r12,r12 ; Anyone own the FPU?
405 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
406 beq-- fsret ; Nobody owns the FPU, no save required...
407
408 cmplw cr1,r3,r12 ; Is the specified context live?
409
410 isync ; Force owner check first
411
412 lwz r9,FPUcpu(r12) ; Get the cpu that context was last on
413 bne-- cr1,fsret ; No, it is not...
414
415 cmplw cr1,r9,r11 ; Was the context for this processor?
416 beq-- cr1,fsgoodcpu ; Facility last used on this processor...
417
418 b fsret ; Someone else claimed it...
419
420 .align 5
421
422 fsgoodcpu: lwz r3,FPUsave(r12) ; Get the current FPU savearea for the thread
423 lwz r9,FPUlevel(r12) ; Get our current level indicator
424
425 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
426 beq- cr1,fsneedone ; Never saved it, so go do it...
427
428 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
429 cmplw cr1,r9,r8 ; Correct level?
430 beq-- cr1,fsret ; The current level is already saved, bail out...
431
432 fsneedone: bl EXT(save_get) ; Get a savearea for the context
433
434 mfsprg r6,0 ; Get back per_processor block
435 li r4,SAVfloat ; Get floating point tag
436 lwz r12,FPUowner(r6) ; Get back our thread
437 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
438 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
439 beq-- fsbackout ; If disowned, just toss savearea...
440 lwz r4,facAct(r12) ; Get the activation associated with live context
441 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
442 stw r4,SAVact(r3) ; Indicate the right activation for this context
443 lwz r9,FPUlevel(r12) ; Get our current level indicator again
444 stw r3,FPUsave(r12) ; Set this as the most current floating point context
445 stw r8,SAVprev+4(r3) ; And then chain this in front
446
447 stw r9,SAVlevel(r3) ; Show level in savearea
448
449 bl fp_store ; save all 32 FPRs in the save area at r3
450 mtlr r2 ; Restore return
451
452 fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
453 isync
454
455 blr
456
457 fsbackout: mr r4,r0 ; restore the original MSR
458 b EXT(save_ret_wMSR) ; Toss savearea and return from there...
459
460 /*
461 * fpu_switch()
462 *
463 * Entered to handle the floating-point unavailable exception and
464 * switch fpu context
465 *
466 * This code is run in virtual address mode on with interrupts off.
467 *
468 * Upon exit, the code returns to the users context with the floating
469 * point facility turned on.
470 *
471 * ENTRY: VM switched ON
472 * Interrupts OFF
473 * State is saved in savearea pointed to by R4.
474 * All other registers are free.
475 *
476 */
477
478 .align 5
479 .globl EXT(fpu_switch)
480
481 LEXT(fpu_switch)
482
483 #if DEBUG
484 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
485 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
486 lwz r1,0(r3)
487 addi r1,r1,1
488 stw r1,0(r3)
489 #endif /* DEBUG */
490
491 mfsprg r26,0 ; Get the per_processor block
492 mfmsr r19 ; Get the current MSR
493 mfsprg r17,1 ; Get the current thread
494
495 mr r25,r4 ; Save the entry savearea
496 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
497 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
498
499 mtmsr r19 ; Enable floating point instructions
500 isync
501
502 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
503 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
504
505 ; R22 has the "old" context anchor
506 ; R29 has the "new" context anchor
507
508 #if FPVECDBG
509 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
510 li r2,0x7F01 ; (TEST/DEBUG)
511 mr r3,r22 ; (TEST/DEBUG)
512 mr r5,r29 ; (TEST/DEBUG)
513 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
514 sc ; (TEST/DEBUG)
515 #endif
516
517 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
518
519 fswretry: mr. r22,r22 ; See if there is any live FP status
520
521 beq- fsnosave ; No live context, so nothing to save...
522
523 isync ; Make sure we see this in the right order
524
525 lwz r30,FPUsave(r22) ; Get the top savearea
526 cmplw cr2,r22,r29 ; Are both old and new the same context?
527 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
528 cmplwi cr1,r30,0 ; Anything saved yet?
529 cmplw r18,r16 ; Make sure we are on the right processor
530 lwz r31,FPUlevel(r22) ; Get the context level
531
532 bne- fsnosave ; No, not on the same processor...
533
534 ;
535 ; Check to see if the live context has already been saved.
536 ; Also check to see if all we are here just to re-enable the MSR
537 ; and handle specially if so.
538 ;
539
540 cmplw r31,r27 ; See if the current and active levels are the same
541 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
542 li r3,0 ; Clear this
543
544 beq- fsthesame ; New and old are the same, just go enable...
545
546 beq- cr1,fsmstsave ; Not saved yet, go do it...
547
548 lwz r11,SAVlevel(r30) ; Get the level of top saved context
549
550 cmplw r31,r11 ; Are live and saved the same?
551
552 #if FPVECDBG
553 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
554 li r2,0x7F02 ; (TEST/DEBUG)
555 mr r3,r30 ; (TEST/DEBUG)
556 mr r5,r31 ; (TEST/DEBUG)
557 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
558 sc ; (TEST/DEBUG)
559 li r3,0 ; (TEST/DEBUG)
560 #endif
561
562 beq+ fsnosave ; Same level, so already saved...
563
564
565 fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
566 eieio ; Make sure everyone sees it
567 bl EXT(save_get) ; Go get a savearea
568
569 mr. r31,r31 ; Are we saving the user state?
570 la r15,FPUsync(r22) ; Point to the sync word
571 beq++ fswusave ; Yeah, no need for lock...
572 ;
573 ; Here we make sure that the live context is not tossed while we are
574 ; trying to push it. This can happen only for kernel context and
575 ; then only by a race with act_machine_sv_free.
576 ;
577 ; We only need to hold this for a very short time, so no sniffing needed.
578 ; If we find any change to the level, we just abandon.
579 ;
580 fswsync: lwarx r19,0,r15 ; Get the sync word
581 li r0,1 ; Get the lock
582 cmplwi cr1,r19,0 ; Is it unlocked?
583 stwcx. r0,0,r15 ; Store lock and test reservation
584 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
585 bne-- fswsync ; Try again if lost reservation or locked...
586
587 isync ; Toss speculation
588
589 lwz r0,FPUlevel(r22) ; Pick up the level again
590 li r7,0 ; Get unlock value
591 cmplw r0,r31 ; Same level?
592 beq++ fswusave ; Yeah, we expect it to be...
593
594 stw r7,FPUsync(r22) ; Unlock lock. No need to sync here
595
596 bl EXT(save_ret) ; Toss save area because we are abandoning save
597 b fsnosave ; Skip the save...
598
599 .align 5
600
601 fswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
602 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
603 mr. r31,r31 ; Check again if we were user level
604 stw r30,SAVprev+4(r3) ; Point us to the old context
605 stw r31,SAVlevel(r3) ; Tag our level
606 li r7,SAVfloat ; Get the floating point ID
607 stw r12,SAVact(r3) ; Make sure we point to the right guy
608 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
609
610 li r7,0 ; Get the unlock value
611
612 beq-- fswnulock ; Skip unlock if user (we did not lock it)...
613 eieio ; Make sure that these updates make it out
614 stw r7,FPUsync(r22) ; Unlock it.
615
616 fswnulock:
617
618 #if FPVECDBG
619 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
620 li r2,0x7F03 ; (TEST/DEBUG)
621 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
622 sc ; (TEST/DEBUG)
623 #endif
624
625 bl fp_store ; store all 32 FPRs
626
627 ;
628 ; The context is all saved now and the facility is free.
629 ;
630 ; If we do not we need to fill the registers with junk, because this level has
631 ; never used them before and some thieving bastard could hack the old values
632 ; of some thread! Just imagine what would happen if they could! Why, nothing
633 ; would be safe! My God! It is terrifying!
634 ;
635
636
637 fsnosave: lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
638 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
639 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
640
641 stw r16,FPUcpu(r29) ; Claim context for us
642 eieio
643
644 #if FPVECDBG
645 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
646 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
647 li r2,0x7F04 ; (TEST/DEBUG)
648 mr r1,r15 ; (TEST/DEBUG)
649 mr r3,r14 ; (TEST/DEBUG)
650 mr r5,r13 ; (TEST/DEBUG)
651 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
652 sc ; (TEST/DEBUG)
653 #endif
654
655 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
656 mulli r19,r19,ppSize ; Find offset to the owner per_proc
657 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
658 li r16,FPUowner ; Displacement to float owner
659 add r19,r18,r19 ; Point to the owner per_proc
660
661 fsinvothr: lwarx r18,r16,r19 ; Get the owner
662 sub r0,r18,r29 ; Subtract one from the other
663 sub r11,r29,r18 ; Subtract the other from the one
664 or r11,r11,r0 ; Combine them
665 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
666 and r18,r18,r11 ; Make 0 if same, unchanged if not
667 stwcx. r18,r16,r19 ; Try to invalidate it
668 bne-- fsinvothr ; Try again if there was a collision...
669
670 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
671 la r11,savefp0(r14) ; Point to first line to bring in
672 stw r15,FPUlevel(r29) ; Set the "new" active level
673 eieio
674 stw r29,FPUowner(r26) ; Mark us as having the live context
675
676 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
677
678 dcbt 0,r11 ; Touch line in
679
680 lwz r3,SAVprev+4(r14) ; Get the previous context
681 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
682 cmplw r0,r15 ; Top level correct to load?
683 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
684
685 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
686
687 #if FPVECDBG
688 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
689 li r2,0x7F05 ; (TEST/DEBUG)
690 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
691 sc ; (TEST/DEBUG)
692 #endif
693
694 // Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
695 // are executed on a 128-byte machine, but that is better than a mispredicted branch.
696
697 la r11,savefp4(r14) ; Point to next line
698 dcbt 0,r11 ; Touch line in
699 lfd f0, savefp0(r14)
700 lfd f1,savefp1(r14)
701 lfd f2,savefp2(r14)
702 la r11,savefp8(r14) ; Point to next line
703 lfd f3,savefp3(r14)
704 dcbt 0,r11 ; Touch line in
705 lfd f4,savefp4(r14)
706 lfd f5,savefp5(r14)
707 lfd f6,savefp6(r14)
708 la r11,savefp12(r14) ; Point to next line
709 lfd f7,savefp7(r14)
710 dcbt 0,r11 ; Touch line in
711 lfd f8,savefp8(r14)
712 lfd f9,savefp9(r14)
713 lfd f10,savefp10(r14)
714 la r11,savefp16(r14) ; Point to next line
715 lfd f11,savefp11(r14)
716 dcbt 0,r11 ; Touch line in
717 lfd f12,savefp12(r14)
718 lfd f13,savefp13(r14)
719 lfd f14,savefp14(r14)
720 la r11,savefp20(r14) ; Point to next line
721 lfd f15,savefp15(r14)
722 dcbt 0,r11 ; Touch line in
723 lfd f16,savefp16(r14)
724 lfd f17,savefp17(r14)
725 lfd f18,savefp18(r14)
726 la r11,savefp24(r14) ; Point to next line
727 lfd f19,savefp19(r14)
728 dcbt 0,r11 ; Touch line in
729 lfd f20,savefp20(r14)
730 lfd f21,savefp21(r14)
731 la r11,savefp28(r14) ; Point to next line
732 lfd f22,savefp22(r14)
733 lfd f23,savefp23(r14)
734 dcbt 0,r11 ; Touch line in
735 lfd f24,savefp24(r14)
736 lfd f25,savefp25(r14)
737 lfd f26,savefp26(r14)
738 lfd f27,savefp27(r14)
739 lfd f28,savefp28(r14)
740 lfd f29,savefp29(r14)
741 lfd f30,savefp30(r14)
742 lfd f31,savefp31(r14)
743
744 mr r3,r14 ; Get the old savearea (we popped it before)
745 bl EXT(save_ret) ; Toss it
746
747 fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
748 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
749 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
750 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
751 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
752 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
753 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
754 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
755 mr r3,r25 ; Pass the virtual addres of savearea
756 beq- fsnuser ; We are not user state...
757 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
758 stw r11,spcFlags(r26) ; Set per_proc copy
759
760 fsnuser:
761 #if FPVECDBG
762 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
763 li r2,0x7F07 ; (TEST/DEBUG)
764 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
765 sc ; (TEST/DEBUG)
766 #endif
767
768 b EXT(exception_exit) ; Exit to the fray...
769
770 /*
771 * Initialize the registers to some bogus value
772 */
773
774 MakeSureThatNoTerroristsCanHurtUsByGod:
775
776 #if FPVECDBG
777 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
778 li r2,0x7F06 ; (TEST/DEBUG)
779 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
780 sc ; (TEST/DEBUG)
781 #endif
782 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
783 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
784 lfd f0,0(r5) ; Initialize FP0
785 fmr f1,f0 ; Do them all
786 fmr f2,f0
787 fmr f3,f0
788 fmr f4,f0
789 fmr f5,f0
790 fmr f6,f0
791 fmr f7,f0
792 fmr f8,f0
793 fmr f9,f0
794 fmr f10,f0
795 fmr f11,f0
796 fmr f12,f0
797 fmr f13,f0
798 fmr f14,f0
799 fmr f15,f0
800 fmr f16,f0
801 fmr f17,f0
802 fmr f18,f0
803 fmr f19,f0
804 fmr f20,f0
805 fmr f21,f0
806 fmr f22,f0
807 fmr f23,f0
808 fmr f24,f0
809 fmr f25,f0
810 fmr f26,f0
811 fmr f27,f0
812 fmr f28,f0
813 fmr f29,f0
814 fmr f30,f0
815 fmr f31,f0
816 b fsenable ; Finish setting it all up...
817
818
819 ;
820 ; We get here when we are switching to the same context at the same level and the context
821 ; is still live. Essentially, all we are doing is turning on the faility. It may have
822 ; gotten turned off due to doing a context save for the current level or a context switch
823 ; back to the live guy.
824 ;
825
826 .align 5
827
828 fsthesame:
829
830 #if FPVECDBG
831 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
832 li r2,0x7F0A ; (TEST/DEBUG)
833 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
834 sc ; (TEST/DEBUG)
835 #endif
836 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
837
838 lwz r11,SAVlevel(r30) ; Get the level of top saved context
839 lwz r14,SAVprev+4(r30) ; Get the previous savearea
840
841 cmplw r11,r31 ; Are live and saved the same?
842
843 bne+ fsenable ; Level not the same, nothing to pop, go enable and exit...
844
845 mr r3,r30 ; Get the old savearea (we popped it before)
846 stw r14,FPUsave(r22) ; Pop the savearea from the stack
847 bl EXT(save_ret) ; Toss it
848 b fsenable ; Go enable and exit...
849
850
851 ;
852 ; This function invalidates any live floating point context for the passed in facility_context.
853 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
854 ;
855
856 .align 5
857 .globl EXT(toss_live_fpu)
858
859 LEXT(toss_live_fpu)
860
861 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
862 mfmsr r9 ; Get the MSR
863 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
864 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
865 andc r9,r9,r0 ; Force off VEC and FP
866 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
867 andc r0,r9,r0 ; Turn off EE now
868 mtmsr r0 ; No interruptions
869 isync
870 beq+ tlfnotours ; Floats off, can not be live here...
871
872 mfsprg r8,0 ; Get the per proc
873
874 ;
875 ; Note that at this point, since floats are on, we are the owner
876 ; of live state on this processor
877 ;
878
879 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
880 li r0,0 ; Clear this just in case we need it
881 cmplw r6,r3 ; Are we tossing our own context?
882 bne-- tlfnotours ; Nope...
883
884 lfd f1,Zero(0) ; Make a 0
885 mtfsf 0xFF,f1 ; Clear it
886
887 tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
888 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
889 mulli r11,r11,ppSize ; Find offset to the owner per_proc
890 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
891 li r10,FPUowner ; Displacement to float owner
892 add r11,r12,r11 ; Point to the owner per_proc
893
894 tlfinvothr: lwarx r12,r10,r11 ; Get the owner
895
896 sub r0,r12,r3 ; Subtract one from the other
897 sub r8,r3,r12 ; Subtract the other from the one
898 or r8,r8,r0 ; Combine them
899 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
900 and r12,r12,r8 ; Make 0 if same, unchanged if not
901 stwcx. r12,r10,r11 ; Try to invalidate it
902 bne-- tlfinvothr ; Try again if there was a collision...
903
904 mtmsr r9 ; Restore interruptions
905 isync ; Could be turning off floats here
906 blr ; Leave...
907
908
909 /*
910 * Altivec stuff is here. The techniques used are pretty identical to
911 * the floating point. Except that we will honor the VRSAVE register
912 * settings when loading and restoring registers.
913 *
914 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
915 * mask. VRSAVE is set by the vector user and represents the VRs that they
916 * say that they are using. The vrvalid mask indicates which vector registers
917 * are saved in the savearea. Whenever context is saved, it is saved according
918 * to the VRSAVE register. It is loaded based on VRSAVE anded with
919 * vrvalid (all other registers are splatted with 0s). This is done because we
920 * don't want to load any registers we don't have a copy of, we want to set them
921 * to zero instead.
922 *
923 * Note that there are some oddities here when we save a context we are using.
924 * It is really not too cool to do this, but what the hey... Anyway,
925 * we turn vectors and fpu off before we leave.
926 * The oddity is that if you use vectors after this, the
927 * savearea containing the context just saved will go away. So, bottom line is
928 * that don't use vectors until after you are done with the saved context.
929 *
930 */
931
932 .align 5
933 .globl EXT(vec_save)
934
935 LEXT(vec_save)
936
937
938 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
939 mfmsr r0 ; Get the MSR
940 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
941 andc r0,r0,r2 ; Force off VEC and FP
942 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
943 andc r2,r0,r2 ; Clear EE for now
944 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
945 mtmsr r2 ; Set the MSR
946 isync
947
948 mfsprg r6,0 ; Get the per_processor block
949 lwz r12,VMXowner(r6) ; Get the context ID for owner
950
951 #if FPVECDBG
952 mr r7,r0 ; (TEST/DEBUG)
953 li r4,0 ; (TEST/DEBUG)
954 mr r10,r3 ; (TEST/DEBUG)
955 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
956 mr. r3,r12 ; (TEST/DEBUG)
957 li r2,0x5F00 ; (TEST/DEBUG)
958 li r5,0 ; (TEST/DEBUG)
959 beq- noowneryeu ; (TEST/DEBUG)
960 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
961 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
962
963 noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
964 sc ; (TEST/DEBUG)
965 mr r0,r7 ; (TEST/DEBUG)
966 mr r3,r10 ; (TEST/DEBUG)
967 #endif
968 mflr r2 ; Save the return address
969
970 vsretry: mr. r12,r12 ; Anyone own the vector?
971 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
972 beq- vsret ; Nobody owns the vector, no save required...
973
974 cmplw cr1,r3,r12 ; Is the specified context live?
975
976 isync ; Force owner check first
977
978 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
979 bne- cr1,vsret ; Specified context is not live
980
981 cmplw cr1,r9,r11 ; Was the context for this processor?
982 beq+ cr1,vsgoodcpu ; Facility last used on this processor...
983
984 b vsret ; Someone else claimed this...
985
986 .align 5
987
988 vsgoodcpu: lwz r3,VMXsave(r12) ; Get the current vector savearea for the thread
989 lwz r10,liveVRS(r6) ; Get the right VRSave register
990 lwz r9,VMXlevel(r12) ; Get our current level indicator
991
992
993 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
994 beq- cr1,vsneedone ; Never saved it, so we need an area...
995
996 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
997 mr. r10,r10 ; Is VRsave set to 0?
998 cmplw cr1,r9,r8 ; Correct level?
999 bne- cr1,vsneedone ; Different level, so we need to save...
1000
1001 bne+ vsret ; VRsave is non-zero so we need to keep what is saved...
1002
1003 lwz r4,SAVprev+4(r3) ; Pick up the previous area
1004 lwz r5,SAVlevel(r4) ; Get the level associated with save
1005 stw r4,VMXsave(r12) ; Dequeue this savearea
1006 li r4,0 ; Clear
1007 stw r5,VMXlevel(r12) ; Save the level
1008
1009 stw r4,VMXowner(r12) ; Show no live context here
1010 eieio
1011
1012 vsbackout: mr r4,r0 ; restore the saved MSR
1013 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
1014
1015 .align 5
1016
1017 vsneedone: mr. r10,r10 ; Is VRsave set to 0?
1018 beq- vsret ; Yeah, they do not care about any of them...
1019
1020 bl EXT(save_get) ; Get a savearea for the context
1021
1022 mfsprg r6,0 ; Get back per_processor block
1023 li r4,SAVvector ; Get vector tag
1024 lwz r12,VMXowner(r6) ; Get back our context ID
1025 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1026 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1027 beq- vsbackout ; If disowned, just toss savearea...
1028 lwz r4,facAct(r12) ; Get the activation associated with live context
1029 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1030 stw r4,SAVact(r3) ; Indicate the right activation for this context
1031 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1032 stw r3,VMXsave(r12) ; Set this as the most current floating point context
1033 stw r8,SAVprev+4(r3) ; And then chain this in front
1034
1035 stw r9,SAVlevel(r3) ; Set level in savearea
1036 mfcr r12 ; save CRs across call to vr_store
1037 lwz r10,liveVRS(r6) ; Get the right VRSave register
1038
1039 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
1040
1041 mtcrf 255,r12 ; Restore the non-volatile CRs
1042 mtlr r2 ; restore return address
1043
1044 vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1045 isync
1046
1047 blr
1048
1049 /*
1050 * vec_switch()
1051 *
1052 * Entered to handle the vector unavailable exception and
1053 * switch vector context
1054 *
1055 * This code is run with virtual address mode on and interrupts off.
1056 *
1057 * Upon exit, the code returns to the users context with the vector
1058 * facility turned on.
1059 *
1060 * ENTRY: VM switched ON
1061 * Interrupts OFF
1062 * State is saved in savearea pointed to by R4.
1063 * All other registers are free.
1064 *
1065 */
1066
1067 .align 5
1068 .globl EXT(vec_switch)
1069
1070 LEXT(vec_switch)
1071
1072 #if DEBUG
1073 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1074 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1075 lwz r1,0(r3)
1076 addi r1,r1,1
1077 stw r1,0(r3)
1078 #endif /* DEBUG */
1079
1080 mfsprg r26,0 ; Get the per_processor block
1081 mfmsr r19 ; Get the current MSR
1082 mfsprg r17,1 ; Get the current thread
1083
1084 mr r25,r4 ; Save the entry savearea
1085 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1086 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1087
1088 mtmsr r19 ; Enable vector instructions
1089 isync
1090
1091 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1092 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
1093
1094 ; R22 has the "old" context anchor
1095 ; R29 has the "new" context anchor
1096
1097 #if FPVECDBG
1098 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1099 li r2,0x5F01 ; (TEST/DEBUG)
1100 mr r3,r22 ; (TEST/DEBUG)
1101 mr r5,r29 ; (TEST/DEBUG)
1102 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1103 sc ; (TEST/DEBUG)
1104 #endif
1105
1106 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1107
1108 vsvretry: mr. r22,r22 ; See if there is any live vector status
1109
1110 beq- vsnosave ; No live context, so nothing to save...
1111
1112 isync ; Make sure we see this in the right order
1113
1114 lwz r30,VMXsave(r22) ; Get the top savearea
1115 cmplw cr2,r22,r29 ; Are both old and new the same context?
1116 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1117 cmplwi cr1,r30,0 ; Anything saved yet?
1118 cmplw r18,r16 ; Make sure we are on the right processor
1119 lwz r31,VMXlevel(r22) ; Get the context level
1120
1121 lwz r10,liveVRS(r26) ; Get the right VRSave register
1122
1123 bne- vsnosave ; No, not on the same processor...
1124
1125 ;
1126 ; Check to see if the live context has already been saved.
1127 ; Also check to see if all we are here just to re-enable the MSR
1128 ; and handle specially if so.
1129 ;
1130
1131 cmplw r31,r27 ; See if the current and active levels are the same
1132 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1133 li r8,0 ; Clear this
1134
1135 beq- vsthesame ; New and old are the same, just go enable...
1136
1137 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1138 beq- cr1,vsmstsave ; Not saved yet, go do it...
1139
1140 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1141
1142 cmplw r31,r11 ; Are live and saved the same?
1143
1144 #if FPVECDBG
1145 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1146 li r2,0x5F02 ; (TEST/DEBUG)
1147 mr r3,r30 ; (TEST/DEBUG)
1148 mr r5,r31 ; (TEST/DEBUG)
1149 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1150 sc ; (TEST/DEBUG)
1151 #endif
1152
1153 bne- vsmstsave ; Live context has not been saved yet...
1154
1155 bne- cr2,vsnosave ; Live context saved and VRSave not 0, no save and keep context...
1156
1157 lwz r4,SAVprev+4(r30) ; Pick up the previous area
1158 li r5,0 ; Assume this is the only one (which should be the ususal case)
1159 mr. r4,r4 ; Was this the only one?
1160 stw r4,VMXsave(r22) ; Dequeue this savearea
1161 beq+ vsonlyone ; This was the only one...
1162 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1163
1164 vsonlyone: stw r5,VMXlevel(r22) ; Save the level
1165 stw r8,VMXowner(r26) ; Clear owner
1166 eieio
1167 mr r3,r30 ; Copy the savearea we are tossing
1168 bl EXT(save_ret) ; Toss the savearea
1169 b vsnosave ; Go load up the context...
1170
1171 .align 5
1172
1173
1174 vsmstsave: stw r8,VMXowner(r26) ; Clear owner
1175 eieio
1176 beq- cr2,vsnosave ; The VRSave was 0, so there is nothing to save...
1177
1178 bl EXT(save_get) ; Go get a savearea
1179
1180 mr. r31,r31 ; Are we saving the user state?
1181 la r15,VMXsync(r22) ; Point to the sync word
1182 beq++ vswusave ; Yeah, no need for lock...
1183 ;
1184 ; Here we make sure that the live context is not tossed while we are
1185 ; trying to push it. This can happen only for kernel context and
1186 ; then only by a race with act_machine_sv_free.
1187 ;
1188 ; We only need to hold this for a very short time, so no sniffing needed.
1189 ; If we find any change to the level, we just abandon.
1190 ;
1191 vswsync: lwarx r19,0,r15 ; Get the sync word
1192 li r0,1 ; Get the lock
1193 cmplwi cr1,r19,0 ; Is it unlocked?
1194 stwcx. r0,0,r15 ; Store lock and test reservation
1195 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
1196 bne-- vswsync ; Try again if lost reservation or locked...
1197
1198 isync ; Toss speculation
1199
1200 lwz r0,VMXlevel(r22) ; Pick up the level again
1201 li r7,0 ; Get unlock value
1202 cmplw r0,r31 ; Same level?
1203 beq++ vswusave ; Yeah, we expect it to be...
1204
1205 stw r7,VMXsync(r22) ; Unlock lock. No need to sync here
1206
1207 bl EXT(save_ret) ; Toss save area because we are abandoning save
1208 b vsnosave ; Skip the save...
1209
1210 .align 5
1211
1212 vswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
1213 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1214 mr. r31,r31 ; Check again if we were user level
1215 stw r30,SAVprev+4(r3) ; Point us to the old context
1216 stw r31,SAVlevel(r3) ; Tag our level
1217 li r7,SAVvector ; Get the vector ID
1218 stw r12,SAVact(r3) ; Make sure we point to the right guy
1219 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
1220
1221 li r7,0 ; Get the unlock value
1222
1223 beq-- vswnulock ; Skip unlock if user (we did not lock it)...
1224 eieio ; Make sure that these updates make it out
1225 stw r7,VMXsync(r22) ; Unlock it.
1226
1227 vswnulock:
1228
1229 #if FPVECDBG
1230 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1231 li r2,0x5F03 ; (TEST/DEBUG)
1232 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1233 sc ; (TEST/DEBUG)
1234 #endif
1235
1236 lwz r10,liveVRS(r26) ; Get the right VRSave register
1237 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1238
1239
1240 ;
1241 ; The context is all saved now and the facility is free.
1242 ;
1243 ; If we do not we need to fill the registers with junk, because this level has
1244 ; never used them before and some thieving bastard could hack the old values
1245 ; of some thread! Just imagine what would happen if they could! Why, nothing
1246 ; would be safe! My God! It is terrifying!
1247 ;
1248 ; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1249 ; constant that we may need to fill unused vector registers.
1250 ;
1251
1252
1253
1254
1255 vsnosave: vspltisb v31,-10 ; Get 0xF6F6F6F6
1256 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1257 vspltisb v30,5 ; Get 0x05050505
1258 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1259 vspltish v29,4 ; Get 0x00040004
1260 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1261 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1262
1263 stw r16,VMXcpu(r29) ; Claim context for us
1264 eieio
1265
1266 #if FPVECDBG
1267 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1268 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1269 li r2,0x5F04 ; (TEST/DEBUG)
1270 mr r1,r15 ; (TEST/DEBUG)
1271 mr r3,r14 ; (TEST/DEBUG)
1272 mr r5,r13 ; (TEST/DEBUG)
1273 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1274 sc ; (TEST/DEBUG)
1275 #endif
1276
1277 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
1278 vspltisb v28,-2 ; Get 0xFEFEFEFE
1279 mulli r19,r19,ppSize ; Find offset to the owner per_proc
1280 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1281 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
1282 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1283 li r16,VMXowner ; Displacement to vector owner
1284 add r19,r18,r19 ; Point to the owner per_proc
1285 vrlb v31,v31,v29 ; Get 0xDEADDEAD
1286
1287 vsinvothr: lwarx r18,r16,r19 ; Get the owner
1288
1289 sub r0,r18,r29 ; Subtract one from the other
1290 sub r11,r29,r18 ; Subtract the other from the one
1291 or r11,r11,r0 ; Combine them
1292 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1293 and r18,r18,r11 ; Make 0 if same, unchanged if not
1294 stwcx. r18,r16,r19 ; Try to invalidate it
1295 bne-- vsinvothr ; Try again if there was a collision...
1296
1297 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
1298 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1299 stw r15,VMXlevel(r29) ; Set the "new" active level
1300 eieio
1301 stw r29,VMXowner(r26) ; Mark us as having the live context
1302
1303 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
1304
1305 lwz r3,SAVprev+4(r14) ; Get the previous context
1306 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1307 cmplw r0,r15 ; Top level correct to load?
1308 bne-- ProtectTheAmericanWay ; No, go initialize...
1309
1310 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1311
1312 #if FPVECDBG
1313 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1314 li r2,0x5F05 ; (TEST/DEBUG)
1315 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1316 sc ; (TEST/DEBUG)
1317 #endif
1318
1319 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
1320 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
1321 and r10,r10,r22 ; Figure out just what registers need to be loaded
1322 mr r3,r14 ; r3 <- ptr to savearea with VRs
1323 bl vr_load ; load VRs from save area based on vrsave in r10
1324
1325 bl EXT(save_ret) ; Toss the save area after loading VRs
1326
1327 vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
1328 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
1329 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1330 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1331 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1332 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1333 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
1334 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1335 mr r3,r25 ; Pass virtual address of the savearea
1336 beq- vrnuser ; We are not user state...
1337 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
1338 stw r11,spcFlags(r26) ; Set per_proc copy
1339
1340 vrnuser:
1341 #if FPVECDBG
1342 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1343 li r2,0x5F07 ; (TEST/DEBUG)
1344 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1345 sc ; (TEST/DEBUG)
1346 #endif
1347 b EXT(exception_exit) ; Exit to the fray...
1348
1349 /*
1350 * Initialize the registers to some bogus value
1351 */
1352
1353 ProtectTheAmericanWay:
1354
1355 #if FPVECDBG
1356 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1357 li r2,0x5F06 ; (TEST/DEBUG)
1358 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1359 sc ; (TEST/DEBUG)
1360 #endif
1361
1362 vor v0,v31,v31 ; Copy into the next register
1363 vor v1,v31,v31 ; Copy into the next register
1364 vor v2,v31,v31 ; Copy into the next register
1365 vor v3,v31,v31 ; Copy into the next register
1366 vor v4,v31,v31 ; Copy into the next register
1367 vor v5,v31,v31 ; Copy into the next register
1368 vor v6,v31,v31 ; Copy into the next register
1369 vor v7,v31,v31 ; Copy into the next register
1370 vor v8,v31,v31 ; Copy into the next register
1371 vor v9,v31,v31 ; Copy into the next register
1372 vor v10,v31,v31 ; Copy into the next register
1373 vor v11,v31,v31 ; Copy into the next register
1374 vor v12,v31,v31 ; Copy into the next register
1375 vor v13,v31,v31 ; Copy into the next register
1376 vor v14,v31,v31 ; Copy into the next register
1377 vor v15,v31,v31 ; Copy into the next register
1378 vor v16,v31,v31 ; Copy into the next register
1379 vor v17,v31,v31 ; Copy into the next register
1380 vor v18,v31,v31 ; Copy into the next register
1381 vor v19,v31,v31 ; Copy into the next register
1382 vor v20,v31,v31 ; Copy into the next register
1383 vor v21,v31,v31 ; Copy into the next register
1384 vor v22,v31,v31 ; Copy into the next register
1385 vor v23,v31,v31 ; Copy into the next register
1386 vor v24,v31,v31 ; Copy into the next register
1387 vor v25,v31,v31 ; Copy into the next register
1388 vor v26,v31,v31 ; Copy into the next register
1389 vor v27,v31,v31 ; Copy into the next register
1390 vor v28,v31,v31 ; Copy into the next register
1391 vor v29,v31,v31 ; Copy into the next register
1392 vor v30,v31,v31 ; Copy into the next register
1393 b vrenable ; Finish setting it all up...
1394
1395
1396
1397 ;
1398 ; We get here when we are switching to the same context at the same level and the context
1399 ; is still live. Essentially, all we are doing is turning on the faility. It may have
1400 ; gotten turned off due to doing a context save for the current level or a context switch
1401 ; back to the live guy.
1402 ;
1403
1404 .align 5
1405
1406 vsthesame:
1407
1408 #if FPVECDBG
1409 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1410 li r2,0x5F0A ; (TEST/DEBUG)
1411 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1412 sc ; (TEST/DEBUG)
1413 #endif
1414 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1415
1416 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1417 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1418
1419 cmplw r11,r31 ; Are live and saved the same?
1420
1421 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1422
1423 mr r3,r30 ; Get the old savearea (we popped it before)
1424 stw r11,VMXsave(r22) ; Pop the vector stack
1425 bl EXT(save_ret) ; Toss it
1426 b vrenable ; Go enable and exit...
1427
1428
1429 ;
1430 ; This function invalidates any live vector context for the passed in facility_context.
1431 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1432 ;
1433
1434 .align 5
1435 .globl EXT(toss_live_vec)
1436
1437 LEXT(toss_live_vec)
1438
1439 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1440 mfmsr r9 ; Get the MSR
1441 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1442 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1443 andc r9,r9,r0 ; Force off VEC and FP
1444 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1445 andc r0,r9,r0 ; Turn off EE now
1446 mtmsr r0 ; No interruptions
1447 isync
1448 beq+ tlvnotours ; Vector off, can not be live here...
1449
1450 mfsprg r8,0 ; Get the per proc
1451
1452 ;
1453 ; Note that at this point, since vecs are on, we are the owner
1454 ; of live state on this processor
1455 ;
1456
1457 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1458 li r0,0 ; Clear this just in case we need it
1459 cmplw r6,r3 ; Are we tossing our own context?
1460 bne- tlvnotours ; Nope...
1461
1462 vspltish v1,1 ; Turn on the non-Java bit and saturate
1463 vspltisw v0,1 ; Turn on the saturate bit
1464 vxor v1,v1,v0 ; Turn off saturate
1465 mtspr vrsave,r0 ; Clear VRSAVE
1466 mtvscr v1 ; Set the non-java, no saturate status
1467
1468 tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1469 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1470 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1471 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1472 li r10,VMXowner ; Displacement to vector owner
1473 add r11,r12,r11 ; Point to the owner per_proc
1474 li r0,0 ; Set a 0 to invalidate context
1475
1476 tlvinvothr: lwarx r12,r10,r11 ; Get the owner
1477
1478 sub r0,r12,r3 ; Subtract one from the other
1479 sub r8,r3,r12 ; Subtract the other from the one
1480 or r8,r8,r0 ; Combine them
1481 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1482 and r12,r12,r8 ; Make 0 if same, unchanged if not
1483 stwcx. r12,r10,r11 ; Try to invalidate it
1484 bne-- tlvinvothr ; Try again if there was a collision...
1485
1486 mtmsr r9 ; Restore interruptions
1487 isync ; Could be turning off vectors here
1488 blr ; Leave....
1489
1490 #if 0
1491 ;
1492 ; This function invalidates any live vector context for the passed in facility_context
1493 ; if the level is current. It also tosses the corresponding savearea if there is one.
1494 ; This function is primarily used whenever we detect a VRSave that is all zeros.
1495 ;
1496
1497 .align 5
1498 .globl EXT(vec_trash)
1499
1500 LEXT(vec_trash)
1501
1502 lwz r12,facAct(r3) ; Get the activation
1503 lwz r11,VMXlevel(r3) ; Get the context level
1504 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1505 lwz r9,VMXsave(r3) ; Get the savearea, if any
1506 cmplw r10,r11 ; Are we at the right level?
1507 cmplwi cr1,r9,0 ; Remember if there is a savearea
1508 bnelr+ ; No, we do nothing...
1509
1510 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1511 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1512 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1513 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1514 li r10,VMXowner ; Displacement to vector owner
1515 add r11,r12,r11 ; Point to the owner per_proc
1516
1517 vtinvothr: lwarx r12,r10,r11 ; Get the owner
1518
1519 sub r0,r12,r3 ; Subtract one from the other
1520 sub r8,r3,r12 ; Subtract the other from the one
1521 or r8,r8,r0 ; Combine them
1522 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1523 and r12,r12,r8 ; Make 0 if same, unchanged if not
1524 stwcx. r12,r10,r11 ; Try to invalidate it
1525 bne-- vtinvothr ; Try again if there was a collision...
1526
1527
1528 beqlr++ cr1 ; Leave if there is no savearea
1529 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1530 cmplw r8,r11 ; Savearea for the current level?
1531 bnelr++ ; No, nothing to release...
1532
1533 lwz r8,SAVprev+4(r9) ; Pick up the previous area
1534 mr. r8,r8 ; Is there a previous?
1535 beq-- vtnoprev ; Nope...
1536 lwz r7,SAVlevel(r8) ; Get the level associated with save
1537
1538 vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1539 stw r7,VMXlevel(r3) ; Pop the level
1540
1541 mr r3,r9 ; Get the savearea to release
1542 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1543 #endif
1544
1545 ;
1546 ; Just some test code to force vector and/or floating point in the kernel
1547 ;
1548
1549 .align 5
1550 .globl EXT(fctx_test)
1551
1552 LEXT(fctx_test)
1553
1554 mfsprg r3,1 ; Get the current thread
1555 mr. r3,r3 ; Are we actually up and running?
1556 beqlr- ; No...
1557
1558 fmr f0,f0 ; Use floating point
1559 mftb r4 ; Get time base for a random number
1560 li r5,1 ; Get a potential vrsave to use
1561 andi. r4,r4,0x3F ; Get a number from 0 - 63
1562 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1563 mtspr vrsave,r5 ; Set VRSave
1564 vor v0,v0,v0 ; Use vectors
1565 blr
1566
1567
1568 // *******************
1569 // * f p _ s t o r e *
1570 // *******************
1571 //
1572 // Store FPRs into a save area. Called by fpu_save and fpu_switch.
1573 //
1574 // When called:
1575 // floating pt is enabled
1576 // r3 = ptr to save area
1577 //
1578 // We destroy:
1579 // r11.
1580
1581 fp_store:
1582 mfsprg r11,2 ; get feature flags
1583 mtcrf 0x02,r11 ; put cache line size bits in cr6
1584 la r11,savefp0(r3) ; point to 1st line
1585 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1586 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1587
1588 // Store the FPRs on a 128-byte machine.
1589
1590 stfd f0,savefp0(r3)
1591 stfd f1,savefp1(r3)
1592 la r11,savefp16(r3) ; Point to the 2nd cache line
1593 stfd f2,savefp2(r3)
1594 stfd f3,savefp3(r3)
1595 dcbz128 0,r11 ; establish 2nd line
1596 stfd f4,savefp4(r3)
1597 stfd f5,savefp5(r3)
1598 stfd f6,savefp6(r3)
1599 stfd f7,savefp7(r3)
1600 stfd f8,savefp8(r3)
1601 stfd f9,savefp9(r3)
1602 stfd f10,savefp10(r3)
1603 stfd f11,savefp11(r3)
1604 stfd f12,savefp12(r3)
1605 stfd f13,savefp13(r3)
1606 stfd f14,savefp14(r3)
1607 stfd f15,savefp15(r3)
1608 stfd f16,savefp16(r3)
1609 stfd f17,savefp17(r3)
1610 stfd f18,savefp18(r3)
1611 stfd f19,savefp19(r3)
1612 stfd f20,savefp20(r3)
1613 stfd f21,savefp21(r3)
1614 stfd f22,savefp22(r3)
1615 stfd f23,savefp23(r3)
1616 stfd f24,savefp24(r3)
1617 stfd f25,savefp25(r3)
1618 stfd f26,savefp26(r3)
1619 stfd f27,savefp27(r3)
1620 stfd f28,savefp28(r3)
1621 stfd f29,savefp29(r3)
1622 stfd f30,savefp30(r3)
1623 stfd f31,savefp31(r3)
1624 blr
1625
1626 // Store FPRs on a 32-byte machine.
1627
1628 fp_st32:
1629 la r11,savefp4(r3) ; Point to the 2nd line
1630 stfd f0,savefp0(r3)
1631 dcbz 0,r11 ; Allocate cache
1632 stfd f1,savefp1(r3)
1633 stfd f2,savefp2(r3)
1634 la r11,savefp8(r3) ; Point to the 3rd line
1635 stfd f3,savefp3(r3)
1636 dcbz 0,r11 ; Allocate cache
1637 stfd f4,savefp4(r3)
1638 stfd f5,savefp5(r3)
1639 stfd f6,savefp6(r3)
1640 la r11,savefp12(r3) ; Point to the 4th line
1641 stfd f7,savefp7(r3)
1642 dcbz 0,r11 ; Allocate cache
1643 stfd f8,savefp8(r3)
1644 stfd f9,savefp9(r3)
1645 stfd f10,savefp10(r3)
1646 la r11,savefp16(r3) ; Point to the 5th line
1647 stfd f11,savefp11(r3)
1648 dcbz 0,r11 ; Allocate cache
1649 stfd f12,savefp12(r3)
1650 stfd f13,savefp13(r3)
1651 stfd f14,savefp14(r3)
1652 la r11,savefp20(r3) ; Point to the 6th line
1653 stfd f15,savefp15(r3)
1654 dcbz 0,r11 ; Allocate cache
1655 stfd f16,savefp16(r3)
1656 stfd f17,savefp17(r3)
1657 stfd f18,savefp18(r3)
1658 la r11,savefp24(r3) ; Point to the 7th line
1659 stfd f19,savefp19(r3)
1660 dcbz 0,r11 ; Allocate cache
1661 stfd f20,savefp20(r3)
1662
1663 stfd f21,savefp21(r3)
1664 stfd f22,savefp22(r3)
1665 la r11,savefp28(r3) ; Point to the 8th line
1666 stfd f23,savefp23(r3)
1667 dcbz 0,r11 ; allocate it
1668 stfd f24,savefp24(r3)
1669 stfd f25,savefp25(r3)
1670 stfd f26,savefp26(r3)
1671 stfd f27,savefp27(r3)
1672
1673 stfd f28,savefp28(r3)
1674 stfd f29,savefp29(r3)
1675 stfd f30,savefp30(r3)
1676 stfd f31,savefp31(r3)
1677 blr
1678
1679
1680 // *******************
1681 // * v r _ s t o r e *
1682 // *******************
1683 //
1684 // Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1685 // both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1686 // unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1687 // for each cache block size.
1688 //
1689 // When called:
1690 // interrupts are off, vectors are enabled
1691 // r3 = ptr to save area
1692 // r10 = vrsave (not 0)
1693 //
1694 // We destroy:
1695 // r4 - r11, all CRs.
1696
1697 vr_store:
1698 mfsprg r9,2 ; get feature flags
1699 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1700 slwi r8,r10,1 ; Shift over 1
1701 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1702 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1703 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1704
1705
1706 ; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1707 ; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1708
1709 slwi r7,r8,2 ; shift groups-of-2 over by 2
1710 li r4,16 ; load offsets for X-form stores
1711 or r8,r7,r8 ; show if any in group of 4 are in use
1712 li r5,32
1713 slwi r7,r8,4 ; shift groups-of-4 over by 4
1714 li r6,48
1715 or r11,r7,r8 ; show if any in group of 8 are in use
1716 li r7,64
1717 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1718 li r8,80
1719 mtcrf 0x20,r11
1720 li r9,96
1721 mtcrf 0x08,r11
1722 li r10,112
1723 mtcrf 0x02,r11
1724
1725 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1726 la r11,savevr0(r3) ; get address of this group of registers in save area
1727 dcbz128 0,r11 ; zero the line
1728 stvxl v0,0,r11 ; save 8 VRs in the line
1729 stvxl v1,r4,r11
1730 stvxl v2,r5,r11
1731 stvxl v3,r6,r11
1732 stvxl v4,r7,r11
1733 stvxl v5,r8,r11
1734 stvxl v6,r9,r11
1735 stvxl v7,r10,r11
1736
1737 vr_st64b:
1738 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
1739 la r11,savevr8(r3) ; get address of this group of registers in save area
1740 dcbz128 0,r11 ; zero the line
1741 stvxl v8,0,r11 ; save 8 VRs in the line
1742 stvxl v9,r4,r11
1743 stvxl v10,r5,r11
1744 stvxl v11,r6,r11
1745 stvxl v12,r7,r11
1746 stvxl v13,r8,r11
1747 stvxl v14,r9,r11
1748 stvxl v15,r10,r11
1749
1750 vr_st64c:
1751 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
1752 la r11,savevr16(r3) ; get address of this group of registers in save area
1753 dcbz128 0,r11 ; zero the line
1754 stvxl v16,0,r11 ; save 8 VRs in the line
1755 stvxl v17,r4,r11
1756 stvxl v18,r5,r11
1757 stvxl v19,r6,r11
1758 stvxl v20,r7,r11
1759 stvxl v21,r8,r11
1760 stvxl v22,r9,r11
1761 stvxl v23,r10,r11
1762
1763 vr_st64d:
1764 bflr 24 ; done if none of vr24-vr31 are in use
1765 la r11,savevr24(r3) ; get address of this group of registers in save area
1766 dcbz128 0,r11 ; zero the line
1767 stvxl v24,0,r11 ; save 8 VRs in the line
1768 stvxl v25,r4,r11
1769 stvxl v26,r5,r11
1770 stvxl v27,r6,r11
1771 stvxl v28,r7,r11
1772 stvxl v29,r8,r11
1773 stvxl v30,r9,r11
1774 stvxl v31,r10,r11
1775 blr
1776
1777 ; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
1778 ; or neither in each group. This cuts down on conditional branches.
1779 ; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
1780 ; r3 = savearea
1781
1782 vr_st32:
1783 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
1784 li r4,16 ; load offset for X-form stores
1785
1786 bf 0,vr_st32b ; skip if neither VR in this pair is in use
1787 la r11,savevr0(r3) ; get address of this group of registers in save area
1788 dcba 0,r11 ; establish the line wo reading it
1789 stvxl v0,0,r11 ; save the two VRs in the line
1790 stvxl v1,r4,r11
1791
1792 vr_st32b:
1793 bf 2,vr_st32c ; skip if neither VR in this pair is in use
1794 la r11,savevr2(r3) ; get address of this group of registers in save area
1795 dcba 0,r11 ; establish the line wo reading it
1796 stvxl v2,0,r11 ; save the two VRs in the line
1797 stvxl v3,r4,r11
1798
1799 vr_st32c:
1800 bf 4,vr_st32d ; skip if neither VR in this pair is in use
1801 la r11,savevr4(r3) ; get address of this group of registers in save area
1802 dcba 0,r11 ; establish the line wo reading it
1803 stvxl v4,0,r11 ; save the two VRs in the line
1804 stvxl v5,r4,r11
1805
1806 vr_st32d:
1807 bf 6,vr_st32e ; skip if neither VR in this pair is in use
1808 la r11,savevr6(r3) ; get address of this group of registers in save area
1809 dcba 0,r11 ; establish the line wo reading it
1810 stvxl v6,0,r11 ; save the two VRs in the line
1811 stvxl v7,r4,r11
1812
1813 vr_st32e:
1814 bf 8,vr_st32f ; skip if neither VR in this pair is in use
1815 la r11,savevr8(r3) ; get address of this group of registers in save area
1816 dcba 0,r11 ; establish the line wo reading it
1817 stvxl v8,0,r11 ; save the two VRs in the line
1818 stvxl v9,r4,r11
1819
1820 vr_st32f:
1821 bf 10,vr_st32g ; skip if neither VR in this pair is in use
1822 la r11,savevr10(r3) ; get address of this group of registers in save area
1823 dcba 0,r11 ; establish the line wo reading it
1824 stvxl v10,0,r11 ; save the two VRs in the line
1825 stvxl v11,r4,r11
1826
1827 vr_st32g:
1828 bf 12,vr_st32h ; skip if neither VR in this pair is in use
1829 la r11,savevr12(r3) ; get address of this group of registers in save area
1830 dcba 0,r11 ; establish the line wo reading it
1831 stvxl v12,0,r11 ; save the two VRs in the line
1832 stvxl v13,r4,r11
1833
1834 vr_st32h:
1835 bf 14,vr_st32i ; skip if neither VR in this pair is in use
1836 la r11,savevr14(r3) ; get address of this group of registers in save area
1837 dcba 0,r11 ; establish the line wo reading it
1838 stvxl v14,0,r11 ; save the two VRs in the line
1839 stvxl v15,r4,r11
1840
1841 vr_st32i:
1842 bf 16,vr_st32j ; skip if neither VR in this pair is in use
1843 la r11,savevr16(r3) ; get address of this group of registers in save area
1844 dcba 0,r11 ; establish the line wo reading it
1845 stvxl v16,0,r11 ; save the two VRs in the line
1846 stvxl v17,r4,r11
1847
1848 vr_st32j:
1849 bf 18,vr_st32k ; skip if neither VR in this pair is in use
1850 la r11,savevr18(r3) ; get address of this group of registers in save area
1851 dcba 0,r11 ; establish the line wo reading it
1852 stvxl v18,0,r11 ; save the two VRs in the line
1853 stvxl v19,r4,r11
1854
1855 vr_st32k:
1856 bf 20,vr_st32l ; skip if neither VR in this pair is in use
1857 la r11,savevr20(r3) ; get address of this group of registers in save area
1858 dcba 0,r11 ; establish the line wo reading it
1859 stvxl v20,0,r11 ; save the two VRs in the line
1860 stvxl v21,r4,r11
1861
1862 vr_st32l:
1863 bf 22,vr_st32m ; skip if neither VR in this pair is in use
1864 la r11,savevr22(r3) ; get address of this group of registers in save area
1865 dcba 0,r11 ; establish the line wo reading it
1866 stvxl v22,0,r11 ; save the two VRs in the line
1867 stvxl v23,r4,r11
1868
1869 vr_st32m:
1870 bf 24,vr_st32n ; skip if neither VR in this pair is in use
1871 la r11,savevr24(r3) ; get address of this group of registers in save area
1872 dcba 0,r11 ; establish the line wo reading it
1873 stvxl v24,0,r11 ; save the two VRs in the line
1874 stvxl v25,r4,r11
1875
1876 vr_st32n:
1877 bf 26,vr_st32o ; skip if neither VR in this pair is in use
1878 la r11,savevr26(r3) ; get address of this group of registers in save area
1879 dcba 0,r11 ; establish the line wo reading it
1880 stvxl v26,0,r11 ; save the two VRs in the line
1881 stvxl v27,r4,r11
1882
1883 vr_st32o:
1884 bf 28,vr_st32p ; skip if neither VR in this pair is in use
1885 la r11,savevr28(r3) ; get address of this group of registers in save area
1886 dcba 0,r11 ; establish the line wo reading it
1887 stvxl v28,0,r11 ; save the two VRs in the line
1888 stvxl v29,r4,r11
1889
1890 vr_st32p:
1891 bflr 30 ; done if neither VR in this pair is in use
1892 la r11,savevr30(r3) ; get address of this group of registers in save area
1893 dcba 0,r11 ; establish the line wo reading it
1894 stvxl v30,0,r11 ; save the two VRs in the line
1895 stvxl v31,r4,r11
1896 blr
1897
1898
1899 // *****************
1900 // * v r _ l o a d *
1901 // *****************
1902 //
1903 // Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
1904 // of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
1905 // branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
1906 // in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
1907 // which would bug all VRs that aren't live.
1908 //
1909 // When called:
1910 // interrupts are off, vectors are enabled
1911 // r3 = ptr to save area
1912 // r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
1913 // v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
1914 //
1915 // We destroy:
1916 // r4 - r11, all CRs.
1917
1918 vr_load:
1919 mfsprg r9,2 ; get feature flags
1920 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
1921 mtcrf 0x02,r9 ; set cache line size bits in cr6
1922 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
1923 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
1924 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
1925 lis r7,0xFF00 ; so set bits 0-7
1926
1927 // Loop touching in cache blocks we will load from.
1928 // r3 = savearea ptr
1929 // r5 = we light bits for the VRs we will be loading
1930 // r6 = 1 if 32-byte, 7 if 128-byte
1931 // r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
1932 // r10 = live VR bits
1933 // v31 = bugbug constant
1934
1935 vr_ld0:
1936 li r5,0 ; initialize set of VRs to load
1937 la r11,savevr0(r3) ; get address of register file
1938 b vr_ld2 ; enter loop in middle
1939
1940 .align 5
1941 vr_ld1: ; loop over each cache line we will load
1942 dcbt r4,r11 ; start prefetch of the line
1943 andc r10,r10,r9 ; turn off the bits in this line
1944 or r5,r5,r9 ; we will load all these
1945 vr_ld2: ; initial entry pt
1946 cntlzw r4,r10 ; get offset to next live VR
1947 andc r4,r4,r6 ; cacheline align it
1948 srw. r9,r7,r4 ; position bits for VRs in that cache line
1949 slwi r4,r4,4 ; get byte offset within register file to that line
1950 bne vr_ld1 ; loop if more bits in r10
1951
1952 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
1953
1954 // Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
1955 // r3 = savearea ptr
1956 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
1957 // r11 = addr(savevr0)
1958 // v31 = bugbug constant
1959
1960 mtcrf 0x80,r5 ; set up bits for conditional branches
1961 li r4,16 ; load offsets for X-form stores
1962 li r6,48
1963 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
1964 li r7,64
1965 li r8,80
1966 mtcrf 0x08,r5
1967 li r9,96
1968 li r10,112
1969 mtcrf 0x02,r5
1970 li r5,32
1971
1972 bt 0,vr_ld128a ; skip if this line must be loaded
1973 vor v0,v31,v31 ; no VR must be loaded, so bug them all
1974 vor v1,v31,v31
1975 vor v2,v31,v31
1976 vor v3,v31,v31
1977 vor v4,v31,v31
1978 vor v5,v31,v31
1979 vor v6,v31,v31
1980 vor v7,v31,v31
1981 b vr_ld128b
1982 vr_ld128a: ; must load from this line
1983 lvxl v0,0,r11
1984 lvxl v1,r4,r11
1985 lvxl v2,r5,r11
1986 lvxl v3,r6,r11
1987 lvxl v4,r7,r11
1988 lvxl v5,r8,r11
1989 lvxl v6,r9,r11
1990 lvxl v7,r10,r11
1991
1992 vr_ld128b: ; here to handle next cache line
1993 la r11,savevr8(r3) ; load offset to it
1994 bt 8,vr_ld128c ; skip if this line must be loaded
1995 vor v8,v31,v31 ; no VR must be loaded, so bug them all
1996 vor v9,v31,v31
1997 vor v10,v31,v31
1998 vor v11,v31,v31
1999 vor v12,v31,v31
2000 vor v13,v31,v31
2001 vor v14,v31,v31
2002 vor v15,v31,v31
2003 b vr_ld128d
2004 vr_ld128c: ; must load from this line
2005 lvxl v8,0,r11
2006 lvxl v9,r4,r11
2007 lvxl v10,r5,r11
2008 lvxl v11,r6,r11
2009 lvxl v12,r7,r11
2010 lvxl v13,r8,r11
2011 lvxl v14,r9,r11
2012 lvxl v15,r10,r11
2013
2014 vr_ld128d: ; here to handle next cache line
2015 la r11,savevr16(r3) ; load offset to it
2016 bt 16,vr_ld128e ; skip if this line must be loaded
2017 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2018 vor v17,v31,v31
2019 vor v18,v31,v31
2020 vor v19,v31,v31
2021 vor v20,v31,v31
2022 vor v21,v31,v31
2023 vor v22,v31,v31
2024 vor v23,v31,v31
2025 b vr_ld128f
2026 vr_ld128e: ; must load from this line
2027 lvxl v16,0,r11
2028 lvxl v17,r4,r11
2029 lvxl v18,r5,r11
2030 lvxl v19,r6,r11
2031 lvxl v20,r7,r11
2032 lvxl v21,r8,r11
2033 lvxl v22,r9,r11
2034 lvxl v23,r10,r11
2035
2036 vr_ld128f: ; here to handle next cache line
2037 la r11,savevr24(r3) ; load offset to it
2038 bt 24,vr_ld128g ; skip if this line must be loaded
2039 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2040 vor v25,v31,v31
2041 vor v26,v31,v31
2042 vor v27,v31,v31
2043 vor v28,v31,v31
2044 vor v29,v31,v31
2045 vor v30,v31,v31
2046 blr
2047 vr_ld128g: ; must load from this line
2048 lvxl v24,0,r11
2049 lvxl v25,r4,r11
2050 lvxl v26,r5,r11
2051 lvxl v27,r6,r11
2052 lvxl v28,r7,r11
2053 lvxl v29,r8,r11
2054 lvxl v30,r9,r11
2055 lvxl v31,r10,r11
2056 blr
2057
2058 // Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2059 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2060 // r11 = addr(savevr0)
2061
2062 vr_ld32:
2063 mtcrf 0xFF,r5 ; set up bits for conditional branches
2064 li r4,16 ; load offset for X-form stores
2065
2066 bt 0,vr_ld32load0 ; skip if we must load this line
2067 vor v0,v31,v31 ; neither VR is live, so bug them both
2068 vor v1,v31,v31
2069 b vr_ld32test2
2070 vr_ld32load0: ; must load VRs in this line
2071 lvxl v0,0,r11
2072 lvxl v1,r4,r11
2073
2074 vr_ld32test2: ; here to handle next cache line
2075 la r11,savevr2(r3) ; get offset to next cache line
2076 bt 2,vr_ld32load2 ; skip if we must load this line
2077 vor v2,v31,v31 ; neither VR is live, so bug them both
2078 vor v3,v31,v31
2079 b vr_ld32test4
2080 vr_ld32load2: ; must load VRs in this line
2081 lvxl v2,0,r11
2082 lvxl v3,r4,r11
2083
2084 vr_ld32test4: ; here to handle next cache line
2085 la r11,savevr4(r3) ; get offset to next cache line
2086 bt 4,vr_ld32load4 ; skip if we must load this line
2087 vor v4,v31,v31 ; neither VR is live, so bug them both
2088 vor v5,v31,v31
2089 b vr_ld32test6
2090 vr_ld32load4: ; must load VRs in this line
2091 lvxl v4,0,r11
2092 lvxl v5,r4,r11
2093
2094 vr_ld32test6: ; here to handle next cache line
2095 la r11,savevr6(r3) ; get offset to next cache line
2096 bt 6,vr_ld32load6 ; skip if we must load this line
2097 vor v6,v31,v31 ; neither VR is live, so bug them both
2098 vor v7,v31,v31
2099 b vr_ld32test8
2100 vr_ld32load6: ; must load VRs in this line
2101 lvxl v6,0,r11
2102 lvxl v7,r4,r11
2103
2104 vr_ld32test8: ; here to handle next cache line
2105 la r11,savevr8(r3) ; get offset to next cache line
2106 bt 8,vr_ld32load8 ; skip if we must load this line
2107 vor v8,v31,v31 ; neither VR is live, so bug them both
2108 vor v9,v31,v31
2109 b vr_ld32test10
2110 vr_ld32load8: ; must load VRs in this line
2111 lvxl v8,0,r11
2112 lvxl v9,r4,r11
2113
2114 vr_ld32test10: ; here to handle next cache line
2115 la r11,savevr10(r3) ; get offset to next cache line
2116 bt 10,vr_ld32load10 ; skip if we must load this line
2117 vor v10,v31,v31 ; neither VR is live, so bug them both
2118 vor v11,v31,v31
2119 b vr_ld32test12
2120 vr_ld32load10: ; must load VRs in this line
2121 lvxl v10,0,r11
2122 lvxl v11,r4,r11
2123
2124 vr_ld32test12: ; here to handle next cache line
2125 la r11,savevr12(r3) ; get offset to next cache line
2126 bt 12,vr_ld32load12 ; skip if we must load this line
2127 vor v12,v31,v31 ; neither VR is live, so bug them both
2128 vor v13,v31,v31
2129 b vr_ld32test14
2130 vr_ld32load12: ; must load VRs in this line
2131 lvxl v12,0,r11
2132 lvxl v13,r4,r11
2133
2134 vr_ld32test14: ; here to handle next cache line
2135 la r11,savevr14(r3) ; get offset to next cache line
2136 bt 14,vr_ld32load14 ; skip if we must load this line
2137 vor v14,v31,v31 ; neither VR is live, so bug them both
2138 vor v15,v31,v31
2139 b vr_ld32test16
2140 vr_ld32load14: ; must load VRs in this line
2141 lvxl v14,0,r11
2142 lvxl v15,r4,r11
2143
2144 vr_ld32test16: ; here to handle next cache line
2145 la r11,savevr16(r3) ; get offset to next cache line
2146 bt 16,vr_ld32load16 ; skip if we must load this line
2147 vor v16,v31,v31 ; neither VR is live, so bug them both
2148 vor v17,v31,v31
2149 b vr_ld32test18
2150 vr_ld32load16: ; must load VRs in this line
2151 lvxl v16,0,r11
2152 lvxl v17,r4,r11
2153
2154 vr_ld32test18: ; here to handle next cache line
2155 la r11,savevr18(r3) ; get offset to next cache line
2156 bt 18,vr_ld32load18 ; skip if we must load this line
2157 vor v18,v31,v31 ; neither VR is live, so bug them both
2158 vor v19,v31,v31
2159 b vr_ld32test20
2160 vr_ld32load18: ; must load VRs in this line
2161 lvxl v18,0,r11
2162 lvxl v19,r4,r11
2163
2164 vr_ld32test20: ; here to handle next cache line
2165 la r11,savevr20(r3) ; get offset to next cache line
2166 bt 20,vr_ld32load20 ; skip if we must load this line
2167 vor v20,v31,v31 ; neither VR is live, so bug them both
2168 vor v21,v31,v31
2169 b vr_ld32test22
2170 vr_ld32load20: ; must load VRs in this line
2171 lvxl v20,0,r11
2172 lvxl v21,r4,r11
2173
2174 vr_ld32test22: ; here to handle next cache line
2175 la r11,savevr22(r3) ; get offset to next cache line
2176 bt 22,vr_ld32load22 ; skip if we must load this line
2177 vor v22,v31,v31 ; neither VR is live, so bug them both
2178 vor v23,v31,v31
2179 b vr_ld32test24
2180 vr_ld32load22: ; must load VRs in this line
2181 lvxl v22,0,r11
2182 lvxl v23,r4,r11
2183
2184 vr_ld32test24: ; here to handle next cache line
2185 la r11,savevr24(r3) ; get offset to next cache line
2186 bt 24,vr_ld32load24 ; skip if we must load this line
2187 vor v24,v31,v31 ; neither VR is live, so bug them both
2188 vor v25,v31,v31
2189 b vr_ld32test26
2190 vr_ld32load24: ; must load VRs in this line
2191 lvxl v24,0,r11
2192 lvxl v25,r4,r11
2193
2194 vr_ld32test26: ; here to handle next cache line
2195 la r11,savevr26(r3) ; get offset to next cache line
2196 bt 26,vr_ld32load26 ; skip if we must load this line
2197 vor v26,v31,v31 ; neither VR is live, so bug them both
2198 vor v27,v31,v31
2199 b vr_ld32test28
2200 vr_ld32load26: ; must load VRs in this line
2201 lvxl v26,0,r11
2202 lvxl v27,r4,r11
2203
2204 vr_ld32test28: ; here to handle next cache line
2205 la r11,savevr28(r3) ; get offset to next cache line
2206 bt 28,vr_ld32load28 ; skip if we must load this line
2207 vor v28,v31,v31 ; neither VR is live, so bug them both
2208 vor v29,v31,v31
2209 b vr_ld32test30
2210 vr_ld32load28: ; must load VRs in this line
2211 lvxl v28,0,r11
2212 lvxl v29,r4,r11
2213
2214 vr_ld32test30: ; here to handle next cache line
2215 la r11,savevr30(r3) ; get offset to next cache line
2216 bt 30,vr_ld32load30 ; skip if we must load this line
2217 vor v30,v31,v31 ; neither VR is live, so bug them both
2218 blr
2219 vr_ld32load30: ; must load VRs in this line
2220 lvxl v30,0,r11
2221 lvxl v31,r4,r11
2222 blr
mirror of XNU