]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ppc/cswtch.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-344.21.74.tar.gz
[apple/xnu.git] / osfmk / ppc / cswtch.s
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
7 *
8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * file.
14 *
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
22 *
23 * @APPLE_LICENSE_HEADER_END@
24 */
25 /*
26 * @OSF_COPYRIGHT@
27 */
28
29 #include <ppc/asm.h>
30 #include <ppc/proc_reg.h>
31 #include <cpus.h>
32 #include <assym.s>
33 #include <debug.h>
34 #include <mach/ppc/vm_param.h>
35 #include <ppc/exception.h>
36 #include <ppc/savearea.h>
37
38 #define FPVECDBG 0
39 #define GDDBG 0
40
41 .text
42
43 /*
44 * void load_context(thread_t thread)
45 *
46 * Load the context for the first kernel thread, and go.
47 *
48 * NOTE - if DEBUG is set, the former routine is a piece
49 * of C capable of printing out debug info before calling the latter,
50 * otherwise both entry points are identical.
51 */
52
53 .align 5
54 .globl EXT(load_context)
55
56 LEXT(load_context)
57
58 .globl EXT(Load_context)
59
60 LEXT(Load_context)
61
62 /*
63 * Since this is the first thread, we came in on the interrupt
64 * stack. The first thread never returns, so there is no need to
65 * worry about saving its frame, hence we can reset the istackptr
66 * back to the saved_state structure at it's top
67 */
68
69
70 /*
71 * get new thread pointer and set it into the active_threads pointer
72 *
73 */
74
75 mfsprg r6,0
76 lwz r0,PP_INTSTACK_TOP_SS(r6)
77 stw r0,PP_ISTACKPTR(r6)
78 stw r3,PP_ACTIVE_THREAD(r6)
79
80 /* Find the new stack and store it in active_stacks */
81
82 lwz r12,PP_ACTIVE_STACKS(r6)
83 lwz r1,THREAD_KERNEL_STACK(r3)
84 lwz r9,THREAD_TOP_ACT(r3) /* Point to the active activation */
85 mtsprg 1,r9
86 stw r1,0(r12)
87 li r0,0 /* Clear a register */
88 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
89 mfmsr r5 /* Since we are passing control, get our MSR values */
90 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
91 lwz r1,saver1+4(r3) /* Load new stack pointer */
92 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
93 stw r0,FM_BACKPTR(r1) /* zero backptr */
94 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
95 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
96 b EXT(exception_exit) /* Go end it all... */
97
98 /* struct thread_shuttle *Switch_context(struct thread_shuttle *old,
99 * void (*cont)(void),
100 * struct thread_shuttle *new)
101 *
102 * Switch from one thread to another. If a continuation is supplied, then
103 * we do not need to save callee save registers.
104 *
105 */
106
107 /* void Call_continuation( void (*continuation)(void), vm_offset_t stack_ptr)
108 */
109
110 .align 5
111 .globl EXT(Call_continuation)
112
113 LEXT(Call_continuation)
114
115 mtlr r3
116 mr r1, r4 /* Load new stack pointer */
117 blr /* Jump to the continuation */
118
119 /*
120 * Get the old kernel stack, and store into the thread structure.
121 * See if a continuation is supplied, and skip state save if so.
122 *
123 * Note that interrupts must be disabled before we get here (i.e., splsched)
124 */
125
126 /* Context switches are double jumps. We pass the following to the
127 * context switch firmware call:
128 *
129 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
130 * R4 = old thread
131 * R5 = new SRR0
132 * R6 = new SRR1
133 * R7 = high order physical address of savearea for full switch
134 *
135 * savesrr0 is set to go to switch_in
136 * savesrr1 is set to uninterruptible with translation on
137 */
138
139
140 .align 5
141 .globl EXT(Switch_context)
142
143 LEXT(Switch_context)
144
145 lwz r11,THREAD_KERNEL_STACK(r5) ; Get the new stack pointer
146 mfsprg r12,0 ; Get the per_proc block
147 lwz r10,PP_ACTIVE_STACKS(r12) ; Get the pointer to the current stack
148 #if DEBUG
149 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
150 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
151 bne++ notonintstack ; (DEBUG/TRACE) stack
152 BREAKPOINT_TRAP
153 notonintstack:
154 #endif
155
156 #if 0
157 lwz r8,lgPPStart(0) ; (TEST/DEBUG) Get the start of per_procs
158 sub r7,r12,r8 ; (TEST/DEBUG) Find offset to our per_proc
159 xori r7,r7,0x1000 ; (TEST/DEBUG) Switch to other proc
160 add r8,r8,r7 ; (TEST/DEBUG) Switch to it
161 lwz r8,PP_ACTIVE_THREAD(r8) ; (TEST/DEBUG) Get the other active thread
162 cmplw r8,r5 ; (TEST/DEBUG) Trying to switch to an active thread?
163 bne++ snively ; (TEST/DEBUG) Nope...
164 BREAKPOINT_TRAP ; (TEST/DEBUG) Get to debugger...
165
166 snively: ; (TEST/DEBUG)
167 #endif
168
169 stw r5,PP_ACTIVE_THREAD(r12) ; Make the new thread current
170 lwz r5,THREAD_TOP_ACT(r5) ; Get the new activation
171 stw r4,THREAD_CONTINUATION(r3) ; Set continuation into the thread
172 lwz r7,0(r10) ; Get the current stack
173 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
174 stw r11,0(r10) ; Save the new kernel stack address
175
176 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
177 lwz r9,cioSpace(r5) ; Get copyin/out address space
178 stw r7,THREAD_KERNEL_STACK(r3) ; Remember the current stack in the thread (do not need???)
179 mtsprg 1,r5 ; Set the current activation pointer
180 lwz r7,CTHREAD_SELF(r5) ; Pick up the user assist word
181 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
182 lwz r6,cioRelo(r5) ; Get copyin/out relocation top
183 lwz r2,cioRelo+4(r5) ; Get copyin/out relocation bottom
184
185 stw r7,UAW(r12) ; Save the assist word for the "ultra fast path"
186
187 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
188
189 lwz r0,ACT_KLOADED(r5)
190 sth r9,ppCIOmp+mpSpace(r12) ; Save the space
191 stw r6,ppCIOmp+mpNestReloc(r12) ; Save top part of physical address
192 stw r2,ppCIOmp+mpNestReloc+4(r12) ; Save bottom part of physical address
193 lwz r10,PP_ACTIVE_KLOADED(r12) ; Get kernel loaded flag address
194 subfic r0,r0,0 ; Get bit 0 to 0 if not kloaded, 1 otherwise
195 lwz r2,traceMask(0) ; Get the enabled traces
196 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
197 srawi r0,r0,31 ; Get 0 if not kloaded, ffffffff otherwise
198 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
199 and r0,r5,r0 ; Get 0 if not kloaded, activation otherwise
200
201 mr. r2,r2 ; Any tracing going on?
202 stw r0,0(r10) ; Set the kloaded stuff
203 lis r0,hi16(CutTrace) ; Trace FW call
204 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
205 ori r0,r0,lo16(CutTrace) ; Trace FW call
206 mr r10,r3 ; Save across trace
207 beq++ cswNoTrc ; No trace today, dude...
208 lwz r2,THREAD_TOP_ACT(r3) ; Trace old activation
209 mr r3,r11 ; Trace prev savearea
210 sc ; Cut trace entry of context switch
211 mr r3,r10 ; Restore
212
213 cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
214 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
215 lwz r9,VMXowner(r12) ; Grab the owner of the vector
216 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
217 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
218 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
219 li r0,1 ; Get set to hold off quickfret
220
221 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
222 cmplw r10,r2 ; Do we have the live float context?
223 lwz r10,FPUlevel(r2) ; Get the live level
224 mr r4,r3 ; Save our old thread to pass back
225 cmplw cr5,r9,r2 ; Do we have the live vector context?
226 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
227 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
228 bne++ cswnofloat ; Float is not ours...
229
230 cmplw r10,r11 ; Is the level the same?
231 lwz r5,FPUcpu(r2) ; Get the owning cpu
232 bne++ cswnofloat ; Level not the same, this is not live...
233
234 cmplw r5,r0 ; Still owned by this cpu?
235 lwz r10,FPUsave(r2) ; Get the level
236 bne++ cswnofloat ; CPU claimed by someone else...
237
238 mr. r10,r10 ; Is there a savearea here?
239 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
240
241 beq-- cswnofloat ; No savearea to check...
242
243 lwz r3,SAVlevel(r10) ; Get the level
244 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
245 cmplw r3,r11 ; Is it for the current level?
246
247 bne++ cswnofloat ; Nope...
248
249 stw r5,FPUsave(r2) ; Pop off this savearea
250
251 rlwinm r3,r10,0,0,19 ; Move back to start of page
252
253 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
254 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
255 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
256 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
257 stw r5,SAVprev(r10) ; Link the old in (top)
258 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
259 xor r3,r10,r3 ; Convert to physical
260 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
261 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
262
263 #if FPVECDBG
264 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
265 mr r7,r2 ; (TEST/DEBUG)
266 li r2,0x4401 ; (TEST/DEBUG)
267 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
268 sc ; (TEST/DEBUG)
269 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
270 mr r2,r7 ; (TEST/DEBUG)
271 #endif
272
273 cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
274
275 lwz r10,VMXlevel(r2) ; Get the live level
276
277 cmplw r10,r11 ; Is the level the same?
278 lwz r5,VMXcpu(r2) ; Get the owning cpu
279 bne++ cswnovect ; Level not the same, this is not live...
280
281 cmplw r5,r0 ; Still owned by this cpu?
282 lwz r10,VMXsave(r2) ; Get the level
283 bne++ cswnovect ; CPU claimed by someone else...
284
285 mr. r10,r10 ; Is there a savearea here?
286 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
287
288 beq-- cswnovect ; No savearea to check...
289
290 lwz r3,SAVlevel(r10) ; Get the level
291 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
292 cmplw r3,r11 ; Is it for the current level?
293
294 bne++ cswnovect ; Nope...
295
296 stw r5,VMXsave(r2) ; Pop off this savearea
297 rlwinm r3,r10,0,0,19 ; Move back to start of page
298
299 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
300 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
301 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
302 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
303 stw r5,SAVprev(r10) ; Link the old in (top)
304 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
305 xor r3,r10,r3 ; Convert to physical
306 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
307 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
308
309 #if FPVECDBG
310 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
311 li r2,0x4501 ; (TEST/DEBUG)
312 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
313 sc ; (TEST/DEBUG)
314 #endif
315
316 cswnovect: li r0,0 ; Get set to release quickfret holdoff
317 rlwinm r11,r8,0,0,19 ; Switch to savearea base
318 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
319 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
320 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
321 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
322 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
323 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
324 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
325
326 lwz r9,SAVflags(r8) /* Get the flags */
327 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
328 li r10,MSR_SUPERVISOR_INT_OFF /* Get the switcher's MSR */
329 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
330 stw r10,savesrr1+4(r8) /* Set up for switch in */
331 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
332 xor r3,r11,r8 /* Get the physical address of the new context save area */
333 stw r9,SAVflags(r8) /* Set the flags */
334
335 bne cr1,swtchtocont ; Switch to the continuation
336 sc /* Switch to the new context */
337
338 /* We come back here in the new thread context
339 * R4 was set to hold the old thread pointer, but switch_in will put it into
340 * R3 where it belongs.
341 */
342 blr /* Jump into the new thread */
343
344 ;
345 ; This is where we go when a continuation is set. We are actually
346 ; killing off the old context of the new guy so we need to pop off
347 ; any float or vector states for the ditched level.
348 ;
349 ; Note that we do the same kind of thing a chkfac in hw_exceptions.s
350 ;
351
352
353 swtchtocont:
354
355 stw r5,savesrr0+4(r8) ; Set the pc
356 stw r6,savesrr1+4(r8) ; Set the next MSR to use
357 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
358 mr r3,r8 ; Pass in the virtual address of savearea
359
360 b EXT(exception_exit) ; Blocking on continuation, toss old context...
361
362
363
364 /*
365 * All switched to threads come here first to clean up the old thread.
366 * We need to do the following contortions because we need to keep
367 * the LR clean. And because we need to manipulate the savearea chain
368 * with translation on. If we could, this should be done in lowmem_vectors
369 * before translation is turned on. But we can't, dang it!
370 *
371 * R3 = switcher's savearea (32-bit virtual)
372 * saver4 = old thread in switcher's save
373 * saver5 = new SRR0 in switcher's save
374 * saver6 = new SRR1 in switcher's save
375
376
377 */
378
379
380 .align 5
381 .globl EXT(switch_in)
382
383 LEXT(switch_in)
384
385 lwz r4,saver4+4(r3) ; Get the old thread
386 lwz r5,saver5+4(r3) ; Get the srr0 value
387
388 mfsprg r0,2 ; Get feature flags
389 lwz r9,THREAD_TOP_ACT(r4) ; Get the switched from ACT
390 lwz r6,saver6+4(r3) ; Get the srr1 value
391 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
392 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
393
394 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
395 stw r10,SAVprev+4(r3) ; Chain on the old one
396
397 mr r3,r4 ; Pass back the old thread
398
399 mtsrr0 r5 ; Set return point
400 mtsrr1 r6 ; Set return MSR
401
402 bne++ siSixtyFour ; Go do 64-bit...
403
404 rfi ; Jam...
405
406 siSixtyFour:
407 rfid ; Jam...
408
409 /*
410 * void fpu_save(facility_context ctx)
411 *
412 * Note that there are some oddities here when we save a context we are using.
413 * It is really not too cool to do this, but what the hey... Anyway,
414 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
415 * savearea containing the context just saved will go away. So, bottom line is
416 * that don't use fpus until after you are done with the saved context.
417 */
418 .align 5
419 .globl EXT(fpu_save)
420
421 LEXT(fpu_save)
422
423 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
424 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
425 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
426
427 mfmsr r0 ; Get the MSR
428 andc r0,r0,r2 ; Clear FP, VEC
429 andc r2,r0,r12 ; Clear EE
430 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
431 mtmsr r2 ; Set the MSR
432 isync
433
434 mfsprg r6,0 ; Get the per_processor block
435 lwz r12,FPUowner(r6) ; Get the context ID for owner
436
437 #if FPVECDBG
438 mr r7,r0 ; (TEST/DEBUG)
439 li r4,0 ; (TEST/DEBUG)
440 mr r10,r3 ; (TEST/DEBUG)
441 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
442 mr. r3,r12 ; (TEST/DEBUG)
443 li r2,0x6F00 ; (TEST/DEBUG)
444 li r5,0 ; (TEST/DEBUG)
445 beq-- noowneryet ; (TEST/DEBUG)
446 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
447 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
448
449 noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
450 sc ; (TEST/DEBUG)
451 mr r0,r7 ; (TEST/DEBUG)
452 mr r3,r10 ; (TEST/DEBUG)
453 #endif
454 mflr r2 ; Save the return address
455
456 fsretry: mr. r12,r12 ; Anyone own the FPU?
457 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
458 beq-- fsret ; Nobody owns the FPU, no save required...
459
460 cmplw cr1,r3,r12 ; Is the specified context live?
461
462 isync ; Force owner check first
463
464 lwz r9,FPUcpu(r12) ; Get the cpu that context was last on
465 bne-- cr1,fsret ; No, it is not...
466
467 cmplw cr1,r9,r11 ; Was the context for this processor?
468 beq-- cr1,fsgoodcpu ; Facility last used on this processor...
469
470 b fsret ; Someone else claimed it...
471
472 .align 5
473
474 fsgoodcpu: lwz r3,FPUsave(r12) ; Get the current FPU savearea for the thread
475 lwz r9,FPUlevel(r12) ; Get our current level indicator
476
477 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
478 beq- cr1,fsneedone ; Never saved it, so go do it...
479
480 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
481 cmplw cr1,r9,r8 ; Correct level?
482 beq-- cr1,fsret ; The current level is already saved, bail out...
483
484 fsneedone: bl EXT(save_get) ; Get a savearea for the context
485
486 mfsprg r6,0 ; Get back per_processor block
487 li r4,SAVfloat ; Get floating point tag
488 lwz r12,FPUowner(r6) ; Get back our thread
489 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
490 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
491 beq-- fsbackout ; If disowned, just toss savearea...
492 lwz r4,facAct(r12) ; Get the activation associated with live context
493 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
494 stw r4,SAVact(r3) ; Indicate the right activation for this context
495 lwz r9,FPUlevel(r12) ; Get our current level indicator again
496 stw r3,FPUsave(r12) ; Set this as the most current floating point context
497 stw r8,SAVprev+4(r3) ; And then chain this in front
498
499 stw r9,SAVlevel(r3) ; Show level in savearea
500
501 bl fp_store ; save all 32 FPRs in the save area at r3
502 mtlr r2 ; Restore return
503
504 fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
505 isync
506
507 blr
508
509 fsbackout: mr r4,r0 ; restore the original MSR
510 b EXT(save_ret_wMSR) ; Toss savearea and return from there...
511
512 /*
513 * fpu_switch()
514 *
515 * Entered to handle the floating-point unavailable exception and
516 * switch fpu context
517 *
518 * This code is run in virtual address mode on with interrupts off.
519 *
520 * Upon exit, the code returns to the users context with the floating
521 * point facility turned on.
522 *
523 * ENTRY: VM switched ON
524 * Interrupts OFF
525 * State is saved in savearea pointed to by R4.
526 * All other registers are free.
527 *
528 */
529
530 .align 5
531 .globl EXT(fpu_switch)
532
533 LEXT(fpu_switch)
534
535 #if DEBUG
536 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
537 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
538 lwz r1,0(r3)
539 addi r1,r1,1
540 stw r1,0(r3)
541 #endif /* DEBUG */
542
543 mfsprg r26,0 ; Get the per_processor block
544 mfmsr r19 ; Get the current MSR
545
546 mr r25,r4 ; Save the entry savearea
547 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
548 lwz r10,PP_ACTIVE_THREAD(r26) ; Get the pointer to the active thread
549 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
550 lwz r17,THREAD_TOP_ACT(r10) ; Now get the activation that is running
551
552 mtmsr r19 ; Enable floating point instructions
553 isync
554
555 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
556 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
557
558 ; R22 has the "old" context anchor
559 ; R29 has the "new" context anchor
560
561 #if FPVECDBG
562 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
563 li r2,0x7F01 ; (TEST/DEBUG)
564 mr r3,r22 ; (TEST/DEBUG)
565 mr r5,r29 ; (TEST/DEBUG)
566 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
567 sc ; (TEST/DEBUG)
568 #endif
569
570 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
571
572 fswretry: mr. r22,r22 ; See if there is any live FP status
573
574 beq- fsnosave ; No live context, so nothing to save...
575
576 isync ; Make sure we see this in the right order
577
578 lwz r30,FPUsave(r22) ; Get the top savearea
579 cmplw cr2,r22,r29 ; Are both old and new the same context?
580 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
581 cmplwi cr1,r30,0 ; Anything saved yet?
582 cmplw r18,r16 ; Make sure we are on the right processor
583 lwz r31,FPUlevel(r22) ; Get the context level
584
585 bne- fsnosave ; No, not on the same processor...
586
587 ;
588 ; Check to see if the live context has already been saved.
589 ; Also check to see if all we are here just to re-enable the MSR
590 ; and handle specially if so.
591 ;
592
593 cmplw r31,r27 ; See if the current and active levels are the same
594 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
595 li r3,0 ; Clear this
596
597 beq- fsthesame ; New and old are the same, just go enable...
598
599 beq- cr1,fsmstsave ; Not saved yet, go do it...
600
601 lwz r11,SAVlevel(r30) ; Get the level of top saved context
602
603 cmplw r31,r11 ; Are live and saved the same?
604
605 #if FPVECDBG
606 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
607 li r2,0x7F02 ; (TEST/DEBUG)
608 mr r3,r30 ; (TEST/DEBUG)
609 mr r5,r31 ; (TEST/DEBUG)
610 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
611 sc ; (TEST/DEBUG)
612 li r3,0 ; (TEST/DEBUG)
613 #endif
614
615 beq+ fsnosave ; Same level, so already saved...
616
617
618 fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
619 eieio ; Make sure everyone sees it
620 bl EXT(save_get) ; Go get a savearea
621
622 mr. r31,r31 ; Are we saving the user state?
623 la r15,FPUsync(r22) ; Point to the sync word
624 beq++ fswusave ; Yeah, no need for lock...
625 ;
626 ; Here we make sure that the live context is not tossed while we are
627 ; trying to push it. This can happen only for kernel context and
628 ; then only by a race with act_machine_sv_free.
629 ;
630 ; We only need to hold this for a very short time, so no sniffing needed.
631 ; If we find any change to the level, we just abandon.
632 ;
633 fswsync: lwarx r19,0,r15 ; Get the sync word
634 li r0,1 ; Get the lock
635 cmplwi cr1,r19,0 ; Is it unlocked?
636 stwcx. r0,0,r15 ; Store lock and test reservation
637 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
638 bne-- fswsync ; Try again if lost reservation or locked...
639
640 isync ; Toss speculation
641
642 lwz r0,FPUlevel(r22) ; Pick up the level again
643 li r7,0 ; Get unlock value
644 cmplw r0,r31 ; Same level?
645 beq++ fswusave ; Yeah, we expect it to be...
646
647 stw r7,FPUsync(r22) ; Unlock lock. No need to sync here
648
649 bl EXT(save_ret) ; Toss save area because we are abandoning save
650 b fsnosave ; Skip the save...
651
652 .align 5
653
654 fswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
655 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
656 mr. r31,r31 ; Check again if we were user level
657 stw r30,SAVprev+4(r3) ; Point us to the old context
658 stw r31,SAVlevel(r3) ; Tag our level
659 li r7,SAVfloat ; Get the floating point ID
660 stw r12,SAVact(r3) ; Make sure we point to the right guy
661 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
662
663 li r7,0 ; Get the unlock value
664
665 beq-- fswnulock ; Skip unlock if user (we did not lock it)...
666 eieio ; Make sure that these updates make it out
667 stw r7,FPUsync(r22) ; Unlock it.
668
669 fswnulock:
670
671 #if FPVECDBG
672 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
673 li r2,0x7F03 ; (TEST/DEBUG)
674 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
675 sc ; (TEST/DEBUG)
676 #endif
677
678 bl fp_store ; store all 32 FPRs
679
680 ;
681 ; The context is all saved now and the facility is free.
682 ;
683 ; If we do not we need to fill the registers with junk, because this level has
684 ; never used them before and some thieving bastard could hack the old values
685 ; of some thread! Just imagine what would happen if they could! Why, nothing
686 ; would be safe! My God! It is terrifying!
687 ;
688
689
690 fsnosave: lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
691 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
692 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
693
694 stw r16,FPUcpu(r29) ; Claim context for us
695 eieio
696
697 #if FPVECDBG
698 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
699 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
700 li r2,0x7F04 ; (TEST/DEBUG)
701 mr r1,r15 ; (TEST/DEBUG)
702 mr r3,r14 ; (TEST/DEBUG)
703 mr r5,r13 ; (TEST/DEBUG)
704 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
705 sc ; (TEST/DEBUG)
706 #endif
707
708 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
709 mulli r19,r19,ppSize ; Find offset to the owner per_proc
710 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
711 li r16,FPUowner ; Displacement to float owner
712 add r19,r18,r19 ; Point to the owner per_proc
713
714 fsinvothr: lwarx r18,r16,r19 ; Get the owner
715 sub r0,r18,r29 ; Subtract one from the other
716 sub r11,r29,r18 ; Subtract the other from the one
717 or r11,r11,r0 ; Combine them
718 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
719 and r18,r18,r11 ; Make 0 if same, unchanged if not
720 stwcx. r18,r16,r19 ; Try to invalidate it
721 bne-- fsinvothr ; Try again if there was a collision...
722
723 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
724 la r11,savefp0(r14) ; Point to first line to bring in
725 stw r15,FPUlevel(r29) ; Set the "new" active level
726 eieio
727 stw r29,FPUowner(r26) ; Mark us as having the live context
728
729 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
730
731 dcbt 0,r11 ; Touch line in
732
733 lwz r3,SAVprev+4(r14) ; Get the previous context
734 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
735 cmplw r0,r15 ; Top level correct to load?
736 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
737
738 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
739
740 #if FPVECDBG
741 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
742 li r2,0x7F05 ; (TEST/DEBUG)
743 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
744 sc ; (TEST/DEBUG)
745 #endif
746
747 // Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
748 // are executed on a 128-byte machine, but that is better than a mispredicted branch.
749
750 la r11,savefp4(r14) ; Point to next line
751 dcbt 0,r11 ; Touch line in
752 lfd f0, savefp0(r14)
753 lfd f1,savefp1(r14)
754 lfd f2,savefp2(r14)
755 la r11,savefp8(r14) ; Point to next line
756 lfd f3,savefp3(r14)
757 dcbt 0,r11 ; Touch line in
758 lfd f4,savefp4(r14)
759 lfd f5,savefp5(r14)
760 lfd f6,savefp6(r14)
761 la r11,savefp12(r14) ; Point to next line
762 lfd f7,savefp7(r14)
763 dcbt 0,r11 ; Touch line in
764 lfd f8,savefp8(r14)
765 lfd f9,savefp9(r14)
766 lfd f10,savefp10(r14)
767 la r11,savefp16(r14) ; Point to next line
768 lfd f11,savefp11(r14)
769 dcbt 0,r11 ; Touch line in
770 lfd f12,savefp12(r14)
771 lfd f13,savefp13(r14)
772 lfd f14,savefp14(r14)
773 la r11,savefp20(r14) ; Point to next line
774 lfd f15,savefp15(r14)
775 dcbt 0,r11 ; Touch line in
776 lfd f16,savefp16(r14)
777 lfd f17,savefp17(r14)
778 lfd f18,savefp18(r14)
779 la r11,savefp24(r14) ; Point to next line
780 lfd f19,savefp19(r14)
781 dcbt 0,r11 ; Touch line in
782 lfd f20,savefp20(r14)
783 lfd f21,savefp21(r14)
784 la r11,savefp28(r14) ; Point to next line
785 lfd f22,savefp22(r14)
786 lfd f23,savefp23(r14)
787 dcbt 0,r11 ; Touch line in
788 lfd f24,savefp24(r14)
789 lfd f25,savefp25(r14)
790 lfd f26,savefp26(r14)
791 lfd f27,savefp27(r14)
792 lfd f28,savefp28(r14)
793 lfd f29,savefp29(r14)
794 lfd f30,savefp30(r14)
795 lfd f31,savefp31(r14)
796
797 mr r3,r14 ; Get the old savearea (we popped it before)
798 bl EXT(save_ret) ; Toss it
799
800 fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
801 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
802 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
803 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
804 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
805 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
806 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
807 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
808 mr r3,r25 ; Pass the virtual addres of savearea
809 beq- fsnuser ; We are not user state...
810 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
811 stw r11,spcFlags(r26) ; Set per_proc copy
812
813 fsnuser:
814 #if FPVECDBG
815 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
816 li r2,0x7F07 ; (TEST/DEBUG)
817 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
818 sc ; (TEST/DEBUG)
819 #endif
820
821 b EXT(exception_exit) ; Exit to the fray...
822
823 /*
824 * Initialize the registers to some bogus value
825 */
826
827 MakeSureThatNoTerroristsCanHurtUsByGod:
828
829 #if FPVECDBG
830 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
831 li r2,0x7F06 ; (TEST/DEBUG)
832 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
833 sc ; (TEST/DEBUG)
834 #endif
835 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
836 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
837 lfd f0,0(r5) ; Initialize FP0
838 fmr f1,f0 ; Do them all
839 fmr f2,f0
840 fmr f3,f0
841 fmr f4,f0
842 fmr f5,f0
843 fmr f6,f0
844 fmr f7,f0
845 fmr f8,f0
846 fmr f9,f0
847 fmr f10,f0
848 fmr f11,f0
849 fmr f12,f0
850 fmr f13,f0
851 fmr f14,f0
852 fmr f15,f0
853 fmr f16,f0
854 fmr f17,f0
855 fmr f18,f0
856 fmr f19,f0
857 fmr f20,f0
858 fmr f21,f0
859 fmr f22,f0
860 fmr f23,f0
861 fmr f24,f0
862 fmr f25,f0
863 fmr f26,f0
864 fmr f27,f0
865 fmr f28,f0
866 fmr f29,f0
867 fmr f30,f0
868 fmr f31,f0
869 b fsenable ; Finish setting it all up...
870
871
872 ;
873 ; We get here when we are switching to the same context at the same level and the context
874 ; is still live. Essentially, all we are doing is turning on the faility. It may have
875 ; gotten turned off due to doing a context save for the current level or a context switch
876 ; back to the live guy.
877 ;
878
879 .align 5
880
881 fsthesame:
882
883 #if FPVECDBG
884 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
885 li r2,0x7F0A ; (TEST/DEBUG)
886 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
887 sc ; (TEST/DEBUG)
888 #endif
889 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
890
891 lwz r11,SAVlevel(r30) ; Get the level of top saved context
892 lwz r14,SAVprev+4(r30) ; Get the previous savearea
893
894 cmplw r11,r31 ; Are live and saved the same?
895
896 bne+ fsenable ; Level not the same, nothing to pop, go enable and exit...
897
898 mr r3,r30 ; Get the old savearea (we popped it before)
899 stw r14,FPUsave(r22) ; Pop the savearea from the stack
900 bl EXT(save_ret) ; Toss it
901 b fsenable ; Go enable and exit...
902
903
904 ;
905 ; This function invalidates any live floating point context for the passed in facility_context.
906 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
907 ;
908
909 .align 5
910 .globl EXT(toss_live_fpu)
911
912 LEXT(toss_live_fpu)
913
914 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
915 mfmsr r9 ; Get the MSR
916 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
917 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
918 andc r9,r9,r0 ; Force off VEC and FP
919 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
920 andc r0,r9,r0 ; Turn off EE now
921 mtmsr r0 ; No interruptions
922 isync
923 beq+ tlfnotours ; Floats off, can not be live here...
924
925 mfsprg r8,0 ; Get the per proc
926
927 ;
928 ; Note that at this point, since floats are on, we are the owner
929 ; of live state on this processor
930 ;
931
932 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
933 li r0,0 ; Clear this just in case we need it
934 cmplw r6,r3 ; Are we tossing our own context?
935 bne-- tlfnotours ; Nope...
936
937 lfd f1,Zero(0) ; Make a 0
938 mtfsf 0xFF,f1 ; Clear it
939
940 tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
941 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
942 mulli r11,r11,ppSize ; Find offset to the owner per_proc
943 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
944 li r10,FPUowner ; Displacement to float owner
945 add r11,r12,r11 ; Point to the owner per_proc
946
947 tlfinvothr: lwarx r12,r10,r11 ; Get the owner
948
949 sub r0,r12,r3 ; Subtract one from the other
950 sub r8,r3,r12 ; Subtract the other from the one
951 or r8,r8,r0 ; Combine them
952 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
953 and r12,r12,r8 ; Make 0 if same, unchanged if not
954 stwcx. r12,r10,r11 ; Try to invalidate it
955 bne-- tlfinvothr ; Try again if there was a collision...
956
957 mtmsr r9 ; Restore interruptions
958 isync ; Could be turning off floats here
959 blr ; Leave...
960
961
962 /*
963 * Altivec stuff is here. The techniques used are pretty identical to
964 * the floating point. Except that we will honor the VRSAVE register
965 * settings when loading and restoring registers.
966 *
967 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
968 * mask. VRSAVE is set by the vector user and represents the VRs that they
969 * say that they are using. The vrvalid mask indicates which vector registers
970 * are saved in the savearea. Whenever context is saved, it is saved according
971 * to the VRSAVE register. It is loaded based on VRSAVE anded with
972 * vrvalid (all other registers are splatted with 0s). This is done because we
973 * don't want to load any registers we don't have a copy of, we want to set them
974 * to zero instead.
975 *
976 * Note that there are some oddities here when we save a context we are using.
977 * It is really not too cool to do this, but what the hey... Anyway,
978 * we turn vectors and fpu off before we leave.
979 * The oddity is that if you use vectors after this, the
980 * savearea containing the context just saved will go away. So, bottom line is
981 * that don't use vectors until after you are done with the saved context.
982 *
983 */
984
985 .align 5
986 .globl EXT(vec_save)
987
988 LEXT(vec_save)
989
990
991 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
992 mfmsr r0 ; Get the MSR
993 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
994 andc r0,r0,r2 ; Force off VEC and FP
995 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
996 andc r2,r0,r2 ; Clear EE for now
997 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
998 mtmsr r2 ; Set the MSR
999 isync
1000
1001 mfsprg r6,0 ; Get the per_processor block
1002 lwz r12,VMXowner(r6) ; Get the context ID for owner
1003
1004 #if FPVECDBG
1005 mr r7,r0 ; (TEST/DEBUG)
1006 li r4,0 ; (TEST/DEBUG)
1007 mr r10,r3 ; (TEST/DEBUG)
1008 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1009 mr. r3,r12 ; (TEST/DEBUG)
1010 li r2,0x5F00 ; (TEST/DEBUG)
1011 li r5,0 ; (TEST/DEBUG)
1012 beq- noowneryeu ; (TEST/DEBUG)
1013 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
1014 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
1015
1016 noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1017 sc ; (TEST/DEBUG)
1018 mr r0,r7 ; (TEST/DEBUG)
1019 mr r3,r10 ; (TEST/DEBUG)
1020 #endif
1021 mflr r2 ; Save the return address
1022
1023 vsretry: mr. r12,r12 ; Anyone own the vector?
1024 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
1025 beq- vsret ; Nobody owns the vector, no save required...
1026
1027 cmplw cr1,r3,r12 ; Is the specified context live?
1028
1029 isync ; Force owner check first
1030
1031 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
1032 bne- cr1,vsret ; Specified context is not live
1033
1034 cmplw cr1,r9,r11 ; Was the context for this processor?
1035 beq+ cr1,vsgoodcpu ; Facility last used on this processor...
1036
1037 b vsret ; Someone else claimed this...
1038
1039 .align 5
1040
1041 vsgoodcpu: lwz r3,VMXsave(r12) ; Get the current vector savearea for the thread
1042 lwz r10,liveVRS(r6) ; Get the right VRSave register
1043 lwz r9,VMXlevel(r12) ; Get our current level indicator
1044
1045
1046 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
1047 beq- cr1,vsneedone ; Never saved it, so we need an area...
1048
1049 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
1050 mr. r10,r10 ; Is VRsave set to 0?
1051 cmplw cr1,r9,r8 ; Correct level?
1052 bne- cr1,vsneedone ; Different level, so we need to save...
1053
1054 bne+ vsret ; VRsave is non-zero so we need to keep what is saved...
1055
1056 lwz r4,SAVprev+4(r3) ; Pick up the previous area
1057 lwz r5,SAVlevel(r4) ; Get the level associated with save
1058 stw r4,VMXsave(r12) ; Dequeue this savearea
1059 li r4,0 ; Clear
1060 stw r5,VMXlevel(r12) ; Save the level
1061
1062 stw r4,VMXowner(r12) ; Show no live context here
1063 eieio
1064
1065 vsbackout: mr r4,r0 ; restore the saved MSR
1066 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
1067
1068 .align 5
1069
1070 vsneedone: mr. r10,r10 ; Is VRsave set to 0?
1071 beq- vsret ; Yeah, they do not care about any of them...
1072
1073 bl EXT(save_get) ; Get a savearea for the context
1074
1075 mfsprg r6,0 ; Get back per_processor block
1076 li r4,SAVvector ; Get vector tag
1077 lwz r12,VMXowner(r6) ; Get back our context ID
1078 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1079 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1080 beq- vsbackout ; If disowned, just toss savearea...
1081 lwz r4,facAct(r12) ; Get the activation associated with live context
1082 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1083 stw r4,SAVact(r3) ; Indicate the right activation for this context
1084 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1085 stw r3,VMXsave(r12) ; Set this as the most current floating point context
1086 stw r8,SAVprev+4(r3) ; And then chain this in front
1087
1088 stw r9,SAVlevel(r3) ; Set level in savearea
1089 mfcr r12 ; save CRs across call to vr_store
1090 lwz r10,liveVRS(r6) ; Get the right VRSave register
1091
1092 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
1093
1094 mtcrf 255,r12 ; Restore the non-volatile CRs
1095 mtlr r2 ; restore return address
1096
1097 vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1098 isync
1099
1100 blr
1101
1102 /*
1103 * vec_switch()
1104 *
1105 * Entered to handle the vector unavailable exception and
1106 * switch vector context
1107 *
1108 * This code is run with virtual address mode on and interrupts off.
1109 *
1110 * Upon exit, the code returns to the users context with the vector
1111 * facility turned on.
1112 *
1113 * ENTRY: VM switched ON
1114 * Interrupts OFF
1115 * State is saved in savearea pointed to by R4.
1116 * All other registers are free.
1117 *
1118 */
1119
1120 .align 5
1121 .globl EXT(vec_switch)
1122
1123 LEXT(vec_switch)
1124
1125 #if DEBUG
1126 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1127 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1128 lwz r1,0(r3)
1129 addi r1,r1,1
1130 stw r1,0(r3)
1131 #endif /* DEBUG */
1132
1133 mfsprg r26,0 ; Get the per_processor block
1134 mfmsr r19 ; Get the current MSR
1135
1136 mr r25,r4 ; Save the entry savearea
1137 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1138 lwz r10,PP_ACTIVE_THREAD(r26) ; Get the pointer to the active thread
1139 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1140 lwz r17,THREAD_TOP_ACT(r10) ; Now get the activation that is running
1141
1142 mtmsr r19 ; Enable vector instructions
1143 isync
1144
1145 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1146 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
1147
1148 ; R22 has the "old" context anchor
1149 ; R29 has the "new" context anchor
1150
1151 #if FPVECDBG
1152 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1153 li r2,0x5F01 ; (TEST/DEBUG)
1154 mr r3,r22 ; (TEST/DEBUG)
1155 mr r5,r29 ; (TEST/DEBUG)
1156 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1157 sc ; (TEST/DEBUG)
1158 #endif
1159
1160 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1161
1162 vsvretry: mr. r22,r22 ; See if there is any live vector status
1163
1164 beq- vsnosave ; No live context, so nothing to save...
1165
1166 isync ; Make sure we see this in the right order
1167
1168 lwz r30,VMXsave(r22) ; Get the top savearea
1169 cmplw cr2,r22,r29 ; Are both old and new the same context?
1170 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1171 cmplwi cr1,r30,0 ; Anything saved yet?
1172 cmplw r18,r16 ; Make sure we are on the right processor
1173 lwz r31,VMXlevel(r22) ; Get the context level
1174
1175 lwz r10,liveVRS(r26) ; Get the right VRSave register
1176
1177 bne- vsnosave ; No, not on the same processor...
1178
1179 ;
1180 ; Check to see if the live context has already been saved.
1181 ; Also check to see if all we are here just to re-enable the MSR
1182 ; and handle specially if so.
1183 ;
1184
1185 cmplw r31,r27 ; See if the current and active levels are the same
1186 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1187 li r8,0 ; Clear this
1188
1189 beq- vsthesame ; New and old are the same, just go enable...
1190
1191 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1192 beq- cr1,vsmstsave ; Not saved yet, go do it...
1193
1194 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1195
1196 cmplw r31,r11 ; Are live and saved the same?
1197
1198 #if FPVECDBG
1199 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1200 li r2,0x5F02 ; (TEST/DEBUG)
1201 mr r3,r30 ; (TEST/DEBUG)
1202 mr r5,r31 ; (TEST/DEBUG)
1203 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1204 sc ; (TEST/DEBUG)
1205 #endif
1206
1207 bne- vsmstsave ; Live context has not been saved yet...
1208
1209 bne- cr2,vsnosave ; Live context saved and VRSave not 0, no save and keep context...
1210
1211 lwz r4,SAVprev+4(r30) ; Pick up the previous area
1212 li r5,0 ; Assume this is the only one (which should be the ususal case)
1213 mr. r4,r4 ; Was this the only one?
1214 stw r4,VMXsave(r22) ; Dequeue this savearea
1215 beq+ vsonlyone ; This was the only one...
1216 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1217
1218 vsonlyone: stw r5,VMXlevel(r22) ; Save the level
1219 stw r8,VMXowner(r26) ; Clear owner
1220 eieio
1221 mr r3,r30 ; Copy the savearea we are tossing
1222 bl EXT(save_ret) ; Toss the savearea
1223 b vsnosave ; Go load up the context...
1224
1225 .align 5
1226
1227
1228 vsmstsave: stw r8,VMXowner(r26) ; Clear owner
1229 eieio
1230 beq- cr2,vsnosave ; The VRSave was 0, so there is nothing to save...
1231
1232 bl EXT(save_get) ; Go get a savearea
1233
1234 mr. r31,r31 ; Are we saving the user state?
1235 la r15,VMXsync(r22) ; Point to the sync word
1236 beq++ vswusave ; Yeah, no need for lock...
1237 ;
1238 ; Here we make sure that the live context is not tossed while we are
1239 ; trying to push it. This can happen only for kernel context and
1240 ; then only by a race with act_machine_sv_free.
1241 ;
1242 ; We only need to hold this for a very short time, so no sniffing needed.
1243 ; If we find any change to the level, we just abandon.
1244 ;
1245 vswsync: lwarx r19,0,r15 ; Get the sync word
1246 li r0,1 ; Get the lock
1247 cmplwi cr1,r19,0 ; Is it unlocked?
1248 stwcx. r0,0,r15 ; Store lock and test reservation
1249 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
1250 bne-- vswsync ; Try again if lost reservation or locked...
1251
1252 isync ; Toss speculation
1253
1254 lwz r0,VMXlevel(r22) ; Pick up the level again
1255 li r7,0 ; Get unlock value
1256 cmplw r0,r31 ; Same level?
1257 beq++ fswusave ; Yeah, we expect it to be...
1258
1259 stw r7,VMXsync(r22) ; Unlock lock. No need to sync here
1260
1261 bl EXT(save_ret) ; Toss save area because we are abandoning save
1262 b fsnosave ; Skip the save...
1263
1264 .align 5
1265
1266 vswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
1267 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1268 mr. r31,r31 ; Check again if we were user level
1269 stw r30,SAVprev+4(r3) ; Point us to the old context
1270 stw r31,SAVlevel(r3) ; Tag our level
1271 li r7,SAVvector ; Get the vector ID
1272 stw r12,SAVact(r3) ; Make sure we point to the right guy
1273 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
1274
1275 li r7,0 ; Get the unlock value
1276
1277 beq-- vswnulock ; Skip unlock if user (we did not lock it)...
1278 eieio ; Make sure that these updates make it out
1279 stw r7,VMXsync(r22) ; Unlock it.
1280
1281 vswnulock:
1282
1283 #if FPVECDBG
1284 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1285 li r2,0x5F03 ; (TEST/DEBUG)
1286 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1287 sc ; (TEST/DEBUG)
1288 #endif
1289
1290 lwz r10,liveVRS(r26) ; Get the right VRSave register
1291 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1292
1293
1294 ;
1295 ; The context is all saved now and the facility is free.
1296 ;
1297 ; If we do not we need to fill the registers with junk, because this level has
1298 ; never used them before and some thieving bastard could hack the old values
1299 ; of some thread! Just imagine what would happen if they could! Why, nothing
1300 ; would be safe! My God! It is terrifying!
1301 ;
1302 ; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1303 ; constant that we may need to fill unused vector registers.
1304 ;
1305
1306
1307
1308
1309 vsnosave: vspltisb v31,-10 ; Get 0xF6F6F6F6
1310 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1311 vspltisb v30,5 ; Get 0x05050505
1312 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1313 vspltish v29,4 ; Get 0x00040004
1314 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1315 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1316
1317 stw r16,VMXcpu(r29) ; Claim context for us
1318 eieio
1319
1320 #if FPVECDBG
1321 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1322 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1323 li r2,0x5F04 ; (TEST/DEBUG)
1324 mr r1,r15 ; (TEST/DEBUG)
1325 mr r3,r14 ; (TEST/DEBUG)
1326 mr r5,r13 ; (TEST/DEBUG)
1327 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1328 sc ; (TEST/DEBUG)
1329 #endif
1330
1331 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
1332 vspltisb v28,-2 ; Get 0xFEFEFEFE
1333 mulli r19,r19,ppSize ; Find offset to the owner per_proc
1334 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1335 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
1336 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1337 li r16,VMXowner ; Displacement to vector owner
1338 add r19,r18,r19 ; Point to the owner per_proc
1339 vrlb v31,v31,v29 ; Get 0xDEADDEAD
1340
1341 vsinvothr: lwarx r18,r16,r19 ; Get the owner
1342
1343 sub r0,r18,r29 ; Subtract one from the other
1344 sub r11,r29,r18 ; Subtract the other from the one
1345 or r11,r11,r0 ; Combine them
1346 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1347 and r18,r18,r11 ; Make 0 if same, unchanged if not
1348 stwcx. r18,r16,r19 ; Try to invalidate it
1349 bne-- vsinvothr ; Try again if there was a collision...
1350
1351 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
1352 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1353 stw r15,VMXlevel(r29) ; Set the "new" active level
1354 eieio
1355 stw r29,VMXowner(r26) ; Mark us as having the live context
1356
1357 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
1358
1359 lwz r3,SAVprev+4(r14) ; Get the previous context
1360 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1361 cmplw r0,r15 ; Top level correct to load?
1362 bne-- ProtectTheAmericanWay ; No, go initialize...
1363
1364 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1365
1366 #if FPVECDBG
1367 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1368 li r2,0x5F05 ; (TEST/DEBUG)
1369 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1370 sc ; (TEST/DEBUG)
1371 #endif
1372
1373 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
1374 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
1375 and r10,r10,r22 ; Figure out just what registers need to be loaded
1376 mr r3,r14 ; r3 <- ptr to savearea with VRs
1377 bl vr_load ; load VRs from save area based on vrsave in r10
1378
1379 bl EXT(save_ret) ; Toss the save area after loading VRs
1380
1381 vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
1382 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
1383 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1384 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1385 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1386 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1387 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
1388 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1389 mr r3,r25 ; Pass virtual address of the savearea
1390 beq- vrnuser ; We are not user state...
1391 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
1392 stw r11,spcFlags(r26) ; Set per_proc copy
1393
1394 vrnuser:
1395 #if FPVECDBG
1396 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1397 li r2,0x5F07 ; (TEST/DEBUG)
1398 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1399 sc ; (TEST/DEBUG)
1400 #endif
1401 b EXT(exception_exit) ; Exit to the fray...
1402
1403 /*
1404 * Initialize the registers to some bogus value
1405 */
1406
1407 ProtectTheAmericanWay:
1408
1409 #if FPVECDBG
1410 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1411 li r2,0x5F06 ; (TEST/DEBUG)
1412 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1413 sc ; (TEST/DEBUG)
1414 #endif
1415
1416 vor v0,v31,v31 ; Copy into the next register
1417 vor v1,v31,v31 ; Copy into the next register
1418 vor v2,v31,v31 ; Copy into the next register
1419 vor v3,v31,v31 ; Copy into the next register
1420 vor v4,v31,v31 ; Copy into the next register
1421 vor v5,v31,v31 ; Copy into the next register
1422 vor v6,v31,v31 ; Copy into the next register
1423 vor v7,v31,v31 ; Copy into the next register
1424 vor v8,v31,v31 ; Copy into the next register
1425 vor v9,v31,v31 ; Copy into the next register
1426 vor v10,v31,v31 ; Copy into the next register
1427 vor v11,v31,v31 ; Copy into the next register
1428 vor v12,v31,v31 ; Copy into the next register
1429 vor v13,v31,v31 ; Copy into the next register
1430 vor v14,v31,v31 ; Copy into the next register
1431 vor v15,v31,v31 ; Copy into the next register
1432 vor v16,v31,v31 ; Copy into the next register
1433 vor v17,v31,v31 ; Copy into the next register
1434 vor v18,v31,v31 ; Copy into the next register
1435 vor v19,v31,v31 ; Copy into the next register
1436 vor v20,v31,v31 ; Copy into the next register
1437 vor v21,v31,v31 ; Copy into the next register
1438 vor v22,v31,v31 ; Copy into the next register
1439 vor v23,v31,v31 ; Copy into the next register
1440 vor v24,v31,v31 ; Copy into the next register
1441 vor v25,v31,v31 ; Copy into the next register
1442 vor v26,v31,v31 ; Copy into the next register
1443 vor v27,v31,v31 ; Copy into the next register
1444 vor v28,v31,v31 ; Copy into the next register
1445 vor v29,v31,v31 ; Copy into the next register
1446 vor v30,v31,v31 ; Copy into the next register
1447 b vrenable ; Finish setting it all up...
1448
1449
1450
1451 ;
1452 ; We get here when we are switching to the same context at the same level and the context
1453 ; is still live. Essentially, all we are doing is turning on the faility. It may have
1454 ; gotten turned off due to doing a context save for the current level or a context switch
1455 ; back to the live guy.
1456 ;
1457
1458 .align 5
1459
1460 vsthesame:
1461
1462 #if FPVECDBG
1463 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1464 li r2,0x5F0A ; (TEST/DEBUG)
1465 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1466 sc ; (TEST/DEBUG)
1467 #endif
1468 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1469
1470 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1471 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1472
1473 cmplw r11,r31 ; Are live and saved the same?
1474
1475 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1476
1477 mr r3,r30 ; Get the old savearea (we popped it before)
1478 stw r11,VMXsave(r22) ; Pop the vector stack
1479 bl EXT(save_ret) ; Toss it
1480 b vrenable ; Go enable and exit...
1481
1482
1483 ;
1484 ; This function invalidates any live vector context for the passed in facility_context.
1485 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1486 ;
1487
1488 .align 5
1489 .globl EXT(toss_live_vec)
1490
1491 LEXT(toss_live_vec)
1492
1493 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1494 mfmsr r9 ; Get the MSR
1495 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1496 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1497 andc r9,r9,r0 ; Force off VEC and FP
1498 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1499 andc r0,r9,r0 ; Turn off EE now
1500 mtmsr r0 ; No interruptions
1501 isync
1502 beq+ tlvnotours ; Vector off, can not be live here...
1503
1504 mfsprg r8,0 ; Get the per proc
1505
1506 ;
1507 ; Note that at this point, since vecs are on, we are the owner
1508 ; of live state on this processor
1509 ;
1510
1511 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1512 li r0,0 ; Clear this just in case we need it
1513 cmplw r6,r3 ; Are we tossing our own context?
1514 bne- tlvnotours ; Nope...
1515
1516 vspltish v1,1 ; Turn on the non-Java bit and saturate
1517 vspltisw v0,1 ; Turn on the saturate bit
1518 vxor v1,v1,v0 ; Turn off saturate
1519 mtspr vrsave,r0 ; Clear VRSAVE
1520 mtvscr v1 ; Set the non-java, no saturate status
1521
1522 tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1523 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1524 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1525 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1526 li r10,VMXowner ; Displacement to vector owner
1527 add r11,r12,r11 ; Point to the owner per_proc
1528 li r0,0 ; Set a 0 to invalidate context
1529
1530 tlvinvothr: lwarx r12,r10,r11 ; Get the owner
1531
1532 sub r0,r12,r3 ; Subtract one from the other
1533 sub r8,r3,r12 ; Subtract the other from the one
1534 or r8,r8,r0 ; Combine them
1535 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1536 and r12,r12,r8 ; Make 0 if same, unchanged if not
1537 stwcx. r12,r10,r11 ; Try to invalidate it
1538 bne-- tlvinvothr ; Try again if there was a collision...
1539
1540 mtmsr r9 ; Restore interruptions
1541 isync ; Could be turning off vectors here
1542 blr ; Leave....
1543
1544 #if 0
1545 ;
1546 ; This function invalidates any live vector context for the passed in facility_context
1547 ; if the level is current. It also tosses the corresponding savearea if there is one.
1548 ; This function is primarily used whenever we detect a VRSave that is all zeros.
1549 ;
1550
1551 .align 5
1552 .globl EXT(vec_trash)
1553
1554 LEXT(vec_trash)
1555
1556 lwz r12,facAct(r3) ; Get the activation
1557 lwz r11,VMXlevel(r3) ; Get the context level
1558 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1559 lwz r9,VMXsave(r3) ; Get the savearea, if any
1560 cmplw r10,r11 ; Are we at the right level?
1561 cmplwi cr1,r9,0 ; Remember if there is a savearea
1562 bnelr+ ; No, we do nothing...
1563
1564 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1565 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1566 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1567 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1568 li r10,VMXowner ; Displacement to vector owner
1569 add r11,r12,r11 ; Point to the owner per_proc
1570
1571 vtinvothr: lwarx r12,r10,r11 ; Get the owner
1572
1573 sub r0,r12,r3 ; Subtract one from the other
1574 sub r8,r3,r12 ; Subtract the other from the one
1575 or r8,r8,r0 ; Combine them
1576 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1577 and r12,r12,r8 ; Make 0 if same, unchanged if not
1578 stwcx. r12,r10,r11 ; Try to invalidate it
1579 bne-- vtinvothr ; Try again if there was a collision...
1580
1581
1582 beqlr++ cr1 ; Leave if there is no savearea
1583 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1584 cmplw r8,r11 ; Savearea for the current level?
1585 bnelr++ ; No, nothing to release...
1586
1587 lwz r8,SAVprev+4(r9) ; Pick up the previous area
1588 mr. r8,r8 ; Is there a previous?
1589 beq-- vtnoprev ; Nope...
1590 lwz r7,SAVlevel(r8) ; Get the level associated with save
1591
1592 vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1593 stw r7,VMXlevel(r3) ; Pop the level
1594
1595 mr r3,r9 ; Get the savearea to release
1596 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1597 #endif
1598
1599 ;
1600 ; Just some test code to force vector and/or floating point in the kernel
1601 ;
1602
1603 .align 5
1604 .globl EXT(fctx_test)
1605
1606 LEXT(fctx_test)
1607
1608 mfsprg r3,0 ; Get the per_proc block
1609 lwz r3,PP_ACTIVE_THREAD(r3) ; Get the thread pointer
1610 mr. r3,r3 ; Are we actually up and running?
1611 beqlr- ; No...
1612
1613 fmr f0,f0 ; Use floating point
1614 mftb r4 ; Get time base for a random number
1615 li r5,1 ; Get a potential vrsave to use
1616 andi. r4,r4,0x3F ; Get a number from 0 - 63
1617 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1618 mtspr vrsave,r5 ; Set VRSave
1619 vor v0,v0,v0 ; Use vectors
1620 blr
1621
1622
1623 // *******************
1624 // * f p _ s t o r e *
1625 // *******************
1626 //
1627 // Store FPRs into a save area. Called by fpu_save and fpu_switch.
1628 //
1629 // When called:
1630 // floating pt is enabled
1631 // r3 = ptr to save area
1632 //
1633 // We destroy:
1634 // r11.
1635
1636 fp_store:
1637 mfsprg r11,2 ; get feature flags
1638 mtcrf 0x02,r11 ; put cache line size bits in cr6
1639 la r11,savefp0(r3) ; point to 1st line
1640 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1641 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1642
1643 // Store the FPRs on a 128-byte machine.
1644
1645 stfd f0,savefp0(r3)
1646 stfd f1,savefp1(r3)
1647 la r11,savefp16(r3) ; Point to the 2nd cache line
1648 stfd f2,savefp2(r3)
1649 stfd f3,savefp3(r3)
1650 dcbz128 0,r11 ; establish 2nd line
1651 stfd f4,savefp4(r3)
1652 stfd f5,savefp5(r3)
1653 stfd f6,savefp6(r3)
1654 stfd f7,savefp7(r3)
1655 stfd f8,savefp8(r3)
1656 stfd f9,savefp9(r3)
1657 stfd f10,savefp10(r3)
1658 stfd f11,savefp11(r3)
1659 stfd f12,savefp12(r3)
1660 stfd f13,savefp13(r3)
1661 stfd f14,savefp14(r3)
1662 stfd f15,savefp15(r3)
1663 stfd f16,savefp16(r3)
1664 stfd f17,savefp17(r3)
1665 stfd f18,savefp18(r3)
1666 stfd f19,savefp19(r3)
1667 stfd f20,savefp20(r3)
1668 stfd f21,savefp21(r3)
1669 stfd f22,savefp22(r3)
1670 stfd f23,savefp23(r3)
1671 stfd f24,savefp24(r3)
1672 stfd f25,savefp25(r3)
1673 stfd f26,savefp26(r3)
1674 stfd f27,savefp27(r3)
1675 stfd f28,savefp28(r3)
1676 stfd f29,savefp29(r3)
1677 stfd f30,savefp30(r3)
1678 stfd f31,savefp31(r3)
1679 blr
1680
1681 // Store FPRs on a 32-byte machine.
1682
1683 fp_st32:
1684 la r11,savefp4(r3) ; Point to the 2nd line
1685 stfd f0,savefp0(r3)
1686 dcbz 0,r11 ; Allocate cache
1687 stfd f1,savefp1(r3)
1688 stfd f2,savefp2(r3)
1689 la r11,savefp8(r3) ; Point to the 3rd line
1690 stfd f3,savefp3(r3)
1691 dcbz 0,r11 ; Allocate cache
1692 stfd f4,savefp4(r3)
1693 stfd f5,savefp5(r3)
1694 stfd f6,savefp6(r3)
1695 la r11,savefp12(r3) ; Point to the 4th line
1696 stfd f7,savefp7(r3)
1697 dcbz 0,r11 ; Allocate cache
1698 stfd f8,savefp8(r3)
1699 stfd f9,savefp9(r3)
1700 stfd f10,savefp10(r3)
1701 la r11,savefp16(r3) ; Point to the 5th line
1702 stfd f11,savefp11(r3)
1703 dcbz 0,r11 ; Allocate cache
1704 stfd f12,savefp12(r3)
1705 stfd f13,savefp13(r3)
1706 stfd f14,savefp14(r3)
1707 la r11,savefp20(r3) ; Point to the 6th line
1708 stfd f15,savefp15(r3)
1709 dcbz 0,r11 ; Allocate cache
1710 stfd f16,savefp16(r3)
1711 stfd f17,savefp17(r3)
1712 stfd f18,savefp18(r3)
1713 la r11,savefp24(r3) ; Point to the 7th line
1714 stfd f19,savefp19(r3)
1715 dcbz 0,r11 ; Allocate cache
1716 stfd f20,savefp20(r3)
1717
1718 stfd f21,savefp21(r3)
1719 stfd f22,savefp22(r3)
1720 la r11,savefp28(r3) ; Point to the 8th line
1721 stfd f23,savefp23(r3)
1722 dcbz 0,r11 ; allocate it
1723 stfd f24,savefp24(r3)
1724 stfd f25,savefp25(r3)
1725 stfd f26,savefp26(r3)
1726 stfd f27,savefp27(r3)
1727
1728 stfd f28,savefp28(r3)
1729 stfd f29,savefp29(r3)
1730 stfd f30,savefp30(r3)
1731 stfd f31,savefp31(r3)
1732 blr
1733
1734
1735 // *******************
1736 // * v r _ s t o r e *
1737 // *******************
1738 //
1739 // Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1740 // both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1741 // unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1742 // for each cache block size.
1743 //
1744 // When called:
1745 // interrupts are off, vectors are enabled
1746 // r3 = ptr to save area
1747 // r10 = vrsave (not 0)
1748 //
1749 // We destroy:
1750 // r4 - r11, all CRs.
1751
1752 vr_store:
1753 mfsprg r9,2 ; get feature flags
1754 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1755 slwi r8,r10,1 ; Shift over 1
1756 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1757 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1758 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1759
1760
1761 ; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1762 ; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1763
1764 slwi r7,r8,2 ; shift groups-of-2 over by 2
1765 li r4,16 ; load offsets for X-form stores
1766 or r8,r7,r8 ; show if any in group of 4 are in use
1767 li r5,32
1768 slwi r7,r8,4 ; shift groups-of-4 over by 4
1769 li r6,48
1770 or r11,r7,r8 ; show if any in group of 8 are in use
1771 li r7,64
1772 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1773 li r8,80
1774 mtcrf 0x20,r11
1775 li r9,96
1776 mtcrf 0x08,r11
1777 li r10,112
1778 mtcrf 0x02,r11
1779
1780 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1781 la r11,savevr0(r3) ; get address of this group of registers in save area
1782 dcbz128 0,r11 ; zero the line
1783 stvxl v0,0,r11 ; save 8 VRs in the line
1784 stvxl v1,r4,r11
1785 stvxl v2,r5,r11
1786 stvxl v3,r6,r11
1787 stvxl v4,r7,r11
1788 stvxl v5,r8,r11
1789 stvxl v6,r9,r11
1790 stvxl v7,r10,r11
1791
1792 vr_st64b:
1793 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
1794 la r11,savevr8(r3) ; get address of this group of registers in save area
1795 dcbz128 0,r11 ; zero the line
1796 stvxl v8,0,r11 ; save 8 VRs in the line
1797 stvxl v9,r4,r11
1798 stvxl v10,r5,r11
1799 stvxl v11,r6,r11
1800 stvxl v12,r7,r11
1801 stvxl v13,r8,r11
1802 stvxl v14,r9,r11
1803 stvxl v15,r10,r11
1804
1805 vr_st64c:
1806 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
1807 la r11,savevr16(r3) ; get address of this group of registers in save area
1808 dcbz128 0,r11 ; zero the line
1809 stvxl v16,0,r11 ; save 8 VRs in the line
1810 stvxl v17,r4,r11
1811 stvxl v18,r5,r11
1812 stvxl v19,r6,r11
1813 stvxl v20,r7,r11
1814 stvxl v21,r8,r11
1815 stvxl v22,r9,r11
1816 stvxl v23,r10,r11
1817
1818 vr_st64d:
1819 bflr 24 ; done if none of vr24-vr31 are in use
1820 la r11,savevr24(r3) ; get address of this group of registers in save area
1821 dcbz128 0,r11 ; zero the line
1822 stvxl v24,0,r11 ; save 8 VRs in the line
1823 stvxl v25,r4,r11
1824 stvxl v26,r5,r11
1825 stvxl v27,r6,r11
1826 stvxl v28,r7,r11
1827 stvxl v29,r8,r11
1828 stvxl v30,r9,r11
1829 stvxl v31,r10,r11
1830 blr
1831
1832 ; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
1833 ; or neither in each group. This cuts down on conditional branches.
1834 ; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
1835 ; r3 = savearea
1836
1837 vr_st32:
1838 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
1839 li r4,16 ; load offset for X-form stores
1840
1841 bf 0,vr_st32b ; skip if neither VR in this pair is in use
1842 la r11,savevr0(r3) ; get address of this group of registers in save area
1843 dcba 0,r11 ; establish the line wo reading it
1844 stvxl v0,0,r11 ; save the two VRs in the line
1845 stvxl v1,r4,r11
1846
1847 vr_st32b:
1848 bf 2,vr_st32c ; skip if neither VR in this pair is in use
1849 la r11,savevr2(r3) ; get address of this group of registers in save area
1850 dcba 0,r11 ; establish the line wo reading it
1851 stvxl v2,0,r11 ; save the two VRs in the line
1852 stvxl v3,r4,r11
1853
1854 vr_st32c:
1855 bf 4,vr_st32d ; skip if neither VR in this pair is in use
1856 la r11,savevr4(r3) ; get address of this group of registers in save area
1857 dcba 0,r11 ; establish the line wo reading it
1858 stvxl v4,0,r11 ; save the two VRs in the line
1859 stvxl v5,r4,r11
1860
1861 vr_st32d:
1862 bf 6,vr_st32e ; skip if neither VR in this pair is in use
1863 la r11,savevr6(r3) ; get address of this group of registers in save area
1864 dcba 0,r11 ; establish the line wo reading it
1865 stvxl v6,0,r11 ; save the two VRs in the line
1866 stvxl v7,r4,r11
1867
1868 vr_st32e:
1869 bf 8,vr_st32f ; skip if neither VR in this pair is in use
1870 la r11,savevr8(r3) ; get address of this group of registers in save area
1871 dcba 0,r11 ; establish the line wo reading it
1872 stvxl v8,0,r11 ; save the two VRs in the line
1873 stvxl v9,r4,r11
1874
1875 vr_st32f:
1876 bf 10,vr_st32g ; skip if neither VR in this pair is in use
1877 la r11,savevr10(r3) ; get address of this group of registers in save area
1878 dcba 0,r11 ; establish the line wo reading it
1879 stvxl v10,0,r11 ; save the two VRs in the line
1880 stvxl v11,r4,r11
1881
1882 vr_st32g:
1883 bf 12,vr_st32h ; skip if neither VR in this pair is in use
1884 la r11,savevr12(r3) ; get address of this group of registers in save area
1885 dcba 0,r11 ; establish the line wo reading it
1886 stvxl v12,0,r11 ; save the two VRs in the line
1887 stvxl v13,r4,r11
1888
1889 vr_st32h:
1890 bf 14,vr_st32i ; skip if neither VR in this pair is in use
1891 la r11,savevr14(r3) ; get address of this group of registers in save area
1892 dcba 0,r11 ; establish the line wo reading it
1893 stvxl v14,0,r11 ; save the two VRs in the line
1894 stvxl v15,r4,r11
1895
1896 vr_st32i:
1897 bf 16,vr_st32j ; skip if neither VR in this pair is in use
1898 la r11,savevr16(r3) ; get address of this group of registers in save area
1899 dcba 0,r11 ; establish the line wo reading it
1900 stvxl v16,0,r11 ; save the two VRs in the line
1901 stvxl v17,r4,r11
1902
1903 vr_st32j:
1904 bf 18,vr_st32k ; skip if neither VR in this pair is in use
1905 la r11,savevr18(r3) ; get address of this group of registers in save area
1906 dcba 0,r11 ; establish the line wo reading it
1907 stvxl v18,0,r11 ; save the two VRs in the line
1908 stvxl v19,r4,r11
1909
1910 vr_st32k:
1911 bf 20,vr_st32l ; skip if neither VR in this pair is in use
1912 la r11,savevr20(r3) ; get address of this group of registers in save area
1913 dcba 0,r11 ; establish the line wo reading it
1914 stvxl v20,0,r11 ; save the two VRs in the line
1915 stvxl v21,r4,r11
1916
1917 vr_st32l:
1918 bf 22,vr_st32m ; skip if neither VR in this pair is in use
1919 la r11,savevr22(r3) ; get address of this group of registers in save area
1920 dcba 0,r11 ; establish the line wo reading it
1921 stvxl v22,0,r11 ; save the two VRs in the line
1922 stvxl v23,r4,r11
1923
1924 vr_st32m:
1925 bf 24,vr_st32n ; skip if neither VR in this pair is in use
1926 la r11,savevr24(r3) ; get address of this group of registers in save area
1927 dcba 0,r11 ; establish the line wo reading it
1928 stvxl v24,0,r11 ; save the two VRs in the line
1929 stvxl v25,r4,r11
1930
1931 vr_st32n:
1932 bf 26,vr_st32o ; skip if neither VR in this pair is in use
1933 la r11,savevr26(r3) ; get address of this group of registers in save area
1934 dcba 0,r11 ; establish the line wo reading it
1935 stvxl v26,0,r11 ; save the two VRs in the line
1936 stvxl v27,r4,r11
1937
1938 vr_st32o:
1939 bf 28,vr_st32p ; skip if neither VR in this pair is in use
1940 la r11,savevr28(r3) ; get address of this group of registers in save area
1941 dcba 0,r11 ; establish the line wo reading it
1942 stvxl v28,0,r11 ; save the two VRs in the line
1943 stvxl v29,r4,r11
1944
1945 vr_st32p:
1946 bflr 30 ; done if neither VR in this pair is in use
1947 la r11,savevr30(r3) ; get address of this group of registers in save area
1948 dcba 0,r11 ; establish the line wo reading it
1949 stvxl v30,0,r11 ; save the two VRs in the line
1950 stvxl v31,r4,r11
1951 blr
1952
1953
1954 // *****************
1955 // * v r _ l o a d *
1956 // *****************
1957 //
1958 // Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
1959 // of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
1960 // branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
1961 // in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
1962 // which would bug all VRs that aren't live.
1963 //
1964 // When called:
1965 // interrupts are off, vectors are enabled
1966 // r3 = ptr to save area
1967 // r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
1968 // v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
1969 //
1970 // We destroy:
1971 // r4 - r11, all CRs.
1972
1973 vr_load:
1974 mfsprg r9,2 ; get feature flags
1975 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
1976 mtcrf 0x02,r9 ; set cache line size bits in cr6
1977 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
1978 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
1979 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
1980 lis r7,0xFF00 ; so set bits 0-7
1981
1982 // Loop touching in cache blocks we will load from.
1983 // r3 = savearea ptr
1984 // r5 = we light bits for the VRs we will be loading
1985 // r6 = 1 if 32-byte, 7 if 128-byte
1986 // r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
1987 // r10 = live VR bits
1988 // v31 = bugbug constant
1989
1990 vr_ld0:
1991 li r5,0 ; initialize set of VRs to load
1992 la r11,savevr0(r3) ; get address of register file
1993 b vr_ld2 ; enter loop in middle
1994
1995 .align 5
1996 vr_ld1: ; loop over each cache line we will load
1997 dcbt r4,r11 ; start prefetch of the line
1998 andc r10,r10,r9 ; turn off the bits in this line
1999 or r5,r5,r9 ; we will load all these
2000 vr_ld2: ; initial entry pt
2001 cntlzw r4,r10 ; get offset to next live VR
2002 andc r4,r4,r6 ; cacheline align it
2003 srw. r9,r7,r4 ; position bits for VRs in that cache line
2004 slwi r4,r4,4 ; get byte offset within register file to that line
2005 bne vr_ld1 ; loop if more bits in r10
2006
2007 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
2008
2009 // Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
2010 // r3 = savearea ptr
2011 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2012 // r11 = addr(savevr0)
2013 // v31 = bugbug constant
2014
2015 mtcrf 0x80,r5 ; set up bits for conditional branches
2016 li r4,16 ; load offsets for X-form stores
2017 li r6,48
2018 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
2019 li r7,64
2020 li r8,80
2021 mtcrf 0x08,r5
2022 li r9,96
2023 li r10,112
2024 mtcrf 0x02,r5
2025 li r5,32
2026
2027 bt 0,vr_ld128a ; skip if this line must be loaded
2028 vor v0,v31,v31 ; no VR must be loaded, so bug them all
2029 vor v1,v31,v31
2030 vor v2,v31,v31
2031 vor v3,v31,v31
2032 vor v4,v31,v31
2033 vor v5,v31,v31
2034 vor v6,v31,v31
2035 vor v7,v31,v31
2036 b vr_ld128b
2037 vr_ld128a: ; must load from this line
2038 lvxl v0,0,r11
2039 lvxl v1,r4,r11
2040 lvxl v2,r5,r11
2041 lvxl v3,r6,r11
2042 lvxl v4,r7,r11
2043 lvxl v5,r8,r11
2044 lvxl v6,r9,r11
2045 lvxl v7,r10,r11
2046
2047 vr_ld128b: ; here to handle next cache line
2048 la r11,savevr8(r3) ; load offset to it
2049 bt 8,vr_ld128c ; skip if this line must be loaded
2050 vor v8,v31,v31 ; no VR must be loaded, so bug them all
2051 vor v9,v31,v31
2052 vor v10,v31,v31
2053 vor v11,v31,v31
2054 vor v12,v31,v31
2055 vor v13,v31,v31
2056 vor v14,v31,v31
2057 vor v15,v31,v31
2058 b vr_ld128d
2059 vr_ld128c: ; must load from this line
2060 lvxl v8,0,r11
2061 lvxl v9,r4,r11
2062 lvxl v10,r5,r11
2063 lvxl v11,r6,r11
2064 lvxl v12,r7,r11
2065 lvxl v13,r8,r11
2066 lvxl v14,r9,r11
2067 lvxl v15,r10,r11
2068
2069 vr_ld128d: ; here to handle next cache line
2070 la r11,savevr16(r3) ; load offset to it
2071 bt 16,vr_ld128e ; skip if this line must be loaded
2072 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2073 vor v17,v31,v31
2074 vor v18,v31,v31
2075 vor v19,v31,v31
2076 vor v20,v31,v31
2077 vor v21,v31,v31
2078 vor v22,v31,v31
2079 vor v23,v31,v31
2080 b vr_ld128f
2081 vr_ld128e: ; must load from this line
2082 lvxl v16,0,r11
2083 lvxl v17,r4,r11
2084 lvxl v18,r5,r11
2085 lvxl v19,r6,r11
2086 lvxl v20,r7,r11
2087 lvxl v21,r8,r11
2088 lvxl v22,r9,r11
2089 lvxl v23,r10,r11
2090
2091 vr_ld128f: ; here to handle next cache line
2092 la r11,savevr24(r3) ; load offset to it
2093 bt 24,vr_ld128g ; skip if this line must be loaded
2094 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2095 vor v25,v31,v31
2096 vor v26,v31,v31
2097 vor v27,v31,v31
2098 vor v28,v31,v31
2099 vor v29,v31,v31
2100 vor v30,v31,v31
2101 blr
2102 vr_ld128g: ; must load from this line
2103 lvxl v24,0,r11
2104 lvxl v25,r4,r11
2105 lvxl v26,r5,r11
2106 lvxl v27,r6,r11
2107 lvxl v28,r7,r11
2108 lvxl v29,r8,r11
2109 lvxl v30,r9,r11
2110 lvxl v31,r10,r11
2111 blr
2112
2113 // Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2114 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2115 // r11 = addr(savevr0)
2116
2117 vr_ld32:
2118 mtcrf 0xFF,r5 ; set up bits for conditional branches
2119 li r4,16 ; load offset for X-form stores
2120
2121 bt 0,vr_ld32load0 ; skip if we must load this line
2122 vor v0,v31,v31 ; neither VR is live, so bug them both
2123 vor v1,v31,v31
2124 b vr_ld32test2
2125 vr_ld32load0: ; must load VRs in this line
2126 lvxl v0,0,r11
2127 lvxl v1,r4,r11
2128
2129 vr_ld32test2: ; here to handle next cache line
2130 la r11,savevr2(r3) ; get offset to next cache line
2131 bt 2,vr_ld32load2 ; skip if we must load this line
2132 vor v2,v31,v31 ; neither VR is live, so bug them both
2133 vor v3,v31,v31
2134 b vr_ld32test4
2135 vr_ld32load2: ; must load VRs in this line
2136 lvxl v2,0,r11
2137 lvxl v3,r4,r11
2138
2139 vr_ld32test4: ; here to handle next cache line
2140 la r11,savevr4(r3) ; get offset to next cache line
2141 bt 4,vr_ld32load4 ; skip if we must load this line
2142 vor v4,v31,v31 ; neither VR is live, so bug them both
2143 vor v5,v31,v31
2144 b vr_ld32test6
2145 vr_ld32load4: ; must load VRs in this line
2146 lvxl v4,0,r11
2147 lvxl v5,r4,r11
2148
2149 vr_ld32test6: ; here to handle next cache line
2150 la r11,savevr6(r3) ; get offset to next cache line
2151 bt 6,vr_ld32load6 ; skip if we must load this line
2152 vor v6,v31,v31 ; neither VR is live, so bug them both
2153 vor v7,v31,v31
2154 b vr_ld32test8
2155 vr_ld32load6: ; must load VRs in this line
2156 lvxl v6,0,r11
2157 lvxl v7,r4,r11
2158
2159 vr_ld32test8: ; here to handle next cache line
2160 la r11,savevr8(r3) ; get offset to next cache line
2161 bt 8,vr_ld32load8 ; skip if we must load this line
2162 vor v8,v31,v31 ; neither VR is live, so bug them both
2163 vor v9,v31,v31
2164 b vr_ld32test10
2165 vr_ld32load8: ; must load VRs in this line
2166 lvxl v8,0,r11
2167 lvxl v9,r4,r11
2168
2169 vr_ld32test10: ; here to handle next cache line
2170 la r11,savevr10(r3) ; get offset to next cache line
2171 bt 10,vr_ld32load10 ; skip if we must load this line
2172 vor v10,v31,v31 ; neither VR is live, so bug them both
2173 vor v11,v31,v31
2174 b vr_ld32test12
2175 vr_ld32load10: ; must load VRs in this line
2176 lvxl v10,0,r11
2177 lvxl v11,r4,r11
2178
2179 vr_ld32test12: ; here to handle next cache line
2180 la r11,savevr12(r3) ; get offset to next cache line
2181 bt 12,vr_ld32load12 ; skip if we must load this line
2182 vor v12,v31,v31 ; neither VR is live, so bug them both
2183 vor v13,v31,v31
2184 b vr_ld32test14
2185 vr_ld32load12: ; must load VRs in this line
2186 lvxl v12,0,r11
2187 lvxl v13,r4,r11
2188
2189 vr_ld32test14: ; here to handle next cache line
2190 la r11,savevr14(r3) ; get offset to next cache line
2191 bt 14,vr_ld32load14 ; skip if we must load this line
2192 vor v14,v31,v31 ; neither VR is live, so bug them both
2193 vor v15,v31,v31
2194 b vr_ld32test16
2195 vr_ld32load14: ; must load VRs in this line
2196 lvxl v14,0,r11
2197 lvxl v15,r4,r11
2198
2199 vr_ld32test16: ; here to handle next cache line
2200 la r11,savevr16(r3) ; get offset to next cache line
2201 bt 16,vr_ld32load16 ; skip if we must load this line
2202 vor v16,v31,v31 ; neither VR is live, so bug them both
2203 vor v17,v31,v31
2204 b vr_ld32test18
2205 vr_ld32load16: ; must load VRs in this line
2206 lvxl v16,0,r11
2207 lvxl v17,r4,r11
2208
2209 vr_ld32test18: ; here to handle next cache line
2210 la r11,savevr18(r3) ; get offset to next cache line
2211 bt 18,vr_ld32load18 ; skip if we must load this line
2212 vor v18,v31,v31 ; neither VR is live, so bug them both
2213 vor v19,v31,v31
2214 b vr_ld32test20
2215 vr_ld32load18: ; must load VRs in this line
2216 lvxl v18,0,r11
2217 lvxl v19,r4,r11
2218
2219 vr_ld32test20: ; here to handle next cache line
2220 la r11,savevr20(r3) ; get offset to next cache line
2221 bt 20,vr_ld32load20 ; skip if we must load this line
2222 vor v20,v31,v31 ; neither VR is live, so bug them both
2223 vor v21,v31,v31
2224 b vr_ld32test22
2225 vr_ld32load20: ; must load VRs in this line
2226 lvxl v20,0,r11
2227 lvxl v21,r4,r11
2228
2229 vr_ld32test22: ; here to handle next cache line
2230 la r11,savevr22(r3) ; get offset to next cache line
2231 bt 22,vr_ld32load22 ; skip if we must load this line
2232 vor v22,v31,v31 ; neither VR is live, so bug them both
2233 vor v23,v31,v31
2234 b vr_ld32test24
2235 vr_ld32load22: ; must load VRs in this line
2236 lvxl v22,0,r11
2237 lvxl v23,r4,r11
2238
2239 vr_ld32test24: ; here to handle next cache line
2240 la r11,savevr24(r3) ; get offset to next cache line
2241 bt 24,vr_ld32load24 ; skip if we must load this line
2242 vor v24,v31,v31 ; neither VR is live, so bug them both
2243 vor v25,v31,v31
2244 b vr_ld32test26
2245 vr_ld32load24: ; must load VRs in this line
2246 lvxl v24,0,r11
2247 lvxl v25,r4,r11
2248
2249 vr_ld32test26: ; here to handle next cache line
2250 la r11,savevr26(r3) ; get offset to next cache line
2251 bt 26,vr_ld32load26 ; skip if we must load this line
2252 vor v26,v31,v31 ; neither VR is live, so bug them both
2253 vor v27,v31,v31
2254 b vr_ld32test28
2255 vr_ld32load26: ; must load VRs in this line
2256 lvxl v26,0,r11
2257 lvxl v27,r4,r11
2258
2259 vr_ld32test28: ; here to handle next cache line
2260 la r11,savevr28(r3) ; get offset to next cache line
2261 bt 28,vr_ld32load28 ; skip if we must load this line
2262 vor v28,v31,v31 ; neither VR is live, so bug them both
2263 vor v29,v31,v31
2264 b vr_ld32test30
2265 vr_ld32load28: ; must load VRs in this line
2266 lvxl v28,0,r11
2267 lvxl v29,r4,r11
2268
2269 vr_ld32test30: ; here to handle next cache line
2270 la r11,savevr30(r3) ; get offset to next cache line
2271 bt 30,vr_ld32load30 ; skip if we must load this line
2272 vor v30,v31,v31 ; neither VR is live, so bug them both
2273 blr
2274 vr_ld32load30: ; must load VRs in this line
2275 lvxl v30,0,r11
2276 lvxl v31,r4,r11
2277 blr
mirror of XNU