]> git.saurik.com Git - apple/xnu.git/blame - osfmk/ppc/cswtch.s
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-344.21.74.tar.gz
[apple/xnu.git] / osfmk / ppc / cswtch.s
CommitLineData
1c79356b
A		 1/*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
d7e50217 6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b 7 *
d7e50217
A		 8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * file.
14 *
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b
A		 17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
d7e50217
A		 19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
1c79356b
A		 22 *
23 * @APPLE_LICENSE_HEADER_END@
24 */
25/*
26 * @OSF_COPYRIGHT@
27 */
28
29#include <ppc/asm.h>
30#include <ppc/proc_reg.h>
31#include <cpus.h>
32#include <assym.s>
33#include <debug.h>
34#include <mach/ppc/vm_param.h>
35#include <ppc/exception.h>
9bccf70c 36#include <ppc/savearea.h>
1c79356b
A		 37
38#define FPVECDBG 0
39#define GDDBG 0
40
41 .text
42
43/*
44 * void load_context(thread_t thread)
45 *
46 * Load the context for the first kernel thread, and go.
47 *
48 * NOTE - if DEBUG is set, the former routine is a piece
49 * of C capable of printing out debug info before calling the latter,
50 * otherwise both entry points are identical.
51 */
52
9bccf70c
A		 53 .align 5
54 .globl EXT(load_context)
55
56LEXT(load_context)
57
58 .globl EXT(Load_context)
59
60LEXT(Load_context)
1c79356b
A		 61
62/*
63 * Since this is the first thread, we came in on the interrupt
64 * stack. The first thread never returns, so there is no need to
d7e50217 65 * worry about saving its frame, hence we can reset the istackptr
1c79356b
A		 66 * back to the saved_state structure at it's top
67 */
68
69
70/*
71 * get new thread pointer and set it into the active_threads pointer
72 *
73 */
74
75 mfsprg r6,0
76 lwz r0,PP_INTSTACK_TOP_SS(r6)
1c79356b 77 stw r0,PP_ISTACKPTR(r6)
9bccf70c 78 stw r3,PP_ACTIVE_THREAD(r6)
1c79356b
A		 79
80/* Find the new stack and store it in active_stacks */
81
82 lwz r12,PP_ACTIVE_STACKS(r6)
83 lwz r1,THREAD_KERNEL_STACK(r3)
84 lwz r9,THREAD_TOP_ACT(r3) /* Point to the active activation */
d7e50217 85 mtsprg 1,r9
1c79356b
A		 86 stw r1,0(r12)
87 li r0,0 /* Clear a register */
d7e50217 88 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
1c79356b 89 mfmsr r5 /* Since we are passing control, get our MSR values */
d7e50217
A		 90 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
91 lwz r1,saver1+4(r3) /* Load new stack pointer */
92 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
1c79356b 93 stw r0,FM_BACKPTR(r1) /* zero backptr */
d7e50217 94 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
1c79356b
A		 95 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
96 b EXT(exception_exit) /* Go end it all... */
97
98/* struct thread_shuttle *Switch_context(struct thread_shuttle *old,
99 * void (*cont)(void),
100 * struct thread_shuttle *new)
101 *
102 * Switch from one thread to another. If a continuation is supplied, then
103 * we do not need to save callee save registers.
104 *
105 */
106
107/* void Call_continuation( void (*continuation)(void), vm_offset_t stack_ptr)
108 */
109
9bccf70c
A		 110 .align 5
111 .globl EXT(Call_continuation)
112
113LEXT(Call_continuation)
114
115 mtlr r3
116 mr r1, r4 /* Load new stack pointer */
117 blr /* Jump to the continuation */
1c79356b
A		 118
119/*
120 * Get the old kernel stack, and store into the thread structure.
121 * See if a continuation is supplied, and skip state save if so.
d7e50217
A		 122 *
123 * Note that interrupts must be disabled before we get here (i.e., splsched)
1c79356b
A		 124 */
125
126/* Context switches are double jumps. We pass the following to the
127 * context switch firmware call:
128 *
d7e50217 129 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
1c79356b
A		 130 * R4 = old thread
131 * R5 = new SRR0
132 * R6 = new SRR1
d7e50217 133 * R7 = high order physical address of savearea for full switch
1c79356b
A		 134 *
135 * savesrr0 is set to go to switch_in
136 * savesrr1 is set to uninterruptible with translation on
137 */
138
139
9bccf70c
A		 140 .align 5
141 .globl EXT(Switch_context)
1c79356b 142
9bccf70c
A		 143LEXT(Switch_context)
144
d7e50217 145 lwz r11,THREAD_KERNEL_STACK(r5) ; Get the new stack pointer
9bccf70c
A		 146 mfsprg r12,0 ; Get the per_proc block
147 lwz r10,PP_ACTIVE_STACKS(r12) ; Get the pointer to the current stack
1c79356b 148#if DEBUG
d7e50217
A		 149 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
150 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
151 bne++ notonintstack ; (DEBUG/TRACE) stack
1c79356b
A		 152 BREAKPOINT_TRAP
153notonintstack:
154#endif
d7e50217
A		 155
156#if 0
157 lwz r8,lgPPStart(0) ; (TEST/DEBUG) Get the start of per_procs
158 sub r7,r12,r8 ; (TEST/DEBUG) Find offset to our per_proc
159 xori r7,r7,0x1000 ; (TEST/DEBUG) Switch to other proc
160 add r8,r8,r7 ; (TEST/DEBUG) Switch to it
161 lwz r8,PP_ACTIVE_THREAD(r8) ; (TEST/DEBUG) Get the other active thread
162 cmplw r8,r5 ; (TEST/DEBUG) Trying to switch to an active thread?
163 bne++ snively ; (TEST/DEBUG) Nope...
164 BREAKPOINT_TRAP ; (TEST/DEBUG) Get to debugger...
165
166snively: ; (TEST/DEBUG)
167#endif
168
169 stw r5,PP_ACTIVE_THREAD(r12) ; Make the new thread current
170 lwz r5,THREAD_TOP_ACT(r5) ; Get the new activation
171 stw r4,THREAD_CONTINUATION(r3) ; Set continuation into the thread
9bccf70c 172 lwz r7,0(r10) ; Get the current stack
d7e50217
A		 173 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
174 stw r11,0(r10) ; Save the new kernel stack address
175
176 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
177 lwz r9,cioSpace(r5) ; Get copyin/out address space
9bccf70c 178 stw r7,THREAD_KERNEL_STACK(r3) ; Remember the current stack in the thread (do not need???)
d7e50217 179 mtsprg 1,r5 ; Set the current activation pointer
1c79356b 180 lwz r7,CTHREAD_SELF(r5) ; Pick up the user assist word
d7e50217
A		 181 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
182 lwz r6,cioRelo(r5) ; Get copyin/out relocation top
183 lwz r2,cioRelo+4(r5) ; Get copyin/out relocation bottom
9bccf70c 184
9bccf70c 185 stw r7,UAW(r12) ; Save the assist word for the "ultra fast path"
d7e50217 186
1c79356b
A		 187 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
188
d7e50217
A		 189 lwz r0,ACT_KLOADED(r5)
190 sth r9,ppCIOmp+mpSpace(r12) ; Save the space
191 stw r6,ppCIOmp+mpNestReloc(r12) ; Save top part of physical address
192 stw r2,ppCIOmp+mpNestReloc+4(r12) ; Save bottom part of physical address
193 lwz r10,PP_ACTIVE_KLOADED(r12) ; Get kernel loaded flag address
194 subfic r0,r0,0 ; Get bit 0 to 0 if not kloaded, 1 otherwise
195 lwz r2,traceMask(0) ; Get the enabled traces
9bccf70c 196 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
d7e50217 197 srawi r0,r0,31 ; Get 0 if not kloaded, ffffffff otherwise
9bccf70c 198 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
d7e50217 199 and r0,r5,r0 ; Get 0 if not kloaded, activation otherwise
1c79356b 200
d7e50217
A		 201 mr. r2,r2 ; Any tracing going on?
202 stw r0,0(r10) ; Set the kloaded stuff
1c79356b 203 lis r0,hi16(CutTrace) ; Trace FW call
d7e50217 204 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
1c79356b 205 ori r0,r0,lo16(CutTrace) ; Trace FW call
1c79356b 206 mr r10,r3 ; Save across trace
d7e50217 207 beq++ cswNoTrc ; No trace today, dude...
1c79356b
A		 208 lwz r2,THREAD_TOP_ACT(r3) ; Trace old activation
209 mr r3,r11 ; Trace prev savearea
210 sc ; Cut trace entry of context switch
211 mr r3,r10 ; Restore
212
d7e50217 213cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
9bccf70c
A		 214 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
215 lwz r9,VMXowner(r12) ; Grab the owner of the vector
d7e50217
A		 216 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
217 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
218 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
219 li r0,1 ; Get set to hold off quickfret
220
221 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
9bccf70c
A		 222 cmplw r10,r2 ; Do we have the live float context?
223 lwz r10,FPUlevel(r2) ; Get the live level
d7e50217 224 mr r4,r3 ; Save our old thread to pass back
9bccf70c 225 cmplw cr5,r9,r2 ; Do we have the live vector context?
d7e50217
A		 226 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
227 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
228 bne++ cswnofloat ; Float is not ours...
9bccf70c
A		 229
230 cmplw r10,r11 ; Is the level the same?
231 lwz r5,FPUcpu(r2) ; Get the owning cpu
d7e50217 232 bne++ cswnofloat ; Level not the same, this is not live...
9bccf70c
A		 233
234 cmplw r5,r0 ; Still owned by this cpu?
235 lwz r10,FPUsave(r2) ; Get the level
d7e50217 236 bne++ cswnofloat ; CPU claimed by someone else...
9bccf70c
A		 237
238 mr. r10,r10 ; Is there a savearea here?
239 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
240
d7e50217 241 beq-- cswnofloat ; No savearea to check...
9bccf70c
A		 242
243 lwz r3,SAVlevel(r10) ; Get the level
d7e50217 244 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
9bccf70c
A		 245 cmplw r3,r11 ; Is it for the current level?
246
d7e50217 247 bne++ cswnofloat ; Nope...
9bccf70c
A		 248
249 stw r5,FPUsave(r2) ; Pop off this savearea
d7e50217
A		 250
251 rlwinm r3,r10,0,0,19 ; Move back to start of page
252
253 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
254 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
255 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
256 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
257 stw r5,SAVprev(r10) ; Link the old in (top)
258 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
259 xor r3,r10,r3 ; Convert to physical
260 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
261 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
9bccf70c
A		 262
263#if FPVECDBG
264 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
d7e50217 265 mr r7,r2 ; (TEST/DEBUG)
9bccf70c
A		 266 li r2,0x4401 ; (TEST/DEBUG)
267 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
268 sc ; (TEST/DEBUG)
269 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
d7e50217 270 mr r2,r7 ; (TEST/DEBUG)
9bccf70c
A		 271#endif
272
d7e50217 273cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
9bccf70c
A		 274
275 lwz r10,VMXlevel(r2) ; Get the live level
276
277 cmplw r10,r11 ; Is the level the same?
278 lwz r5,VMXcpu(r2) ; Get the owning cpu
d7e50217 279 bne++ cswnovect ; Level not the same, this is not live...
9bccf70c
A		 280
281 cmplw r5,r0 ; Still owned by this cpu?
282 lwz r10,VMXsave(r2) ; Get the level
d7e50217 283 bne++ cswnovect ; CPU claimed by someone else...
9bccf70c
A		 284
285 mr. r10,r10 ; Is there a savearea here?
286 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
287
d7e50217 288 beq-- cswnovect ; No savearea to check...
9bccf70c
A		 289
290 lwz r3,SAVlevel(r10) ; Get the level
d7e50217 291 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
9bccf70c
A		 292 cmplw r3,r11 ; Is it for the current level?
293
d7e50217 294 bne++ cswnovect ; Nope...
9bccf70c
A		 295
296 stw r5,VMXsave(r2) ; Pop off this savearea
d7e50217
A		 297 rlwinm r3,r10,0,0,19 ; Move back to start of page
298
299 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
300 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
301 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
302 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
303 stw r5,SAVprev(r10) ; Link the old in (top)
304 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
305 xor r3,r10,r3 ; Convert to physical
306 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
307 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
9bccf70c
A		 308
309#if FPVECDBG
310 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
311 li r2,0x4501 ; (TEST/DEBUG)
312 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
313 sc ; (TEST/DEBUG)
314#endif
315
d7e50217
A		 316cswnovect: li r0,0 ; Get set to release quickfret holdoff
317 rlwinm r11,r8,0,0,19 ; Switch to savearea base
318 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
319 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
320 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
321 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
322 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
323 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
324 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
9bccf70c 325
d7e50217 326 lwz r9,SAVflags(r8) /* Get the flags */
9bccf70c 327 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
1c79356b 328 li r10,MSR_SUPERVISOR_INT_OFF /* Get the switcher's MSR */
1c79356b 329 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
d7e50217
A		 330 stw r10,savesrr1+4(r8) /* Set up for switch in */
331 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
332 xor r3,r11,r8 /* Get the physical address of the new context save area */
1c79356b 333 stw r9,SAVflags(r8) /* Set the flags */
9bccf70c
A		 334
335 bne cr1,swtchtocont ; Switch to the continuation
1c79356b
A		 336 sc /* Switch to the new context */
337
338/* We come back here in the new thread context
339 * R4 was set to hold the old thread pointer, but switch_in will put it into
340 * R3 where it belongs.
341 */
342 blr /* Jump into the new thread */
9bccf70c
A		 343
344;
345; This is where we go when a continuation is set. We are actually
346; killing off the old context of the new guy so we need to pop off
347; any float or vector states for the ditched level.
348;
349; Note that we do the same kind of thing a chkfac in hw_exceptions.s
350;
351
1c79356b 352
9bccf70c 353swtchtocont:
d7e50217
A		 354
355 stw r5,savesrr0+4(r8) ; Set the pc
356 stw r6,savesrr1+4(r8) ; Set the next MSR to use
357 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
358 mr r3,r8 ; Pass in the virtual address of savearea
9bccf70c
A		 359
360 b EXT(exception_exit) ; Blocking on continuation, toss old context...
1c79356b
A		 361
362
363
364/*
365 * All switched to threads come here first to clean up the old thread.
366 * We need to do the following contortions because we need to keep
367 * the LR clean. And because we need to manipulate the savearea chain
368 * with translation on. If we could, this should be done in lowmem_vectors
369 * before translation is turned on. But we can't, dang it!
370 *
d7e50217 371 * R3 = switcher's savearea (32-bit virtual)
1c79356b
A		 372 * saver4 = old thread in switcher's save
373 * saver5 = new SRR0 in switcher's save
374 * saver6 = new SRR1 in switcher's save
375
376
377 */
378
9bccf70c
A		 379
380 .align 5
381 .globl EXT(switch_in)
382
383LEXT(switch_in)
1c79356b 384
d7e50217
A		 385 lwz r4,saver4+4(r3) ; Get the old thread
386 lwz r5,saver5+4(r3) ; Get the srr0 value
387
388 mfsprg r0,2 ; Get feature flags
389 lwz r9,THREAD_TOP_ACT(r4) ; Get the switched from ACT
390 lwz r6,saver6+4(r3) ; Get the srr1 value
391 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
392 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
1c79356b 393
d7e50217
A		 394 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
395 stw r10,SAVprev+4(r3) ; Chain on the old one
1c79356b 396
d7e50217 397 mr r3,r4 ; Pass back the old thread
1c79356b 398
d7e50217
A		 399 mtsrr0 r5 ; Set return point
400 mtsrr1 r6 ; Set return MSR
401
402 bne++ siSixtyFour ; Go do 64-bit...
1c79356b 403
d7e50217
A		 404 rfi ; Jam...
405
406siSixtyFour:
407 rfid ; Jam...
1c79356b
A		 408
409/*
9bccf70c 410 * void fpu_save(facility_context ctx)
1c79356b 411 *
9bccf70c
A		 412 * Note that there are some oddities here when we save a context we are using.
413 * It is really not too cool to do this, but what the hey... Anyway,
414 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
415 * savearea containing the context just saved will go away. So, bottom line is
416 * that don't use fpus until after you are done with the saved context.
1c79356b 417 */
9bccf70c
A		 418 .align 5
419 .globl EXT(fpu_save)
1c79356b 420
9bccf70c
A		 421LEXT(fpu_save)
422
d7e50217
A		 423 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
424 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
425 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
9bccf70c
A		 426
427 mfmsr r0 ; Get the MSR
d7e50217
A		 428 andc r0,r0,r2 ; Clear FP, VEC
429 andc r2,r0,r12 ; Clear EE
9bccf70c 430 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
9bccf70c 431 mtmsr r2 ; Set the MSR
1c79356b 432 isync
9bccf70c
A		 433
434 mfsprg r6,0 ; Get the per_processor block
435 lwz r12,FPUowner(r6) ; Get the context ID for owner
436
1c79356b 437#if FPVECDBG
9bccf70c
A		 438 mr r7,r0 ; (TEST/DEBUG)
439 li r4,0 ; (TEST/DEBUG)
440 mr r10,r3 ; (TEST/DEBUG)
441 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
442 mr. r3,r12 ; (TEST/DEBUG)
443 li r2,0x6F00 ; (TEST/DEBUG)
444 li r5,0 ; (TEST/DEBUG)
d7e50217 445 beq-- noowneryet ; (TEST/DEBUG)
9bccf70c
A		 446 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
447 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
448
449noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
450 sc ; (TEST/DEBUG)
451 mr r0,r7 ; (TEST/DEBUG)
452 mr r3,r10 ; (TEST/DEBUG)
1c79356b 453#endif
9bccf70c
A		 454 mflr r2 ; Save the return address
455
456fsretry: mr. r12,r12 ; Anyone own the FPU?
457 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
d7e50217 458 beq-- fsret ; Nobody owns the FPU, no save required...
1c79356b 459
9bccf70c 460 cmplw cr1,r3,r12 ; Is the specified context live?
1c79356b 461
9bccf70c 462 isync ; Force owner check first
1c79356b 463
9bccf70c 464 lwz r9,FPUcpu(r12) ; Get the cpu that context was last on
d7e50217 465 bne-- cr1,fsret ; No, it is not...
1c79356b 466
9bccf70c 467 cmplw cr1,r9,r11 ; Was the context for this processor?
d7e50217 468 beq-- cr1,fsgoodcpu ; Facility last used on this processor...
0b4e3aa0 469
9bccf70c 470 b fsret ; Someone else claimed it...
1c79356b 471
9bccf70c 472 .align 5
1c79356b 473
9bccf70c
A		 474fsgoodcpu: lwz r3,FPUsave(r12) ; Get the current FPU savearea for the thread
475 lwz r9,FPUlevel(r12) ; Get our current level indicator
1c79356b 476
9bccf70c
A		 477 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
478 beq- cr1,fsneedone ; Never saved it, so go do it...
1c79356b 479
9bccf70c
A		 480 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
481 cmplw cr1,r9,r8 ; Correct level?
d7e50217 482 beq-- cr1,fsret ; The current level is already saved, bail out...
1c79356b 483
9bccf70c
A		 484fsneedone: bl EXT(save_get) ; Get a savearea for the context
485
486 mfsprg r6,0 ; Get back per_processor block
487 li r4,SAVfloat ; Get floating point tag
488 lwz r12,FPUowner(r6) ; Get back our thread
489 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
490 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
d7e50217 491 beq-- fsbackout ; If disowned, just toss savearea...
9bccf70c 492 lwz r4,facAct(r12) ; Get the activation associated with live context
9bccf70c
A		 493 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
494 stw r4,SAVact(r3) ; Indicate the right activation for this context
495 lwz r9,FPUlevel(r12) ; Get our current level indicator again
496 stw r3,FPUsave(r12) ; Set this as the most current floating point context
d7e50217 497 stw r8,SAVprev+4(r3) ; And then chain this in front
1c79356b 498
9bccf70c 499 stw r9,SAVlevel(r3) ; Show level in savearea
1c79356b 500
d7e50217
A		 501 bl fp_store ; save all 32 FPRs in the save area at r3
502 mtlr r2 ; Restore return
503
9bccf70c 504fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
1c79356b
A		 505 isync
506
507 blr
508
d7e50217
A		 509fsbackout: mr r4,r0 ; restore the original MSR
510 b EXT(save_ret_wMSR) ; Toss savearea and return from there...
9bccf70c 511
1c79356b
A		 512/*
513 * fpu_switch()
514 *
515 * Entered to handle the floating-point unavailable exception and
516 * switch fpu context
517 *
518 * This code is run in virtual address mode on with interrupts off.
519 *
520 * Upon exit, the code returns to the users context with the floating
521 * point facility turned on.
522 *
523 * ENTRY: VM switched ON
524 * Interrupts OFF
525 * State is saved in savearea pointed to by R4.
526 * All other registers are free.
527 *
528 */
529
9bccf70c
A		 530 .align 5
531 .globl EXT(fpu_switch)
532
533LEXT(fpu_switch)
534
1c79356b 535#if DEBUG
1c79356b
A		 536 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
537 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
538 lwz r1,0(r3)
539 addi r1,r1,1
540 stw r1,0(r3)
1c79356b
A		 541#endif /* DEBUG */
542
9bccf70c
A		 543 mfsprg r26,0 ; Get the per_processor block
544 mfmsr r19 ; Get the current MSR
1c79356b 545
9bccf70c
A		 546 mr r25,r4 ; Save the entry savearea
547 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
548 lwz r10,PP_ACTIVE_THREAD(r26) ; Get the pointer to the active thread
549 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
550 lwz r17,THREAD_TOP_ACT(r10) ; Now get the activation that is running
1c79356b 551
9bccf70c 552 mtmsr r19 ; Enable floating point instructions
1c79356b 553 isync
1c79356b 554
9bccf70c
A		 555 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
556 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
0b4e3aa0 557
9bccf70c
A		 558; R22 has the "old" context anchor
559; R29 has the "new" context anchor
0b4e3aa0 560
1c79356b 561#if FPVECDBG
9bccf70c
A		 562 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
563 li r2,0x7F01 ; (TEST/DEBUG)
564 mr r3,r22 ; (TEST/DEBUG)
565 mr r5,r29 ; (TEST/DEBUG)
566 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
567 sc ; (TEST/DEBUG)
1c79356b 568#endif
9bccf70c
A		 569
570 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1c79356b 571
9bccf70c 572fswretry: mr. r22,r22 ; See if there is any live FP status
1c79356b 573
9bccf70c 574 beq- fsnosave ; No live context, so nothing to save...
1c79356b 575
9bccf70c 576 isync ; Make sure we see this in the right order
1c79356b 577
9bccf70c
A		 578 lwz r30,FPUsave(r22) ; Get the top savearea
579 cmplw cr2,r22,r29 ; Are both old and new the same context?
580 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
581 cmplwi cr1,r30,0 ; Anything saved yet?
582 cmplw r18,r16 ; Make sure we are on the right processor
583 lwz r31,FPUlevel(r22) ; Get the context level
1c79356b 584
9bccf70c
A		 585 bne- fsnosave ; No, not on the same processor...
586
1c79356b 587;
9bccf70c
A		 588; Check to see if the live context has already been saved.
589; Also check to see if all we are here just to re-enable the MSR
590; and handle specially if so.
1c79356b 591;
9bccf70c
A		 592
593 cmplw r31,r27 ; See if the current and active levels are the same
594 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
595 li r3,0 ; Clear this
1c79356b 596
9bccf70c
A		 597 beq- fsthesame ; New and old are the same, just go enable...
598
599 beq- cr1,fsmstsave ; Not saved yet, go do it...
1c79356b 600
9bccf70c 601 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1c79356b 602
9bccf70c
A		 603 cmplw r31,r11 ; Are live and saved the same?
604
1c79356b 605#if FPVECDBG
9bccf70c
A		 606 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
607 li r2,0x7F02 ; (TEST/DEBUG)
608 mr r3,r30 ; (TEST/DEBUG)
609 mr r5,r31 ; (TEST/DEBUG)
610 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
611 sc ; (TEST/DEBUG)
d7e50217 612 li r3,0 ; (TEST/DEBUG)
1c79356b 613#endif
9bccf70c
A		 614
615 beq+ fsnosave ; Same level, so already saved...
616
1c79356b 617
9bccf70c
A		 618fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
619 eieio ; Make sure everyone sees it
620 bl EXT(save_get) ; Go get a savearea
621
d7e50217
A		 622 mr. r31,r31 ; Are we saving the user state?
623 la r15,FPUsync(r22) ; Point to the sync word
624 beq++ fswusave ; Yeah, no need for lock...
625;
626; Here we make sure that the live context is not tossed while we are
627; trying to push it. This can happen only for kernel context and
628; then only by a race with act_machine_sv_free.
629;
630; We only need to hold this for a very short time, so no sniffing needed.
631; If we find any change to the level, we just abandon.
632;
633fswsync: lwarx r19,0,r15 ; Get the sync word
634 li r0,1 ; Get the lock
635 cmplwi cr1,r19,0 ; Is it unlocked?
636 stwcx. r0,0,r15 ; Store lock and test reservation
637 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
638 bne-- fswsync ; Try again if lost reservation or locked...
1c79356b 639
d7e50217
A		 640 isync ; Toss speculation
641
642 lwz r0,FPUlevel(r22) ; Pick up the level again
643 li r7,0 ; Get unlock value
644 cmplw r0,r31 ; Same level?
645 beq++ fswusave ; Yeah, we expect it to be...
646
647 stw r7,FPUsync(r22) ; Unlock lock. No need to sync here
648
649 bl EXT(save_ret) ; Toss save area because we are abandoning save
650 b fsnosave ; Skip the save...
651
652 .align 5
653
654fswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
655 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
656 mr. r31,r31 ; Check again if we were user level
657 stw r30,SAVprev+4(r3) ; Point us to the old context
9bccf70c
A		 658 stw r31,SAVlevel(r3) ; Tag our level
659 li r7,SAVfloat ; Get the floating point ID
660 stw r12,SAVact(r3) ; Make sure we point to the right guy
661 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
d7e50217
A		 662
663 li r7,0 ; Get the unlock value
664
665 beq-- fswnulock ; Skip unlock if user (we did not lock it)...
666 eieio ; Make sure that these updates make it out
667 stw r7,FPUsync(r22) ; Unlock it.
668
669fswnulock:
1c79356b 670
1c79356b 671#if FPVECDBG
9bccf70c
A		 672 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
673 li r2,0x7F03 ; (TEST/DEBUG)
674 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
675 sc ; (TEST/DEBUG)
1c79356b
A		 676#endif
677
d7e50217 678 bl fp_store ; store all 32 FPRs
1c79356b 679
1c79356b
A		 680;
681; The context is all saved now and the facility is free.
682;
9bccf70c 683; If we do not we need to fill the registers with junk, because this level has
1c79356b
A		 684; never used them before and some thieving bastard could hack the old values
685; of some thread! Just imagine what would happen if they could! Why, nothing
686; would be safe! My God! It is terrifying!
687;
688
0b4e3aa0 689
9bccf70c
A		 690fsnosave: lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
691 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
692 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
0b4e3aa0 693
9bccf70c
A		 694 stw r16,FPUcpu(r29) ; Claim context for us
695 eieio
0b4e3aa0 696
1c79356b 697#if FPVECDBG
9bccf70c
A		 698 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
699 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
700 li r2,0x7F04 ; (TEST/DEBUG)
701 mr r1,r15 ; (TEST/DEBUG)
702 mr r3,r14 ; (TEST/DEBUG)
703 mr r5,r13 ; (TEST/DEBUG)
704 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
705 sc ; (TEST/DEBUG)
1c79356b 706#endif
9bccf70c
A		 707
708 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
709 mulli r19,r19,ppSize ; Find offset to the owner per_proc
710 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
711 li r16,FPUowner ; Displacement to float owner
712 add r19,r18,r19 ; Point to the owner per_proc
9bccf70c
A		 713
714fsinvothr: lwarx r18,r16,r19 ; Get the owner
d7e50217
A		 715 sub r0,r18,r29 ; Subtract one from the other
716 sub r11,r29,r18 ; Subtract the other from the one
717 or r11,r11,r0 ; Combine them
718 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
719 and r18,r18,r11 ; Make 0 if same, unchanged if not
720 stwcx. r18,r16,r19 ; Try to invalidate it
721 bne-- fsinvothr ; Try again if there was a collision...
9bccf70c 722
d7e50217 723 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
9bccf70c
A		 724 la r11,savefp0(r14) ; Point to first line to bring in
725 stw r15,FPUlevel(r29) ; Set the "new" active level
726 eieio
727 stw r29,FPUowner(r26) ; Mark us as having the live context
1c79356b 728
d7e50217 729 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
9bccf70c
A		 730
731 dcbt 0,r11 ; Touch line in
732
d7e50217 733 lwz r3,SAVprev+4(r14) ; Get the previous context
9bccf70c
A		 734 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
735 cmplw r0,r15 ; Top level correct to load?
d7e50217 736 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
9bccf70c
A		 737
738 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
1c79356b
A		 739
740#if FPVECDBG
9bccf70c
A		 741 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
742 li r2,0x7F05 ; (TEST/DEBUG)
743 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
744 sc ; (TEST/DEBUG)
1c79356b
A		 745#endif
746
d7e50217
A		 747// Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
748// are executed on a 128-byte machine, but that is better than a mispredicted branch.
749
9bccf70c
A		 750 la r11,savefp4(r14) ; Point to next line
751 dcbt 0,r11 ; Touch line in
1c79356b
A		 752 lfd f0, savefp0(r14)
753 lfd f1,savefp1(r14)
1c79356b 754 lfd f2,savefp2(r14)
9bccf70c 755 la r11,savefp8(r14) ; Point to next line
1c79356b 756 lfd f3,savefp3(r14)
9bccf70c 757 dcbt 0,r11 ; Touch line in
1c79356b
A		 758 lfd f4,savefp4(r14)
759 lfd f5,savefp5(r14)
760 lfd f6,savefp6(r14)
9bccf70c 761 la r11,savefp12(r14) ; Point to next line
1c79356b 762 lfd f7,savefp7(r14)
9bccf70c 763 dcbt 0,r11 ; Touch line in
1c79356b
A		 764 lfd f8,savefp8(r14)
765 lfd f9,savefp9(r14)
766 lfd f10,savefp10(r14)
9bccf70c 767 la r11,savefp16(r14) ; Point to next line
1c79356b 768 lfd f11,savefp11(r14)
9bccf70c 769 dcbt 0,r11 ; Touch line in
1c79356b
A		 770 lfd f12,savefp12(r14)
771 lfd f13,savefp13(r14)
772 lfd f14,savefp14(r14)
9bccf70c 773 la r11,savefp20(r14) ; Point to next line
1c79356b 774 lfd f15,savefp15(r14)
9bccf70c 775 dcbt 0,r11 ; Touch line in
1c79356b
A		 776 lfd f16,savefp16(r14)
777 lfd f17,savefp17(r14)
778 lfd f18,savefp18(r14)
9bccf70c 779 la r11,savefp24(r14) ; Point to next line
1c79356b 780 lfd f19,savefp19(r14)
9bccf70c 781 dcbt 0,r11 ; Touch line in
1c79356b
A		 782 lfd f20,savefp20(r14)
783 lfd f21,savefp21(r14)
9bccf70c 784 la r11,savefp28(r14) ; Point to next line
1c79356b
A		 785 lfd f22,savefp22(r14)
786 lfd f23,savefp23(r14)
9bccf70c 787 dcbt 0,r11 ; Touch line in
1c79356b
A		 788 lfd f24,savefp24(r14)
789 lfd f25,savefp25(r14)
790 lfd f26,savefp26(r14)
791 lfd f27,savefp27(r14)
792 lfd f28,savefp28(r14)
793 lfd f29,savefp29(r14)
794 lfd f30,savefp30(r14)
795 lfd f31,savefp31(r14)
796
9bccf70c
A		 797 mr r3,r14 ; Get the old savearea (we popped it before)
798 bl EXT(save_ret) ; Toss it
799
d7e50217 800fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
9bccf70c 801 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
d7e50217
A		 802 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
803 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1c79356b 804 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
d7e50217 805 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
9bccf70c 806 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
d7e50217
A		 807 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
808 mr r3,r25 ; Pass the virtual addres of savearea
b4c24cb9 809 beq- fsnuser ; We are not user state...
9bccf70c 810 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
d7e50217 811 stw r11,spcFlags(r26) ; Set per_proc copy
1c79356b
A		 812
813fsnuser:
814#if FPVECDBG
9bccf70c
A		 815 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
816 li r2,0x7F07 ; (TEST/DEBUG)
817 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
818 sc ; (TEST/DEBUG)
1c79356b 819#endif
1c79356b 820
9bccf70c 821 b EXT(exception_exit) ; Exit to the fray...
1c79356b
A		 822
823/*
824 * Initialize the registers to some bogus value
825 */
826
827MakeSureThatNoTerroristsCanHurtUsByGod:
0b4e3aa0 828
1c79356b 829#if FPVECDBG
9bccf70c
A		 830 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
831 li r2,0x7F06 ; (TEST/DEBUG)
832 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
833 sc ; (TEST/DEBUG)
1c79356b 834#endif
9bccf70c
A		 835 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
836 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
837 lfd f0,0(r5) ; Initialize FP0
838 fmr f1,f0 ; Do them all
1c79356b
A		 839 fmr f2,f0
840 fmr f3,f0
841 fmr f4,f0
842 fmr f5,f0
843 fmr f6,f0
844 fmr f7,f0
845 fmr f8,f0
846 fmr f9,f0
847 fmr f10,f0
848 fmr f11,f0
849 fmr f12,f0
850 fmr f13,f0
851 fmr f14,f0
852 fmr f15,f0
853 fmr f16,f0
854 fmr f17,f0
1c79356b
A		 855 fmr f18,f0
856 fmr f19,f0
857 fmr f20,f0
1c79356b
A		 858 fmr f21,f0
859 fmr f22,f0
860 fmr f23,f0
861 fmr f24,f0
862 fmr f25,f0
863 fmr f26,f0
864 fmr f27,f0
865 fmr f28,f0
866 fmr f29,f0
867 fmr f30,f0
868 fmr f31,f0
9bccf70c
A		 869 b fsenable ; Finish setting it all up...
870
1c79356b
A		 871
872;
9bccf70c
A		 873; We get here when we are switching to the same context at the same level and the context
874; is still live. Essentially, all we are doing is turning on the faility. It may have
875; gotten turned off due to doing a context save for the current level or a context switch
876; back to the live guy.
1c79356b 877;
9bccf70c
A		 878
879 .align 5
1c79356b 880
9bccf70c 881fsthesame:
1c79356b 882
9bccf70c
A		 883#if FPVECDBG
884 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
885 li r2,0x7F0A ; (TEST/DEBUG)
886 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
887 sc ; (TEST/DEBUG)
888#endif
889 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
1c79356b 890
9bccf70c 891 lwz r11,SAVlevel(r30) ; Get the level of top saved context
d7e50217 892 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1c79356b 893
9bccf70c 894 cmplw r11,r31 ; Are live and saved the same?
1c79356b 895
9bccf70c 896 bne+ fsenable ; Level not the same, nothing to pop, go enable and exit...
1c79356b 897
9bccf70c 898 mr r3,r30 ; Get the old savearea (we popped it before)
d7e50217 899 stw r14,FPUsave(r22) ; Pop the savearea from the stack
9bccf70c
A		 900 bl EXT(save_ret) ; Toss it
901 b fsenable ; Go enable and exit...
902
903
904;
905; This function invalidates any live floating point context for the passed in facility_context.
906; This is intended to be called just before act_machine_sv_free tosses saveareas.
907;
908
909 .align 5
910 .globl EXT(toss_live_fpu)
911
912LEXT(toss_live_fpu)
913
d7e50217 914 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
9bccf70c 915 mfmsr r9 ; Get the MSR
d7e50217 916 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
9bccf70c 917 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
d7e50217
A		 918 andc r9,r9,r0 ; Force off VEC and FP
919 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
920 andc r0,r9,r0 ; Turn off EE now
9bccf70c
A		 921 mtmsr r0 ; No interruptions
922 isync
923 beq+ tlfnotours ; Floats off, can not be live here...
924
925 mfsprg r8,0 ; Get the per proc
926
927;
928; Note that at this point, since floats are on, we are the owner
929; of live state on this processor
1c79356b 930;
9bccf70c
A		 931
932 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
933 li r0,0 ; Clear this just in case we need it
934 cmplw r6,r3 ; Are we tossing our own context?
d7e50217 935 bne-- tlfnotours ; Nope...
9bccf70c 936
d7e50217 937 lfd f1,Zero(0) ; Make a 0
9bccf70c
A		 938 mtfsf 0xFF,f1 ; Clear it
939
940tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
941 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
942 mulli r11,r11,ppSize ; Find offset to the owner per_proc
943 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
944 li r10,FPUowner ; Displacement to float owner
945 add r11,r12,r11 ; Point to the owner per_proc
9bccf70c
A		 946
947tlfinvothr: lwarx r12,r10,r11 ; Get the owner
9bccf70c 948
d7e50217
A		 949 sub r0,r12,r3 ; Subtract one from the other
950 sub r8,r3,r12 ; Subtract the other from the one
951 or r8,r8,r0 ; Combine them
952 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
953 and r12,r12,r8 ; Make 0 if same, unchanged if not
954 stwcx. r12,r10,r11 ; Try to invalidate it
955 bne-- tlfinvothr ; Try again if there was a collision...
956
957 mtmsr r9 ; Restore interruptions
9bccf70c
A		 958 isync ; Could be turning off floats here
959 blr ; Leave...
960
1c79356b
A		 961
962/*
963 * Altivec stuff is here. The techniques used are pretty identical to
964 * the floating point. Except that we will honor the VRSAVE register
965 * settings when loading and restoring registers.
966 *
967 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
968 * mask. VRSAVE is set by the vector user and represents the VRs that they
969 * say that they are using. The vrvalid mask indicates which vector registers
970 * are saved in the savearea. Whenever context is saved, it is saved according
971 * to the VRSAVE register. It is loaded based on VRSAVE anded with
972 * vrvalid (all other registers are splatted with 0s). This is done because we
973 * don't want to load any registers we don't have a copy of, we want to set them
974 * to zero instead.
975 *
9bccf70c
A		 976 * Note that there are some oddities here when we save a context we are using.
977 * It is really not too cool to do this, but what the hey... Anyway,
978 * we turn vectors and fpu off before we leave.
979 * The oddity is that if you use vectors after this, the
980 * savearea containing the context just saved will go away. So, bottom line is
981 * that don't use vectors until after you are done with the saved context.
982 *
1c79356b
A		 983 */
984
9bccf70c
A		 985 .align 5
986 .globl EXT(vec_save)
987
988LEXT(vec_save)
1c79356b 989
d7e50217
A		 990
991 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
9bccf70c 992 mfmsr r0 ; Get the MSR
d7e50217
A		 993 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
994 andc r0,r0,r2 ; Force off VEC and FP
995 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
996 andc r2,r0,r2 ; Clear EE for now
9bccf70c 997 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
9bccf70c 998 mtmsr r2 ; Set the MSR
1c79356b
A		 999 isync
1000
9bccf70c
A		 1001 mfsprg r6,0 ; Get the per_processor block
1002 lwz r12,VMXowner(r6) ; Get the context ID for owner
1003
1c79356b 1004#if FPVECDBG
9bccf70c
A		 1005 mr r7,r0 ; (TEST/DEBUG)
1006 li r4,0 ; (TEST/DEBUG)
1007 mr r10,r3 ; (TEST/DEBUG)
1008 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1009 mr. r3,r12 ; (TEST/DEBUG)
1010 li r2,0x5F00 ; (TEST/DEBUG)
1011 li r5,0 ; (TEST/DEBUG)
1012 beq- noowneryeu ; (TEST/DEBUG)
1013 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
1014 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
1015
1016noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1017 sc ; (TEST/DEBUG)
1018 mr r0,r7 ; (TEST/DEBUG)
1019 mr r3,r10 ; (TEST/DEBUG)
1c79356b 1020#endif
9bccf70c
A		 1021 mflr r2 ; Save the return address
1022
1023vsretry: mr. r12,r12 ; Anyone own the vector?
1024 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
1025 beq- vsret ; Nobody owns the vector, no save required...
1026
1027 cmplw cr1,r3,r12 ; Is the specified context live?
1028
1029 isync ; Force owner check first
1030
1031 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
1032 bne- cr1,vsret ; Specified context is not live
1033
1034 cmplw cr1,r9,r11 ; Was the context for this processor?
1035 beq+ cr1,vsgoodcpu ; Facility last used on this processor...
1036
1037 b vsret ; Someone else claimed this...
1038
1039 .align 5
1040
1041vsgoodcpu: lwz r3,VMXsave(r12) ; Get the current vector savearea for the thread
1042 lwz r10,liveVRS(r6) ; Get the right VRSave register
1043 lwz r9,VMXlevel(r12) ; Get our current level indicator
1044
1045
1046 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
1047 beq- cr1,vsneedone ; Never saved it, so we need an area...
1048
1049 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
1050 mr. r10,r10 ; Is VRsave set to 0?
1051 cmplw cr1,r9,r8 ; Correct level?
1052 bne- cr1,vsneedone ; Different level, so we need to save...
1c79356b 1053
9bccf70c 1054 bne+ vsret ; VRsave is non-zero so we need to keep what is saved...
0b4e3aa0 1055
d7e50217 1056 lwz r4,SAVprev+4(r3) ; Pick up the previous area
9bccf70c
A		 1057 lwz r5,SAVlevel(r4) ; Get the level associated with save
1058 stw r4,VMXsave(r12) ; Dequeue this savearea
d7e50217 1059 li r4,0 ; Clear
9bccf70c
A		 1060 stw r5,VMXlevel(r12) ; Save the level
1061
d7e50217 1062 stw r4,VMXowner(r12) ; Show no live context here
9bccf70c
A		 1063 eieio
1064
d7e50217
A		 1065vsbackout: mr r4,r0 ; restore the saved MSR
1066 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
9bccf70c
A		 1067
1068 .align 5
1069
1070vsneedone: mr. r10,r10 ; Is VRsave set to 0?
1071 beq- vsret ; Yeah, they do not care about any of them...
1072
1073 bl EXT(save_get) ; Get a savearea for the context
1074
1075 mfsprg r6,0 ; Get back per_processor block
1076 li r4,SAVvector ; Get vector tag
1077 lwz r12,VMXowner(r6) ; Get back our context ID
1078 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1079 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1080 beq- vsbackout ; If disowned, just toss savearea...
1081 lwz r4,facAct(r12) ; Get the activation associated with live context
9bccf70c
A		 1082 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1083 stw r4,SAVact(r3) ; Indicate the right activation for this context
1084 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1085 stw r3,VMXsave(r12) ; Set this as the most current floating point context
d7e50217 1086 stw r8,SAVprev+4(r3) ; And then chain this in front
9bccf70c
A		 1087
1088 stw r9,SAVlevel(r3) ; Set level in savearea
d7e50217
A		 1089 mfcr r12 ; save CRs across call to vr_store
1090 lwz r10,liveVRS(r6) ; Get the right VRSave register
1091
1092 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
9bccf70c 1093
d7e50217
A		 1094 mtcrf 255,r12 ; Restore the non-volatile CRs
1095 mtlr r2 ; restore return address
1096
1097vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1c79356b
A		 1098 isync
1099
1100 blr
1101
1102/*
1103 * vec_switch()
1104 *
1105 * Entered to handle the vector unavailable exception and
1106 * switch vector context
1107 *
1108 * This code is run with virtual address mode on and interrupts off.
1109 *
1110 * Upon exit, the code returns to the users context with the vector
1111 * facility turned on.
1112 *
1113 * ENTRY: VM switched ON
1114 * Interrupts OFF
1115 * State is saved in savearea pointed to by R4.
1116 * All other registers are free.
1117 *
1118 */
1119
9bccf70c
A		 1120 .align 5
1121 .globl EXT(vec_switch)
1122
1123LEXT(vec_switch)
1c79356b
A		 1124
1125#if DEBUG
1c79356b
A		 1126 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1127 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1128 lwz r1,0(r3)
1129 addi r1,r1,1
1130 stw r1,0(r3)
1c79356b
A		 1131#endif /* DEBUG */
1132
9bccf70c
A		 1133 mfsprg r26,0 ; Get the per_processor block
1134 mfmsr r19 ; Get the current MSR
1c79356b 1135
9bccf70c
A		 1136 mr r25,r4 ; Save the entry savearea
1137 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1138 lwz r10,PP_ACTIVE_THREAD(r26) ; Get the pointer to the active thread
1139 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1140 lwz r17,THREAD_TOP_ACT(r10) ; Now get the activation that is running
1141
1142 mtmsr r19 ; Enable vector instructions
1c79356b
A		 1143 isync
1144
9bccf70c
A		 1145 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1146 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
0b4e3aa0 1147
9bccf70c
A		 1148; R22 has the "old" context anchor
1149; R29 has the "new" context anchor
0b4e3aa0 1150
1c79356b 1151#if FPVECDBG
9bccf70c
A		 1152 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1153 li r2,0x5F01 ; (TEST/DEBUG)
1154 mr r3,r22 ; (TEST/DEBUG)
1155 mr r5,r29 ; (TEST/DEBUG)
1156 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1157 sc ; (TEST/DEBUG)
1c79356b
A		 1158#endif
1159
9bccf70c 1160 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1c79356b 1161
9bccf70c 1162vsvretry: mr. r22,r22 ; See if there is any live vector status
1c79356b 1163
9bccf70c 1164 beq- vsnosave ; No live context, so nothing to save...
1c79356b 1165
9bccf70c 1166 isync ; Make sure we see this in the right order
1c79356b 1167
9bccf70c
A		 1168 lwz r30,VMXsave(r22) ; Get the top savearea
1169 cmplw cr2,r22,r29 ; Are both old and new the same context?
1170 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1171 cmplwi cr1,r30,0 ; Anything saved yet?
1172 cmplw r18,r16 ; Make sure we are on the right processor
1173 lwz r31,VMXlevel(r22) ; Get the context level
1c79356b 1174
9bccf70c
A		 1175 lwz r10,liveVRS(r26) ; Get the right VRSave register
1176
1177 bne- vsnosave ; No, not on the same processor...
1178
1c79356b 1179;
9bccf70c
A		 1180; Check to see if the live context has already been saved.
1181; Also check to see if all we are here just to re-enable the MSR
1182; and handle specially if so.
1c79356b 1183;
9bccf70c
A		 1184
1185 cmplw r31,r27 ; See if the current and active levels are the same
1186 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1187 li r8,0 ; Clear this
1c79356b 1188
9bccf70c
A		 1189 beq- vsthesame ; New and old are the same, just go enable...
1190
1191 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1192 beq- cr1,vsmstsave ; Not saved yet, go do it...
1c79356b 1193
9bccf70c 1194 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1c79356b 1195
9bccf70c
A		 1196 cmplw r31,r11 ; Are live and saved the same?
1197
1c79356b 1198#if FPVECDBG
9bccf70c
A		 1199 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1200 li r2,0x5F02 ; (TEST/DEBUG)
1201 mr r3,r30 ; (TEST/DEBUG)
1202 mr r5,r31 ; (TEST/DEBUG)
1203 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1204 sc ; (TEST/DEBUG)
1c79356b 1205#endif
9bccf70c
A		 1206
1207 bne- vsmstsave ; Live context has not been saved yet...
1208
1209 bne- cr2,vsnosave ; Live context saved and VRSave not 0, no save and keep context...
1c79356b 1210
d7e50217 1211 lwz r4,SAVprev+4(r30) ; Pick up the previous area
9bccf70c
A		 1212 li r5,0 ; Assume this is the only one (which should be the ususal case)
1213 mr. r4,r4 ; Was this the only one?
1214 stw r4,VMXsave(r22) ; Dequeue this savearea
1215 beq+ vsonlyone ; This was the only one...
1216 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1217
1218vsonlyone: stw r5,VMXlevel(r22) ; Save the level
1219 stw r8,VMXowner(r26) ; Clear owner
1220 eieio
1221 mr r3,r30 ; Copy the savearea we are tossing
1222 bl EXT(save_ret) ; Toss the savearea
1223 b vsnosave ; Go load up the context...
1224
1225 .align 5
1c79356b 1226
9bccf70c
A		 1227
1228vsmstsave: stw r8,VMXowner(r26) ; Clear owner
1229 eieio
1230 beq- cr2,vsnosave ; The VRSave was 0, so there is nothing to save...
1231
1232 bl EXT(save_get) ; Go get a savearea
1233
d7e50217
A		 1234 mr. r31,r31 ; Are we saving the user state?
1235 la r15,VMXsync(r22) ; Point to the sync word
1236 beq++ vswusave ; Yeah, no need for lock...
1237;
1238; Here we make sure that the live context is not tossed while we are
1239; trying to push it. This can happen only for kernel context and
1240; then only by a race with act_machine_sv_free.
1241;
1242; We only need to hold this for a very short time, so no sniffing needed.
1243; If we find any change to the level, we just abandon.
1244;
1245vswsync: lwarx r19,0,r15 ; Get the sync word
1246 li r0,1 ; Get the lock
1247 cmplwi cr1,r19,0 ; Is it unlocked?
1248 stwcx. r0,0,r15 ; Store lock and test reservation
1249 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
1250 bne-- vswsync ; Try again if lost reservation or locked...
1251
1252 isync ; Toss speculation
1253
1254 lwz r0,VMXlevel(r22) ; Pick up the level again
1255 li r7,0 ; Get unlock value
1256 cmplw r0,r31 ; Same level?
1257 beq++ fswusave ; Yeah, we expect it to be...
1258
1259 stw r7,VMXsync(r22) ; Unlock lock. No need to sync here
1260
1261 bl EXT(save_ret) ; Toss save area because we are abandoning save
1262 b fsnosave ; Skip the save...
9bccf70c 1263
d7e50217
A		 1264 .align 5
1265
1266vswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
1267 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1268 mr. r31,r31 ; Check again if we were user level
1269 stw r30,SAVprev+4(r3) ; Point us to the old context
9bccf70c
A		 1270 stw r31,SAVlevel(r3) ; Tag our level
1271 li r7,SAVvector ; Get the vector ID
1272 stw r12,SAVact(r3) ; Make sure we point to the right guy
1273 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
d7e50217
A		 1274
1275 li r7,0 ; Get the unlock value
1276
1277 beq-- vswnulock ; Skip unlock if user (we did not lock it)...
1278 eieio ; Make sure that these updates make it out
1279 stw r7,VMXsync(r22) ; Unlock it.
1280
1281vswnulock:
1c79356b 1282
1c79356b 1283#if FPVECDBG
9bccf70c
A		 1284 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1285 li r2,0x5F03 ; (TEST/DEBUG)
1286 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1287 sc ; (TEST/DEBUG)
1c79356b
A		 1288#endif
1289
9bccf70c 1290 lwz r10,liveVRS(r26) ; Get the right VRSave register
d7e50217 1291 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1c79356b
A		 1292
1293
9bccf70c
A		 1294;
1295; The context is all saved now and the facility is free.
1296;
1297; If we do not we need to fill the registers with junk, because this level has
1298; never used them before and some thieving bastard could hack the old values
1299; of some thread! Just imagine what would happen if they could! Why, nothing
1300; would be safe! My God! It is terrifying!
1301;
1302; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1303; constant that we may need to fill unused vector registers.
1304;
1c79356b 1305
0b4e3aa0
A		 1306
1307
0b4e3aa0 1308
9bccf70c
A		 1309vsnosave: vspltisb v31,-10 ; Get 0xF6F6F6F6
1310 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1311 vspltisb v30,5 ; Get 0x05050505
1312 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1313 vspltish v29,4 ; Get 0x00040004
1314 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1315 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1316
1317 stw r16,VMXcpu(r29) ; Claim context for us
1318 eieio
1c79356b
A		 1319
1320#if FPVECDBG
9bccf70c
A		 1321 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1322 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1323 li r2,0x5F04 ; (TEST/DEBUG)
1324 mr r1,r15 ; (TEST/DEBUG)
1325 mr r3,r14 ; (TEST/DEBUG)
1326 mr r5,r13 ; (TEST/DEBUG)
1327 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1328 sc ; (TEST/DEBUG)
1c79356b 1329#endif
9bccf70c
A		 1330
1331 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
1332 vspltisb v28,-2 ; Get 0xFEFEFEFE
1333 mulli r19,r19,ppSize ; Find offset to the owner per_proc
1334 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1335 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
1336 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1337 li r16,VMXowner ; Displacement to vector owner
1338 add r19,r18,r19 ; Point to the owner per_proc
1339 vrlb v31,v31,v29 ; Get 0xDEADDEAD
9bccf70c
A		 1340
1341vsinvothr: lwarx r18,r16,r19 ; Get the owner
d7e50217
A		 1342
1343 sub r0,r18,r29 ; Subtract one from the other
1344 sub r11,r29,r18 ; Subtract the other from the one
1345 or r11,r11,r0 ; Combine them
1346 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1347 and r18,r18,r11 ; Make 0 if same, unchanged if not
1348 stwcx. r18,r16,r19 ; Try to invalidate it
1349 bne-- vsinvothr ; Try again if there was a collision...
9bccf70c 1350
d7e50217 1351 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
9bccf70c
A		 1352 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1353 stw r15,VMXlevel(r29) ; Set the "new" active level
1354 eieio
1355 stw r29,VMXowner(r26) ; Mark us as having the live context
1c79356b 1356
d7e50217 1357 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
9bccf70c 1358
d7e50217 1359 lwz r3,SAVprev+4(r14) ; Get the previous context
9bccf70c
A		 1360 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1361 cmplw r0,r15 ; Top level correct to load?
d7e50217 1362 bne-- ProtectTheAmericanWay ; No, go initialize...
1c79356b 1363
9bccf70c
A		 1364 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1365
1c79356b 1366#if FPVECDBG
9bccf70c
A		 1367 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1368 li r2,0x5F05 ; (TEST/DEBUG)
1369 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1370 sc ; (TEST/DEBUG)
1c79356b
A		 1371#endif
1372
9bccf70c 1373 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
d7e50217 1374 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
9bccf70c 1375 and r10,r10,r22 ; Figure out just what registers need to be loaded
d7e50217
A		 1376 mr r3,r14 ; r3 <- ptr to savearea with VRs
1377 bl vr_load ; load VRs from save area based on vrsave in r10
1378
1379 bl EXT(save_ret) ; Toss the save area after loading VRs
1c79356b 1380
d7e50217 1381vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
9bccf70c 1382 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
d7e50217
A		 1383 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1384 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1c79356b 1385 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
d7e50217 1386 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
9bccf70c 1387 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
d7e50217
A		 1388 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1389 mr r3,r25 ; Pass virtual address of the savearea
b4c24cb9 1390 beq- vrnuser ; We are not user state...
9bccf70c 1391 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
d7e50217 1392 stw r11,spcFlags(r26) ; Set per_proc copy
1c79356b
A		 1393
1394vrnuser:
1395#if FPVECDBG
9bccf70c
A		 1396 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1397 li r2,0x5F07 ; (TEST/DEBUG)
1398 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1399 sc ; (TEST/DEBUG)
1c79356b 1400#endif
9bccf70c 1401 b EXT(exception_exit) ; Exit to the fray...
1c79356b
A		 1402
1403/*
1404 * Initialize the registers to some bogus value
1c79356b
A		 1405 */
1406
1407ProtectTheAmericanWay:
1408
1409#if FPVECDBG
9bccf70c
A		 1410 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1411 li r2,0x5F06 ; (TEST/DEBUG)
1412 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1413 sc ; (TEST/DEBUG)
1c79356b 1414#endif
9bccf70c
A		 1415
1416 vor v0,v31,v31 ; Copy into the next register
1417 vor v1,v31,v31 ; Copy into the next register
1418 vor v2,v31,v31 ; Copy into the next register
1419 vor v3,v31,v31 ; Copy into the next register
1420 vor v4,v31,v31 ; Copy into the next register
1421 vor v5,v31,v31 ; Copy into the next register
1422 vor v6,v31,v31 ; Copy into the next register
1423 vor v7,v31,v31 ; Copy into the next register
1424 vor v8,v31,v31 ; Copy into the next register
1425 vor v9,v31,v31 ; Copy into the next register
1426 vor v10,v31,v31 ; Copy into the next register
1427 vor v11,v31,v31 ; Copy into the next register
1428 vor v12,v31,v31 ; Copy into the next register
1429 vor v13,v31,v31 ; Copy into the next register
1430 vor v14,v31,v31 ; Copy into the next register
1431 vor v15,v31,v31 ; Copy into the next register
1432 vor v16,v31,v31 ; Copy into the next register
1433 vor v17,v31,v31 ; Copy into the next register
1434 vor v18,v31,v31 ; Copy into the next register
1435 vor v19,v31,v31 ; Copy into the next register
1436 vor v20,v31,v31 ; Copy into the next register
1437 vor v21,v31,v31 ; Copy into the next register
1438 vor v22,v31,v31 ; Copy into the next register
1439 vor v23,v31,v31 ; Copy into the next register
1440 vor v24,v31,v31 ; Copy into the next register
1441 vor v25,v31,v31 ; Copy into the next register
1442 vor v26,v31,v31 ; Copy into the next register
1443 vor v27,v31,v31 ; Copy into the next register
1444 vor v28,v31,v31 ; Copy into the next register
1445 vor v29,v31,v31 ; Copy into the next register
1446 vor v30,v31,v31 ; Copy into the next register
1447 b vrenable ; Finish setting it all up...
1448
1449
1450
1451;
1452; We get here when we are switching to the same context at the same level and the context
1453; is still live. Essentially, all we are doing is turning on the faility. It may have
1454; gotten turned off due to doing a context save for the current level or a context switch
1455; back to the live guy.
1456;
1457
1458 .align 5
1459
1460vsthesame:
1461
1462#if FPVECDBG
1463 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1464 li r2,0x5F0A ; (TEST/DEBUG)
1465 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1466 sc ; (TEST/DEBUG)
1467#endif
1468 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1469
1470 lwz r11,SAVlevel(r30) ; Get the level of top saved context
d7e50217 1471 lwz r14,SAVprev+4(r30) ; Get the previous savearea
9bccf70c
A		 1472
1473 cmplw r11,r31 ; Are live and saved the same?
1474
1475 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1476
1477 mr r3,r30 ; Get the old savearea (we popped it before)
d7e50217 1478 stw r11,VMXsave(r22) ; Pop the vector stack
9bccf70c
A		 1479 bl EXT(save_ret) ; Toss it
1480 b vrenable ; Go enable and exit...
1481
1482
1483;
1484; This function invalidates any live vector context for the passed in facility_context.
1485; This is intended to be called just before act_machine_sv_free tosses saveareas.
1c79356b 1486;
1c79356b 1487
9bccf70c
A		 1488 .align 5
1489 .globl EXT(toss_live_vec)
1c79356b 1490
9bccf70c
A		 1491LEXT(toss_live_vec)
1492
d7e50217 1493 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
9bccf70c 1494 mfmsr r9 ; Get the MSR
d7e50217
A		 1495 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1496 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1497 andc r9,r9,r0 ; Force off VEC and FP
1498 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1499 andc r0,r9,r0 ; Turn off EE now
9bccf70c
A		 1500 mtmsr r0 ; No interruptions
1501 isync
1502 beq+ tlvnotours ; Vector off, can not be live here...
1c79356b 1503
9bccf70c
A		 1504 mfsprg r8,0 ; Get the per proc
1505
1506;
1507; Note that at this point, since vecs are on, we are the owner
1508; of live state on this processor
1509;
1510
1511 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1512 li r0,0 ; Clear this just in case we need it
1513 cmplw r6,r3 ; Are we tossing our own context?
1514 bne- tlvnotours ; Nope...
1515
1516 vspltish v1,1 ; Turn on the non-Java bit and saturate
1517 vspltisw v0,1 ; Turn on the saturate bit
1518 vxor v1,v1,v0 ; Turn off saturate
1519 mtspr vrsave,r0 ; Clear VRSAVE
1520 mtvscr v1 ; Set the non-java, no saturate status
1521
1522tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1523 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1524 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1525 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1526 li r10,VMXowner ; Displacement to vector owner
1527 add r11,r12,r11 ; Point to the owner per_proc
1528 li r0,0 ; Set a 0 to invalidate context
1529
1530tlvinvothr: lwarx r12,r10,r11 ; Get the owner
9bccf70c 1531
d7e50217
A		 1532 sub r0,r12,r3 ; Subtract one from the other
1533 sub r8,r3,r12 ; Subtract the other from the one
1534 or r8,r8,r0 ; Combine them
1535 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1536 and r12,r12,r8 ; Make 0 if same, unchanged if not
1537 stwcx. r12,r10,r11 ; Try to invalidate it
1538 bne-- tlvinvothr ; Try again if there was a collision...
1539
1540 mtmsr r9 ; Restore interruptions
9bccf70c
A		 1541 isync ; Could be turning off vectors here
1542 blr ; Leave....
1543
1544#if 0
1545;
1546; This function invalidates any live vector context for the passed in facility_context
1547; if the level is current. It also tosses the corresponding savearea if there is one.
1548; This function is primarily used whenever we detect a VRSave that is all zeros.
1549;
1550
1551 .align 5
1552 .globl EXT(vec_trash)
1553
1554LEXT(vec_trash)
1555
1556 lwz r12,facAct(r3) ; Get the activation
1557 lwz r11,VMXlevel(r3) ; Get the context level
1558 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1559 lwz r9,VMXsave(r3) ; Get the savearea, if any
1560 cmplw r10,r11 ; Are we at the right level?
1561 cmplwi cr1,r9,0 ; Remember if there is a savearea
1562 bnelr+ ; No, we do nothing...
1563
1564 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1565 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1566 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1567 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1568 li r10,VMXowner ; Displacement to vector owner
1569 add r11,r12,r11 ; Point to the owner per_proc
9bccf70c
A		 1570
1571vtinvothr: lwarx r12,r10,r11 ; Get the owner
9bccf70c 1572
d7e50217
A		 1573 sub r0,r12,r3 ; Subtract one from the other
1574 sub r8,r3,r12 ; Subtract the other from the one
1575 or r8,r8,r0 ; Combine them
1576 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1577 and r12,r12,r8 ; Make 0 if same, unchanged if not
1578 stwcx. r12,r10,r11 ; Try to invalidate it
1579 bne-- vtinvothr ; Try again if there was a collision...
1580
1581
1582 beqlr++ cr1 ; Leave if there is no savearea
9bccf70c
A		 1583 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1584 cmplw r8,r11 ; Savearea for the current level?
d7e50217 1585 bnelr++ ; No, nothing to release...
9bccf70c 1586
d7e50217 1587 lwz r8,SAVprev+4(r9) ; Pick up the previous area
9bccf70c 1588 mr. r8,r8 ; Is there a previous?
d7e50217 1589 beq-- vtnoprev ; Nope...
9bccf70c
A		 1590 lwz r7,SAVlevel(r8) ; Get the level associated with save
1591
1592vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1593 stw r7,VMXlevel(r3) ; Pop the level
1594
1595 mr r3,r9 ; Get the savearea to release
1596 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1597#endif
1598
1599;
1600; Just some test code to force vector and/or floating point in the kernel
1601;
1602
1603 .align 5
1604 .globl EXT(fctx_test)
1c79356b 1605
9bccf70c
A		 1606LEXT(fctx_test)
1607
1608 mfsprg r3,0 ; Get the per_proc block
1609 lwz r3,PP_ACTIVE_THREAD(r3) ; Get the thread pointer
1610 mr. r3,r3 ; Are we actually up and running?
1611 beqlr- ; No...
1612
1613 fmr f0,f0 ; Use floating point
1614 mftb r4 ; Get time base for a random number
1615 li r5,1 ; Get a potential vrsave to use
1616 andi. r4,r4,0x3F ; Get a number from 0 - 63
1617 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1618 mtspr vrsave,r5 ; Set VRSave
1619 vor v0,v0,v0 ; Use vectors
1620 blr
d7e50217
A		 1621
1622
1623// *******************
1624// * f p _ s t o r e *
1625// *******************
1626//
1627// Store FPRs into a save area. Called by fpu_save and fpu_switch.
1628//
1629// When called:
1630// floating pt is enabled
1631// r3 = ptr to save area
1632//
1633// We destroy:
1634// r11.
1635
1636fp_store:
1637 mfsprg r11,2 ; get feature flags
1638 mtcrf 0x02,r11 ; put cache line size bits in cr6
1639 la r11,savefp0(r3) ; point to 1st line
1640 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1641 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1642
1643// Store the FPRs on a 128-byte machine.
1644
1645 stfd f0,savefp0(r3)
1646 stfd f1,savefp1(r3)
1647 la r11,savefp16(r3) ; Point to the 2nd cache line
1648 stfd f2,savefp2(r3)
1649 stfd f3,savefp3(r3)
1650 dcbz128 0,r11 ; establish 2nd line
1651 stfd f4,savefp4(r3)
1652 stfd f5,savefp5(r3)
1653 stfd f6,savefp6(r3)
1654 stfd f7,savefp7(r3)
1655 stfd f8,savefp8(r3)
1656 stfd f9,savefp9(r3)
1657 stfd f10,savefp10(r3)
1658 stfd f11,savefp11(r3)
1659 stfd f12,savefp12(r3)
1660 stfd f13,savefp13(r3)
1661 stfd f14,savefp14(r3)
1662 stfd f15,savefp15(r3)
1663 stfd f16,savefp16(r3)
1664 stfd f17,savefp17(r3)
1665 stfd f18,savefp18(r3)
1666 stfd f19,savefp19(r3)
1667 stfd f20,savefp20(r3)
1668 stfd f21,savefp21(r3)
1669 stfd f22,savefp22(r3)
1670 stfd f23,savefp23(r3)
1671 stfd f24,savefp24(r3)
1672 stfd f25,savefp25(r3)
1673 stfd f26,savefp26(r3)
1674 stfd f27,savefp27(r3)
1675 stfd f28,savefp28(r3)
1676 stfd f29,savefp29(r3)
1677 stfd f30,savefp30(r3)
1678 stfd f31,savefp31(r3)
1679 blr
1680
1681// Store FPRs on a 32-byte machine.
1682
1683fp_st32:
1684 la r11,savefp4(r3) ; Point to the 2nd line
1685 stfd f0,savefp0(r3)
1686 dcbz 0,r11 ; Allocate cache
1687 stfd f1,savefp1(r3)
1688 stfd f2,savefp2(r3)
1689 la r11,savefp8(r3) ; Point to the 3rd line
1690 stfd f3,savefp3(r3)
1691 dcbz 0,r11 ; Allocate cache
1692 stfd f4,savefp4(r3)
1693 stfd f5,savefp5(r3)
1694 stfd f6,savefp6(r3)
1695 la r11,savefp12(r3) ; Point to the 4th line
1696 stfd f7,savefp7(r3)
1697 dcbz 0,r11 ; Allocate cache
1698 stfd f8,savefp8(r3)
1699 stfd f9,savefp9(r3)
1700 stfd f10,savefp10(r3)
1701 la r11,savefp16(r3) ; Point to the 5th line
1702 stfd f11,savefp11(r3)
1703 dcbz 0,r11 ; Allocate cache
1704 stfd f12,savefp12(r3)
1705 stfd f13,savefp13(r3)
1706 stfd f14,savefp14(r3)
1707 la r11,savefp20(r3) ; Point to the 6th line
1708 stfd f15,savefp15(r3)
1709 dcbz 0,r11 ; Allocate cache
1710 stfd f16,savefp16(r3)
1711 stfd f17,savefp17(r3)
1712 stfd f18,savefp18(r3)
1713 la r11,savefp24(r3) ; Point to the 7th line
1714 stfd f19,savefp19(r3)
1715 dcbz 0,r11 ; Allocate cache
1716 stfd f20,savefp20(r3)
1717
1718 stfd f21,savefp21(r3)
1719 stfd f22,savefp22(r3)
1720 la r11,savefp28(r3) ; Point to the 8th line
1721 stfd f23,savefp23(r3)
1722 dcbz 0,r11 ; allocate it
1723 stfd f24,savefp24(r3)
1724 stfd f25,savefp25(r3)
1725 stfd f26,savefp26(r3)
1726 stfd f27,savefp27(r3)
1727
1728 stfd f28,savefp28(r3)
1729 stfd f29,savefp29(r3)
1730 stfd f30,savefp30(r3)
1731 stfd f31,savefp31(r3)
1732 blr
1733
1734
1735// *******************
1736// * v r _ s t o r e *
1737// *******************
1738//
1739// Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1740// both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1741// unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1742// for each cache block size.
1743//
1744// When called:
1745// interrupts are off, vectors are enabled
1746// r3 = ptr to save area
1747// r10 = vrsave (not 0)
1748//
1749// We destroy:
1750// r4 - r11, all CRs.
1751
1752vr_store:
1753 mfsprg r9,2 ; get feature flags
1754 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1755 slwi r8,r10,1 ; Shift over 1
1756 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1757 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1758 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1759
1760
1761; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1762; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1763
1764 slwi r7,r8,2 ; shift groups-of-2 over by 2
1765 li r4,16 ; load offsets for X-form stores
1766 or r8,r7,r8 ; show if any in group of 4 are in use
1767 li r5,32
1768 slwi r7,r8,4 ; shift groups-of-4 over by 4
1769 li r6,48
1770 or r11,r7,r8 ; show if any in group of 8 are in use
1771 li r7,64
1772 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1773 li r8,80
1774 mtcrf 0x20,r11
1775 li r9,96
1776 mtcrf 0x08,r11
1777 li r10,112
1778 mtcrf 0x02,r11
1779
1780 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1781 la r11,savevr0(r3) ; get address of this group of registers in save area
1782 dcbz128 0,r11 ; zero the line
1783 stvxl v0,0,r11 ; save 8 VRs in the line
1784 stvxl v1,r4,r11
1785 stvxl v2,r5,r11
1786 stvxl v3,r6,r11
1787 stvxl v4,r7,r11
1788 stvxl v5,r8,r11
1789 stvxl v6,r9,r11
1790 stvxl v7,r10,r11
1791
1792vr_st64b:
1793 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
1794 la r11,savevr8(r3) ; get address of this group of registers in save area
1795 dcbz128 0,r11 ; zero the line
1796 stvxl v8,0,r11 ; save 8 VRs in the line
1797 stvxl v9,r4,r11
1798 stvxl v10,r5,r11
1799 stvxl v11,r6,r11
1800 stvxl v12,r7,r11
1801 stvxl v13,r8,r11
1802 stvxl v14,r9,r11
1803 stvxl v15,r10,r11
1804
1805vr_st64c:
1806 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
1807 la r11,savevr16(r3) ; get address of this group of registers in save area
1808 dcbz128 0,r11 ; zero the line
1809 stvxl v16,0,r11 ; save 8 VRs in the line
1810 stvxl v17,r4,r11
1811 stvxl v18,r5,r11
1812 stvxl v19,r6,r11
1813 stvxl v20,r7,r11
1814 stvxl v21,r8,r11
1815 stvxl v22,r9,r11
1816 stvxl v23,r10,r11
1817
1818vr_st64d:
1819 bflr 24 ; done if none of vr24-vr31 are in use
1820 la r11,savevr24(r3) ; get address of this group of registers in save area
1821 dcbz128 0,r11 ; zero the line
1822 stvxl v24,0,r11 ; save 8 VRs in the line
1823 stvxl v25,r4,r11
1824 stvxl v26,r5,r11
1825 stvxl v27,r6,r11
1826 stvxl v28,r7,r11
1827 stvxl v29,r8,r11
1828 stvxl v30,r9,r11
1829 stvxl v31,r10,r11
1830 blr
1831
1832; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
1833; or neither in each group. This cuts down on conditional branches.
1834; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
1835; r3 = savearea
1836
1837vr_st32:
1838 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
1839 li r4,16 ; load offset for X-form stores
1840
1841 bf 0,vr_st32b ; skip if neither VR in this pair is in use
1842 la r11,savevr0(r3) ; get address of this group of registers in save area
1843 dcba 0,r11 ; establish the line wo reading it
1844 stvxl v0,0,r11 ; save the two VRs in the line
1845 stvxl v1,r4,r11
1846
1847vr_st32b:
1848 bf 2,vr_st32c ; skip if neither VR in this pair is in use
1849 la r11,savevr2(r3) ; get address of this group of registers in save area
1850 dcba 0,r11 ; establish the line wo reading it
1851 stvxl v2,0,r11 ; save the two VRs in the line
1852 stvxl v3,r4,r11
1853
1854vr_st32c:
1855 bf 4,vr_st32d ; skip if neither VR in this pair is in use
1856 la r11,savevr4(r3) ; get address of this group of registers in save area
1857 dcba 0,r11 ; establish the line wo reading it
1858 stvxl v4,0,r11 ; save the two VRs in the line
1859 stvxl v5,r4,r11
1860
1861vr_st32d:
1862 bf 6,vr_st32e ; skip if neither VR in this pair is in use
1863 la r11,savevr6(r3) ; get address of this group of registers in save area
1864 dcba 0,r11 ; establish the line wo reading it
1865 stvxl v6,0,r11 ; save the two VRs in the line
1866 stvxl v7,r4,r11
1867
1868vr_st32e:
1869 bf 8,vr_st32f ; skip if neither VR in this pair is in use
1870 la r11,savevr8(r3) ; get address of this group of registers in save area
1871 dcba 0,r11 ; establish the line wo reading it
1872 stvxl v8,0,r11 ; save the two VRs in the line
1873 stvxl v9,r4,r11
1874
1875vr_st32f:
1876 bf 10,vr_st32g ; skip if neither VR in this pair is in use
1877 la r11,savevr10(r3) ; get address of this group of registers in save area
1878 dcba 0,r11 ; establish the line wo reading it
1879 stvxl v10,0,r11 ; save the two VRs in the line
1880 stvxl v11,r4,r11
1881
1882vr_st32g:
1883 bf 12,vr_st32h ; skip if neither VR in this pair is in use
1884 la r11,savevr12(r3) ; get address of this group of registers in save area
1885 dcba 0,r11 ; establish the line wo reading it
1886 stvxl v12,0,r11 ; save the two VRs in the line
1887 stvxl v13,r4,r11
1888
1889vr_st32h:
1890 bf 14,vr_st32i ; skip if neither VR in this pair is in use
1891 la r11,savevr14(r3) ; get address of this group of registers in save area
1892 dcba 0,r11 ; establish the line wo reading it
1893 stvxl v14,0,r11 ; save the two VRs in the line
1894 stvxl v15,r4,r11
1895
1896vr_st32i:
1897 bf 16,vr_st32j ; skip if neither VR in this pair is in use
1898 la r11,savevr16(r3) ; get address of this group of registers in save area
1899 dcba 0,r11 ; establish the line wo reading it
1900 stvxl v16,0,r11 ; save the two VRs in the line
1901 stvxl v17,r4,r11
1902
1903vr_st32j:
1904 bf 18,vr_st32k ; skip if neither VR in this pair is in use
1905 la r11,savevr18(r3) ; get address of this group of registers in save area
1906 dcba 0,r11 ; establish the line wo reading it
1907 stvxl v18,0,r11 ; save the two VRs in the line
1908 stvxl v19,r4,r11
1909
1910vr_st32k:
1911 bf 20,vr_st32l ; skip if neither VR in this pair is in use
1912 la r11,savevr20(r3) ; get address of this group of registers in save area
1913 dcba 0,r11 ; establish the line wo reading it
1914 stvxl v20,0,r11 ; save the two VRs in the line
1915 stvxl v21,r4,r11
1916
1917vr_st32l:
1918 bf 22,vr_st32m ; skip if neither VR in this pair is in use
1919 la r11,savevr22(r3) ; get address of this group of registers in save area
1920 dcba 0,r11 ; establish the line wo reading it
1921 stvxl v22,0,r11 ; save the two VRs in the line
1922 stvxl v23,r4,r11
1923
1924vr_st32m:
1925 bf 24,vr_st32n ; skip if neither VR in this pair is in use
1926 la r11,savevr24(r3) ; get address of this group of registers in save area
1927 dcba 0,r11 ; establish the line wo reading it
1928 stvxl v24,0,r11 ; save the two VRs in the line
1929 stvxl v25,r4,r11
1930
1931vr_st32n:
1932 bf 26,vr_st32o ; skip if neither VR in this pair is in use
1933 la r11,savevr26(r3) ; get address of this group of registers in save area
1934 dcba 0,r11 ; establish the line wo reading it
1935 stvxl v26,0,r11 ; save the two VRs in the line
1936 stvxl v27,r4,r11
1937
1938vr_st32o:
1939 bf 28,vr_st32p ; skip if neither VR in this pair is in use
1940 la r11,savevr28(r3) ; get address of this group of registers in save area
1941 dcba 0,r11 ; establish the line wo reading it
1942 stvxl v28,0,r11 ; save the two VRs in the line
1943 stvxl v29,r4,r11
1944
1945vr_st32p:
1946 bflr 30 ; done if neither VR in this pair is in use
1947 la r11,savevr30(r3) ; get address of this group of registers in save area
1948 dcba 0,r11 ; establish the line wo reading it
1949 stvxl v30,0,r11 ; save the two VRs in the line
1950 stvxl v31,r4,r11
1951 blr
1952
1953
1954// *****************
1955// * v r _ l o a d *
1956// *****************
1957//
1958// Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
1959// of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
1960// branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
1961// in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
1962// which would bug all VRs that aren't live.
1963//
1964// When called:
1965// interrupts are off, vectors are enabled
1966// r3 = ptr to save area
1967// r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
1968// v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
1969//
1970// We destroy:
1971// r4 - r11, all CRs.
1972
1973vr_load:
1974 mfsprg r9,2 ; get feature flags
1975 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
1976 mtcrf 0x02,r9 ; set cache line size bits in cr6
1977 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
1978 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
1979 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
1980 lis r7,0xFF00 ; so set bits 0-7
1981
1982// Loop touching in cache blocks we will load from.
1983// r3 = savearea ptr
1984// r5 = we light bits for the VRs we will be loading
1985// r6 = 1 if 32-byte, 7 if 128-byte
1986// r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
1987// r10 = live VR bits
1988// v31 = bugbug constant
1989
1990vr_ld0:
1991 li r5,0 ; initialize set of VRs to load
1992 la r11,savevr0(r3) ; get address of register file
1993 b vr_ld2 ; enter loop in middle
1994
1995 .align 5
1996vr_ld1: ; loop over each cache line we will load
1997 dcbt r4,r11 ; start prefetch of the line
1998 andc r10,r10,r9 ; turn off the bits in this line
1999 or r5,r5,r9 ; we will load all these
2000vr_ld2: ; initial entry pt
2001 cntlzw r4,r10 ; get offset to next live VR
2002 andc r4,r4,r6 ; cacheline align it
2003 srw. r9,r7,r4 ; position bits for VRs in that cache line
2004 slwi r4,r4,4 ; get byte offset within register file to that line
2005 bne vr_ld1 ; loop if more bits in r10
2006
2007 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
2008
2009// Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
2010// r3 = savearea ptr
2011// r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2012// r11 = addr(savevr0)
2013// v31 = bugbug constant
2014
2015 mtcrf 0x80,r5 ; set up bits for conditional branches
2016 li r4,16 ; load offsets for X-form stores
2017 li r6,48
2018 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
2019 li r7,64
2020 li r8,80
2021 mtcrf 0x08,r5
2022 li r9,96
2023 li r10,112
2024 mtcrf 0x02,r5
2025 li r5,32
2026
2027 bt 0,vr_ld128a ; skip if this line must be loaded
2028 vor v0,v31,v31 ; no VR must be loaded, so bug them all
2029 vor v1,v31,v31
2030 vor v2,v31,v31
2031 vor v3,v31,v31
2032 vor v4,v31,v31
2033 vor v5,v31,v31
2034 vor v6,v31,v31
2035 vor v7,v31,v31
2036 b vr_ld128b
2037vr_ld128a: ; must load from this line
2038 lvxl v0,0,r11
2039 lvxl v1,r4,r11
2040 lvxl v2,r5,r11
2041 lvxl v3,r6,r11
2042 lvxl v4,r7,r11
2043 lvxl v5,r8,r11
2044 lvxl v6,r9,r11
2045 lvxl v7,r10,r11
2046
2047vr_ld128b: ; here to handle next cache line
2048 la r11,savevr8(r3) ; load offset to it
2049 bt 8,vr_ld128c ; skip if this line must be loaded
2050 vor v8,v31,v31 ; no VR must be loaded, so bug them all
2051 vor v9,v31,v31
2052 vor v10,v31,v31
2053 vor v11,v31,v31
2054 vor v12,v31,v31
2055 vor v13,v31,v31
2056 vor v14,v31,v31
2057 vor v15,v31,v31
2058 b vr_ld128d
2059vr_ld128c: ; must load from this line
2060 lvxl v8,0,r11
2061 lvxl v9,r4,r11
2062 lvxl v10,r5,r11
2063 lvxl v11,r6,r11
2064 lvxl v12,r7,r11
2065 lvxl v13,r8,r11
2066 lvxl v14,r9,r11
2067 lvxl v15,r10,r11
2068
2069vr_ld128d: ; here to handle next cache line
2070 la r11,savevr16(r3) ; load offset to it
2071 bt 16,vr_ld128e ; skip if this line must be loaded
2072 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2073 vor v17,v31,v31
2074 vor v18,v31,v31
2075 vor v19,v31,v31
2076 vor v20,v31,v31
2077 vor v21,v31,v31
2078 vor v22,v31,v31
2079 vor v23,v31,v31
2080 b vr_ld128f
2081vr_ld128e: ; must load from this line
2082 lvxl v16,0,r11
2083 lvxl v17,r4,r11
2084 lvxl v18,r5,r11
2085 lvxl v19,r6,r11
2086 lvxl v20,r7,r11
2087 lvxl v21,r8,r11
2088 lvxl v22,r9,r11
2089 lvxl v23,r10,r11
2090
2091vr_ld128f: ; here to handle next cache line
2092 la r11,savevr24(r3) ; load offset to it
2093 bt 24,vr_ld128g ; skip if this line must be loaded
2094 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2095 vor v25,v31,v31
2096 vor v26,v31,v31
2097 vor v27,v31,v31
2098 vor v28,v31,v31
2099 vor v29,v31,v31
2100 vor v30,v31,v31
2101 blr
2102vr_ld128g: ; must load from this line
2103 lvxl v24,0,r11
2104 lvxl v25,r4,r11
2105 lvxl v26,r5,r11
2106 lvxl v27,r6,r11
2107 lvxl v28,r7,r11
2108 lvxl v29,r8,r11
2109 lvxl v30,r9,r11
2110 lvxl v31,r10,r11
2111 blr
2112
2113// Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2114// r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2115// r11 = addr(savevr0)
2116
2117vr_ld32:
2118 mtcrf 0xFF,r5 ; set up bits for conditional branches
2119 li r4,16 ; load offset for X-form stores
2120
2121 bt 0,vr_ld32load0 ; skip if we must load this line
2122 vor v0,v31,v31 ; neither VR is live, so bug them both
2123 vor v1,v31,v31
2124 b vr_ld32test2
2125vr_ld32load0: ; must load VRs in this line
2126 lvxl v0,0,r11
2127 lvxl v1,r4,r11
2128
2129vr_ld32test2: ; here to handle next cache line
2130 la r11,savevr2(r3) ; get offset to next cache line
2131 bt 2,vr_ld32load2 ; skip if we must load this line
2132 vor v2,v31,v31 ; neither VR is live, so bug them both
2133 vor v3,v31,v31
2134 b vr_ld32test4
2135vr_ld32load2: ; must load VRs in this line
2136 lvxl v2,0,r11
2137 lvxl v3,r4,r11
2138
2139vr_ld32test4: ; here to handle next cache line
2140 la r11,savevr4(r3) ; get offset to next cache line
2141 bt 4,vr_ld32load4 ; skip if we must load this line
2142 vor v4,v31,v31 ; neither VR is live, so bug them both
2143 vor v5,v31,v31
2144 b vr_ld32test6
2145vr_ld32load4: ; must load VRs in this line
2146 lvxl v4,0,r11
2147 lvxl v5,r4,r11
2148
2149vr_ld32test6: ; here to handle next cache line
2150 la r11,savevr6(r3) ; get offset to next cache line
2151 bt 6,vr_ld32load6 ; skip if we must load this line
2152 vor v6,v31,v31 ; neither VR is live, so bug them both
2153 vor v7,v31,v31
2154 b vr_ld32test8
2155vr_ld32load6: ; must load VRs in this line
2156 lvxl v6,0,r11
2157 lvxl v7,r4,r11
2158
2159vr_ld32test8: ; here to handle next cache line
2160 la r11,savevr8(r3) ; get offset to next cache line
2161 bt 8,vr_ld32load8 ; skip if we must load this line
2162 vor v8,v31,v31 ; neither VR is live, so bug them both
2163 vor v9,v31,v31
2164 b vr_ld32test10
2165vr_ld32load8: ; must load VRs in this line
2166 lvxl v8,0,r11
2167 lvxl v9,r4,r11
2168
2169vr_ld32test10: ; here to handle next cache line
2170 la r11,savevr10(r3) ; get offset to next cache line
2171 bt 10,vr_ld32load10 ; skip if we must load this line
2172 vor v10,v31,v31 ; neither VR is live, so bug them both
2173 vor v11,v31,v31
2174 b vr_ld32test12
2175vr_ld32load10: ; must load VRs in this line
2176 lvxl v10,0,r11
2177 lvxl v11,r4,r11
2178
2179vr_ld32test12: ; here to handle next cache line
2180 la r11,savevr12(r3) ; get offset to next cache line
2181 bt 12,vr_ld32load12 ; skip if we must load this line
2182 vor v12,v31,v31 ; neither VR is live, so bug them both
2183 vor v13,v31,v31
2184 b vr_ld32test14
2185vr_ld32load12: ; must load VRs in this line
2186 lvxl v12,0,r11
2187 lvxl v13,r4,r11
2188
2189vr_ld32test14: ; here to handle next cache line
2190 la r11,savevr14(r3) ; get offset to next cache line
2191 bt 14,vr_ld32load14 ; skip if we must load this line
2192 vor v14,v31,v31 ; neither VR is live, so bug them both
2193 vor v15,v31,v31
2194 b vr_ld32test16
2195vr_ld32load14: ; must load VRs in this line
2196 lvxl v14,0,r11
2197 lvxl v15,r4,r11
2198
2199vr_ld32test16: ; here to handle next cache line
2200 la r11,savevr16(r3) ; get offset to next cache line
2201 bt 16,vr_ld32load16 ; skip if we must load this line
2202 vor v16,v31,v31 ; neither VR is live, so bug them both
2203 vor v17,v31,v31
2204 b vr_ld32test18
2205vr_ld32load16: ; must load VRs in this line
2206 lvxl v16,0,r11
2207 lvxl v17,r4,r11
2208
2209vr_ld32test18: ; here to handle next cache line
2210 la r11,savevr18(r3) ; get offset to next cache line
2211 bt 18,vr_ld32load18 ; skip if we must load this line
2212 vor v18,v31,v31 ; neither VR is live, so bug them both
2213 vor v19,v31,v31
2214 b vr_ld32test20
2215vr_ld32load18: ; must load VRs in this line
2216 lvxl v18,0,r11
2217 lvxl v19,r4,r11
2218
2219vr_ld32test20: ; here to handle next cache line
2220 la r11,savevr20(r3) ; get offset to next cache line
2221 bt 20,vr_ld32load20 ; skip if we must load this line
2222 vor v20,v31,v31 ; neither VR is live, so bug them both
2223 vor v21,v31,v31
2224 b vr_ld32test22
2225vr_ld32load20: ; must load VRs in this line
2226 lvxl v20,0,r11
2227 lvxl v21,r4,r11
2228
2229vr_ld32test22: ; here to handle next cache line
2230 la r11,savevr22(r3) ; get offset to next cache line
2231 bt 22,vr_ld32load22 ; skip if we must load this line
2232 vor v22,v31,v31 ; neither VR is live, so bug them both
2233 vor v23,v31,v31
2234 b vr_ld32test24
2235vr_ld32load22: ; must load VRs in this line
2236 lvxl v22,0,r11
2237 lvxl v23,r4,r11
2238
2239vr_ld32test24: ; here to handle next cache line
2240 la r11,savevr24(r3) ; get offset to next cache line
2241 bt 24,vr_ld32load24 ; skip if we must load this line
2242 vor v24,v31,v31 ; neither VR is live, so bug them both
2243 vor v25,v31,v31
2244 b vr_ld32test26
2245vr_ld32load24: ; must load VRs in this line
2246 lvxl v24,0,r11
2247 lvxl v25,r4,r11
2248
2249vr_ld32test26: ; here to handle next cache line
2250 la r11,savevr26(r3) ; get offset to next cache line
2251 bt 26,vr_ld32load26 ; skip if we must load this line
2252 vor v26,v31,v31 ; neither VR is live, so bug them both
2253 vor v27,v31,v31
2254 b vr_ld32test28
2255vr_ld32load26: ; must load VRs in this line
2256 lvxl v26,0,r11
2257 lvxl v27,r4,r11
2258
2259vr_ld32test28: ; here to handle next cache line
2260 la r11,savevr28(r3) ; get offset to next cache line
2261 bt 28,vr_ld32load28 ; skip if we must load this line
2262 vor v28,v31,v31 ; neither VR is live, so bug them both
2263 vor v29,v31,v31
2264 b vr_ld32test30
2265vr_ld32load28: ; must load VRs in this line
2266 lvxl v28,0,r11
2267 lvxl v29,r4,r11
2268
2269vr_ld32test30: ; here to handle next cache line
2270 la r11,savevr30(r3) ; get offset to next cache line
2271 bt 30,vr_ld32load30 ; skip if we must load this line
2272 vor v30,v31,v31 ; neither VR is live, so bug them both
2273 blr
2274vr_ld32load30: ; must load VRs in this line
2275 lvxl v30,0,r11
2276 lvxl v31,r4,r11
2277 blr
mirror of XNU