2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
35 #include <ppc/proc_reg.h>
38 #include <mach/ppc/vm_param.h>
39 #include <ppc/exception.h>
40 #include <ppc/savearea.h>
47 * void machine_load_context(thread_t thread)
49 * Load the context for the first thread to run on a
54 .globl EXT(machine_load_context)
56 LEXT(machine_load_context)
57 mfsprg r6,1 ; Get the current activation
58 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
59 lwz r0,PP_INTSTACK_TOP_SS(r6)
60 stw r0,PP_ISTACKPTR(r6)
61 mr r9,r3 /* Set up the current thread */
63 li r0,0 /* Clear a register */
64 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
65 mfmsr r5 /* Since we are passing control, get our MSR values */
66 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
67 lwz r1,saver1+4(r3) /* Load new stack pointer */
68 lwz r10,ACT_MACT_SPF(r9) /* Get the special flags */
69 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
70 stw r0,FM_BACKPTR(r1) /* zero backptr */
71 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
72 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
73 oris r10,r10,hi16(OnProc) /* Set OnProc bit */
74 stw r0,ACT_PREEMPT_CNT(r9) /* Enable preemption */
75 stw r10,ACT_MACT_SPF(r9) /* Update the special flags */
76 stw r10,spcFlags(r6) /* Set per_proc copy of the special flags */
77 b EXT(exception_exit) /* Go for it */
79 /* thread_t Switch_context(thread_t old,
83 * Switch from one thread to another. If a continuation is supplied, then
84 * we do not need to save callee save registers.
88 /* void Call_continuation( void (*continuation)(void), void *param, wait_result_t wresult, vm_offset_t stack_ptr)
92 .globl EXT(Call_continuation)
94 LEXT(Call_continuation)
95 mtlr r3 /* continuation */
96 mr r3,r4 /* parameter */
97 mr r4,r5 /* wait result */
98 mr r1,r6 /* Load new stack pointer */
99 blrl /* Jump to the continuation */
101 b EXT(thread_terminate)
104 * Get the old kernel stack, and store into the thread structure.
105 * See if a continuation is supplied, and skip state save if so.
107 * Note that interrupts must be disabled before we get here (i.e., splsched)
111 * Switch_context(old, continuation, new)
113 * Context switches are double jumps. We pass the following to the
114 * context switch firmware call:
116 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
120 * R7 = high order physical address of savearea for full switch
122 * savesrr0 is set to go to switch_in
123 * savesrr1 is set to uninterruptible with translation on
128 .globl EXT(Switch_context)
132 lwz r12,ACT_PER_PROC(r3) ; Get the per_proc block
134 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
135 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
136 bne++ notonintstack ; (DEBUG/TRACE) stack
140 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
141 lwz r9,umwSpace(r5) ; Get user memory window address space
142 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
143 lwz r0,CTHREAD_SELF+0(r5) ; Pick up the user assist "word" (actually a double)
144 lwz r7,CTHREAD_SELF+4(r5) ; both halves
145 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
146 lwz r6,umwRelo(r5) ; Get user memory window relocation top
147 stw r12,ACT_PER_PROC(r5) ; Set per_proc in new activation
149 lwz r2,umwRelo+4(r5) ; Get user memory window relocation bottom
151 stw r0,UAW+0(r12) ; Save the assist word for the "ultra fast path"
154 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
156 sth r9,ppUMWmp+mpSpace(r12) ; Save the space
157 stw r6,ppUMWmp+mpNestReloc(r12) ; Save top part of physical address
158 stw r2,ppUMWmp+mpNestReloc+4(r12) ; Save bottom part of physical address
159 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
160 lwz r2,traceMask(0) ; Get the enabled traces
161 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
162 lis r0,hi16(CutTrace) ; Trace FW call
163 mr. r2,r2 ; Any tracing going on?
164 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
165 ori r0,r0,lo16(CutTrace) ; Trace FW call
166 beq++ cswNoTrc ; No trace today, dude...
168 li r2,0x4400 ; Trace ID
169 mr r6,r11 ; Trace prev savearea
170 sc ; Cut trace entry of context switch
172 cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
173 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
174 lwz r9,VMXowner(r12) ; Grab the owner of the vector
175 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
176 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
177 li r0,1 ; Get set to hold off quickfret
179 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
180 cmplw r10,r2 ; Do we have the live float context?
181 lwz r10,FPUlevel(r2) ; Get the live level
182 mr r4,r3 ; Save our old thread to pass back
183 cmplw cr5,r9,r2 ; Do we have the live vector context?
184 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
185 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
186 bne++ cswnofloat ; Float is not ours...
188 cmplw r10,r11 ; Is the level the same?
189 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
190 lwz r5,FPUcpu(r2) ; Get the owning cpu
191 bne++ cswnofloat ; Level not the same, this is not live...
193 cmplw r5,r0 ; Still owned by this cpu?
194 lwz r10,FPUsave(r2) ; Get the pointer to next saved context
195 bne++ cswnofloat ; CPU claimed by someone else...
197 mr. r10,r10 ; Is there a savearea here?
198 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
200 beq-- cswnofloat ; No savearea to check...
202 lwz r3,SAVlevel(r10) ; Get the level
203 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
204 cmplw r3,r11 ; Is it for the current level?
206 bne++ cswnofloat ; Nope...
208 stw r5,FPUsave(r2) ; Pop off this savearea
210 rlwinm r3,r10,0,0,19 ; Move back to start of page
212 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
213 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
214 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
215 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
216 stw r5,SAVprev(r10) ; Link the old in (top)
217 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
218 xor r3,r10,r3 ; Convert to physical
219 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
220 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
223 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
224 mr r7,r2 ; (TEST/DEBUG)
225 li r2,0x4401 ; (TEST/DEBUG)
226 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
228 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
229 mr r2,r7 ; (TEST/DEBUG)
232 cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
234 lwz r10,VMXlevel(r2) ; Get the live level
236 cmplw r10,r11 ; Is the level the same?
237 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
238 lwz r5,VMXcpu(r2) ; Get the owning cpu
239 bne++ cswnovect ; Level not the same, this is not live...
241 cmplw r5,r0 ; Still owned by this cpu?
242 lwz r10,VMXsave(r2) ; Get the level
243 bne++ cswnovect ; CPU claimed by someone else...
245 mr. r10,r10 ; Is there a savearea here?
246 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
248 beq-- cswnovect ; No savearea to check...
250 lwz r3,SAVlevel(r10) ; Get the level
251 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
252 cmplw r3,r11 ; Is it for the current level?
254 bne++ cswnovect ; Nope...
256 stw r5,VMXsave(r2) ; Pop off this savearea
257 rlwinm r3,r10,0,0,19 ; Move back to start of page
259 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
260 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
261 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
262 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
263 stw r5,SAVprev(r10) ; Link the old in (top)
264 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
265 xor r3,r10,r3 ; Convert to physical
266 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
267 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
270 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
271 li r2,0x4501 ; (TEST/DEBUG)
272 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
276 cswnovect: li r0,0 ; Get set to release quickfret holdoff
277 rlwinm r11,r8,0,0,19 ; Switch to savearea base
278 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
279 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
281 ; Note that the low-level code requires the R7 contain the high order half of the savearea's
282 ; physical address. This is hack city, but it is the way it is.
284 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
285 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
286 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
287 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
288 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
290 lwz r9,SAVflags(r8) /* Get the flags */
291 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
292 li r10,(MASK(MSR_ME)|MASK(MSR_DR)) /* Get the switcher's MSR */
293 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
294 stw r10,savesrr1+4(r8) /* Set up for switch in */
295 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
296 xor r3,r11,r8 /* Get the physical address of the new context save area */
297 stw r9,SAVflags(r8) /* Set the flags */
299 bne cr1,swtchtocont ; Switch to the continuation
300 sc /* Switch to the new context */
302 /* We come back here in the new thread context
303 * R4 was set to hold the old thread pointer, but switch_in will put it into
304 * R3 where it belongs.
306 blr /* Jump into the new thread */
309 ; This is where we go when a continuation is set. We are actually
310 ; killing off the old context of the new guy so we need to pop off
311 ; any float or vector states for the ditched level.
313 ; Note that we do the same kind of thing a chkfac in hw_exceptions.s
319 stw r5,savesrr0+4(r8) ; Set the pc
320 stw r6,savesrr1+4(r8) ; Set the next MSR to use
321 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
322 mr r3,r8 ; Pass in the virtual address of savearea
324 b EXT(exception_exit) ; Blocking on continuation, toss old context...
329 * All switched to threads come here first to clean up the old thread.
330 * We need to do the following contortions because we need to keep
331 * the LR clean. And because we need to manipulate the savearea chain
332 * with translation on. If we could, this should be done in lowmem_vectors
333 * before translation is turned on. But we can't, dang it!
335 * switch_in() runs with DR on and IR off
337 * R3 = switcher's savearea (32-bit virtual)
338 * saver4 = old thread in switcher's save
339 * saver5 = new SRR0 in switcher's save
340 * saver6 = new SRR1 in switcher's save
347 .globl EXT(switch_in)
351 lwz r4,saver4+4(r3) ; Get the old thread
352 lwz r5,saver5+4(r3) ; Get the srr0 value
354 mfsprg r0,2 ; Get feature flags
355 mr r9,r4 ; Get the switched from ACT
356 lwz r6,saver6+4(r3) ; Get the srr1 value
357 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
358 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
360 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
361 stw r10,SAVprev+4(r3) ; Chain on the old one
363 mr r3,r4 ; Pass back the old thread
365 mtsrr0 r5 ; Set return point
366 mtsrr1 r6 ; Set return MSR
368 bne++ siSixtyFour ; Go do 64-bit...
376 * void fpu_save(facility_context ctx)
378 * Note that there are some oddities here when we save a context we are using.
379 * It is really not too cool to do this, but what the hey... Anyway,
380 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
381 * savearea containing the context just saved will go away. So, bottom line is
382 * that don't use fpus until after you are done with the saved context.
389 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
390 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
391 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
393 mfmsr r0 ; Get the MSR
394 andc r0,r0,r2 ; Clear FP, VEC
395 andc r2,r0,r12 ; Clear EE
396 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
397 mtmsr r2 ; Set the MSR
400 mfsprg r6,1 ; Get the current activation
401 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
402 lwz r12,FPUowner(r6) ; Get the context ID for owner
405 mr r7,r0 ; (TEST/DEBUG)
406 li r4,0 ; (TEST/DEBUG)
407 mr r10,r3 ; (TEST/DEBUG)
408 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
409 mr. r3,r12 ; (TEST/DEBUG)
410 li r2,0x6F00 ; (TEST/DEBUG)
411 li r5,0 ; (TEST/DEBUG)
412 beq-- noowneryet ; (TEST/DEBUG)
413 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
414 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
416 noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
418 mr r0,r7 ; (TEST/DEBUG)
419 mr r3,r10 ; (TEST/DEBUG)
421 mflr r2 ; Save the return address
423 cmplw r3,r12 ; Is the specified context live?
424 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
425 lwz r9,FPUcpu(r3) ; Get the cpu that context was last on
426 bne-- fsret ; Nobody owns the FPU, no save required...
428 cmplw r9,r11 ; Was the context for this processor?
429 la r5,FPUsync(r3) ; Point to the sync word
430 bne-- fsret ; Facility not last used on this processor...
433 ; It looks like we need to save this one.
435 ; First, make sure that the live context block is not mucked with while
436 ; we are trying to save it on out. Then we will give it the final check.
439 lis r9,ha16(EXT(LockTimeOut)) ; Get the high part
440 mftb r8 ; Get the time now
441 lwz r9,lo16(EXT(LockTimeOut))(r9) ; Get the timeout value
442 b fssync0a ; Jump to the lock...
446 fssync0: li r7,lgKillResv ; Get killing field
447 stwcx. r7,0,r7 ; Kill reservation
449 fssync0a: lwz r7,0(r5) ; Sniff the lock
450 mftb r10 ; Is it time yet?
451 cmplwi cr1,r7,0 ; Is it locked?
452 sub r10,r10,r8 ; How long have we been spinning?
453 cmplw r10,r9 ; Has it been too long?
454 bgt-- fstimeout ; Way too long, panic...
455 bne-- cr1,fssync0a ; Yea, still locked so sniff harder...
457 fssync1: lwarx r7,0,r5 ; Get the sync word
458 li r12,1 ; Get the lock
459 mr. r7,r7 ; Is it unlocked?
461 stwcx. r12,0,r5 ; Store lock and test reservation
462 bne-- fssync1 ; Try again if lost reservation...
464 isync ; Toss speculation
466 lwz r12,FPUowner(r6) ; Get the context ID for owner
467 cmplw r3,r12 ; Check again if we own the FPU?
468 bne-- fsretlk ; Go unlock and return since we no longer own context
470 lwz r5,FPUcpu(r12) ; Get the cpu that context was last on
471 lwz r7,FPUsave(r12) ; Get the current FPU savearea for the thread
472 cmplw r5,r11 ; Is this for the same processor?
473 lwz r9,FPUlevel(r12) ; Get our current level indicator
474 bne-- fsretlk ; Not the same processor, skip any save...
476 cmplwi r7,0 ; Have we ever saved this facility context?
477 beq-- fsneedone ; Never saved it, so go do it...
479 lwz r8,SAVlevel(r7) ; Get the level of this savearea
480 cmplw r9,r8 ; Correct level?
481 beq-- fsretlk ; The current level is already saved, bail out...
483 fsneedone: bl EXT(save_get) ; Get a savearea for the context
485 mfsprg r6,1 ; Get the current activation
486 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
487 li r4,SAVfloat ; Get floating point tag
488 lwz r12,FPUowner(r6) ; Get back our thread
489 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
490 lwz r4,facAct(r12) ; Get the activation associated with live context
491 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
492 stw r4,SAVact(r3) ; Indicate the right activation for this context
493 lwz r9,FPUlevel(r12) ; Get our current level indicator again
494 stw r3,FPUsave(r12) ; Set this as the most current floating point context
495 stw r8,SAVprev+4(r3) ; And then chain this in front
497 stw r9,SAVlevel(r3) ; Show level in savearea
499 bl fp_store ; save all 32 FPRs in the save area at r3
500 mtlr r2 ; Restore return
502 fsretlk: li r7,0 ; Get the unlock value
503 eieio ; Make sure that these updates make it out
504 stw r7,FPUsync(r12) ; Unlock it
506 fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
511 fstimeout: mr r4,r5 ; Set the lock address
512 mr r5,r7 ; Set the lock word data
513 lis r3,hi16(fstimeout_str) ; Get the failed lck message
514 ori r3,r3,lo16(fstimeout_str) ; Get the failed lck message
516 BREAKPOINT_TRAP ; We die here anyway
520 STRINGD "fpu_save: timeout on sync lock (0x%08X), value = 0x%08X\n\000"
527 * Entered to handle the floating-point unavailable exception and
530 * This code is run in virtual address mode on with interrupts off.
532 * Upon exit, the code returns to the users context with the floating
533 * point facility turned on.
535 * ENTRY: VM switched ON
537 * State is saved in savearea pointed to by R4.
538 * All other registers are free.
543 .globl EXT(fpu_switch)
548 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
549 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
555 mfsprg r17,1 ; Get the current activation
556 lwz r26,ACT_PER_PROC(r17) ; Get the per_proc block
557 mfmsr r19 ; Get the current MSR
559 mr r25,r4 ; Save the entry savearea
560 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
561 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
563 mtmsr r19 ; Enable floating point instructions
566 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
567 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
569 ; R22 has the "old" context anchor
570 ; R29 has the "new" context anchor
573 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
574 li r2,0x7F01 ; (TEST/DEBUG)
575 mr r3,r22 ; (TEST/DEBUG)
576 mr r5,r29 ; (TEST/DEBUG)
577 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
581 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
583 mr. r22,r22 ; See if there is any live FP status
584 la r15,FPUsync(r22) ; Point to the sync word
586 beq-- fsnosave ; No live context, so nothing to save...
588 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
589 cmplw cr2,r22,r29 ; Are both old and new the same context?
590 lwz r30,FPUsave(r22) ; Get the top savearea
591 cmplw r18,r16 ; Make sure we are on the right processor
592 lwz r31,FPUlevel(r22) ; Get the context level
593 cmplwi cr1,r30,0 ; Anything saved yet?
595 bne-- fsnosave ; No, not on the same processor...
598 ; Check to see if the live context has already been saved.
599 ; Also check to see if all we are here just to re-enable the MSR
600 ; and handle specially if so.
603 cmplw r31,r27 ; See if the current and active levels are the same
604 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
606 beq-- fsthesame ; New and old are the same, just go enable...
610 ; Note it turns out that on a G5, the following load has about a 50-50 chance of
611 ; taking a segment exception in a system that is doing heavy file I/O. We
612 ; make a dummy access right now in order to get that resolved before we take the lock.
613 ; We do not use the data returned because it may change over the lock
616 beq-- cr1,fswsync ; Nothing saved, skip the probe attempt...
617 lwz r11,SAVlevel(r30) ; Touch the context in order to fault in the segment
620 ; Make sure that the live context block is not mucked with while
621 ; we are trying to save it on out
624 fswsync: lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
625 mftb r3 ; Get the time now
626 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
627 b fswsync0a ; Jump to the lock...
631 fswsync0: li r19,lgKillResv ; Get killing field
632 stwcx. r19,0,r19 ; Kill reservation
634 fswsync0a: lwz r19,0(r15) ; Sniff the lock
635 mftb r18 ; Is it time yet?
636 cmplwi cr1,r19,0 ; Is it locked?
637 sub r18,r18,r3 ; How long have we been spinning?
638 cmplw r18,r11 ; Has it been too long?
639 bgt-- fswtimeout ; Way too long, panic...
640 bne-- cr1,fswsync0a ; Yea, still locked so sniff harder...
642 fswsync1: lwarx r19,0,r15 ; Get the sync word
643 li r0,1 ; Get the lock
644 mr. r19,r19 ; Is it unlocked?
646 stwcx. r0,0,r15 ; Store lock and test reservation
647 bne-- fswsync1 ; Try again if lost reservation...
649 isync ; Toss speculation
652 ; Note that now that we have the lock, we need to check if anything changed.
653 ; Also note that the possible changes are limited. The context owner can
654 ; never change to a different thread or level although it can be invalidated.
655 ; A new context can not be pushed on top of us, but it can be popped. The
656 ; cpu indicator will always change if another processor mucked with any
659 ; It should be very rare that any of the context stuff changes across the lock.
662 lwz r0,FPUowner(r26) ; Get the thread that owns the FPU again
663 lwz r11,FPUsave(r22) ; Get the top savearea again
664 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on again
665 sub r0,r0,r22 ; Non-zero if we lost ownership, 0 if not
666 xor r11,r11,r30 ; Non-zero if saved context changed, 0 if not
667 xor r18,r18,r16 ; Non-zero if cpu changed, 0 if not
668 cmplwi cr1,r30,0 ; Is anything saved?
669 or r0,r0,r11 ; Zero only if both owner and context are unchanged
670 or. r0,r0,r18 ; Zero only if nothing has changed
673 bne-- fsnosavelk ; Something has changed, so this is not ours to save...
674 beq-- cr1,fsmstsave ; There is no context saved yet...
676 lwz r11,SAVlevel(r30) ; Get the level of top saved context
678 cmplw r31,r11 ; Are live and saved the same?
681 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
682 li r2,0x7F02 ; (TEST/DEBUG)
683 mr r3,r11 ; (TEST/DEBUG)
684 mr r5,r31 ; (TEST/DEBUG)
685 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
687 li r3,0 ; (TEST/DEBUG)
690 beq++ fsnosavelk ; Same level, so already saved...
692 fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
693 eieio ; Make sure everyone sees it
694 bl EXT(save_get) ; Go get a savearea
696 lwz r12,facAct(r22) ; Get the activation associated with the context
697 stw r30,SAVprev+4(r3) ; Point us to the old context
698 stw r31,SAVlevel(r3) ; Tag our level
699 li r7,SAVfloat ; Get the floating point ID
700 stw r12,SAVact(r3) ; Make sure we point to the right guy
701 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
702 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
705 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
706 li r2,0x7F03 ; (TEST/DEBUG)
707 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
711 bl fp_store ; store all 32 FPRs
713 fsnosavelk: li r7,0 ; Get the unlock value
714 eieio ; Make sure that these updates make it out
715 stw r7,FPUsync(r22) ; Unlock it.
718 ; The context is all saved now and the facility is free.
720 ; Check if we need to fill the registers with junk, because this level has
721 ; never used them before and some thieving bastard could hack the old values
722 ; of some thread! Just imagine what would happen if they could! Why, nothing
723 ; would be safe! My God! It is terrifying!
725 ; Make sure that the live context block is not mucked with while
726 ; we are trying to load it up
729 fsnosave: la r15,FPUsync(r29) ; Point to the sync word
730 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
731 mftb r3 ; Get the time now
732 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
733 b fsnsync0a ; Jump to the lock...
737 fsnsync0: li r19,lgKillResv ; Get killing field
738 stwcx. r19,0,r19 ; Kill reservation
740 fsnsync0a: lwz r19,0(r15) ; Sniff the lock
741 mftb r18 ; Is it time yet?
742 cmplwi cr1,r19,0 ; Is it locked?
743 sub r18,r18,r3 ; How long have we been spinning?
744 cmplw r18,r11 ; Has it been too long?
745 bgt-- fsntimeout ; Way too long, panic...
746 bne-- cr1,fsnsync0a ; Yea, still locked so sniff harder...
748 fsnsync1: lwarx r19,0,r15 ; Get the sync word
749 li r0,1 ; Get the lock
750 mr. r19,r19 ; Is it unlocked?
751 bne-- fsnsync0 ; Unfortunately, it is locked...
752 stwcx. r0,0,r15 ; Store lock and test reservation
753 bne-- fsnsync1 ; Try again if lost reservation...
755 isync ; Toss speculation
757 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
758 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
759 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
761 stw r16,FPUcpu(r29) ; Claim context for us
765 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
766 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
767 li r2,0x7F04 ; (TEST/DEBUG)
768 mr r1,r15 ; (TEST/DEBUG)
769 mr r3,r14 ; (TEST/DEBUG)
770 mr r5,r13 ; (TEST/DEBUG)
771 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
775 lis r18,hi16(EXT(PerProcTable)) ; Set base PerProcTable
776 mulli r19,r19,ppeSize ; Find offset to the owner per_proc_entry
777 ori r18,r18,lo16(EXT(PerProcTable)) ; Set base PerProcTable
778 li r16,FPUowner ; Displacement to float owner
779 add r19,r18,r19 ; Point to the owner per_proc_entry
780 lwz r19,ppe_vaddr(r19) ; Point to the owner per_proc
782 fsinvothr: lwarx r18,r16,r19 ; Get the owner
783 sub r0,r18,r29 ; Subtract one from the other
784 sub r11,r29,r18 ; Subtract the other from the one
785 or r11,r11,r0 ; Combine them
786 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
787 and r18,r18,r11 ; Make 0 if same, unchanged if not
788 stwcx. r18,r16,r19 ; Try to invalidate it
789 bne-- fsinvothr ; Try again if there was a collision...
791 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
792 la r11,savefp0(r14) ; Point to first line to bring in
793 stw r15,FPUlevel(r29) ; Set the "new" active level
795 stw r29,FPUowner(r26) ; Mark us as having the live context
797 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
799 dcbt 0,r11 ; Touch line in
801 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
802 lwz r3,SAVprev+4(r14) ; Get the previous context
803 cmplw r0,r15 ; Top level correct to load?
804 li r7,0 ; Get the unlock value
805 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
807 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
810 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
811 li r2,0x7F05 ; (TEST/DEBUG)
812 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
816 eieio ; Make sure that these updates make it out
817 stw r7,FPUsync(r29) ; Unlock context now that the context save has been removed
819 // Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
820 // are executed on a 128-byte machine, but that is better than a mispredicted branch.
822 la r11,savefp4(r14) ; Point to next line
823 dcbt 0,r11 ; Touch line in
827 la r11,savefp8(r14) ; Point to next line
829 dcbt 0,r11 ; Touch line in
833 la r11,savefp12(r14) ; Point to next line
835 dcbt 0,r11 ; Touch line in
838 lfd f10,savefp10(r14)
839 la r11,savefp16(r14) ; Point to next line
840 lfd f11,savefp11(r14)
841 dcbt 0,r11 ; Touch line in
842 lfd f12,savefp12(r14)
843 lfd f13,savefp13(r14)
844 lfd f14,savefp14(r14)
845 la r11,savefp20(r14) ; Point to next line
846 lfd f15,savefp15(r14)
847 dcbt 0,r11 ; Touch line in
848 lfd f16,savefp16(r14)
849 lfd f17,savefp17(r14)
850 lfd f18,savefp18(r14)
851 la r11,savefp24(r14) ; Point to next line
852 lfd f19,savefp19(r14)
853 dcbt 0,r11 ; Touch line in
854 lfd f20,savefp20(r14)
855 lfd f21,savefp21(r14)
856 la r11,savefp28(r14) ; Point to next line
857 lfd f22,savefp22(r14)
858 lfd f23,savefp23(r14)
859 dcbt 0,r11 ; Touch line in
860 lfd f24,savefp24(r14)
861 lfd f25,savefp25(r14)
862 lfd f26,savefp26(r14)
863 lfd f27,savefp27(r14)
864 lfd f28,savefp28(r14)
865 lfd f29,savefp29(r14)
866 lfd f30,savefp30(r14)
867 lfd f31,savefp31(r14)
869 mr r3,r14 ; Get the old savearea (we popped it before)
870 bl EXT(save_ret) ; Toss it
872 fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
873 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
874 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
875 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
876 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
877 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
878 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
879 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
880 mr r3,r25 ; Pass the virtual addres of savearea
881 beq- fsnuser ; We are not user state...
882 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
883 stw r11,spcFlags(r26) ; Set per_proc copy
887 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
888 li r2,0x7F07 ; (TEST/DEBUG)
889 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
893 b EXT(exception_exit) ; Exit to the fray...
896 * Initialize the registers to some bogus value
899 MakeSureThatNoTerroristsCanHurtUsByGod:
902 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
903 li r2,0x7F06 ; (TEST/DEBUG)
904 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
907 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
908 li r7,0 ; Get the unlock value
909 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
910 eieio ; Make sure that these updates make it out
911 stw r7,FPUsync(r29) ; Unlock it now that the context has been removed
913 lfd f0,0(r5) ; Initialize FP0
914 fmr f1,f0 ; Do them all
945 b fsenable ; Finish setting it all up...
949 ; We get here when we are switching to the same context at the same level and the context
950 ; is still live. Essentially, all we are doing is turning on the facility. It may have
951 ; gotten turned off due to doing a context save for the current level or a context switch
952 ; back to the live guy.
958 fsthesamel: li r7,0 ; Get the unlock value
959 eieio ; Make sure that these updates make it out
960 stw r7,FPUsync(r22) ; Unlock it.
965 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
966 li r2,0x7F0A ; (TEST/DEBUG)
967 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
970 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
972 lwz r11,SAVlevel(r30) ; Get the level of top saved context
973 lwz r14,SAVprev+4(r30) ; Get the previous savearea
975 cmplw r11,r31 ; Are live and saved the same?
977 bne++ fsenable ; Level not the same, nothing to pop, go enable and exit...
979 mr r3,r30 ; Get the old savearea (we popped it before)
980 stw r14,FPUsave(r22) ; Pop the savearea from the stack
981 bl EXT(save_ret) ; Toss it
982 b fsenable ; Go enable and exit...
985 ; Note that we need to choke in this code rather than panic because there is no
989 fswtimeout: lis r0,hi16(Choke) ; Choke code
990 ori r0,r0,lo16(Choke) ; and the rest
991 li r3,failTimeout ; Timeout code
994 fsntimeout: lis r0,hi16(Choke) ; Choke code
995 ori r0,r0,lo16(Choke) ; and the rest
996 li r3,failTimeout ; Timeout code
1000 lis r0,hi16(Choke) ; Choke code
1001 ori r0,r0,lo16(Choke) ; and the rest
1002 li r3,failTimeout ; Timeout code
1006 lis r0,hi16(Choke) ; Choke code
1007 ori r0,r0,lo16(Choke) ; and the rest
1008 li r3,failTimeout ; Timeout code
1012 ; This function invalidates any live floating point context for the passed in facility_context.
1013 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1017 .globl EXT(toss_live_fpu)
1021 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1022 mfmsr r9 ; Get the MSR
1023 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1024 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
1025 andc r9,r9,r0 ; Force off VEC and FP
1026 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1027 andc r0,r9,r0 ; Turn off EE now
1028 mtmsr r0 ; No interruptions
1030 beq+ tlfnotours ; Floats off, can not be live here...
1032 mfsprg r8,1 ; Get the current activation
1033 lwz r8,ACT_PER_PROC(r8) ; Get the per_proc block
1036 ; Note that at this point, since floats are on, we are the owner
1037 ; of live state on this processor
1040 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
1041 li r0,0 ; Clear this just in case we need it
1042 cmplw r6,r3 ; Are we tossing our own context?
1043 bne-- tlfnotours ; Nope...
1045 lfd f1,Zero(0) ; Make a 0
1046 mtfsf 0xFF,f1 ; Clear it
1048 tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
1049 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1050 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1051 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1052 li r10,FPUowner ; Displacement to float owner
1053 add r11,r12,r11 ; Point to the owner per_proc_entry
1054 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1056 tlfinvothr: lwarx r12,r10,r11 ; Get the owner
1058 sub r0,r12,r3 ; Subtract one from the other
1059 sub r8,r3,r12 ; Subtract the other from the one
1060 or r8,r8,r0 ; Combine them
1061 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1062 and r12,r12,r8 ; Make 0 if same, unchanged if not
1063 stwcx. r12,r10,r11 ; Try to invalidate it
1064 bne-- tlfinvothr ; Try again if there was a collision...
1066 mtmsr r9 ; Restore interruptions
1067 isync ; Could be turning off floats here
1072 * Altivec stuff is here. The techniques used are pretty identical to
1073 * the floating point. Except that we will honor the VRSAVE register
1074 * settings when loading and restoring registers.
1076 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
1077 * mask. VRSAVE is set by the vector user and represents the VRs that they
1078 * say that they are using. The vrvalid mask indicates which vector registers
1079 * are saved in the savearea. Whenever context is saved, it is saved according
1080 * to the VRSAVE register. It is loaded based on VRSAVE anded with
1081 * vrvalid (all other registers are splatted with 0s). This is done because we
1082 * don't want to load any registers we don't have a copy of, we want to set them
1085 * Note that there are some oddities here when we save a context we are using.
1086 * It is really not too cool to do this, but what the hey... Anyway,
1087 * we turn vectors and fpu off before we leave.
1088 * The oddity is that if you use vectors after this, the
1089 * savearea containing the context just saved will go away. So, bottom line is
1090 * that don't use vectors until after you are done with the saved context.
1095 .globl EXT(vec_save)
1100 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
1101 mfmsr r0 ; Get the MSR
1102 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
1103 andc r0,r0,r2 ; Force off VEC and FP
1104 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
1105 andc r2,r0,r2 ; Clear EE for now
1106 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
1107 mtmsr r2 ; Set the MSR
1110 mfsprg r6,1 ; Get the current activation
1111 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1112 lwz r12,VMXowner(r6) ; Get the context ID for owner
1115 mr r11,r6 ; (TEST/DEBUG)
1116 mr r7,r0 ; (TEST/DEBUG)
1117 li r4,0 ; (TEST/DEBUG)
1118 mr r10,r3 ; (TEST/DEBUG)
1119 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1120 mr. r3,r12 ; (TEST/DEBUG)
1121 li r2,0x5F00 ; (TEST/DEBUG)
1122 li r5,0 ; (TEST/DEBUG)
1123 lwz r6,liveVRS(r6) ; (TEST/DEBUG)
1124 beq-- noowneryeu ; (TEST/DEBUG)
1125 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
1126 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
1128 noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1130 mr r0,r7 ; (TEST/DEBUG)
1131 mr r3,r10 ; (TEST/DEBUG)
1132 mr r6,r11 ; (TEST/DEBUG)
1134 mflr r2 ; Save the return address
1136 cmplw r3,r12 ; Is the specified context live?
1137 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
1138 bne-- vsret ; We do not own the vector, no save required...
1139 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
1141 cmplw r9,r11 ; Was the context for this processor?
1142 la r5,VMXsync(r3) ; Point to the sync word
1143 bne-- vsret ; Specified context is not live
1146 ; It looks like we need to save this one. Or possibly toss a saved one if
1149 ; First, make sure that the live context block is not mucked with while
1150 ; we are trying to save it on out. Then we will give it the final check.
1153 lis r9,ha16(EXT(LockTimeOut)) ; Get the high part
1154 mftb r8 ; Get the time now
1155 lwz r9,lo16(EXT(LockTimeOut))(r9) ; Get the timeout value
1156 b vssync0a ; Jump to the lock...
1160 vssync0: li r7,lgKillResv ; Get killing field
1161 stwcx. r7,0,r7 ; Kill reservation
1163 vssync0a: lwz r7,0(r5) ; Sniff the lock
1164 mftb r10 ; Is it time yet?
1165 cmplwi cr1,r7,0 ; Is it locked?
1166 sub r10,r10,r8 ; How long have we been spinning?
1167 cmplw r10,r9 ; Has it been too long?
1168 bgt-- vswtimeout0 ; Way too long, panic...
1169 bne-- cr1,vssync0a ; Yea, still locked so sniff harder...
1171 vssync1: lwarx r7,0,r5 ; Get the sync word
1172 li r12,1 ; Get the lock
1173 mr. r7,r7 ; Is it unlocked?
1174 bne-- vssync0 ; No, it is unlocked...
1175 stwcx. r12,0,r5 ; Store lock and test reservation
1176 bne-- vssync1 ; Try again if lost reservation...
1178 isync ; Toss speculation
1180 lwz r12,VMXowner(r6) ; Get the context ID for owner
1181 cmplw r3,r12 ; Check again if we own VMX?
1182 lwz r10,liveVRS(r6) ; Get the right VRSave register
1183 bne-- vsretlk ; Go unlock and return since we no longer own context
1185 lwz r5,VMXcpu(r12) ; Get the cpu that context was last on
1186 lwz r7,VMXsave(r12) ; Get the current vector savearea for the thread
1187 cmplwi cr1,r10,0 ; Is VRsave set to 0?
1188 cmplw r5,r11 ; Is this for the same processor?
1189 lwz r9,VMXlevel(r12) ; Get our current level indicator
1190 bne-- vsretlk ; Not the same processor, skip any save...
1192 cmplwi r7,0 ; Have we ever saved this facility context?
1193 beq-- vsneedone ; Never saved it, so we need an area...
1195 lwz r8,SAVlevel(r7) ; Get the level this savearea is for
1196 cmplw r9,r8 ; Correct level?
1197 bne-- vsneedone ; Different level, so we need to save...
1199 bne++ cr1,vsretlk ; VRsave is non-zero so we need to keep what is saved...
1201 lwz r4,SAVprev+4(r7) ; Pick up the previous area
1202 li r5,0 ; Assume we just dumped the last
1203 mr. r4,r4 ; Is there one?
1204 stw r4,VMXsave(r12) ; Dequeue this savearea
1205 beq-- vsnomore ; We do not have another...
1207 lwz r5,SAVlevel(r4) ; Get the level associated with save
1209 vsnomore: stw r5,VMXlevel(r12) ; Save the level
1211 stw r7,VMXowner(r6) ; Show no live context here
1213 vsbackout: mr r4,r0 ; restore the saved MSR
1215 stw r7,VMXsync(r12) ; Unlock the context
1217 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
1221 vsneedone: beq-- cr1,vsclrlive ; VRSave is zero, go blow away the context...
1223 bl EXT(save_get) ; Get a savearea for the context
1225 mfsprg r6,1 ; Get the current activation
1226 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1227 li r4,SAVvector ; Get vector tag
1228 lwz r12,VMXowner(r6) ; Get back our context ID
1229 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1230 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1232 beq-- vsbackout ; If disowned, just toss savearea...
1233 lwz r4,facAct(r12) ; Get the activation associated with live context
1234 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1235 stw r4,SAVact(r3) ; Indicate the right activation for this context
1236 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1237 stw r3,VMXsave(r12) ; Set this as the most current floating point context
1238 stw r8,SAVprev+4(r3) ; And then chain this in front
1240 stw r9,SAVlevel(r3) ; Set level in savearea
1241 mfcr r12 ; save CRs across call to vr_store
1242 lwz r10,liveVRS(r6) ; Get the right VRSave register
1244 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
1246 mfsprg r6,1 ; Get the current activation
1247 mtcrf 255,r12 ; Restore the non-volatile CRs
1248 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1249 mtlr r2 ; Restore return address
1250 lwz r12,VMXowner(r6) ; Get back our context ID
1252 vsretlk: li r7,0 ; Get the unlock value
1253 eieio ; Make sure that these updates make it out
1254 stw r7,VMXsync(r12) ; Unlock it
1256 vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1261 vsclrlive: li r7,0 ; Clear
1262 stw r7,VMXowner(r6) ; Show no live context here
1263 b vsretlk ; Go unlock and leave...
1268 * Entered to handle the vector unavailable exception and
1269 * switch vector context
1271 * This code is run with virtual address mode on and interrupts off.
1273 * Upon exit, the code returns to the users context with the vector
1274 * facility turned on.
1276 * ENTRY: VM switched ON
1278 * State is saved in savearea pointed to by R4.
1279 * All other registers are free.
1284 .globl EXT(vec_switch)
1289 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1290 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1296 mfsprg r17,1 ; Get the current activation
1297 lwz r26,ACT_PER_PROC(r17) ; Get the per_proc block
1298 mfmsr r19 ; Get the current MSR
1300 mr r25,r4 ; Save the entry savearea
1301 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1302 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1304 mtmsr r19 ; Enable vector instructions
1307 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1308 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
1310 ; R22 has the "old" context anchor
1311 ; R29 has the "new" context anchor
1314 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1315 li r2,0x5F01 ; (TEST/DEBUG)
1316 mr r3,r22 ; (TEST/DEBUG)
1317 mr r5,r29 ; (TEST/DEBUG)
1318 lwz r6,liveVRS(r26) ; (TEST/DEBUG)
1319 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1323 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1325 mr. r22,r22 ; See if there is any live vector status
1326 la r15,VMXsync(r22) ; Point to the sync word
1328 beq-- vswnosave ; No live context, so nothing to save...
1330 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1331 cmplw cr2,r22,r29 ; Are both old and new the same context?
1332 lwz r30,VMXsave(r22) ; Get the top savearea
1333 cmplwi cr1,r30,0 ; Anything saved yet?
1334 lwz r31,VMXlevel(r22) ; Get the context level
1335 cmplw r18,r16 ; Make sure we are on the right processor
1337 lwz r10,liveVRS(r26) ; Get the right VRSave register
1339 bne-- vswnosave ; No, not on the same processor...
1342 ; Check to see if the live context has already been saved.
1343 ; Also check to see if all we are here just to re-enable the MSR
1344 ; and handle specially if so.
1347 cmplw r31,r27 ; See if the current and active levels are the same
1348 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1350 beq-- vswthesame ; New and old are the same, just go enable...
1353 ; Make sure that the live context block is not mucked with while
1354 ; we are trying to save it on out
1357 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
1358 mftb r3 ; Get the time now
1359 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
1360 b vswsync0a ; Jump to the lock...
1364 vswsync0: li r19,lgKillResv ; Get killing field
1365 stwcx. r19,0,r19 ; Kill reservation
1367 vswsync0a: lwz r19,0(r15) ; Sniff the lock
1368 mftb r18 ; Is it time yet?
1369 cmplwi cr1,r19,0 ; Is it locked?
1370 sub r18,r18,r3 ; How long have we been spinning?
1371 cmplw r18,r11 ; Has it been too long?
1372 bgt-- vswtimeout0 ; Way too long, panic...
1373 bne-- cr1,vswsync0a ; Yea, still locked so sniff harder...
1375 vswsync1: lwarx r19,0,r15 ; Get the sync word
1376 li r0,1 ; Get the lock
1377 mr. r19,r19 ; Is it unlocked?
1379 stwcx. r0,0,r15 ; Store lock and test reservation
1380 bne-- vswsync1 ; Try again if lost reservation...
1382 isync ; Toss speculation
1385 ; Note that now that we have the lock, we need to check if anything changed.
1386 ; Also note that the possible changes are limited. The context owner can
1387 ; never change to a different thread or level although it can be invalidated.
1388 ; A new context can not be pushed on top of us, but it can be popped. The
1389 ; cpu indicator will always change if another processor mucked with any
1392 ; It should be very rare that any of the context stuff changes across the lock.
1395 lwz r0,VMXowner(r26) ; Get the thread that owns the vectors again
1396 lwz r11,VMXsave(r22) ; Get the top savearea again
1397 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on again
1398 sub r0,r0,r22 ; Non-zero if we lost ownership, 0 if not
1399 xor r11,r11,r30 ; Non-zero if saved context changed, 0 if not
1400 xor r18,r18,r16 ; Non-zero if cpu changed, 0 if not
1401 cmplwi cr1,r30,0 ; Is anything saved?
1402 or r0,r0,r11 ; Zero only if both owner and context are unchanged
1403 or. r0,r0,r18 ; Zero only if nothing has changed
1404 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1407 bne-- vswnosavelk ; Something has changed, so this is not ours to save...
1408 beq-- cr1,vswmstsave ; There is no context saved yet...
1410 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1412 cmplw r31,r11 ; Are live and saved the same?
1415 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1416 li r2,0x5F02 ; (TEST/DEBUG)
1417 mr r3,r30 ; (TEST/DEBUG)
1418 mr r5,r31 ; (TEST/DEBUG)
1419 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1423 beq++ vswnosavelk ; Same level, already saved...
1424 bne-- cr2,vswnosavelk ; Live context saved and VRSave not 0, no save and keep context...
1426 lwz r4,SAVprev+4(r30) ; Pick up the previous area
1427 li r5,0 ; Assume this is the only one (which should be the ususal case)
1428 mr. r4,r4 ; Was this the only one?
1429 stw r4,VMXsave(r22) ; Dequeue this savearea
1430 beq++ vswonlyone ; This was the only one...
1431 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1433 vswonlyone: stw r5,VMXlevel(r22) ; Save the level
1434 stw r8,VMXowner(r26) ; Clear owner
1436 mr r3,r30 ; Copy the savearea we are tossing
1437 bl EXT(save_ret) ; Toss the savearea
1438 b vswnosavelk ; Go load up the context...
1442 vswmstsave: stw r8,VMXowner(r26) ; Clear owner
1443 beq-- cr2,vswnosavelk ; The VRSave was 0, so there is nothing to save...
1445 bl EXT(save_get) ; Go get a savearea
1447 lwz r12,facAct(r22) ; Get the activation associated with the context
1448 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1449 stw r30,SAVprev+4(r3) ; Point us to the old context
1450 stw r31,SAVlevel(r3) ; Tag our level
1451 li r7,SAVvector ; Get the vector ID
1452 stw r12,SAVact(r3) ; Make sure we point to the right guy
1453 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
1456 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1457 li r2,0x5F03 ; (TEST/DEBUG)
1458 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1462 lwz r10,liveVRS(r26) ; Get the right VRSave register
1463 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1466 ; The context is all saved now and the facility is free.
1468 ; Check if we need to fill the registers with junk, because this level has
1469 ; never used them before and some thieving bastard could hack the old values
1470 ; of some thread! Just imagine what would happen if they could! Why, nothing
1471 ; would be safe! My God! It is terrifying!
1473 ; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1474 ; constant that we may need to fill unused vector registers.
1476 ; Make sure that the live context block is not mucked with while
1477 ; we are trying to load it up
1481 li r7,0 ; Get the unlock value
1482 eieio ; Make sure that these updates make it out
1483 stw r7,VMXsync(r22) ; Unlock the old context
1485 vswnosave: la r15,VMXsync(r29) ; Point to the sync word
1486 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
1487 mftb r3 ; Get the time now
1488 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
1489 b vswnsync0a ; Jump to the lock...
1493 vswnsync0: li r19,lgKillResv ; Get killing field
1494 stwcx. r19,0,r19 ; Kill reservation
1496 vswnsync0a: lwz r19,0(r15) ; Sniff the lock
1497 mftb r18 ; Is it time yet?
1498 cmplwi cr1,r19,0 ; Is it locked?
1499 sub r18,r18,r3 ; How long have we been spinning?
1500 cmplw r18,r11 ; Has it been too long?
1501 bgt-- vswtimeout1 ; Way too long, panic...
1502 bne-- cr1,vswnsync0a ; Yea, still locked so sniff harder...
1504 vswnsync1: lwarx r19,0,r15 ; Get the sync word
1505 li r0,1 ; Get the lock
1506 mr. r19,r19 ; Is it unlocked?
1507 bne-- vswnsync0 ; Unfortunately, it is locked...
1508 stwcx. r0,0,r15 ; Store lock and test reservation
1509 bne-- vswnsync1 ; Try again if lost reservation...
1511 isync ; Toss speculation
1513 vspltisb v31,-10 ; Get 0xF6F6F6F6
1514 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1515 vspltisb v30,5 ; Get 0x05050505
1516 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1517 vspltish v29,4 ; Get 0x00040004
1518 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1519 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1521 stw r16,VMXcpu(r29) ; Claim context for us
1525 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1526 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1527 li r2,0x5F04 ; (TEST/DEBUG)
1528 mr r1,r15 ; (TEST/DEBUG)
1529 mr r3,r14 ; (TEST/DEBUG)
1530 mr r5,r13 ; (TEST/DEBUG)
1531 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1535 lis r18,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1536 vspltisb v28,-2 ; Get 0xFEFEFEFE
1537 mulli r19,r19,ppeSize ; Find offset to the owner per_proc_entry
1538 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1539 ori r18,r18,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1540 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1541 li r16,VMXowner ; Displacement to vector owner
1542 add r19,r18,r19 ; Point to the owner per_proc_entry
1543 lwz r19,ppe_vaddr(r19) ; Point to the owner per_proc
1544 vrlb v31,v31,v29 ; Get 0xDEADDEAD
1546 vswinvothr: lwarx r18,r16,r19 ; Get the owner
1548 sub r0,r18,r29 ; Subtract one from the other
1549 sub r11,r29,r18 ; Subtract the other from the one
1550 or r11,r11,r0 ; Combine them
1551 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1552 and r18,r18,r11 ; Make 0 if same, unchanged if not
1553 stwcx. r18,r16,r19 ; Try to invalidate it
1554 bne-- vswinvothr ; Try again if there was a collision...
1556 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
1557 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1558 stw r15,VMXlevel(r29) ; Set the "new" active level
1560 stw r29,VMXowner(r26) ; Mark us as having the live context
1562 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
1564 lwz r3,SAVprev+4(r14) ; Get the previous context
1565 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1566 cmplw r0,r15 ; Top level correct to load?
1567 bne-- ProtectTheAmericanWay ; No, go initialize...
1569 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1572 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1573 li r2,0x5F05 ; (TEST/DEBUG)
1574 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1578 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
1579 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
1580 and r10,r10,r22 ; Figure out just what registers need to be loaded
1581 mr r3,r14 ; r3 <- ptr to savearea with VRs
1582 bl vr_load ; load VRs from save area based on vrsave in r10
1584 bl EXT(save_ret) ; Toss the save area after loading VRs
1586 vrenablelk: li r7,0 ; Get the unlock value
1587 eieio ; Make sure that these updates make it out
1588 stw r7,VMXsync(r29) ; Unlock the new context
1590 vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
1591 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
1592 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1593 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1594 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1595 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1596 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
1597 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1598 mr r3,r25 ; Pass virtual address of the savearea
1599 beq- vrnuser ; We are not user state...
1600 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
1601 stw r11,spcFlags(r26) ; Set per_proc copy
1605 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1606 li r2,0x5F07 ; (TEST/DEBUG)
1607 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1610 b EXT(exception_exit) ; Exit to the fray...
1613 * Initialize the registers to some bogus value
1616 ProtectTheAmericanWay:
1619 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1620 li r2,0x5F06 ; (TEST/DEBUG)
1621 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1625 vor v0,v31,v31 ; Copy into the next register
1626 vor v1,v31,v31 ; Copy into the next register
1627 vor v2,v31,v31 ; Copy into the next register
1628 vor v3,v31,v31 ; Copy into the next register
1629 vor v4,v31,v31 ; Copy into the next register
1630 vor v5,v31,v31 ; Copy into the next register
1631 vor v6,v31,v31 ; Copy into the next register
1632 vor v7,v31,v31 ; Copy into the next register
1633 vor v8,v31,v31 ; Copy into the next register
1634 vor v9,v31,v31 ; Copy into the next register
1635 vor v10,v31,v31 ; Copy into the next register
1636 vor v11,v31,v31 ; Copy into the next register
1637 vor v12,v31,v31 ; Copy into the next register
1638 vor v13,v31,v31 ; Copy into the next register
1639 vor v14,v31,v31 ; Copy into the next register
1640 vor v15,v31,v31 ; Copy into the next register
1641 vor v16,v31,v31 ; Copy into the next register
1642 vor v17,v31,v31 ; Copy into the next register
1643 vor v18,v31,v31 ; Copy into the next register
1644 vor v19,v31,v31 ; Copy into the next register
1645 vor v20,v31,v31 ; Copy into the next register
1646 vor v21,v31,v31 ; Copy into the next register
1647 vor v22,v31,v31 ; Copy into the next register
1648 vor v23,v31,v31 ; Copy into the next register
1649 vor v24,v31,v31 ; Copy into the next register
1650 vor v25,v31,v31 ; Copy into the next register
1651 vor v26,v31,v31 ; Copy into the next register
1652 vor v27,v31,v31 ; Copy into the next register
1653 vor v28,v31,v31 ; Copy into the next register
1654 vor v29,v31,v31 ; Copy into the next register
1655 vor v30,v31,v31 ; Copy into the next register
1656 b vrenablelk ; Finish setting it all up...
1661 ; We get here when we are switching to the same context at the same level and the context
1662 ; is still live. Essentially, all we are doing is turning on the faility. It may have
1663 ; gotten turned off due to doing a context save for the current level or a context switch
1664 ; back to the live guy.
1672 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1673 li r2,0x5F0A ; (TEST/DEBUG)
1674 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1677 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1679 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1680 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1682 cmplw r11,r31 ; Are live and saved the same?
1684 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1686 mr r3,r30 ; Get the old savearea (we popped it before)
1687 stw r11,VMXsave(r22) ; Pop the vector stack
1688 bl EXT(save_ret) ; Toss it
1689 b vrenable ; Go enable and exit...
1693 ; This function invalidates any live vector context for the passed in facility_context.
1694 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1698 .globl EXT(toss_live_vec)
1702 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1703 mfmsr r9 ; Get the MSR
1704 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1705 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1706 andc r9,r9,r0 ; Force off VEC and FP
1707 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1708 andc r0,r9,r0 ; Turn off EE now
1709 mtmsr r0 ; No interruptions
1711 beq+ tlvnotours ; Vector off, can not be live here...
1713 mfsprg r8,1 ; Get the current activation
1714 lwz r8,ACT_PER_PROC(r8) ; Get the per_proc block
1717 ; Note that at this point, since vecs are on, we are the owner
1718 ; of live state on this processor
1721 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1722 li r0,0 ; Clear this just in case we need it
1723 cmplw r6,r3 ; Are we tossing our own context?
1724 bne- tlvnotours ; Nope...
1726 vspltish v1,1 ; Turn on the non-Java bit and saturate
1727 vspltisw v0,1 ; Turn on the saturate bit
1728 vxor v1,v1,v0 ; Turn off saturate
1729 mtspr vrsave,r0 ; Clear VRSAVE
1730 mtvscr v1 ; Set the non-java, no saturate status
1732 tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1733 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1734 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1735 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1736 li r10,VMXowner ; Displacement to vector owner
1737 add r11,r12,r11 ; Point to the owner per_proc_entry
1738 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1739 li r0,0 ; Set a 0 to invalidate context
1741 tlvinvothr: lwarx r12,r10,r11 ; Get the owner
1743 sub r0,r12,r3 ; Subtract one from the other
1744 sub r8,r3,r12 ; Subtract the other from the one
1745 or r8,r8,r0 ; Combine them
1746 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1747 and r12,r12,r8 ; Make 0 if same, unchanged if not
1748 stwcx. r12,r10,r11 ; Try to invalidate it
1749 bne-- tlvinvothr ; Try again if there was a collision...
1751 mtmsr r9 ; Restore interruptions
1752 isync ; Could be turning off vectors here
1757 ; This function invalidates any live vector context for the passed in facility_context
1758 ; if the level is current. It also tosses the corresponding savearea if there is one.
1759 ; This function is primarily used whenever we detect a VRSave that is all zeros.
1763 .globl EXT(vec_trash)
1767 lwz r12,facAct(r3) ; Get the activation
1768 lwz r11,VMXlevel(r3) ; Get the context level
1769 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1770 lwz r9,VMXsave(r3) ; Get the savearea, if any
1771 cmplw r10,r11 ; Are we at the right level?
1772 cmplwi cr1,r9,0 ; Remember if there is a savearea
1773 bnelr+ ; No, we do nothing...
1775 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1776 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1777 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1778 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1779 li r10,VMXowner ; Displacement to vector owner
1780 add r11,r12,r11 ; Point to the owner per_proc_entry
1781 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1783 vtinvothr: lwarx r12,r10,r11 ; Get the owner
1785 sub r0,r12,r3 ; Subtract one from the other
1786 sub r8,r3,r12 ; Subtract the other from the one
1787 or r8,r8,r0 ; Combine them
1788 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1789 and r12,r12,r8 ; Make 0 if same, unchanged if not
1790 stwcx. r12,r10,r11 ; Try to invalidate it
1791 bne-- vtinvothr ; Try again if there was a collision...
1794 beqlr++ cr1 ; Leave if there is no savearea
1795 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1796 cmplw r8,r11 ; Savearea for the current level?
1797 bnelr++ ; No, nothing to release...
1799 lwz r8,SAVprev+4(r9) ; Pick up the previous area
1800 mr. r8,r8 ; Is there a previous?
1801 beq-- vtnoprev ; Nope...
1802 lwz r7,SAVlevel(r8) ; Get the level associated with save
1804 vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1805 stw r7,VMXlevel(r3) ; Pop the level
1807 mr r3,r9 ; Get the savearea to release
1808 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1812 ; Just some test code to force vector and/or floating point in the kernel
1816 .globl EXT(fctx_test)
1820 mfsprg r3,1 ; Get the current thread
1821 mr. r3,r3 ; Are we actually up and running?
1824 fmr f0,f0 ; Use floating point
1825 mftb r4 ; Get time base for a random number
1826 li r5,1 ; Get a potential vrsave to use
1827 andi. r4,r4,0x3F ; Get a number from 0 - 63
1828 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1829 mtspr vrsave,r5 ; Set VRSave
1830 vor v0,v0,v0 ; Use vectors
1834 // *******************
1835 // * f p _ s t o r e *
1836 // *******************
1838 // Store FPRs into a save area. Called by fpu_save and fpu_switch.
1841 // floating pt is enabled
1842 // r3 = ptr to save area
1848 mfsprg r11,2 ; get feature flags
1849 mtcrf 0x02,r11 ; put cache line size bits in cr6
1850 la r11,savefp0(r3) ; point to 1st line
1851 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1852 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1854 // Store the FPRs on a 128-byte machine.
1858 la r11,savefp16(r3) ; Point to the 2nd cache line
1861 dcbz128 0,r11 ; establish 2nd line
1868 stfd f10,savefp10(r3)
1869 stfd f11,savefp11(r3)
1870 stfd f12,savefp12(r3)
1871 stfd f13,savefp13(r3)
1872 stfd f14,savefp14(r3)
1873 stfd f15,savefp15(r3)
1874 stfd f16,savefp16(r3)
1875 stfd f17,savefp17(r3)
1876 stfd f18,savefp18(r3)
1877 stfd f19,savefp19(r3)
1878 stfd f20,savefp20(r3)
1879 stfd f21,savefp21(r3)
1880 stfd f22,savefp22(r3)
1881 stfd f23,savefp23(r3)
1882 stfd f24,savefp24(r3)
1883 stfd f25,savefp25(r3)
1884 stfd f26,savefp26(r3)
1885 stfd f27,savefp27(r3)
1886 stfd f28,savefp28(r3)
1887 stfd f29,savefp29(r3)
1888 stfd f30,savefp30(r3)
1889 stfd f31,savefp31(r3)
1892 // Store FPRs on a 32-byte machine.
1895 la r11,savefp4(r3) ; Point to the 2nd line
1897 dcbz 0,r11 ; Allocate cache
1900 la r11,savefp8(r3) ; Point to the 3rd line
1902 dcbz 0,r11 ; Allocate cache
1906 la r11,savefp12(r3) ; Point to the 4th line
1908 dcbz 0,r11 ; Allocate cache
1911 stfd f10,savefp10(r3)
1912 la r11,savefp16(r3) ; Point to the 5th line
1913 stfd f11,savefp11(r3)
1914 dcbz 0,r11 ; Allocate cache
1915 stfd f12,savefp12(r3)
1916 stfd f13,savefp13(r3)
1917 stfd f14,savefp14(r3)
1918 la r11,savefp20(r3) ; Point to the 6th line
1919 stfd f15,savefp15(r3)
1920 dcbz 0,r11 ; Allocate cache
1921 stfd f16,savefp16(r3)
1922 stfd f17,savefp17(r3)
1923 stfd f18,savefp18(r3)
1924 la r11,savefp24(r3) ; Point to the 7th line
1925 stfd f19,savefp19(r3)
1926 dcbz 0,r11 ; Allocate cache
1927 stfd f20,savefp20(r3)
1929 stfd f21,savefp21(r3)
1930 stfd f22,savefp22(r3)
1931 la r11,savefp28(r3) ; Point to the 8th line
1932 stfd f23,savefp23(r3)
1933 dcbz 0,r11 ; allocate it
1934 stfd f24,savefp24(r3)
1935 stfd f25,savefp25(r3)
1936 stfd f26,savefp26(r3)
1937 stfd f27,savefp27(r3)
1939 stfd f28,savefp28(r3)
1940 stfd f29,savefp29(r3)
1941 stfd f30,savefp30(r3)
1942 stfd f31,savefp31(r3)
1946 // *******************
1947 // * v r _ s t o r e *
1948 // *******************
1950 // Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1951 // both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1952 // unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1953 // for each cache block size.
1956 // interrupts are off, vectors are enabled
1957 // r3 = ptr to save area
1958 // r10 = vrsave (not 0)
1961 // r4 - r11, all CRs.
1964 mfsprg r9,2 ; get feature flags
1965 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1966 slwi r8,r10,1 ; Shift over 1
1967 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1968 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1969 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1972 ; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1973 ; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1975 slwi r7,r8,2 ; shift groups-of-2 over by 2
1976 li r4,16 ; load offsets for X-form stores
1977 or r8,r7,r8 ; show if any in group of 4 are in use
1979 slwi r7,r8,4 ; shift groups-of-4 over by 4
1981 or r11,r7,r8 ; show if any in group of 8 are in use
1983 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1991 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1992 la r11,savevr0(r3) ; get address of this group of registers in save area
1993 dcbz128 0,r11 ; zero the line
1994 stvxl v0,0,r11 ; save 8 VRs in the line
2004 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
2005 la r11,savevr8(r3) ; get address of this group of registers in save area
2006 dcbz128 0,r11 ; zero the line
2007 stvxl v8,0,r11 ; save 8 VRs in the line
2017 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
2018 la r11,savevr16(r3) ; get address of this group of registers in save area
2019 dcbz128 0,r11 ; zero the line
2020 stvxl v16,0,r11 ; save 8 VRs in the line
2030 bflr 24 ; done if none of vr24-vr31 are in use
2031 la r11,savevr24(r3) ; get address of this group of registers in save area
2032 dcbz128 0,r11 ; zero the line
2033 stvxl v24,0,r11 ; save 8 VRs in the line
2043 ; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
2044 ; or neither in each group. This cuts down on conditional branches.
2045 ; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
2049 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
2050 li r4,16 ; load offset for X-form stores
2052 bf 0,vr_st32b ; skip if neither VR in this pair is in use
2053 la r11,savevr0(r3) ; get address of this group of registers in save area
2054 dcba 0,r11 ; establish the line wo reading it
2055 stvxl v0,0,r11 ; save the two VRs in the line
2059 bf 2,vr_st32c ; skip if neither VR in this pair is in use
2060 la r11,savevr2(r3) ; get address of this group of registers in save area
2061 dcba 0,r11 ; establish the line wo reading it
2062 stvxl v2,0,r11 ; save the two VRs in the line
2066 bf 4,vr_st32d ; skip if neither VR in this pair is in use
2067 la r11,savevr4(r3) ; get address of this group of registers in save area
2068 dcba 0,r11 ; establish the line wo reading it
2069 stvxl v4,0,r11 ; save the two VRs in the line
2073 bf 6,vr_st32e ; skip if neither VR in this pair is in use
2074 la r11,savevr6(r3) ; get address of this group of registers in save area
2075 dcba 0,r11 ; establish the line wo reading it
2076 stvxl v6,0,r11 ; save the two VRs in the line
2080 bf 8,vr_st32f ; skip if neither VR in this pair is in use
2081 la r11,savevr8(r3) ; get address of this group of registers in save area
2082 dcba 0,r11 ; establish the line wo reading it
2083 stvxl v8,0,r11 ; save the two VRs in the line
2087 bf 10,vr_st32g ; skip if neither VR in this pair is in use
2088 la r11,savevr10(r3) ; get address of this group of registers in save area
2089 dcba 0,r11 ; establish the line wo reading it
2090 stvxl v10,0,r11 ; save the two VRs in the line
2094 bf 12,vr_st32h ; skip if neither VR in this pair is in use
2095 la r11,savevr12(r3) ; get address of this group of registers in save area
2096 dcba 0,r11 ; establish the line wo reading it
2097 stvxl v12,0,r11 ; save the two VRs in the line
2101 bf 14,vr_st32i ; skip if neither VR in this pair is in use
2102 la r11,savevr14(r3) ; get address of this group of registers in save area
2103 dcba 0,r11 ; establish the line wo reading it
2104 stvxl v14,0,r11 ; save the two VRs in the line
2108 bf 16,vr_st32j ; skip if neither VR in this pair is in use
2109 la r11,savevr16(r3) ; get address of this group of registers in save area
2110 dcba 0,r11 ; establish the line wo reading it
2111 stvxl v16,0,r11 ; save the two VRs in the line
2115 bf 18,vr_st32k ; skip if neither VR in this pair is in use
2116 la r11,savevr18(r3) ; get address of this group of registers in save area
2117 dcba 0,r11 ; establish the line wo reading it
2118 stvxl v18,0,r11 ; save the two VRs in the line
2122 bf 20,vr_st32l ; skip if neither VR in this pair is in use
2123 la r11,savevr20(r3) ; get address of this group of registers in save area
2124 dcba 0,r11 ; establish the line wo reading it
2125 stvxl v20,0,r11 ; save the two VRs in the line
2129 bf 22,vr_st32m ; skip if neither VR in this pair is in use
2130 la r11,savevr22(r3) ; get address of this group of registers in save area
2131 dcba 0,r11 ; establish the line wo reading it
2132 stvxl v22,0,r11 ; save the two VRs in the line
2136 bf 24,vr_st32n ; skip if neither VR in this pair is in use
2137 la r11,savevr24(r3) ; get address of this group of registers in save area
2138 dcba 0,r11 ; establish the line wo reading it
2139 stvxl v24,0,r11 ; save the two VRs in the line
2143 bf 26,vr_st32o ; skip if neither VR in this pair is in use
2144 la r11,savevr26(r3) ; get address of this group of registers in save area
2145 dcba 0,r11 ; establish the line wo reading it
2146 stvxl v26,0,r11 ; save the two VRs in the line
2150 bf 28,vr_st32p ; skip if neither VR in this pair is in use
2151 la r11,savevr28(r3) ; get address of this group of registers in save area
2152 dcba 0,r11 ; establish the line wo reading it
2153 stvxl v28,0,r11 ; save the two VRs in the line
2157 bflr 30 ; done if neither VR in this pair is in use
2158 la r11,savevr30(r3) ; get address of this group of registers in save area
2159 dcba 0,r11 ; establish the line wo reading it
2160 stvxl v30,0,r11 ; save the two VRs in the line
2165 // *****************
2166 // * v r _ l o a d *
2167 // *****************
2169 // Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
2170 // of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
2171 // branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
2172 // in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
2173 // which would bug all VRs that aren't live.
2176 // interrupts are off, vectors are enabled
2177 // r3 = ptr to save area
2178 // r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
2179 // v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
2182 // r4 - r11, all CRs.
2185 mfsprg r9,2 ; get feature flags
2186 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
2187 mtcrf 0x02,r9 ; set cache line size bits in cr6
2188 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
2189 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
2190 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
2191 lis r7,0xFF00 ; so set bits 0-7
2193 // Loop touching in cache blocks we will load from.
2194 // r3 = savearea ptr
2195 // r5 = we light bits for the VRs we will be loading
2196 // r6 = 1 if 32-byte, 7 if 128-byte
2197 // r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
2198 // r10 = live VR bits
2199 // v31 = bugbug constant
2202 li r5,0 ; initialize set of VRs to load
2203 la r11,savevr0(r3) ; get address of register file
2204 b vr_ld2 ; enter loop in middle
2207 vr_ld1: ; loop over each cache line we will load
2208 dcbt r4,r11 ; start prefetch of the line
2209 andc r10,r10,r9 ; turn off the bits in this line
2210 or r5,r5,r9 ; we will load all these
2211 vr_ld2: ; initial entry pt
2212 cntlzw r4,r10 ; get offset to next live VR
2213 andc r4,r4,r6 ; cacheline align it
2214 srw. r9,r7,r4 ; position bits for VRs in that cache line
2215 slwi r4,r4,4 ; get byte offset within register file to that line
2216 bne vr_ld1 ; loop if more bits in r10
2218 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
2220 // Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
2221 // r3 = savearea ptr
2222 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2223 // r11 = addr(savevr0)
2224 // v31 = bugbug constant
2226 mtcrf 0x80,r5 ; set up bits for conditional branches
2227 li r4,16 ; load offsets for X-form stores
2229 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
2238 bt 0,vr_ld128a ; skip if this line must be loaded
2239 vor v0,v31,v31 ; no VR must be loaded, so bug them all
2248 vr_ld128a: ; must load from this line
2258 vr_ld128b: ; here to handle next cache line
2259 la r11,savevr8(r3) ; load offset to it
2260 bt 8,vr_ld128c ; skip if this line must be loaded
2261 vor v8,v31,v31 ; no VR must be loaded, so bug them all
2270 vr_ld128c: ; must load from this line
2280 vr_ld128d: ; here to handle next cache line
2281 la r11,savevr16(r3) ; load offset to it
2282 bt 16,vr_ld128e ; skip if this line must be loaded
2283 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2292 vr_ld128e: ; must load from this line
2302 vr_ld128f: ; here to handle next cache line
2303 la r11,savevr24(r3) ; load offset to it
2304 bt 24,vr_ld128g ; skip if this line must be loaded
2305 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2313 vr_ld128g: ; must load from this line
2324 // Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2325 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2326 // r11 = addr(savevr0)
2329 mtcrf 0xFF,r5 ; set up bits for conditional branches
2330 li r4,16 ; load offset for X-form stores
2332 bt 0,vr_ld32load0 ; skip if we must load this line
2333 vor v0,v31,v31 ; neither VR is live, so bug them both
2336 vr_ld32load0: ; must load VRs in this line
2340 vr_ld32test2: ; here to handle next cache line
2341 la r11,savevr2(r3) ; get offset to next cache line
2342 bt 2,vr_ld32load2 ; skip if we must load this line
2343 vor v2,v31,v31 ; neither VR is live, so bug them both
2346 vr_ld32load2: ; must load VRs in this line
2350 vr_ld32test4: ; here to handle next cache line
2351 la r11,savevr4(r3) ; get offset to next cache line
2352 bt 4,vr_ld32load4 ; skip if we must load this line
2353 vor v4,v31,v31 ; neither VR is live, so bug them both
2356 vr_ld32load4: ; must load VRs in this line
2360 vr_ld32test6: ; here to handle next cache line
2361 la r11,savevr6(r3) ; get offset to next cache line
2362 bt 6,vr_ld32load6 ; skip if we must load this line
2363 vor v6,v31,v31 ; neither VR is live, so bug them both
2366 vr_ld32load6: ; must load VRs in this line
2370 vr_ld32test8: ; here to handle next cache line
2371 la r11,savevr8(r3) ; get offset to next cache line
2372 bt 8,vr_ld32load8 ; skip if we must load this line
2373 vor v8,v31,v31 ; neither VR is live, so bug them both
2376 vr_ld32load8: ; must load VRs in this line
2380 vr_ld32test10: ; here to handle next cache line
2381 la r11,savevr10(r3) ; get offset to next cache line
2382 bt 10,vr_ld32load10 ; skip if we must load this line
2383 vor v10,v31,v31 ; neither VR is live, so bug them both
2386 vr_ld32load10: ; must load VRs in this line
2390 vr_ld32test12: ; here to handle next cache line
2391 la r11,savevr12(r3) ; get offset to next cache line
2392 bt 12,vr_ld32load12 ; skip if we must load this line
2393 vor v12,v31,v31 ; neither VR is live, so bug them both
2396 vr_ld32load12: ; must load VRs in this line
2400 vr_ld32test14: ; here to handle next cache line
2401 la r11,savevr14(r3) ; get offset to next cache line
2402 bt 14,vr_ld32load14 ; skip if we must load this line
2403 vor v14,v31,v31 ; neither VR is live, so bug them both
2406 vr_ld32load14: ; must load VRs in this line
2410 vr_ld32test16: ; here to handle next cache line
2411 la r11,savevr16(r3) ; get offset to next cache line
2412 bt 16,vr_ld32load16 ; skip if we must load this line
2413 vor v16,v31,v31 ; neither VR is live, so bug them both
2416 vr_ld32load16: ; must load VRs in this line
2420 vr_ld32test18: ; here to handle next cache line
2421 la r11,savevr18(r3) ; get offset to next cache line
2422 bt 18,vr_ld32load18 ; skip if we must load this line
2423 vor v18,v31,v31 ; neither VR is live, so bug them both
2426 vr_ld32load18: ; must load VRs in this line
2430 vr_ld32test20: ; here to handle next cache line
2431 la r11,savevr20(r3) ; get offset to next cache line
2432 bt 20,vr_ld32load20 ; skip if we must load this line
2433 vor v20,v31,v31 ; neither VR is live, so bug them both
2436 vr_ld32load20: ; must load VRs in this line
2440 vr_ld32test22: ; here to handle next cache line
2441 la r11,savevr22(r3) ; get offset to next cache line
2442 bt 22,vr_ld32load22 ; skip if we must load this line
2443 vor v22,v31,v31 ; neither VR is live, so bug them both
2446 vr_ld32load22: ; must load VRs in this line
2450 vr_ld32test24: ; here to handle next cache line
2451 la r11,savevr24(r3) ; get offset to next cache line
2452 bt 24,vr_ld32load24 ; skip if we must load this line
2453 vor v24,v31,v31 ; neither VR is live, so bug them both
2456 vr_ld32load24: ; must load VRs in this line
2460 vr_ld32test26: ; here to handle next cache line
2461 la r11,savevr26(r3) ; get offset to next cache line
2462 bt 26,vr_ld32load26 ; skip if we must load this line
2463 vor v26,v31,v31 ; neither VR is live, so bug them both
2466 vr_ld32load26: ; must load VRs in this line
2470 vr_ld32test28: ; here to handle next cache line
2471 la r11,savevr28(r3) ; get offset to next cache line
2472 bt 28,vr_ld32load28 ; skip if we must load this line
2473 vor v28,v31,v31 ; neither VR is live, so bug them both
2476 vr_ld32load28: ; must load VRs in this line
2480 vr_ld32test30: ; here to handle next cache line
2481 la r11,savevr30(r3) ; get offset to next cache line
2482 bt 30,vr_ld32load30 ; skip if we must load this line
2483 vor v30,v31,v31 ; neither VR is live, so bug them both
2485 vr_ld32load30: ; must load VRs in this line
projects / apple / xnu.git / blob