2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
33 #include <ppc/proc_reg.h>
36 #include <mach/ppc/vm_param.h>
37 #include <ppc/exception.h>
38 #include <ppc/savearea.h>
45 * void machine_load_context(thread_t thread)
47 * Load the context for the first thread to run on a
52 .globl EXT(machine_load_context)
54 LEXT(machine_load_context)
55 mfsprg r6,1 ; Get the current activation
56 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
57 lwz r0,PP_INTSTACK_TOP_SS(r6)
58 stw r0,PP_ISTACKPTR(r6)
59 mr r9,r3 /* Set up the current thread */
61 li r0,0 /* Clear a register */
62 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
63 mfmsr r5 /* Since we are passing control, get our MSR values */
64 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
65 lwz r1,saver1+4(r3) /* Load new stack pointer */
66 lwz r10,ACT_MACT_SPF(r9) /* Get the special flags */
67 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
68 stw r0,FM_BACKPTR(r1) /* zero backptr */
69 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
70 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
71 oris r10,r10,hi16(OnProc) /* Set OnProc bit */
72 stw r0,ACT_PREEMPT_CNT(r9) /* Enable preemption */
73 stw r10,ACT_MACT_SPF(r9) /* Update the special flags */
74 stw r10,spcFlags(r6) /* Set per_proc copy of the special flags */
75 b EXT(exception_exit) /* Go for it */
77 /* thread_t Switch_context(thread_t old,
81 * Switch from one thread to another. If a continuation is supplied, then
82 * we do not need to save callee save registers.
86 /* void Call_continuation( void (*continuation)(void), void *param, wait_result_t wresult, vm_offset_t stack_ptr)
90 .globl EXT(Call_continuation)
92 LEXT(Call_continuation)
93 mtlr r3 /* continuation */
94 mr r3,r4 /* parameter */
95 mr r4,r5 /* wait result */
96 mr r1,r6 /* Load new stack pointer */
97 blrl /* Jump to the continuation */
99 b EXT(thread_terminate)
102 * Get the old kernel stack, and store into the thread structure.
103 * See if a continuation is supplied, and skip state save if so.
105 * Note that interrupts must be disabled before we get here (i.e., splsched)
109 * Switch_context(old, continuation, new)
111 * Context switches are double jumps. We pass the following to the
112 * context switch firmware call:
114 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
118 * R7 = high order physical address of savearea for full switch
120 * savesrr0 is set to go to switch_in
121 * savesrr1 is set to uninterruptible with translation on
126 .globl EXT(Switch_context)
130 lwz r12,ACT_PER_PROC(r3) ; Get the per_proc block
132 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
133 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
134 bne++ notonintstack ; (DEBUG/TRACE) stack
138 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
139 lwz r9,umwSpace(r5) ; Get user memory window address space
140 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
141 lwz r0,CTHREAD_SELF+0(r5) ; Pick up the user assist "word" (actually a double)
142 lwz r7,CTHREAD_SELF+4(r5) ; both halves
143 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
144 lwz r6,umwRelo(r5) ; Get user memory window relocation top
145 stw r12,ACT_PER_PROC(r5) ; Set per_proc in new activation
147 lwz r2,umwRelo+4(r5) ; Get user memory window relocation bottom
149 stw r0,UAW+0(r12) ; Save the assist word for the "ultra fast path"
152 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
154 sth r9,ppUMWmp+mpSpace(r12) ; Save the space
155 stw r6,ppUMWmp+mpNestReloc(r12) ; Save top part of physical address
156 stw r2,ppUMWmp+mpNestReloc+4(r12) ; Save bottom part of physical address
157 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
158 lwz r2,traceMask(0) ; Get the enabled traces
159 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
160 lis r0,hi16(CutTrace) ; Trace FW call
161 mr. r2,r2 ; Any tracing going on?
162 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
163 ori r0,r0,lo16(CutTrace) ; Trace FW call
164 beq++ cswNoTrc ; No trace today, dude...
166 li r2,0x4400 ; Trace ID
167 mr r6,r11 ; Trace prev savearea
168 sc ; Cut trace entry of context switch
170 cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
171 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
172 lwz r9,VMXowner(r12) ; Grab the owner of the vector
173 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
174 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
175 li r0,1 ; Get set to hold off quickfret
177 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
178 cmplw r10,r2 ; Do we have the live float context?
179 lwz r10,FPUlevel(r2) ; Get the live level
180 mr r4,r3 ; Save our old thread to pass back
181 cmplw cr5,r9,r2 ; Do we have the live vector context?
182 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
183 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
184 bne++ cswnofloat ; Float is not ours...
186 cmplw r10,r11 ; Is the level the same?
187 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
188 lwz r5,FPUcpu(r2) ; Get the owning cpu
189 bne++ cswnofloat ; Level not the same, this is not live...
191 cmplw r5,r0 ; Still owned by this cpu?
192 lwz r10,FPUsave(r2) ; Get the pointer to next saved context
193 bne++ cswnofloat ; CPU claimed by someone else...
195 mr. r10,r10 ; Is there a savearea here?
196 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
198 beq-- cswnofloat ; No savearea to check...
200 lwz r3,SAVlevel(r10) ; Get the level
201 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
202 cmplw r3,r11 ; Is it for the current level?
204 bne++ cswnofloat ; Nope...
206 stw r5,FPUsave(r2) ; Pop off this savearea
208 rlwinm r3,r10,0,0,19 ; Move back to start of page
210 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
211 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
212 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
213 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
214 stw r5,SAVprev(r10) ; Link the old in (top)
215 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
216 xor r3,r10,r3 ; Convert to physical
217 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
218 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
221 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
222 mr r7,r2 ; (TEST/DEBUG)
223 li r2,0x4401 ; (TEST/DEBUG)
224 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
226 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
227 mr r2,r7 ; (TEST/DEBUG)
230 cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
232 lwz r10,VMXlevel(r2) ; Get the live level
234 cmplw r10,r11 ; Is the level the same?
235 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
236 lwz r5,VMXcpu(r2) ; Get the owning cpu
237 bne++ cswnovect ; Level not the same, this is not live...
239 cmplw r5,r0 ; Still owned by this cpu?
240 lwz r10,VMXsave(r2) ; Get the level
241 bne++ cswnovect ; CPU claimed by someone else...
243 mr. r10,r10 ; Is there a savearea here?
244 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
246 beq-- cswnovect ; No savearea to check...
248 lwz r3,SAVlevel(r10) ; Get the level
249 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
250 cmplw r3,r11 ; Is it for the current level?
252 bne++ cswnovect ; Nope...
254 stw r5,VMXsave(r2) ; Pop off this savearea
255 rlwinm r3,r10,0,0,19 ; Move back to start of page
257 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
258 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
259 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
260 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
261 stw r5,SAVprev(r10) ; Link the old in (top)
262 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
263 xor r3,r10,r3 ; Convert to physical
264 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
265 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
268 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
269 li r2,0x4501 ; (TEST/DEBUG)
270 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
274 cswnovect: li r0,0 ; Get set to release quickfret holdoff
275 rlwinm r11,r8,0,0,19 ; Switch to savearea base
276 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
277 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
279 ; Note that the low-level code requires the R7 contain the high order half of the savearea's
280 ; physical address. This is hack city, but it is the way it is.
282 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
283 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
284 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
285 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
286 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
288 lwz r9,SAVflags(r8) /* Get the flags */
289 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
290 li r10,(MASK(MSR_ME)|MASK(MSR_DR)) /* Get the switcher's MSR */
291 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
292 stw r10,savesrr1+4(r8) /* Set up for switch in */
293 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
294 xor r3,r11,r8 /* Get the physical address of the new context save area */
295 stw r9,SAVflags(r8) /* Set the flags */
297 bne cr1,swtchtocont ; Switch to the continuation
298 sc /* Switch to the new context */
300 /* We come back here in the new thread context
301 * R4 was set to hold the old thread pointer, but switch_in will put it into
302 * R3 where it belongs.
304 blr /* Jump into the new thread */
307 ; This is where we go when a continuation is set. We are actually
308 ; killing off the old context of the new guy so we need to pop off
309 ; any float or vector states for the ditched level.
311 ; Note that we do the same kind of thing a chkfac in hw_exceptions.s
317 stw r5,savesrr0+4(r8) ; Set the pc
318 stw r6,savesrr1+4(r8) ; Set the next MSR to use
319 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
320 mr r3,r8 ; Pass in the virtual address of savearea
322 b EXT(exception_exit) ; Blocking on continuation, toss old context...
327 * All switched to threads come here first to clean up the old thread.
328 * We need to do the following contortions because we need to keep
329 * the LR clean. And because we need to manipulate the savearea chain
330 * with translation on. If we could, this should be done in lowmem_vectors
331 * before translation is turned on. But we can't, dang it!
333 * switch_in() runs with DR on and IR off
335 * R3 = switcher's savearea (32-bit virtual)
336 * saver4 = old thread in switcher's save
337 * saver5 = new SRR0 in switcher's save
338 * saver6 = new SRR1 in switcher's save
345 .globl EXT(switch_in)
349 lwz r4,saver4+4(r3) ; Get the old thread
350 lwz r5,saver5+4(r3) ; Get the srr0 value
352 mfsprg r0,2 ; Get feature flags
353 mr r9,r4 ; Get the switched from ACT
354 lwz r6,saver6+4(r3) ; Get the srr1 value
355 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
356 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
358 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
359 stw r10,SAVprev+4(r3) ; Chain on the old one
361 mr r3,r4 ; Pass back the old thread
363 mtsrr0 r5 ; Set return point
364 mtsrr1 r6 ; Set return MSR
366 bne++ siSixtyFour ; Go do 64-bit...
374 * void fpu_save(facility_context ctx)
376 * Note that there are some oddities here when we save a context we are using.
377 * It is really not too cool to do this, but what the hey... Anyway,
378 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
379 * savearea containing the context just saved will go away. So, bottom line is
380 * that don't use fpus until after you are done with the saved context.
387 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
388 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
389 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
391 mfmsr r0 ; Get the MSR
392 andc r0,r0,r2 ; Clear FP, VEC
393 andc r2,r0,r12 ; Clear EE
394 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
395 mtmsr r2 ; Set the MSR
398 mfsprg r6,1 ; Get the current activation
399 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
400 lwz r12,FPUowner(r6) ; Get the context ID for owner
403 mr r7,r0 ; (TEST/DEBUG)
404 li r4,0 ; (TEST/DEBUG)
405 mr r10,r3 ; (TEST/DEBUG)
406 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
407 mr. r3,r12 ; (TEST/DEBUG)
408 li r2,0x6F00 ; (TEST/DEBUG)
409 li r5,0 ; (TEST/DEBUG)
410 beq-- noowneryet ; (TEST/DEBUG)
411 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
412 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
414 noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
416 mr r0,r7 ; (TEST/DEBUG)
417 mr r3,r10 ; (TEST/DEBUG)
419 mflr r2 ; Save the return address
421 cmplw r3,r12 ; Is the specified context live?
422 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
423 lwz r9,FPUcpu(r3) ; Get the cpu that context was last on
424 bne-- fsret ; Nobody owns the FPU, no save required...
426 cmplw r9,r11 ; Was the context for this processor?
427 la r5,FPUsync(r3) ; Point to the sync word
428 bne-- fsret ; Facility not last used on this processor...
431 ; It looks like we need to save this one.
433 ; First, make sure that the live context block is not mucked with while
434 ; we are trying to save it on out. Then we will give it the final check.
437 lis r9,ha16(EXT(LockTimeOut)) ; Get the high part
438 mftb r8 ; Get the time now
439 lwz r9,lo16(EXT(LockTimeOut))(r9) ; Get the timeout value
440 b fssync0a ; Jump to the lock...
444 fssync0: li r7,lgKillResv ; Get killing field
445 stwcx. r7,0,r7 ; Kill reservation
447 fssync0a: lwz r7,0(r5) ; Sniff the lock
448 mftb r10 ; Is it time yet?
449 cmplwi cr1,r7,0 ; Is it locked?
450 sub r10,r10,r8 ; How long have we been spinning?
451 cmplw r10,r9 ; Has it been too long?
452 bgt-- fstimeout ; Way too long, panic...
453 bne-- cr1,fssync0a ; Yea, still locked so sniff harder...
455 fssync1: lwarx r7,0,r5 ; Get the sync word
456 li r12,1 ; Get the lock
457 mr. r7,r7 ; Is it unlocked?
459 stwcx. r12,0,r5 ; Store lock and test reservation
460 bne-- fssync1 ; Try again if lost reservation...
462 isync ; Toss speculation
464 lwz r12,FPUowner(r6) ; Get the context ID for owner
465 cmplw r3,r12 ; Check again if we own the FPU?
466 bne-- fsretlk ; Go unlock and return since we no longer own context
468 lwz r5,FPUcpu(r12) ; Get the cpu that context was last on
469 lwz r7,FPUsave(r12) ; Get the current FPU savearea for the thread
470 cmplw r5,r11 ; Is this for the same processor?
471 lwz r9,FPUlevel(r12) ; Get our current level indicator
472 bne-- fsretlk ; Not the same processor, skip any save...
474 cmplwi r7,0 ; Have we ever saved this facility context?
475 beq-- fsneedone ; Never saved it, so go do it...
477 lwz r8,SAVlevel(r7) ; Get the level of this savearea
478 cmplw r9,r8 ; Correct level?
479 beq-- fsretlk ; The current level is already saved, bail out...
481 fsneedone: bl EXT(save_get) ; Get a savearea for the context
483 mfsprg r6,1 ; Get the current activation
484 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
485 li r4,SAVfloat ; Get floating point tag
486 lwz r12,FPUowner(r6) ; Get back our thread
487 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
488 lwz r4,facAct(r12) ; Get the activation associated with live context
489 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
490 stw r4,SAVact(r3) ; Indicate the right activation for this context
491 lwz r9,FPUlevel(r12) ; Get our current level indicator again
492 stw r3,FPUsave(r12) ; Set this as the most current floating point context
493 stw r8,SAVprev+4(r3) ; And then chain this in front
495 stw r9,SAVlevel(r3) ; Show level in savearea
497 bl fp_store ; save all 32 FPRs in the save area at r3
498 mtlr r2 ; Restore return
500 fsretlk: li r7,0 ; Get the unlock value
501 eieio ; Make sure that these updates make it out
502 stw r7,FPUsync(r12) ; Unlock it
504 fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
509 fstimeout: mr r4,r5 ; Set the lock address
510 mr r5,r7 ; Set the lock word data
511 lis r3,hi16(fstimeout_str) ; Get the failed lck message
512 ori r3,r3,lo16(fstimeout_str) ; Get the failed lck message
514 BREAKPOINT_TRAP ; We die here anyway
518 STRINGD "fpu_save: timeout on sync lock (0x%08X), value = 0x%08X\n\000"
525 * Entered to handle the floating-point unavailable exception and
528 * This code is run in virtual address mode on with interrupts off.
530 * Upon exit, the code returns to the users context with the floating
531 * point facility turned on.
533 * ENTRY: VM switched ON
535 * State is saved in savearea pointed to by R4.
536 * All other registers are free.
541 .globl EXT(fpu_switch)
546 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
547 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
553 mfsprg r17,1 ; Get the current activation
554 lwz r26,ACT_PER_PROC(r17) ; Get the per_proc block
555 mfmsr r19 ; Get the current MSR
557 mr r25,r4 ; Save the entry savearea
558 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
559 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
561 mtmsr r19 ; Enable floating point instructions
564 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
565 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
567 ; R22 has the "old" context anchor
568 ; R29 has the "new" context anchor
571 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
572 li r2,0x7F01 ; (TEST/DEBUG)
573 mr r3,r22 ; (TEST/DEBUG)
574 mr r5,r29 ; (TEST/DEBUG)
575 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
579 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
581 mr. r22,r22 ; See if there is any live FP status
582 la r15,FPUsync(r22) ; Point to the sync word
584 beq-- fsnosave ; No live context, so nothing to save...
586 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
587 cmplw cr2,r22,r29 ; Are both old and new the same context?
588 lwz r30,FPUsave(r22) ; Get the top savearea
589 cmplw r18,r16 ; Make sure we are on the right processor
590 lwz r31,FPUlevel(r22) ; Get the context level
591 cmplwi cr1,r30,0 ; Anything saved yet?
593 bne-- fsnosave ; No, not on the same processor...
596 ; Check to see if the live context has already been saved.
597 ; Also check to see if all we are here just to re-enable the MSR
598 ; and handle specially if so.
601 cmplw r31,r27 ; See if the current and active levels are the same
602 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
604 beq-- fsthesame ; New and old are the same, just go enable...
608 ; Note it turns out that on a G5, the following load has about a 50-50 chance of
609 ; taking a segment exception in a system that is doing heavy file I/O. We
610 ; make a dummy access right now in order to get that resolved before we take the lock.
611 ; We do not use the data returned because it may change over the lock
614 beq-- cr1,fswsync ; Nothing saved, skip the probe attempt...
615 lwz r11,SAVlevel(r30) ; Touch the context in order to fault in the segment
618 ; Make sure that the live context block is not mucked with while
619 ; we are trying to save it on out
622 fswsync: lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
623 mftb r3 ; Get the time now
624 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
625 b fswsync0a ; Jump to the lock...
629 fswsync0: li r19,lgKillResv ; Get killing field
630 stwcx. r19,0,r19 ; Kill reservation
632 fswsync0a: lwz r19,0(r15) ; Sniff the lock
633 mftb r18 ; Is it time yet?
634 cmplwi cr1,r19,0 ; Is it locked?
635 sub r18,r18,r3 ; How long have we been spinning?
636 cmplw r18,r11 ; Has it been too long?
637 bgt-- fswtimeout ; Way too long, panic...
638 bne-- cr1,fswsync0a ; Yea, still locked so sniff harder...
640 fswsync1: lwarx r19,0,r15 ; Get the sync word
641 li r0,1 ; Get the lock
642 mr. r19,r19 ; Is it unlocked?
644 stwcx. r0,0,r15 ; Store lock and test reservation
645 bne-- fswsync1 ; Try again if lost reservation...
647 isync ; Toss speculation
650 ; Note that now that we have the lock, we need to check if anything changed.
651 ; Also note that the possible changes are limited. The context owner can
652 ; never change to a different thread or level although it can be invalidated.
653 ; A new context can not be pushed on top of us, but it can be popped. The
654 ; cpu indicator will always change if another processor mucked with any
657 ; It should be very rare that any of the context stuff changes across the lock.
660 lwz r0,FPUowner(r26) ; Get the thread that owns the FPU again
661 lwz r11,FPUsave(r22) ; Get the top savearea again
662 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on again
663 sub r0,r0,r22 ; Non-zero if we lost ownership, 0 if not
664 xor r11,r11,r30 ; Non-zero if saved context changed, 0 if not
665 xor r18,r18,r16 ; Non-zero if cpu changed, 0 if not
666 cmplwi cr1,r30,0 ; Is anything saved?
667 or r0,r0,r11 ; Zero only if both owner and context are unchanged
668 or. r0,r0,r18 ; Zero only if nothing has changed
671 bne-- fsnosavelk ; Something has changed, so this is not ours to save...
672 beq-- cr1,fsmstsave ; There is no context saved yet...
674 lwz r11,SAVlevel(r30) ; Get the level of top saved context
676 cmplw r31,r11 ; Are live and saved the same?
679 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
680 li r2,0x7F02 ; (TEST/DEBUG)
681 mr r3,r11 ; (TEST/DEBUG)
682 mr r5,r31 ; (TEST/DEBUG)
683 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
685 li r3,0 ; (TEST/DEBUG)
688 beq++ fsnosavelk ; Same level, so already saved...
690 fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
691 eieio ; Make sure everyone sees it
692 bl EXT(save_get) ; Go get a savearea
694 lwz r12,facAct(r22) ; Get the activation associated with the context
695 stw r30,SAVprev+4(r3) ; Point us to the old context
696 stw r31,SAVlevel(r3) ; Tag our level
697 li r7,SAVfloat ; Get the floating point ID
698 stw r12,SAVact(r3) ; Make sure we point to the right guy
699 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
700 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
703 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
704 li r2,0x7F03 ; (TEST/DEBUG)
705 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
709 bl fp_store ; store all 32 FPRs
711 fsnosavelk: li r7,0 ; Get the unlock value
712 eieio ; Make sure that these updates make it out
713 stw r7,FPUsync(r22) ; Unlock it.
716 ; The context is all saved now and the facility is free.
718 ; Check if we need to fill the registers with junk, because this level has
719 ; never used them before and some thieving bastard could hack the old values
720 ; of some thread! Just imagine what would happen if they could! Why, nothing
721 ; would be safe! My God! It is terrifying!
723 ; Make sure that the live context block is not mucked with while
724 ; we are trying to load it up
727 fsnosave: la r15,FPUsync(r29) ; Point to the sync word
728 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
729 mftb r3 ; Get the time now
730 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
731 b fsnsync0a ; Jump to the lock...
735 fsnsync0: li r19,lgKillResv ; Get killing field
736 stwcx. r19,0,r19 ; Kill reservation
738 fsnsync0a: lwz r19,0(r15) ; Sniff the lock
739 mftb r18 ; Is it time yet?
740 cmplwi cr1,r19,0 ; Is it locked?
741 sub r18,r18,r3 ; How long have we been spinning?
742 cmplw r18,r11 ; Has it been too long?
743 bgt-- fsntimeout ; Way too long, panic...
744 bne-- cr1,fsnsync0a ; Yea, still locked so sniff harder...
746 fsnsync1: lwarx r19,0,r15 ; Get the sync word
747 li r0,1 ; Get the lock
748 mr. r19,r19 ; Is it unlocked?
749 bne-- fsnsync0 ; Unfortunately, it is locked...
750 stwcx. r0,0,r15 ; Store lock and test reservation
751 bne-- fsnsync1 ; Try again if lost reservation...
753 isync ; Toss speculation
755 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
756 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
757 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
759 stw r16,FPUcpu(r29) ; Claim context for us
763 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
764 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
765 li r2,0x7F04 ; (TEST/DEBUG)
766 mr r1,r15 ; (TEST/DEBUG)
767 mr r3,r14 ; (TEST/DEBUG)
768 mr r5,r13 ; (TEST/DEBUG)
769 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
773 lis r18,hi16(EXT(PerProcTable)) ; Set base PerProcTable
774 mulli r19,r19,ppeSize ; Find offset to the owner per_proc_entry
775 ori r18,r18,lo16(EXT(PerProcTable)) ; Set base PerProcTable
776 li r16,FPUowner ; Displacement to float owner
777 add r19,r18,r19 ; Point to the owner per_proc_entry
778 lwz r19,ppe_vaddr(r19) ; Point to the owner per_proc
780 fsinvothr: lwarx r18,r16,r19 ; Get the owner
781 sub r0,r18,r29 ; Subtract one from the other
782 sub r11,r29,r18 ; Subtract the other from the one
783 or r11,r11,r0 ; Combine them
784 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
785 and r18,r18,r11 ; Make 0 if same, unchanged if not
786 stwcx. r18,r16,r19 ; Try to invalidate it
787 bne-- fsinvothr ; Try again if there was a collision...
789 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
790 la r11,savefp0(r14) ; Point to first line to bring in
791 stw r15,FPUlevel(r29) ; Set the "new" active level
793 stw r29,FPUowner(r26) ; Mark us as having the live context
795 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
797 dcbt 0,r11 ; Touch line in
799 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
800 lwz r3,SAVprev+4(r14) ; Get the previous context
801 cmplw r0,r15 ; Top level correct to load?
802 li r7,0 ; Get the unlock value
803 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
805 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
808 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
809 li r2,0x7F05 ; (TEST/DEBUG)
810 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
814 eieio ; Make sure that these updates make it out
815 stw r7,FPUsync(r29) ; Unlock context now that the context save has been removed
817 // Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
818 // are executed on a 128-byte machine, but that is better than a mispredicted branch.
820 la r11,savefp4(r14) ; Point to next line
821 dcbt 0,r11 ; Touch line in
825 la r11,savefp8(r14) ; Point to next line
827 dcbt 0,r11 ; Touch line in
831 la r11,savefp12(r14) ; Point to next line
833 dcbt 0,r11 ; Touch line in
836 lfd f10,savefp10(r14)
837 la r11,savefp16(r14) ; Point to next line
838 lfd f11,savefp11(r14)
839 dcbt 0,r11 ; Touch line in
840 lfd f12,savefp12(r14)
841 lfd f13,savefp13(r14)
842 lfd f14,savefp14(r14)
843 la r11,savefp20(r14) ; Point to next line
844 lfd f15,savefp15(r14)
845 dcbt 0,r11 ; Touch line in
846 lfd f16,savefp16(r14)
847 lfd f17,savefp17(r14)
848 lfd f18,savefp18(r14)
849 la r11,savefp24(r14) ; Point to next line
850 lfd f19,savefp19(r14)
851 dcbt 0,r11 ; Touch line in
852 lfd f20,savefp20(r14)
853 lfd f21,savefp21(r14)
854 la r11,savefp28(r14) ; Point to next line
855 lfd f22,savefp22(r14)
856 lfd f23,savefp23(r14)
857 dcbt 0,r11 ; Touch line in
858 lfd f24,savefp24(r14)
859 lfd f25,savefp25(r14)
860 lfd f26,savefp26(r14)
861 lfd f27,savefp27(r14)
862 lfd f28,savefp28(r14)
863 lfd f29,savefp29(r14)
864 lfd f30,savefp30(r14)
865 lfd f31,savefp31(r14)
867 mr r3,r14 ; Get the old savearea (we popped it before)
868 bl EXT(save_ret) ; Toss it
870 fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
871 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
872 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
873 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
874 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
875 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
876 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
877 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
878 mr r3,r25 ; Pass the virtual addres of savearea
879 beq- fsnuser ; We are not user state...
880 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
881 stw r11,spcFlags(r26) ; Set per_proc copy
885 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
886 li r2,0x7F07 ; (TEST/DEBUG)
887 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
891 b EXT(exception_exit) ; Exit to the fray...
894 * Initialize the registers to some bogus value
897 MakeSureThatNoTerroristsCanHurtUsByGod:
900 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
901 li r2,0x7F06 ; (TEST/DEBUG)
902 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
905 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
906 li r7,0 ; Get the unlock value
907 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
908 eieio ; Make sure that these updates make it out
909 stw r7,FPUsync(r29) ; Unlock it now that the context has been removed
911 lfd f0,0(r5) ; Initialize FP0
912 fmr f1,f0 ; Do them all
943 b fsenable ; Finish setting it all up...
947 ; We get here when we are switching to the same context at the same level and the context
948 ; is still live. Essentially, all we are doing is turning on the facility. It may have
949 ; gotten turned off due to doing a context save for the current level or a context switch
950 ; back to the live guy.
956 fsthesamel: li r7,0 ; Get the unlock value
957 eieio ; Make sure that these updates make it out
958 stw r7,FPUsync(r22) ; Unlock it.
963 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
964 li r2,0x7F0A ; (TEST/DEBUG)
965 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
968 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
970 lwz r11,SAVlevel(r30) ; Get the level of top saved context
971 lwz r14,SAVprev+4(r30) ; Get the previous savearea
973 cmplw r11,r31 ; Are live and saved the same?
975 bne++ fsenable ; Level not the same, nothing to pop, go enable and exit...
977 mr r3,r30 ; Get the old savearea (we popped it before)
978 stw r14,FPUsave(r22) ; Pop the savearea from the stack
979 bl EXT(save_ret) ; Toss it
980 b fsenable ; Go enable and exit...
983 ; Note that we need to choke in this code rather than panic because there is no
987 fswtimeout: lis r0,hi16(Choke) ; Choke code
988 ori r0,r0,lo16(Choke) ; and the rest
989 li r3,failTimeout ; Timeout code
992 fsntimeout: lis r0,hi16(Choke) ; Choke code
993 ori r0,r0,lo16(Choke) ; and the rest
994 li r3,failTimeout ; Timeout code
998 lis r0,hi16(Choke) ; Choke code
999 ori r0,r0,lo16(Choke) ; and the rest
1000 li r3,failTimeout ; Timeout code
1004 lis r0,hi16(Choke) ; Choke code
1005 ori r0,r0,lo16(Choke) ; and the rest
1006 li r3,failTimeout ; Timeout code
1010 ; This function invalidates any live floating point context for the passed in facility_context.
1011 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1015 .globl EXT(toss_live_fpu)
1019 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1020 mfmsr r9 ; Get the MSR
1021 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1022 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
1023 andc r9,r9,r0 ; Force off VEC and FP
1024 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1025 andc r0,r9,r0 ; Turn off EE now
1026 mtmsr r0 ; No interruptions
1028 beq+ tlfnotours ; Floats off, can not be live here...
1030 mfsprg r8,1 ; Get the current activation
1031 lwz r8,ACT_PER_PROC(r8) ; Get the per_proc block
1034 ; Note that at this point, since floats are on, we are the owner
1035 ; of live state on this processor
1038 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
1039 li r0,0 ; Clear this just in case we need it
1040 cmplw r6,r3 ; Are we tossing our own context?
1041 bne-- tlfnotours ; Nope...
1043 lfd f1,Zero(0) ; Make a 0
1044 mtfsf 0xFF,f1 ; Clear it
1046 tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
1047 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1048 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1049 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1050 li r10,FPUowner ; Displacement to float owner
1051 add r11,r12,r11 ; Point to the owner per_proc_entry
1052 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1054 tlfinvothr: lwarx r12,r10,r11 ; Get the owner
1056 sub r0,r12,r3 ; Subtract one from the other
1057 sub r8,r3,r12 ; Subtract the other from the one
1058 or r8,r8,r0 ; Combine them
1059 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1060 and r12,r12,r8 ; Make 0 if same, unchanged if not
1061 stwcx. r12,r10,r11 ; Try to invalidate it
1062 bne-- tlfinvothr ; Try again if there was a collision...
1064 mtmsr r9 ; Restore interruptions
1065 isync ; Could be turning off floats here
1070 * Altivec stuff is here. The techniques used are pretty identical to
1071 * the floating point. Except that we will honor the VRSAVE register
1072 * settings when loading and restoring registers.
1074 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
1075 * mask. VRSAVE is set by the vector user and represents the VRs that they
1076 * say that they are using. The vrvalid mask indicates which vector registers
1077 * are saved in the savearea. Whenever context is saved, it is saved according
1078 * to the VRSAVE register. It is loaded based on VRSAVE anded with
1079 * vrvalid (all other registers are splatted with 0s). This is done because we
1080 * don't want to load any registers we don't have a copy of, we want to set them
1083 * Note that there are some oddities here when we save a context we are using.
1084 * It is really not too cool to do this, but what the hey... Anyway,
1085 * we turn vectors and fpu off before we leave.
1086 * The oddity is that if you use vectors after this, the
1087 * savearea containing the context just saved will go away. So, bottom line is
1088 * that don't use vectors until after you are done with the saved context.
1093 .globl EXT(vec_save)
1098 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
1099 mfmsr r0 ; Get the MSR
1100 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
1101 andc r0,r0,r2 ; Force off VEC and FP
1102 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
1103 andc r2,r0,r2 ; Clear EE for now
1104 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
1105 mtmsr r2 ; Set the MSR
1108 mfsprg r6,1 ; Get the current activation
1109 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1110 lwz r12,VMXowner(r6) ; Get the context ID for owner
1113 mr r11,r6 ; (TEST/DEBUG)
1114 mr r7,r0 ; (TEST/DEBUG)
1115 li r4,0 ; (TEST/DEBUG)
1116 mr r10,r3 ; (TEST/DEBUG)
1117 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1118 mr. r3,r12 ; (TEST/DEBUG)
1119 li r2,0x5F00 ; (TEST/DEBUG)
1120 li r5,0 ; (TEST/DEBUG)
1121 lwz r6,liveVRS(r6) ; (TEST/DEBUG)
1122 beq-- noowneryeu ; (TEST/DEBUG)
1123 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
1124 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
1126 noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1128 mr r0,r7 ; (TEST/DEBUG)
1129 mr r3,r10 ; (TEST/DEBUG)
1130 mr r6,r11 ; (TEST/DEBUG)
1132 mflr r2 ; Save the return address
1134 cmplw r3,r12 ; Is the specified context live?
1135 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
1136 bne-- vsret ; We do not own the vector, no save required...
1137 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
1139 cmplw r9,r11 ; Was the context for this processor?
1140 la r5,VMXsync(r3) ; Point to the sync word
1141 bne-- vsret ; Specified context is not live
1144 ; It looks like we need to save this one. Or possibly toss a saved one if
1147 ; First, make sure that the live context block is not mucked with while
1148 ; we are trying to save it on out. Then we will give it the final check.
1151 lis r9,ha16(EXT(LockTimeOut)) ; Get the high part
1152 mftb r8 ; Get the time now
1153 lwz r9,lo16(EXT(LockTimeOut))(r9) ; Get the timeout value
1154 b vssync0a ; Jump to the lock...
1158 vssync0: li r7,lgKillResv ; Get killing field
1159 stwcx. r7,0,r7 ; Kill reservation
1161 vssync0a: lwz r7,0(r5) ; Sniff the lock
1162 mftb r10 ; Is it time yet?
1163 cmplwi cr1,r7,0 ; Is it locked?
1164 sub r10,r10,r8 ; How long have we been spinning?
1165 cmplw r10,r9 ; Has it been too long?
1166 bgt-- vswtimeout0 ; Way too long, panic...
1167 bne-- cr1,vssync0a ; Yea, still locked so sniff harder...
1169 vssync1: lwarx r7,0,r5 ; Get the sync word
1170 li r12,1 ; Get the lock
1171 mr. r7,r7 ; Is it unlocked?
1172 bne-- vssync0 ; No, it is unlocked...
1173 stwcx. r12,0,r5 ; Store lock and test reservation
1174 bne-- vssync1 ; Try again if lost reservation...
1176 isync ; Toss speculation
1178 lwz r12,VMXowner(r6) ; Get the context ID for owner
1179 cmplw r3,r12 ; Check again if we own VMX?
1180 lwz r10,liveVRS(r6) ; Get the right VRSave register
1181 bne-- vsretlk ; Go unlock and return since we no longer own context
1183 lwz r5,VMXcpu(r12) ; Get the cpu that context was last on
1184 lwz r7,VMXsave(r12) ; Get the current vector savearea for the thread
1185 cmplwi cr1,r10,0 ; Is VRsave set to 0?
1186 cmplw r5,r11 ; Is this for the same processor?
1187 lwz r9,VMXlevel(r12) ; Get our current level indicator
1188 bne-- vsretlk ; Not the same processor, skip any save...
1190 cmplwi r7,0 ; Have we ever saved this facility context?
1191 beq-- vsneedone ; Never saved it, so we need an area...
1193 lwz r8,SAVlevel(r7) ; Get the level this savearea is for
1194 cmplw r9,r8 ; Correct level?
1195 bne-- vsneedone ; Different level, so we need to save...
1197 bne++ cr1,vsretlk ; VRsave is non-zero so we need to keep what is saved...
1199 lwz r4,SAVprev+4(r7) ; Pick up the previous area
1200 li r5,0 ; Assume we just dumped the last
1201 mr. r4,r4 ; Is there one?
1202 stw r4,VMXsave(r12) ; Dequeue this savearea
1203 beq-- vsnomore ; We do not have another...
1205 lwz r5,SAVlevel(r4) ; Get the level associated with save
1207 vsnomore: stw r5,VMXlevel(r12) ; Save the level
1209 stw r7,VMXowner(r6) ; Show no live context here
1211 vsbackout: mr r4,r0 ; restore the saved MSR
1213 stw r7,VMXsync(r12) ; Unlock the context
1215 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
1219 vsneedone: beq-- cr1,vsclrlive ; VRSave is zero, go blow away the context...
1221 bl EXT(save_get) ; Get a savearea for the context
1223 mfsprg r6,1 ; Get the current activation
1224 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1225 li r4,SAVvector ; Get vector tag
1226 lwz r12,VMXowner(r6) ; Get back our context ID
1227 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1228 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1230 beq-- vsbackout ; If disowned, just toss savearea...
1231 lwz r4,facAct(r12) ; Get the activation associated with live context
1232 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1233 stw r4,SAVact(r3) ; Indicate the right activation for this context
1234 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1235 stw r3,VMXsave(r12) ; Set this as the most current floating point context
1236 stw r8,SAVprev+4(r3) ; And then chain this in front
1238 stw r9,SAVlevel(r3) ; Set level in savearea
1239 mfcr r12 ; save CRs across call to vr_store
1240 lwz r10,liveVRS(r6) ; Get the right VRSave register
1242 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
1244 mfsprg r6,1 ; Get the current activation
1245 mtcrf 255,r12 ; Restore the non-volatile CRs
1246 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1247 mtlr r2 ; Restore return address
1248 lwz r12,VMXowner(r6) ; Get back our context ID
1250 vsretlk: li r7,0 ; Get the unlock value
1251 eieio ; Make sure that these updates make it out
1252 stw r7,VMXsync(r12) ; Unlock it
1254 vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1259 vsclrlive: li r7,0 ; Clear
1260 stw r7,VMXowner(r6) ; Show no live context here
1261 b vsretlk ; Go unlock and leave...
1266 * Entered to handle the vector unavailable exception and
1267 * switch vector context
1269 * This code is run with virtual address mode on and interrupts off.
1271 * Upon exit, the code returns to the users context with the vector
1272 * facility turned on.
1274 * ENTRY: VM switched ON
1276 * State is saved in savearea pointed to by R4.
1277 * All other registers are free.
1282 .globl EXT(vec_switch)
1287 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1288 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1294 mfsprg r17,1 ; Get the current activation
1295 lwz r26,ACT_PER_PROC(r17) ; Get the per_proc block
1296 mfmsr r19 ; Get the current MSR
1298 mr r25,r4 ; Save the entry savearea
1299 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1300 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1302 mtmsr r19 ; Enable vector instructions
1305 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1306 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
1308 ; R22 has the "old" context anchor
1309 ; R29 has the "new" context anchor
1312 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1313 li r2,0x5F01 ; (TEST/DEBUG)
1314 mr r3,r22 ; (TEST/DEBUG)
1315 mr r5,r29 ; (TEST/DEBUG)
1316 lwz r6,liveVRS(r26) ; (TEST/DEBUG)
1317 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1321 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1323 mr. r22,r22 ; See if there is any live vector status
1324 la r15,VMXsync(r22) ; Point to the sync word
1326 beq-- vswnosave ; No live context, so nothing to save...
1328 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1329 cmplw cr2,r22,r29 ; Are both old and new the same context?
1330 lwz r30,VMXsave(r22) ; Get the top savearea
1331 cmplwi cr1,r30,0 ; Anything saved yet?
1332 lwz r31,VMXlevel(r22) ; Get the context level
1333 cmplw r18,r16 ; Make sure we are on the right processor
1335 lwz r10,liveVRS(r26) ; Get the right VRSave register
1337 bne-- vswnosave ; No, not on the same processor...
1340 ; Check to see if the live context has already been saved.
1341 ; Also check to see if all we are here just to re-enable the MSR
1342 ; and handle specially if so.
1345 cmplw r31,r27 ; See if the current and active levels are the same
1346 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1348 beq-- vswthesame ; New and old are the same, just go enable...
1351 ; Make sure that the live context block is not mucked with while
1352 ; we are trying to save it on out
1355 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
1356 mftb r3 ; Get the time now
1357 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
1358 b vswsync0a ; Jump to the lock...
1362 vswsync0: li r19,lgKillResv ; Get killing field
1363 stwcx. r19,0,r19 ; Kill reservation
1365 vswsync0a: lwz r19,0(r15) ; Sniff the lock
1366 mftb r18 ; Is it time yet?
1367 cmplwi cr1,r19,0 ; Is it locked?
1368 sub r18,r18,r3 ; How long have we been spinning?
1369 cmplw r18,r11 ; Has it been too long?
1370 bgt-- vswtimeout0 ; Way too long, panic...
1371 bne-- cr1,vswsync0a ; Yea, still locked so sniff harder...
1373 vswsync1: lwarx r19,0,r15 ; Get the sync word
1374 li r0,1 ; Get the lock
1375 mr. r19,r19 ; Is it unlocked?
1377 stwcx. r0,0,r15 ; Store lock and test reservation
1378 bne-- vswsync1 ; Try again if lost reservation...
1380 isync ; Toss speculation
1383 ; Note that now that we have the lock, we need to check if anything changed.
1384 ; Also note that the possible changes are limited. The context owner can
1385 ; never change to a different thread or level although it can be invalidated.
1386 ; A new context can not be pushed on top of us, but it can be popped. The
1387 ; cpu indicator will always change if another processor mucked with any
1390 ; It should be very rare that any of the context stuff changes across the lock.
1393 lwz r0,VMXowner(r26) ; Get the thread that owns the vectors again
1394 lwz r11,VMXsave(r22) ; Get the top savearea again
1395 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on again
1396 sub r0,r0,r22 ; Non-zero if we lost ownership, 0 if not
1397 xor r11,r11,r30 ; Non-zero if saved context changed, 0 if not
1398 xor r18,r18,r16 ; Non-zero if cpu changed, 0 if not
1399 cmplwi cr1,r30,0 ; Is anything saved?
1400 or r0,r0,r11 ; Zero only if both owner and context are unchanged
1401 or. r0,r0,r18 ; Zero only if nothing has changed
1402 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1405 bne-- vswnosavelk ; Something has changed, so this is not ours to save...
1406 beq-- cr1,vswmstsave ; There is no context saved yet...
1408 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1410 cmplw r31,r11 ; Are live and saved the same?
1413 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1414 li r2,0x5F02 ; (TEST/DEBUG)
1415 mr r3,r30 ; (TEST/DEBUG)
1416 mr r5,r31 ; (TEST/DEBUG)
1417 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1421 beq++ vswnosavelk ; Same level, already saved...
1422 bne-- cr2,vswnosavelk ; Live context saved and VRSave not 0, no save and keep context...
1424 lwz r4,SAVprev+4(r30) ; Pick up the previous area
1425 li r5,0 ; Assume this is the only one (which should be the ususal case)
1426 mr. r4,r4 ; Was this the only one?
1427 stw r4,VMXsave(r22) ; Dequeue this savearea
1428 beq++ vswonlyone ; This was the only one...
1429 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1431 vswonlyone: stw r5,VMXlevel(r22) ; Save the level
1432 stw r8,VMXowner(r26) ; Clear owner
1434 mr r3,r30 ; Copy the savearea we are tossing
1435 bl EXT(save_ret) ; Toss the savearea
1436 b vswnosavelk ; Go load up the context...
1440 vswmstsave: stw r8,VMXowner(r26) ; Clear owner
1441 beq-- cr2,vswnosavelk ; The VRSave was 0, so there is nothing to save...
1443 bl EXT(save_get) ; Go get a savearea
1445 lwz r12,facAct(r22) ; Get the activation associated with the context
1446 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1447 stw r30,SAVprev+4(r3) ; Point us to the old context
1448 stw r31,SAVlevel(r3) ; Tag our level
1449 li r7,SAVvector ; Get the vector ID
1450 stw r12,SAVact(r3) ; Make sure we point to the right guy
1451 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
1454 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1455 li r2,0x5F03 ; (TEST/DEBUG)
1456 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1460 lwz r10,liveVRS(r26) ; Get the right VRSave register
1461 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1464 ; The context is all saved now and the facility is free.
1466 ; Check if we need to fill the registers with junk, because this level has
1467 ; never used them before and some thieving bastard could hack the old values
1468 ; of some thread! Just imagine what would happen if they could! Why, nothing
1469 ; would be safe! My God! It is terrifying!
1471 ; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1472 ; constant that we may need to fill unused vector registers.
1474 ; Make sure that the live context block is not mucked with while
1475 ; we are trying to load it up
1479 li r7,0 ; Get the unlock value
1480 eieio ; Make sure that these updates make it out
1481 stw r7,VMXsync(r22) ; Unlock the old context
1483 vswnosave: la r15,VMXsync(r29) ; Point to the sync word
1484 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
1485 mftb r3 ; Get the time now
1486 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
1487 b vswnsync0a ; Jump to the lock...
1491 vswnsync0: li r19,lgKillResv ; Get killing field
1492 stwcx. r19,0,r19 ; Kill reservation
1494 vswnsync0a: lwz r19,0(r15) ; Sniff the lock
1495 mftb r18 ; Is it time yet?
1496 cmplwi cr1,r19,0 ; Is it locked?
1497 sub r18,r18,r3 ; How long have we been spinning?
1498 cmplw r18,r11 ; Has it been too long?
1499 bgt-- vswtimeout1 ; Way too long, panic...
1500 bne-- cr1,vswnsync0a ; Yea, still locked so sniff harder...
1502 vswnsync1: lwarx r19,0,r15 ; Get the sync word
1503 li r0,1 ; Get the lock
1504 mr. r19,r19 ; Is it unlocked?
1505 bne-- vswnsync0 ; Unfortunately, it is locked...
1506 stwcx. r0,0,r15 ; Store lock and test reservation
1507 bne-- vswnsync1 ; Try again if lost reservation...
1509 isync ; Toss speculation
1511 vspltisb v31,-10 ; Get 0xF6F6F6F6
1512 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1513 vspltisb v30,5 ; Get 0x05050505
1514 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1515 vspltish v29,4 ; Get 0x00040004
1516 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1517 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1519 stw r16,VMXcpu(r29) ; Claim context for us
1523 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1524 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1525 li r2,0x5F04 ; (TEST/DEBUG)
1526 mr r1,r15 ; (TEST/DEBUG)
1527 mr r3,r14 ; (TEST/DEBUG)
1528 mr r5,r13 ; (TEST/DEBUG)
1529 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1533 lis r18,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1534 vspltisb v28,-2 ; Get 0xFEFEFEFE
1535 mulli r19,r19,ppeSize ; Find offset to the owner per_proc_entry
1536 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1537 ori r18,r18,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1538 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1539 li r16,VMXowner ; Displacement to vector owner
1540 add r19,r18,r19 ; Point to the owner per_proc_entry
1541 lwz r19,ppe_vaddr(r19) ; Point to the owner per_proc
1542 vrlb v31,v31,v29 ; Get 0xDEADDEAD
1544 vswinvothr: lwarx r18,r16,r19 ; Get the owner
1546 sub r0,r18,r29 ; Subtract one from the other
1547 sub r11,r29,r18 ; Subtract the other from the one
1548 or r11,r11,r0 ; Combine them
1549 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1550 and r18,r18,r11 ; Make 0 if same, unchanged if not
1551 stwcx. r18,r16,r19 ; Try to invalidate it
1552 bne-- vswinvothr ; Try again if there was a collision...
1554 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
1555 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1556 stw r15,VMXlevel(r29) ; Set the "new" active level
1558 stw r29,VMXowner(r26) ; Mark us as having the live context
1560 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
1562 lwz r3,SAVprev+4(r14) ; Get the previous context
1563 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1564 cmplw r0,r15 ; Top level correct to load?
1565 bne-- ProtectTheAmericanWay ; No, go initialize...
1567 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1570 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1571 li r2,0x5F05 ; (TEST/DEBUG)
1572 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1576 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
1577 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
1578 and r10,r10,r22 ; Figure out just what registers need to be loaded
1579 mr r3,r14 ; r3 <- ptr to savearea with VRs
1580 bl vr_load ; load VRs from save area based on vrsave in r10
1582 bl EXT(save_ret) ; Toss the save area after loading VRs
1584 vrenablelk: li r7,0 ; Get the unlock value
1585 eieio ; Make sure that these updates make it out
1586 stw r7,VMXsync(r29) ; Unlock the new context
1588 vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
1589 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
1590 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1591 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1592 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1593 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1594 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
1595 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1596 mr r3,r25 ; Pass virtual address of the savearea
1597 beq- vrnuser ; We are not user state...
1598 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
1599 stw r11,spcFlags(r26) ; Set per_proc copy
1603 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1604 li r2,0x5F07 ; (TEST/DEBUG)
1605 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1608 b EXT(exception_exit) ; Exit to the fray...
1611 * Initialize the registers to some bogus value
1614 ProtectTheAmericanWay:
1617 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1618 li r2,0x5F06 ; (TEST/DEBUG)
1619 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1623 vor v0,v31,v31 ; Copy into the next register
1624 vor v1,v31,v31 ; Copy into the next register
1625 vor v2,v31,v31 ; Copy into the next register
1626 vor v3,v31,v31 ; Copy into the next register
1627 vor v4,v31,v31 ; Copy into the next register
1628 vor v5,v31,v31 ; Copy into the next register
1629 vor v6,v31,v31 ; Copy into the next register
1630 vor v7,v31,v31 ; Copy into the next register
1631 vor v8,v31,v31 ; Copy into the next register
1632 vor v9,v31,v31 ; Copy into the next register
1633 vor v10,v31,v31 ; Copy into the next register
1634 vor v11,v31,v31 ; Copy into the next register
1635 vor v12,v31,v31 ; Copy into the next register
1636 vor v13,v31,v31 ; Copy into the next register
1637 vor v14,v31,v31 ; Copy into the next register
1638 vor v15,v31,v31 ; Copy into the next register
1639 vor v16,v31,v31 ; Copy into the next register
1640 vor v17,v31,v31 ; Copy into the next register
1641 vor v18,v31,v31 ; Copy into the next register
1642 vor v19,v31,v31 ; Copy into the next register
1643 vor v20,v31,v31 ; Copy into the next register
1644 vor v21,v31,v31 ; Copy into the next register
1645 vor v22,v31,v31 ; Copy into the next register
1646 vor v23,v31,v31 ; Copy into the next register
1647 vor v24,v31,v31 ; Copy into the next register
1648 vor v25,v31,v31 ; Copy into the next register
1649 vor v26,v31,v31 ; Copy into the next register
1650 vor v27,v31,v31 ; Copy into the next register
1651 vor v28,v31,v31 ; Copy into the next register
1652 vor v29,v31,v31 ; Copy into the next register
1653 vor v30,v31,v31 ; Copy into the next register
1654 b vrenablelk ; Finish setting it all up...
1659 ; We get here when we are switching to the same context at the same level and the context
1660 ; is still live. Essentially, all we are doing is turning on the faility. It may have
1661 ; gotten turned off due to doing a context save for the current level or a context switch
1662 ; back to the live guy.
1670 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1671 li r2,0x5F0A ; (TEST/DEBUG)
1672 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1675 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1677 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1678 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1680 cmplw r11,r31 ; Are live and saved the same?
1682 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1684 mr r3,r30 ; Get the old savearea (we popped it before)
1685 stw r11,VMXsave(r22) ; Pop the vector stack
1686 bl EXT(save_ret) ; Toss it
1687 b vrenable ; Go enable and exit...
1691 ; This function invalidates any live vector context for the passed in facility_context.
1692 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1696 .globl EXT(toss_live_vec)
1700 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1701 mfmsr r9 ; Get the MSR
1702 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1703 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1704 andc r9,r9,r0 ; Force off VEC and FP
1705 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1706 andc r0,r9,r0 ; Turn off EE now
1707 mtmsr r0 ; No interruptions
1709 beq+ tlvnotours ; Vector off, can not be live here...
1711 mfsprg r8,1 ; Get the current activation
1712 lwz r8,ACT_PER_PROC(r8) ; Get the per_proc block
1715 ; Note that at this point, since vecs are on, we are the owner
1716 ; of live state on this processor
1719 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1720 li r0,0 ; Clear this just in case we need it
1721 cmplw r6,r3 ; Are we tossing our own context?
1722 bne- tlvnotours ; Nope...
1724 vspltish v1,1 ; Turn on the non-Java bit and saturate
1725 vspltisw v0,1 ; Turn on the saturate bit
1726 vxor v1,v1,v0 ; Turn off saturate
1727 mtspr vrsave,r0 ; Clear VRSAVE
1728 mtvscr v1 ; Set the non-java, no saturate status
1730 tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1731 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1732 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1733 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1734 li r10,VMXowner ; Displacement to vector owner
1735 add r11,r12,r11 ; Point to the owner per_proc_entry
1736 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1737 li r0,0 ; Set a 0 to invalidate context
1739 tlvinvothr: lwarx r12,r10,r11 ; Get the owner
1741 sub r0,r12,r3 ; Subtract one from the other
1742 sub r8,r3,r12 ; Subtract the other from the one
1743 or r8,r8,r0 ; Combine them
1744 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1745 and r12,r12,r8 ; Make 0 if same, unchanged if not
1746 stwcx. r12,r10,r11 ; Try to invalidate it
1747 bne-- tlvinvothr ; Try again if there was a collision...
1749 mtmsr r9 ; Restore interruptions
1750 isync ; Could be turning off vectors here
1755 ; This function invalidates any live vector context for the passed in facility_context
1756 ; if the level is current. It also tosses the corresponding savearea if there is one.
1757 ; This function is primarily used whenever we detect a VRSave that is all zeros.
1761 .globl EXT(vec_trash)
1765 lwz r12,facAct(r3) ; Get the activation
1766 lwz r11,VMXlevel(r3) ; Get the context level
1767 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1768 lwz r9,VMXsave(r3) ; Get the savearea, if any
1769 cmplw r10,r11 ; Are we at the right level?
1770 cmplwi cr1,r9,0 ; Remember if there is a savearea
1771 bnelr+ ; No, we do nothing...
1773 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1774 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1775 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1776 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1777 li r10,VMXowner ; Displacement to vector owner
1778 add r11,r12,r11 ; Point to the owner per_proc_entry
1779 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1781 vtinvothr: lwarx r12,r10,r11 ; Get the owner
1783 sub r0,r12,r3 ; Subtract one from the other
1784 sub r8,r3,r12 ; Subtract the other from the one
1785 or r8,r8,r0 ; Combine them
1786 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1787 and r12,r12,r8 ; Make 0 if same, unchanged if not
1788 stwcx. r12,r10,r11 ; Try to invalidate it
1789 bne-- vtinvothr ; Try again if there was a collision...
1792 beqlr++ cr1 ; Leave if there is no savearea
1793 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1794 cmplw r8,r11 ; Savearea for the current level?
1795 bnelr++ ; No, nothing to release...
1797 lwz r8,SAVprev+4(r9) ; Pick up the previous area
1798 mr. r8,r8 ; Is there a previous?
1799 beq-- vtnoprev ; Nope...
1800 lwz r7,SAVlevel(r8) ; Get the level associated with save
1802 vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1803 stw r7,VMXlevel(r3) ; Pop the level
1805 mr r3,r9 ; Get the savearea to release
1806 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1810 ; Just some test code to force vector and/or floating point in the kernel
1814 .globl EXT(fctx_test)
1818 mfsprg r3,1 ; Get the current thread
1819 mr. r3,r3 ; Are we actually up and running?
1822 fmr f0,f0 ; Use floating point
1823 mftb r4 ; Get time base for a random number
1824 li r5,1 ; Get a potential vrsave to use
1825 andi. r4,r4,0x3F ; Get a number from 0 - 63
1826 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1827 mtspr vrsave,r5 ; Set VRSave
1828 vor v0,v0,v0 ; Use vectors
1832 // *******************
1833 // * f p _ s t o r e *
1834 // *******************
1836 // Store FPRs into a save area. Called by fpu_save and fpu_switch.
1839 // floating pt is enabled
1840 // r3 = ptr to save area
1846 mfsprg r11,2 ; get feature flags
1847 mtcrf 0x02,r11 ; put cache line size bits in cr6
1848 la r11,savefp0(r3) ; point to 1st line
1849 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1850 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1852 // Store the FPRs on a 128-byte machine.
1856 la r11,savefp16(r3) ; Point to the 2nd cache line
1859 dcbz128 0,r11 ; establish 2nd line
1866 stfd f10,savefp10(r3)
1867 stfd f11,savefp11(r3)
1868 stfd f12,savefp12(r3)
1869 stfd f13,savefp13(r3)
1870 stfd f14,savefp14(r3)
1871 stfd f15,savefp15(r3)
1872 stfd f16,savefp16(r3)
1873 stfd f17,savefp17(r3)
1874 stfd f18,savefp18(r3)
1875 stfd f19,savefp19(r3)
1876 stfd f20,savefp20(r3)
1877 stfd f21,savefp21(r3)
1878 stfd f22,savefp22(r3)
1879 stfd f23,savefp23(r3)
1880 stfd f24,savefp24(r3)
1881 stfd f25,savefp25(r3)
1882 stfd f26,savefp26(r3)
1883 stfd f27,savefp27(r3)
1884 stfd f28,savefp28(r3)
1885 stfd f29,savefp29(r3)
1886 stfd f30,savefp30(r3)
1887 stfd f31,savefp31(r3)
1890 // Store FPRs on a 32-byte machine.
1893 la r11,savefp4(r3) ; Point to the 2nd line
1895 dcbz 0,r11 ; Allocate cache
1898 la r11,savefp8(r3) ; Point to the 3rd line
1900 dcbz 0,r11 ; Allocate cache
1904 la r11,savefp12(r3) ; Point to the 4th line
1906 dcbz 0,r11 ; Allocate cache
1909 stfd f10,savefp10(r3)
1910 la r11,savefp16(r3) ; Point to the 5th line
1911 stfd f11,savefp11(r3)
1912 dcbz 0,r11 ; Allocate cache
1913 stfd f12,savefp12(r3)
1914 stfd f13,savefp13(r3)
1915 stfd f14,savefp14(r3)
1916 la r11,savefp20(r3) ; Point to the 6th line
1917 stfd f15,savefp15(r3)
1918 dcbz 0,r11 ; Allocate cache
1919 stfd f16,savefp16(r3)
1920 stfd f17,savefp17(r3)
1921 stfd f18,savefp18(r3)
1922 la r11,savefp24(r3) ; Point to the 7th line
1923 stfd f19,savefp19(r3)
1924 dcbz 0,r11 ; Allocate cache
1925 stfd f20,savefp20(r3)
1927 stfd f21,savefp21(r3)
1928 stfd f22,savefp22(r3)
1929 la r11,savefp28(r3) ; Point to the 8th line
1930 stfd f23,savefp23(r3)
1931 dcbz 0,r11 ; allocate it
1932 stfd f24,savefp24(r3)
1933 stfd f25,savefp25(r3)
1934 stfd f26,savefp26(r3)
1935 stfd f27,savefp27(r3)
1937 stfd f28,savefp28(r3)
1938 stfd f29,savefp29(r3)
1939 stfd f30,savefp30(r3)
1940 stfd f31,savefp31(r3)
1944 // *******************
1945 // * v r _ s t o r e *
1946 // *******************
1948 // Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1949 // both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1950 // unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1951 // for each cache block size.
1954 // interrupts are off, vectors are enabled
1955 // r3 = ptr to save area
1956 // r10 = vrsave (not 0)
1959 // r4 - r11, all CRs.
1962 mfsprg r9,2 ; get feature flags
1963 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1964 slwi r8,r10,1 ; Shift over 1
1965 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1966 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1967 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1970 ; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1971 ; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1973 slwi r7,r8,2 ; shift groups-of-2 over by 2
1974 li r4,16 ; load offsets for X-form stores
1975 or r8,r7,r8 ; show if any in group of 4 are in use
1977 slwi r7,r8,4 ; shift groups-of-4 over by 4
1979 or r11,r7,r8 ; show if any in group of 8 are in use
1981 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1989 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1990 la r11,savevr0(r3) ; get address of this group of registers in save area
1991 dcbz128 0,r11 ; zero the line
1992 stvxl v0,0,r11 ; save 8 VRs in the line
2002 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
2003 la r11,savevr8(r3) ; get address of this group of registers in save area
2004 dcbz128 0,r11 ; zero the line
2005 stvxl v8,0,r11 ; save 8 VRs in the line
2015 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
2016 la r11,savevr16(r3) ; get address of this group of registers in save area
2017 dcbz128 0,r11 ; zero the line
2018 stvxl v16,0,r11 ; save 8 VRs in the line
2028 bflr 24 ; done if none of vr24-vr31 are in use
2029 la r11,savevr24(r3) ; get address of this group of registers in save area
2030 dcbz128 0,r11 ; zero the line
2031 stvxl v24,0,r11 ; save 8 VRs in the line
2041 ; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
2042 ; or neither in each group. This cuts down on conditional branches.
2043 ; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
2047 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
2048 li r4,16 ; load offset for X-form stores
2050 bf 0,vr_st32b ; skip if neither VR in this pair is in use
2051 la r11,savevr0(r3) ; get address of this group of registers in save area
2052 dcba 0,r11 ; establish the line wo reading it
2053 stvxl v0,0,r11 ; save the two VRs in the line
2057 bf 2,vr_st32c ; skip if neither VR in this pair is in use
2058 la r11,savevr2(r3) ; get address of this group of registers in save area
2059 dcba 0,r11 ; establish the line wo reading it
2060 stvxl v2,0,r11 ; save the two VRs in the line
2064 bf 4,vr_st32d ; skip if neither VR in this pair is in use
2065 la r11,savevr4(r3) ; get address of this group of registers in save area
2066 dcba 0,r11 ; establish the line wo reading it
2067 stvxl v4,0,r11 ; save the two VRs in the line
2071 bf 6,vr_st32e ; skip if neither VR in this pair is in use
2072 la r11,savevr6(r3) ; get address of this group of registers in save area
2073 dcba 0,r11 ; establish the line wo reading it
2074 stvxl v6,0,r11 ; save the two VRs in the line
2078 bf 8,vr_st32f ; skip if neither VR in this pair is in use
2079 la r11,savevr8(r3) ; get address of this group of registers in save area
2080 dcba 0,r11 ; establish the line wo reading it
2081 stvxl v8,0,r11 ; save the two VRs in the line
2085 bf 10,vr_st32g ; skip if neither VR in this pair is in use
2086 la r11,savevr10(r3) ; get address of this group of registers in save area
2087 dcba 0,r11 ; establish the line wo reading it
2088 stvxl v10,0,r11 ; save the two VRs in the line
2092 bf 12,vr_st32h ; skip if neither VR in this pair is in use
2093 la r11,savevr12(r3) ; get address of this group of registers in save area
2094 dcba 0,r11 ; establish the line wo reading it
2095 stvxl v12,0,r11 ; save the two VRs in the line
2099 bf 14,vr_st32i ; skip if neither VR in this pair is in use
2100 la r11,savevr14(r3) ; get address of this group of registers in save area
2101 dcba 0,r11 ; establish the line wo reading it
2102 stvxl v14,0,r11 ; save the two VRs in the line
2106 bf 16,vr_st32j ; skip if neither VR in this pair is in use
2107 la r11,savevr16(r3) ; get address of this group of registers in save area
2108 dcba 0,r11 ; establish the line wo reading it
2109 stvxl v16,0,r11 ; save the two VRs in the line
2113 bf 18,vr_st32k ; skip if neither VR in this pair is in use
2114 la r11,savevr18(r3) ; get address of this group of registers in save area
2115 dcba 0,r11 ; establish the line wo reading it
2116 stvxl v18,0,r11 ; save the two VRs in the line
2120 bf 20,vr_st32l ; skip if neither VR in this pair is in use
2121 la r11,savevr20(r3) ; get address of this group of registers in save area
2122 dcba 0,r11 ; establish the line wo reading it
2123 stvxl v20,0,r11 ; save the two VRs in the line
2127 bf 22,vr_st32m ; skip if neither VR in this pair is in use
2128 la r11,savevr22(r3) ; get address of this group of registers in save area
2129 dcba 0,r11 ; establish the line wo reading it
2130 stvxl v22,0,r11 ; save the two VRs in the line
2134 bf 24,vr_st32n ; skip if neither VR in this pair is in use
2135 la r11,savevr24(r3) ; get address of this group of registers in save area
2136 dcba 0,r11 ; establish the line wo reading it
2137 stvxl v24,0,r11 ; save the two VRs in the line
2141 bf 26,vr_st32o ; skip if neither VR in this pair is in use
2142 la r11,savevr26(r3) ; get address of this group of registers in save area
2143 dcba 0,r11 ; establish the line wo reading it
2144 stvxl v26,0,r11 ; save the two VRs in the line
2148 bf 28,vr_st32p ; skip if neither VR in this pair is in use
2149 la r11,savevr28(r3) ; get address of this group of registers in save area
2150 dcba 0,r11 ; establish the line wo reading it
2151 stvxl v28,0,r11 ; save the two VRs in the line
2155 bflr 30 ; done if neither VR in this pair is in use
2156 la r11,savevr30(r3) ; get address of this group of registers in save area
2157 dcba 0,r11 ; establish the line wo reading it
2158 stvxl v30,0,r11 ; save the two VRs in the line
2163 // *****************
2164 // * v r _ l o a d *
2165 // *****************
2167 // Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
2168 // of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
2169 // branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
2170 // in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
2171 // which would bug all VRs that aren't live.
2174 // interrupts are off, vectors are enabled
2175 // r3 = ptr to save area
2176 // r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
2177 // v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
2180 // r4 - r11, all CRs.
2183 mfsprg r9,2 ; get feature flags
2184 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
2185 mtcrf 0x02,r9 ; set cache line size bits in cr6
2186 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
2187 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
2188 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
2189 lis r7,0xFF00 ; so set bits 0-7
2191 // Loop touching in cache blocks we will load from.
2192 // r3 = savearea ptr
2193 // r5 = we light bits for the VRs we will be loading
2194 // r6 = 1 if 32-byte, 7 if 128-byte
2195 // r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
2196 // r10 = live VR bits
2197 // v31 = bugbug constant
2200 li r5,0 ; initialize set of VRs to load
2201 la r11,savevr0(r3) ; get address of register file
2202 b vr_ld2 ; enter loop in middle
2205 vr_ld1: ; loop over each cache line we will load
2206 dcbt r4,r11 ; start prefetch of the line
2207 andc r10,r10,r9 ; turn off the bits in this line
2208 or r5,r5,r9 ; we will load all these
2209 vr_ld2: ; initial entry pt
2210 cntlzw r4,r10 ; get offset to next live VR
2211 andc r4,r4,r6 ; cacheline align it
2212 srw. r9,r7,r4 ; position bits for VRs in that cache line
2213 slwi r4,r4,4 ; get byte offset within register file to that line
2214 bne vr_ld1 ; loop if more bits in r10
2216 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
2218 // Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
2219 // r3 = savearea ptr
2220 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2221 // r11 = addr(savevr0)
2222 // v31 = bugbug constant
2224 mtcrf 0x80,r5 ; set up bits for conditional branches
2225 li r4,16 ; load offsets for X-form stores
2227 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
2236 bt 0,vr_ld128a ; skip if this line must be loaded
2237 vor v0,v31,v31 ; no VR must be loaded, so bug them all
2246 vr_ld128a: ; must load from this line
2256 vr_ld128b: ; here to handle next cache line
2257 la r11,savevr8(r3) ; load offset to it
2258 bt 8,vr_ld128c ; skip if this line must be loaded
2259 vor v8,v31,v31 ; no VR must be loaded, so bug them all
2268 vr_ld128c: ; must load from this line
2278 vr_ld128d: ; here to handle next cache line
2279 la r11,savevr16(r3) ; load offset to it
2280 bt 16,vr_ld128e ; skip if this line must be loaded
2281 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2290 vr_ld128e: ; must load from this line
2300 vr_ld128f: ; here to handle next cache line
2301 la r11,savevr24(r3) ; load offset to it
2302 bt 24,vr_ld128g ; skip if this line must be loaded
2303 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2311 vr_ld128g: ; must load from this line
2322 // Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2323 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2324 // r11 = addr(savevr0)
2327 mtcrf 0xFF,r5 ; set up bits for conditional branches
2328 li r4,16 ; load offset for X-form stores
2330 bt 0,vr_ld32load0 ; skip if we must load this line
2331 vor v0,v31,v31 ; neither VR is live, so bug them both
2334 vr_ld32load0: ; must load VRs in this line
2338 vr_ld32test2: ; here to handle next cache line
2339 la r11,savevr2(r3) ; get offset to next cache line
2340 bt 2,vr_ld32load2 ; skip if we must load this line
2341 vor v2,v31,v31 ; neither VR is live, so bug them both
2344 vr_ld32load2: ; must load VRs in this line
2348 vr_ld32test4: ; here to handle next cache line
2349 la r11,savevr4(r3) ; get offset to next cache line
2350 bt 4,vr_ld32load4 ; skip if we must load this line
2351 vor v4,v31,v31 ; neither VR is live, so bug them both
2354 vr_ld32load4: ; must load VRs in this line
2358 vr_ld32test6: ; here to handle next cache line
2359 la r11,savevr6(r3) ; get offset to next cache line
2360 bt 6,vr_ld32load6 ; skip if we must load this line
2361 vor v6,v31,v31 ; neither VR is live, so bug them both
2364 vr_ld32load6: ; must load VRs in this line
2368 vr_ld32test8: ; here to handle next cache line
2369 la r11,savevr8(r3) ; get offset to next cache line
2370 bt 8,vr_ld32load8 ; skip if we must load this line
2371 vor v8,v31,v31 ; neither VR is live, so bug them both
2374 vr_ld32load8: ; must load VRs in this line
2378 vr_ld32test10: ; here to handle next cache line
2379 la r11,savevr10(r3) ; get offset to next cache line
2380 bt 10,vr_ld32load10 ; skip if we must load this line
2381 vor v10,v31,v31 ; neither VR is live, so bug them both
2384 vr_ld32load10: ; must load VRs in this line
2388 vr_ld32test12: ; here to handle next cache line
2389 la r11,savevr12(r3) ; get offset to next cache line
2390 bt 12,vr_ld32load12 ; skip if we must load this line
2391 vor v12,v31,v31 ; neither VR is live, so bug them both
2394 vr_ld32load12: ; must load VRs in this line
2398 vr_ld32test14: ; here to handle next cache line
2399 la r11,savevr14(r3) ; get offset to next cache line
2400 bt 14,vr_ld32load14 ; skip if we must load this line
2401 vor v14,v31,v31 ; neither VR is live, so bug them both
2404 vr_ld32load14: ; must load VRs in this line
2408 vr_ld32test16: ; here to handle next cache line
2409 la r11,savevr16(r3) ; get offset to next cache line
2410 bt 16,vr_ld32load16 ; skip if we must load this line
2411 vor v16,v31,v31 ; neither VR is live, so bug them both
2414 vr_ld32load16: ; must load VRs in this line
2418 vr_ld32test18: ; here to handle next cache line
2419 la r11,savevr18(r3) ; get offset to next cache line
2420 bt 18,vr_ld32load18 ; skip if we must load this line
2421 vor v18,v31,v31 ; neither VR is live, so bug them both
2424 vr_ld32load18: ; must load VRs in this line
2428 vr_ld32test20: ; here to handle next cache line
2429 la r11,savevr20(r3) ; get offset to next cache line
2430 bt 20,vr_ld32load20 ; skip if we must load this line
2431 vor v20,v31,v31 ; neither VR is live, so bug them both
2434 vr_ld32load20: ; must load VRs in this line
2438 vr_ld32test22: ; here to handle next cache line
2439 la r11,savevr22(r3) ; get offset to next cache line
2440 bt 22,vr_ld32load22 ; skip if we must load this line
2441 vor v22,v31,v31 ; neither VR is live, so bug them both
2444 vr_ld32load22: ; must load VRs in this line
2448 vr_ld32test24: ; here to handle next cache line
2449 la r11,savevr24(r3) ; get offset to next cache line
2450 bt 24,vr_ld32load24 ; skip if we must load this line
2451 vor v24,v31,v31 ; neither VR is live, so bug them both
2454 vr_ld32load24: ; must load VRs in this line
2458 vr_ld32test26: ; here to handle next cache line
2459 la r11,savevr26(r3) ; get offset to next cache line
2460 bt 26,vr_ld32load26 ; skip if we must load this line
2461 vor v26,v31,v31 ; neither VR is live, so bug them both
2464 vr_ld32load26: ; must load VRs in this line
2468 vr_ld32test28: ; here to handle next cache line
2469 la r11,savevr28(r3) ; get offset to next cache line
2470 bt 28,vr_ld32load28 ; skip if we must load this line
2471 vor v28,v31,v31 ; neither VR is live, so bug them both
2474 vr_ld32load28: ; must load VRs in this line
2478 vr_ld32test30: ; here to handle next cache line
2479 la r11,savevr30(r3) ; get offset to next cache line
2480 bt 30,vr_ld32load30 ; skip if we must load this line
2481 vor v30,v31,v31 ; neither VR is live, so bug them both
2483 vr_ld32load30: ; must load VRs in this line
projects / apple / xnu.git / blob