2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
27 #include <ppc/proc_reg.h>
30 #include <mach/ppc/vm_param.h>
31 #include <ppc/exception.h>
32 #include <ppc/savearea.h>
39 * void machine_load_context(thread_t thread)
41 * Load the context for the first thread to run on a
46 .globl EXT(machine_load_context)
48 LEXT(machine_load_context)
49 mfsprg r6,1 ; Get the current activation
50 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
51 lwz r0,PP_INTSTACK_TOP_SS(r6)
52 stw r0,PP_ISTACKPTR(r6)
53 mr r9,r3 /* Set up the current thread */
55 li r0,0 /* Clear a register */
56 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
57 mfmsr r5 /* Since we are passing control, get our MSR values */
58 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
59 lwz r1,saver1+4(r3) /* Load new stack pointer */
60 lwz r10,ACT_MACT_SPF(r9) /* Get the special flags */
61 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
62 stw r0,FM_BACKPTR(r1) /* zero backptr */
63 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
64 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
65 oris r10,r10,hi16(OnProc) /* Set OnProc bit */
66 stw r0,ACT_PREEMPT_CNT(r9) /* Enable preemption */
67 stw r10,ACT_MACT_SPF(r9) /* Update the special flags */
68 stw r10,spcFlags(r6) /* Set per_proc copy of the special flags */
69 b EXT(exception_exit) /* Go for it */
71 /* thread_t Switch_context(thread_t old,
75 * Switch from one thread to another. If a continuation is supplied, then
76 * we do not need to save callee save registers.
80 /* void Call_continuation( void (*continuation)(void), void *param, wait_result_t wresult, vm_offset_t stack_ptr)
84 .globl EXT(Call_continuation)
86 LEXT(Call_continuation)
87 mtlr r3 /* continuation */
88 mr r3,r4 /* parameter */
89 mr r4,r5 /* wait result */
90 mr r1,r6 /* Load new stack pointer */
91 blrl /* Jump to the continuation */
93 b EXT(thread_terminate)
96 * Get the old kernel stack, and store into the thread structure.
97 * See if a continuation is supplied, and skip state save if so.
99 * Note that interrupts must be disabled before we get here (i.e., splsched)
103 * Switch_context(old, continuation, new)
105 * Context switches are double jumps. We pass the following to the
106 * context switch firmware call:
108 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
112 * R7 = high order physical address of savearea for full switch
114 * savesrr0 is set to go to switch_in
115 * savesrr1 is set to uninterruptible with translation on
120 .globl EXT(Switch_context)
124 lwz r12,ACT_PER_PROC(r3) ; Get the per_proc block
126 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
127 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
128 bne++ notonintstack ; (DEBUG/TRACE) stack
132 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
133 lwz r9,umwSpace(r5) ; Get user memory window address space
134 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
135 lwz r0,CTHREAD_SELF+0(r5) ; Pick up the user assist "word" (actually a double)
136 lwz r7,CTHREAD_SELF+4(r5) ; both halves
137 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
138 lwz r6,umwRelo(r5) ; Get user memory window relocation top
139 stw r12,ACT_PER_PROC(r5) ; Set per_proc in new activation
141 lwz r2,umwRelo+4(r5) ; Get user memory window relocation bottom
143 stw r0,UAW+0(r12) ; Save the assist word for the "ultra fast path"
146 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
148 sth r9,ppUMWmp+mpSpace(r12) ; Save the space
149 stw r6,ppUMWmp+mpNestReloc(r12) ; Save top part of physical address
150 stw r2,ppUMWmp+mpNestReloc+4(r12) ; Save bottom part of physical address
151 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
152 lwz r2,traceMask(0) ; Get the enabled traces
153 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
154 lis r0,hi16(CutTrace) ; Trace FW call
155 mr. r2,r2 ; Any tracing going on?
156 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
157 ori r0,r0,lo16(CutTrace) ; Trace FW call
158 beq++ cswNoTrc ; No trace today, dude...
160 li r2,0x4400 ; Trace ID
161 mr r6,r11 ; Trace prev savearea
162 sc ; Cut trace entry of context switch
164 cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
165 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
166 lwz r9,VMXowner(r12) ; Grab the owner of the vector
167 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
168 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
169 li r0,1 ; Get set to hold off quickfret
171 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
172 cmplw r10,r2 ; Do we have the live float context?
173 lwz r10,FPUlevel(r2) ; Get the live level
174 mr r4,r3 ; Save our old thread to pass back
175 cmplw cr5,r9,r2 ; Do we have the live vector context?
176 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
177 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
178 bne++ cswnofloat ; Float is not ours...
180 cmplw r10,r11 ; Is the level the same?
181 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
182 lwz r5,FPUcpu(r2) ; Get the owning cpu
183 bne++ cswnofloat ; Level not the same, this is not live...
185 cmplw r5,r0 ; Still owned by this cpu?
186 lwz r10,FPUsave(r2) ; Get the pointer to next saved context
187 bne++ cswnofloat ; CPU claimed by someone else...
189 mr. r10,r10 ; Is there a savearea here?
190 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
192 beq-- cswnofloat ; No savearea to check...
194 lwz r3,SAVlevel(r10) ; Get the level
195 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
196 cmplw r3,r11 ; Is it for the current level?
198 bne++ cswnofloat ; Nope...
200 stw r5,FPUsave(r2) ; Pop off this savearea
202 rlwinm r3,r10,0,0,19 ; Move back to start of page
204 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
205 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
206 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
207 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
208 stw r5,SAVprev(r10) ; Link the old in (top)
209 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
210 xor r3,r10,r3 ; Convert to physical
211 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
212 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
215 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
216 mr r7,r2 ; (TEST/DEBUG)
217 li r2,0x4401 ; (TEST/DEBUG)
218 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
220 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
221 mr r2,r7 ; (TEST/DEBUG)
224 cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
226 lwz r10,VMXlevel(r2) ; Get the live level
228 cmplw r10,r11 ; Is the level the same?
229 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
230 lwz r5,VMXcpu(r2) ; Get the owning cpu
231 bne++ cswnovect ; Level not the same, this is not live...
233 cmplw r5,r0 ; Still owned by this cpu?
234 lwz r10,VMXsave(r2) ; Get the level
235 bne++ cswnovect ; CPU claimed by someone else...
237 mr. r10,r10 ; Is there a savearea here?
238 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
240 beq-- cswnovect ; No savearea to check...
242 lwz r3,SAVlevel(r10) ; Get the level
243 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
244 cmplw r3,r11 ; Is it for the current level?
246 bne++ cswnovect ; Nope...
248 stw r5,VMXsave(r2) ; Pop off this savearea
249 rlwinm r3,r10,0,0,19 ; Move back to start of page
251 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
252 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
253 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
254 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
255 stw r5,SAVprev(r10) ; Link the old in (top)
256 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
257 xor r3,r10,r3 ; Convert to physical
258 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
259 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
262 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
263 li r2,0x4501 ; (TEST/DEBUG)
264 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
268 cswnovect: li r0,0 ; Get set to release quickfret holdoff
269 rlwinm r11,r8,0,0,19 ; Switch to savearea base
270 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
271 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
273 ; Note that the low-level code requires the R7 contain the high order half of the savearea's
274 ; physical address. This is hack city, but it is the way it is.
276 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
277 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
278 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
279 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
280 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
282 lwz r9,SAVflags(r8) /* Get the flags */
283 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
284 li r10,(MASK(MSR_ME)|MASK(MSR_DR)) /* Get the switcher's MSR */
285 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
286 stw r10,savesrr1+4(r8) /* Set up for switch in */
287 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
288 xor r3,r11,r8 /* Get the physical address of the new context save area */
289 stw r9,SAVflags(r8) /* Set the flags */
291 bne cr1,swtchtocont ; Switch to the continuation
292 sc /* Switch to the new context */
294 /* We come back here in the new thread context
295 * R4 was set to hold the old thread pointer, but switch_in will put it into
296 * R3 where it belongs.
298 blr /* Jump into the new thread */
301 ; This is where we go when a continuation is set. We are actually
302 ; killing off the old context of the new guy so we need to pop off
303 ; any float or vector states for the ditched level.
305 ; Note that we do the same kind of thing a chkfac in hw_exceptions.s
311 stw r5,savesrr0+4(r8) ; Set the pc
312 stw r6,savesrr1+4(r8) ; Set the next MSR to use
313 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
314 mr r3,r8 ; Pass in the virtual address of savearea
316 b EXT(exception_exit) ; Blocking on continuation, toss old context...
321 * All switched to threads come here first to clean up the old thread.
322 * We need to do the following contortions because we need to keep
323 * the LR clean. And because we need to manipulate the savearea chain
324 * with translation on. If we could, this should be done in lowmem_vectors
325 * before translation is turned on. But we can't, dang it!
327 * switch_in() runs with DR on and IR off
329 * R3 = switcher's savearea (32-bit virtual)
330 * saver4 = old thread in switcher's save
331 * saver5 = new SRR0 in switcher's save
332 * saver6 = new SRR1 in switcher's save
339 .globl EXT(switch_in)
343 lwz r4,saver4+4(r3) ; Get the old thread
344 lwz r5,saver5+4(r3) ; Get the srr0 value
346 mfsprg r0,2 ; Get feature flags
347 mr r9,r4 ; Get the switched from ACT
348 lwz r6,saver6+4(r3) ; Get the srr1 value
349 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
350 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
352 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
353 stw r10,SAVprev+4(r3) ; Chain on the old one
355 mr r3,r4 ; Pass back the old thread
357 mtsrr0 r5 ; Set return point
358 mtsrr1 r6 ; Set return MSR
360 bne++ siSixtyFour ; Go do 64-bit...
368 * void fpu_save(facility_context ctx)
370 * Note that there are some oddities here when we save a context we are using.
371 * It is really not too cool to do this, but what the hey... Anyway,
372 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
373 * savearea containing the context just saved will go away. So, bottom line is
374 * that don't use fpus until after you are done with the saved context.
381 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
382 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
383 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
385 mfmsr r0 ; Get the MSR
386 andc r0,r0,r2 ; Clear FP, VEC
387 andc r2,r0,r12 ; Clear EE
388 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
389 mtmsr r2 ; Set the MSR
392 mfsprg r6,1 ; Get the current activation
393 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
394 lwz r12,FPUowner(r6) ; Get the context ID for owner
397 mr r7,r0 ; (TEST/DEBUG)
398 li r4,0 ; (TEST/DEBUG)
399 mr r10,r3 ; (TEST/DEBUG)
400 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
401 mr. r3,r12 ; (TEST/DEBUG)
402 li r2,0x6F00 ; (TEST/DEBUG)
403 li r5,0 ; (TEST/DEBUG)
404 beq-- noowneryet ; (TEST/DEBUG)
405 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
406 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
408 noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
410 mr r0,r7 ; (TEST/DEBUG)
411 mr r3,r10 ; (TEST/DEBUG)
413 mflr r2 ; Save the return address
415 cmplw r3,r12 ; Is the specified context live?
416 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
417 lwz r9,FPUcpu(r3) ; Get the cpu that context was last on
418 bne-- fsret ; Nobody owns the FPU, no save required...
420 cmplw r9,r11 ; Was the context for this processor?
421 la r5,FPUsync(r3) ; Point to the sync word
422 bne-- fsret ; Facility not last used on this processor...
425 ; It looks like we need to save this one.
427 ; First, make sure that the live context block is not mucked with while
428 ; we are trying to save it on out. Then we will give it the final check.
431 lis r9,ha16(EXT(LockTimeOut)) ; Get the high part
432 mftb r8 ; Get the time now
433 lwz r9,lo16(EXT(LockTimeOut))(r9) ; Get the timeout value
434 b fssync0a ; Jump to the lock...
438 fssync0: li r7,lgKillResv ; Get killing field
439 stwcx. r7,0,r7 ; Kill reservation
441 fssync0a: lwz r7,0(r5) ; Sniff the lock
442 mftb r10 ; Is it time yet?
443 cmplwi cr1,r7,0 ; Is it locked?
444 sub r10,r10,r8 ; How long have we been spinning?
445 cmplw r10,r9 ; Has it been too long?
446 bgt-- fstimeout ; Way too long, panic...
447 bne-- cr1,fssync0a ; Yea, still locked so sniff harder...
449 fssync1: lwarx r7,0,r5 ; Get the sync word
450 li r12,1 ; Get the lock
451 mr. r7,r7 ; Is it unlocked?
453 stwcx. r12,0,r5 ; Store lock and test reservation
454 bne-- fssync1 ; Try again if lost reservation...
456 isync ; Toss speculation
458 lwz r12,FPUowner(r6) ; Get the context ID for owner
459 cmplw r3,r12 ; Check again if we own the FPU?
460 bne-- fsretlk ; Go unlock and return since we no longer own context
462 lwz r5,FPUcpu(r12) ; Get the cpu that context was last on
463 lwz r7,FPUsave(r12) ; Get the current FPU savearea for the thread
464 cmplw r5,r11 ; Is this for the same processor?
465 lwz r9,FPUlevel(r12) ; Get our current level indicator
466 bne-- fsretlk ; Not the same processor, skip any save...
468 cmplwi r7,0 ; Have we ever saved this facility context?
469 beq-- fsneedone ; Never saved it, so go do it...
471 lwz r8,SAVlevel(r7) ; Get the level of this savearea
472 cmplw r9,r8 ; Correct level?
473 beq-- fsretlk ; The current level is already saved, bail out...
475 fsneedone: bl EXT(save_get) ; Get a savearea for the context
477 mfsprg r6,1 ; Get the current activation
478 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
479 li r4,SAVfloat ; Get floating point tag
480 lwz r12,FPUowner(r6) ; Get back our thread
481 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
482 lwz r4,facAct(r12) ; Get the activation associated with live context
483 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
484 stw r4,SAVact(r3) ; Indicate the right activation for this context
485 lwz r9,FPUlevel(r12) ; Get our current level indicator again
486 stw r3,FPUsave(r12) ; Set this as the most current floating point context
487 stw r8,SAVprev+4(r3) ; And then chain this in front
489 stw r9,SAVlevel(r3) ; Show level in savearea
491 bl fp_store ; save all 32 FPRs in the save area at r3
492 mtlr r2 ; Restore return
494 fsretlk: li r7,0 ; Get the unlock value
495 eieio ; Make sure that these updates make it out
496 stw r7,FPUsync(r12) ; Unlock it
498 fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
503 fstimeout: mr r4,r5 ; Set the lock address
504 mr r5,r7 ; Set the lock word data
505 lis r3,hi16(fstimeout_str) ; Get the failed lck message
506 ori r3,r3,lo16(fstimeout_str) ; Get the failed lck message
508 BREAKPOINT_TRAP ; We die here anyway
512 STRINGD "fpu_save: timeout on sync lock (0x%08X), value = 0x%08X\n\000"
519 * Entered to handle the floating-point unavailable exception and
522 * This code is run in virtual address mode on with interrupts off.
524 * Upon exit, the code returns to the users context with the floating
525 * point facility turned on.
527 * ENTRY: VM switched ON
529 * State is saved in savearea pointed to by R4.
530 * All other registers are free.
535 .globl EXT(fpu_switch)
540 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
541 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
547 mfsprg r17,1 ; Get the current activation
548 lwz r26,ACT_PER_PROC(r17) ; Get the per_proc block
549 mfmsr r19 ; Get the current MSR
551 mr r25,r4 ; Save the entry savearea
552 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
553 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
555 mtmsr r19 ; Enable floating point instructions
558 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
559 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
561 ; R22 has the "old" context anchor
562 ; R29 has the "new" context anchor
565 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
566 li r2,0x7F01 ; (TEST/DEBUG)
567 mr r3,r22 ; (TEST/DEBUG)
568 mr r5,r29 ; (TEST/DEBUG)
569 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
573 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
575 mr. r22,r22 ; See if there is any live FP status
576 la r15,FPUsync(r22) ; Point to the sync word
578 beq-- fsnosave ; No live context, so nothing to save...
580 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
581 cmplw cr2,r22,r29 ; Are both old and new the same context?
582 lwz r30,FPUsave(r22) ; Get the top savearea
583 cmplw r18,r16 ; Make sure we are on the right processor
584 lwz r31,FPUlevel(r22) ; Get the context level
585 cmplwi cr1,r30,0 ; Anything saved yet?
587 bne-- fsnosave ; No, not on the same processor...
590 ; Check to see if the live context has already been saved.
591 ; Also check to see if all we are here just to re-enable the MSR
592 ; and handle specially if so.
595 cmplw r31,r27 ; See if the current and active levels are the same
596 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
598 beq-- fsthesame ; New and old are the same, just go enable...
602 ; Note it turns out that on a G5, the following load has about a 50-50 chance of
603 ; taking a segment exception in a system that is doing heavy file I/O. We
604 ; make a dummy access right now in order to get that resolved before we take the lock.
605 ; We do not use the data returned because it may change over the lock
608 beq-- cr1,fswsync ; Nothing saved, skip the probe attempt...
609 lwz r11,SAVlevel(r30) ; Touch the context in order to fault in the segment
612 ; Make sure that the live context block is not mucked with while
613 ; we are trying to save it on out
616 fswsync: lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
617 mftb r3 ; Get the time now
618 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
619 b fswsync0a ; Jump to the lock...
623 fswsync0: li r19,lgKillResv ; Get killing field
624 stwcx. r19,0,r19 ; Kill reservation
626 fswsync0a: lwz r19,0(r15) ; Sniff the lock
627 mftb r18 ; Is it time yet?
628 cmplwi cr1,r19,0 ; Is it locked?
629 sub r18,r18,r3 ; How long have we been spinning?
630 cmplw r18,r11 ; Has it been too long?
631 bgt-- fswtimeout ; Way too long, panic...
632 bne-- cr1,fswsync0a ; Yea, still locked so sniff harder...
634 fswsync1: lwarx r19,0,r15 ; Get the sync word
635 li r0,1 ; Get the lock
636 mr. r19,r19 ; Is it unlocked?
638 stwcx. r0,0,r15 ; Store lock and test reservation
639 bne-- fswsync1 ; Try again if lost reservation...
641 isync ; Toss speculation
644 ; Note that now that we have the lock, we need to check if anything changed.
645 ; Also note that the possible changes are limited. The context owner can
646 ; never change to a different thread or level although it can be invalidated.
647 ; A new context can not be pushed on top of us, but it can be popped. The
648 ; cpu indicator will always change if another processor mucked with any
651 ; It should be very rare that any of the context stuff changes across the lock.
654 lwz r0,FPUowner(r26) ; Get the thread that owns the FPU again
655 lwz r11,FPUsave(r22) ; Get the top savearea again
656 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on again
657 sub r0,r0,r22 ; Non-zero if we lost ownership, 0 if not
658 xor r11,r11,r30 ; Non-zero if saved context changed, 0 if not
659 xor r18,r18,r16 ; Non-zero if cpu changed, 0 if not
660 cmplwi cr1,r30,0 ; Is anything saved?
661 or r0,r0,r11 ; Zero only if both owner and context are unchanged
662 or. r0,r0,r18 ; Zero only if nothing has changed
665 bne-- fsnosavelk ; Something has changed, so this is not ours to save...
666 beq-- cr1,fsmstsave ; There is no context saved yet...
668 lwz r11,SAVlevel(r30) ; Get the level of top saved context
670 cmplw r31,r11 ; Are live and saved the same?
673 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
674 li r2,0x7F02 ; (TEST/DEBUG)
675 mr r3,r11 ; (TEST/DEBUG)
676 mr r5,r31 ; (TEST/DEBUG)
677 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
679 li r3,0 ; (TEST/DEBUG)
682 beq++ fsnosavelk ; Same level, so already saved...
684 fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
685 eieio ; Make sure everyone sees it
686 bl EXT(save_get) ; Go get a savearea
688 lwz r12,facAct(r22) ; Get the activation associated with the context
689 stw r30,SAVprev+4(r3) ; Point us to the old context
690 stw r31,SAVlevel(r3) ; Tag our level
691 li r7,SAVfloat ; Get the floating point ID
692 stw r12,SAVact(r3) ; Make sure we point to the right guy
693 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
694 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
697 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
698 li r2,0x7F03 ; (TEST/DEBUG)
699 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
703 bl fp_store ; store all 32 FPRs
705 fsnosavelk: li r7,0 ; Get the unlock value
706 eieio ; Make sure that these updates make it out
707 stw r7,FPUsync(r22) ; Unlock it.
710 ; The context is all saved now and the facility is free.
712 ; Check if we need to fill the registers with junk, because this level has
713 ; never used them before and some thieving bastard could hack the old values
714 ; of some thread! Just imagine what would happen if they could! Why, nothing
715 ; would be safe! My God! It is terrifying!
717 ; Make sure that the live context block is not mucked with while
718 ; we are trying to load it up
721 fsnosave: la r15,FPUsync(r29) ; Point to the sync word
722 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
723 mftb r3 ; Get the time now
724 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
725 b fsnsync0a ; Jump to the lock...
729 fsnsync0: li r19,lgKillResv ; Get killing field
730 stwcx. r19,0,r19 ; Kill reservation
732 fsnsync0a: lwz r19,0(r15) ; Sniff the lock
733 mftb r18 ; Is it time yet?
734 cmplwi cr1,r19,0 ; Is it locked?
735 sub r18,r18,r3 ; How long have we been spinning?
736 cmplw r18,r11 ; Has it been too long?
737 bgt-- fsntimeout ; Way too long, panic...
738 bne-- cr1,fsnsync0a ; Yea, still locked so sniff harder...
740 fsnsync1: lwarx r19,0,r15 ; Get the sync word
741 li r0,1 ; Get the lock
742 mr. r19,r19 ; Is it unlocked?
743 bne-- fsnsync0 ; Unfortunately, it is locked...
744 stwcx. r0,0,r15 ; Store lock and test reservation
745 bne-- fsnsync1 ; Try again if lost reservation...
747 isync ; Toss speculation
749 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
750 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
751 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
753 stw r16,FPUcpu(r29) ; Claim context for us
757 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
758 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
759 li r2,0x7F04 ; (TEST/DEBUG)
760 mr r1,r15 ; (TEST/DEBUG)
761 mr r3,r14 ; (TEST/DEBUG)
762 mr r5,r13 ; (TEST/DEBUG)
763 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
767 lis r18,hi16(EXT(PerProcTable)) ; Set base PerProcTable
768 mulli r19,r19,ppeSize ; Find offset to the owner per_proc_entry
769 ori r18,r18,lo16(EXT(PerProcTable)) ; Set base PerProcTable
770 li r16,FPUowner ; Displacement to float owner
771 add r19,r18,r19 ; Point to the owner per_proc_entry
772 lwz r19,ppe_vaddr(r19) ; Point to the owner per_proc
774 fsinvothr: lwarx r18,r16,r19 ; Get the owner
775 sub r0,r18,r29 ; Subtract one from the other
776 sub r11,r29,r18 ; Subtract the other from the one
777 or r11,r11,r0 ; Combine them
778 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
779 and r18,r18,r11 ; Make 0 if same, unchanged if not
780 stwcx. r18,r16,r19 ; Try to invalidate it
781 bne-- fsinvothr ; Try again if there was a collision...
783 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
784 la r11,savefp0(r14) ; Point to first line to bring in
785 stw r15,FPUlevel(r29) ; Set the "new" active level
787 stw r29,FPUowner(r26) ; Mark us as having the live context
789 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
791 dcbt 0,r11 ; Touch line in
793 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
794 lwz r3,SAVprev+4(r14) ; Get the previous context
795 cmplw r0,r15 ; Top level correct to load?
796 li r7,0 ; Get the unlock value
797 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
799 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
802 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
803 li r2,0x7F05 ; (TEST/DEBUG)
804 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
808 eieio ; Make sure that these updates make it out
809 stw r7,FPUsync(r29) ; Unlock context now that the context save has been removed
811 // Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
812 // are executed on a 128-byte machine, but that is better than a mispredicted branch.
814 la r11,savefp4(r14) ; Point to next line
815 dcbt 0,r11 ; Touch line in
819 la r11,savefp8(r14) ; Point to next line
821 dcbt 0,r11 ; Touch line in
825 la r11,savefp12(r14) ; Point to next line
827 dcbt 0,r11 ; Touch line in
830 lfd f10,savefp10(r14)
831 la r11,savefp16(r14) ; Point to next line
832 lfd f11,savefp11(r14)
833 dcbt 0,r11 ; Touch line in
834 lfd f12,savefp12(r14)
835 lfd f13,savefp13(r14)
836 lfd f14,savefp14(r14)
837 la r11,savefp20(r14) ; Point to next line
838 lfd f15,savefp15(r14)
839 dcbt 0,r11 ; Touch line in
840 lfd f16,savefp16(r14)
841 lfd f17,savefp17(r14)
842 lfd f18,savefp18(r14)
843 la r11,savefp24(r14) ; Point to next line
844 lfd f19,savefp19(r14)
845 dcbt 0,r11 ; Touch line in
846 lfd f20,savefp20(r14)
847 lfd f21,savefp21(r14)
848 la r11,savefp28(r14) ; Point to next line
849 lfd f22,savefp22(r14)
850 lfd f23,savefp23(r14)
851 dcbt 0,r11 ; Touch line in
852 lfd f24,savefp24(r14)
853 lfd f25,savefp25(r14)
854 lfd f26,savefp26(r14)
855 lfd f27,savefp27(r14)
856 lfd f28,savefp28(r14)
857 lfd f29,savefp29(r14)
858 lfd f30,savefp30(r14)
859 lfd f31,savefp31(r14)
861 mr r3,r14 ; Get the old savearea (we popped it before)
862 bl EXT(save_ret) ; Toss it
864 fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
865 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
866 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
867 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
868 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
869 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
870 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
871 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
872 mr r3,r25 ; Pass the virtual addres of savearea
873 beq- fsnuser ; We are not user state...
874 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
875 stw r11,spcFlags(r26) ; Set per_proc copy
879 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
880 li r2,0x7F07 ; (TEST/DEBUG)
881 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
885 b EXT(exception_exit) ; Exit to the fray...
888 * Initialize the registers to some bogus value
891 MakeSureThatNoTerroristsCanHurtUsByGod:
894 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
895 li r2,0x7F06 ; (TEST/DEBUG)
896 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
899 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
900 li r7,0 ; Get the unlock value
901 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
902 eieio ; Make sure that these updates make it out
903 stw r7,FPUsync(r29) ; Unlock it now that the context has been removed
905 lfd f0,0(r5) ; Initialize FP0
906 fmr f1,f0 ; Do them all
937 b fsenable ; Finish setting it all up...
941 ; We get here when we are switching to the same context at the same level and the context
942 ; is still live. Essentially, all we are doing is turning on the facility. It may have
943 ; gotten turned off due to doing a context save for the current level or a context switch
944 ; back to the live guy.
950 fsthesamel: li r7,0 ; Get the unlock value
951 eieio ; Make sure that these updates make it out
952 stw r7,FPUsync(r22) ; Unlock it.
957 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
958 li r2,0x7F0A ; (TEST/DEBUG)
959 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
962 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
964 lwz r11,SAVlevel(r30) ; Get the level of top saved context
965 lwz r14,SAVprev+4(r30) ; Get the previous savearea
967 cmplw r11,r31 ; Are live and saved the same?
969 bne++ fsenable ; Level not the same, nothing to pop, go enable and exit...
971 mr r3,r30 ; Get the old savearea (we popped it before)
972 stw r14,FPUsave(r22) ; Pop the savearea from the stack
973 bl EXT(save_ret) ; Toss it
974 b fsenable ; Go enable and exit...
977 ; Note that we need to choke in this code rather than panic because there is no
981 fswtimeout: lis r0,hi16(Choke) ; Choke code
982 ori r0,r0,lo16(Choke) ; and the rest
983 li r3,failTimeout ; Timeout code
986 fsntimeout: lis r0,hi16(Choke) ; Choke code
987 ori r0,r0,lo16(Choke) ; and the rest
988 li r3,failTimeout ; Timeout code
992 lis r0,hi16(Choke) ; Choke code
993 ori r0,r0,lo16(Choke) ; and the rest
994 li r3,failTimeout ; Timeout code
998 lis r0,hi16(Choke) ; Choke code
999 ori r0,r0,lo16(Choke) ; and the rest
1000 li r3,failTimeout ; Timeout code
1004 ; This function invalidates any live floating point context for the passed in facility_context.
1005 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1009 .globl EXT(toss_live_fpu)
1013 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1014 mfmsr r9 ; Get the MSR
1015 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1016 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
1017 andc r9,r9,r0 ; Force off VEC and FP
1018 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1019 andc r0,r9,r0 ; Turn off EE now
1020 mtmsr r0 ; No interruptions
1022 beq+ tlfnotours ; Floats off, can not be live here...
1024 mfsprg r8,1 ; Get the current activation
1025 lwz r8,ACT_PER_PROC(r8) ; Get the per_proc block
1028 ; Note that at this point, since floats are on, we are the owner
1029 ; of live state on this processor
1032 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
1033 li r0,0 ; Clear this just in case we need it
1034 cmplw r6,r3 ; Are we tossing our own context?
1035 bne-- tlfnotours ; Nope...
1037 lfd f1,Zero(0) ; Make a 0
1038 mtfsf 0xFF,f1 ; Clear it
1040 tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
1041 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1042 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1043 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1044 li r10,FPUowner ; Displacement to float owner
1045 add r11,r12,r11 ; Point to the owner per_proc_entry
1046 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1048 tlfinvothr: lwarx r12,r10,r11 ; Get the owner
1050 sub r0,r12,r3 ; Subtract one from the other
1051 sub r8,r3,r12 ; Subtract the other from the one
1052 or r8,r8,r0 ; Combine them
1053 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1054 and r12,r12,r8 ; Make 0 if same, unchanged if not
1055 stwcx. r12,r10,r11 ; Try to invalidate it
1056 bne-- tlfinvothr ; Try again if there was a collision...
1058 mtmsr r9 ; Restore interruptions
1059 isync ; Could be turning off floats here
1064 * Altivec stuff is here. The techniques used are pretty identical to
1065 * the floating point. Except that we will honor the VRSAVE register
1066 * settings when loading and restoring registers.
1068 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
1069 * mask. VRSAVE is set by the vector user and represents the VRs that they
1070 * say that they are using. The vrvalid mask indicates which vector registers
1071 * are saved in the savearea. Whenever context is saved, it is saved according
1072 * to the VRSAVE register. It is loaded based on VRSAVE anded with
1073 * vrvalid (all other registers are splatted with 0s). This is done because we
1074 * don't want to load any registers we don't have a copy of, we want to set them
1077 * Note that there are some oddities here when we save a context we are using.
1078 * It is really not too cool to do this, but what the hey... Anyway,
1079 * we turn vectors and fpu off before we leave.
1080 * The oddity is that if you use vectors after this, the
1081 * savearea containing the context just saved will go away. So, bottom line is
1082 * that don't use vectors until after you are done with the saved context.
1087 .globl EXT(vec_save)
1092 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
1093 mfmsr r0 ; Get the MSR
1094 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
1095 andc r0,r0,r2 ; Force off VEC and FP
1096 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
1097 andc r2,r0,r2 ; Clear EE for now
1098 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
1099 mtmsr r2 ; Set the MSR
1102 mfsprg r6,1 ; Get the current activation
1103 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1104 lwz r12,VMXowner(r6) ; Get the context ID for owner
1107 mr r11,r6 ; (TEST/DEBUG)
1108 mr r7,r0 ; (TEST/DEBUG)
1109 li r4,0 ; (TEST/DEBUG)
1110 mr r10,r3 ; (TEST/DEBUG)
1111 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1112 mr. r3,r12 ; (TEST/DEBUG)
1113 li r2,0x5F00 ; (TEST/DEBUG)
1114 li r5,0 ; (TEST/DEBUG)
1115 lwz r6,liveVRS(r6) ; (TEST/DEBUG)
1116 beq-- noowneryeu ; (TEST/DEBUG)
1117 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
1118 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
1120 noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1122 mr r0,r7 ; (TEST/DEBUG)
1123 mr r3,r10 ; (TEST/DEBUG)
1124 mr r6,r11 ; (TEST/DEBUG)
1126 mflr r2 ; Save the return address
1128 cmplw r3,r12 ; Is the specified context live?
1129 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
1130 bne-- vsret ; We do not own the vector, no save required...
1131 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
1133 cmplw r9,r11 ; Was the context for this processor?
1134 la r5,VMXsync(r3) ; Point to the sync word
1135 bne-- vsret ; Specified context is not live
1138 ; It looks like we need to save this one. Or possibly toss a saved one if
1141 ; First, make sure that the live context block is not mucked with while
1142 ; we are trying to save it on out. Then we will give it the final check.
1145 lis r9,ha16(EXT(LockTimeOut)) ; Get the high part
1146 mftb r8 ; Get the time now
1147 lwz r9,lo16(EXT(LockTimeOut))(r9) ; Get the timeout value
1148 b vssync0a ; Jump to the lock...
1152 vssync0: li r7,lgKillResv ; Get killing field
1153 stwcx. r7,0,r7 ; Kill reservation
1155 vssync0a: lwz r7,0(r5) ; Sniff the lock
1156 mftb r10 ; Is it time yet?
1157 cmplwi cr1,r7,0 ; Is it locked?
1158 sub r10,r10,r8 ; How long have we been spinning?
1159 cmplw r10,r9 ; Has it been too long?
1160 bgt-- vswtimeout0 ; Way too long, panic...
1161 bne-- cr1,vssync0a ; Yea, still locked so sniff harder...
1163 vssync1: lwarx r7,0,r5 ; Get the sync word
1164 li r12,1 ; Get the lock
1165 mr. r7,r7 ; Is it unlocked?
1166 bne-- vssync0 ; No, it is unlocked...
1167 stwcx. r12,0,r5 ; Store lock and test reservation
1168 bne-- vssync1 ; Try again if lost reservation...
1170 isync ; Toss speculation
1172 lwz r12,VMXowner(r6) ; Get the context ID for owner
1173 cmplw r3,r12 ; Check again if we own VMX?
1174 lwz r10,liveVRS(r6) ; Get the right VRSave register
1175 bne-- vsretlk ; Go unlock and return since we no longer own context
1177 lwz r5,VMXcpu(r12) ; Get the cpu that context was last on
1178 lwz r7,VMXsave(r12) ; Get the current vector savearea for the thread
1179 cmplwi cr1,r10,0 ; Is VRsave set to 0?
1180 cmplw r5,r11 ; Is this for the same processor?
1181 lwz r9,VMXlevel(r12) ; Get our current level indicator
1182 bne-- vsretlk ; Not the same processor, skip any save...
1184 cmplwi r7,0 ; Have we ever saved this facility context?
1185 beq-- vsneedone ; Never saved it, so we need an area...
1187 lwz r8,SAVlevel(r7) ; Get the level this savearea is for
1188 cmplw r9,r8 ; Correct level?
1189 bne-- vsneedone ; Different level, so we need to save...
1191 bne++ cr1,vsretlk ; VRsave is non-zero so we need to keep what is saved...
1193 lwz r4,SAVprev+4(r7) ; Pick up the previous area
1194 li r5,0 ; Assume we just dumped the last
1195 mr. r4,r4 ; Is there one?
1196 stw r4,VMXsave(r12) ; Dequeue this savearea
1197 beq-- vsnomore ; We do not have another...
1199 lwz r5,SAVlevel(r4) ; Get the level associated with save
1201 vsnomore: stw r5,VMXlevel(r12) ; Save the level
1203 stw r7,VMXowner(r6) ; Show no live context here
1205 vsbackout: mr r4,r0 ; restore the saved MSR
1207 stw r7,VMXsync(r12) ; Unlock the context
1209 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
1213 vsneedone: beq-- cr1,vsclrlive ; VRSave is zero, go blow away the context...
1215 bl EXT(save_get) ; Get a savearea for the context
1217 mfsprg r6,1 ; Get the current activation
1218 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1219 li r4,SAVvector ; Get vector tag
1220 lwz r12,VMXowner(r6) ; Get back our context ID
1221 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1222 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1224 beq-- vsbackout ; If disowned, just toss savearea...
1225 lwz r4,facAct(r12) ; Get the activation associated with live context
1226 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1227 stw r4,SAVact(r3) ; Indicate the right activation for this context
1228 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1229 stw r3,VMXsave(r12) ; Set this as the most current floating point context
1230 stw r8,SAVprev+4(r3) ; And then chain this in front
1232 stw r9,SAVlevel(r3) ; Set level in savearea
1233 mfcr r12 ; save CRs across call to vr_store
1234 lwz r10,liveVRS(r6) ; Get the right VRSave register
1236 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
1238 mfsprg r6,1 ; Get the current activation
1239 mtcrf 255,r12 ; Restore the non-volatile CRs
1240 lwz r6,ACT_PER_PROC(r6) ; Get the per_proc block
1241 mtlr r2 ; Restore return address
1242 lwz r12,VMXowner(r6) ; Get back our context ID
1244 vsretlk: li r7,0 ; Get the unlock value
1245 eieio ; Make sure that these updates make it out
1246 stw r7,VMXsync(r12) ; Unlock it
1248 vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1253 vsclrlive: li r7,0 ; Clear
1254 stw r7,VMXowner(r6) ; Show no live context here
1255 b vsretlk ; Go unlock and leave...
1260 * Entered to handle the vector unavailable exception and
1261 * switch vector context
1263 * This code is run with virtual address mode on and interrupts off.
1265 * Upon exit, the code returns to the users context with the vector
1266 * facility turned on.
1268 * ENTRY: VM switched ON
1270 * State is saved in savearea pointed to by R4.
1271 * All other registers are free.
1276 .globl EXT(vec_switch)
1281 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1282 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1288 mfsprg r17,1 ; Get the current activation
1289 lwz r26,ACT_PER_PROC(r17) ; Get the per_proc block
1290 mfmsr r19 ; Get the current MSR
1292 mr r25,r4 ; Save the entry savearea
1293 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1294 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1296 mtmsr r19 ; Enable vector instructions
1299 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1300 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
1302 ; R22 has the "old" context anchor
1303 ; R29 has the "new" context anchor
1306 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1307 li r2,0x5F01 ; (TEST/DEBUG)
1308 mr r3,r22 ; (TEST/DEBUG)
1309 mr r5,r29 ; (TEST/DEBUG)
1310 lwz r6,liveVRS(r26) ; (TEST/DEBUG)
1311 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1315 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1317 mr. r22,r22 ; See if there is any live vector status
1318 la r15,VMXsync(r22) ; Point to the sync word
1320 beq-- vswnosave ; No live context, so nothing to save...
1322 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1323 cmplw cr2,r22,r29 ; Are both old and new the same context?
1324 lwz r30,VMXsave(r22) ; Get the top savearea
1325 cmplwi cr1,r30,0 ; Anything saved yet?
1326 lwz r31,VMXlevel(r22) ; Get the context level
1327 cmplw r18,r16 ; Make sure we are on the right processor
1329 lwz r10,liveVRS(r26) ; Get the right VRSave register
1331 bne-- vswnosave ; No, not on the same processor...
1334 ; Check to see if the live context has already been saved.
1335 ; Also check to see if all we are here just to re-enable the MSR
1336 ; and handle specially if so.
1339 cmplw r31,r27 ; See if the current and active levels are the same
1340 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1342 beq-- vswthesame ; New and old are the same, just go enable...
1345 ; Make sure that the live context block is not mucked with while
1346 ; we are trying to save it on out
1349 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
1350 mftb r3 ; Get the time now
1351 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
1352 b vswsync0a ; Jump to the lock...
1356 vswsync0: li r19,lgKillResv ; Get killing field
1357 stwcx. r19,0,r19 ; Kill reservation
1359 vswsync0a: lwz r19,0(r15) ; Sniff the lock
1360 mftb r18 ; Is it time yet?
1361 cmplwi cr1,r19,0 ; Is it locked?
1362 sub r18,r18,r3 ; How long have we been spinning?
1363 cmplw r18,r11 ; Has it been too long?
1364 bgt-- vswtimeout0 ; Way too long, panic...
1365 bne-- cr1,vswsync0a ; Yea, still locked so sniff harder...
1367 vswsync1: lwarx r19,0,r15 ; Get the sync word
1368 li r0,1 ; Get the lock
1369 mr. r19,r19 ; Is it unlocked?
1371 stwcx. r0,0,r15 ; Store lock and test reservation
1372 bne-- vswsync1 ; Try again if lost reservation...
1374 isync ; Toss speculation
1377 ; Note that now that we have the lock, we need to check if anything changed.
1378 ; Also note that the possible changes are limited. The context owner can
1379 ; never change to a different thread or level although it can be invalidated.
1380 ; A new context can not be pushed on top of us, but it can be popped. The
1381 ; cpu indicator will always change if another processor mucked with any
1384 ; It should be very rare that any of the context stuff changes across the lock.
1387 lwz r0,VMXowner(r26) ; Get the thread that owns the vectors again
1388 lwz r11,VMXsave(r22) ; Get the top savearea again
1389 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on again
1390 sub r0,r0,r22 ; Non-zero if we lost ownership, 0 if not
1391 xor r11,r11,r30 ; Non-zero if saved context changed, 0 if not
1392 xor r18,r18,r16 ; Non-zero if cpu changed, 0 if not
1393 cmplwi cr1,r30,0 ; Is anything saved?
1394 or r0,r0,r11 ; Zero only if both owner and context are unchanged
1395 or. r0,r0,r18 ; Zero only if nothing has changed
1396 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1399 bne-- vswnosavelk ; Something has changed, so this is not ours to save...
1400 beq-- cr1,vswmstsave ; There is no context saved yet...
1402 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1404 cmplw r31,r11 ; Are live and saved the same?
1407 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1408 li r2,0x5F02 ; (TEST/DEBUG)
1409 mr r3,r30 ; (TEST/DEBUG)
1410 mr r5,r31 ; (TEST/DEBUG)
1411 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1415 beq++ vswnosavelk ; Same level, already saved...
1416 bne-- cr2,vswnosavelk ; Live context saved and VRSave not 0, no save and keep context...
1418 lwz r4,SAVprev+4(r30) ; Pick up the previous area
1419 li r5,0 ; Assume this is the only one (which should be the ususal case)
1420 mr. r4,r4 ; Was this the only one?
1421 stw r4,VMXsave(r22) ; Dequeue this savearea
1422 beq++ vswonlyone ; This was the only one...
1423 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1425 vswonlyone: stw r5,VMXlevel(r22) ; Save the level
1426 stw r8,VMXowner(r26) ; Clear owner
1428 mr r3,r30 ; Copy the savearea we are tossing
1429 bl EXT(save_ret) ; Toss the savearea
1430 b vswnosavelk ; Go load up the context...
1434 vswmstsave: stw r8,VMXowner(r26) ; Clear owner
1435 beq-- cr2,vswnosavelk ; The VRSave was 0, so there is nothing to save...
1437 bl EXT(save_get) ; Go get a savearea
1439 lwz r12,facAct(r22) ; Get the activation associated with the context
1440 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1441 stw r30,SAVprev+4(r3) ; Point us to the old context
1442 stw r31,SAVlevel(r3) ; Tag our level
1443 li r7,SAVvector ; Get the vector ID
1444 stw r12,SAVact(r3) ; Make sure we point to the right guy
1445 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
1448 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1449 li r2,0x5F03 ; (TEST/DEBUG)
1450 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1454 lwz r10,liveVRS(r26) ; Get the right VRSave register
1455 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1458 ; The context is all saved now and the facility is free.
1460 ; Check if we need to fill the registers with junk, because this level has
1461 ; never used them before and some thieving bastard could hack the old values
1462 ; of some thread! Just imagine what would happen if they could! Why, nothing
1463 ; would be safe! My God! It is terrifying!
1465 ; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1466 ; constant that we may need to fill unused vector registers.
1468 ; Make sure that the live context block is not mucked with while
1469 ; we are trying to load it up
1473 li r7,0 ; Get the unlock value
1474 eieio ; Make sure that these updates make it out
1475 stw r7,VMXsync(r22) ; Unlock the old context
1477 vswnosave: la r15,VMXsync(r29) ; Point to the sync word
1478 lis r11,ha16(EXT(LockTimeOut)) ; Get the high part
1479 mftb r3 ; Get the time now
1480 lwz r11,lo16(EXT(LockTimeOut))(r11) ; Get the timeout value
1481 b vswnsync0a ; Jump to the lock...
1485 vswnsync0: li r19,lgKillResv ; Get killing field
1486 stwcx. r19,0,r19 ; Kill reservation
1488 vswnsync0a: lwz r19,0(r15) ; Sniff the lock
1489 mftb r18 ; Is it time yet?
1490 cmplwi cr1,r19,0 ; Is it locked?
1491 sub r18,r18,r3 ; How long have we been spinning?
1492 cmplw r18,r11 ; Has it been too long?
1493 bgt-- vswtimeout1 ; Way too long, panic...
1494 bne-- cr1,vswnsync0a ; Yea, still locked so sniff harder...
1496 vswnsync1: lwarx r19,0,r15 ; Get the sync word
1497 li r0,1 ; Get the lock
1498 mr. r19,r19 ; Is it unlocked?
1499 bne-- vswnsync0 ; Unfortunately, it is locked...
1500 stwcx. r0,0,r15 ; Store lock and test reservation
1501 bne-- vswnsync1 ; Try again if lost reservation...
1503 isync ; Toss speculation
1505 vspltisb v31,-10 ; Get 0xF6F6F6F6
1506 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1507 vspltisb v30,5 ; Get 0x05050505
1508 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1509 vspltish v29,4 ; Get 0x00040004
1510 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1511 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1513 stw r16,VMXcpu(r29) ; Claim context for us
1517 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1518 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1519 li r2,0x5F04 ; (TEST/DEBUG)
1520 mr r1,r15 ; (TEST/DEBUG)
1521 mr r3,r14 ; (TEST/DEBUG)
1522 mr r5,r13 ; (TEST/DEBUG)
1523 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1527 lis r18,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1528 vspltisb v28,-2 ; Get 0xFEFEFEFE
1529 mulli r19,r19,ppeSize ; Find offset to the owner per_proc_entry
1530 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1531 ori r18,r18,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1532 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1533 li r16,VMXowner ; Displacement to vector owner
1534 add r19,r18,r19 ; Point to the owner per_proc_entry
1535 lwz r19,ppe_vaddr(r19) ; Point to the owner per_proc
1536 vrlb v31,v31,v29 ; Get 0xDEADDEAD
1538 vswinvothr: lwarx r18,r16,r19 ; Get the owner
1540 sub r0,r18,r29 ; Subtract one from the other
1541 sub r11,r29,r18 ; Subtract the other from the one
1542 or r11,r11,r0 ; Combine them
1543 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1544 and r18,r18,r11 ; Make 0 if same, unchanged if not
1545 stwcx. r18,r16,r19 ; Try to invalidate it
1546 bne-- vswinvothr ; Try again if there was a collision...
1548 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
1549 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1550 stw r15,VMXlevel(r29) ; Set the "new" active level
1552 stw r29,VMXowner(r26) ; Mark us as having the live context
1554 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
1556 lwz r3,SAVprev+4(r14) ; Get the previous context
1557 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1558 cmplw r0,r15 ; Top level correct to load?
1559 bne-- ProtectTheAmericanWay ; No, go initialize...
1561 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1564 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1565 li r2,0x5F05 ; (TEST/DEBUG)
1566 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1570 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
1571 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
1572 and r10,r10,r22 ; Figure out just what registers need to be loaded
1573 mr r3,r14 ; r3 <- ptr to savearea with VRs
1574 bl vr_load ; load VRs from save area based on vrsave in r10
1576 bl EXT(save_ret) ; Toss the save area after loading VRs
1578 vrenablelk: li r7,0 ; Get the unlock value
1579 eieio ; Make sure that these updates make it out
1580 stw r7,VMXsync(r29) ; Unlock the new context
1582 vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
1583 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
1584 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1585 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1586 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1587 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1588 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
1589 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1590 mr r3,r25 ; Pass virtual address of the savearea
1591 beq- vrnuser ; We are not user state...
1592 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
1593 stw r11,spcFlags(r26) ; Set per_proc copy
1597 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1598 li r2,0x5F07 ; (TEST/DEBUG)
1599 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1602 b EXT(exception_exit) ; Exit to the fray...
1605 * Initialize the registers to some bogus value
1608 ProtectTheAmericanWay:
1611 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1612 li r2,0x5F06 ; (TEST/DEBUG)
1613 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1617 vor v0,v31,v31 ; Copy into the next register
1618 vor v1,v31,v31 ; Copy into the next register
1619 vor v2,v31,v31 ; Copy into the next register
1620 vor v3,v31,v31 ; Copy into the next register
1621 vor v4,v31,v31 ; Copy into the next register
1622 vor v5,v31,v31 ; Copy into the next register
1623 vor v6,v31,v31 ; Copy into the next register
1624 vor v7,v31,v31 ; Copy into the next register
1625 vor v8,v31,v31 ; Copy into the next register
1626 vor v9,v31,v31 ; Copy into the next register
1627 vor v10,v31,v31 ; Copy into the next register
1628 vor v11,v31,v31 ; Copy into the next register
1629 vor v12,v31,v31 ; Copy into the next register
1630 vor v13,v31,v31 ; Copy into the next register
1631 vor v14,v31,v31 ; Copy into the next register
1632 vor v15,v31,v31 ; Copy into the next register
1633 vor v16,v31,v31 ; Copy into the next register
1634 vor v17,v31,v31 ; Copy into the next register
1635 vor v18,v31,v31 ; Copy into the next register
1636 vor v19,v31,v31 ; Copy into the next register
1637 vor v20,v31,v31 ; Copy into the next register
1638 vor v21,v31,v31 ; Copy into the next register
1639 vor v22,v31,v31 ; Copy into the next register
1640 vor v23,v31,v31 ; Copy into the next register
1641 vor v24,v31,v31 ; Copy into the next register
1642 vor v25,v31,v31 ; Copy into the next register
1643 vor v26,v31,v31 ; Copy into the next register
1644 vor v27,v31,v31 ; Copy into the next register
1645 vor v28,v31,v31 ; Copy into the next register
1646 vor v29,v31,v31 ; Copy into the next register
1647 vor v30,v31,v31 ; Copy into the next register
1648 b vrenablelk ; Finish setting it all up...
1653 ; We get here when we are switching to the same context at the same level and the context
1654 ; is still live. Essentially, all we are doing is turning on the faility. It may have
1655 ; gotten turned off due to doing a context save for the current level or a context switch
1656 ; back to the live guy.
1664 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1665 li r2,0x5F0A ; (TEST/DEBUG)
1666 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1669 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1671 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1672 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1674 cmplw r11,r31 ; Are live and saved the same?
1676 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1678 mr r3,r30 ; Get the old savearea (we popped it before)
1679 stw r11,VMXsave(r22) ; Pop the vector stack
1680 bl EXT(save_ret) ; Toss it
1681 b vrenable ; Go enable and exit...
1685 ; This function invalidates any live vector context for the passed in facility_context.
1686 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1690 .globl EXT(toss_live_vec)
1694 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1695 mfmsr r9 ; Get the MSR
1696 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1697 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1698 andc r9,r9,r0 ; Force off VEC and FP
1699 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1700 andc r0,r9,r0 ; Turn off EE now
1701 mtmsr r0 ; No interruptions
1703 beq+ tlvnotours ; Vector off, can not be live here...
1705 mfsprg r8,1 ; Get the current activation
1706 lwz r8,ACT_PER_PROC(r8) ; Get the per_proc block
1709 ; Note that at this point, since vecs are on, we are the owner
1710 ; of live state on this processor
1713 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1714 li r0,0 ; Clear this just in case we need it
1715 cmplw r6,r3 ; Are we tossing our own context?
1716 bne- tlvnotours ; Nope...
1718 vspltish v1,1 ; Turn on the non-Java bit and saturate
1719 vspltisw v0,1 ; Turn on the saturate bit
1720 vxor v1,v1,v0 ; Turn off saturate
1721 mtspr vrsave,r0 ; Clear VRSAVE
1722 mtvscr v1 ; Set the non-java, no saturate status
1724 tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1725 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1726 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1727 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1728 li r10,VMXowner ; Displacement to vector owner
1729 add r11,r12,r11 ; Point to the owner per_proc_entry
1730 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1731 li r0,0 ; Set a 0 to invalidate context
1733 tlvinvothr: lwarx r12,r10,r11 ; Get the owner
1735 sub r0,r12,r3 ; Subtract one from the other
1736 sub r8,r3,r12 ; Subtract the other from the one
1737 or r8,r8,r0 ; Combine them
1738 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1739 and r12,r12,r8 ; Make 0 if same, unchanged if not
1740 stwcx. r12,r10,r11 ; Try to invalidate it
1741 bne-- tlvinvothr ; Try again if there was a collision...
1743 mtmsr r9 ; Restore interruptions
1744 isync ; Could be turning off vectors here
1749 ; This function invalidates any live vector context for the passed in facility_context
1750 ; if the level is current. It also tosses the corresponding savearea if there is one.
1751 ; This function is primarily used whenever we detect a VRSave that is all zeros.
1755 .globl EXT(vec_trash)
1759 lwz r12,facAct(r3) ; Get the activation
1760 lwz r11,VMXlevel(r3) ; Get the context level
1761 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1762 lwz r9,VMXsave(r3) ; Get the savearea, if any
1763 cmplw r10,r11 ; Are we at the right level?
1764 cmplwi cr1,r9,0 ; Remember if there is a savearea
1765 bnelr+ ; No, we do nothing...
1767 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1768 lis r12,hi16(EXT(PerProcTable)) ; Set base PerProcTable
1769 mulli r11,r11,ppeSize ; Find offset to the owner per_proc_entry
1770 ori r12,r12,lo16(EXT(PerProcTable)) ; Set base PerProcTable
1771 li r10,VMXowner ; Displacement to vector owner
1772 add r11,r12,r11 ; Point to the owner per_proc_entry
1773 lwz r11,ppe_vaddr(r11) ; Point to the owner per_proc
1775 vtinvothr: lwarx r12,r10,r11 ; Get the owner
1777 sub r0,r12,r3 ; Subtract one from the other
1778 sub r8,r3,r12 ; Subtract the other from the one
1779 or r8,r8,r0 ; Combine them
1780 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1781 and r12,r12,r8 ; Make 0 if same, unchanged if not
1782 stwcx. r12,r10,r11 ; Try to invalidate it
1783 bne-- vtinvothr ; Try again if there was a collision...
1786 beqlr++ cr1 ; Leave if there is no savearea
1787 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1788 cmplw r8,r11 ; Savearea for the current level?
1789 bnelr++ ; No, nothing to release...
1791 lwz r8,SAVprev+4(r9) ; Pick up the previous area
1792 mr. r8,r8 ; Is there a previous?
1793 beq-- vtnoprev ; Nope...
1794 lwz r7,SAVlevel(r8) ; Get the level associated with save
1796 vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1797 stw r7,VMXlevel(r3) ; Pop the level
1799 mr r3,r9 ; Get the savearea to release
1800 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1804 ; Just some test code to force vector and/or floating point in the kernel
1808 .globl EXT(fctx_test)
1812 mfsprg r3,1 ; Get the current thread
1813 mr. r3,r3 ; Are we actually up and running?
1816 fmr f0,f0 ; Use floating point
1817 mftb r4 ; Get time base for a random number
1818 li r5,1 ; Get a potential vrsave to use
1819 andi. r4,r4,0x3F ; Get a number from 0 - 63
1820 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1821 mtspr vrsave,r5 ; Set VRSave
1822 vor v0,v0,v0 ; Use vectors
1826 // *******************
1827 // * f p _ s t o r e *
1828 // *******************
1830 // Store FPRs into a save area. Called by fpu_save and fpu_switch.
1833 // floating pt is enabled
1834 // r3 = ptr to save area
1840 mfsprg r11,2 ; get feature flags
1841 mtcrf 0x02,r11 ; put cache line size bits in cr6
1842 la r11,savefp0(r3) ; point to 1st line
1843 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1844 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1846 // Store the FPRs on a 128-byte machine.
1850 la r11,savefp16(r3) ; Point to the 2nd cache line
1853 dcbz128 0,r11 ; establish 2nd line
1860 stfd f10,savefp10(r3)
1861 stfd f11,savefp11(r3)
1862 stfd f12,savefp12(r3)
1863 stfd f13,savefp13(r3)
1864 stfd f14,savefp14(r3)
1865 stfd f15,savefp15(r3)
1866 stfd f16,savefp16(r3)
1867 stfd f17,savefp17(r3)
1868 stfd f18,savefp18(r3)
1869 stfd f19,savefp19(r3)
1870 stfd f20,savefp20(r3)
1871 stfd f21,savefp21(r3)
1872 stfd f22,savefp22(r3)
1873 stfd f23,savefp23(r3)
1874 stfd f24,savefp24(r3)
1875 stfd f25,savefp25(r3)
1876 stfd f26,savefp26(r3)
1877 stfd f27,savefp27(r3)
1878 stfd f28,savefp28(r3)
1879 stfd f29,savefp29(r3)
1880 stfd f30,savefp30(r3)
1881 stfd f31,savefp31(r3)
1884 // Store FPRs on a 32-byte machine.
1887 la r11,savefp4(r3) ; Point to the 2nd line
1889 dcbz 0,r11 ; Allocate cache
1892 la r11,savefp8(r3) ; Point to the 3rd line
1894 dcbz 0,r11 ; Allocate cache
1898 la r11,savefp12(r3) ; Point to the 4th line
1900 dcbz 0,r11 ; Allocate cache
1903 stfd f10,savefp10(r3)
1904 la r11,savefp16(r3) ; Point to the 5th line
1905 stfd f11,savefp11(r3)
1906 dcbz 0,r11 ; Allocate cache
1907 stfd f12,savefp12(r3)
1908 stfd f13,savefp13(r3)
1909 stfd f14,savefp14(r3)
1910 la r11,savefp20(r3) ; Point to the 6th line
1911 stfd f15,savefp15(r3)
1912 dcbz 0,r11 ; Allocate cache
1913 stfd f16,savefp16(r3)
1914 stfd f17,savefp17(r3)
1915 stfd f18,savefp18(r3)
1916 la r11,savefp24(r3) ; Point to the 7th line
1917 stfd f19,savefp19(r3)
1918 dcbz 0,r11 ; Allocate cache
1919 stfd f20,savefp20(r3)
1921 stfd f21,savefp21(r3)
1922 stfd f22,savefp22(r3)
1923 la r11,savefp28(r3) ; Point to the 8th line
1924 stfd f23,savefp23(r3)
1925 dcbz 0,r11 ; allocate it
1926 stfd f24,savefp24(r3)
1927 stfd f25,savefp25(r3)
1928 stfd f26,savefp26(r3)
1929 stfd f27,savefp27(r3)
1931 stfd f28,savefp28(r3)
1932 stfd f29,savefp29(r3)
1933 stfd f30,savefp30(r3)
1934 stfd f31,savefp31(r3)
1938 // *******************
1939 // * v r _ s t o r e *
1940 // *******************
1942 // Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1943 // both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1944 // unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1945 // for each cache block size.
1948 // interrupts are off, vectors are enabled
1949 // r3 = ptr to save area
1950 // r10 = vrsave (not 0)
1953 // r4 - r11, all CRs.
1956 mfsprg r9,2 ; get feature flags
1957 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1958 slwi r8,r10,1 ; Shift over 1
1959 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1960 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1961 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1964 ; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1965 ; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1967 slwi r7,r8,2 ; shift groups-of-2 over by 2
1968 li r4,16 ; load offsets for X-form stores
1969 or r8,r7,r8 ; show if any in group of 4 are in use
1971 slwi r7,r8,4 ; shift groups-of-4 over by 4
1973 or r11,r7,r8 ; show if any in group of 8 are in use
1975 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1983 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1984 la r11,savevr0(r3) ; get address of this group of registers in save area
1985 dcbz128 0,r11 ; zero the line
1986 stvxl v0,0,r11 ; save 8 VRs in the line
1996 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
1997 la r11,savevr8(r3) ; get address of this group of registers in save area
1998 dcbz128 0,r11 ; zero the line
1999 stvxl v8,0,r11 ; save 8 VRs in the line
2009 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
2010 la r11,savevr16(r3) ; get address of this group of registers in save area
2011 dcbz128 0,r11 ; zero the line
2012 stvxl v16,0,r11 ; save 8 VRs in the line
2022 bflr 24 ; done if none of vr24-vr31 are in use
2023 la r11,savevr24(r3) ; get address of this group of registers in save area
2024 dcbz128 0,r11 ; zero the line
2025 stvxl v24,0,r11 ; save 8 VRs in the line
2035 ; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
2036 ; or neither in each group. This cuts down on conditional branches.
2037 ; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
2041 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
2042 li r4,16 ; load offset for X-form stores
2044 bf 0,vr_st32b ; skip if neither VR in this pair is in use
2045 la r11,savevr0(r3) ; get address of this group of registers in save area
2046 dcba 0,r11 ; establish the line wo reading it
2047 stvxl v0,0,r11 ; save the two VRs in the line
2051 bf 2,vr_st32c ; skip if neither VR in this pair is in use
2052 la r11,savevr2(r3) ; get address of this group of registers in save area
2053 dcba 0,r11 ; establish the line wo reading it
2054 stvxl v2,0,r11 ; save the two VRs in the line
2058 bf 4,vr_st32d ; skip if neither VR in this pair is in use
2059 la r11,savevr4(r3) ; get address of this group of registers in save area
2060 dcba 0,r11 ; establish the line wo reading it
2061 stvxl v4,0,r11 ; save the two VRs in the line
2065 bf 6,vr_st32e ; skip if neither VR in this pair is in use
2066 la r11,savevr6(r3) ; get address of this group of registers in save area
2067 dcba 0,r11 ; establish the line wo reading it
2068 stvxl v6,0,r11 ; save the two VRs in the line
2072 bf 8,vr_st32f ; skip if neither VR in this pair is in use
2073 la r11,savevr8(r3) ; get address of this group of registers in save area
2074 dcba 0,r11 ; establish the line wo reading it
2075 stvxl v8,0,r11 ; save the two VRs in the line
2079 bf 10,vr_st32g ; skip if neither VR in this pair is in use
2080 la r11,savevr10(r3) ; get address of this group of registers in save area
2081 dcba 0,r11 ; establish the line wo reading it
2082 stvxl v10,0,r11 ; save the two VRs in the line
2086 bf 12,vr_st32h ; skip if neither VR in this pair is in use
2087 la r11,savevr12(r3) ; get address of this group of registers in save area
2088 dcba 0,r11 ; establish the line wo reading it
2089 stvxl v12,0,r11 ; save the two VRs in the line
2093 bf 14,vr_st32i ; skip if neither VR in this pair is in use
2094 la r11,savevr14(r3) ; get address of this group of registers in save area
2095 dcba 0,r11 ; establish the line wo reading it
2096 stvxl v14,0,r11 ; save the two VRs in the line
2100 bf 16,vr_st32j ; skip if neither VR in this pair is in use
2101 la r11,savevr16(r3) ; get address of this group of registers in save area
2102 dcba 0,r11 ; establish the line wo reading it
2103 stvxl v16,0,r11 ; save the two VRs in the line
2107 bf 18,vr_st32k ; skip if neither VR in this pair is in use
2108 la r11,savevr18(r3) ; get address of this group of registers in save area
2109 dcba 0,r11 ; establish the line wo reading it
2110 stvxl v18,0,r11 ; save the two VRs in the line
2114 bf 20,vr_st32l ; skip if neither VR in this pair is in use
2115 la r11,savevr20(r3) ; get address of this group of registers in save area
2116 dcba 0,r11 ; establish the line wo reading it
2117 stvxl v20,0,r11 ; save the two VRs in the line
2121 bf 22,vr_st32m ; skip if neither VR in this pair is in use
2122 la r11,savevr22(r3) ; get address of this group of registers in save area
2123 dcba 0,r11 ; establish the line wo reading it
2124 stvxl v22,0,r11 ; save the two VRs in the line
2128 bf 24,vr_st32n ; skip if neither VR in this pair is in use
2129 la r11,savevr24(r3) ; get address of this group of registers in save area
2130 dcba 0,r11 ; establish the line wo reading it
2131 stvxl v24,0,r11 ; save the two VRs in the line
2135 bf 26,vr_st32o ; skip if neither VR in this pair is in use
2136 la r11,savevr26(r3) ; get address of this group of registers in save area
2137 dcba 0,r11 ; establish the line wo reading it
2138 stvxl v26,0,r11 ; save the two VRs in the line
2142 bf 28,vr_st32p ; skip if neither VR in this pair is in use
2143 la r11,savevr28(r3) ; get address of this group of registers in save area
2144 dcba 0,r11 ; establish the line wo reading it
2145 stvxl v28,0,r11 ; save the two VRs in the line
2149 bflr 30 ; done if neither VR in this pair is in use
2150 la r11,savevr30(r3) ; get address of this group of registers in save area
2151 dcba 0,r11 ; establish the line wo reading it
2152 stvxl v30,0,r11 ; save the two VRs in the line
2157 // *****************
2158 // * v r _ l o a d *
2159 // *****************
2161 // Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
2162 // of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
2163 // branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
2164 // in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
2165 // which would bug all VRs that aren't live.
2168 // interrupts are off, vectors are enabled
2169 // r3 = ptr to save area
2170 // r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
2171 // v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
2174 // r4 - r11, all CRs.
2177 mfsprg r9,2 ; get feature flags
2178 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
2179 mtcrf 0x02,r9 ; set cache line size bits in cr6
2180 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
2181 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
2182 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
2183 lis r7,0xFF00 ; so set bits 0-7
2185 // Loop touching in cache blocks we will load from.
2186 // r3 = savearea ptr
2187 // r5 = we light bits for the VRs we will be loading
2188 // r6 = 1 if 32-byte, 7 if 128-byte
2189 // r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
2190 // r10 = live VR bits
2191 // v31 = bugbug constant
2194 li r5,0 ; initialize set of VRs to load
2195 la r11,savevr0(r3) ; get address of register file
2196 b vr_ld2 ; enter loop in middle
2199 vr_ld1: ; loop over each cache line we will load
2200 dcbt r4,r11 ; start prefetch of the line
2201 andc r10,r10,r9 ; turn off the bits in this line
2202 or r5,r5,r9 ; we will load all these
2203 vr_ld2: ; initial entry pt
2204 cntlzw r4,r10 ; get offset to next live VR
2205 andc r4,r4,r6 ; cacheline align it
2206 srw. r9,r7,r4 ; position bits for VRs in that cache line
2207 slwi r4,r4,4 ; get byte offset within register file to that line
2208 bne vr_ld1 ; loop if more bits in r10
2210 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
2212 // Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
2213 // r3 = savearea ptr
2214 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2215 // r11 = addr(savevr0)
2216 // v31 = bugbug constant
2218 mtcrf 0x80,r5 ; set up bits for conditional branches
2219 li r4,16 ; load offsets for X-form stores
2221 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
2230 bt 0,vr_ld128a ; skip if this line must be loaded
2231 vor v0,v31,v31 ; no VR must be loaded, so bug them all
2240 vr_ld128a: ; must load from this line
2250 vr_ld128b: ; here to handle next cache line
2251 la r11,savevr8(r3) ; load offset to it
2252 bt 8,vr_ld128c ; skip if this line must be loaded
2253 vor v8,v31,v31 ; no VR must be loaded, so bug them all
2262 vr_ld128c: ; must load from this line
2272 vr_ld128d: ; here to handle next cache line
2273 la r11,savevr16(r3) ; load offset to it
2274 bt 16,vr_ld128e ; skip if this line must be loaded
2275 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2284 vr_ld128e: ; must load from this line
2294 vr_ld128f: ; here to handle next cache line
2295 la r11,savevr24(r3) ; load offset to it
2296 bt 24,vr_ld128g ; skip if this line must be loaded
2297 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2305 vr_ld128g: ; must load from this line
2316 // Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2317 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2318 // r11 = addr(savevr0)
2321 mtcrf 0xFF,r5 ; set up bits for conditional branches
2322 li r4,16 ; load offset for X-form stores
2324 bt 0,vr_ld32load0 ; skip if we must load this line
2325 vor v0,v31,v31 ; neither VR is live, so bug them both
2328 vr_ld32load0: ; must load VRs in this line
2332 vr_ld32test2: ; here to handle next cache line
2333 la r11,savevr2(r3) ; get offset to next cache line
2334 bt 2,vr_ld32load2 ; skip if we must load this line
2335 vor v2,v31,v31 ; neither VR is live, so bug them both
2338 vr_ld32load2: ; must load VRs in this line
2342 vr_ld32test4: ; here to handle next cache line
2343 la r11,savevr4(r3) ; get offset to next cache line
2344 bt 4,vr_ld32load4 ; skip if we must load this line
2345 vor v4,v31,v31 ; neither VR is live, so bug them both
2348 vr_ld32load4: ; must load VRs in this line
2352 vr_ld32test6: ; here to handle next cache line
2353 la r11,savevr6(r3) ; get offset to next cache line
2354 bt 6,vr_ld32load6 ; skip if we must load this line
2355 vor v6,v31,v31 ; neither VR is live, so bug them both
2358 vr_ld32load6: ; must load VRs in this line
2362 vr_ld32test8: ; here to handle next cache line
2363 la r11,savevr8(r3) ; get offset to next cache line
2364 bt 8,vr_ld32load8 ; skip if we must load this line
2365 vor v8,v31,v31 ; neither VR is live, so bug them both
2368 vr_ld32load8: ; must load VRs in this line
2372 vr_ld32test10: ; here to handle next cache line
2373 la r11,savevr10(r3) ; get offset to next cache line
2374 bt 10,vr_ld32load10 ; skip if we must load this line
2375 vor v10,v31,v31 ; neither VR is live, so bug them both
2378 vr_ld32load10: ; must load VRs in this line
2382 vr_ld32test12: ; here to handle next cache line
2383 la r11,savevr12(r3) ; get offset to next cache line
2384 bt 12,vr_ld32load12 ; skip if we must load this line
2385 vor v12,v31,v31 ; neither VR is live, so bug them both
2388 vr_ld32load12: ; must load VRs in this line
2392 vr_ld32test14: ; here to handle next cache line
2393 la r11,savevr14(r3) ; get offset to next cache line
2394 bt 14,vr_ld32load14 ; skip if we must load this line
2395 vor v14,v31,v31 ; neither VR is live, so bug them both
2398 vr_ld32load14: ; must load VRs in this line
2402 vr_ld32test16: ; here to handle next cache line
2403 la r11,savevr16(r3) ; get offset to next cache line
2404 bt 16,vr_ld32load16 ; skip if we must load this line
2405 vor v16,v31,v31 ; neither VR is live, so bug them both
2408 vr_ld32load16: ; must load VRs in this line
2412 vr_ld32test18: ; here to handle next cache line
2413 la r11,savevr18(r3) ; get offset to next cache line
2414 bt 18,vr_ld32load18 ; skip if we must load this line
2415 vor v18,v31,v31 ; neither VR is live, so bug them both
2418 vr_ld32load18: ; must load VRs in this line
2422 vr_ld32test20: ; here to handle next cache line
2423 la r11,savevr20(r3) ; get offset to next cache line
2424 bt 20,vr_ld32load20 ; skip if we must load this line
2425 vor v20,v31,v31 ; neither VR is live, so bug them both
2428 vr_ld32load20: ; must load VRs in this line
2432 vr_ld32test22: ; here to handle next cache line
2433 la r11,savevr22(r3) ; get offset to next cache line
2434 bt 22,vr_ld32load22 ; skip if we must load this line
2435 vor v22,v31,v31 ; neither VR is live, so bug them both
2438 vr_ld32load22: ; must load VRs in this line
2442 vr_ld32test24: ; here to handle next cache line
2443 la r11,savevr24(r3) ; get offset to next cache line
2444 bt 24,vr_ld32load24 ; skip if we must load this line
2445 vor v24,v31,v31 ; neither VR is live, so bug them both
2448 vr_ld32load24: ; must load VRs in this line
2452 vr_ld32test26: ; here to handle next cache line
2453 la r11,savevr26(r3) ; get offset to next cache line
2454 bt 26,vr_ld32load26 ; skip if we must load this line
2455 vor v26,v31,v31 ; neither VR is live, so bug them both
2458 vr_ld32load26: ; must load VRs in this line
2462 vr_ld32test28: ; here to handle next cache line
2463 la r11,savevr28(r3) ; get offset to next cache line
2464 bt 28,vr_ld32load28 ; skip if we must load this line
2465 vor v28,v31,v31 ; neither VR is live, so bug them both
2468 vr_ld32load28: ; must load VRs in this line
2472 vr_ld32test30: ; here to handle next cache line
2473 la r11,savevr30(r3) ; get offset to next cache line
2474 bt 30,vr_ld32load30 ; skip if we must load this line
2475 vor v30,v31,v31 ; neither VR is live, so bug them both
2477 vr_ld32load30: ; must load VRs in this line
projects / apple / xnu.git / blob