2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
27 #include <ppc/proc_reg.h>
31 #include <mach/ppc/vm_param.h>
32 #include <ppc/exception.h>
33 #include <ppc/savearea.h>
41 * void machine_load_context(thread_t thread)
43 * Load the context for the first thread to run on a
48 .globl EXT(machine_load_context)
50 LEXT(machine_load_context)
52 lwz r0,PP_INTSTACK_TOP_SS(r6)
53 stw r0,PP_ISTACKPTR(r6)
54 lwz r9,THREAD_TOP_ACT(r3) /* Set up the current thread */
56 li r0,0 /* Clear a register */
57 lwz r3,ACT_MACT_PCB(r9) /* Get the savearea used */
58 mfmsr r5 /* Since we are passing control, get our MSR values */
59 lwz r11,SAVprev+4(r3) /* Get the previous savearea */
60 lwz r1,saver1+4(r3) /* Load new stack pointer */
61 lwz r10,ACT_MACT_SPF(r9) /* Get the special flags */
62 stw r0,saver3+4(r3) /* Make sure we pass in a 0 for the continuation */
63 stw r0,FM_BACKPTR(r1) /* zero backptr */
64 stw r5,savesrr1+4(r3) /* Pass our MSR to the new guy */
65 stw r11,ACT_MACT_PCB(r9) /* Unstack our savearea */
66 oris r10,r10,hi16(OnProc) /* Set OnProc bit */
67 stw r0,ACT_PREEMPT_CNT(r9) /* Enable preemption */
68 stw r10,ACT_MACT_SPF(r9) /* Update the special flags */
69 stw r10,spcFlags(r6) /* Set per_proc copy of the special flags */
70 b EXT(exception_exit) /* Go for it */
72 /* thread_t Switch_context(thread_t old,
76 * Switch from one thread to another. If a continuation is supplied, then
77 * we do not need to save callee save registers.
81 /* void Call_continuation( void (*continuation)(void), vm_offset_t stack_ptr)
85 .globl EXT(Call_continuation)
87 LEXT(Call_continuation)
90 mr r1, r4 /* Load new stack pointer */
91 blr /* Jump to the continuation */
94 * Get the old kernel stack, and store into the thread structure.
95 * See if a continuation is supplied, and skip state save if so.
97 * Note that interrupts must be disabled before we get here (i.e., splsched)
100 /* Context switches are double jumps. We pass the following to the
101 * context switch firmware call:
103 * R3 = switchee's savearea, virtual if continuation, low order physical for full switch
107 * R7 = high order physical address of savearea for full switch
109 * savesrr0 is set to go to switch_in
110 * savesrr1 is set to uninterruptible with translation on
115 .globl EXT(Switch_context)
119 mfsprg r12,0 ; Get the per_proc block
121 lwz r0,PP_ISTACKPTR(r12) ; (DEBUG/TRACE) make sure we are not
122 mr. r0,r0 ; (DEBUG/TRACE) on the interrupt
123 bne++ notonintstack ; (DEBUG/TRACE) stack
127 lwz r5,THREAD_TOP_ACT(r5) ; Get the new activation
128 lwz r8,ACT_MACT_PCB(r5) ; Get the PCB for the new guy
129 lwz r9,cioSpace(r5) ; Get copyin/out address space
130 cmpwi cr1,r4,0 ; Remeber if there is a continuation - used waaaay down below
131 lwz r7,CTHREAD_SELF(r5) ; Pick up the user assist word
132 lwz r11,ACT_MACT_BTE(r5) ; Get BlueBox Task Environment
133 lwz r6,cioRelo(r5) ; Get copyin/out relocation top
135 lwz r2,cioRelo+4(r5) ; Get copyin/out relocation bottom
137 stw r7,UAW(r12) ; Save the assist word for the "ultra fast path"
139 lwz r7,ACT_MACT_SPF(r5) ; Get the special flags
141 sth r9,ppCIOmp+mpSpace(r12) ; Save the space
142 stw r6,ppCIOmp+mpNestReloc(r12) ; Save top part of physical address
143 stw r2,ppCIOmp+mpNestReloc+4(r12) ; Save bottom part of physical address
144 stw r11,ppbbTaskEnv(r12) ; Save the bb task env
145 lwz r2,traceMask(0) ; Get the enabled traces
146 stw r7,spcFlags(r12) ; Set per_proc copy of the special flags
147 lis r0,hi16(CutTrace) ; Trace FW call
148 mr. r2,r2 ; Any tracing going on?
149 lwz r11,SAVprev+4(r8) ; Get the previous of the switchee savearea
150 ori r0,r0,lo16(CutTrace) ; Trace FW call
151 beq++ cswNoTrc ; No trace today, dude...
152 mr r10,r3 ; Save across trace
153 lwz r2,THREAD_TOP_ACT(r3) ; Trace old activation
154 mr r3,r11 ; Trace prev savearea
155 sc ; Cut trace entry of context switch
158 cswNoTrc: lwz r2,curctx(r5) ; Grab our current context pointer
159 lwz r10,FPUowner(r12) ; Grab the owner of the FPU
160 lwz r9,VMXowner(r12) ; Grab the owner of the vector
161 lhz r0,PP_CPU_NUMBER(r12) ; Get our CPU number
162 mfmsr r6 ; Get the MSR because the switched to thread should inherit it
163 stw r11,ACT_MACT_PCB(r5) ; Dequeue the savearea we are switching to
164 li r0,1 ; Get set to hold off quickfret
166 rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off the FP
167 cmplw r10,r2 ; Do we have the live float context?
168 lwz r10,FPUlevel(r2) ; Get the live level
169 mr r4,r3 ; Save our old thread to pass back
170 cmplw cr5,r9,r2 ; Do we have the live vector context?
171 rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off the vector
172 stw r0,holdQFret(r12) ; Make sure we hold off releasing quickfret
173 bne++ cswnofloat ; Float is not ours...
175 cmplw r10,r11 ; Is the level the same?
176 lwz r5,FPUcpu(r2) ; Get the owning cpu
177 bne++ cswnofloat ; Level not the same, this is not live...
179 cmplw r5,r0 ; Still owned by this cpu?
180 lwz r10,FPUsave(r2) ; Get the level
181 bne++ cswnofloat ; CPU claimed by someone else...
183 mr. r10,r10 ; Is there a savearea here?
184 ori r6,r6,lo16(MASK(MSR_FP)) ; Enable floating point
186 beq-- cswnofloat ; No savearea to check...
188 lwz r3,SAVlevel(r10) ; Get the level
189 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
190 cmplw r3,r11 ; Is it for the current level?
192 bne++ cswnofloat ; Nope...
194 stw r5,FPUsave(r2) ; Pop off this savearea
196 rlwinm r3,r10,0,0,19 ; Move back to start of page
198 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
199 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
200 lwz r7,SACvrswap(r3) ; Get the virtual to real conversion (top)
201 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
202 stw r5,SAVprev(r10) ; Link the old in (top)
203 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
204 xor r3,r10,r3 ; Convert to physical
205 stw r7,quickfret(r12) ; Set the first in quickfret list (top)
206 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
209 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
210 mr r7,r2 ; (TEST/DEBUG)
211 li r2,0x4401 ; (TEST/DEBUG)
212 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
214 lhz r0,PP_CPU_NUMBER(r12) ; (TEST/DEBUG)
215 mr r2,r7 ; (TEST/DEBUG)
218 cswnofloat: bne++ cr5,cswnovect ; Vector is not ours...
220 lwz r10,VMXlevel(r2) ; Get the live level
222 cmplw r10,r11 ; Is the level the same?
223 lwz r5,VMXcpu(r2) ; Get the owning cpu
224 bne++ cswnovect ; Level not the same, this is not live...
226 cmplw r5,r0 ; Still owned by this cpu?
227 lwz r10,VMXsave(r2) ; Get the level
228 bne++ cswnovect ; CPU claimed by someone else...
230 mr. r10,r10 ; Is there a savearea here?
231 oris r6,r6,hi16(MASK(MSR_VEC)) ; Enable vector
233 beq-- cswnovect ; No savearea to check...
235 lwz r3,SAVlevel(r10) ; Get the level
236 lwz r5,SAVprev+4(r10) ; Get the previous of this savearea
237 cmplw r3,r11 ; Is it for the current level?
239 bne++ cswnovect ; Nope...
241 stw r5,VMXsave(r2) ; Pop off this savearea
242 rlwinm r3,r10,0,0,19 ; Move back to start of page
244 lwz r5,quickfret(r12) ; Get the first in quickfret list (top)
245 lwz r9,quickfret+4(r12) ; Get the first in quickfret list (bottom)
246 lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
247 lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
248 stw r5,SAVprev(r10) ; Link the old in (top)
249 stw r9,SAVprev+4(r10) ; Link the old in (bottom)
250 xor r3,r10,r3 ; Convert to physical
251 stw r2,quickfret(r12) ; Set the first in quickfret list (top)
252 stw r3,quickfret+4(r12) ; Set the first in quickfret list (bottom)
255 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
256 li r2,0x4501 ; (TEST/DEBUG)
257 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
261 cswnovect: li r0,0 ; Get set to release quickfret holdoff
262 rlwinm r11,r8,0,0,19 ; Switch to savearea base
263 lis r9,hi16(EXT(switch_in)) ; Get top of switch in routine
264 lwz r5,savesrr0+4(r8) ; Set up the new SRR0
265 lwz r7,SACvrswap(r11) ; Get the high order V to R translation
266 lwz r11,SACvrswap+4(r11) ; Get the low order V to R translation
267 ori r9,r9,lo16(EXT(switch_in)) ; Bottom half of switch in
268 stw r0,holdQFret(r12) ; Make sure we release quickfret holdoff
269 stw r9,savesrr0+4(r8) ; Make us jump to the switch in routine
271 lwz r9,SAVflags(r8) /* Get the flags */
272 lis r0,hi16(SwitchContextCall) /* Top part of switch context */
273 li r10,MSR_SUPERVISOR_INT_OFF /* Get the switcher's MSR */
274 ori r0,r0,lo16(SwitchContextCall) /* Bottom part of switch context */
275 stw r10,savesrr1+4(r8) /* Set up for switch in */
276 rlwinm r9,r9,0,15,13 /* Reset the syscall flag */
277 xor r3,r11,r8 /* Get the physical address of the new context save area */
278 stw r9,SAVflags(r8) /* Set the flags */
280 bne cr1,swtchtocont ; Switch to the continuation
281 sc /* Switch to the new context */
283 /* We come back here in the new thread context
284 * R4 was set to hold the old thread pointer, but switch_in will put it into
285 * R3 where it belongs.
287 blr /* Jump into the new thread */
290 ; This is where we go when a continuation is set. We are actually
291 ; killing off the old context of the new guy so we need to pop off
292 ; any float or vector states for the ditched level.
294 ; Note that we do the same kind of thing a chkfac in hw_exceptions.s
300 stw r5,savesrr0+4(r8) ; Set the pc
301 stw r6,savesrr1+4(r8) ; Set the next MSR to use
302 stw r4,saver3+4(r8) ; Make sure we pass back the old thread
303 mr r3,r8 ; Pass in the virtual address of savearea
305 b EXT(exception_exit) ; Blocking on continuation, toss old context...
310 * All switched to threads come here first to clean up the old thread.
311 * We need to do the following contortions because we need to keep
312 * the LR clean. And because we need to manipulate the savearea chain
313 * with translation on. If we could, this should be done in lowmem_vectors
314 * before translation is turned on. But we can't, dang it!
316 * R3 = switcher's savearea (32-bit virtual)
317 * saver4 = old thread in switcher's save
318 * saver5 = new SRR0 in switcher's save
319 * saver6 = new SRR1 in switcher's save
326 .globl EXT(switch_in)
330 lwz r4,saver4+4(r3) ; Get the old thread
331 lwz r5,saver5+4(r3) ; Get the srr0 value
333 mfsprg r0,2 ; Get feature flags
334 lwz r9,THREAD_TOP_ACT(r4) ; Get the switched from ACT
335 lwz r6,saver6+4(r3) ; Get the srr1 value
336 rlwinm. r0,r0,0,pf64Bitb,pf64Bitb ; Check for 64-bit
337 lwz r10,ACT_MACT_PCB(r9) ; Get the top PCB on the old thread
339 stw r3,ACT_MACT_PCB(r9) ; Put the new one on top
340 stw r10,SAVprev+4(r3) ; Chain on the old one
342 mr r3,r4 ; Pass back the old thread
344 mtsrr0 r5 ; Set return point
345 mtsrr1 r6 ; Set return MSR
347 bne++ siSixtyFour ; Go do 64-bit...
355 * void fpu_save(facility_context ctx)
357 * Note that there are some oddities here when we save a context we are using.
358 * It is really not too cool to do this, but what the hey... Anyway,
359 * we turn fpus and vecs off before we leave., The oddity is that if you use fpus after this, the
360 * savearea containing the context just saved will go away. So, bottom line is
361 * that don't use fpus until after you are done with the saved context.
368 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector enable
369 li r12,lo16(MASK(MSR_EE)) ; Get the EE bit
370 ori r2,r2,lo16(MASK(MSR_FP)) ; Get FP
372 mfmsr r0 ; Get the MSR
373 andc r0,r0,r2 ; Clear FP, VEC
374 andc r2,r0,r12 ; Clear EE
375 ori r2,r2,MASK(MSR_FP) ; Enable the floating point feature for now also
376 mtmsr r2 ; Set the MSR
379 mfsprg r6,0 ; Get the per_processor block
380 lwz r12,FPUowner(r6) ; Get the context ID for owner
383 mr r7,r0 ; (TEST/DEBUG)
384 li r4,0 ; (TEST/DEBUG)
385 mr r10,r3 ; (TEST/DEBUG)
386 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
387 mr. r3,r12 ; (TEST/DEBUG)
388 li r2,0x6F00 ; (TEST/DEBUG)
389 li r5,0 ; (TEST/DEBUG)
390 beq-- noowneryet ; (TEST/DEBUG)
391 lwz r4,FPUlevel(r12) ; (TEST/DEBUG)
392 lwz r5,FPUsave(r12) ; (TEST/DEBUG)
394 noowneryet: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
396 mr r0,r7 ; (TEST/DEBUG)
397 mr r3,r10 ; (TEST/DEBUG)
399 mflr r2 ; Save the return address
401 fsretry: mr. r12,r12 ; Anyone own the FPU?
402 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
403 beq-- fsret ; Nobody owns the FPU, no save required...
405 cmplw cr1,r3,r12 ; Is the specified context live?
407 isync ; Force owner check first
409 lwz r9,FPUcpu(r12) ; Get the cpu that context was last on
410 bne-- cr1,fsret ; No, it is not...
412 cmplw cr1,r9,r11 ; Was the context for this processor?
413 beq-- cr1,fsgoodcpu ; Facility last used on this processor...
415 b fsret ; Someone else claimed it...
419 fsgoodcpu: lwz r3,FPUsave(r12) ; Get the current FPU savearea for the thread
420 lwz r9,FPUlevel(r12) ; Get our current level indicator
422 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
423 beq- cr1,fsneedone ; Never saved it, so go do it...
425 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
426 cmplw cr1,r9,r8 ; Correct level?
427 beq-- cr1,fsret ; The current level is already saved, bail out...
429 fsneedone: bl EXT(save_get) ; Get a savearea for the context
431 mfsprg r6,0 ; Get back per_processor block
432 li r4,SAVfloat ; Get floating point tag
433 lwz r12,FPUowner(r6) ; Get back our thread
434 stb r4,SAVflags+2(r3) ; Mark this savearea as a float
435 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
436 beq-- fsbackout ; If disowned, just toss savearea...
437 lwz r4,facAct(r12) ; Get the activation associated with live context
438 lwz r8,FPUsave(r12) ; Get the current top floating point savearea
439 stw r4,SAVact(r3) ; Indicate the right activation for this context
440 lwz r9,FPUlevel(r12) ; Get our current level indicator again
441 stw r3,FPUsave(r12) ; Set this as the most current floating point context
442 stw r8,SAVprev+4(r3) ; And then chain this in front
444 stw r9,SAVlevel(r3) ; Show level in savearea
446 bl fp_store ; save all 32 FPRs in the save area at r3
447 mtlr r2 ; Restore return
449 fsret: mtmsr r0 ; Put interrupts on if they were and floating point off
454 fsbackout: mr r4,r0 ; restore the original MSR
455 b EXT(save_ret_wMSR) ; Toss savearea and return from there...
460 * Entered to handle the floating-point unavailable exception and
463 * This code is run in virtual address mode on with interrupts off.
465 * Upon exit, the code returns to the users context with the floating
466 * point facility turned on.
468 * ENTRY: VM switched ON
470 * State is saved in savearea pointed to by R4.
471 * All other registers are free.
476 .globl EXT(fpu_switch)
481 lis r3,hi16(EXT(fpu_trap_count)) ; Get address of FP trap counter
482 ori r3,r3,lo16(EXT(fpu_trap_count)) ; Get address of FP trap counter
488 mfsprg r26,0 ; Get the per_processor block
489 mfmsr r19 ; Get the current MSR
490 mfsprg r17,1 ; Get the current thread
492 mr r25,r4 ; Save the entry savearea
493 lwz r22,FPUowner(r26) ; Get the thread that owns the FPU
494 ori r19,r19,lo16(MASK(MSR_FP)) ; Enable the floating point feature
496 mtmsr r19 ; Enable floating point instructions
499 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
500 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
502 ; R22 has the "old" context anchor
503 ; R29 has the "new" context anchor
506 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
507 li r2,0x7F01 ; (TEST/DEBUG)
508 mr r3,r22 ; (TEST/DEBUG)
509 mr r5,r29 ; (TEST/DEBUG)
510 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
514 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
516 fswretry: mr. r22,r22 ; See if there is any live FP status
518 beq- fsnosave ; No live context, so nothing to save...
520 isync ; Make sure we see this in the right order
522 lwz r30,FPUsave(r22) ; Get the top savearea
523 cmplw cr2,r22,r29 ; Are both old and new the same context?
524 lwz r18,FPUcpu(r22) ; Get the last CPU we ran on
525 cmplwi cr1,r30,0 ; Anything saved yet?
526 cmplw r18,r16 ; Make sure we are on the right processor
527 lwz r31,FPUlevel(r22) ; Get the context level
529 bne- fsnosave ; No, not on the same processor...
532 ; Check to see if the live context has already been saved.
533 ; Also check to see if all we are here just to re-enable the MSR
534 ; and handle specially if so.
537 cmplw r31,r27 ; See if the current and active levels are the same
538 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
541 beq- fsthesame ; New and old are the same, just go enable...
543 beq- cr1,fsmstsave ; Not saved yet, go do it...
545 lwz r11,SAVlevel(r30) ; Get the level of top saved context
547 cmplw r31,r11 ; Are live and saved the same?
550 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
551 li r2,0x7F02 ; (TEST/DEBUG)
552 mr r3,r30 ; (TEST/DEBUG)
553 mr r5,r31 ; (TEST/DEBUG)
554 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
556 li r3,0 ; (TEST/DEBUG)
559 beq+ fsnosave ; Same level, so already saved...
562 fsmstsave: stw r3,FPUowner(r26) ; Kill the context now
563 eieio ; Make sure everyone sees it
564 bl EXT(save_get) ; Go get a savearea
566 mr. r31,r31 ; Are we saving the user state?
567 la r15,FPUsync(r22) ; Point to the sync word
568 beq++ fswusave ; Yeah, no need for lock...
570 ; Here we make sure that the live context is not tossed while we are
571 ; trying to push it. This can happen only for kernel context and
572 ; then only by a race with act_machine_sv_free.
574 ; We only need to hold this for a very short time, so no sniffing needed.
575 ; If we find any change to the level, we just abandon.
577 fswsync: lwarx r19,0,r15 ; Get the sync word
578 li r0,1 ; Get the lock
579 cmplwi cr1,r19,0 ; Is it unlocked?
580 stwcx. r0,0,r15 ; Store lock and test reservation
581 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
582 bne-- fswsync ; Try again if lost reservation or locked...
584 isync ; Toss speculation
586 lwz r0,FPUlevel(r22) ; Pick up the level again
587 li r7,0 ; Get unlock value
588 cmplw r0,r31 ; Same level?
589 beq++ fswusave ; Yeah, we expect it to be...
591 stw r7,FPUsync(r22) ; Unlock lock. No need to sync here
593 bl EXT(save_ret) ; Toss save area because we are abandoning save
594 b fsnosave ; Skip the save...
598 fswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
599 stw r3,FPUsave(r22) ; Set this as the latest context savearea for the thread
600 mr. r31,r31 ; Check again if we were user level
601 stw r30,SAVprev+4(r3) ; Point us to the old context
602 stw r31,SAVlevel(r3) ; Tag our level
603 li r7,SAVfloat ; Get the floating point ID
604 stw r12,SAVact(r3) ; Make sure we point to the right guy
605 stb r7,SAVflags+2(r3) ; Set that we have a floating point save area
607 li r7,0 ; Get the unlock value
609 beq-- fswnulock ; Skip unlock if user (we did not lock it)...
610 eieio ; Make sure that these updates make it out
611 stw r7,FPUsync(r22) ; Unlock it.
616 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
617 li r2,0x7F03 ; (TEST/DEBUG)
618 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
622 bl fp_store ; store all 32 FPRs
625 ; The context is all saved now and the facility is free.
627 ; If we do not we need to fill the registers with junk, because this level has
628 ; never used them before and some thieving bastard could hack the old values
629 ; of some thread! Just imagine what would happen if they could! Why, nothing
630 ; would be safe! My God! It is terrifying!
634 fsnosave: lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
635 lwz r19,FPUcpu(r29) ; Get the last CPU we ran on
636 lwz r14,FPUsave(r29) ; Point to the top of the "new" context stack
638 stw r16,FPUcpu(r29) ; Claim context for us
642 lwz r13,FPUlevel(r29) ; (TEST/DEBUG)
643 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
644 li r2,0x7F04 ; (TEST/DEBUG)
645 mr r1,r15 ; (TEST/DEBUG)
646 mr r3,r14 ; (TEST/DEBUG)
647 mr r5,r13 ; (TEST/DEBUG)
648 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
652 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
653 mulli r19,r19,ppSize ; Find offset to the owner per_proc
654 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
655 li r16,FPUowner ; Displacement to float owner
656 add r19,r18,r19 ; Point to the owner per_proc
658 fsinvothr: lwarx r18,r16,r19 ; Get the owner
659 sub r0,r18,r29 ; Subtract one from the other
660 sub r11,r29,r18 ; Subtract the other from the one
661 or r11,r11,r0 ; Combine them
662 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
663 and r18,r18,r11 ; Make 0 if same, unchanged if not
664 stwcx. r18,r16,r19 ; Try to invalidate it
665 bne-- fsinvothr ; Try again if there was a collision...
667 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
668 la r11,savefp0(r14) ; Point to first line to bring in
669 stw r15,FPUlevel(r29) ; Set the "new" active level
671 stw r29,FPUowner(r26) ; Mark us as having the live context
673 beq++ cr1,MakeSureThatNoTerroristsCanHurtUsByGod ; No "new" context to load...
675 dcbt 0,r11 ; Touch line in
677 lwz r3,SAVprev+4(r14) ; Get the previous context
678 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
679 cmplw r0,r15 ; Top level correct to load?
680 bne-- MakeSureThatNoTerroristsCanHurtUsByGod ; No, go initialize...
682 stw r3,FPUsave(r29) ; Pop the context (we will toss the savearea later)
685 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
686 li r2,0x7F05 ; (TEST/DEBUG)
687 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
691 // Note this code is used both by 32- and 128-byte processors. This means six extra DCBTs
692 // are executed on a 128-byte machine, but that is better than a mispredicted branch.
694 la r11,savefp4(r14) ; Point to next line
695 dcbt 0,r11 ; Touch line in
699 la r11,savefp8(r14) ; Point to next line
701 dcbt 0,r11 ; Touch line in
705 la r11,savefp12(r14) ; Point to next line
707 dcbt 0,r11 ; Touch line in
710 lfd f10,savefp10(r14)
711 la r11,savefp16(r14) ; Point to next line
712 lfd f11,savefp11(r14)
713 dcbt 0,r11 ; Touch line in
714 lfd f12,savefp12(r14)
715 lfd f13,savefp13(r14)
716 lfd f14,savefp14(r14)
717 la r11,savefp20(r14) ; Point to next line
718 lfd f15,savefp15(r14)
719 dcbt 0,r11 ; Touch line in
720 lfd f16,savefp16(r14)
721 lfd f17,savefp17(r14)
722 lfd f18,savefp18(r14)
723 la r11,savefp24(r14) ; Point to next line
724 lfd f19,savefp19(r14)
725 dcbt 0,r11 ; Touch line in
726 lfd f20,savefp20(r14)
727 lfd f21,savefp21(r14)
728 la r11,savefp28(r14) ; Point to next line
729 lfd f22,savefp22(r14)
730 lfd f23,savefp23(r14)
731 dcbt 0,r11 ; Touch line in
732 lfd f24,savefp24(r14)
733 lfd f25,savefp25(r14)
734 lfd f26,savefp26(r14)
735 lfd f27,savefp27(r14)
736 lfd f28,savefp28(r14)
737 lfd f29,savefp29(r14)
738 lfd f30,savefp30(r14)
739 lfd f31,savefp31(r14)
741 mr r3,r14 ; Get the old savearea (we popped it before)
742 bl EXT(save_ret) ; Toss it
744 fsenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
745 ori r8,r8,MASK(MSR_FP) ; Enable the floating point feature
746 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
747 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
748 oris r10,r10,hi16(floatUsed|floatCng) ; Set that we used floating point
749 oris r11,r11,hi16(floatUsed|floatCng) ; Set that we used floating point
750 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
751 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
752 mr r3,r25 ; Pass the virtual addres of savearea
753 beq- fsnuser ; We are not user state...
754 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
755 stw r11,spcFlags(r26) ; Set per_proc copy
759 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
760 li r2,0x7F07 ; (TEST/DEBUG)
761 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
765 b EXT(exception_exit) ; Exit to the fray...
768 * Initialize the registers to some bogus value
771 MakeSureThatNoTerroristsCanHurtUsByGod:
774 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
775 li r2,0x7F06 ; (TEST/DEBUG)
776 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
779 lis r5,hi16(EXT(FloatInit)) ; Get top secret floating point init value address
780 ori r5,r5,lo16(EXT(FloatInit)) ; Slam bottom
781 lfd f0,0(r5) ; Initialize FP0
782 fmr f1,f0 ; Do them all
813 b fsenable ; Finish setting it all up...
817 ; We get here when we are switching to the same context at the same level and the context
818 ; is still live. Essentially, all we are doing is turning on the faility. It may have
819 ; gotten turned off due to doing a context save for the current level or a context switch
820 ; back to the live guy.
828 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
829 li r2,0x7F0A ; (TEST/DEBUG)
830 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
833 beq- cr1,fsenable ; Not saved yet, nothing to pop, go enable and exit...
835 lwz r11,SAVlevel(r30) ; Get the level of top saved context
836 lwz r14,SAVprev+4(r30) ; Get the previous savearea
838 cmplw r11,r31 ; Are live and saved the same?
840 bne+ fsenable ; Level not the same, nothing to pop, go enable and exit...
842 mr r3,r30 ; Get the old savearea (we popped it before)
843 stw r14,FPUsave(r22) ; Pop the savearea from the stack
844 bl EXT(save_ret) ; Toss it
845 b fsenable ; Go enable and exit...
849 ; This function invalidates any live floating point context for the passed in facility_context.
850 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
854 .globl EXT(toss_live_fpu)
858 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
859 mfmsr r9 ; Get the MSR
860 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
861 rlwinm. r8,r9,0,MSR_FP_BIT,MSR_FP_BIT ; Are floats on right now?
862 andc r9,r9,r0 ; Force off VEC and FP
863 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
864 andc r0,r9,r0 ; Turn off EE now
865 mtmsr r0 ; No interruptions
867 beq+ tlfnotours ; Floats off, can not be live here...
869 mfsprg r8,0 ; Get the per proc
872 ; Note that at this point, since floats are on, we are the owner
873 ; of live state on this processor
876 lwz r6,FPUowner(r8) ; Get the thread that owns the floats
877 li r0,0 ; Clear this just in case we need it
878 cmplw r6,r3 ; Are we tossing our own context?
879 bne-- tlfnotours ; Nope...
881 lfd f1,Zero(0) ; Make a 0
882 mtfsf 0xFF,f1 ; Clear it
884 tlfnotours: lwz r11,FPUcpu(r3) ; Get the cpu on which we last loaded context
885 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
886 mulli r11,r11,ppSize ; Find offset to the owner per_proc
887 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
888 li r10,FPUowner ; Displacement to float owner
889 add r11,r12,r11 ; Point to the owner per_proc
891 tlfinvothr: lwarx r12,r10,r11 ; Get the owner
893 sub r0,r12,r3 ; Subtract one from the other
894 sub r8,r3,r12 ; Subtract the other from the one
895 or r8,r8,r0 ; Combine them
896 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
897 and r12,r12,r8 ; Make 0 if same, unchanged if not
898 stwcx. r12,r10,r11 ; Try to invalidate it
899 bne-- tlfinvothr ; Try again if there was a collision...
901 mtmsr r9 ; Restore interruptions
902 isync ; Could be turning off floats here
907 * Altivec stuff is here. The techniques used are pretty identical to
908 * the floating point. Except that we will honor the VRSAVE register
909 * settings when loading and restoring registers.
911 * There are two indications of saved VRs: the VRSAVE register and the vrvalid
912 * mask. VRSAVE is set by the vector user and represents the VRs that they
913 * say that they are using. The vrvalid mask indicates which vector registers
914 * are saved in the savearea. Whenever context is saved, it is saved according
915 * to the VRSAVE register. It is loaded based on VRSAVE anded with
916 * vrvalid (all other registers are splatted with 0s). This is done because we
917 * don't want to load any registers we don't have a copy of, we want to set them
920 * Note that there are some oddities here when we save a context we are using.
921 * It is really not too cool to do this, but what the hey... Anyway,
922 * we turn vectors and fpu off before we leave.
923 * The oddity is that if you use vectors after this, the
924 * savearea containing the context just saved will go away. So, bottom line is
925 * that don't use vectors until after you are done with the saved context.
935 lis r2,hi16(MASK(MSR_VEC)) ; Get VEC
936 mfmsr r0 ; Get the MSR
937 ori r2,r2,lo16(MASK(MSR_FP)) ; Add in FP
938 andc r0,r0,r2 ; Force off VEC and FP
939 ori r2,r2,lo16(MASK(MSR_EE)) ; Clear EE
940 andc r2,r0,r2 ; Clear EE for now
941 oris r2,r2,hi16(MASK(MSR_VEC)) ; Enable the vector facility for now also
942 mtmsr r2 ; Set the MSR
945 mfsprg r6,0 ; Get the per_processor block
946 lwz r12,VMXowner(r6) ; Get the context ID for owner
949 mr r7,r0 ; (TEST/DEBUG)
950 li r4,0 ; (TEST/DEBUG)
951 mr r10,r3 ; (TEST/DEBUG)
952 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
953 mr. r3,r12 ; (TEST/DEBUG)
954 li r2,0x5F00 ; (TEST/DEBUG)
955 li r5,0 ; (TEST/DEBUG)
956 beq- noowneryeu ; (TEST/DEBUG)
957 lwz r4,VMXlevel(r12) ; (TEST/DEBUG)
958 lwz r5,VMXsave(r12) ; (TEST/DEBUG)
960 noowneryeu: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
962 mr r0,r7 ; (TEST/DEBUG)
963 mr r3,r10 ; (TEST/DEBUG)
965 mflr r2 ; Save the return address
967 vsretry: mr. r12,r12 ; Anyone own the vector?
968 lhz r11,PP_CPU_NUMBER(r6) ; Get our CPU number
969 beq- vsret ; Nobody owns the vector, no save required...
971 cmplw cr1,r3,r12 ; Is the specified context live?
973 isync ; Force owner check first
975 lwz r9,VMXcpu(r12) ; Get the cpu that context was last on
976 bne- cr1,vsret ; Specified context is not live
978 cmplw cr1,r9,r11 ; Was the context for this processor?
979 beq+ cr1,vsgoodcpu ; Facility last used on this processor...
981 b vsret ; Someone else claimed this...
985 vsgoodcpu: lwz r3,VMXsave(r12) ; Get the current vector savearea for the thread
986 lwz r10,liveVRS(r6) ; Get the right VRSave register
987 lwz r9,VMXlevel(r12) ; Get our current level indicator
990 cmplwi cr1,r3,0 ; Have we ever saved this facility context?
991 beq- cr1,vsneedone ; Never saved it, so we need an area...
993 lwz r8,SAVlevel(r3) ; Get the level this savearea is for
994 mr. r10,r10 ; Is VRsave set to 0?
995 cmplw cr1,r9,r8 ; Correct level?
996 bne- cr1,vsneedone ; Different level, so we need to save...
998 bne+ vsret ; VRsave is non-zero so we need to keep what is saved...
1000 lwz r4,SAVprev+4(r3) ; Pick up the previous area
1001 lwz r5,SAVlevel(r4) ; Get the level associated with save
1002 stw r4,VMXsave(r12) ; Dequeue this savearea
1004 stw r5,VMXlevel(r12) ; Save the level
1006 stw r4,VMXowner(r12) ; Show no live context here
1009 vsbackout: mr r4,r0 ; restore the saved MSR
1010 b EXT(save_ret_wMSR) ; Toss the savearea and return from there...
1014 vsneedone: mr. r10,r10 ; Is VRsave set to 0?
1015 beq- vsret ; Yeah, they do not care about any of them...
1017 bl EXT(save_get) ; Get a savearea for the context
1019 mfsprg r6,0 ; Get back per_processor block
1020 li r4,SAVvector ; Get vector tag
1021 lwz r12,VMXowner(r6) ; Get back our context ID
1022 stb r4,SAVflags+2(r3) ; Mark this savearea as a vector
1023 mr. r12,r12 ; See if we were disowned while away. Very, very small chance of it...
1024 beq- vsbackout ; If disowned, just toss savearea...
1025 lwz r4,facAct(r12) ; Get the activation associated with live context
1026 lwz r8,VMXsave(r12) ; Get the current top vector savearea
1027 stw r4,SAVact(r3) ; Indicate the right activation for this context
1028 lwz r9,VMXlevel(r12) ; Get our current level indicator again
1029 stw r3,VMXsave(r12) ; Set this as the most current floating point context
1030 stw r8,SAVprev+4(r3) ; And then chain this in front
1032 stw r9,SAVlevel(r3) ; Set level in savearea
1033 mfcr r12 ; save CRs across call to vr_store
1034 lwz r10,liveVRS(r6) ; Get the right VRSave register
1036 bl vr_store ; store live VRs into savearea as required (uses r4-r11)
1038 mtcrf 255,r12 ; Restore the non-volatile CRs
1039 mtlr r2 ; restore return address
1041 vsret: mtmsr r0 ; Put interrupts on if they were and vector off
1049 * Entered to handle the vector unavailable exception and
1050 * switch vector context
1052 * This code is run with virtual address mode on and interrupts off.
1054 * Upon exit, the code returns to the users context with the vector
1055 * facility turned on.
1057 * ENTRY: VM switched ON
1059 * State is saved in savearea pointed to by R4.
1060 * All other registers are free.
1065 .globl EXT(vec_switch)
1070 lis r3,hi16(EXT(vec_trap_count)) ; Get address of vector trap counter
1071 ori r3,r3,lo16(EXT(vec_trap_count)) ; Get address of vector trap counter
1077 mfsprg r26,0 ; Get the per_processor block
1078 mfmsr r19 ; Get the current MSR
1079 mfsprg r17,1 ; Get the current thread
1081 mr r25,r4 ; Save the entry savearea
1082 oris r19,r19,hi16(MASK(MSR_VEC)) ; Enable the vector feature
1083 lwz r22,VMXowner(r26) ; Get the thread that owns the vector
1085 mtmsr r19 ; Enable vector instructions
1088 lwz r27,ACT_MACT_PCB(r17) ; Get the current level
1089 lwz r29,curctx(r17) ; Grab the current context anchor of the current thread
1091 ; R22 has the "old" context anchor
1092 ; R29 has the "new" context anchor
1095 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
1096 li r2,0x5F01 ; (TEST/DEBUG)
1097 mr r3,r22 ; (TEST/DEBUG)
1098 mr r5,r29 ; (TEST/DEBUG)
1099 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
1103 lhz r16,PP_CPU_NUMBER(r26) ; Get the current CPU number
1105 vsvretry: mr. r22,r22 ; See if there is any live vector status
1107 beq- vsnosave ; No live context, so nothing to save...
1109 isync ; Make sure we see this in the right order
1111 lwz r30,VMXsave(r22) ; Get the top savearea
1112 cmplw cr2,r22,r29 ; Are both old and new the same context?
1113 lwz r18,VMXcpu(r22) ; Get the last CPU we ran on
1114 cmplwi cr1,r30,0 ; Anything saved yet?
1115 cmplw r18,r16 ; Make sure we are on the right processor
1116 lwz r31,VMXlevel(r22) ; Get the context level
1118 lwz r10,liveVRS(r26) ; Get the right VRSave register
1120 bne- vsnosave ; No, not on the same processor...
1123 ; Check to see if the live context has already been saved.
1124 ; Also check to see if all we are here just to re-enable the MSR
1125 ; and handle specially if so.
1128 cmplw r31,r27 ; See if the current and active levels are the same
1129 crand cr0_eq,cr2_eq,cr0_eq ; Remember if both the levels and contexts are the same
1130 li r8,0 ; Clear this
1132 beq- vsthesame ; New and old are the same, just go enable...
1134 cmplwi cr2,r10,0 ; Check VRSave to see if we really need to save anything...
1135 beq- cr1,vsmstsave ; Not saved yet, go do it...
1137 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1139 cmplw r31,r11 ; Are live and saved the same?
1142 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1143 li r2,0x5F02 ; (TEST/DEBUG)
1144 mr r3,r30 ; (TEST/DEBUG)
1145 mr r5,r31 ; (TEST/DEBUG)
1146 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1150 bne- vsmstsave ; Live context has not been saved yet...
1152 bne- cr2,vsnosave ; Live context saved and VRSave not 0, no save and keep context...
1154 lwz r4,SAVprev+4(r30) ; Pick up the previous area
1155 li r5,0 ; Assume this is the only one (which should be the ususal case)
1156 mr. r4,r4 ; Was this the only one?
1157 stw r4,VMXsave(r22) ; Dequeue this savearea
1158 beq+ vsonlyone ; This was the only one...
1159 lwz r5,SAVlevel(r4) ; Get the level associated with previous save
1161 vsonlyone: stw r5,VMXlevel(r22) ; Save the level
1162 stw r8,VMXowner(r26) ; Clear owner
1164 mr r3,r30 ; Copy the savearea we are tossing
1165 bl EXT(save_ret) ; Toss the savearea
1166 b vsnosave ; Go load up the context...
1171 vsmstsave: stw r8,VMXowner(r26) ; Clear owner
1173 beq- cr2,vsnosave ; The VRSave was 0, so there is nothing to save...
1175 bl EXT(save_get) ; Go get a savearea
1177 mr. r31,r31 ; Are we saving the user state?
1178 la r15,VMXsync(r22) ; Point to the sync word
1179 beq++ vswusave ; Yeah, no need for lock...
1181 ; Here we make sure that the live context is not tossed while we are
1182 ; trying to push it. This can happen only for kernel context and
1183 ; then only by a race with act_machine_sv_free.
1185 ; We only need to hold this for a very short time, so no sniffing needed.
1186 ; If we find any change to the level, we just abandon.
1188 vswsync: lwarx r19,0,r15 ; Get the sync word
1189 li r0,1 ; Get the lock
1190 cmplwi cr1,r19,0 ; Is it unlocked?
1191 stwcx. r0,0,r15 ; Store lock and test reservation
1192 cror cr0_eq,cr1_eq,cr0_eq ; Combine lost reservation and previously locked
1193 bne-- vswsync ; Try again if lost reservation or locked...
1195 isync ; Toss speculation
1197 lwz r0,VMXlevel(r22) ; Pick up the level again
1198 li r7,0 ; Get unlock value
1199 cmplw r0,r31 ; Same level?
1200 beq++ vswusave ; Yeah, we expect it to be...
1202 stw r7,VMXsync(r22) ; Unlock lock. No need to sync here
1204 bl EXT(save_ret) ; Toss save area because we are abandoning save
1205 b vsnosave ; Skip the save...
1209 vswusave: lwz r12,facAct(r22) ; Get the activation associated with the context
1210 stw r3,VMXsave(r22) ; Set this as the latest context savearea for the thread
1211 mr. r31,r31 ; Check again if we were user level
1212 stw r30,SAVprev+4(r3) ; Point us to the old context
1213 stw r31,SAVlevel(r3) ; Tag our level
1214 li r7,SAVvector ; Get the vector ID
1215 stw r12,SAVact(r3) ; Make sure we point to the right guy
1216 stb r7,SAVflags+2(r3) ; Set that we have a vector save area
1218 li r7,0 ; Get the unlock value
1220 beq-- vswnulock ; Skip unlock if user (we did not lock it)...
1221 eieio ; Make sure that these updates make it out
1222 stw r7,VMXsync(r22) ; Unlock it.
1227 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1228 li r2,0x5F03 ; (TEST/DEBUG)
1229 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1233 lwz r10,liveVRS(r26) ; Get the right VRSave register
1234 bl vr_store ; store VRs into savearea according to vrsave (uses r4-r11)
1238 ; The context is all saved now and the facility is free.
1240 ; If we do not we need to fill the registers with junk, because this level has
1241 ; never used them before and some thieving bastard could hack the old values
1242 ; of some thread! Just imagine what would happen if they could! Why, nothing
1243 ; would be safe! My God! It is terrifying!
1245 ; Also, along the way, thanks to Ian Ollmann, we generate the 0x7FFFDEAD (QNaNbarbarian)
1246 ; constant that we may need to fill unused vector registers.
1252 vsnosave: vspltisb v31,-10 ; Get 0xF6F6F6F6
1253 lwz r15,ACT_MACT_PCB(r17) ; Get the current level of the "new" one
1254 vspltisb v30,5 ; Get 0x05050505
1255 lwz r19,VMXcpu(r29) ; Get the last CPU we ran on
1256 vspltish v29,4 ; Get 0x00040004
1257 lwz r14,VMXsave(r29) ; Point to the top of the "new" context stack
1258 vrlb v31,v31,v30 ; Get 0xDEDEDEDE
1260 stw r16,VMXcpu(r29) ; Claim context for us
1264 lwz r13,VMXlevel(r29) ; (TEST/DEBUG)
1265 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1266 li r2,0x5F04 ; (TEST/DEBUG)
1267 mr r1,r15 ; (TEST/DEBUG)
1268 mr r3,r14 ; (TEST/DEBUG)
1269 mr r5,r13 ; (TEST/DEBUG)
1270 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1274 lis r18,hi16(EXT(per_proc_info)) ; Set base per_proc
1275 vspltisb v28,-2 ; Get 0xFEFEFEFE
1276 mulli r19,r19,ppSize ; Find offset to the owner per_proc
1277 vsubuhm v31,v31,v29 ; Get 0xDEDADEDA
1278 ori r18,r18,lo16(EXT(per_proc_info)) ; Set base per_proc
1279 vpkpx v30,v28,v3 ; Get 0x7FFF7FFF
1280 li r16,VMXowner ; Displacement to vector owner
1281 add r19,r18,r19 ; Point to the owner per_proc
1282 vrlb v31,v31,v29 ; Get 0xDEADDEAD
1284 vsinvothr: lwarx r18,r16,r19 ; Get the owner
1286 sub r0,r18,r29 ; Subtract one from the other
1287 sub r11,r29,r18 ; Subtract the other from the one
1288 or r11,r11,r0 ; Combine them
1289 srawi r11,r11,31 ; Get a 0 if equal or -1 of not
1290 and r18,r18,r11 ; Make 0 if same, unchanged if not
1291 stwcx. r18,r16,r19 ; Try to invalidate it
1292 bne-- vsinvothr ; Try again if there was a collision...
1294 cmplwi cr1,r14,0 ; Do we possibly have some context to load?
1295 vmrghh v31,v30,v31 ; Get 0x7FFFDEAD. V31 keeps this value until the bitter end
1296 stw r15,VMXlevel(r29) ; Set the "new" active level
1298 stw r29,VMXowner(r26) ; Mark us as having the live context
1300 beq-- cr1,ProtectTheAmericanWay ; Nothing to restore, first time use...
1302 lwz r3,SAVprev+4(r14) ; Get the previous context
1303 lwz r0,SAVlevel(r14) ; Get the level of first facility savearea
1304 cmplw r0,r15 ; Top level correct to load?
1305 bne-- ProtectTheAmericanWay ; No, go initialize...
1307 stw r3,VMXsave(r29) ; Pop the context (we will toss the savearea later)
1310 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1311 li r2,0x5F05 ; (TEST/DEBUG)
1312 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1316 lwz r10,savevrvalid(r14) ; Get the valid VRs in the savearea
1317 lwz r22,savevrsave(r25) ; Get the most current VRSAVE
1318 and r10,r10,r22 ; Figure out just what registers need to be loaded
1319 mr r3,r14 ; r3 <- ptr to savearea with VRs
1320 bl vr_load ; load VRs from save area based on vrsave in r10
1322 bl EXT(save_ret) ; Toss the save area after loading VRs
1324 vrenable: lwz r8,savesrr1+4(r25) ; Get the msr of the interrupted guy
1325 oris r8,r8,hi16(MASK(MSR_VEC)) ; Enable the vector facility
1326 lwz r10,ACT_MACT_SPF(r17) ; Get the act special flags
1327 lwz r11,spcFlags(r26) ; Get per_proc spec flags cause not in sync with act
1328 oris r10,r10,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1329 oris r11,r11,hi16(vectorUsed|vectorCng) ; Set that we used vectors
1330 rlwinm. r0,r8,0,MSR_PR_BIT,MSR_PR_BIT ; See if we are doing this for user state
1331 stw r8,savesrr1+4(r25) ; Set the msr of the interrupted guy
1332 mr r3,r25 ; Pass virtual address of the savearea
1333 beq- vrnuser ; We are not user state...
1334 stw r10,ACT_MACT_SPF(r17) ; Set the activation copy
1335 stw r11,spcFlags(r26) ; Set per_proc copy
1339 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1340 li r2,0x5F07 ; (TEST/DEBUG)
1341 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1344 b EXT(exception_exit) ; Exit to the fray...
1347 * Initialize the registers to some bogus value
1350 ProtectTheAmericanWay:
1353 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1354 li r2,0x5F06 ; (TEST/DEBUG)
1355 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1359 vor v0,v31,v31 ; Copy into the next register
1360 vor v1,v31,v31 ; Copy into the next register
1361 vor v2,v31,v31 ; Copy into the next register
1362 vor v3,v31,v31 ; Copy into the next register
1363 vor v4,v31,v31 ; Copy into the next register
1364 vor v5,v31,v31 ; Copy into the next register
1365 vor v6,v31,v31 ; Copy into the next register
1366 vor v7,v31,v31 ; Copy into the next register
1367 vor v8,v31,v31 ; Copy into the next register
1368 vor v9,v31,v31 ; Copy into the next register
1369 vor v10,v31,v31 ; Copy into the next register
1370 vor v11,v31,v31 ; Copy into the next register
1371 vor v12,v31,v31 ; Copy into the next register
1372 vor v13,v31,v31 ; Copy into the next register
1373 vor v14,v31,v31 ; Copy into the next register
1374 vor v15,v31,v31 ; Copy into the next register
1375 vor v16,v31,v31 ; Copy into the next register
1376 vor v17,v31,v31 ; Copy into the next register
1377 vor v18,v31,v31 ; Copy into the next register
1378 vor v19,v31,v31 ; Copy into the next register
1379 vor v20,v31,v31 ; Copy into the next register
1380 vor v21,v31,v31 ; Copy into the next register
1381 vor v22,v31,v31 ; Copy into the next register
1382 vor v23,v31,v31 ; Copy into the next register
1383 vor v24,v31,v31 ; Copy into the next register
1384 vor v25,v31,v31 ; Copy into the next register
1385 vor v26,v31,v31 ; Copy into the next register
1386 vor v27,v31,v31 ; Copy into the next register
1387 vor v28,v31,v31 ; Copy into the next register
1388 vor v29,v31,v31 ; Copy into the next register
1389 vor v30,v31,v31 ; Copy into the next register
1390 b vrenable ; Finish setting it all up...
1395 ; We get here when we are switching to the same context at the same level and the context
1396 ; is still live. Essentially, all we are doing is turning on the faility. It may have
1397 ; gotten turned off due to doing a context save for the current level or a context switch
1398 ; back to the live guy.
1406 lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1407 li r2,0x5F0A ; (TEST/DEBUG)
1408 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1411 beq- cr1,vrenable ; Not saved yet, nothing to pop, go enable and exit...
1413 lwz r11,SAVlevel(r30) ; Get the level of top saved context
1414 lwz r14,SAVprev+4(r30) ; Get the previous savearea
1416 cmplw r11,r31 ; Are live and saved the same?
1418 bne+ vrenable ; Level not the same, nothing to pop, go enable and exit...
1420 mr r3,r30 ; Get the old savearea (we popped it before)
1421 stw r11,VMXsave(r22) ; Pop the vector stack
1422 bl EXT(save_ret) ; Toss it
1423 b vrenable ; Go enable and exit...
1427 ; This function invalidates any live vector context for the passed in facility_context.
1428 ; This is intended to be called just before act_machine_sv_free tosses saveareas.
1432 .globl EXT(toss_live_vec)
1436 lis r0,hi16(MASK(MSR_VEC)) ; Get VEC
1437 mfmsr r9 ; Get the MSR
1438 ori r0,r0,lo16(MASK(MSR_FP)) ; Add in FP
1439 rlwinm. r8,r9,0,MSR_VEC_BIT,MSR_VEC_BIT ; Are vectors on right now?
1440 andc r9,r9,r0 ; Force off VEC and FP
1441 ori r0,r0,lo16(MASK(MSR_EE)) ; Turn off EE
1442 andc r0,r9,r0 ; Turn off EE now
1443 mtmsr r0 ; No interruptions
1445 beq+ tlvnotours ; Vector off, can not be live here...
1447 mfsprg r8,0 ; Get the per proc
1450 ; Note that at this point, since vecs are on, we are the owner
1451 ; of live state on this processor
1454 lwz r6,VMXowner(r8) ; Get the thread that owns the vector
1455 li r0,0 ; Clear this just in case we need it
1456 cmplw r6,r3 ; Are we tossing our own context?
1457 bne- tlvnotours ; Nope...
1459 vspltish v1,1 ; Turn on the non-Java bit and saturate
1460 vspltisw v0,1 ; Turn on the saturate bit
1461 vxor v1,v1,v0 ; Turn off saturate
1462 mtspr vrsave,r0 ; Clear VRSAVE
1463 mtvscr v1 ; Set the non-java, no saturate status
1465 tlvnotours: lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1466 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1467 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1468 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1469 li r10,VMXowner ; Displacement to vector owner
1470 add r11,r12,r11 ; Point to the owner per_proc
1471 li r0,0 ; Set a 0 to invalidate context
1473 tlvinvothr: lwarx r12,r10,r11 ; Get the owner
1475 sub r0,r12,r3 ; Subtract one from the other
1476 sub r8,r3,r12 ; Subtract the other from the one
1477 or r8,r8,r0 ; Combine them
1478 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1479 and r12,r12,r8 ; Make 0 if same, unchanged if not
1480 stwcx. r12,r10,r11 ; Try to invalidate it
1481 bne-- tlvinvothr ; Try again if there was a collision...
1483 mtmsr r9 ; Restore interruptions
1484 isync ; Could be turning off vectors here
1489 ; This function invalidates any live vector context for the passed in facility_context
1490 ; if the level is current. It also tosses the corresponding savearea if there is one.
1491 ; This function is primarily used whenever we detect a VRSave that is all zeros.
1495 .globl EXT(vec_trash)
1499 lwz r12,facAct(r3) ; Get the activation
1500 lwz r11,VMXlevel(r3) ; Get the context level
1501 lwz r10,ACT_MACT_PCB(r12) ; Grab the current level for the thread
1502 lwz r9,VMXsave(r3) ; Get the savearea, if any
1503 cmplw r10,r11 ; Are we at the right level?
1504 cmplwi cr1,r9,0 ; Remember if there is a savearea
1505 bnelr+ ; No, we do nothing...
1507 lwz r11,VMXcpu(r3) ; Get the cpu on which we last loaded context
1508 lis r12,hi16(EXT(per_proc_info)) ; Set base per_proc
1509 mulli r11,r11,ppSize ; Find offset to the owner per_proc
1510 ori r12,r12,lo16(EXT(per_proc_info)) ; Set base per_proc
1511 li r10,VMXowner ; Displacement to vector owner
1512 add r11,r12,r11 ; Point to the owner per_proc
1514 vtinvothr: lwarx r12,r10,r11 ; Get the owner
1516 sub r0,r12,r3 ; Subtract one from the other
1517 sub r8,r3,r12 ; Subtract the other from the one
1518 or r8,r8,r0 ; Combine them
1519 srawi r8,r8,31 ; Get a 0 if equal or -1 of not
1520 and r12,r12,r8 ; Make 0 if same, unchanged if not
1521 stwcx. r12,r10,r11 ; Try to invalidate it
1522 bne-- vtinvothr ; Try again if there was a collision...
1525 beqlr++ cr1 ; Leave if there is no savearea
1526 lwz r8,SAVlevel(r9) ; Get the level of the savearea
1527 cmplw r8,r11 ; Savearea for the current level?
1528 bnelr++ ; No, nothing to release...
1530 lwz r8,SAVprev+4(r9) ; Pick up the previous area
1531 mr. r8,r8 ; Is there a previous?
1532 beq-- vtnoprev ; Nope...
1533 lwz r7,SAVlevel(r8) ; Get the level associated with save
1535 vtnoprev: stw r8,VMXsave(r3) ; Dequeue this savearea
1536 stw r7,VMXlevel(r3) ; Pop the level
1538 mr r3,r9 ; Get the savearea to release
1539 b EXT(save_ret) ; Go and toss the save area (note, we will return from there)...
1543 ; Just some test code to force vector and/or floating point in the kernel
1547 .globl EXT(fctx_test)
1551 mfsprg r3,1 ; Get the current thread
1552 mr. r3,r3 ; Are we actually up and running?
1555 fmr f0,f0 ; Use floating point
1556 mftb r4 ; Get time base for a random number
1557 li r5,1 ; Get a potential vrsave to use
1558 andi. r4,r4,0x3F ; Get a number from 0 - 63
1559 slw r5,r5,r4 ; Choose a register to save (should be 0 half the time)
1560 mtspr vrsave,r5 ; Set VRSave
1561 vor v0,v0,v0 ; Use vectors
1565 // *******************
1566 // * f p _ s t o r e *
1567 // *******************
1569 // Store FPRs into a save area. Called by fpu_save and fpu_switch.
1572 // floating pt is enabled
1573 // r3 = ptr to save area
1579 mfsprg r11,2 ; get feature flags
1580 mtcrf 0x02,r11 ; put cache line size bits in cr6
1581 la r11,savefp0(r3) ; point to 1st line
1582 dcbz128 0,r11 ; establish 1st line no matter what linesize is
1583 bt-- pf32Byteb,fp_st32 ; skip if a 32-byte machine
1585 // Store the FPRs on a 128-byte machine.
1589 la r11,savefp16(r3) ; Point to the 2nd cache line
1592 dcbz128 0,r11 ; establish 2nd line
1599 stfd f10,savefp10(r3)
1600 stfd f11,savefp11(r3)
1601 stfd f12,savefp12(r3)
1602 stfd f13,savefp13(r3)
1603 stfd f14,savefp14(r3)
1604 stfd f15,savefp15(r3)
1605 stfd f16,savefp16(r3)
1606 stfd f17,savefp17(r3)
1607 stfd f18,savefp18(r3)
1608 stfd f19,savefp19(r3)
1609 stfd f20,savefp20(r3)
1610 stfd f21,savefp21(r3)
1611 stfd f22,savefp22(r3)
1612 stfd f23,savefp23(r3)
1613 stfd f24,savefp24(r3)
1614 stfd f25,savefp25(r3)
1615 stfd f26,savefp26(r3)
1616 stfd f27,savefp27(r3)
1617 stfd f28,savefp28(r3)
1618 stfd f29,savefp29(r3)
1619 stfd f30,savefp30(r3)
1620 stfd f31,savefp31(r3)
1623 // Store FPRs on a 32-byte machine.
1626 la r11,savefp4(r3) ; Point to the 2nd line
1628 dcbz 0,r11 ; Allocate cache
1631 la r11,savefp8(r3) ; Point to the 3rd line
1633 dcbz 0,r11 ; Allocate cache
1637 la r11,savefp12(r3) ; Point to the 4th line
1639 dcbz 0,r11 ; Allocate cache
1642 stfd f10,savefp10(r3)
1643 la r11,savefp16(r3) ; Point to the 5th line
1644 stfd f11,savefp11(r3)
1645 dcbz 0,r11 ; Allocate cache
1646 stfd f12,savefp12(r3)
1647 stfd f13,savefp13(r3)
1648 stfd f14,savefp14(r3)
1649 la r11,savefp20(r3) ; Point to the 6th line
1650 stfd f15,savefp15(r3)
1651 dcbz 0,r11 ; Allocate cache
1652 stfd f16,savefp16(r3)
1653 stfd f17,savefp17(r3)
1654 stfd f18,savefp18(r3)
1655 la r11,savefp24(r3) ; Point to the 7th line
1656 stfd f19,savefp19(r3)
1657 dcbz 0,r11 ; Allocate cache
1658 stfd f20,savefp20(r3)
1660 stfd f21,savefp21(r3)
1661 stfd f22,savefp22(r3)
1662 la r11,savefp28(r3) ; Point to the 8th line
1663 stfd f23,savefp23(r3)
1664 dcbz 0,r11 ; allocate it
1665 stfd f24,savefp24(r3)
1666 stfd f25,savefp25(r3)
1667 stfd f26,savefp26(r3)
1668 stfd f27,savefp27(r3)
1670 stfd f28,savefp28(r3)
1671 stfd f29,savefp29(r3)
1672 stfd f30,savefp30(r3)
1673 stfd f31,savefp31(r3)
1677 // *******************
1678 // * v r _ s t o r e *
1679 // *******************
1681 // Store VRs into savearea, according to bits set in passed vrsave bitfield. This routine is used
1682 // both by vec_save and vec_switch. In order to minimize conditional branches and touching in
1683 // unnecessary cache blocks, we either save all or none of the VRs in a block. We have separate paths
1684 // for each cache block size.
1687 // interrupts are off, vectors are enabled
1688 // r3 = ptr to save area
1689 // r10 = vrsave (not 0)
1692 // r4 - r11, all CRs.
1695 mfsprg r9,2 ; get feature flags
1696 stw r10,savevrvalid(r3) ; Save the validity information in savearea
1697 slwi r8,r10,1 ; Shift over 1
1698 mtcrf 0x02,r9 ; put cache line size bits in cr6 where we can test
1699 or r8,r10,r8 ; r8 <- even bits show which pairs are in use
1700 bt-- pf32Byteb,vr_st32 ; skip if 32-byte cacheline processor
1703 ; Save vectors on a 128-byte linesize processor. We save all or none of the 8 registers in each of
1704 ; the four cache lines. This minimizes mispredicted branches yet handles cache lines optimally.
1706 slwi r7,r8,2 ; shift groups-of-2 over by 2
1707 li r4,16 ; load offsets for X-form stores
1708 or r8,r7,r8 ; show if any in group of 4 are in use
1710 slwi r7,r8,4 ; shift groups-of-4 over by 4
1712 or r11,r7,r8 ; show if any in group of 8 are in use
1714 mtcrf 0x80,r11 ; set CRs one at a time (faster)
1722 bf 0,vr_st64b ; skip if none of vr0-vr7 are in use
1723 la r11,savevr0(r3) ; get address of this group of registers in save area
1724 dcbz128 0,r11 ; zero the line
1725 stvxl v0,0,r11 ; save 8 VRs in the line
1735 bf 8,vr_st64c ; skip if none of vr8-vr15 are in use
1736 la r11,savevr8(r3) ; get address of this group of registers in save area
1737 dcbz128 0,r11 ; zero the line
1738 stvxl v8,0,r11 ; save 8 VRs in the line
1748 bf 16,vr_st64d ; skip if none of vr16-vr23 are in use
1749 la r11,savevr16(r3) ; get address of this group of registers in save area
1750 dcbz128 0,r11 ; zero the line
1751 stvxl v16,0,r11 ; save 8 VRs in the line
1761 bflr 24 ; done if none of vr24-vr31 are in use
1762 la r11,savevr24(r3) ; get address of this group of registers in save area
1763 dcbz128 0,r11 ; zero the line
1764 stvxl v24,0,r11 ; save 8 VRs in the line
1774 ; Save vectors on a 32-byte linesize processor. We save in 16 groups of 2: we either save both
1775 ; or neither in each group. This cuts down on conditional branches.
1776 ; r8 = bitmask with bit n set (for even n) if either of that pair of VRs is in use
1780 mtcrf 0xFF,r8 ; set CR bits so we can branch on them
1781 li r4,16 ; load offset for X-form stores
1783 bf 0,vr_st32b ; skip if neither VR in this pair is in use
1784 la r11,savevr0(r3) ; get address of this group of registers in save area
1785 dcba 0,r11 ; establish the line wo reading it
1786 stvxl v0,0,r11 ; save the two VRs in the line
1790 bf 2,vr_st32c ; skip if neither VR in this pair is in use
1791 la r11,savevr2(r3) ; get address of this group of registers in save area
1792 dcba 0,r11 ; establish the line wo reading it
1793 stvxl v2,0,r11 ; save the two VRs in the line
1797 bf 4,vr_st32d ; skip if neither VR in this pair is in use
1798 la r11,savevr4(r3) ; get address of this group of registers in save area
1799 dcba 0,r11 ; establish the line wo reading it
1800 stvxl v4,0,r11 ; save the two VRs in the line
1804 bf 6,vr_st32e ; skip if neither VR in this pair is in use
1805 la r11,savevr6(r3) ; get address of this group of registers in save area
1806 dcba 0,r11 ; establish the line wo reading it
1807 stvxl v6,0,r11 ; save the two VRs in the line
1811 bf 8,vr_st32f ; skip if neither VR in this pair is in use
1812 la r11,savevr8(r3) ; get address of this group of registers in save area
1813 dcba 0,r11 ; establish the line wo reading it
1814 stvxl v8,0,r11 ; save the two VRs in the line
1818 bf 10,vr_st32g ; skip if neither VR in this pair is in use
1819 la r11,savevr10(r3) ; get address of this group of registers in save area
1820 dcba 0,r11 ; establish the line wo reading it
1821 stvxl v10,0,r11 ; save the two VRs in the line
1825 bf 12,vr_st32h ; skip if neither VR in this pair is in use
1826 la r11,savevr12(r3) ; get address of this group of registers in save area
1827 dcba 0,r11 ; establish the line wo reading it
1828 stvxl v12,0,r11 ; save the two VRs in the line
1832 bf 14,vr_st32i ; skip if neither VR in this pair is in use
1833 la r11,savevr14(r3) ; get address of this group of registers in save area
1834 dcba 0,r11 ; establish the line wo reading it
1835 stvxl v14,0,r11 ; save the two VRs in the line
1839 bf 16,vr_st32j ; skip if neither VR in this pair is in use
1840 la r11,savevr16(r3) ; get address of this group of registers in save area
1841 dcba 0,r11 ; establish the line wo reading it
1842 stvxl v16,0,r11 ; save the two VRs in the line
1846 bf 18,vr_st32k ; skip if neither VR in this pair is in use
1847 la r11,savevr18(r3) ; get address of this group of registers in save area
1848 dcba 0,r11 ; establish the line wo reading it
1849 stvxl v18,0,r11 ; save the two VRs in the line
1853 bf 20,vr_st32l ; skip if neither VR in this pair is in use
1854 la r11,savevr20(r3) ; get address of this group of registers in save area
1855 dcba 0,r11 ; establish the line wo reading it
1856 stvxl v20,0,r11 ; save the two VRs in the line
1860 bf 22,vr_st32m ; skip if neither VR in this pair is in use
1861 la r11,savevr22(r3) ; get address of this group of registers in save area
1862 dcba 0,r11 ; establish the line wo reading it
1863 stvxl v22,0,r11 ; save the two VRs in the line
1867 bf 24,vr_st32n ; skip if neither VR in this pair is in use
1868 la r11,savevr24(r3) ; get address of this group of registers in save area
1869 dcba 0,r11 ; establish the line wo reading it
1870 stvxl v24,0,r11 ; save the two VRs in the line
1874 bf 26,vr_st32o ; skip if neither VR in this pair is in use
1875 la r11,savevr26(r3) ; get address of this group of registers in save area
1876 dcba 0,r11 ; establish the line wo reading it
1877 stvxl v26,0,r11 ; save the two VRs in the line
1881 bf 28,vr_st32p ; skip if neither VR in this pair is in use
1882 la r11,savevr28(r3) ; get address of this group of registers in save area
1883 dcba 0,r11 ; establish the line wo reading it
1884 stvxl v28,0,r11 ; save the two VRs in the line
1888 bflr 30 ; done if neither VR in this pair is in use
1889 la r11,savevr30(r3) ; get address of this group of registers in save area
1890 dcba 0,r11 ; establish the line wo reading it
1891 stvxl v30,0,r11 ; save the two VRs in the line
1896 // *****************
1897 // * v r _ l o a d *
1898 // *****************
1900 // Load live VRs from a savearea, according to bits set in a passed vector. This is the reverse
1901 // of "vr_store". Like it, we avoid touching unnecessary cache blocks and minimize conditional
1902 // branches by loading all VRs from a cache line, if we have to load any. If we don't load the VRs
1903 // in a cache line, we bug them. Note that this behavior is slightly different from earlier kernels,
1904 // which would bug all VRs that aren't live.
1907 // interrupts are off, vectors are enabled
1908 // r3 = ptr to save area
1909 // r10 = vector of live regs to load (ie, savevrsave & savevrvalid, may be 0)
1910 // v31 = bugbug constant (0x7FFFDEAD7FFFDEAD7FFFDEAD7FFFDEAD)
1913 // r4 - r11, all CRs.
1916 mfsprg r9,2 ; get feature flags
1917 li r6,1 ; assuming 32-byte, get (#VRs)-1 in a cacheline
1918 mtcrf 0x02,r9 ; set cache line size bits in cr6
1919 lis r7,0xC000 ; assuming 32-byte, set bits 0-1
1920 bt-- pf32Byteb,vr_ld0 ; skip if 32-bit processor
1921 li r6,7 ; 128-byte machines have 8 VRs in a cacheline
1922 lis r7,0xFF00 ; so set bits 0-7
1924 // Loop touching in cache blocks we will load from.
1925 // r3 = savearea ptr
1926 // r5 = we light bits for the VRs we will be loading
1927 // r6 = 1 if 32-byte, 7 if 128-byte
1928 // r7 = 0xC0000000 if 32-byte, 0xFF000000 if 128-byte
1929 // r10 = live VR bits
1930 // v31 = bugbug constant
1933 li r5,0 ; initialize set of VRs to load
1934 la r11,savevr0(r3) ; get address of register file
1935 b vr_ld2 ; enter loop in middle
1938 vr_ld1: ; loop over each cache line we will load
1939 dcbt r4,r11 ; start prefetch of the line
1940 andc r10,r10,r9 ; turn off the bits in this line
1941 or r5,r5,r9 ; we will load all these
1942 vr_ld2: ; initial entry pt
1943 cntlzw r4,r10 ; get offset to next live VR
1944 andc r4,r4,r6 ; cacheline align it
1945 srw. r9,r7,r4 ; position bits for VRs in that cache line
1946 slwi r4,r4,4 ; get byte offset within register file to that line
1947 bne vr_ld1 ; loop if more bits in r10
1949 bf-- pf128Byteb,vr_ld32 ; skip if not 128-byte lines
1951 // Handle a processor with 128-byte cache lines. Four groups of 8 VRs.
1952 // r3 = savearea ptr
1953 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
1954 // r11 = addr(savevr0)
1955 // v31 = bugbug constant
1957 mtcrf 0x80,r5 ; set up bits for conditional branches
1958 li r4,16 ; load offsets for X-form stores
1960 mtcrf 0x20,r5 ; load CRs ona at a time, which is faster
1969 bt 0,vr_ld128a ; skip if this line must be loaded
1970 vor v0,v31,v31 ; no VR must be loaded, so bug them all
1979 vr_ld128a: ; must load from this line
1989 vr_ld128b: ; here to handle next cache line
1990 la r11,savevr8(r3) ; load offset to it
1991 bt 8,vr_ld128c ; skip if this line must be loaded
1992 vor v8,v31,v31 ; no VR must be loaded, so bug them all
2001 vr_ld128c: ; must load from this line
2011 vr_ld128d: ; here to handle next cache line
2012 la r11,savevr16(r3) ; load offset to it
2013 bt 16,vr_ld128e ; skip if this line must be loaded
2014 vor v16,v31,v31 ; no VR must be loaded, so bug them all
2023 vr_ld128e: ; must load from this line
2033 vr_ld128f: ; here to handle next cache line
2034 la r11,savevr24(r3) ; load offset to it
2035 bt 24,vr_ld128g ; skip if this line must be loaded
2036 vor v24,v31,v31 ; no VR must be loaded, so bug them all
2044 vr_ld128g: ; must load from this line
2055 // Handle a processor with 32-byte cache lines. Sixteen groups of two VRs.
2056 // r5 = 1st bit in each cacheline is 1 iff any reg in that line must be loaded
2057 // r11 = addr(savevr0)
2060 mtcrf 0xFF,r5 ; set up bits for conditional branches
2061 li r4,16 ; load offset for X-form stores
2063 bt 0,vr_ld32load0 ; skip if we must load this line
2064 vor v0,v31,v31 ; neither VR is live, so bug them both
2067 vr_ld32load0: ; must load VRs in this line
2071 vr_ld32test2: ; here to handle next cache line
2072 la r11,savevr2(r3) ; get offset to next cache line
2073 bt 2,vr_ld32load2 ; skip if we must load this line
2074 vor v2,v31,v31 ; neither VR is live, so bug them both
2077 vr_ld32load2: ; must load VRs in this line
2081 vr_ld32test4: ; here to handle next cache line
2082 la r11,savevr4(r3) ; get offset to next cache line
2083 bt 4,vr_ld32load4 ; skip if we must load this line
2084 vor v4,v31,v31 ; neither VR is live, so bug them both
2087 vr_ld32load4: ; must load VRs in this line
2091 vr_ld32test6: ; here to handle next cache line
2092 la r11,savevr6(r3) ; get offset to next cache line
2093 bt 6,vr_ld32load6 ; skip if we must load this line
2094 vor v6,v31,v31 ; neither VR is live, so bug them both
2097 vr_ld32load6: ; must load VRs in this line
2101 vr_ld32test8: ; here to handle next cache line
2102 la r11,savevr8(r3) ; get offset to next cache line
2103 bt 8,vr_ld32load8 ; skip if we must load this line
2104 vor v8,v31,v31 ; neither VR is live, so bug them both
2107 vr_ld32load8: ; must load VRs in this line
2111 vr_ld32test10: ; here to handle next cache line
2112 la r11,savevr10(r3) ; get offset to next cache line
2113 bt 10,vr_ld32load10 ; skip if we must load this line
2114 vor v10,v31,v31 ; neither VR is live, so bug them both
2117 vr_ld32load10: ; must load VRs in this line
2121 vr_ld32test12: ; here to handle next cache line
2122 la r11,savevr12(r3) ; get offset to next cache line
2123 bt 12,vr_ld32load12 ; skip if we must load this line
2124 vor v12,v31,v31 ; neither VR is live, so bug them both
2127 vr_ld32load12: ; must load VRs in this line
2131 vr_ld32test14: ; here to handle next cache line
2132 la r11,savevr14(r3) ; get offset to next cache line
2133 bt 14,vr_ld32load14 ; skip if we must load this line
2134 vor v14,v31,v31 ; neither VR is live, so bug them both
2137 vr_ld32load14: ; must load VRs in this line
2141 vr_ld32test16: ; here to handle next cache line
2142 la r11,savevr16(r3) ; get offset to next cache line
2143 bt 16,vr_ld32load16 ; skip if we must load this line
2144 vor v16,v31,v31 ; neither VR is live, so bug them both
2147 vr_ld32load16: ; must load VRs in this line
2151 vr_ld32test18: ; here to handle next cache line
2152 la r11,savevr18(r3) ; get offset to next cache line
2153 bt 18,vr_ld32load18 ; skip if we must load this line
2154 vor v18,v31,v31 ; neither VR is live, so bug them both
2157 vr_ld32load18: ; must load VRs in this line
2161 vr_ld32test20: ; here to handle next cache line
2162 la r11,savevr20(r3) ; get offset to next cache line
2163 bt 20,vr_ld32load20 ; skip if we must load this line
2164 vor v20,v31,v31 ; neither VR is live, so bug them both
2167 vr_ld32load20: ; must load VRs in this line
2171 vr_ld32test22: ; here to handle next cache line
2172 la r11,savevr22(r3) ; get offset to next cache line
2173 bt 22,vr_ld32load22 ; skip if we must load this line
2174 vor v22,v31,v31 ; neither VR is live, so bug them both
2177 vr_ld32load22: ; must load VRs in this line
2181 vr_ld32test24: ; here to handle next cache line
2182 la r11,savevr24(r3) ; get offset to next cache line
2183 bt 24,vr_ld32load24 ; skip if we must load this line
2184 vor v24,v31,v31 ; neither VR is live, so bug them both
2187 vr_ld32load24: ; must load VRs in this line
2191 vr_ld32test26: ; here to handle next cache line
2192 la r11,savevr26(r3) ; get offset to next cache line
2193 bt 26,vr_ld32load26 ; skip if we must load this line
2194 vor v26,v31,v31 ; neither VR is live, so bug them both
2197 vr_ld32load26: ; must load VRs in this line
2201 vr_ld32test28: ; here to handle next cache line
2202 la r11,savevr28(r3) ; get offset to next cache line
2203 bt 28,vr_ld32load28 ; skip if we must load this line
2204 vor v28,v31,v31 ; neither VR is live, so bug them both
2207 vr_ld32load28: ; must load VRs in this line
2211 vr_ld32test30: ; here to handle next cache line
2212 la r11,savevr30(r3) ; get offset to next cache line
2213 bt 30,vr_ld32load30 ; skip if we must load this line
2214 vor v30,v31,v31 ; neither VR is live, so bug them both
2216 vr_ld32load30: ; must load VRs in this line
projects / apple / xnu.git / blob