/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
- *
- * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* Please see the License for the specific language governing rights and
* limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
#include <debug.h>
#include <mach_assert.h>
#include <mach/exception_types.h>
+#include <mach/kern_return.h>
#include <mach/ppc/vm_param.h>
#include <assym.s>
#include <ppc/trap.h>
#include <ppc/exception.h>
#include <ppc/savearea.h>
-#include <ppc/spl.h>
#define VERIFYSAVE 0
#define FPVECDBG 0
-
+#define FPFLOOD 0
+#define INSTRUMENT 0
+
/*
* thandler(type)
*
.globl EXT(thandler)
LEXT(thandler) ; Trap handler
- mfsprg r25,0 ; Get the per_proc
+ mfsprg r13,1 ; Get the current activation
+ lwz r25,ACT_PER_PROC(r13) ; Get the per_proc block
lwz r1,PP_ISTACKPTR(r25) ; Get interrupt stack pointer
cmpwi cr0,r1,0 ; Are we on interrupt stack?
- lwz r6,PP_ACTIVE_THREAD(r25) ; Get the pointer to the currently active thread
+ mr r6,r13
beq- cr0,EXT(ihandler) ; If on interrupt stack, treat this as interrupt...
- lwz r13,THREAD_TOP_ACT(r6) ; Point to the active activation
lwz r26,ACT_MACT_SPF(r13) ; Get special flags
lwz r8,ACT_MACT_PCB(r13) ; Get the last savearea used
rlwinm. r26,r26,0,bbThreadbit,bbThreadbit ; Do we have Blue Box Assist active?
lwz r1,ACT_MACT_KSP(r13) ; Get the top of kernel stack
bnel- checkassist ; See if we should assist this
stw r4,ACT_MACT_PCB(r13) ; Point to our savearea
- stw r8,SAVprev(r4) ; Queue the new save area in the front
+ stw r8,SAVprev+4(r4) ; Queue the new save area in the front
#if VERIFYSAVE
bl versave ; (TEST/DEBUG)
lwz r9,THREAD_KERNEL_STACK(r6) ; Get our kernel stack start
cmpwi cr1,r1,0 ; Are we already on kernel stack?
stw r13,SAVact(r4) ; Mark the savearea as belonging to this activation
- lwz r26,saver1(r4) ; Get the stack at interrupt time
+ lwz r26,saver1+4(r4) ; Get the stack at interrupt time
bne+ cr1,.L_kstackfree ; We are not on kernel stack yet...
subi r1,r26,FM_REDZONE ; Make a red zone on interrupt time kernel stack
.L_kstackfree:
- lwz r7,savesrr1(r4) ; Pick up the entry MSR
+ lwz r31,savesrr1+4(r4) ; Pick up the entry MSR
sub r9,r1,r9 ; Get displacment into the kernel stack
li r0,0 ; Make this 0
+ rlwinm. r0,r9,0,28,31 ; Verify that we have a 16-byte aligned stack (and get a 0)
cmplwi cr2,r9,KERNEL_STACK_SIZE ; Do we still have room on the stack?
beq cr1,.L_state_on_kstack ; using above test for pcb/stack
.L_state_on_kstack:
lwz r9,savevrsave(r4) ; Get the VRSAVE register
- rlwinm. r6,r7,0,MSR_VEC_BIT,MSR_VEC_BIT ; Was vector on?
+ bne-- kernelStackUnaligned ; Stack is unaligned...
+ rlwinm. r6,r31,0,MSR_VEC_BIT,MSR_VEC_BIT ; Was vector on?
subi r1,r1,FM_SIZE ; Push a header onto the current stack
- bgt- cr2,kernelStackBad ; Kernel stack is bogus...
+ bgt-- cr2,kernelStackBad ; Kernel stack is bogus...
kernelStackNotBad: ; Vector was off
- beq+ tvecoff ; Vector off, do not save vrsave...
+ beq++ tvecoff ; Vector off, do not save vrsave...
stw r9,liveVRS(r25) ; Set the live value
tvecoff: stw r26,FM_BACKPTR(r1) ; Link back to the previous frame
* which links back to the trapped routine. The second is
* that which the C routine below will need
*/
- lwz r3,savesrr0(r4) ; Get the point of interruption
+ lwz r3,savesrr0+4(r4) ; Get the point of interruption
stw r3,FM_LR_SAVE(r1) ; save old instr ptr as LR value
stwu r1, -FM_SIZE(r1) ; and make new frame
#endif /* DEBUG */
+ mr r30,r4
+ lwz r3,SAVtime(r4)
+ lwz r4,SAVtime+4(r4)
+ addi r5,r13,SYSTEM_TIMER
+ bl EXT(thread_timer_event)
+ addi r5,r25,SYSTEM_STATE
+ bl EXT(state_event)
+
+ lwz r7,ACT_TASK(r13)
+ lwz r8,TASK_VTIMERS(r7)
+ cmpwi r8,0
+ beq++ 0f
+
+ lwz r7,ACT_PER_PROC(r13)
+ li r4,AST_BSD
+ lwz r8,PP_PENDING_AST(r7)
+ or r8,r8,r4
+ stw r8,PP_PENDING_AST(r7)
+ addi r3,r13,ACT_AST
+ bl EXT(hw_atomic_or)
+0:
/* call trap handler proper, with
- * ARG0 = type (not yet, holds pcb ptr)
- * ARG1 = saved_state ptr (already there)
- * ARG2 = dsisr (already there)
- * ARG3 = dar (already there)
+ * ARG0 = type
+ * ARG1 = saved_state ptr
+ * ARG2 = dsisr
+ * ARG3 = dar
*/
-
- lwz r3,saveexception(r4) ; Get the exception code
+ mr r4,r30
+ lwz r3,saveexception(r30) ; Get the exception code
lwz r0,ACT_MACT_SPF(r13) ; Get the special flags
addi r5,r3,-T_DATA_ACCESS ; Adjust to start of range
lwz r5,savedsisr(r4) ; Get the saved DSISR
crnor cr7_eq,cr0_eq,cr2_gt ; We should intercept if in VM and is a true trap (cr7_eq == 1 if yes)
- rlwinm. r0,r7,0,MSR_PR_BIT,MSR_PR_BIT ; Are we trapping from supervisor state? (cr0_eq == 1 if yes)
+ rlwinm. r0,r31,0,MSR_PR_BIT,MSR_PR_BIT ; Are we trapping from supervisor state? (cr0_eq == 1 if yes)
cmpi cr2,r3,T_PREEMPT ; Is this a preemption?
+
+ beq-- .L_check_VM
+ stw r4,ACT_MACT_UPCB(r13) ; Store user savearea
+.L_check_VM:
crandc cr0_eq,cr7_eq,cr0_eq ; Do not intercept if we are in the kernel (cr0_eq == 1 if yes)
- lwz r6,savedar(r4) ; Get the DAR
+ lwz r6,savedar(r4) ; Get the DAR (top)
+ lwz r7,savedar+4(r4) ; Get the DAR (bottom)
- beq- cr2, .L_call_trap ; Do not turn on interrupts for T_PREEMPT
+ beq- cr2,.L_call_trap ; Do not turn on interrupts for T_PREEMPT
beq- exitFromVM ; Any true trap but T_MACHINE_CHECK exits us from the VM...
/* syscall exception might warp here if there's nothing left
.L_call_trap:
+#if FPFLOOD
+ stfd f31,emfp31(r25) ; (TEST/DEBUG)
+#endif
+
bl EXT(trap)
+ lis r10,hi16(MASK(MSR_VEC)) ; Get the vector enable
mfmsr r7 ; Get the MSR
- rlwinm r7,r7,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r7,r7,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r7,r7,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Clear the interrupt enable mask
+ ori r10,r10,lo16(MASK(MSR_FP)|MASK(MSR_EE)) ; Add in FP and EE
+ andc r7,r7,r10 ; Turn off VEC, FP, and EE
mtmsr r7 ; Disable for interrupts
- mfsprg r10,0 ; Restore the per_proc info
+ mfsprg r8,1 ; Get the current activation
+ lwz r10,ACT_PER_PROC(r8) ; Get the per_proc block
/*
* This is also the point where new threads come when they are created.
* The new thread is setup to look like a thread that took an
*/
thread_return:
- lwz r4,SAVprev(r3) ; Pick up the previous savearea
lwz r11,SAVflags(r3) ; Get the flags of the current savearea
- lwz r8,savesrr1(r3) ; Get the MSR we are going to
+ lwz r0,savesrr1+4(r3) ; Get the MSR we are going to
+ lwz r4,SAVprev+4(r3) ; Pick up the previous savearea
+ mfsprg r8,1 ; Get the current thread
rlwinm r11,r11,0,15,13 ; Clear the syscall flag
- lwz r1,PP_ACTIVE_THREAD(r10) ; Get the active thread
- rlwinm. r8,r8,0,MSR_PR_BIT,MSR_PR_BIT ; Are we going to the user?
- mfsprg r8,1 ; Get the current activation
+ rlwinm. r0,r0,0,MSR_PR_BIT,MSR_PR_BIT ; Are we going to the user?
+ mr r1,r8
stw r11,SAVflags(r3) ; Save back the flags (with reset stack cleared)
-
+
+ lwz r5,THREAD_KERNEL_STACK(r1) ; Get the base pointer to the stack
stw r4,ACT_MACT_PCB(r8) ; Point to the previous savearea (or 0 if none)
+ addi r5,r5,KERNEL_STACK_SIZE-FM_SIZE ; Reset to empty
- beq- chkfac ; We are not leaving the kernel yet...
+ beq-- chkfac ; We are not leaving the kernel yet...
- lwz r5,THREAD_KERNEL_STACK(r1) ; Get the base pointer to the stack
- addi r5,r5,KERNEL_STACK_SIZE-FM_SIZE ; Reset to empty
stw r5,ACT_MACT_KSP(r8) ; Save the empty stack pointer
b chkfac ; Go end it all...
li r3,failStack ; Bad stack code
sc ; System ABEND
+kernelStackUnaligned:
+ lis r0,hi16(Choke) ; Choke code
+ ori r0,r0,lo16(Choke) ; and the rest
+ li r3,failUnalignedStk ; Unaligned stack code
+ sc ; System ABEND
+
/*
* shandler(type)
.globl EXT(shandler)
LEXT(shandler) ; System call handler
- mfsprg r25,0 ; Get the per proc area
- lwz r0,saver0(r4) ; Get the original syscall number
+ lwz r7,savesrr1+4(r4) ; Get the SRR1 value
+ mfsprg r13,1 ; Get the current activation
+ lwz r25,ACT_PER_PROC(r13) ; Get the per_proc block
+ lwz r0,saver0+4(r4) ; Get the original syscall number
lwz r17,PP_ISTACKPTR(r25) ; Get interrupt stack pointer
rlwinm r15,r0,0,0,19 ; Clear the bottom of call number for fast check
mr. r17,r17 ; Are we on interrupt stack?
- lwz r7,savesrr1(r4) ; Get the SRR1 value
- beq- EXT(ihandler) ; On interrupt stack, not allowed...
lwz r9,savevrsave(r4) ; Get the VRsave register
+ beq-- EXT(ihandler) ; On interrupt stack, not allowed...
rlwinm. r6,r7,0,MSR_VEC_BIT,MSR_VEC_BIT ; Was vector on?
- lwz r16,PP_ACTIVE_THREAD(r25) ; Get the thread pointer
- mfsprg r13,1 ; Pick up the active thread
+ mr r16,r13
- beq+ svecoff ; Vector off, do not save vrsave...
+ beq++ svecoff ; Vector off, do not save vrsave...
stw r9,liveVRS(r25) ; Set the live value
;
; Check if SCs are being redirected for the BlueBox or to VMM
;
svecoff: lwz r6,ACT_MACT_SPF(r13) ; Pick up activation special flags
- mtcrf 0x41,r6 ; Check special flags
+ mtcrf 0x40,r6 ; Check special flags
+ mtcrf 0x01,r6 ; Check special flags
crmove cr6_eq,runningVMbit ; Remember if we are in VMM
- bne cr6,sVMchecked ; Not running VM
+ bne++ cr6,sVMchecked ; Not running VM
lwz r18,spcFlags(r25) ; Load per_proc special flags
rlwinm. r18,r18,0,FamVMmodebit,FamVMmodebit ; Is FamVMmodebit set?
beq sVMchecked ; Not in FAM
cmpwi r0,0x6004 ; Is it vmm_dispatch syscall:
bne sVMchecked
- lwz r26,saver3(r4) ; Get the original syscall number
+ lwz r26,saver3+4(r4) ; Get the original syscall number
cmpwi cr6,r26,kvmmExitToHost ; vmm_exit_to_host request
sVMchecked:
- bf+ bbNoMachSCbit,noassist ; Take branch if SCs are not redirected
+ bf++ bbNoMachSCbit,noassist ; Take branch if SCs are not redirected
lwz r26,ACT_MACT_BEDA(r13) ; Pick up the pointer to the blue box exception area
b EXT(atomic_switch_syscall) ; Go to the assist...
noassist: cmplwi r15,0x7000 ; Do we have a fast path trap?
lwz r14,ACT_MACT_PCB(r13) ; Now point to the PCB
- beql+ fastpath ; We think it is a fastpath...
+ beql fastpath ; We think it is a fastpath...
lwz r1,ACT_MACT_KSP(r13) ; Get the kernel stack pointer
#if DEBUG
#endif /* DEBUG */
stw r4,ACT_MACT_PCB(r13) ; Point to our savearea
+ stw r4,ACT_MACT_UPCB(r13) ; Store user savearea
li r0,0 ; Clear this out
- stw r14,SAVprev(r4) ; Queue the new save area in the front
+ stw r14,SAVprev+4(r4) ; Queue the new save area in the front
stw r13,SAVact(r4) ; Point the savearea at its activation
#if VERIFYSAVE
bl versave ; (TEST/DEBUG)
#endif
+ lwz r15,saver1+4(r4) ; Grab interrupt time stack
mr r30,r4 ; Save pointer to the new context savearea
- lwz r15,saver1(r4) ; Grab interrupt time stack
stw r0,ACT_MACT_KSP(r13) ; Mark stack as busy with 0 val
stw r15,FM_BACKPTR(r1) ; Link stack frame backwards
+
+ lwz r3,SAVtime(r30)
+ lwz r4,SAVtime+4(r30)
+ addi r5,r13,SYSTEM_TIMER
+ bl EXT(thread_timer_event)
+ addi r5,r25,SYSTEM_STATE
+ bl EXT(state_event)
+
+ lwz r7,ACT_TASK(r13)
+ lwz r8,TASK_VTIMERS(r7)
+ cmpwi r8,0
+ beq++ 0f
+
+ lwz r7,ACT_PER_PROC(r13)
+ li r4,AST_BSD
+ lwz r8,PP_PENDING_AST(r7)
+ or r8,r8,r4
+ stw r8,PP_PENDING_AST(r7)
+ addi r3,r13,ACT_AST
+ bl EXT(hw_atomic_or)
+0:
#if DEBUG
/* If debugging, we need two frames, the first being a dummy
* which links back to the trapped routine. The second is
* that which the C routine below will need
*/
- lwz r8,savesrr0(r30) ; Get the point of interruption
+ lwz r8,savesrr0+4(r30) ; Get the point of interruption
stw r8,FM_LR_SAVE(r1) ; Save old instr ptr as LR value
stwu r1, -FM_SIZE(r1) ; and make new frame
#endif /* DEBUG */
- mfmsr r11 ; Get the MSR
+ mr r4,r30
+
lwz r15,SAVflags(r30) ; Get the savearea flags
+ lwz r0,saver0+4(r30) ; Get R0 back
+ mfmsr r11 ; Get the MSR
+ stwu r1,-(FM_SIZE+ARG_SIZE+MUNGE_ARGS_SIZE)(r1) ; Make a stack frame
ori r11,r11,lo16(MASK(MSR_EE)) ; Turn on interruption enabled bit
- lwz r0,saver0(r30) ; Get R0 back
- oris r15,r15,SAVsyscall >> 16 ; Mark that it this is a syscall
rlwinm r10,r0,0,0,19 ; Keep only the top part
- stwu r1,-(FM_SIZE+ARG_SIZE)(r1) ; Make a stack frame
+ oris r15,r15,SAVsyscall >> 16 ; Mark that it this is a syscall
cmplwi r10,0x6000 ; Is it the special ppc-only guy?
stw r15,SAVflags(r30) ; Save syscall marker
- beq- cr6,exitFromVM ; It is time to exit from alternate context...
+ beq-- cr6,exitFromVM ; It is time to exit from alternate context...
- beq- ppcscall ; Call the ppc-only system call handler...
+ beq-- ppcscall ; Call the ppc-only system call handler...
+ mr. r0,r0 ; What kind is it?
mtmsr r11 ; Enable interruptions
- lwz r0,saver0(r30) ; Get the system call selector
- mr. r0,r0 ; What kind is it?
- blt- .L_kernel_syscall ; System call number if negative, this is a mach call...
+ blt-- .L_kernel_syscall ; System call number if negative, this is a mach call...
+ lwz r8,ACT_TASK(r13) ; Get our task
cmpwi cr0,r0,0x7FFA ; Special blue box call?
- beq- .L_notify_interrupt_syscall ; Yeah, call it...
+ beq-- .L_notify_interrupt_syscall ; Yeah, call it...
- lwz r8,ACT_TASK(r13) ; Get our task
- lis r10,hi16(EXT(c_syscalls_unix)) ; Get top half of counter address
lwz r7,TASK_SYSCALLS_UNIX(r8) ; Get the current count
- ori r10,r10,lo16(EXT(c_syscalls_unix)) ; Get low half of counter address
- addi r7,r7,1 ; Bump it
- lwz r9,0(r10) ; Get counter
- stw r7,TASK_SYSCALLS_UNIX(r8) ; Save it
mr r3,r30 ; Get PCB/savearea
mr r4,r13 ; current activation
- addi r9,r9,1 ; Add 1
- stw r9,0(r10) ; Save it back
+ addi r7,r7,1 ; Bump it
+ stw r7,TASK_SYSCALLS_UNIX(r8) ; Save it
+
+#if FPFLOOD
+ stfd f31,emfp31(r25) ; (TEST/DEBUG)
+#endif
+
bl EXT(unix_syscall) ; Check out unix...
.L_call_server_syscall_exception:
b EXT(doexception) ; Go away, never to return...
.L_notify_interrupt_syscall:
- lwz r3,saver3(r30) ; Get the new PC address to pass in
+ lwz r3,saver3+4(r30) ; Get the new PC address to pass in
bl EXT(syscall_notify_interrupt)
- b .L_syscall_return
+/*
+ * Ok, return from C function, R3 = return value
+ *
+ * saved state is still in R30 and the active thread is in R16 .
+ */
+ mr r31,r16 ; Move the current thread pointer
+ stw r3,saver3+4(r30) ; Stash the return code
+ b .L_thread_syscall_ret_check_ast
;
; Handle PPC-only system call interface
; and the savearea/pcb as the first parameter.
; It is up to the callee to enable interruptions if
; they should be. We are in a state here where
-; both interrupts and preemption is ok, but because we could
+; both interrupts and preemption are ok, but because we could
; be calling diagnostic code we will not enable.
;
; Also, the callee is responsible for finding any parameters
mr r3,r30 ; Pass the savearea
mr r4,r13 ; Pass the activation
mr. r11,r11 ; See if there is a function here
- mtlr r11 ; Set the function address
+ mtctr r11 ; Set the function address
beq- .L_call_server_syscall_exception ; Disabled call...
- blrl ; Call it
+#if INSTRUMENT
+ mfspr r4,pmc1 ; Get stamp
+ stw r4,0x6100+(9*16)+0x0(0) ; Save it
+ mfspr r4,pmc2 ; Get stamp
+ stw r4,0x6100+(9*16)+0x4(0) ; Save it
+ mfspr r4,pmc3 ; Get stamp
+ stw r4,0x6100+(9*16)+0x8(0) ; Save it
+ mfspr r4,pmc4 ; Get stamp
+ stw r4,0x6100+(9*16)+0xC(0) ; Save it
+#endif
+ bctrl ; Call it
.globl EXT(ppcscret)
mr. r3,r3 ; See what we should do
mr r31,r16 ; Restore the current thread pointer
bgt+ .L_thread_syscall_ret_check_ast ; Take normal AST checking return....
- mfsprg r10,0 ; Get the per_proc
+ mfsprg r10,1 ; Get the current activation
+ lwz r10,ACT_PER_PROC(r10) ; Get the per_proc block
blt+ .L_thread_syscall_return ; Return, but no ASTs....
- lwz r0,saver0(r30) ; Restore the system call number
+ lwz r0,saver0+4(r30) ; Restore the system call number
b .L_call_server_syscall_exception ; Go to common exit...
+
+/*
+ * we get here for mach system calls
+ * when kdebug tracing is enabled
+ */
+
+ksystrace:
+ mr r4,r30 ; Pass in saved state
+ bl EXT(syscall_trace)
+
+ cmplw r31,r29 ; Is this syscall in the table?
+ add r31,r27,r28 ; Point right to the syscall table entry
+
+ bge- .L_call_server_syscall_exception ; The syscall number is invalid
+
+ lwz r0,savesrr1(r30) ; Get the saved srr1
+ rlwinm. r0,r0,0,MSR_SF_BIT,MSR_SF_BIT ; Test for 64 bit caller
+ lwz r0,MACH_TRAP_ARG_MUNGE32(r31) ; Pick up the 32 bit munge function address
+ beq-- .L_ksystrace_munge
+ lwz r0,MACH_TRAP_ARG_MUNGE64(r31) ; Pick up the 64 bit munge function address
+
+.L_ksystrace_munge:
+ cmplwi r0,0 ; do we have a munger to call?
+ mtctr r0 ; Set the function call address
+ addi r3,r30,saver3 ; Pointer to args from save area
+ addi r4,r1,FM_ARG0+ARG_SIZE ; Pointer for munged args
+ beq-- .L_ksystrace_trapcall ; just make the trap call
+ bctrl ; Call the munge function
+
+.L_ksystrace_trapcall:
+ lwz r0,MACH_TRAP_FUNCTION(r31) ; Pick up the function address
+ mtctr r0 ; Set the function call address
+ addi r3,r1,FM_ARG0+ARG_SIZE ; Pointer to munged args
+ bctrl
+
+ mr r4,r30 ; Pass in the savearea
+ bl EXT(syscall_trace_end) ; Trace the exit of the system call
+ b .L_mach_return
+
+
+
/* Once here, we know that the syscall was -ve
* we should still have r1=ksp,
* r16 = pointer to current thread,
* r30 = pointer to saved state (in pcb)
*/
- .align 5
+ .align 5
.L_kernel_syscall:
;
; Call a function that can print out our syscall info
; Note that we don t care about any volatiles yet
;
- mr r4,r30
- bl EXT(syscall_trace)
- lwz r0,saver0(r30) ; Get the system call selector */
- neg r31,r0 ; Make system call number positive and put in r31
- lis r29,hi16(EXT(mach_trap_count)) ; High part of valid trap number
- ori r29,r29,lo16(EXT(mach_trap_count)) ; Low part of valid trap number
- lis r28,hi16(EXT(mach_trap_table)) ; High part of trap table
- lwz r29,0(r29) ; Get the first invalid system call number
- ori r28,r28,lo16(EXT(mach_trap_table)) ; Low part of trap table
-
- cmplw r31,r29 ; See if we have a valid system call number
- slwi r31,r31,MACH_TRAP_OFFSET_POW2 ; Get offset into table
-
- bge- .L_call_server_syscall_exception ; System call number of bogus
-
- add r31,r31,r28 ; Point to the system call entry
- lis r28,hi16(EXT(kern_invalid)) ; Get the high invalid routine address
- lwz r0,MACH_TRAP_FUNCTION(r31) ; Grab the system call routine address
- ori r28,r28,lo16(EXT(kern_invalid)) ; Get the low part of the invalid routine address
- lwz r29,MACH_TRAP_ARGC(r31) ; Get the number of arguments in the call
- cmplw r0,r28 ; Is this an invalid entry?
- beq- .L_call_server_syscall_exception ; Yes, it is invalid...
-
-/* get arg count. If argc > 8 then not all args were in regs,
- * so we must perform copyin.
- */
- cmpwi cr0,r29,8 ; Do we have more than 8 arguments?
- ble+ .L_syscall_got_args ; Nope, no copy in needed...
-
-/* argc > 8 - perform a copyin */
-/* if the syscall came from kernel space, we can just copy */
-
- lwz r0,savesrr1(r30) ; Pick up exception time MSR
- andi. r0,r0,MASK(MSR_PR) ; Check the priv bit
- bne+ .L_syscall_arg_copyin ; We are not priviliged...
-
-/* we came from a privilaged task, just do a copy */
-/* get user's stack pointer */
-
- lwz r28,saver1(r30) ; Get the stack pointer
-
- subi r29,r29,8 ; Get the number of arguments to copy
-
- addi r28,r28,COPYIN_ARG0_OFFSET-4 ; Point to source - 4
- addi r27,r1,FM_ARG0-4 ; Point to sink - 4
-
-.L_syscall_copy_word_loop:
- addic. r29,r29,-1 ; Count down the number of arguments left
- lwz r0,4(r28) ; Pick up the argument from the stack
- addi r28,r28,4 ; Point to the next source
- stw r0,4(r27) ; Store the argument
- addi r27,r27,4 ; Point to the next sink
- bne+ .L_syscall_copy_word_loop ; Move all arguments...
- b .L_syscall_got_args ; Go call it now...
-
-
-/* we came from a user task, pay the price of a real copyin */
-/* set recovery point */
-
- .align 5
-
-.L_syscall_arg_copyin:
- lwz r8,ACT_VMMAP(r13) ; Get the vm_map for this activation
- lis r28,hi16(.L_syscall_copyin_recover)
- lwz r8,VMMAP_PMAP(r8) ; Get the pmap
- ori r28,r28,lo16(.L_syscall_copyin_recover)
- addi r8,r8,PMAP_SEGS ; Point to the pmap SR slots
- stw r28,THREAD_RECOVER(r16) ; R16 still holds thread ptr
-
-/* We can manipulate the COPYIN segment register quite easily
- * here, but we've also got to make sure we don't go over a
- * segment boundary - hence some mess.
- * Registers from 12-29 are free for our use.
- */
-
-
- lwz r28,saver1(r30) ; Get the stack pointer
- subi r29,r29,8 ; Get the number of arguments to copy
- addi r28,r28,COPYIN_ARG0_OFFSET ; Set source in user land
-
-/* set up SR_COPYIN to allow us to copy, we may need to loop
- * around if we change segments. We know that this previously
- * pointed to user space, so the sid doesn't need setting.
- */
-
- rlwinm r7,r28,6,26,29 ; Get index to the segment slot
-
-.L_syscall_copyin_seg_loop:
- lwzx r10,r8,r7 ; Get the source SR value
- rlwinm r26,r28,0,4,31 ; Clear the segment number from source address
- mtsr SR_COPYIN,r10 ; Set the copyin SR
- isync
-
- oris r26,r26,(SR_COPYIN_NUM << (28-16)) ; Insert the copyin segment number into source address
-
- addi r27,r1,FM_ARG0-4 ; Point to area - 4 where we will store the arguments
-
-.L_syscall_copyin_word_loop:
- lwz r0,0(r26) ; MAY CAUSE PAGE FAULT!
- subi r29,r29,1 ; Decrement count
- addi r26,r26,4 ; Bump input
- stw r0,4(r27) ; Save the copied in word
- mr. r29,r29 ; Are they all moved?
- addi r27,r27,4 ; Bump output
- beq+ .L_syscall_copyin_done ; Escape if we are done...
-
- rlwinm. r0,r26,0,4,29 ; Did we just step into a new segment?
- addi r28,r28,4 ; Bump up user state address also
- bne+ .L_syscall_copyin_word_loop ; We are still on the same segment...
-
- addi r7,r7,4 ; Bump to next slot
- b .L_syscall_copyin_seg_loop ; On new segment! remap
-
-/* Don't bother restoring SR_COPYIN, we can leave it trashed */
-/* clear thread recovery as we're done touching user data */
-
- .align 5
-
-.L_syscall_copyin_done:
- li r0,0
- stw r0,THREAD_RECOVER(r16) ; R16 still holds thread ptr
-
-.L_syscall_got_args:
- lwz r0,MACH_TRAP_FUNCTION(r31) ; Get function address
- lwz r8,ACT_TASK(r13) ; Get our task
- lis r10,hi16(EXT(c_syscalls_mach)) ; Get top half of counter address
- lwz r7,TASK_SYSCALLS_MACH(r8) ; Get the current count
- lwz r3,saver3(r30) ; Restore r3
- addi r7,r7,1 ; Bump it
- ori r10,r10,lo16(EXT(c_syscalls_mach)) ; Get low half of counter address
- stw r7,TASK_SYSCALLS_MACH(r8) ; Save it
- lwz r4,saver4(r30) ; Restore r4
- lwz r9,0(r10) ; Get counter
- mtctr r0 ; Set function address
- lwz r5,saver5(r30) ; Restore r5
- lwz r6,saver6(r30) ; Restore r6
- addi r9,r9,1 ; Add 1
- lwz r7,saver7(r30) ; Restore r7
- lwz r8,saver8(r30) ; Restore r8
- stw r9,0(r10) ; Save it back
- lwz r9,saver9(r30) ; Restore r9
- lwz r10,saver10(r30) ; Restore r10
-
-
-;
-; Note that all arguments from the system call are passed into the function
-;
-
- bctrl ; Perform the actual syscall
+ lwz r10,ACT_TASK(r13) ; Get our task
+ lwz r0,saver0+4(r30)
+ lis r8,hi16(EXT(kdebug_enable)) ; Get top of kdebug_enable
+ lis r28,hi16(EXT(mach_trap_table)) ; Get address of table
+ ori r8,r8,lo16(EXT(kdebug_enable)) ; Get bottom of kdebug_enable
+ lwz r8,0(r8) ; Get kdebug_enable
+
+ lwz r7,TASK_SYSCALLS_MACH(r10) ; Get the current count
+ neg r31,r0 ; Make this positive
+ mr r3,r31 ; save it
+ slwi r27,r3,4 ; multiply by 16
+ slwi r3,r3,2 ; and the original by 4
+ ori r28,r28,lo16(EXT(mach_trap_table)) ; Get address of table
+ add r27,r27,r3 ; for a total of 20x (5 words/entry)
+ addi r7,r7,1 ; Bump TASK_SYSCALLS_MACH count
+ cmplwi r8,0 ; Is kdebug_enable non-zero
+ stw r7,TASK_SYSCALLS_MACH(r10) ; Save count
+ bne-- ksystrace ; yes, tracing enabled
+
+ cmplwi r31,MACH_TRAP_TABLE_COUNT ; Is this syscall in the table?
+ add r31,r27,r28 ; Point right to the syscall table entry
+
+ bge-- .L_call_server_syscall_exception ; The syscall number is invalid
+
+ lwz r0,savesrr1(r30) ; Get the saved srr1
+ rlwinm. r0,r0,0,MSR_SF_BIT,MSR_SF_BIT ; Test for 64 bit caller
+ lwz r0,MACH_TRAP_ARG_MUNGE32(r31) ; Pick up the 32 bit munge function address
+ beq-- .L_kernel_syscall_munge
+ lwz r0,MACH_TRAP_ARG_MUNGE64(r31) ; Pick up the 64 bit munge function address
+
+.L_kernel_syscall_munge:
+ cmplwi r0,0 ; test for null munger
+ mtctr r0 ; Set the function call address
+ addi r3,r30,saver3 ; Pointer to args from save area
+ addi r4,r1,FM_ARG0+ARG_SIZE ; Pointer for munged args
+ beq-- .L_kernel_syscall_trapcall ; null munger - skip to trap call
+ bctrl ; Call the munge function
+
+.L_kernel_syscall_trapcall:
+ lwz r0,MACH_TRAP_FUNCTION(r31) ; Pick up the function address
+ mtctr r0 ; Set the function call address
+ addi r3,r1,FM_ARG0+ARG_SIZE ; Pointer to munged args
+
+#if FPFLOOD
+ stfd f31,emfp31(r25) ; (TEST/DEBUG)
+#endif
-/* 'standard' syscall returns here - INTERRUPTS ARE STILL ON */
+ bctrl
-/* r3 contains value that we're going to return to the user
- */
/*
* Ok, return from C function, R3 = return value
*
* get the active thread's PCB pointer and thus pointer to user state
- * saved state is still in R30 and the active thread is in R16 .
+ * saved state is still in R30 and the active thread is in R16
*/
-/* Store return value into saved state structure, since
- * we need to pick up the value from here later - the
- * syscall may perform a thread_set_syscall_return
+.L_mach_return:
+ srawi r0,r3,31 ; properly extend the return code
+ cmpi cr0,r3,KERN_INVALID_ARGUMENT ; deal with invalid system calls
+ mr r31,r16 ; Move the current thread pointer
+ stw r0, saver3(r30) ; stash the high part of the return code
+ stw r3,saver3+4(r30) ; Stash the low part of the return code
+ beq-- cr0,.L_mach_invalid_ret ; otherwise fall through into the normal return path
+.L_mach_invalid_arg:
+
+
+/* 'standard' syscall returns here - INTERRUPTS ARE STILL ON
+ * the syscall may perform a thread_set_syscall_return
* followed by a thread_exception_return, ending up
* at thread_syscall_return below, with SS_R3 having
* been set up already
- */
-
-/* When we are here, r16 should point to the current thread,
+ *
+ * When we are here, r31 should point to the current thread,
* r30 should point to the current pcb
+ * r3 contains value that we're going to return to the user
+ * which has already been stored back into the save area
*/
-
-/* save off return value, we must load it
- * back anyway for thread_exception_return
- */
-
-.L_syscall_return:
- mr r31,r16 ; Move the current thread pointer
- stw r3,saver3(r30) ; Stash the return code
-
- mr r4,r30 ; Pass in the savearea
- bl EXT(syscall_trace_end) ; Trace the exit of the system call
-
+
.L_thread_syscall_ret_check_ast:
+ lis r10,hi16(MASK(MSR_VEC)) ; Get the vector enable
mfmsr r12 ; Get the current MSR
- rlwinm r12,r12,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r12,r12,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r12,r12,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Turn off interruptions enable bit
+ ori r10,r10,lo16(MASK(MSR_FP)|MASK(MSR_EE)) ; Add in FP and EE
+ andc r12,r12,r10 ; Turn off VEC, FP, and EE
mtmsr r12 ; Turn interruptions off
- mfsprg r10,0 ; Get the per_processor block
+ mfsprg r10,1 ; Get the current activation
+ lwz r10,ACT_PER_PROC(r10) ; Get the per_proc block
/* Check to see if there's an outstanding AST */
- lwz r4,PP_NEED_AST(r10) ; Get the pointer to the ast requests
- lwz r4,0(r4) ; Get the flags
+ lwz r4,PP_PENDING_AST(r10)
cmpi cr0,r4, 0 ; Any pending asts?
- beq+ cr0,.L_syscall_no_ast ; Nope...
+ beq++ cr0,.L_syscall_no_ast ; Nope...
/* Yes there is, call ast_taken
* pretending that the user thread took an AST exception here,
#if DEBUG
/* debug assert - make sure that we're not returning to kernel */
- lwz r3,savesrr1(r30)
+ lwz r3,savesrr1+4(r30)
andi. r3,r3,MASK(MSR_PR)
- bne+ scrnotkern ; returning to user level, check
+ bne++ scrnotkern ; returning to user level, check
lis r0,hi16(Choke) ; Choke code
ori r0,r0,lo16(Choke) ; and the rest
scrnotkern:
#endif /* DEBUG */
- li r3,AST_ALL ; Set ast flags
+ lis r3,hi16(AST_ALL) ; Set ast flags
li r4,1 ; Set interrupt allowed
+ ori r3,r3,lo16(AST_ALL)
bl EXT(ast_taken) ; Process the pending ast
b .L_thread_syscall_ret_check_ast ; Go see if there was another...
+.L_mach_invalid_ret:
+/*
+ * need to figure out why we got an KERN_INVALID_ARG
+ * if it was due to a non-existent system call
+ * then we want to throw an exception... otherwise
+ * we want to pass the error code back to the caller
+ */
+ lwz r0,saver0+4(r30) ; reload the original syscall number
+ neg r28,r0 ; Make this positive
+ mr r4,r28 ; save a copy
+ slwi r27,r4,4 ; multiply by 16
+ slwi r4,r4,2 ; and another 4
+ lis r28,hi16(EXT(mach_trap_table)) ; Get address of table
+ add r27,r27,r4 ; for a total of 20x (5 words/entry)
+ ori r28,r28,lo16(EXT(mach_trap_table)) ; Get address of table
+ add r28,r27,r28 ; Point right to the syscall table entry
+ lwz r27,MACH_TRAP_FUNCTION(r28) ; Pick up the function address
+ lis r28,hi16(EXT(kern_invalid)) ; Get high half of invalid syscall function
+ ori r28,r28,lo16(EXT(kern_invalid)) ; Get low half of invalid syscall function
+ cmpw cr0,r27,r28 ; Check if this is an invalid system call
+ beq-- .L_call_server_syscall_exception ; We have a bad system call
+ b .L_mach_invalid_arg ; a system call returned KERN_INVALID_ARG
+
+
/* thread_exception_return returns to here, almost all
* registers intact. It expects a full context restore
* of what it hasn't restored itself (ie. what we use).
.L_thread_syscall_return:
mr r3,r30 ; Get savearea to the correct register for common exit
- mfsprg r8,1 ; Now find the current activation
lwz r11,SAVflags(r30) ; Get the flags
lwz r5,THREAD_KERNEL_STACK(r31) ; Get the base pointer to the stack
+ lwz r4,SAVprev+4(r30) ; Get the previous save area
rlwinm r11,r11,0,15,13 ; Clear the syscall flag
- lwz r4,SAVprev(r30) ; Get the previous save area
- stw r11,SAVflags(r30) ; Stick back the flags
+ mfsprg r8,1 ; Now find the current activation
addi r5,r5,KERNEL_STACK_SIZE-FM_SIZE ; Reset to empty
- stw r4,ACT_MACT_PCB(r8) ; Save previous save area
+ stw r11,SAVflags(r30) ; Stick back the flags
stw r5,ACT_MACT_KSP(r8) ; Save the empty stack pointer
+ stw r4,ACT_MACT_PCB(r8) ; Save previous save area
b chkfac ; Go end it all...
- .align 5
-
-.L_syscall_copyin_recover:
-
-/* This is the catcher for any data faults in the copyin
- * of arguments from the user's stack.
- * r30 still holds a pointer to the PCB
- *
- * call syscall_error(EXC_BAD_ACCESS, EXC_PPC_VM_PROT_READ, sp, ssp),
- *
- * we already had a frame so we can do this
- */
-
- li r3,EXC_BAD_ACCESS ; Set bad access code
- li r4,EXC_PPC_VM_PROT_READ ; Set protection exception
- lwz r5,saver1(r30) ; Point to the stack
- mr r6,r30 ; Pass savearea
-
- bl EXT(syscall_error) ; Generate error...
- b .L_syscall_return ; Continue out...
-
-
/*
* thread_exception_return()
*
LEXT(thread_exception_return) ; Directly return to user mode
.L_thread_exc_ret_check_ast:
+ lis r10,hi16(MASK(MSR_VEC)) ; Get the vector enable
mfmsr r3 ; Get the MSR
- rlwinm r3,r3,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r3,r3,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r3,r3,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Clear EE
+ ori r10,r10,lo16(MASK(MSR_FP)|MASK(MSR_EE)) ; Add in FP and EE
+ andc r3,r3,r10 ; Turn off VEC, FP, and EE
mtmsr r3 ; Disable interrupts
/* Check to see if there's an outstanding AST */
frame, given that we're not going to return.
*/
- mfsprg r10,0 ; Get the per_processor block
- lwz r4,PP_NEED_AST(r10)
- lwz r4,0(r4)
+ mfsprg r10,1 ; Get the current activation
+ lwz r10,ACT_PER_PROC(r10) ; Get the per_proc block
+ lwz r4,PP_PENDING_AST(r10)
cmpi cr0,r4, 0
- beq cr0,.L_exc_ret_no_ast
+ beq+ cr0,.L_exc_ret_no_ast
/* Yes there is, call ast_taken
* pretending that the user thread took an AST exception here,
* ast_taken will save all state and bring us back here
*/
- li r3,AST_ALL
+ lis r3,hi16(AST_ALL)
li r4,1
+ ori r3,r3,lo16(AST_ALL)
bl EXT(ast_taken)
b .L_thread_exc_ret_check_ast ; check for a second AST (rare)
.L_exc_ret_no_ast:
mfsprg r30,1 ; Get the currrent activation
- lwz r31,ACT_THREAD(r30) ; Get the current thread
+ mr r31,r30
lwz r30,ACT_MACT_PCB(r30)
mr. r30,r30 ; Is there any context yet?
* get the active thread's PCB pointer and thus pointer to user state
*/
- lwz r3,savesrr1(r30)
+ lwz r3,savesrr1+4(r30)
andi. r3,r3,MASK(MSR_PR)
bne+ ret_user2 ; We are ok...
* which takes PCB pointer in R3, not in r30!
*/
lwz r0,SAVflags(r30) ; Grab the savearea flags
- mr r3,r30 ; Copy pcb pointer into r3 in case we need it
andis. r0,r0,SAVsyscall>>16 ; Are we returning from a syscall?
- beq- cr0,thread_return ; Nope, must be a thread return...
+ mr r3,r30 ; Copy pcb pointer into r3 in case we need it
+ beq-- cr0,thread_return ; Nope, must be a thread return...
b .L_thread_syscall_return ; Join up with the system call return...
;
li r4,SAVgeneral ; Get the general context type
li r0,0 ; Get a 0
stb r4,SAVflags+2(r3) ; Set type
- addi r2,r3,savevscr ; Point past what we are clearing
+ addi r2,r3,savefpscr+4 ; Point past what we are clearing
mr r4,r3 ; Save the start
cleardummy: stw r0,0(r4) ; Clear stuff
lis r2,hi16(MSR_EXPORT_MASK_SET) ; Set the high part of the user MSR
ori r2,r2,lo16(MSR_EXPORT_MASK_SET) ; And the low part
- stw r2,savesrr1(r3) ; Set the default user MSR
+ stw r2,savesrr1+4(r3) ; Set the default user MSR
b thread_return ; Go let em try to execute, hah!
* interrupt stack.
*/
- lwz r10,savesrr1(r4) ; Get SRR1
+ lwz r10,savesrr1+4(r4) ; Get SRR1
lwz r7,savevrsave(r4) ; Get the VRSAVE register
- mfsprg r25,0 ; Get the per_proc block
+ mfsprg r13,1 ; Get the current activation
+ lwz r25,ACT_PER_PROC(r13) ; Get the per_proc block
li r14,0 ; Zero this for now
- rlwinm. r13,r10,0,MSR_VEC_BIT,MSR_VEC_BIT ; Was vector on?
+ rlwinm. r16,r10,0,MSR_VEC_BIT,MSR_VEC_BIT ; Was vector on?
lwz r1,PP_ISTACKPTR(r25) ; Get the interrupt stack
- li r13,0 ; Zero this for now
- lwz r16,PP_ACTIVE_THREAD(r25) ; Get the thread pointer
+ li r16,0 ; Zero this for now
beq+ ivecoff ; Vector off, do not save vrsave...
stw r7,liveVRS(r25) ; Set the live value
ivecoff: li r0,0 ; Get a constant 0
- cmplwi cr1,r16,0 ; Are we still booting?
-
-ifpoff: mr. r1,r1 ; Is it active?
- beq- cr1,ihboot1 ; We are still coming up...
- lwz r13,THREAD_TOP_ACT(r16) ; Pick up the active thread
+ rlwinm r5,r10,0,MSR_PR_BIT,MSR_PR_BIT ; Are we trapping from supervisor state?
+ mr. r1,r1 ; Is it active?
+ cmplwi cr2,r5,0 ; cr2_eq == 1 if yes
+ mr r16,r13
lwz r14,ACT_MACT_PCB(r13) ; Now point to the PCB
-
-ihboot1: lwz r9,saver1(r4) ; Pick up the rupt time stack
- stw r14,SAVprev(r4) ; Queue the new save area in the front
+ lwz r9,saver1+4(r4) ; Pick up the rupt time stack
+ stw r14,SAVprev+4(r4) ; Queue the new save area in the front
stw r13,SAVact(r4) ; Point the savearea at its activation
- beq- cr1,ihboot4 ; We are still coming up...
stw r4,ACT_MACT_PCB(r13) ; Point to our savearea
+ beq cr2,ifromk
+ stw r4,ACT_MACT_UPCB(r13) ; Store user savearea
-ihboot4: bne .L_istackfree ; Nope...
+ifromk: bne .L_istackfree ; Nope...
/* We're already on the interrupt stack, get back the old
* stack pointer and make room for a frame
subi r5,r5,KERNEL_STACK_SIZE-FM_SIZE ; Adjust to start of stack
sub r5,r1,r5 ; Get displacement into debug stack
cmplwi cr2,r5,KERNEL_STACK_SIZE-FM_SIZE ; Check if we are on debug stack
- blt+ ihsetback ; Yeah, that is ok too...
+ blt+ cr2,ihsetback ; Yeah, that is ok too...
lis r0,hi16(Choke) ; Choke code
ori r0,r0,lo16(Choke) ; and the rest
li r3,failStack ; Bad stack code
sc ; System ABEND
+intUnalignedStk:
+ lis r0,hi16(Choke) ; Choke code
+ ori r0,r0,lo16(Choke) ; and the rest
+ li r3,failUnalignedStk ; Unaligned stack code
+ sc ; System ABEND
+
.align 5
.L_istackfree:
- lwz r10,SAVflags(r4)
+ rlwinm. r0,r1,0,28,31 ; Check if stack is aligned (and get 0)
+ lwz r10,SAVflags(r4) ; Get savearea flags
+ bne-- intUnalignedStk ; Stack is unaligned...
stw r0,PP_ISTACKPTR(r25) ; Mark the stack in use
- oris r10,r10,HIGH_ADDR(SAVrststk) ; Indicate we reset stack when we return from this one
+ oris r10,r10,hi16(SAVrststk) ; Indicate we reset stack when we return from this one
stw r10,SAVflags(r4) ; Stick it back
/*
* which links back to the trapped routine. The second is
* that which the C routine below will need
*/
- lwz r5,savesrr0(r4) ; Get interrupt address
+ lwz r5,savesrr0+4(r4) ; Get interrupt address
stw r5,FM_LR_SAVE(r1) ; save old instr ptr as LR value
stwu r1,-FM_SIZE(r1) ; Make another new frame for C routine
#endif /* DEBUG */
- lwz r5,savedsisr(r4) ; Get the DSISR
- lwz r6,savedar(r4) ; Get the DAR
-
- bl EXT(interrupt)
-
+ mr r31,r3
+ mr r30,r4
+
+ lwz r3,SAVtime(r4)
+ lwz r4,SAVtime+4(r4)
+ addi r5,r25,PP_PROCESSOR
+ lwz r5,KERNEL_TIMER(r5)
+ bl EXT(thread_timer_event)
+ addi r6,r25,PP_PROCESSOR
+ lwz r5,CURRENT_STATE(r6)
+ addi r7,r6,USER_STATE
+ cmplw r5,r7
+ bne 0f
+ addi r5,r6,SYSTEM_STATE
+ bl EXT(state_event)
+0:
+
+ lwz r7,ACT_TASK(r13)
+ lwz r8,TASK_VTIMERS(r7)
+ cmpwi r8,0
+ beq++ 0f
+
+ lwz r7,ACT_PER_PROC(r13)
+ li r4,AST_BSD
+ lwz r8,PP_PENDING_AST(r7)
+ or r8,r8,r4
+ stw r8,PP_PENDING_AST(r7)
+ addi r3,r13,ACT_AST
+ bl EXT(hw_atomic_or)
+0:
+
+ mr r3,r31
+ mr r4,r30
+ lwz r5,savedsisr(r30) ; Get the DSISR
+ lwz r6,savedar+4(r30) ; Get the DAR
+
+#if FPFLOOD
+ stfd f31,emfp31(r25) ; (TEST/DEBUG)
+#endif
-/* interrupt() returns a pointer to the saved state in r3
- *
- * Ok, back from C. Disable interrupts while we restore things
- */
- .globl EXT(ihandler_ret)
+ bl EXT(interrupt)
-LEXT(ihandler_ret) ; Marks our return point from debugger entry
+/* interrupt() returns a pointer to the saved state in r3 */
+ lis r10,hi16(MASK(MSR_VEC)) ; Get the vector enable
mfmsr r0 ; Get our MSR
- rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
- rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
- rlwinm r0,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Flip off the interrupt enabled bit
+ ori r10,r10,lo16(MASK(MSR_FP)|MASK(MSR_EE)) ; Add in FP and EE
+ andc r0,r0,r10 ; Turn off VEC, FP, and EE
mtmsr r0 ; Make sure interrupts are disabled
- mfsprg r10,0 ; Get the per_proc block
+ mfsprg r8,1 ; Get the current activation
+ lwz r10,ACT_PER_PROC(r8) ; Get the per_proc block
lwz r7,SAVflags(r3) ; Pick up the flags
- lwz r8,PP_ACTIVE_THREAD(r10) ; and the active thread
- lwz r9,SAVprev(r3) ; Get previous save area
+ lwz r9,SAVprev+4(r3) ; Get previous save area
cmplwi cr1,r8,0 ; Are we still initializing?
- lwz r12,savesrr1(r3) ; Get the MSR we will load on return
- beq- cr1,ihboot2 ; Skip if we are still in init...
- lwz r8,THREAD_TOP_ACT(r8) ; Pick up the active thread
-
-ihboot2: andis. r11,r7,hi16(SAVrststk) ; Is this the first on the stack?
- beq- cr1,ihboot3 ; Skip if we are still in init...
+ lwz r12,savesrr1+4(r3) ; Get the MSR we will load on return
+ andis. r11,r7,hi16(SAVrststk) ; Is this the first on the stack?
stw r9,ACT_MACT_PCB(r8) ; Point to previous context savearea
-
-ihboot3: mr r4,r3 ; Move the savearea pointer
+ mr r4,r3 ; Move the savearea pointer
beq .L_no_int_ast2 ; Get going if not the top-o-stack...
lwz r9,PP_INTSTACK_TOP_SS(r10) ; Get the empty stack value
andc r7,r7,r11 ; Remove the stack reset bit in case we pass this one
stw r9,PP_ISTACKPTR(r10) ; Save that saved state ptr
- lwz r3,PP_PREEMPT_CNT(r10) ; Get preemption level
+ lwz r3,ACT_PREEMPT_CNT(r8) ; Get preemption level
stw r7,SAVflags(r4) ; Save the flags
cmplwi r3, 0 ; Check for preemption
bne .L_no_int_ast ; Do not preempt if level is not zero
andi. r6,r12,MASK(MSR_PR) ; privilege mode
- lwz r11,PP_NEED_AST(r10) ; Get the AST request address
- lwz r11,0(r11) ; Get the request
+ lwz r11,PP_PENDING_AST(r10) ; Get the pending AST mask
beq- .L_kernel_int_ast ; In kernel space, AST_URGENT check
li r3,T_AST ; Assume the worst
mr. r11,r11 ; Are there any pending?
rlwinm r7,r7,0,15,13 ; Clear the syscall flag
li r4,0 ; Assume for a moment that we are in init
stw r7,SAVflags(r3) ; Set the flags with cleared syscall flag
- beq- cr1,chkfac ; Jump away if we are in init...
+ beq-- cr1,chkfac ; Jump away if we are in init...
lwz r4,ACT_MACT_PCB(r8) ; Get the new level marker
; are going to user state. CR2_eq will be set to indicate deferred.
;
-chkfac: mr r31,r10 ; Move per_proc address
- mr r30,r4 ; Preserve new level
- lwz r29,savesrr1(r3) ; Get the current MSR
+chkfac: lwz r29,savesrr1+4(r3) ; Get the current MSR
mr. r28,r8 ; Are we still in boot?
+ mr r31,r10 ; Move per_proc address
+ mr r30,r4 ; Preserve new level
mr r27,r3 ; Save the old level
- beq- chkenax ; Yeah, skip it all...
+ beq-- chkenax ; Yeah, skip it all...
rlwinm. r0,r29,0,MSR_PR_BIT,MSR_PR_BIT ; Are we going into user state?
-#if 0
- beq+ lllll ; (TEST/DEBUG)
- BREAKPOINT_TRAP ; (TEST/DEBUG)
-lllll:
-#endif
-
lwz r20,curctx(r28) ; Get our current context
lwz r26,deferctx(r28) ; Get any deferred context switch
+ li r0,1 ; Get set to hold off quickfret
rlwinm r29,r29,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Turn off floating point for now
lwz r21,FPUlevel(r20) ; Get the facility level
cmplwi cr2,r26,0 ; Are we going into a deferred context later?
rlwinm r29,r29,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Turn off vector for now
crnor cr2_eq,cr0_eq,cr2_eq ; Set cr2_eq if going to user state and there is deferred
- cmplw r27,r21 ; Are we returning from the active level?
lhz r19,PP_CPU_NUMBER(r31) ; Get our CPU number
- bne+ fpuchkena ; Nope...
+ cmplw r27,r21 ; Are we returning from the active level?
+ stw r0,holdQFret(r31) ; Make sure we hold off releasing quickfret
+ bne++ fpuchkena ; Nope...
;
; First clean up any live context we are returning from
eieio ; Make sure this gets out before owner clear
- lis r23,hi16(EXT(per_proc_info)) ; Set base per_proc
- mulli r22,r22,ppSize ; Find offset to the owner per_proc
- ori r23,r23,lo16(EXT(per_proc_info)) ; Set base per_proc
- li r24,FPUowner ; Displacement to FPU owner
- add r22,r23,r22 ; Point to the owner per_proc
- li r0,0 ; We need this in a bit
+#if ppeSize != 16
+#error per_proc_entry is not 16bytes in size
+#endif
-fpuinvothr: lwarx r23,r24,r22 ; Get the owner
- cmplw r23,r20 ; Does he still have this context?
- bne fpuinvoths ; Nope...
- stwcx. r0,r24,r22 ; Try to invalidate it
- bne- fpuinvothr ; Try again if there was a collision...
+ lis r23,hi16(EXT(PerProcTable)) ; Set base PerProcTable
+ slwi r22,r22,4 ; Find offset to the owner per_proc_entry
+ ori r23,r23,lo16(EXT(PerProcTable)) ; Set base PerProcTable
+ li r24,FPUowner ; Displacement to float owner
+ add r22,r23,r22 ; Point to the owner per_proc_entry
+ lwz r22,ppe_vaddr(r22) ; Point to the owner per_proc
-fpuinvoths: isync
+fpuinvothr: lwarx r23,r24,r22 ; Get the owner
+
+ sub r0,r23,r20 ; Subtract one from the other
+ sub r21,r20,r23 ; Subtract the other from the one
+ or r21,r21,r0 ; Combine them
+ srawi r21,r21,31 ; Get a 0 if equal or -1 of not
+ and r23,r23,r21 ; Make 0 if same, unchanged if not
+ stwcx. r23,r24,r22 ; Try to invalidate it
+ bne-- fpuinvothr ; Try again if there was a collision...
+
+ isync
;
; Now if there is a savearea associated with the popped context, release it.
lwz r22,FPUsave(r20) ; Get pointer to the first savearea
li r21,0 ; Assume we popped all the way out
mr. r22,r22 ; Is there anything there?
- beq+ fpusetlvl ; No, see if we need to enable...
+ beq++ fpusetlvl ; No, see if we need to enable...
lwz r21,SAVlevel(r22) ; Get the level of that savearea
cmplw r21,r27 ; Is this the saved copy of the live stuff?
bne fpusetlvl ; No, leave as is...
- lwz r24,SAVprev(r22) ; Pick up the previous area
+ lwz r24,SAVprev+4(r22) ; Pick up the previous area
li r21,0 ; Assume we popped all the way out
mr. r24,r24 ; Any more context stacked?
- beq- fpuonlyone ; Nope...
+ beq-- fpuonlyone ; Nope...
lwz r21,SAVlevel(r24) ; Get the level associated with save
fpuonlyone: stw r24,FPUsave(r20) ; Dequeue this savearea
rlwinm r3,r22,0,0,19 ; Find main savearea header
- lwz r3,SACvrswap(r3) ; Get the virtual to real conversion
- la r9,quickfret(r31) ; Point to the quickfret chain header
+
+ lwz r8,quickfret(r31) ; Get the first in quickfret list (top)
+ lwz r9,quickfret+4(r31) ; Get the first in quickfret list (bottom)
+ lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
+ lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
+ stw r8,SAVprev(r22) ; Link the old in (top)
+ stw r9,SAVprev+4(r22) ; Link the old in (bottom)
xor r3,r22,r3 ; Convert to physical
+ stw r2,quickfret(r31) ; Set the first in quickfret list (top)
+ stw r3,quickfret+4(r31) ; Set the first in quickfret list (bottom)
#if FPVECDBG
lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
sc ; (TEST/DEBUG)
#endif
-fpufpucdq: lwarx r0,0,r9 ; Pick up the old chain head
- stw r0,SAVprev(r22) ; Move it to the current guy
- stwcx. r3,0,r9 ; Save it
- bne- fpufpucdq ; Someone chaged the list...
-
fpusetlvl: stw r21,FPUlevel(r20) ; Save the level
;
; going into user state.
;
-fpuchkena: bt- cr2_eq,fpuhasdfrd ; Skip if deferred, R26 already set up...
+fpuchkena: bt-- cr2_eq,fpuhasdfrd ; Skip if deferred, R26 already set up...
mr r26,r20 ; Use the non-deferred value
-fpuhasdfrd: lwz r21,FPUowner(r31) ; Get the ID of the live context
+fpuhasdfrd:
+#if 0
+ rlwinm. r0,r29,0,MSR_PR_BIT,MSR_PR_BIT ; (TEST/DEBUG) Going into user state?
+ beq fpunusrstt ; (TEST/DEBUG) Nope...
+ lwz r23,FPUlevel(r26) ; (TEST/DEBUG) Get the level ID
+ lwz r24,FPUsave(r26) ; (TEST/DEBUG) Get the first savearea
+ mr. r23,r23 ; (TEST/DEBUG) Should be level 0
+ beq++ fpulvl0 ; (TEST/DEBUG) Yes...
+
+ lis r0,hi16(Choke) ; (TEST/DEBUG) Choke code
+ ori r0,r0,lo16(Choke) ; (TEST/DEBUG) and the rest
+ sc ; (TEST/DEBUG) System ABEND
+
+fpulvl0: mr. r24,r24 ; (TEST/DEBUG) Any context?
+ beq fpunusrstt ; (TEST/DEBUG) No...
+ lwz r23,SAVlevel(r24) ; (TEST/DEBUG) Get level of context
+ lwz r21,SAVprev+4(r24) ; (TEST/DEBUG) Get previous pointer
+ mr. r23,r23 ; (TEST/DEBUG) Is this our user context?
+ beq++ fpulvl0b ; (TEST/DEBUG) Yes...
+
+ lis r0,hi16(Choke) ; (TEST/DEBUG) Choke code
+ ori r0,r0,lo16(Choke) ; (TEST/DEBUG) and the rest
+ sc ; (TEST/DEBUG) System ABEND
+
+fpulvl0b: mr. r21,r21 ; (TEST/DEBUG) Is there a forward chain?
+ beq++ fpunusrstt ; (TEST/DEBUG) Nope...
+
+ lis r0,hi16(Choke) ; (TEST/DEBUG) Choke code
+ ori r0,r0,lo16(Choke) ; (TEST/DEBUG) and the rest
+ sc ; (TEST/DEBUG) System ABEND
+
+fpunusrstt: ; (TEST/DEBUG)
+#endif
+
+ lwz r21,FPUowner(r31) ; Get the ID of the live context
lwz r23,FPUlevel(r26) ; Get the level ID
- cmplw cr3,r26,r21 ; Do we have the live context?
lwz r24,FPUcpu(r26) ; Get the CPU that the context was last dispatched on
- bne- cr3,chkvec ; No, can not possibly enable...
+ cmplw cr3,r26,r21 ; Do we have the live context?
cmplw r30,r23 ; Are we about to launch the live level?
+ bne-- cr3,chkvec ; No, can not possibly enable...
cmplw cr1,r19,r24 ; Was facility used on this processor last?
- bne- chkvec ; No, not live...
- bne- cr1,chkvec ; No, wrong cpu, have to enable later....
+ bne-- chkvec ; No, not live...
+ bne-- cr1,chkvec ; No, wrong cpu, have to enable later....
lwz r24,FPUsave(r26) ; Get the first savearea
mr. r24,r24 ; Any savearea?
- beq+ fpuena ; Nope...
+ beq++ fpuena ; Nope...
lwz r25,SAVlevel(r24) ; Get the level of savearea
- lwz r0,SAVprev(r24) ; Get the previous
+ lwz r0,SAVprev+4(r24) ; Get the previous
+
cmplw r30,r25 ; Is savearea for the level we are launching?
- bne+ fpuena ; No, just go enable...
+ bne++ fpuena ; No, just go enable...
stw r0,FPUsave(r26) ; Pop the chain
rlwinm r3,r24,0,0,19 ; Find main savearea header
- lwz r3,SACvrswap(r3) ; Get the virtual to real conversion
- la r9,quickfret(r31) ; Point to the quickfret chain header
+
+ lwz r8,quickfret(r31) ; Get the first in quickfret list (top)
+ lwz r9,quickfret+4(r31) ; Get the first in quickfret list (bottom)
+ lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
+ lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
+ stw r8,SAVprev(r24) ; Link the old in (top)
+ stw r9,SAVprev+4(r24) ; Link the old in (bottom)
xor r3,r24,r3 ; Convert to physical
+ stw r2,quickfret(r31) ; Set the first in quickfret list (top)
+ stw r3,quickfret+4(r31) ; Set the first in quickfret list (bottom)
#if FPVECDBG
lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
sc ; (TEST/DEBUG)
#endif
-
-fpuhascdq: lwarx r0,0,r9 ; Pick up the old chain head
- stw r0,SAVprev(r24) ; Move it to the current guy
- stwcx. r3,0,r9 ; Save it
- bne- fpuhascdq ; Someone chaged the list...
fpuena: ori r29,r29,lo16(MASK(MSR_FP)) ; Enable facility
chkvec:
-#if 0
- rlwinm. r21,r29,0,MSR_PR_BIT,MSR_PR_BIT ; (TEST/DEBUG)
- beq+ ppppp ; (TEST/DEBUG)
- lwz r21,FPUlevel(r26) ; (TEST/DEBUG)
- mr. r21,r21 ; (TEST/DEBUG)
- bne- qqqqq ; (TEST/DEBUG)
- lwz r21,FPUsave(r26) ; (TEST/DEBUG)
- mr. r21,r21 ; (TEST/DEBUG)
- beq+ ppppp ; (TEST/DEBUG)
- lwz r22,SAVlevel(r21) ; (TEST/DEBUG)
- mr. r22,r22 ; (TEST/DEBUG)
- beq+ ppppp ; (TEST/DEBUG)
-qqqqq:
- BREAKPOINT_TRAP ; (TEST/DEBUG)
-
-ppppp: ; (TEST/DEBUG)
-#endif
-
lwz r21,VMXlevel(r20) ; Get the facility level
cmplw r27,r21 ; Are we returning from the active level?
eieio ; Make sure this gets out before owner clear
- lis r23,hi16(EXT(per_proc_info)) ; Set base per_proc
- mulli r22,r22,ppSize ; Find offset to the owner per_proc
- ori r23,r23,lo16(EXT(per_proc_info)) ; Set base per_proc
- li r24,VMXowner ; Displacement to VMX owner
- add r22,r23,r22 ; Point to the owner per_proc
- li r0,0 ; We need this in a bit
+ lis r23,hi16(EXT(PerProcTable)) ; Set base PerProcTable
+ slwi r22,r22,4 ; Find offset to the owner per_proc_entry
+ ori r23,r23,lo16(EXT(PerProcTable)) ; Set base PerProcTable
+ li r24,VMXowner ; Displacement to float owner
+ add r22,r23,r22 ; Point to the owner per_proc_entry
+ lwz r22,ppe_vaddr(r22) ; Point to the owner per_proc
vmxinvothr: lwarx r23,r24,r22 ; Get the owner
- cmplw r23,r20 ; Does he still have this context?
- bne vmxinvoths ; Nope...
- stwcx. r0,r24,r22 ; Try to invalidate it
- bne- vmxinvothr ; Try again if there was a collision...
-
-vmxinvoths: isync
+
+ sub r0,r23,r20 ; Subtract one from the other
+ sub r21,r20,r23 ; Subtract the other from the one
+ or r21,r21,r0 ; Combine them
+ srawi r21,r21,31 ; Get a 0 if equal or -1 of not
+ and r23,r23,r21 ; Make 0 if same, unchanged if not
+ stwcx. r23,r24,r22 ; Try to invalidate it
+ bne-- vmxinvothr ; Try again if there was a collision...
+
+ isync
;
; Now if there is a savearea associated with the popped context, release it.
lwz r22,VMXsave(r20) ; Get pointer to the first savearea
li r21,0 ; Assume we popped all the way out
mr. r22,r22 ; Is there anything there?
- beq+ vmxsetlvl ; No, see if we need to enable...
+ beq++ vmxsetlvl ; No, see if we need to enable...
lwz r21,SAVlevel(r22) ; Get the level of that savearea
cmplw r21,r27 ; Is this the saved copy of the live stuff?
bne vmxsetlvl ; No, leave as is...
- lwz r24,SAVprev(r22) ; Pick up the previous area
+ lwz r24,SAVprev+4(r22) ; Pick up the previous area
li r21,0 ; Assume we popped all the way out
mr. r24,r24 ; Any more context?
- beq- vmxonlyone ; Nope...
+ beq-- vmxonlyone ; Nope...
lwz r21,SAVlevel(r24) ; Get the level associated with save
vmxonlyone: stw r24,VMXsave(r20) ; Dequeue this savearea
rlwinm r3,r22,0,0,19 ; Find main savearea header
- lwz r3,SACvrswap(r3) ; Get the virtual to real conversion
- la r9,quickfret(r31) ; Point to the quickfret chain header
+
+ lwz r8,quickfret(r31) ; Get the first in quickfret list (top)
+ lwz r9,quickfret+4(r31) ; Get the first in quickfret list (bottom)
+ lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
+ lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
+ stw r8,SAVprev(r22) ; Link the old in (top)
+ stw r9,SAVprev+4(r22) ; Link the old in (bottom)
xor r3,r22,r3 ; Convert to physical
+ stw r2,quickfret(r31) ; Set the first in quickfret list (top)
+ stw r3,quickfret+4(r31) ; Set the first in quickfret list (bottom)
#if FPVECDBG
lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG)
sc ; (TEST/DEBUG)
#endif
-
-vmxhscdq: lwarx r0,0,r9 ; Pick up the old chain head
- stw r0,SAVprev(r22) ; Move it to the current guy
- stwcx. r3,0,r9 ; Save it
- bne- vmxhscdq ; Someone chaged the list...
vmxsetlvl: stw r21,VMXlevel(r20) ; Save the level
lwz r23,VMXlevel(r26) ; Get the level ID
cmplw r26,r21 ; Do we have the live context?
lwz r24,VMXcpu(r26) ; Get the CPU that the context was last dispatched on
- bne- setena ; No, can not possibly enable...
+ bne-- setena ; No, can not possibly enable...
cmplw r30,r23 ; Are we about to launch the live level?
cmplw cr1,r19,r24 ; Was facility used on this processor last?
- bne- setena ; No, not live...
- bne- cr1,setena ; No, wrong cpu, have to enable later....
+ bne-- setena ; No, not live...
+ bne-- cr1,setena ; No, wrong cpu, have to enable later....
lwz r24,VMXsave(r26) ; Get the first savearea
mr. r24,r24 ; Any savearea?
- beq+ vmxena ; Nope...
+ beq++ vmxena ; Nope...
lwz r25,SAVlevel(r24) ; Get the level of savearea
- lwz r0,SAVprev(r24) ; Get the previous
+ lwz r0,SAVprev+4(r24) ; Get the previous
cmplw r30,r25 ; Is savearea for the level we are launching?
- bne+ vmxena ; No, just go enable...
+ bne++ vmxena ; No, just go enable...
stw r0,VMXsave(r26) ; Pop the chain
rlwinm r3,r24,0,0,19 ; Find main savearea header
- lwz r3,SACvrswap(r3) ; Get the virtual to real conversion
- la r9,quickfret(r31) ; Point to the quickfret chain header
+
+ lwz r8,quickfret(r31) ; Get the first in quickfret list (top)
+ lwz r9,quickfret+4(r31) ; Get the first in quickfret list (bottom)
+ lwz r2,SACvrswap(r3) ; Get the virtual to real conversion (top)
+ lwz r3,SACvrswap+4(r3) ; Get the virtual to real conversion (bottom)
+ stw r8,SAVprev(r24) ; Link the old in (top)
+ stw r9,SAVprev+4(r24) ; Link the old in (bottom)
xor r3,r24,r3 ; Convert to physical
+ stw r2,quickfret(r31) ; Set the first in quickfret list (top)
+ stw r3,quickfret+4(r31) ; Set the first in quickfret list (bottom)
#if FPVECDBG
lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG)
sc ; (TEST/DEBUG)
#endif
-vmxckcdq: lwarx r0,0,r9 ; Pick up the old chain head
- stw r0,SAVprev(r24) ; Move it to the current guy
- stwcx. r3,0,r9 ; Save it
- bne- vmxckcdq ; Someone chaged the list...
-
vmxena: oris r29,r29,hi16(MASK(MSR_VEC)) ; Enable facility
-
-setena: rlwinm. r0,r29,0,MSR_PR_BIT,MSR_PR_BIT ; Are we about to launch user state?
- rlwinm r20,r29,(((31-vectorCngbit)+(MSR_VEC_BIT+1))&31),vectorCngbit,vectorCngbit ; Set flag if we enabled vector
- stw r29,savesrr1(r27) ; Turn facility on or off
+setena: lwz r18,umwSpace(r28) ; Get the space ID in case we are launching user
+ rlwinm. r0,r29,0,MSR_PR_BIT,MSR_PR_BIT ; Are we about to launch user state?
+ li r0,0 ; Get set to release quickfret holdoff
crmove cr7_eq,cr0_eq ; Remember if we are going to user state
- lwz r19,deferctx(r28) ; Get any deferred facility context switch
rlwimi. r20,r29,(((31-floatCngbit)+(MSR_FP_BIT+1))&31),floatCngbit,floatCngbit ; Set flag if we enabled floats
+ lwz r19,deferctx(r28) ; Get any deferred facility context switch
+ rlwinm r20,r29,(((31-vectorCngbit)+(MSR_VEC_BIT+1))&31),vectorCngbit,vectorCngbit ; Set flag if we enabled vector
+ stw r29,savesrr1+4(r27) ; Turn facility on or off
+ stw r0,holdQFret(r31) ; Release quickfret
+ oris r18,r18,hi16(umwSwitchAway) ; Set the switch-away bit in case we go to user
+
beq setenaa ; Neither float nor vector turned on....
lwz r5,ACT_MACT_SPF(r28) ; Get activation copy
nodefer: lwz r22,qactTimer(r28) ; Get high order quick activation timer
mr. r24,r24 ; See if it has popped already...
lwz r23,qactTimer+4(r28) ; Get low order qact timer
- ble- chkenax ; We have popped or are just about to...
+ ble- chkifuser ; We have popped or are just about to...
segtb: mftbu r20 ; Get the upper time base
mftb r21 ; Get the low
mftbu r19 ; Get upper again
or. r0,r22,r23 ; Any time set?
cmplw cr1,r20,r19 ; Did they change?
- beq+ chkenax ; No time set....
- bne- cr1,segtb ; Timebase ticked, get them again...
+ beq++ chkifuser ; No time set....
+ bne-- cr1,segtb ; Timebase ticked, get them again...
subfc r6,r21,r23 ; Subtract current from qact time
li r0,0 ; Make a 0
subfze r0,r0 ; Get a 0 if qact was bigger than current, -1 otherwise
andc. r12,r5,r0 ; Set 0 if qact has passed
andc r13,r6,r0 ; Set 0 if qact has passed
- bne chkenax ; If high order is non-zero, this is too big for a decrementer
+ bne chkifuser ; If high order is non-zero, this is too big for a decrementer
cmplw r13,r24 ; Is this earlier than the decrementer? (logical compare takes care of high bit on)
- bge+ chkenax ; No, do not reset decrementer...
+ bge++ chkifuser ; No, do not reset decrementer...
mtdec r13 ; Set our value
+chkifuser: bl EXT(mach_absolute_time)
+ lwz r5,ACT_PER_PROC(r28)
+ addi r6,r5,PP_PROCESSOR
+ lwz r5,KERNEL_TIMER(r6)
+ lwz r29,CURRENT_STATE(r6)
+ beq-- cr7,chkifuser1 ; Skip this if we are going to kernel...
+ stw r18,umwSpace(r28) ; Half-invalidate to force MapUserAddressWindow to reload SRs
+ addi r5,r28,USER_TIMER
+ addi r29,r6,USER_STATE
+
+chkifuser1: bl EXT(thread_timer_event)
+ mr r5,r29
+ bl EXT(state_event)
+
chkenax:
-
#if DEBUG
lwz r20,SAVact(r27) ; (TEST/DEBUG) Make sure our restore
- lwz r21,PP_ACTIVE_THREAD(r31) ; (TEST/DEBUG) with the current act.
+ mfsprg r21, 1 ; (TEST/DEBUG) with the current act.
cmpwi r21,0 ; (TEST/DEBUG)
- beq- yeswereok ; (TEST/DEBUG)
- lwz r21,THREAD_TOP_ACT(r21) ; (TEST/DEBUG)
+ beq-- yeswereok ; (TEST/DEBUG)
cmplw r21,r20 ; (TEST/DEBUG)
- beq+ yeswereok ; (TEST/DEBUG)
+ beq++ yeswereok ; (TEST/DEBUG)
lis r0,hi16(Choke) ; (TEST/DEBUG) Choke code
ori r0,r0,lo16(Choke) ; (TEST/DEBUG) and the rest
yeswereok:
#endif
- rlwinm r5,r27,0,0,19 ; Round savearea down to page bndry
- lwz r5,SACvrswap(r5) ; Get the conversion from virtual to real
- xor r3,r27,r5 ; Flip to physical address
+ mr r3,r27 ; Pass savearea back
b EXT(exception_exit) ; We are all done now...
+;
+; Null PPC call - performance testing, does absolutely nothing
+;
+
+ .align 5
+
+ .globl EXT(ppcNull)
+
+LEXT(ppcNull)
+
+ li r3,-1 ; Make sure we test no asts
+ blr
+
+
+;
+; Instrumented null PPC call - performance testing, does absolutely nothing
+; Forces various timestamps to be returned.
+;
+
+ .align 5
+
+ .globl EXT(ppcNullinst)
+
+LEXT(ppcNullinst)
+
+ li r3,-1 ; Make sure we test no asts
+ blr
+
+
/*
* Here's where we handle the fastpath stuff
* We'll do what we can here because registers are already
.align 5
-fastpath: cmplwi cr3,r0,0x7FF1 ; Is it CthreadSetSelfNumber?
- bnelr- cr3 ; Not a fast path...
+fastpath: cmplwi cr3,r0,0x7FF5 ; Is this a null fastpath?
+ beq-- cr3,fastexutl ; Yes, bail fast...
+ cmplwi cr3,r0,0x7FF1 ; Is it CthreadSetSelfNumber?
+ bnelr-- cr3 ; Not a fast path...
/*
* void cthread_set_self(cproc_t p)
*
- * set's thread state "user_value"
+ * Set's thread state "user_value". In practice this is the thread-local-data-pointer (TLDP),
+ * though we do not interpret it. This call is mostly used by 32-bit tasks, but we save all 64 bits
+ * in case a 64-bit task wants to use this facility. They normally do not, because the 64-bit
+ * ABI reserves r13 for the TLDP.
*
* This op is invoked as follows:
* li r0, CthreadSetSelfNumber // load the fast-trap number
* sc // invoke fast-trap
* blr
- *
*/
CthreadSetSelfNumber:
-
- lwz r5,saver3(r4) /* Retrieve the self number */
- stw r5,CTHREAD_SELF(r13) /* Remember it */
- stw r5,UAW(r25) /* Prime the per_proc_info with it */
+ lwz r3,saver3+0(r4) /* get the TLDP passed in r3 */
+ lwz r5,saver3+4(r4) /* (all 64 bits, in case this is a 64-bit task) */
+ stw r3,CTHREAD_SELF+0(r13) /* Remember it in the activation... */
+ stw r5,CTHREAD_SELF+4(r13)
+ stw r3,UAW+0(r25) /* ...and in the per-proc */
+ stw r5,UAW+4(r25)
.globl EXT(fastexit)
EXT(fastexit):
- rlwinm r9,r4,0,0,19 /* Round down to the base savearea block */
- lwz r9,SACvrswap(r9) /* Get the conversion from virtual to real */
- xor r3,r4,r9 /* Switch savearea to physical addressing */
- b EXT(exception_exit) /* Go back to the caller... */
+fastexutl: mr r3,r4 ; Pass back savearea
+ b EXT(exception_exit) ; Go back to the caller...
/*
checkassist:
lwz r0,saveexception(r4) ; Get the exception code
- lwz r23,savesrr1(r4) ; Get the interrupted MSR
+ lwz r23,savesrr1+4(r4) ; Get the interrupted MSR
lwz r26,ACT_MACT_BEDA(r13) ; Get Blue Box Descriptor Area
mtcrf 0x18,r23 ; Check what SRR1 says
lwz r24,ACT_MACT_BTS(r13) ; Get the table start
cmplwi r0,T_AST ; Check for T_AST trap
- lwz r27,savesrr0(r4) ; Get trapped address
+ lwz r27,savesrr0+4(r4) ; Get trapped address
crnand cr1_eq,SRR1_PRG_TRAP_BIT,MSR_PR_BIT ; We need both trap and user state
sub r24,r27,r24 ; See how far into it we are
cror cr0_eq,cr0_eq,cr1_eq ; Need to bail if AST or not trap or not user state
.globl EXT(retFromVM)
LEXT(retFromVM)
- mfsprg r10,0 ; Restore the per_proc info
+ mfsprg r10,1 ; Get the current activation
+ lwz r10,ACT_PER_PROC(r10) ; Get the per_proc block
mr r8,r3 ; Get the activation
- lwz r4,SAVprev(r30) ; Pick up the previous savearea
+ lwz r4,SAVprev+4(r30) ; Pick up the previous savearea
mr r3,r30 ; Put savearea in proper register for common code
lwz r11,SAVflags(r30) ; Get the flags of the current savearea
rlwinm r11,r11,0,15,13 ; Clear the syscall flag
- lwz r1,ACT_THREAD(r8) ; and the active thread
+ mr r1,r8
stw r11,SAVflags(r3) ; Save back the flags (with reset stack cleared)
stw r4,ACT_MACT_PCB(r8) ; Point to the previous savearea (or 0 if none)
.align 5
.globl EXT(chandler)
-LEXT(chandler) /* Choke handler */
+LEXT(chandler) ; Choke handler
- lis r25,hi16(EXT(trcWork)) ; (TEST/DEBUG)
- li r31,0 ; (TEST/DEBUG)
- ori r25,r25,lo16(EXT(trcWork)) ; (TEST/DEBUG)
- stw r31,traceMask(r25) ; (TEST/DEBUG)
+ li r31,0 ; Get a 0
+ mfsprg r25,1 ; Get the current activation
+ lwz r25,ACT_PER_PROC(r25) ; Get the per_proc block
+ stw r31,traceMask(0) ; Force tracing off right now
- mfsprg r25,0 ; Get the per_proc
lwz r1,PP_DEBSTACKPTR(r25) ; Get debug stack pointer
cmpwi r1,-1 ; Are we already choking?
chokefirst: li r0,-1 ; Set choke value
mr. r1,r1 ; See if we are on debug stack yet
- lwz r10,saver1(r4) ;
+ lwz r10,saver1+4(r4) ;
stw r0,PP_DEBSTACKPTR(r25) ; Show we are choking
bne chokestart ; We are not on the debug stack yet...
;
versave:
-#if 1
+#if 0
+ lis r22,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
+ ori r22,r22,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
+ lwz r23,0(r22) ; (TEST/DEBUG)
+ mr. r23,r23 ; (TEST/DEBUG)
+ beqlr- ; (TEST/DEBUG)
+ mfsprg r20,1 ; Get the current activation
+ lwz r20,ACT_PER_PROC(r20) ; Get the per_proc block
+ lwz r21,pfAvailable(r20) ; (TEST/DEBUG)
+ mr. r21,r21 ; (TEST/DEBUG)
+ bnelr+ ; (TEST/DEBUG)
+
+ stw r22,0(r22) ; (TEST/DEBUG) Lock out more checks
+ BREAKPOINT_TRAP ; (TEST/DEBUG) Get into debugger
+#endif
+
+#if 0
+ ;; This code is broken and migration will make the matter even worse
;
; Make sure that all savearea chains have the right type on them
;
stw r22,0(r22) ; (TEST/DEBUG) Lock out more checks
BREAKPOINT_TRAP ; (TEST/DEBUG) Get into debugger
-versvok: lwz r20,SAVprev(r20) ; (TEST/DEBUG) Get the previous one
+versvok: lwz r20,SAVprev+4(r20) ; (TEST/DEBUG) Get the previous one
b versavetype ; (TEST/DEBUG) Go check its type...
#endif
-#if 0
-;
-; Make sure there are no circular links in the float chain
-; And that FP is marked busy in it.
-; And the only the top is marked invalid.
-; And that the owning PCB is correct.
-;
-
- lis r28,hi16(EXT(default_pset)) ; (TEST/DEBUG)
- lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r28,r28,lo16(EXT(default_pset)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- li r20,0 ; (TEST/DEBUG)
- lwz r26,0(r27) ; (TEST/DEBUG)
- lwz r27,psthreadcnt(r28) ; (TEST/DEBUG)
- mr. r26,r26 ; (TEST/DEBUG)
- lwz r28,psthreads(r28) ; (TEST/DEBUG)
- bnelr- ; (TEST/DEBUG)
-
-fcknxtth: mr. r27,r27 ; (TEST/DEBUG)
- beqlr- ; (TEST/DEBUG)
-
- lwz r26,THREAD_TOP_ACT(r28) ; (TEST/DEBUG)
-
-fckact: mr. r26,r26 ; (TEST/DEBUG)
- bne+ fckact2 ; (TEST/DEBUG)
-
- lwz r28,THREAD_PSTHRN(r28) ; (TEST/DEBUG) Next in line
- subi r27,r27,1 ; (TEST/DEBUG)
- b fcknxtth ; (TEST/DEBUG)
-
-fckact2: lwz r20,ACT_MACT_FPU(r26) ; (TEST/DEBUG) Get FPU chain
- li r29,1 ; (TEST/DEBUG)
- li r22,0 ; (TEST/DEBUG)
-
-fckact3: mr. r20,r20 ; (TEST/DEBUG) Are there any?
- beq+ fckact5 ; (TEST/DEBUG) No...
-
- addi r22,r22,1 ; (TEST/DEBUG) Count chain depth
-
- lwz r21,SAVflags(r20) ; (TEST/DEBUG) Get the flags
- rlwinm. r21,r21,0,1,1 ; (TEST/DEBUG) FP busy?
- bne+ fckact3a ; (TEST/DEBUG) Yeah...
- lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- stw r27,0(r27) ; (TEST/DEBUG)
- BREAKPOINT_TRAP ; (TEST/DEBUG) Die
-
-fckact3a: cmplwi r22,1 ; (TEST/DEBUG) At first SA?
- beq+ fckact3b ; (TEST/DEBUG) Yeah, invalid is ok...
- lwz r21,SAVlvlfp(r20) ; (TEST/DEBUG) Get level
- cmplwi r21,1 ; (TEST/DEBUG) Is it invalid?
- bne+ fckact3b ; (TEST/DEBUG) Nope, it is ok...
- lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- stw r27,0(r27) ; (TEST/DEBUG)
- BREAKPOINT_TRAP ; (TEST/DEBUG) Die
-
-fckact3b: lwz r21,SAVact(r20) ; (TEST/DEBUG) Get the owner
- cmplw r21,r26 ; (TEST/DEBUG) Correct activation?
- beq+ fckact3c ; (TEST/DEBUG) Yup...
- lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- stw r27,0(r27) ; (TEST/DEBUG)
- BREAKPOINT_TRAP ; (TEST/DEBUG) Die
-
-fckact3c: ; (TEST/DEBUG)
- lbz r21,SAVflags+3(r20) ; (TEST/DEBUG) Pick up the test byte
- mr. r21,r21 ; (TEST/DEBUG) marked?
- beq+ fckact4 ; (TEST/DEBUG) No, good...
-
- lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- stw r27,0(r27) ; (TEST/DEBUG)
- BREAKPOINT_TRAP ; (TEST/DEBUG)
-
-fckact4: stb r29,SAVflags+3(r20) ; (TEST/DEBUG) Set the test byte
- lwz r20,SAVprefp(r20) ; (TEST/DEBUG) Next in list
- b fckact3 ; (TEST/DEBUG) Try it...
-
-fckact5: lwz r20,ACT_MACT_FPU(r26) ; (TEST/DEBUG) Get FPU chain
- li r29,0 ; (TEST/DEBUG)
-fckact6: mr. r20,r20 ; (TEST/DEBUG) Are there any?
- beq+ fcknact ; (TEST/DEBUG) No...
-
- stb r29,SAVflags+3(r20) ; (TEST/DEBUG) Clear the test byte
- lwz r20,SAVprefp(r20) ; (TEST/DEBUG) Next in list
- b fckact6 ; (TEST/DEBUG) Try it...
-
-fcknact: lwz r26,ACT_LOWER(r26) ; (TEST/DEBUG) Next activation
- b fckact ; (TEST/DEBUG)
-#endif
-
-
-#if 0
-;
-; Make sure in use count matches found savearea. This is
-; not always accurate. There is a variable "fuzz" factor in count.
-
- lis r28,hi16(EXT(default_pset)) ; (TEST/DEBUG)
- lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r28,r28,lo16(EXT(default_pset)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- li r20,0 ; (TEST/DEBUG)
- lwz r26,0(r27) ; (TEST/DEBUG)
- lwz r27,psthreadcnt(r28) ; (TEST/DEBUG)
- mr. r26,r26 ; (TEST/DEBUG)
- lwz r28,psthreads(r28) ; (TEST/DEBUG)
- bnelr- ; (TEST/DEBUG)
-
-cknxtth: mr. r27,r27 ; (TEST/DEBUG)
- beq- cktotal ; (TEST/DEBUG)
-
- lwz r26,THREAD_TOP_ACT(r28) ; (TEST/DEBUG)
-
-ckact: mr. r26,r26 ; (TEST/DEBUG)
- bne+ ckact2 ; (TEST/DEBUG)
-
- lwz r28,THREAD_PSTHRN(r28) ; (TEST/DEBUG) Next in line
- subi r27,r27,1 ; (TEST/DEBUG)
- b cknxtth ; (TEST/DEBUG)
-
-ckact2: lwz r29,ACT_MACT_PCB(r26) ; (TEST/DEBUG)
-
-cknorm: mr. r29,r29 ; (TEST/DEBUG)
- beq- cknormd ; (TEST/DEBUG)
-
- addi r20,r20,1 ; (TEST/DEBUG) Count normal savearea
-
- lwz r29,SAVprev(r29) ; (TEST/DEBUG)
- b cknorm ; (TEST/DEBUG)
-
-cknormd: lwz r29,ACT_MACT_FPU(r26) ; (TEST/DEBUG)
-
-ckfpu: mr. r29,r29 ; (TEST/DEBUG)
- beq- ckfpud ; (TEST/DEBUG)
-
- lwz r21,SAVflags(r29) ; (TEST/DEBUG)
- rlwinm. r21,r21,0,0,0 ; (TEST/DEBUG) See if already counted
- bne- cknfpu ; (TEST/DEBUG)
-
- addi r20,r20,1 ; (TEST/DEBUG) Count fpu savearea
-
-cknfpu: lwz r29,SAVprefp(r29) ; (TEST/DEBUG)
- b ckfpu ; (TEST/DEBUG)
-
-ckfpud: lwz r29,ACT_MACT_VMX(r26) ; (TEST/DEBUG)
-
-ckvmx: mr. r29,r29 ; (TEST/DEBUG)
- beq- ckvmxd ; (TEST/DEBUG)
-
- lwz r21,SAVflags(r29) ; (TEST/DEBUG)
- rlwinm. r21,r21,0,0,1 ; (TEST/DEBUG) See if already counted
- bne- cknvmx ; (TEST/DEBUG)
-
- addi r20,r20,1 ; (TEST/DEBUG) Count vector savearea
-
-cknvmx: lwz r29,SAVprevec(r29) ; (TEST/DEBUG)
- b ckvmx ; (TEST/DEBUG)
-
-ckvmxd: lwz r26,ACT_LOWER(r26) ; (TEST/DEBUG) Next activation
- b ckact ; (TEST/DEBUG)
-
-cktotal: lis r28,hi16(EXT(saveanchor)) ; (TEST/DEBUG)
- lis r27,hi16(EXT(real_ncpus)) ; (TEST/DEBUG)
- ori r28,r28,lo16(EXT(saveanchor)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(real_ncpus)) ; (TEST/DEBUG)
-
- lwz r21,SVinuse(r28) ; (TEST/DEBUG)
- lwz r27,0(r27) ; (TEST/DEBUG) Get the number of CPUs
- sub. r29,r21,r20 ; (TEST/DEBUG) Get number accounted for
- blt- badsave ; (TEST/DEBUG) Have too many in use...
- sub r26,r29,r27 ; (TEST/DEBUG) Should be 1 unaccounted for for each processor
- cmpwi r26,10 ; (TEST/DEBUG) Allow a 10 area slop factor
- bltlr+ ; (TEST/DEBUG)
-
-badsave: lis r27,hi16(EXT(DebugWork)) ; (TEST/DEBUG)
- ori r27,r27,lo16(EXT(DebugWork)) ; (TEST/DEBUG)
- stw r27,0(r27) ; (TEST/DEBUG)
- BREAKPOINT_TRAP ; (TEST/DEBUG)
-#endif
#endif
projects / apple / xnu.git / blobdiff