X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d12e16782ebf8bb779633dff9e14486293bf6d07..0c530ab8987f0ae6a1a3d9284f40182b88852816:/osfmk/ppc/Emulate.s diff --git a/osfmk/ppc/Emulate.s b/osfmk/ppc/Emulate.s index b05eb99ba..7a4ef3ab9 100644 --- a/osfmk/ppc/Emulate.s +++ b/osfmk/ppc/Emulate.s @@ -29,14 +29,13 @@ */ -#include #include #include #include +#include #include #include -#define kernAccess 31 #define traceInst 30 #define dssAllDone 29 @@ -64,13 +63,13 @@ LEXT(Emulate) + bf-- pf64Bitb,emn64 ; Skip if not 64-bit + b EXT(Emulate64) ; Jump to the 64-bit code... - mfsprg r31,0 ; Get the per_proc - lis r30,hi16(EXT(dgWork)) ; Get the high half of diagnostic work area - lwz r12,savesrr1(r13) ; Get the exception info - ori r30,r30,lo16(EXT(dgWork)) ; And the low half +emn64: mfsprg r31,0 ; Get the per_proc + lwz r12,savesrr1+4(r13) ; Get the exception info rlwinm. r0,r12,0,SRR1_PRG_ILL_INS_BIT,SRR1_PRG_ILL_INS_BIT ; Emulation candidate? - lwz r30,dgFlags(r30) ; Get the flags + lwz r30,dgFlags(0) ; Get the flags beq+ eExit ; Nope, do not try to emulate... rlwinm. r0,r30,0,enaDiagEMb,enaDiagEMb ; Do we want to try to emulate something? @@ -111,9 +110,9 @@ eNoVect: bl eIFetch ; Get the instruction image cror cr1_eq,cr1_eq,cr0_eq ; Remember bne cr1_eq,eNotIndex ; Go check non-index forms... - rlwinm. r21,r10,18,25,29 ; Extract index to rA to build EA - rlwinm r22,r10,23,25,29 ; Extract index to rB - addi r24,r13,saver0 ; Point to the start of registers + rlwinm. r21,r10,19,24,28 ; Extract index to rA to build EA + rlwinm r22,r10,24,24,28 ; Extract index to rB + addi r24,r13,saver0+4 ; Point to the start of registers li r19,0 ; Assume 0 base beq eZeroBase ; Yes... lwzx r19,r24,r21 ; Get the base register value @@ -128,15 +127,15 @@ eNotIndex: cmplwi r0,725 ; stswi? cror cr1_eq,cr1_eq,cr0_eq ; Remember bne cr1,eExit ; Not one we handle... - rlwinm. r21,r10,18,25,29 ; Extract index to rA to build EA - addi r24,r13,saver0 ; Point to the start of registers + rlwinm. r21,r10,19,24,28 ; Extract index to rA to build EA + addi r24,r13,saver0+4 ; Point to the start of registers li r22,0 ; Assume 0 base beq eFinishUp ; Yes, it is... lwzx r22,r24,r21 ; Get the base register value eFinishUp: stw r20,savedsisr(r13) ; Set the DSISR li r11,T_ALIGNMENT ; Get the exception code - stw r22,savedar(r13) ; Save the DAR + stw r22,savedar+4(r13) ; Save the DAR stw r11,saveexception(r13) ; Set the exception code b EXT(AlignAssist) ; Go emulate the handler... @@ -153,20 +152,15 @@ eExit: b EXT(EmulExit) ; Just return for now... .align 5 -eIFetch: lwz r23,savesrr1(r13) ; Get old MSR +eIFetch: lwz r23,savesrr1+4(r13) ; Get old MSR mflr r28 ; Save return rlwinm. r22,r23,0,MSR_PR_BIT,MSR_PR_BIT ; Within kernel? mfmsr r30 ; Save the MSR for now - lwz r23,savesrr0(r13) ; Get instruction address - crmove kernAccess,cr0_eq ; Remember if fault was in kernel - li r25,4 ; Set access length - or r22,r22,r30 ; Add PR to access MSR + lwz r23,savesrr0+4(r13) ; Get instruction address - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to - - ori r22,r22,lo16(MASK(MSR_DR)|MASK(MSR_RI)) ; Set RI onto access MSR + ori r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI)) ; Set RI and DR onto access MSR crset cr0_eq ; Set this to see if we failed mtmsr r22 ; Flip DR, RI, and maybe PR on @@ -174,15 +168,10 @@ eIFetch: lwz r23,savesrr1(r13) ; Get old MSR lwz r10,0(r23) ; Fetch the instruction - crmove 28,cr0_eq ; Remember if we failed - li r0,loadMSR ; Set the magic "get back to supervisor" SC - mr r3,r30 ; Get MSR to load - sc ; Get back to supervisor state - - bfl+ kernAccess,aaUnSetSegs ; Go set SRs if we are in user and need to + mtmsr r30 ; Trans and RI off + isync mtlr r28 ; Restore the LR - crmove cr0_eq,28 ; Set CR0_EQ if the fetch succeeded blr ; Return with instruction image in R10 @@ -191,14 +180,14 @@ eIFetch: lwz r23,savesrr1(r13) ; Get old MSR ; eRedriveAsISI: - lwz r6,savesrr1(r13) ; Get the srr1 value + lwz r6,savesrr1+4(r13) ; Get the srr1 value lwz r4,SAVflags(r13) ; Pick up the flags li r11,T_INSTRUCTION_ACCESS ; Set failing instruction fetch code rlwimi r6,r1,0,0,4 ; Move the DSISR bits to the SRR1 oris r4,r4,hi16(SAVredrive) ; Set the redrive bit stw r11,saveexception(r13) ; Set the replacement code stw r4,SAVflags(r13) ; Set redrive request - stw r6,savesrr1(r13) ; Set the srr1 value + stw r6,savesrr1+4(r13) ; Set the srr1 value b EXT(EmulExit) ; Bail out to handle ISI... @@ -225,55 +214,42 @@ eRedriveAsISI: .globl EXT(AlignAssist) LEXT(AlignAssist) - -#if 0 - b EXT(EmulExit) ; Just return for now... -#endif - - + bf-- pf64Bitb,aan64 ; Skip if not 64-bit + b EXT(AlignAssist64) ; Jump to the 64-bit code... + +aan64: lwz r20,savedsisr(r13) ; Get the DSISR + li r0,0 ; Assume we emulate mfsprg r31,0 ; Get the per_proc - lwz r20,savedsisr(r13) ; Get the DSISR + mtcrf 0x10,r20 ; Put instruction ID in CR for later lwz r21,spcFlags(r31) ; Grab the special flags - mtcrf 0x1C,r20 ; Put instruction ID in CR for later + stw r0,savemisc3(r13) ; Assume that we emulate ok + mtcrf 0x08,r20 ; Put instruction ID in CR for later rlwinm. r0,r21,0,runningVMbit,runningVMbit ; Are we running a VM? - lwz r22,savesrr1(r13) ; Get the SRR1 + mtcrf 0x04,r20 ; Put instruction ID in CR for later + lwz r22,savesrr1+4(r13) ; Get the SRR1 bne- aaPassAlong ; We are in a VM, no emulation for alignment exceptions... - rlwinm. r0,r21,0,trapUnalignbit,trapUnalignbit ; Should we trap alignment exceptions? + lwz r19,dgFlags(0) ; Get the diagnostics flags crxor iFloat,iOptype1,iOptype2 ; Set this to 0 if both bits are either 0 or 1 mr r26,r20 ; Save the DSISR - bne- aaPassAlong ; No alignment exceptions allowed... rlwinm. r0,r22,0,MSR_SE_BIT,MSR_SE_BIT ; Were we single stepping? - lwz r23,savedar(r13) ; Pick up the address that we want to access + lwz r23,savedar+4(r13) ; Pick up the address that we want to access crnot traceInst,cr0_eq ; Remember if trace is on - rlwinm. r0,r21,0,notifyUnalignbit,notifyUnalignbit ; Should we notify that an alignment exception happened? - mfsprg r28,2 ; Get the processor features - crnot iNotify,cr0_eq ; Remember to tell someone we did this - rlwinm. r22,r22,0,MSR_PR_BIT,MSR_PR_BIT ; Did we take the exception in the kernel and isolate PR? + + rlwinm. r0,r19,0,enaNotifyEMb,enaNotifyEMb ; Should we notify that an alignment exception happened? mfmsr r30 ; Save the MSR for now + crnot iNotify,cr0_eq ; Remember to tell someone we did this li r29,emfp0 ; Point to work area crxor iFloat,iFloat,iOptype3 ; Set true if we have a floating point instruction - or r22,r22,r30 ; Add PR to access MSR dcbz r29,r31 ; Clear and allocate a cache line for us to work in - rlwinm r24,r20,2,25,29 ; Get displacement to register to update if update form + rlwinm r24,r20,3,24,28 ; Get displacement to register to update if update form rlwimi r20,r20,24,28,28 ; Move load/store indication to the bottom of index - ori r22,r22,lo16(MASK(MSR_DR)|MASK(MSR_RI)) ; Set RI onto access MSR - crmove kernAccess,cr0_eq ; Remember if fault was in kernel - rlwinm. r28,r28,0,pfAltivecb,pfAltivecb ; Do we have Altivec on this machine? + ori r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI)) ; Set RI onto access MSR rlwimi r20,r20,26,27,27 ; Move single/double indication to just above the bottom - beq aaNoVect ; Nope, no Altivec... - - dssall ; We need to kill streams because we are going to flip to problem state - sync - -aaNoVect: lis r29,hi16(aaFPopTable) ; High part of FP branch table + lis r29,hi16(EXT(aaFPopTable)) ; High part of FP branch table bf- iFloat,aaNotFloat ; This is not a floating point instruction... - li r25,8 ; Assume 8-byte access for now - ori r29,r29,lo16(aaFPopTable) ; Low part of FP branch table - bt iDouble,aaFPis8 ; So far, we think we are a double... - li r25,4 ; Set word access + ori r29,r29,lo16(EXT(aaFPopTable)) ; Low part of FP branch table -aaFPis8: rlwimi r29,r20,0,22,28 ; Index into table based upon register||iDouble||iStore - ori r0,r30,lo16(MASK(MSR_FP)) ; Turn on floating point + rlwimi r29,r20,0,22,28 ; Index into table based upon register||iDouble||iStore mtctr r29 ; Get set to call the function bt iStore,aaFPstore ; This is an FP store... @@ -281,11 +257,8 @@ aaFPis8: rlwimi r29,r20,0,22,28 ; Index into table based upon register||iDou ; Here we handle floating point loads ; -aaFPload: bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to - - crset cr0_eq ; Set this to see if we failed - ori r3,r30,lo16(MASK(MSR_FP)) ; We will need FP on in a bit, so turn on when we ditch problem state - mtmsr r22 ; Flip DR, RI, and maybe PR on +aaFPload: crset cr0_eq ; Set this to see if we failed + mtmsr r22 ; Flip DR, RI isync lwz r10,0(r23) ; Get the first word @@ -294,8 +267,9 @@ aaFPload: bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to lwz r11,4(r23) ; Get the second half aaLdNotDbl: mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state and turn on FP + + mtmsr r30 ; Turn off translation again + isync bf- cr0_eq,aaRedriveAsDSI ; Go redrive this as a DSI... @@ -312,24 +286,17 @@ aaLdNotDbl: mr r4,r0 ; Save the DAR if we failed the access .align 5 -aaFPstore: mtmsr r0 ; We need floating point on for the first phase - isync - - bctrl ; Go save the source FP register +aaFPstore: bctrl ; Go save the source FP register lwz r10,emfp0(r31) ; Get first word crandc iDouble,iDouble,iOptype4 ; Change to 4-byte access if stfiwx lwz r11,emfp0+4(r31) ; and the second bf+ iOptype4,aaNotstfiwx ; This is not a stfiwx... - li r25,4 ; Set this is a word mr r10,r11 ; The stfiwx wants to store the second half aaNotstfiwx: - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to - crset cr0_eq ; Set this to see if we failed - mr r3,r30 ; Set the normal MSR - mtmsr r22 ; Flip DR, RI, and maybe PR on + mtmsr r22 ; Flip DR, RI isync stw r10,0(r23) ; Save the first word @@ -338,31 +305,26 @@ aaNotstfiwx: stw r11,4(r23) ; Save the second half aaStNotDbl: mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state - + mtmsr r30 ; Turn off + isync bf- cr0_eq,aaRedriveAsDSI ; Go redrive this as a DSI... - - ; ; Common exit routines ; -aaComExit: lwz r10,savesrr0(r13) ; Get the failing instruction address +aaComExit: lwz r10,savesrr0+4(r13) ; Get the failing instruction address add r24,r24,r13 ; Offset to update register li r11,T_IN_VAIN ; Assume we are all done addi r10,r10,4 ; Step to the next instruction bf iUpdate,aaComExNU ; Skip if not an update form... - stw r23,saver0(r24) ; Update the target + stw r23,saver0+4(r24) ; Update the target aaComExNU: lwz r9,SAVflags(r13) ; Get the flags - stw r10,savesrr0(r13) ; Set new PC + stw r10,savesrr0+4(r13) ; Set new PC bt- traceInst,aaComExitrd ; We are tracing, go emulate trace... bf+ iNotify,aaComExGo ; Nothing special here, go... - - bfl+ kernAccess,aaUnSetSegs ; Go set SRs if we are in user and need to li r11,T_ALIGNMENT ; Set the we just did an alignment exception.... @@ -372,15 +334,11 @@ aaComExGo: b EXT(EmulExit) ; We are done, no tracing on... ; ; This is not a floating point operation ; -; The emulation routines for these are positioned every 64 bytes (16 instructions) -; in a 1024-byte aligned table. It is indexed by taking the low order 4 bits of +; The table of these emulation routines is indexed by taking the low order 4 bits of ; the instruction code in the DSISR and subtracting 7. If this comes up negative, ; the instruction is not to be emulated. Then we add bit 0 of the code * 4. This ; gives us a fairly compact and almost unique index. Both lwm and stmw map to 0 so -; that one needs to be further reduced, and we end up with holes at index 6, 8, and 10. -; -; If the emulation routine takes more than 16 instructions, it must branch elsewhere -; to finish up. +; that one needs to be further reduced, and we end up with holes at a few indexes. ; .align 5 @@ -393,7 +351,7 @@ aaNotFloat: ori r19,r19,lo16(aaEmTable) ; Low part of table address blt- aaPassAlong ; We do not handle any of these (lwarx, stwcx., eciwx, ecowx)... add r19,r19,r3 ; Point to emulation routine - rlwinm r18,r26,29,25,29 ; Get the target/source register displacement + rlwinm r18,r26,30,24,28 ; Get the target/source register displacement mtctr r19 ; Set the routine address @@ -401,9 +359,7 @@ aaNotFloat: ; ; This is the table of non-floating point emulation routines. -; It is indexed by low 4 bits of DSISR op type - 7 + bit 0 of -; op type * 4 -; +; It is indexed by the code immediately above. .align 5 @@ -435,14 +391,12 @@ aaEmTable: .align 5 aaLmwStmw: - subfic r25,r18,32*4 ; Calculate the length of the transfer + rlwinm r17,r18,31,1,29 ; Convert doublword based index to words li r28,0 ; Set no extra bytes to move (used for string instructions) - mr r17,r25 ; Save the word transfer length here + subfic r17,r17,32*4 ; Calculate the length of the transfer -aaLSComm: addi r19,r13,saver0 ; Offset to registers in savearea +aaLSComm: addi r19,r13,saver0+4 ; Offset to registers in savearea mr r16,r23 ; Make a hunk pointer - - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to bt iUpdate,aaStmw ; This is the stmw... @@ -455,7 +409,7 @@ aaLmwNxt: cmplwi cr1,r17,8*4 ; Is there enough to move 8? subi r17,r17,8*4 ; Back off for another hunk crset cr0_eq ; Set this to see if we failed - mtmsr r22 ; Flip DR, RI, and maybe PR on + mtmsr r22 ; Flip DR, RI isync lwz r2,0(r16) ; Load word 0 @@ -475,38 +429,37 @@ aaLmwNxt: cmplwi cr1,r17,8*4 ; Is there enough to move 8? lwz r9,28(r16) ; Load word 7 aaLmwB1: mr r4,r0 ; Remember DAR, jus in case we failed the access - mr r3,r30 ; Set the normal MSR - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Turn off DR, RI + isync bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... addi r16,r16,8*4 ; Point up to next input aread stwx r2,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r15,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r14,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r5,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r6,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r7,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r8,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r9,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed b aaLmwNxt ; Do the next hunk... @@ -530,26 +483,25 @@ aaLmwNxtH: cmplwi cr1,r17,4*4 ; Do we have 4 left? lwz r5,12(r16) ; Load word 3 aaLmwB2: mr r4,r0 ; Remember DAR, jus in case we failed the access - mr r3,r30 ; Set the normal MSR - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Turn off DR, RI + isync bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... addi r16,r16,4*4 ; Point up to next input aread stwx r2,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r15,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r14,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed stwx r5,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed aaLmwL4: or. r5,r17,r28 ; Do we have anything left? cmplwi cr1,r17,(2*4) ; Do we have one, two, or three full words left? @@ -599,26 +551,27 @@ aaLmwDn: rlwinm r5,r5,24,0,7 ; Move first byte to top cmplwi cr1,r17,(2*4) ; Do we have one, two, or three full words left? mr r3,r30 ; Set the normal MSR rlwimi r5,r9,8,8,23 ; Move bytes 1 and 2 after 0 - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + + mtmsr r30 ; Turn off DR, RI + isync bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... beq- cr2,aaLmwCb ; No full words, copy bytes... stwx r2,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed blt cr1,aaLmwCb ; We only had one, we are done... stwx r15,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed beq cr1,aaLmwCb ; We had two, we are done... stwx r14,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed aaLmwCb: mr. r28,r28 ; Any trailing bytes to do? beq+ aaComExit ; Nope, leave... @@ -641,29 +594,29 @@ aaStmwNxt: cmplwi cr1,r17,8*4 ; Is there enough to move 8? subi r17,r17,8*4 ; Back off for another hunk lwzx r2,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r15,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r14,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r5,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r6,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r7,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r8,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r9,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed crset cr0_eq ; Set this to see if we failed mtmsr r22 ; Flip DR, RI, and maybe PR on @@ -689,9 +642,8 @@ aaStmwNxt: cmplwi cr1,r17,8*4 ; Is there enough to move 8? aaStmwB1: mr r4,r0 ; Remember DAR, jus in case we failed the access - mr r3,r30 ; Set the normal MSR - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Normal MSR + isync bt- cr0_eq,aaStmwNxt ; We have more to do and no failed access... b aaRedriveAsDSI ; We failed, go redrive this as a DSI... @@ -703,20 +655,20 @@ aaStmwNxtH: cmplwi cr1,r17,(4*4) ; Do we have at least 4 left? subi r17,r17,4*4 ; Set count properly lwzx r2,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r15,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r14,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed lwzx r5,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed crset cr0_eq ; Set this to see if we failed - mtmsr r22 ; Flip DR, RI, and maybe PR on + mtmsr r22 ; Flip DR, RI isync stw r2,0(r16) ; Store word 0 @@ -730,9 +682,8 @@ aaStmwNxtH: cmplwi cr1,r17,(4*4) ; Do we have at least 4 left? addi r16,r16,4*4 ; Point up to next input aread aaStmwB2: mr r4,r0 ; Remember DAR, jus in case we failed the access - mr r3,r30 ; Set the normal MSR - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Normal MSR + isync bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... @@ -744,18 +695,18 @@ aaStmwL4: or. r5,r17,r28 ; Do we have anything left to do? beq- cr2,aaStmwBy1 ; No full words, check out bytes lwzx r2,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed blt cr1,aaStmwBy1 ; We only had one, go save it... lwzx r15,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed beq cr1,aaStmwBy1 ; We had two, go save it... lwzx r14,r19,r18 ; Store register - addi r18,r18,4 ; Next register - rlwinm r18,r18,0,25,29 ; Wrap back to 0 if needed + addi r18,r18,8 ; Next register + rlwinm r18,r18,0,24,28 ; Wrap back to 0 if needed aaStmwBy1: mr. r28,r28 ; Do we have any trailing bytes? beq+ aaStmwSt ; Nope... @@ -763,22 +714,23 @@ aaStmwBy1: mr. r28,r28 ; Do we have any trailing bytes? lwzx r5,r19,r18 ; Yes, pick up one extra register aaStmwSt: crset cr0_eq ; Set this to see if we failed - mtmsr r22 ; Flip DR, RI, and maybe PR on + mtmsr r22 ; Flip DR, RI isync beq- cr2,aaStmwBy2 ; No words, check trailing bytes... stw r2,0(r16) ; Save first word - bf- cr0_eq,aaStmwDn ; Read failed, escape... + bf- cr0_eq,aaStmwDn ; Store failed, escape... addi r16,r16,4 ; Bump sink blt cr1,aaStmwBy2 ; We only had one, we are done... stw r15,0(r16) ; Save second word - bf- cr0_eq,aaStmwDn ; Read failed, escape... + bf- cr0_eq,aaStmwDn ; Store failed, escape... addi r16,r16,4 ; Bump sink beq cr1,aaStmwBy2 ; We had two, we are done... stw r14,0(r16) ; Save third word + bf- cr0_eq,aaStmwDn ; Store failed, escape... addi r16,r16,4 ; Bump sink aaStmwBy2: rlwinm r2,r5,8,24,31 ; Get byte 0 @@ -801,9 +753,8 @@ aaStmwBy2: rlwinm r2,r5,8,24,31 ; Get byte 0 stb r14,2(r16) ; Save third byte aaStmwDn: mr r4,r0 ; Remember DAR, jus in case we failed the access - mr r3,r30 ; Set the normal MSR - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Normal MSR + isync bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... @@ -816,7 +767,7 @@ aaStmwDn: mr r4,r0 ; Remember DAR, jus in case we failed the access .align 5 -aaLswx: lwz r17,savexer(r13) ; Pick up the XER +aaLswx: lwz r17,savexer+4(r13) ; Pick up the XER crclr iUpdate ; Make sure we think this the load form rlwinm. r25,r17,0,25,31 ; Get the number of bytes to load rlwinm r28,r17,0,30,31 ; Get the number of bytes past an even word @@ -850,7 +801,7 @@ aaLswi: mr r9,r23 ; Save the DAR .align 5 -aaStswx: lwz r17,savexer(r13) ; Pick up the XER +aaStswx: lwz r17,savexer+4(r13) ; Pick up the XER crclr iUpdate ; Make sure this is clear in case we have 0 length rlwinm. r25,r17,0,25,31 ; Get the number of bytes to load rlwinm r28,r17,0,30,31 ; Get the number of bytes past an even word @@ -888,28 +839,24 @@ aaStswi: mr r9,r23 ; Save the DAR aaLwbrx: add r18,r18,r13 ; Index to source register - li r25,4 ; Set the length - - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to crset cr0_eq ; Set this to see if we failed - mr r3,r30 ; Set the normal MSR mtmsr r22 ; Flip DR, RI, and maybe PR on isync lwz r11,0(r23) ; Load the word mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state - + mtmsr r30 ; Restore normal MSR + isync + bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... rlwinm r10,r11,8,0,31 ; Get byte 0 to 3 and byte 2 to 1 rlwimi r10,r11,24,16,23 ; Move byte 1 to byte 2 rlwimi r10,r11,24,0,7 ; Move byte 3 to byte 0 - stw r10,saver0(r18) ; Set the register + stw r10,saver0+4(r18) ; Set the register b aaComExit ; All done, go exit... @@ -923,26 +870,22 @@ aaLwbrx: aaStwbrx: add r18,r18,r13 ; Index to source register - li r25,4 ; Set the length - lwz r11,saver0(r18) ; Get the register to store + lwz r11,saver0+4(r18) ; Get the register to store rlwinm r10,r11,8,0,31 ; Get byte 0 to 3 and byte 2 to 1 rlwimi r10,r11,24,16,23 ; Move byte 1 to byte 2 rlwimi r10,r11,24,0,7 ; Move byte 3 to byte 0 - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to - crset cr0_eq ; Set this to see if we failed - mr r3,r30 ; Set the normal MSR mtmsr r22 ; Flip DR, RI, and maybe PR on isync stw r10,0(r23) ; Store the reversed halfword mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state - + mtmsr r30 ; Restore normal MSR + isync + bt+ cr0_eq,aaComExit ; All done, go exit... b aaRedriveAsDSI ; We failed, go redrive this as a DSI... @@ -956,27 +899,23 @@ aaStwbrx: aaLhbrx: add r18,r18,r13 ; Index to source register - li r25,2 ; Set the length - - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to crset cr0_eq ; Set this to see if we failed - mr r3,r30 ; Set the normal MSR mtmsr r22 ; Flip DR, RI, and maybe PR on isync lhz r11,0(r23) ; Load the halfword mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Restore normal MSR + isync bf- cr0_eq,aaRedriveAsDSI ; We failed, go redrive this as a DSI... rlwinm r10,r11,8,16,23 ; Rotate bottom byte up one and clear everything else rlwimi r10,r11,24,24,31 ; Put old second from bottom into bottom - stw r10,saver0(r18) ; Set the register + stw r10,saver0+4(r18) ; Set the register b aaComExit ; All done, go exit... @@ -989,23 +928,19 @@ aaLhbrx: aaSthbrx: add r18,r18,r13 ; Index to source register - li r25,2 ; Set the length - lwz r10,saver0(r18) ; Get the register to store + lwz r10,saver0+4(r18) ; Get the register to store rlwinm r10,r10,8,0,31 ; Rotate bottom byte up one rlwimi r10,r10,16,24,31 ; Put old second from bottom into bottom - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to - crset cr0_eq ; Set this to see if we failed - mr r3,r30 ; Set the normal MSR mtmsr r22 ; Flip DR, RI, and maybe PR on isync sth r10,0(r23) ; Store the reversed halfword mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Restore normal MSR + isync bt+ cr0_eq,aaComExit ; All done, go exit... b aaRedriveAsDSI ; We failed, go redrive this as a DSI... @@ -1016,14 +951,18 @@ aaSthbrx: .align 5 -aaDcbz: - li r25,32 ; Set the length - rlwinm r23,r23,0,0,26 ; Round back to a 32-byte boundary - - bfl+ kernAccess,aaSetSegs ; Go set SRs if we are in user and need to - +aaDcbz: + lwz r0,savesrr0+4(r13) ; get instruction address + li r4,_COMM_PAGE_BASE_ADDRESS + rlwinm r23,r23,0,0,26 ; Round EA back to a 32-byte boundary + sub r4,r0,r4 ; compute instruction offset from base of commpage + cmplwi r4,_COMM_PAGE_AREA_USED ; did fault occur in commpage? + bge+ aaDcbz1 ; skip if not in commpage + lwz r4,savecr(r13) ; if we take a dcbz in the commpage... + rlwinm r4,r4,0,0,27 ; ...clear users cr7 as a flag for commpage code + stw r4,savecr(r13) +aaDcbz1: crset cr0_eq ; Set this to see if we failed - mr r3,r30 ; Set the normal MSR li r0,0 ; Clear this out mtmsr r22 ; Flip DR, RI, and maybe PR on isync @@ -1045,8 +984,8 @@ aaDcbz: stw r0,28(r23) ; Clear word aaDcbzXit: mr r4,r0 ; Save the DAR if we failed the access - li r0,loadMSR ; Set the magic "get back to supervisor" SC - sc ; Get back to supervisor state + mtmsr r30 ; Restore normal MSR + isync crclr iUpdate ; Make sure we do not think this is an update form @@ -1058,11 +997,9 @@ aaDcbzXit: mr r4,r0 ; Save the DAR if we failed the access ; Unhandled alignment exception, pass it along ; -aaPassAlongUnMap: - bfl+ kernAccess,aaUnSetSegs ; Go set SRs if we are in user and need to - - aaPassAlong: + li r0,1 ; Indicate that we failed to emulate + stw r0,savemisc3(r13) ; Assume that we emulate ok b EXT(EmulExit) @@ -1075,7 +1012,6 @@ aaPassAlong: .align 5 aaComExitrd: - bfl+ kernAccess,aaUnSetSegs ; Go set SRs back if we need to because we are not going back to user yet oris r9,r9,hi16(SAVredrive) ; Set the redrive bit li r11,T_TRACE ; Set trace interrupt rlwinm r12,r12,0,16,31 ; Clear top half of SRR1 @@ -1091,92 +1027,15 @@ aaComExitrd: aaRedriveAsDSI: mr r20,r1 ; Save the DSISR mr r21,r4 - bfl+ kernAccess,aaUnSetSegs ; Go set SRs back if we need to because we are not going back to user yet lwz r4,SAVflags(r13) ; Pick up the flags li r11,T_DATA_ACCESS ; Set failing data access code oris r4,r4,hi16(SAVredrive) ; Set the redrive bit stw r20,savedsisr(r13) ; Set the DSISR of failed access - stw r21,savedar(r13) ; Set the address of the failed access + stw r21,savedar+4(r13) ; Set the address of the failed access stw r11,saveexception(r13) ; Set the replacement code stw r4,SAVflags(r13) ; Set redrive request b EXT(EmulExit) ; Bail out to handle ISI... -; -; Set segment registers for user access. Do not call this if we are trying to get -; supervisor state memory. We do not need this. -; -; Performance-wise, we will usually be setting one SR here. Most memory will be -; allocated before the 1GB mark. Since the kernel maps the first GB, the exception -; handler always sets the SRs before we get here. Therefore, we will usually -; have to remap it. -; -; Also, we need to un-do these mapping ONLY if we take a non-standard -; exit, e.g., emulate DSI, emulate trace exception, etc. This is because -; translation will never be turned on until we return and at that point, -; normal exception exit code will restore the first 4 SRs if needed. -; - - .align 5 - - .globl EXT(aaSetSegsX) - -LEXT(aaSetSegsX) - -aaSetSegs: addi r3,r25,-1 ; Point at last accessed offset in range - lwz r7,PP_USERPMAP(r31) ; Get the current user pmap - lis r0,0x4000 ; This is the address of the first segment outside of the kernel - rlwinm r5,r23,6,26,29 ; Get index into pmap table - add r4,r23,r3 ; Point to the last byte accessed - addi r7,r7,PMAP_SEGS ; Point to the segment slot - cmplw r23,r0 ; See if first segment register needs to be reloaded - cmplw cr2,r4,r0 ; Do we need to set the second (if any) SR? - xor r0,r4,r23 ; See if we are in the same segment as first - bge aaSetS1ok ; Nope, we are in a pure user range - - lwzx r6,r5,r7 ; Get the user address space SR value - mtsrin r6,r23 ; Load the corresponding SR register - -aaSetS1ok: rlwinm. r0,r0,0,0,3 ; Any change in segment? - bgelr- cr2 ; We are in user only space, we do not need to mess with SR - rlwinm r5,r4,6,26,29 ; Get index into pmap table - beqlr+ ; No change in segment, we are done... - - lwzx r6,r5,r7 ; Get the user address space SR value - mtsrin r6,r4 ; Load the corresponding SR register - blr ; Leave... - -; -; Unset segment registers for user access. Do not call unless we had a user access. -; - - .align 5 - - .globl EXT(aaUnSetSegsX) - -LEXT(aaUnSetSegsX) - -aaUnSetSegs: - addi r3,r25,-1 ; Point at last accessed offset in range - lis r0,0x4000 ; This is the address of the first segment outside of the kernel - lis r5,hi16(KERNEL_SEG_REG0_VALUE) ; Get the high half of the kernel SR0 value - add r4,r23,r3 ; Point to the last byte accessed - cmplw r23,r0 ; See if first segment register needs to be reloaded - rlwimi r5,r23,24,8,11 ; Make the correct kernel segment - cmplw cr2,r4,r0 ; Do we need to set the second (if any) SR? - xor r0,r4,r23 ; See if we are in the same segment as first - bge aaUnSetS1ok ; Nope, we are in a pure user range - - mtsrin r5,r23 ; Load the corresponding SR register - -aaUnSetS1ok: - rlwinm. r0,r0,0,0,3 ; Any change in segment? - bgelr cr2 ; We are in user only space, we do not need to mess with SR - rlwimi r5,r4,24,8,11 ; Make the correct kernel segment - beqlr+ ; No change in segment, we are done... - - mtsrin r5,r4 ; Load the corresponding SR register - blr ; Leave... - ; @@ -1189,8 +1048,9 @@ aaUnSetS1ok: ; .align 10 ; Make sure we are on a 1k boundary + .globl EXT(aaFPopTable) -aaFPopTable: +LEXT(aaFPopTable) lfs f0,emfp0(r31) ; Load single variant blr