[apple/xnu.git] / osfmk / ppc / Emulate64.s

/*
 * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */ 																							

/* Emulate64.s
 *
 * Software emulation of instructions not handled in hw, on 64-bit machines.
 */
 
#include <sys/appleapiopts.h>
#include <cpus.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <ppc/exception.h>
#include <mach/machine/vm_param.h>
#include <ppc/cpu_capabilities.h>
#include <assym.s>

// CR bit set if the instruction is an "update" form (LFDU, STWU, etc):
#define	kUpdate	25

// CR bit set if interrupt occured in trace mode (ie, MSR_SE_BIT):
#define kTrace	8

// CR bit set if notification on alignment interrupts is requested (notifyUnalignbit in spcFlags):
#define	kNotify	9

// CR bit distinguishes between alignment and program exceptions:
#define	kAlignment	10


// *************************************
// * P R O G R A M   I N T E R R U P T *
// *************************************
//
// These are floating pt exceptions, illegal instructions, privileged mode violations,
// and traps.  All we're interested in at this low level is illegal instructions.
// The ones we "emulate" are:
//		DCBA,  which is not implemented in the IBM 970.  The emulation is to ignore it,
//			   as it is just a hint.
//		MCRXR, which is not implemented on the IBM 970, but is in the PPC ISA.
//
// Additionally, to facilitate debugging the alignment handler, we recognize a special
// diagnostic mode that is used to simulate alignment exceptions.  When in this mode,
// if the instruction has opcode==0 and the extended opcode is one of the X-form
// instructions that can take an alignment interrupt, then we change the opcode to
// 31 and pretend it got an alignment interrupt.  This exercises paths that
// are hard to drive or perhaps never driven on this particular CPU.

        .text
        .globl	EXT(Emulate64)
        .align	5
LEXT(Emulate64)
        crclr	kAlignment						// not an alignment exception
        b		a64AlignAssistJoin				// join alignment handler
        
        
// Return from alignment handler with all the regs loaded for opcode emulation.
        
a64HandleProgramInt:
        rlwinm.	r0,r29,0,SRR1_PRG_ILL_INS_BIT,SRR1_PRG_ILL_INS_BIT	// illegal opcode?
        beq		a64PassAlong					// No, must have been trap or priv violation etc
        rlwinm	r3,r20,6,26,31					// right justify opcode field (bits 0-5)
        rlwinm	r4,r20,31,22,31					// right justify extended opcode field (bits 21-30)
        cmpwi	cr0,r3,31						// X-form?
        cmpwi	cr1,r4,758						// DCBA?
        cmpwi	cr4,r4,512						// MCRXR?
        crand	cr1_eq,cr0_eq,cr1_eq			// merge the two tests for DCBA
        crand	cr4_eq,cr0_eq,cr4_eq			// and for MCRXR
        beq++	cr1_eq,a64ExitEm				// was DCBA, so ignore
        bne--	cr4_eq,a64NotEmulated			// skip if not MCRXR
        
// Was MCRXR, so emulate.

        ld		r3,savexer(r13)					// get the XER
        lwz		r4,savecr(r13)					// and the CR
        rlwinm	r5,r20,11,27,29					// get (CR# * 4) from instruction
        rlwinm	r6,r3,0,4,31					// zero XER[32-35] (also XER[0-31])
        sld		r4,r4,r5						// move target CR field to bits 32-35
        rlwimi	r4,r3,0,0,3						// move XER[32-35] into CR field
        stw		r6,savexer+4(r13)				// update XER
        srd		r4,r4,r5						// re-position CR
        stw		r4,savecr(r13)					// update CR
        b		a64ExitEm						// done

// Not an opcode we normally emulate.  If in special diagnostic mode and opcode=0,
// emulate as an alignment exception.  This special case is for test software.

a64NotEmulated:
        lwz		r30,dgFlags(0)					// Get the flags
        rlwinm.	r0,r30,0,enaDiagEMb,enaDiagEMb	// Do we want to try to emulate something?
        beq++	a64PassAlong					// No emulation allowed
        cmpwi	r3,0							// opcode==0 ?
        bne		a64PassAlong					// not the special case
        oris	r20,r20,0x7C00					// change opcode to 31
        crset	kAlignment						// say we took alignment exception
        rlwinm	r5,r4,0,26+1,26-1				// mask Update bit (32) out of extended opcode
        rlwinm	r5,r5,0,0,31					// Clean out leftover junk from rlwinm

        cmpwi	r4,1014							// dcbz/dcbz128 ?
        crmove	cr1_eq,cr0_eq
        cmpwi	r5,21							// ldx/ldux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,599							// lfdx/lfdux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,535							// lfsx/lfsux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,343							// lhax/lhaux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,790							// lhbrx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,279							// lhzx/lhzux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,597							// lswi ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,533							// lswx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,341							// lwax/lwaux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,534							// lwbrx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,23							// lwz/lwzx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,149							// stdx/stdux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,727							// stfdx/stfdux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,983							// stfiwx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,663							// stfsx/stfsux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,918							// sthbrx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,407							// sthx/sthux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,725							// stswi ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,661							// stswx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r4,662							// stwbrx ?
        cror	cr1_eq,cr0_eq,cr1_eq
        cmpwi	r5,151							// stwx/stwux ?
        cror	cr1_eq,cr0_eq,cr1_eq
        
        beq++	cr1,a64GotInstruction			// it was one of the X-forms we handle
        crclr	kAlignment						// revert to program interrupt
        b		a64PassAlong					// not recognized extended opcode
        

// *****************************************
// * A L I G N M E N T   I N T E R R U P T *
// *****************************************
//
// We get here in exception context, ie with interrupts disabled, translation off, and
// in 64-bit mode, with:
//		r13 = save-area pointer, with general context already saved in it
//		cr6 = feature flags
// We preserve r13 and cr6.  Other GPRs and CRs, the LR and CTR are used.
//
// Current 64-bit processors (GPUL) handle almost all misaligned operations in hardware,
// so this routine usually isn't called very often.  Only floating pt ops that cross a page
// boundary and are not word aligned, and LMW/STMW can take exceptions to cacheable memory.
// However, in contrast to G3 and G4, any misaligned load/store will get an alignment
// interrupt on uncached memory.
//
// We always emulate scalar ops with a series of byte load/stores.  Doing so is no slower
// than LWZ/STW in cases where a scalar op gets an alignment exception.
//
// This routine supports all legal permutations of alignment interrupts occuring in user or
// supervisor mode, 32 or 64-bit addressing, and translation on or off.  We do not emulate
// instructions that go past the end of an address space, such as "LHZ -1(0)"; we just pass
// along the alignment exception rather than wrap around to byte 0.  (Treatment of address
// space wrap is a moot point in Mac OS X, since we do not map either the last page or
// page 0.)
//
// First, check for a few special cases such as virtual machines, etc.

        .globl	EXT(AlignAssist64)
        .align	5
LEXT(AlignAssist64)
        crset	kAlignment								// mark as alignment interrupt

a64AlignAssistJoin:										// join here from program interrupt handler
        mfsprg	r31,0									// get the per_proc data ptr
        mcrf	cr3,cr6									// save feature flags here...
        lwz		r21,spcFlags(r31)						// grab the special flags
        ld		r29,savesrr1(r13)						// get the MSR etc at the fault
        ld		r28,savesrr0(r13)						// get the EA of faulting instruction
        mfmsr	r26										// save MSR at entry
        rlwinm.	r0,r21,0,runningVMbit,runningVMbit		// Are we running a VM?
        lwz		r19,dgFlags(0)							// Get the diagnostics flags
        bne--	a64PassAlong							// yes, let the virtual machine monitor handle


// Set up the MSR shadow regs.  We turn on FP in this routine, and usually set DR and RI
// when accessing user space (the SLB is still set up with all the user space translations.)
// However, if the interrupt occured in the kernel with DR off, we keep it off while
// accessing the "target" address space.  If we set DR to access the target space, we also
// set RI.  The RI bit tells the exception handlers to clear cr0 beq and return if we get an
// exception accessing the user address space.  We are careful to test cr0 beq after every such
// access.  We keep the following "shadows" of the MSR in global regs across this code:
//		r25 = MSR at entry, plus FP and probably DR and RI (used to access target space)
//		r26 = MSR at entry
//		r27 = free
//		r29 = SRR1 (ie, MSR at interrupt)
// Note that EE and IR are always off, and SF is always on in this code.

		rlwinm	r3,r29,0,MSR_DR_BIT,MSR_DR_BIT			// was translation on at fault?
        rlwimi	r3,r3,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT	// if DR was set, set RI too
        or		r25,r26,r3								// assemble MSR to use accessing target space
        

// Because the DSISR and DAR are either not set or are not to be trusted on some 64-bit
// processors on an alignment interrupt, we must fetch the faulting instruction ourselves,
// then decode/hash the opcode and reconstruct the EA manually.

        mtmsr	r25					// turn on FP and (if it was on at fault) DR and RI
        isync						// wait for it to happen
		cmpw	r0,r0				// turn on beq so we can check for DSIs
        lwz		r20,0(r28)			// fetch faulting instruction, probably with DR on
        bne--	a64RedriveAsISI		// got a DSI trying to fetch it, pretend it was an ISI
        mtmsr	r26					// turn DR back off
        isync						// wait for it to happen


// Set a few flags while we wait for the faulting instruction to arrive from cache.

        rlwinm.	r0,r29,0,MSR_SE_BIT,MSR_SE_BIT				// Were we single stepping?
		stw		r20,savemisc2(r13)	// Save the instruction image in case we notify
        crnot	kTrace,cr0_eq
        rlwinm.	r0,r19,0,enaNotifyEMb,enaNotifyEMb			// Should we notify?
        crnot	kNotify,cr0_eq        


// Hash the intruction into a 5-bit value "AAAAB" used to index the branch table, and a
// 1-bit kUpdate flag, as follows:
Commit	Line	Data
55e303ae A	1	/*
	2	* Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
e5568f75 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
55e303ae	11	*
e5568f75 A	12	* This Original Code and all software distributed under the License are
e5568f75 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
55e303ae A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
55e303ae A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
55e303ae A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	/* Emulate64.s
	24	*
	25	* Software emulation of instructions not handled in hw, on 64-bit machines.
	26	*/
	27
	28	#include <sys/appleapiopts.h>
	29	#include <cpus.h>
	30	#include <ppc/asm.h>
	31	#include <ppc/proc_reg.h>
	32	#include <ppc/exception.h>
	33	#include <mach/machine/vm_param.h>
	34	#include <ppc/cpu_capabilities.h>
	35	#include <assym.s>
	36
	37	// CR bit set if the instruction is an "update" form (LFDU, STWU, etc):
	38	#define kUpdate 25
	39
	40	// CR bit set if interrupt occured in trace mode (ie, MSR_SE_BIT):
	41	#define kTrace 8
	42
	43	// CR bit set if notification on alignment interrupts is requested (notifyUnalignbit in spcFlags):
	44	#define kNotify 9
	45
	46	// CR bit distinguishes between alignment and program exceptions:
	47	#define kAlignment 10
	48
	49
	50
	51	// *************************************
	52	// * P R O G R A M I N T E R R U P T *
	53	// *************************************
	54	//
	55	// These are floating pt exceptions, illegal instructions, privileged mode violations,
	56	// and traps. All we're interested in at this low level is illegal instructions.
	57	// The ones we "emulate" are:
	58	// DCBA, which is not implemented in the IBM 970. The emulation is to ignore it,
	59	// as it is just a hint.
	60	// MCRXR, which is not implemented on the IBM 970, but is in the PPC ISA.
	61	//
	62	// Additionally, to facilitate debugging the alignment handler, we recognize a special
	63	// diagnostic mode that is used to simulate alignment exceptions. When in this mode,
	64	// if the instruction has opcode==0 and the extended opcode is one of the X-form
	65	// instructions that can take an alignment interrupt, then we change the opcode to
	66	// 31 and pretend it got an alignment interrupt. This exercises paths that
	67	// are hard to drive or perhaps never driven on this particular CPU.
	68
	69	.text
	70	.globl EXT(Emulate64)
	71	.align 5
	72	LEXT(Emulate64)
	73	crclr kAlignment // not an alignment exception
	74	b a64AlignAssistJoin // join alignment handler
	75
	76
	77	// Return from alignment handler with all the regs loaded for opcode emulation.
	78
	79	a64HandleProgramInt:
	80	rlwinm. r0,r29,0,SRR1_PRG_ILL_INS_BIT,SRR1_PRG_ILL_INS_BIT // illegal opcode?
	81	beq a64PassAlong // No, must have been trap or priv violation etc
	82	rlwinm r3,r20,6,26,31 // right justify opcode field (bits 0-5)
83	rlwinm r4,r20,31,22,31 // right justify extended opcode field (bits 21-30)
84	cmpwi cr0,r3,31 // X-form?
85	cmpwi cr1,r4,758 // DCBA?
86	cmpwi cr4,r4,512 // MCRXR?
87	crand cr1_eq,cr0_eq,cr1_eq // merge the two tests for DCBA
88	crand cr4_eq,cr0_eq,cr4_eq // and for MCRXR
89	beq++ cr1_eq,a64ExitEm // was DCBA, so ignore
90	bne-- cr4_eq,a64NotEmulated // skip if not MCRXR
91
92	// Was MCRXR, so emulate.
93
94	ld r3,savexer(r13) // get the XER
95	lwz r4,savecr(r13) // and the CR
96	rlwinm r5,r20,11,27,29 // get (CR# * 4) from instruction
97	rlwinm r6,r3,0,4,31 // zero XER[32-35] (also XER[0-31])
98	sld r4,r4,r5 // move target CR field to bits 32-35
99	rlwimi r4,r3,0,0,3 // move XER[32-35] into CR field
100	stw r6,savexer+4(r13) // update XER
101	srd r4,r4,r5 // re-position CR
102	stw r4,savecr(r13) // update CR
103	b a64ExitEm // done
104
105	// Not an opcode we normally emulate. If in special diagnostic mode and opcode=0,
106	// emulate as an alignment exception. This special case is for test software.
107
108	a64NotEmulated:
109	lwz r30,dgFlags(0) // Get the flags
110	rlwinm. r0,r30,0,enaDiagEMb,enaDiagEMb // Do we want to try to emulate something?
111	beq++ a64PassAlong // No emulation allowed
112	cmpwi r3,0 // opcode==0 ?
113	bne a64PassAlong // not the special case
114	oris r20,r20,0x7C00 // change opcode to 31
115	crset kAlignment // say we took alignment exception
116	rlwinm r5,r4,0,26+1,26-1 // mask Update bit (32) out of extended opcode
117	rlwinm r5,r5,0,0,31 // Clean out leftover junk from rlwinm
118
119	cmpwi r4,1014 // dcbz/dcbz128 ?
120	crmove cr1_eq,cr0_eq
121	cmpwi r5,21 // ldx/ldux ?
122	cror cr1_eq,cr0_eq,cr1_eq
123	cmpwi r5,599 // lfdx/lfdux ?
124	cror cr1_eq,cr0_eq,cr1_eq
125	cmpwi r5,535 // lfsx/lfsux ?
126	cror cr1_eq,cr0_eq,cr1_eq
127	cmpwi r5,343 // lhax/lhaux ?
128	cror cr1_eq,cr0_eq,cr1_eq
129	cmpwi r4,790 // lhbrx ?
130	cror cr1_eq,cr0_eq,cr1_eq
131	cmpwi r5,279 // lhzx/lhzux ?
132	cror cr1_eq,cr0_eq,cr1_eq
133	cmpwi r4,597 // lswi ?
134	cror cr1_eq,cr0_eq,cr1_eq
135	cmpwi r4,533 // lswx ?
136	cror cr1_eq,cr0_eq,cr1_eq
137	cmpwi r5,341 // lwax/lwaux ?
138	cror cr1_eq,cr0_eq,cr1_eq
139	cmpwi r4,534 // lwbrx ?
140	cror cr1_eq,cr0_eq,cr1_eq
141	cmpwi r5,23 // lwz/lwzx ?
142	cror cr1_eq,cr0_eq,cr1_eq
143	cmpwi r5,149 // stdx/stdux ?
144	cror cr1_eq,cr0_eq,cr1_eq
145	cmpwi r5,727 // stfdx/stfdux ?
146	cror cr1_eq,cr0_eq,cr1_eq
147	cmpwi r4,983 // stfiwx ?
148	cror cr1_eq,cr0_eq,cr1_eq
149	cmpwi r5,663 // stfsx/stfsux ?
150	cror cr1_eq,cr0_eq,cr1_eq
151	cmpwi r4,918 // sthbrx ?
152	cror cr1_eq,cr0_eq,cr1_eq
153	cmpwi r5,407 // sthx/sthux ?
154	cror cr1_eq,cr0_eq,cr1_eq
155	cmpwi r4,725 // stswi ?
156	cror cr1_eq,cr0_eq,cr1_eq
157	cmpwi r4,661 // stswx ?
158	cror cr1_eq,cr0_eq,cr1_eq
159	cmpwi r4,662 // stwbrx ?
160	cror cr1_eq,cr0_eq,cr1_eq
161	cmpwi r5,151 // stwx/stwux ?
162	cror cr1_eq,cr0_eq,cr1_eq
163
164	beq++ cr1,a64GotInstruction // it was one of the X-forms we handle
165	crclr kAlignment // revert to program interrupt
166	b a64PassAlong // not recognized extended opcode
167
168
169	// *****************************************
170	// * A L I G N M E N T I N T E R R U P T *
171	// *****************************************
172	//
173	// We get here in exception context, ie with interrupts disabled, translation off, and
174	// in 64-bit mode, with:
175	// r13 = save-area pointer, with general context already saved in it
176	// cr6 = feature flags
177	// We preserve r13 and cr6. Other GPRs and CRs, the LR and CTR are used.
178	//
179	// Current 64-bit processors (GPUL) handle almost all misaligned operations in hardware,
180	// so this routine usually isn't called very often. Only floating pt ops that cross a page
181	// boundary and are not word aligned, and LMW/STMW can take exceptions to cacheable memory.
182	// However, in contrast to G3 and G4, any misaligned load/store will get an alignment
183	// interrupt on uncached memory.
184	//
185	// We always emulate scalar ops with a series of byte load/stores. Doing so is no slower
186	// than LWZ/STW in cases where a scalar op gets an alignment exception.
187	//
188	// This routine supports all legal permutations of alignment interrupts occuring in user or
189	// supervisor mode, 32 or 64-bit addressing, and translation on or off. We do not emulate
190	// instructions that go past the end of an address space, such as "LHZ -1(0)"; we just pass
191	// along the alignment exception rather than wrap around to byte 0. (Treatment of address
192	// space wrap is a moot point in Mac OS X, since we do not map either the last page or
193	// page 0.)
194	//
195	// First, check for a few special cases such as virtual machines, etc.
196
197	.globl EXT(AlignAssist64)
198	.align 5
199	LEXT(AlignAssist64)
200	crset kAlignment // mark as alignment interrupt
201
202	a64AlignAssistJoin: // join here from program interrupt handler
203	mfsprg r31,0 // get the per_proc data ptr
204	mcrf cr3,cr6 // save feature flags here...
205	lwz r21,spcFlags(r31) // grab the special flags
206	ld r29,savesrr1(r13) // get the MSR etc at the fault
207	ld r28,savesrr0(r13) // get the EA of faulting instruction
208	mfmsr r26 // save MSR at entry
209	rlwinm. r0,r21,0,runningVMbit,runningVMbit // Are we running a VM?
210	lwz r19,dgFlags(0) // Get the diagnostics flags
211	bne-- a64PassAlong // yes, let the virtual machine monitor handle
212
213
214	// Set up the MSR shadow regs. We turn on FP in this routine, and usually set DR and RI
215	// when accessing user space (the SLB is still set up with all the user space translations.)
216	// However, if the interrupt occured in the kernel with DR off, we keep it off while
217	// accessing the "target" address space. If we set DR to access the target space, we also
218	// set RI. The RI bit tells the exception handlers to clear cr0 beq and return if we get an
219	// exception accessing the user address space. We are careful to test cr0 beq after every such
220	// access. We keep the following "shadows" of the MSR in global regs across this code:
221	// r25 = MSR at entry, plus FP and probably DR and RI (used to access target space)
222	// r26 = MSR at entry
223	// r27 = free
224	// r29 = SRR1 (ie, MSR at interrupt)
225	// Note that EE and IR are always off, and SF is always on in this code.
226
227	rlwinm r3,r29,0,MSR_DR_BIT,MSR_DR_BIT // was translation on at fault?
228	rlwimi r3,r3,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT // if DR was set, set RI too
229	or r25,r26,r3 // assemble MSR to use accessing target space
230
231
232	// Because the DSISR and DAR are either not set or are not to be trusted on some 64-bit
233	// processors on an alignment interrupt, we must fetch the faulting instruction ourselves,
234	// then decode/hash the opcode and reconstruct the EA manually.
235
236	mtmsr r25 // turn on FP and (if it was on at fault) DR and RI
237	isync // wait for it to happen
238	cmpw r0,r0 // turn on beq so we can check for DSIs
239	lwz r20,0(r28) // fetch faulting instruction, probably with DR on
240	bne-- a64RedriveAsISI // got a DSI trying to fetch it, pretend it was an ISI
241	mtmsr r26 // turn DR back off
242	isync // wait for it to happen
243
244
245	// Set a few flags while we wait for the faulting instruction to arrive from cache.
246
247	rlwinm. r0,r29,0,MSR_SE_BIT,MSR_SE_BIT // Were we single stepping?
248	stw r20,savemisc2(r13) // Save the instruction image in case we notify
249	crnot kTrace,cr0_eq
250	rlwinm. r0,r19,0,enaNotifyEMb,enaNotifyEMb // Should we notify?
251	crnot kNotify,cr0_eq
252
253
254	// Hash the intruction into a 5-bit value "AAAAB" used to index the branch table, and a
255	// 1-bit kUpdate flag, as follows:
256