[apple/xnu.git] / osfmk / ppc / movc.s

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
#include <debug.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <mach/ppc/vm_param.h>
#include <assym.s>
#include <sys/errno.h>

/*
 * void pmap_zero_page(vm_offset_t pa)
 *
 * zero a page of physical memory.
 */

#if DEBUG
	/* C debug stub in pmap.c calls this */
ENTRY(pmap_zero_page_assembler, TAG_NO_FRAME_USED)
#else
ENTRY(pmap_zero_page, TAG_NO_FRAME_USED)
#endif /* DEBUG */

		mfmsr	r6								/* Get the MSR */
		rlwinm	r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
		rlwinm	r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
		rlwinm	r7,	r6,	0,	MSR_DR_BIT+1,	MSR_DR_BIT-1	/* Turn off DR */
		rlwinm	r7,r7,0,MSR_EE_BIT+1,MSR_EE_BIT-1	; Disable interruptions
		li		r4,PPC_PGBYTES-CACHE_LINE_SIZE	/* Point to the end of the page */
		mtmsr	r7								/* Set MSR to DR off */
		isync									/* Ensure data translations are off */


.L_phys_zero_loop:	
		subic.	r5,r4,CACHE_LINE_SIZE			/* Point to the next one */
		dcbz	r4, r3							/* Clear the whole thing to 0s */
		subi	r4,r5,CACHE_LINE_SIZE			/* Point to the next one */
		dcbz	r5, r3							/* Clear the next to zeros */
		bgt+	.L_phys_zero_loop				/* Keep going until we do the page... */

		sync									/* Make sure they're all done */
		li		r4,PPC_PGBYTES-CACHE_LINE_SIZE	/* Point to the end of the page */

.L_inst_inval_loop:	
		subic.	r5,r4,CACHE_LINE_SIZE			/* Point to the next one */
		icbi	r4, r3							/* Clear the whole thing to 0s */
		subi	r4,r5,CACHE_LINE_SIZE			/* Point to the next one */
		icbi	r5, r3							/* Clear the next to zeros */
		bgt+	.L_inst_inval_loop				/* Keep going until we do the page... */

		sync									/* Make sure they're all done */

		mtmsr	r6		/* Restore original translations */
		isync			/* Ensure data translations are on */

		blr

/* void
 * phys_copy(src, dst, bytecount)
 *      vm_offset_t     src;
 *      vm_offset_t     dst;
 *      int             bytecount
 *
 * This routine will copy bytecount bytes from physical address src to physical
 * address dst. 
 */

ENTRY(phys_copy, TAG_NO_FRAME_USED)

	/* Switch off data translations */
	mfmsr	r6
	rlwinm	r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
	rlwinm	r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
	rlwinm	r7,	r6,	0,	MSR_DR_BIT+1,	MSR_DR_BIT-1
	rlwinm  r7,     r7,     0,      MSR_EE_BIT+1,   MSR_EE_BIT-1
	mtmsr	r7
	isync			/* Ensure data translations are off */

	subi	r3,	r3,	4
	subi	r4,	r4,	4

	cmpwi	r5,	3
	ble-	.L_phys_copy_bytes
.L_phys_copy_loop:
	lwz	r0,	4(r3)
	addi	r3,	r3,	4
	subi	r5,	r5,	4
	stw	r0,	4(r4)
	addi	r4,	r4,	4
	cmpwi	r5,	3
	bgt+	.L_phys_copy_loop

	/* If no leftover bytes, we're done now */
	cmpwi	r5,	0
	beq+	.L_phys_copy_done
	
.L_phys_copy_bytes:
	addi	r3,	r3,	3
	addi	r4,	r4,	3
.L_phys_copy_byte_loop:	
	lbz	r0,	1(r3)
	addi	r3,	r3,	1
	subi	r5,	r5,	1
	stb	r0,	1(r4)
	addi	r4,	r4,	1
	cmpwi	r5,	0
	bne+	.L_phys_copy_byte_loop

.L_phys_copy_done:
	mtmsr	r6		/* Restore original translations */
	isync			/* Ensure data translations are off */

	blr

/* void
 * pmap_copy_page(src, dst)
 *      vm_offset_t     src;
 *      vm_offset_t     dst;
 *
 * This routine will copy the physical page src to physical page dst
 * 
 * This routine assumes that the src and dst are page aligned and that the
 * destination is cached.
 *
 * We also must assume that noone will be executing within the destination
 * page.  We also assume that this will be used for paging
 *
 */

#if DEBUG
	/* if debug, we have a little piece of C around this
	 * in pmap.c that gives some trace ability
	 */
ENTRY(pmap_copy_page_assembler, TAG_NO_FRAME_USED)
#else
ENTRY(pmap_copy_page, TAG_NO_FRAME_USED)
#endif /* DEBUG */

#if 0
			mfpvr	r9							; Get the PVR
			rlwinm	r9,r9,16,16,31				; Isolate the PPC processor
			cmplwi	r9,PROCESSOR_VERSION_Max	; Do we have Altivec?
			beq+	wegotaltivec				; Yeah...
#endif
		
			mfmsr	r9							; Get the MSR
			rlwinm	r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
			rlwinm	r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
			stwu	r1,-(FM_SIZE+32)(r1)		; Make a frame for us
			rlwinm	r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1	; Disable interruptions
			ori		r7,r7,lo16(MASK(MSR_FP))	; Turn on the FPU
			mtmsr	r7							; Disable rupts and enable FPU
			isync
		
			stfd	f0,FM_SIZE+0(r1)			; Save an FP register
			rlwinm	r7,r7,0,MSR_DR_BIT+1,MSR_DR_BIT-1	; Clear the DDAT bit
			stfd	f1,FM_SIZE+8(r1)			; Save an FP register
			addi	r6,r3,PPC_PGBYTES			; Point to the start of the next page
			stfd	f2,FM_SIZE+16(r1)			; Save an FP register
			mr		r8,r4						; Save the destination
			stfd	f3,FM_SIZE+24(r1)			; Save an FP register
		
			mtmsr	r7							; Set the new MSR
			isync								; Ensure data translations are off

			dcbt	br0, r3						/* Start in first input line */
			li		r5,	CACHE_LINE_SIZE			/* Get the line size */

.L_pmap_copy_page_loop:
			dcbz	0, r4						/* Allocate a line for the output */
			lfd		f0, 0(r3)					/* Get first 8 */
			lfd		f1, 8(r3)					/* Get second 8 */
			lfd		f2, 16(r3)					/* Get third 8 */
			stfd	f0, 0(r4)					/* Put first 8 */
			dcbt	r5, r3						/* Start next line coming in */
			lfd		f3, 24(r3)					/* Get fourth 8 */
			stfd	f1,	8(r4)					/* Put second 8 */
			addi	r3,r3,CACHE_LINE_SIZE		/* Point to the next line in */
			stfd	f2,	16(r4)					/* Put third 8 */
			cmplw	cr0,r3,r6					/* See if we're finished yet */
			stfd	f3,	24(r4)					/* Put fourth 8 */
			dcbst	br0,r4						/* Force it out */
			addi	r4,r4,CACHE_LINE_SIZE		/* Point to the next line out */
			blt+	.L_pmap_copy_page_loop		/* Copy the whole page */
			
			sync								/* Make sure they're all done */
			li		r4,PPC_PGBYTES-CACHE_LINE_SIZE	/* Point to the end of the page */

invalinst:	
			subic.	r5,r4,CACHE_LINE_SIZE		/* Point to the next one */
			icbi	r4, r8						/* Trash the i-cache */
			subi	r4,r5,CACHE_LINE_SIZE		/* Point to the next one */
			icbi	r5, r8						/* Trash the i-cache */
			bgt+	invalinst					/* Keep going until we do the page... */
	
			rlwimi	r7,r9,0,MSR_DR_BIT,MSR_DR_BIT	; Set DDAT if on
			sync								; Make sure all invalidates done
			
			mtmsr	r7							; Set DDAT correctly
			isync		
			
			lfd		f0,FM_SIZE+0(r1)			; Restore an FP register
			lfd		f1,FM_SIZE+8(r1)			; Restore an FP register
			lfd		f2,FM_SIZE+16(r1)			; Restore an FP register
			lfd		f3,FM_SIZE+24(r1)			; Restore an FP register
			
			lwz		r1,0(r1)					; Pop up the stack
	
			mtmsr	r9							; Turn off FPU now and maybe rupts back on
			isync								
			blr
		
#if 0
;
;			This is not very optimal.  We just do it here for a test of 
;			Altivec in the kernel.
;
wegotaltivec:
			mfmsr	r9							; Get the MSR
			lis		r8,hi16(0xC0000000)			; Make sure we keep the first 2 vector registers
			rlwinm	r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1	; Disable interruptions
			lis		r6,lo16(2*256+128)			; Specify 128 blocks of 2 vectors each
			rlwinm	r7,r7,0,MSR_DR_BIT+1,MSR_DR_BIT-1	; Clear the DDAT bit
			ori		r6,r6,32					; Set a 32-byte stride
			mtsprg	256,r8						; Set VRSave
			mtmsr	r7							; Disable rupts and turn xlate off
			isync
	
			addi	r11,r3,4096					; Point to the next page
			li		r10,16						; Get vector size

avmovepg:	lvxl	v0,br0,r3					; Get first half of line
			dcba	br0,r4						; Allocate output
			lvxl	v1,r10,r3					; Get second half of line
			stvxl	v0,br0,r4					; Save first half of line
			addi	r3,r3,32					; Point to the next line
			icbi	br0,r4						; Make the icache go away also
			stvxl	v1,r10,r4					; Save second half of line
			cmplw	r3,r11						; Have we reached the next page?
			dcbst	br0,r4						; Make sure the line is on its way out
			addi	r4,r4,32					; Point to the next line
			blt+	avmovepg					; Move the next line...
			
			li		r8,0						; Clear this
			sync								; Make sure all the memory stuff is done
			mtsprg	256,r8						; Show we are not using VRs any more
			mtmsr	r9							; Translation and interruptions back on
			isync
			blr
#endif
		

/*
 * int
 * copyin(src, dst, count)
 *	vm_offset_t	src;
 *	vm_offset_t	dst;
 *	int		count;
 *
 */

ENTRY2(copyin, copyinmsg, TAG_NO_FRAME_USED)

/* Preamble allowing us to call a sub-function */
		mflr	r0
		stw		r0,FM_LR_SAVE(r1)
		stwu	r1,-(FM_SIZE+16)(r1)
		
		cmpli	cr0,r5,0
		ble-	cr0,.L_copyinout_trivial

/* we know we have a valid copyin to do now */
/* Set up thread_recover in case we hit an illegal address */
		
		mfsprg  r8,1							/* Get the current act */ 
		lwz		r10,ACT_THREAD(r8)
		lis		r11,hi16(.L_copyinout_error)
		lwz		r8,ACT_VMMAP(r8)
		ori		r11,r11,lo16(.L_copyinout_error)
		add		r9,r3,r5						/* Get the end of the source */
		lwz		r8,VMMAP_PMAP(r8)				; Get the pmap
		rlwinm	r12,r3,6,26,29					; Get index to the segment slot
		subi	r9,r9,1							/* Make sure we don't go too far */
		add		r8,r8,r12						; Start indexing to the segment value
		stw		r11,THREAD_RECOVER(r10)
		xor		r9,r9,r3						/* Smoosh 'em together */
		lwz		r8,PMAP_SEGS(r8)				; Get the source SR value
		rlwinm.	r9,r9,0,1,3						/* Top nybble equal? */
		mtsr	SR_COPYIN,r8					; Set the SR
		isync
#if 0
		lis		r0,HIGH_ADDR(EXT(dbgRegsCall))	/* (TEST/DEBUG) */	
		ori		r0,r0,LOW_ADDR(EXT(dbgRegsCall))	/* (TEST/DEBUG) */	
		sc										/* (TEST/DEBUG) */
#endif
	
/* For optimization, we check if the copyin lies on a segment
 * boundary. If it doesn't, we can use a simple copy. If it
 * does, we split it into two separate copies in some C code.
 */
	
		bne-	.L_call_copyin_multiple			/* Nope, we went past the segment boundary... */

		rlwinm	r3,r3,0,4,31
		oris	r3,r3,(SR_COPYIN_NUM << (28-16))	/* Set the copyin segment as the source */
	
		bl		EXT(bcopy)
		
/* Now that copyin is done, we don't need a recovery point */

		addi	r1,r1,FM_SIZE+16
		mfsprg  r6,1							/* Get the current act */ 
		lwz		r10,ACT_THREAD(r6)
		li		r3,0
		lwz		r0,FM_LR_SAVE(r1)
		stw		r3,THREAD_RECOVER(r10)			/* Clear recovery */
		mtlr	r0
		blr
	
/* we get here via the exception handler if an illegal
 * user memory reference was made.
 */
.L_copyinout_error:

/* Now that copyin is done, we don't need a recovery point */
	
		mfsprg  r6,1							/* Get the current act */ 
		addi	r1,r1,FM_SIZE+16
		lwz		r10,ACT_THREAD(r6)
		li		r4,0
		lwz		r0,FM_LR_SAVE(r1)
		stw		r4,THREAD_RECOVER(r10)			/* Clear recovery */
		mtlr	r0
		li		r3,EFAULT							; Indicate error (EFAULT) 
		blr

.L_copyinout_trivial:
	/* The copyin/out was for either 0 bytes or a negative
	 * number of bytes, return an appropriate value (0 == SUCCESS).
	 * cr0 still contains result of comparison of len with 0.
	 */
	li	r3,	0
	beq+	cr0,	.L_copyinout_negative
	li	r3,	1
.L_copyinout_negative:

	/* unwind the stack */
	addi	r1,	r1,	FM_SIZE+16
	lwz	r0,	FM_LR_SAVE(r1)
	mtlr	r0

	blr

.L_call_copyin_multiple:

	/* unwind the stack */
	addi	r1,	r1,	FM_SIZE+16
	lwz	r0,	FM_LR_SAVE(r1)
	mtlr	r0

	b	EXT(copyin_multiple)				/* not a call - a jump! */

/*
 * int
 * copyout(src, dst, count)
 *	vm_offset_t	src;
 *	vm_offset_t	dst;
 *	int		count;
 *
 */

ENTRY2(copyout, copyoutmsg, TAG_NO_FRAME_USED)

/* Preamble allowing us to call a sub-function */

		mflr	r0
		stw		r0,FM_LR_SAVE(r1)
		stwu	r1,-(FM_SIZE+16)(r1)
		
#if 0
		stw		r3,FM_SIZE+0(r1)				/* (TEST/DEBUG) */
		stw		r4,FM_SIZE+4(r1)				/* (TEST/DEBUG) */
		stw		r5,FM_SIZE+8(r1)				/* (TEST/DEBUG) */
		mr		r6,r0							/* (TEST/DEBUG) */
		
		bl		EXT(tracecopyout)				/* (TEST/DEBUG) */
		
		lwz		r3,FM_SIZE+0(r1)				/* (TEST/DEBUG) */
		lwz		r4,FM_SIZE+4(r1)				/* (TEST/DEBUG) */
		lwz		r5,FM_SIZE+8(r1)				/* (TEST/DEBUG) */
#endif
	
		cmpli	cr0,r5,0
		ble-	cr0,.L_copyinout_trivial
/* we know we have a valid copyout to do now */
/* Set up thread_recover in case we hit an illegal address */
		

		mfsprg  r8,1							/* Get the current act */
		lwz		r10,ACT_THREAD(r8)
		lis		r11,HIGH_ADDR(.L_copyinout_error)
		lwz		r8,ACT_VMMAP(r8)
		rlwinm	r12,r4,6,26,29					; Get index to the segment slot
		ori		r11,r11,LOW_ADDR(.L_copyinout_error)
		add		r9,r4,r5						/* Get the end of the destination */
		lwz		r8,VMMAP_PMAP(r8)
		subi	r9,r9,1							/* Make sure we don't go too far */
		add		r8,r8,r12						; Start indexing to the segment value
		stw		r11,THREAD_RECOVER(r10)
		xor		r9,r9,r4						/* Smoosh 'em together */
		lwz		r8,PMAP_SEGS(r8)				; Get the source SR value
		rlwinm.	r9,r9,0,1,3						/* Top nybble equal? */
		mtsr	SR_COPYIN,r8
		isync
	
	
/* For optimisation, we check if the copyout lies on a segment
 * boundary. If it doesn't, we can use a simple copy. If it
 * does, we split it into two separate copies in some C code.
 */
	
		bne-	.L_call_copyout_multiple		/* Nope, we went past the segment boundary... */

		rlwinm	r4,r4,0,4,31
		oris	r4,r4,(SR_COPYIN_NUM << (28-16))	/* Set the copyin segment as the source */
	
		bl	EXT(bcopy)
		
/* Now that copyout is done, we don't need a recovery point */
		mfsprg  r6,1							/* Get the current act */
		addi	r1,r1,FM_SIZE+16
		lwz		r10,ACT_THREAD(r6)
		li		r3,0
		lwz		r0,FM_LR_SAVE(r1)
		stw		r3,THREAD_RECOVER(r10)			/* Clear recovery */
		mtlr	r0
		blr

.L_call_copyout_multiple:
	/* unwind the stack */
	addi	r1,	r1,	FM_SIZE+16
	lwz	r0,	FM_LR_SAVE(r1)
	mtlr	r0

	b	EXT(copyout_multiple)					/* not a call - a jump! */

/*
 * boolean_t
 * copyinstr(src, dst, count, maxcount)
 *	vm_offset_t	src;
 *	vm_offset_t	dst;
 *	vm_size_t	maxcount; 
 *	vm_size_t*	count;
 *
 * Set *count to the number of bytes copied
 * 
 * If dst == NULL, don't copy, just count bytes.
 * Only currently called from klcopyinstr. 
 */

ENTRY(copyinstr, TAG_NO_FRAME_USED)

/* Preamble allowing us to call a sub-function */
		mflr	r0
		stw		r0,FM_LR_SAVE(r1)
		stwu	r1,-(FM_SIZE+16)(r1)

#if 0
		stw		r3,FM_SIZE+0(r1)				/* (TEST/DEBUG) */
		stw		r4,FM_SIZE+4(r1)				/* (TEST/DEBUG) */
		stw		r5,FM_SIZE+8(r1)				/* (TEST/DEBUG) */
		stw		r6,FM_SIZE+12(r1)				/* (TEST/DEBUG) */
		mr		r7,r0							/* (TEST/DEBUG) */
		
		bl		EXT(tracecopystr)				/* (TEST/DEBUG) */
		
		lwz		r3,FM_SIZE+0(r1)				/* (TEST/DEBUG) */
		lwz		r4,FM_SIZE+4(r1)				/* (TEST/DEBUG) */
		lwz		r5,FM_SIZE+8(r1)				/* (TEST/DEBUG) */
		stw		r6,FM_SIZE+12(r1)				/* (TEST/DEBUG) */
#endif
				
		cmpli	cr0,r5,0
		ble-	cr0,.L_copyinout_trivial

/* we know we have a valid copyin to do now */
/* Set up thread_recover in case we hit an illegal address */
		
		li		r0,0							
		mfsprg  r8,1							/* Get the current act */
		lwz		r10,ACT_THREAD(r8)
		stw		r0,0(r6)						/* Clear result length */
		lis		r11,HIGH_ADDR(.L_copyinout_error)
		lwz		r8,ACT_VMMAP(r8)				; Get the map for this activation
		rlwinm	r12,r3,6,26,29					; Get index to the segment slot
		lwz		r8,VMMAP_PMAP(r8)
		ori		r11,r11,LOW_ADDR(.L_copyinout_error)
		add		r8,r8,r12						; Start indexing to the segment value
		stw		r11,THREAD_RECOVER(r10)
		rlwinm	r3,r3,0,4,31
		lwz		r7,PMAP_SEGS(r8)				; Get the source SR value
		oris	r3,r3,(SR_COPYIN_NUM << (28-16))	/* Set the copyin segment as the source */

/* Copy byte by byte for now - TODO NMGS speed this up with
 * some clever (but fairly standard) logic for word copies.
 * We don't use a copyinstr_multiple since copyinstr is called
 * with INT_MAX in the linux server. Eugh.
 */

		li		r9,0							/* Clear byte counter */

/* If the destination is NULL, don't do writes,
 * just count bytes. We set CR7 outside the loop to save time
 */
		cmpwi	cr7,r4,0						/* Is the destination null? */
		
nxtseg:	mtsr	SR_COPYIN,r7					/* Set the source SR */
		isync

.L_copyinstr_loop:
		lbz		r0,0(r3)						/* Get the source */
		addic.	r5,r5,-1						/* Have we gone far enough? */
		addi	r3,r3,1							/* Bump source pointer */
		
		cmpwi	cr1,r0,0						/* Did we hit a null? */

		beq		cr7,.L_copyinstr_no_store		/* If we are just counting, skip the store... */
	
		stb		r0,0(r4)						/* Move to sink */
		addi	r4,r4,1							/* Advance sink pointer */

.L_copyinstr_no_store:

		addi	r9,r9,1							/* Count the character */
		beq-	cr1,.L_copyinstr_done			/* We're done if we did a null... */
		beq-	cr0,L_copyinstr_toobig			/* Also if we maxed the count... */
	
/* Check to see if the copyin pointer has moved out of the
 * copyin segment, if it has we must remap.
 */

		rlwinm.	r0,r3,0,4,31					/* Did we wrap around to 0? */
		bne+	cr0,.L_copyinstr_loop			/* Nope... */

		lwz		r7,PMAP_SEGS+4(r8)				; Get the next source SR value
		addi	r8,r8,4							; Point to the next segment
		oris	r3,r0,(SR_COPYIN_NUM << (28-16))	/* Reset the segment number */
		b		nxtseg							/* Keep going... */
	
L_copyinstr_toobig:
		li		r3,ENAMETOOLONG
		b		L_copyinstr_return
.L_copyinstr_done:
		li		r3,0							/* Normal return */
L_copyinstr_return:
		li		r4,0							/* to clear thread_recover */
		stw		r9,0(r6)						/* Set how many bytes we did */
		stw		r4,THREAD_RECOVER(r10)			/* Clear recovery exit */

		addi	r1,	r1,	FM_SIZE+16
		lwz		r0,	FM_LR_SAVE(r1)
		mtlr	r0
		blr
Commit	Line	Data
1c79356b	1	/*
de355530	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
1c79356b A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
de355530 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
1c79356b	11	*
de355530 A	12	* This Original Code and all software distributed under the License are
de355530 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
de355530 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
1c79356b A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22	/*
	23	* @OSF_COPYRIGHT@
	24	*/
	25	#include <debug.h>
	26	#include <ppc/asm.h>
	27	#include <ppc/proc_reg.h>
	28	#include <mach/ppc/vm_param.h>
	29	#include <assym.s>
	30	#include <sys/errno.h>
	31
	32	/*
	33	* void pmap_zero_page(vm_offset_t pa)
	34	*
de355530	35	* zero a page of physical memory.
1c79356b A	36	*/
1c79356b A	37
de355530 A	38	#if DEBUG
	39	/* C debug stub in pmap.c calls this */
	40	ENTRY(pmap_zero_page_assembler, TAG_NO_FRAME_USED)
	41	#else
	42	ENTRY(pmap_zero_page, TAG_NO_FRAME_USED)
	43	#endif /* DEBUG */
	44
	45	mfmsr r6 /* Get the MSR */
	46	rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	47	rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	48	rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1 /* Turn off DR */
	49	rlwinm r7,r7,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Disable interruptions
	50	li r4,PPC_PGBYTES-CACHE_LINE_SIZE /* Point to the end of the page */
	51	mtmsr r7 /* Set MSR to DR off */
	52	isync /* Ensure data translations are off */
	53
	54
	55	.L_phys_zero_loop:
	56	subic. r5,r4,CACHE_LINE_SIZE /* Point to the next one */
	57	dcbz r4, r3 /* Clear the whole thing to 0s */
	58	subi r4,r5,CACHE_LINE_SIZE /* Point to the next one */
	59	dcbz r5, r3 /* Clear the next to zeros */
	60	bgt+ .L_phys_zero_loop /* Keep going until we do the page... */
	61
	62	sync /* Make sure they're all done */
	63	li r4,PPC_PGBYTES-CACHE_LINE_SIZE /* Point to the end of the page */
	64
	65	.L_inst_inval_loop:
	66	subic. r5,r4,CACHE_LINE_SIZE /* Point to the next one */
	67	icbi r4, r3 /* Clear the whole thing to 0s */
	68	subi r4,r5,CACHE_LINE_SIZE /* Point to the next one */
	69	icbi r5, r3 /* Clear the next to zeros */
	70	bgt+ .L_inst_inval_loop /* Keep going until we do the page... */
	71
	72	sync /* Make sure they're all done */
	73
	74	mtmsr r6 /* Restore original translations */
	75	isync /* Ensure data translations are on */
1c79356b	76
de355530	77	blr
d7e50217	78
1c79356b A	79	/* void
1c79356b A	80	* phys_copy(src, dst, bytecount)
de355530 A	81	* vm_offset_t src;
de355530 A	82	* vm_offset_t dst;
1c79356b A	83	* int bytecount
	84	*
	85	* This routine will copy bytecount bytes from physical address src to physical
de355530	86	* address dst.
1c79356b A	87	*/
1c79356b A	88
de355530 A	89	ENTRY(phys_copy, TAG_NO_FRAME_USED)
	90
	91	/* Switch off data translations */
	92	mfmsr r6
	93	rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	94	rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	95	rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1
	96	rlwinm r7, r7, 0, MSR_EE_BIT+1, MSR_EE_BIT-1
	97	mtmsr r7
	98	isync /* Ensure data translations are off */
	99
	100	subi r3, r3, 4
	101	subi r4, r4, 4
	102
	103	cmpwi r5, 3
	104	ble- .L_phys_copy_bytes
	105	.L_phys_copy_loop:
	106	lwz r0, 4(r3)
	107	addi r3, r3, 4
	108	subi r5, r5, 4
	109	stw r0, 4(r4)
	110	addi r4, r4, 4
	111	cmpwi r5, 3
	112	bgt+ .L_phys_copy_loop
	113
	114	/* If no leftover bytes, we're done now */
	115	cmpwi r5, 0
	116	beq+ .L_phys_copy_done
	117
	118	.L_phys_copy_bytes:
	119	addi r3, r3, 3
	120	addi r4, r4, 3
	121	.L_phys_copy_byte_loop:
	122	lbz r0, 1(r3)
	123	addi r3, r3, 1
	124	subi r5, r5, 1
	125	stb r0, 1(r4)
	126	addi r4, r4, 1
	127	cmpwi r5, 0
	128	bne+ .L_phys_copy_byte_loop
	129
	130	.L_phys_copy_done:
	131	mtmsr r6 /* Restore original translations */
	132	isync /* Ensure data translations are off */
	133
	134	blr
1c79356b A	135
	136	/* void
	137	* pmap_copy_page(src, dst)
de355530 A	138	* vm_offset_t src;
de355530 A	139	* vm_offset_t dst;
1c79356b A	140	*
	141	* This routine will copy the physical page src to physical page dst
	142	*
de355530 A	143	* This routine assumes that the src and dst are page aligned and that the
	144	* destination is cached.
	145	*
	146	* We also must assume that noone will be executing within the destination
	147	* page. We also assume that this will be used for paging
1c79356b A	148	*
	149	*/
	150
de355530 A	151	#if DEBUG
	152	/* if debug, we have a little piece of C around this
	153	* in pmap.c that gives some trace ability
	154	*/
	155	ENTRY(pmap_copy_page_assembler, TAG_NO_FRAME_USED)
	156	#else
1c79356b	157	ENTRY(pmap_copy_page, TAG_NO_FRAME_USED)
de355530 A	158	#endif /* DEBUG */
	159
	160	#if 0
	161	mfpvr r9 ; Get the PVR
	162	rlwinm r9,r9,16,16,31 ; Isolate the PPC processor
	163	cmplwi r9,PROCESSOR_VERSION_Max ; Do we have Altivec?
	164	beq+ wegotaltivec ; Yeah...
	165	#endif
1c79356b	166
de355530 A	167	mfmsr r9 ; Get the MSR
	168	rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	169	rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	170	stwu r1,-(FM_SIZE+32)(r1) ; Make a frame for us
	171	rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Disable interruptions
	172	ori r7,r7,lo16(MASK(MSR_FP)) ; Turn on the FPU
	173	mtmsr r7 ; Disable rupts and enable FPU
	174	isync
	175
	176	stfd f0,FM_SIZE+0(r1) ; Save an FP register
	177	rlwinm r7,r7,0,MSR_DR_BIT+1,MSR_DR_BIT-1 ; Clear the DDAT bit
	178	stfd f1,FM_SIZE+8(r1) ; Save an FP register
	179	addi r6,r3,PPC_PGBYTES ; Point to the start of the next page
	180	stfd f2,FM_SIZE+16(r1) ; Save an FP register
	181	mr r8,r4 ; Save the destination
	182	stfd f3,FM_SIZE+24(r1) ; Save an FP register
	183
	184	mtmsr r7 ; Set the new MSR
	185	isync ; Ensure data translations are off
	186
	187	dcbt br0, r3 /* Start in first input line */
	188	li r5, CACHE_LINE_SIZE /* Get the line size */
	189
	190	.L_pmap_copy_page_loop:
	191	dcbz 0, r4 /* Allocate a line for the output */
	192	lfd f0, 0(r3) /* Get first 8 */
	193	lfd f1, 8(r3) /* Get second 8 */
	194	lfd f2, 16(r3) /* Get third 8 */
	195	stfd f0, 0(r4) /* Put first 8 */
	196	dcbt r5, r3 /* Start next line coming in */
	197	lfd f3, 24(r3) /* Get fourth 8 */
	198	stfd f1, 8(r4) /* Put second 8 */
	199	addi r3,r3,CACHE_LINE_SIZE /* Point to the next line in */
	200	stfd f2, 16(r4) /* Put third 8 */
	201	cmplw cr0,r3,r6 /* See if we're finished yet */
	202	stfd f3, 24(r4) /* Put fourth 8 */
	203	dcbst br0,r4 /* Force it out */
	204	addi r4,r4,CACHE_LINE_SIZE /* Point to the next line out */
	205	blt+ .L_pmap_copy_page_loop /* Copy the whole page */
	206
	207	sync /* Make sure they're all done */
	208	li r4,PPC_PGBYTES-CACHE_LINE_SIZE /* Point to the end of the page */
	209
	210	invalinst:
	211	subic. r5,r4,CACHE_LINE_SIZE /* Point to the next one */
	212	icbi r4, r8 /* Trash the i-cache */
	213	subi r4,r5,CACHE_LINE_SIZE /* Point to the next one */
	214	icbi r5, r8 /* Trash the i-cache */
	215	bgt+ invalinst /* Keep going until we do the page... */
	216
	217	rlwimi r7,r9,0,MSR_DR_BIT,MSR_DR_BIT ; Set DDAT if on
	218	sync ; Make sure all invalidates done
	219
	220	mtmsr r7 ; Set DDAT correctly
	221	isync
	222
	223	lfd f0,FM_SIZE+0(r1) ; Restore an FP register
	224	lfd f1,FM_SIZE+8(r1) ; Restore an FP register
	225	lfd f2,FM_SIZE+16(r1) ; Restore an FP register
	226	lfd f3,FM_SIZE+24(r1) ; Restore an FP register
	227
	228	lwz r1,0(r1) ; Pop up the stack
	229
	230	mtmsr r9 ; Turn off FPU now and maybe rupts back on
231	isync
232	blr
233
234	#if 0
235	;
236	; This is not very optimal. We just do it here for a test of
237	; Altivec in the kernel.
238	;
239	wegotaltivec:
240	mfmsr r9 ; Get the MSR
241	lis r8,hi16(0xC0000000) ; Make sure we keep the first 2 vector registers
242	rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Disable interruptions
243	lis r6,lo16(2*256+128) ; Specify 128 blocks of 2 vectors each
244	rlwinm r7,r7,0,MSR_DR_BIT+1,MSR_DR_BIT-1 ; Clear the DDAT bit
245	ori r6,r6,32 ; Set a 32-byte stride
246	mtsprg 256,r8 ; Set VRSave
247	mtmsr r7 ; Disable rupts and turn xlate off
248	isync
249
250	addi r11,r3,4096 ; Point to the next page
251	li r10,16 ; Get vector size
252
253	avmovepg: lvxl v0,br0,r3 ; Get first half of line
254	dcba br0,r4 ; Allocate output
255	lvxl v1,r10,r3 ; Get second half of line
256	stvxl v0,br0,r4 ; Save first half of line
257	addi r3,r3,32 ; Point to the next line
258	icbi br0,r4 ; Make the icache go away also
259	stvxl v1,r10,r4 ; Save second half of line
260	cmplw r3,r11 ; Have we reached the next page?
261	dcbst br0,r4 ; Make sure the line is on its way out
262	addi r4,r4,32 ; Point to the next line
263	blt+ avmovepg ; Move the next line...
264
265	li r8,0 ; Clear this
266	sync ; Make sure all the memory stuff is done
267	mtsprg 256,r8 ; Show we are not using VRs any more
268	mtmsr r9 ; Translation and interruptions back on
269	isync
270	blr
271	#endif
272
273
274
275
1c79356b A	276	/*
1c79356b A	277	* int
de355530	278	* copyin(src, dst, count)
1c79356b A	279	* vm_offset_t src;
1c79356b A	280	* vm_offset_t dst;
de355530	281	* int count;
1c79356b A	282	*
	283	*/
	284
de355530 A	285	ENTRY2(copyin, copyinmsg, TAG_NO_FRAME_USED)
	286
	287	/* Preamble allowing us to call a sub-function */
	288	mflr r0
	289	stw r0,FM_LR_SAVE(r1)
	290	stwu r1,-(FM_SIZE+16)(r1)
	291
	292	cmpli cr0,r5,0
	293	ble- cr0,.L_copyinout_trivial
	294
	295	/* we know we have a valid copyin to do now */
	296	/* Set up thread_recover in case we hit an illegal address */
	297
	298	mfsprg r8,1 /* Get the current act */
	299	lwz r10,ACT_THREAD(r8)
	300	lis r11,hi16(.L_copyinout_error)
	301	lwz r8,ACT_VMMAP(r8)
	302	ori r11,r11,lo16(.L_copyinout_error)
	303	add r9,r3,r5 /* Get the end of the source */
	304	lwz r8,VMMAP_PMAP(r8) ; Get the pmap
	305	rlwinm r12,r3,6,26,29 ; Get index to the segment slot
	306	subi r9,r9,1 /* Make sure we don't go too far */
	307	add r8,r8,r12 ; Start indexing to the segment value
	308	stw r11,THREAD_RECOVER(r10)
	309	xor r9,r9,r3 /* Smoosh 'em together */
	310	lwz r8,PMAP_SEGS(r8) ; Get the source SR value
	311	rlwinm. r9,r9,0,1,3 /* Top nybble equal? */
	312	mtsr SR_COPYIN,r8 ; Set the SR
	313	isync
	314	#if 0
	315	lis r0,HIGH_ADDR(EXT(dbgRegsCall)) /* (TEST/DEBUG) */
	316	ori r0,r0,LOW_ADDR(EXT(dbgRegsCall)) /* (TEST/DEBUG) */
	317	sc /* (TEST/DEBUG) */
	318	#endif
	319
	320	/* For optimization, we check if the copyin lies on a segment
	321	* boundary. If it doesn't, we can use a simple copy. If it
	322	* does, we split it into two separate copies in some C code.
	323	*/
	324
	325	bne- .L_call_copyin_multiple /* Nope, we went past the segment boundary... */
	326
	327	rlwinm r3,r3,0,4,31
	328	oris r3,r3,(SR_COPYIN_NUM << (28-16)) /* Set the copyin segment as the source */
	329
	330	bl EXT(bcopy)
	331
	332	/* Now that copyin is done, we don't need a recovery point */
1c79356b	333
de355530 A	334	addi r1,r1,FM_SIZE+16
	335	mfsprg r6,1 /* Get the current act */
	336	lwz r10,ACT_THREAD(r6)
	337	li r3,0
	338	lwz r0,FM_LR_SAVE(r1)
	339	stw r3,THREAD_RECOVER(r10) /* Clear recovery */
	340	mtlr r0
	341	blr
	342
	343	/* we get here via the exception handler if an illegal
	344	* user memory reference was made.
	345	*/
	346	.L_copyinout_error:
	347
	348	/* Now that copyin is done, we don't need a recovery point */
	349
	350	mfsprg r6,1 /* Get the current act */
	351	addi r1,r1,FM_SIZE+16
	352	lwz r10,ACT_THREAD(r6)
	353	li r4,0
	354	lwz r0,FM_LR_SAVE(r1)
	355	stw r4,THREAD_RECOVER(r10) /* Clear recovery */
	356	mtlr r0
	357	li r3,EFAULT ; Indicate error (EFAULT)
	358	blr
	359
	360	.L_copyinout_trivial:
	361	/* The copyin/out was for either 0 bytes or a negative
	362	* number of bytes, return an appropriate value (0 == SUCCESS).
	363	* cr0 still contains result of comparison of len with 0.
	364	*/
	365	li r3, 0
	366	beq+ cr0, .L_copyinout_negative
	367	li r3, 1
	368	.L_copyinout_negative:
	369
	370	/* unwind the stack */
	371	addi r1, r1, FM_SIZE+16
	372	lwz r0, FM_LR_SAVE(r1)
	373	mtlr r0
	374
	375	blr
	376
	377	.L_call_copyin_multiple:
	378
	379	/* unwind the stack */
	380	addi r1, r1, FM_SIZE+16
	381	lwz r0, FM_LR_SAVE(r1)
	382	mtlr r0
	383
	384	b EXT(copyin_multiple) /* not a call - a jump! */
1c79356b A	385
1c79356b A	386	/*
d7e50217	387	* int
de355530 A	388	* copyout(src, dst, count)
	389	* vm_offset_t src;
	390	* vm_offset_t dst;
	391	* int count;
	392	*
	393	*/
	394
	395	ENTRY2(copyout, copyoutmsg, TAG_NO_FRAME_USED)
	396
	397	/* Preamble allowing us to call a sub-function */
	398
	399	mflr r0
	400	stw r0,FM_LR_SAVE(r1)
	401	stwu r1,-(FM_SIZE+16)(r1)
	402
	403	#if 0
	404	stw r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
	405	stw r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
	406	stw r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
	407	mr r6,r0 /* (TEST/DEBUG) */
	408
	409	bl EXT(tracecopyout) /* (TEST/DEBUG) */
	410
	411	lwz r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
	412	lwz r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
	413	lwz r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
	414	#endif
	415
	416	cmpli cr0,r5,0
	417	ble- cr0,.L_copyinout_trivial
	418	/* we know we have a valid copyout to do now */
	419	/* Set up thread_recover in case we hit an illegal address */
	420
	421
	422	mfsprg r8,1 /* Get the current act */
	423	lwz r10,ACT_THREAD(r8)
	424	lis r11,HIGH_ADDR(.L_copyinout_error)
	425	lwz r8,ACT_VMMAP(r8)
	426	rlwinm r12,r4,6,26,29 ; Get index to the segment slot
	427	ori r11,r11,LOW_ADDR(.L_copyinout_error)
	428	add r9,r4,r5 /* Get the end of the destination */
	429	lwz r8,VMMAP_PMAP(r8)
	430	subi r9,r9,1 /* Make sure we don't go too far */
	431	add r8,r8,r12 ; Start indexing to the segment value
	432	stw r11,THREAD_RECOVER(r10)
	433	xor r9,r9,r4 /* Smoosh 'em together */
	434	lwz r8,PMAP_SEGS(r8) ; Get the source SR value
	435	rlwinm. r9,r9,0,1,3 /* Top nybble equal? */
	436	mtsr SR_COPYIN,r8
	437	isync
	438
	439
	440	/* For optimisation, we check if the copyout lies on a segment
	441	* boundary. If it doesn't, we can use a simple copy. If it
	442	* does, we split it into two separate copies in some C code.
	443	*/
	444
	445	bne- .L_call_copyout_multiple /* Nope, we went past the segment boundary... */
	446
	447	rlwinm r4,r4,0,4,31
	448	oris r4,r4,(SR_COPYIN_NUM << (28-16)) /* Set the copyin segment as the source */
	449
	450	bl EXT(bcopy)
	451
452	/* Now that copyout is done, we don't need a recovery point */
453	mfsprg r6,1 /* Get the current act */
454	addi r1,r1,FM_SIZE+16
455	lwz r10,ACT_THREAD(r6)
456	li r3,0
457	lwz r0,FM_LR_SAVE(r1)
458	stw r3,THREAD_RECOVER(r10) /* Clear recovery */
459	mtlr r0
460	blr
461
462	.L_call_copyout_multiple:
463	/* unwind the stack */
464	addi r1, r1, FM_SIZE+16
465	lwz r0, FM_LR_SAVE(r1)
466	mtlr r0
467
468	b EXT(copyout_multiple) /* not a call - a jump! */
469
470	/*
471	* boolean_t
472	* copyinstr(src, dst, count, maxcount)
1c79356b A	473	* vm_offset_t src;
	474	* vm_offset_t dst;
	475	* vm_size_t maxcount;
	476	* vm_size_t* count;
	477	*
	478	* Set *count to the number of bytes copied
de355530	479	*
1c79356b A	480	* If dst == NULL, don't copy, just count bytes.
	481	* Only currently called from klcopyinstr.
	482	*/
	483
	484	ENTRY(copyinstr, TAG_NO_FRAME_USED)
1c79356b	485
de355530 A	486	/* Preamble allowing us to call a sub-function */
	487	mflr r0
	488	stw r0,FM_LR_SAVE(r1)
	489	stwu r1,-(FM_SIZE+16)(r1)
	490
	491	#if 0
	492	stw r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
	493	stw r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
	494	stw r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
	495	stw r6,FM_SIZE+12(r1) /* (TEST/DEBUG) */
	496	mr r7,r0 /* (TEST/DEBUG) */
	497
	498	bl EXT(tracecopystr) /* (TEST/DEBUG) */
	499
	500	lwz r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
	501	lwz r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
	502	lwz r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
	503	stw r6,FM_SIZE+12(r1) /* (TEST/DEBUG) */
	504	#endif
	505
	506	cmpli cr0,r5,0
	507	ble- cr0,.L_copyinout_trivial
	508
	509	/* we know we have a valid copyin to do now */
	510	/* Set up thread_recover in case we hit an illegal address */
	511
	512	li r0,0
	513	mfsprg r8,1 /* Get the current act */
	514	lwz r10,ACT_THREAD(r8)
	515	stw r0,0(r6) /* Clear result length */
	516	lis r11,HIGH_ADDR(.L_copyinout_error)
	517	lwz r8,ACT_VMMAP(r8) ; Get the map for this activation
	518	rlwinm r12,r3,6,26,29 ; Get index to the segment slot
	519	lwz r8,VMMAP_PMAP(r8)
	520	ori r11,r11,LOW_ADDR(.L_copyinout_error)
	521	add r8,r8,r12 ; Start indexing to the segment value
	522	stw r11,THREAD_RECOVER(r10)
	523	rlwinm r3,r3,0,4,31
	524	lwz r7,PMAP_SEGS(r8) ; Get the source SR value
	525	oris r3,r3,(SR_COPYIN_NUM << (28-16)) /* Set the copyin segment as the source */
	526
	527	/* Copy byte by byte for now - TODO NMGS speed this up with
	528	* some clever (but fairly standard) logic for word copies.
	529	* We don't use a copyinstr_multiple since copyinstr is called
	530	* with INT_MAX in the linux server. Eugh.
1c79356b	531	*/
1c79356b	532
de355530	533	li r9,0 /* Clear byte counter */
1c79356b	534
de355530 A	535	/* If the destination is NULL, don't do writes,
	536	* just count bytes. We set CR7 outside the loop to save time
	537	*/
	538	cmpwi cr7,r4,0 /* Is the destination null? */
d7e50217	539
de355530 A	540	nxtseg: mtsr SR_COPYIN,r7 /* Set the source SR */
de355530 A	541	isync
1c79356b	542
de355530 A	543	.L_copyinstr_loop:
	544	lbz r0,0(r3) /* Get the source */
	545	addic. r5,r5,-1 /* Have we gone far enough? */
	546	addi r3,r3,1 /* Bump source pointer */
	547
	548	cmpwi cr1,r0,0 /* Did we hit a null? */
1c79356b	549
de355530 A	550	beq cr7,.L_copyinstr_no_store /* If we are just counting, skip the store... */
	551
	552	stb r0,0(r4) /* Move to sink */
	553	addi r4,r4,1 /* Advance sink pointer */
	554
	555	.L_copyinstr_no_store:
1c79356b	556
de355530 A	557	addi r9,r9,1 /* Count the character */
	558	beq- cr1,.L_copyinstr_done /* We're done if we did a null... */
	559	beq- cr0,L_copyinstr_toobig /* Also if we maxed the count... */
	560
	561	/* Check to see if the copyin pointer has moved out of the
	562	* copyin segment, if it has we must remap.
	563	*/
d7e50217	564
de355530 A	565	rlwinm. r0,r3,0,4,31 /* Did we wrap around to 0? */
	566	bne+ cr0,.L_copyinstr_loop /* Nope... */
	567
	568	lwz r7,PMAP_SEGS+4(r8) ; Get the next source SR value
	569	addi r8,r8,4 ; Point to the next segment
	570	oris r3,r0,(SR_COPYIN_NUM << (28-16)) /* Reset the segment number */
	571	b nxtseg /* Keep going... */
	572
	573	L_copyinstr_toobig:
	574	li r3,ENAMETOOLONG
	575	b L_copyinstr_return
	576	.L_copyinstr_done:
	577	li r3,0 /* Normal return */
	578	L_copyinstr_return:
	579	li r4,0 /* to clear thread_recover */
	580	stw r9,0(r6) /* Set how many bytes we did */
	581	stw r4,THREAD_RECOVER(r10) /* Clear recovery exit */
	582
	583	addi r1, r1, FM_SIZE+16
	584	lwz r0, FM_LR_SAVE(r1)
	585	mtlr r0
	586	blr