[apple/xnu.git] / osfmk / ppc / bzero.s

/*
 * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <ppc/asm.h>
#include <ppc/exception.h>
#include <assym.s>

        .text
        .align	2
        .globl	_memset
        .globl	_bzero
        .globl	_bzero_nc
        .globl	_bzero_phys


// ***********************
// * B Z E R O _ P H Y S *
// ***********************
//
// void bzero_phys(addr64_t phys_addr, uint32_t length);
//
// Takes a phys addr in (r3,r4), and length in r5.  We leave cache on.

        .align	5
LEXT(bzero_phys)
        mflr	r12				// save return address
        rlwinm	r3,r3,0,1,0		// coallesce long-long in (r3,r4) into reg64_t in r3
        rlwimi	r3,r4,0,0,31
        mr		r4,r5			// put length where bzero() expects it
        bl		EXT(ml_set_physical_get_ffs)	// turn DR off, SF on, features in cr6, old MSR in r11
        bl		EXT(bzero)		// use normal bzero() routine
        mtlr	r12				// restore return
        b		EXT(ml_restore)		// restore MSR, turning DR on and SF off
        

// *******************
// * B Z E R O _ N C *
// *******************
//
//	void bzero_nc(char	*addr, unsigned int length);
//
// For use with uncached memory.  Doesn't seem to be used at all, so probably not
// performance critical.  NB: we must avoid unaligned stores, because some
// machines (eg, 970) take alignment exceptions on _any_ unaligned op to uncached
// memory.  Of course, we must also avoid dcbz.

LEXT(bzero_nc)
        cmplwi	cr1,r4,20		// too short to bother with 16-byte loops?
        cmplwi	cr7,r4,0		// check for (len==0)
        li		r6,0			// get a 0
        bge		cr1,bznc1		// skip if length >=20
        mtctr	r4				// set up byte loop
        beqlr--	cr7				// done if len=0
        
// Short operands, loop over bytes.

bznc0:
        stb		r6,0(r3)
        addi	r3,r3,1
        bdnz	bznc0
        blr
        
// Handle operands long enough to do doubleword stores; we must doubleword
// align, to avoid alignment exceptions.

bznc1:
        neg		r7,r3			// start to compute #bytes to align
        mfsprg	r10,2			// get feature flags
        andi.	r0,r7,7			// get #bytes to doubleword align
        mr		r5,r3			// make copy of operand ptr as bcopy expects
        mtcrf	0x02,r10		// put pf64Bitb etc in cr6
        beq		bzero_tail		// already doubleword aligned
        sub		r4,r4,r0		// adjust count
        mtctr	r0				// set up loop
bznc2:							// zero bytes until doubleword aligned
        stb		r6,0(r5)
        addi	r5,r5,1
        bdnz	bznc2
        b		bzero_tail		// join bzero, now that r5 is aligned
        

// *************     ***************
// * B Z E R O * and * M E M S E T *
// *************     ***************
//
// void *   memset(void *b, int c, size_t len);
// void		bzero(void *b, size_t len);
//
// These routines support G3, G4, and the 970, and run in both 32 and
// 64-bit mode.  Lengths (size_t) are always 32 bits.
//
// Register use:
//    r0 = temp
//    r2 = temp
//    r3 = original ptr, not changed since memset returns it
//    r4 = count of bytes to set
//    r5 = working operand ptr ("rp")
//    r6 = value to store (usually 0)
// r7-r9 = temps
//   r10 = feature flags
//   r11 = old MSR (if bzero_phys)
//   r12 = return address (if bzero_phys)
//   cr6 = feature flags (pf64Bit, pf128Byte, and pf32Byte)

        .align	5
LEXT(memset)					// void *   memset(void *b, int c, size_t len);
        andi.	r6,r4,0xFF		// copy value to working register, test for 0
        mr		r4,r5			// move length to working register
        bne--	memset1			// skip if nonzero
LEXT(bzero)						// void	bzero(void *b, size_t len);
        dcbtst	0,r3			// touch in 1st cache block
        mfsprg	r10,2			// get features
        li		r6,0			// get a 0
        neg		r7,r3			// start to compute #bytes to align
        andi.	r0,r10,pf128Byte+pf32Byte // get cache line size
        mtcrf	0x02,r10		// put pf128Byte etc in cr6
        cmplw	r4,r0			// operand length >= cache line size?
        mr		r5,r3			// make copy of operand ptr (can't change r3)
        blt		bzero_tail		// too short for dcbz (or dcbz128)
        rlwinm	r0,r7,0,0x1F	// get #bytes to  32-byte align
        rlwinm	r9,r7,0,0x7F	// get #bytes to 128-byte align
        bt++	pf128Byteb,bzero_128 // skip if 128-byte processor

// Operand length >=32 and cache line size is 32.
//		r0 = #bytes to 32-byte align
//		r4 = length
//		r5 = ptr to operand
//		r6 = 0

        sub		r2,r4,r0		// adjust length
        cmpwi	cr1,r0,0		// already 32-byte aligned?
        srwi.	r8,r2,5			// get #32-byte chunks
        beq		bzero_tail		// not long enough to dcbz
        mtctr	r8				// set up loop count
        rlwinm	r4,r2,0,27,31	// mask down to leftover byte count
        beq		cr1,bz_dcbz32 	// skip if already 32-byte aligned
        
// 32-byte align.  We just store 32 0s, rather than test and use conditional
// branches.  This is usually faster, because there are no mispredicts.

        stw		r6,0(r5)		// zero next 32 bytes
        stw		r6,4(r5)
        stw		r6,8(r5)
        stw		r6,12(r5)
        stw		r6,16(r5)
        stw		r6,20(r5)
        stw		r6,24(r5)
        stw		r6,28(r5)
        add		r5,r5,r0		// now r5 is 32-byte aligned
        b		bz_dcbz32

// Loop doing 32-byte version of DCBZ instruction.

        .align	4				// align the inner loop
bz_dcbz32:
        dcbz	0,r5			// zero another 32 bytes
        addi	r5,r5,32
        bdnz	bz_dcbz32

// Store trailing bytes.  This routine is used both by bzero and memset.
//		r4 = #bytes to store (may be large if memset)
//		r5 = address
//		r6 = value to store (in all 8 bytes)
//     cr6 = pf64Bit etc flags

bzero_tail:
        srwi.	r0,r4,4			// get #(16-byte-chunks)
        mtcrf	0x01,r4			// remaining byte count to cr7
        beq		bzt3			// no 16-byte chunks
        mtctr	r0				// set up loop count
        bt++	pf64Bitb,bzt2	// skip if 64-bit processor
        b		bzt1
        .align	5
bzt1:							// loop over 16-byte chunks on 32-bit processor
        stw		r6,0(r5)
        stw		r6,4(r5)
        stw		r6,8(r5)
        stw		r6,12(r5)
        addi	r5,r5,16
        bdnz	bzt1
        b		bzt3
        .align	5
bzt2:							// loop over 16-byte chunks on 64-bit processor
        std		r6,0(r5)
        std		r6,8(r5)
        addi	r5,r5,16
        bdnz	bzt2
        bf		28,bzt4			// 8-byte chunk?
        std		r6,0(r5)
        addi	r5,r5,8
        b		bzt4
bzt3:
        bf		28,bzt4			// 8-byte chunk?
        stw		r6,0(r5)
        stw		r6,4(r5)
        addi	r5,r5,8
bzt4:
        bf		29,bzt5			// word?
        stw		r6,0(r5)
        addi	r5,r5,4
bzt5:
        bf		30,bzt6			// halfword?
        sth		r6,0(r5)
        addi	r5,r5,2
bzt6:
        bflr	31				// byte?
        stb		r6,0(r5)
        blr
        
// Operand length is >=128 and cache line size is 128. We assume that
// because the linesize is 128 bytes, this is a 64-bit processor.
//		r4 = length
//		r5 = ptr to operand
//		r6 = 0
//		r7 = neg(r5)
//		r9 = #bytes to 128-byte align

        .align	5
bzero_128:
        sub		r2,r4,r9		// r2 <- length remaining after cache-line aligning
        rlwinm	r0,r7,0,0xF		// r0 <- #bytes to 16-byte align
        srwi.	r8,r2,7			// r8 <- number of cache lines to 0
        std		r6,0(r5)		// always store 16 bytes to 16-byte align...
        std		r6,8(r5)		// ...even if too short for dcbz128
        add		r5,r5,r0		// 16-byte align ptr
        sub		r4,r4,r0		// adjust count
        beq		bzero_tail		// r8==0, not long enough to dcbz128
        sub.	r7,r9,r0		// get #bytes remaining to 128-byte align
        rlwinm	r4,r2,0,0x7F	// r4 <- length remaining after dcbz128'ing
        mtctr	r8				// set up dcbz128 loop
        beq		bz_dcbz128		// already 128-byte aligned
        b		bz_align		// enter loop over 16-byte chunks

// 128-byte align by looping over 16-byte chunks.
        
        .align	5
bz_align:						// loop over 16-byte chunks
        subic.	r7,r7,16		// more to go?
        std		r6,0(r5)
        std		r6,8(r5)
        addi	r5,r5,16
        bgt		bz_align
        
        b		bz_dcbz128		// enter dcbz128 loop
        
// Loop over 128-byte cache lines.
//		r4 = length remaining after cache lines (0..127)
//		r5 = ptr (128-byte aligned)
//		r6 = 0
//		ctr = count of cache lines to 0

        .align	5
bz_dcbz128:
        dcbz128	0,r5			// zero a 128-byte cache line
        addi	r5,r5,128
        bdnz	bz_dcbz128
        
        b		bzero_tail		// handle leftovers


// Handle memset() for nonzero values.  This case is relatively infrequent;
// the large majority of memset() calls are for 0.
//		r3 = ptr
//		r4 = count
//		r6 = value in lower byte (nonzero)

memset1:
        cmplwi	r4,16			// too short to bother aligning?
        rlwimi	r6,r6,8,16,23	// replicate value to low 2 bytes
        mr		r5,r3			// make working copy of operand ptr
        rlwimi	r6,r6,16,0,15	// value now in all 4 bytes
        blt		bzero_tail		// length<16, we won't be using "std"
        mfsprg	r10,2			// get feature flags
        neg		r7,r5			// start to compute #bytes to align
        rlwinm	r6,r6,0,1,0		// value now in all 8 bytes (if 64-bit)
        andi.	r0,r7,7			// r6 <- #bytes to doubleword align
        stw		r6,0(r5)		// store 8 bytes to avoid a loop
        stw		r6,4(r5)
        mtcrf	0x02,r10		// get pf64Bit flag etc in cr6
        sub		r4,r4,r0		// adjust count
        add		r5,r5,r0		// doubleword align ptr
        b		bzero_tail
Commit	Line	Data
1c79356b	1	/*
d7e50217	2	* Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
1c79356b A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
d7e50217	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b	7	*
d7e50217 A	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
d7e50217 A	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
1c79356b A	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
1c79356b A	25
1c79356b A	26	#include <ppc/asm.h>
d7e50217 A	27	#include <ppc/exception.h>
	28	#include <assym.s>
	29
	30	.text
	31	.align 2
	32	.globl _memset
	33	.globl _bzero
	34	.globl _bzero_nc
	35	.globl _bzero_phys
	36
	37
	38	// ***********************
	39	// * B Z E R O _ P H Y S *
	40	// ***********************
	41	//
	42	// void bzero_phys(addr64_t phys_addr, uint32_t length);
	43	//
	44	// Takes a phys addr in (r3,r4), and length in r5. We leave cache on.
	45
	46	.align 5
	47	LEXT(bzero_phys)
	48	mflr r12 // save return address
	49	rlwinm r3,r3,0,1,0 // coallesce long-long in (r3,r4) into reg64_t in r3
	50	rlwimi r3,r4,0,0,31
	51	mr r4,r5 // put length where bzero() expects it
	52	bl EXT(ml_set_physical_get_ffs) // turn DR off, SF on, features in cr6, old MSR in r11
	53	bl EXT(bzero) // use normal bzero() routine
	54	mtlr r12 // restore return
	55	b EXT(ml_restore) // restore MSR, turning DR on and SF off
	56
	57
	58	// *******************
	59	// * B Z E R O _ N C *
	60	// *******************
	61	//
	62	// void bzero_nc(char *addr, unsigned int length);
	63	//
	64	// For use with uncached memory. Doesn't seem to be used at all, so probably not
	65	// performance critical. NB: we must avoid unaligned stores, because some
	66	// machines (eg, 970) take alignment exceptions on _any_ unaligned op to uncached
	67	// memory. Of course, we must also avoid dcbz.
	68
	69	LEXT(bzero_nc)
	70	cmplwi cr1,r4,20 // too short to bother with 16-byte loops?
	71	cmplwi cr7,r4,0 // check for (len==0)
	72	li r6,0 // get a 0
	73	bge cr1,bznc1 // skip if length >=20
	74	mtctr r4 // set up byte loop
	75	beqlr-- cr7 // done if len=0
	76
	77	// Short operands, loop over bytes.
	78
	79	bznc0:
	80	stb r6,0(r3)
	81	addi r3,r3,1
	82	bdnz bznc0
	83	blr
	84
	85	// Handle operands long enough to do doubleword stores; we must doubleword
	86	// align, to avoid alignment exceptions.
	87
	88	bznc1:
	89	neg r7,r3 // start to compute #bytes to align
	90	mfsprg r10,2 // get feature flags
91	andi. r0,r7,7 // get #bytes to doubleword align
92	mr r5,r3 // make copy of operand ptr as bcopy expects
93	mtcrf 0x02,r10 // put pf64Bitb etc in cr6
94	beq bzero_tail // already doubleword aligned
95	sub r4,r4,r0 // adjust count
96	mtctr r0 // set up loop
97	bznc2: // zero bytes until doubleword aligned
98	stb r6,0(r5)
99	addi r5,r5,1
100	bdnz bznc2
101	b bzero_tail // join bzero, now that r5 is aligned
102
103
104	// *********** *************
105	// * B Z E R O * and * M E M S E T *
106	// *********** *************
107	//
108	// void * memset(void *b, int c, size_t len);
109	// void bzero(void *b, size_t len);
110	//
111	// These routines support G3, G4, and the 970, and run in both 32 and
112	// 64-bit mode. Lengths (size_t) are always 32 bits.
113	//
114	// Register use:
115	// r0 = temp
116	// r2 = temp
117	// r3 = original ptr, not changed since memset returns it
118	// r4 = count of bytes to set
119	// r5 = working operand ptr ("rp")
120	// r6 = value to store (usually 0)
121	// r7-r9 = temps
122	// r10 = feature flags
123	// r11 = old MSR (if bzero_phys)
124	// r12 = return address (if bzero_phys)
125	// cr6 = feature flags (pf64Bit, pf128Byte, and pf32Byte)
126
127	.align 5
128	LEXT(memset) // void * memset(void *b, int c, size_t len);
129	andi. r6,r4,0xFF // copy value to working register, test for 0
130	mr r4,r5 // move length to working register
131	bne-- memset1 // skip if nonzero
132	LEXT(bzero) // void bzero(void *b, size_t len);
133	dcbtst 0,r3 // touch in 1st cache block
134	mfsprg r10,2 // get features
135	li r6,0 // get a 0
136	neg r7,r3 // start to compute #bytes to align
137	andi. r0,r10,pf128Byte+pf32Byte // get cache line size
138	mtcrf 0x02,r10 // put pf128Byte etc in cr6
139	cmplw r4,r0 // operand length >= cache line size?
140	mr r5,r3 // make copy of operand ptr (can't change r3)
141	blt bzero_tail // too short for dcbz (or dcbz128)
142	rlwinm r0,r7,0,0x1F // get #bytes to 32-byte align
143	rlwinm r9,r7,0,0x7F // get #bytes to 128-byte align
144	bt++ pf128Byteb,bzero_128 // skip if 128-byte processor
145
146	// Operand length >=32 and cache line size is 32.
147	// r0 = #bytes to 32-byte align
148	// r4 = length
149	// r5 = ptr to operand
150	// r6 = 0
151
152	sub r2,r4,r0 // adjust length
153	cmpwi cr1,r0,0 // already 32-byte aligned?
154	srwi. r8,r2,5 // get #32-byte chunks
155	beq bzero_tail // not long enough to dcbz
156	mtctr r8 // set up loop count
157	rlwinm r4,r2,0,27,31 // mask down to leftover byte count
158	beq cr1,bz_dcbz32 // skip if already 32-byte aligned
159
160	// 32-byte align. We just store 32 0s, rather than test and use conditional
161	// branches. This is usually faster, because there are no mispredicts.
162
163	stw r6,0(r5) // zero next 32 bytes
164	stw r6,4(r5)
165	stw r6,8(r5)
166	stw r6,12(r5)
167	stw r6,16(r5)
168	stw r6,20(r5)
169	stw r6,24(r5)
170	stw r6,28(r5)
171	add r5,r5,r0 // now r5 is 32-byte aligned
172	b bz_dcbz32
173
174	// Loop doing 32-byte version of DCBZ instruction.
175
176	.align 4 // align the inner loop
177	bz_dcbz32:
178	dcbz 0,r5 // zero another 32 bytes
179	addi r5,r5,32
180	bdnz bz_dcbz32
181
182	// Store trailing bytes. This routine is used both by bzero and memset.
183	// r4 = #bytes to store (may be large if memset)
184	// r5 = address
185	// r6 = value to store (in all 8 bytes)
186	// cr6 = pf64Bit etc flags
187
188	bzero_tail:
189	srwi. r0,r4,4 // get #(16-byte-chunks)
190	mtcrf 0x01,r4 // remaining byte count to cr7
191	beq bzt3 // no 16-byte chunks
192	mtctr r0 // set up loop count
193	bt++ pf64Bitb,bzt2 // skip if 64-bit processor
194	b bzt1
195	.align 5
196	bzt1: // loop over 16-byte chunks on 32-bit processor
197	stw r6,0(r5)
198	stw r6,4(r5)
199	stw r6,8(r5)
200	stw r6,12(r5)
201	addi r5,r5,16
202	bdnz bzt1
203	b bzt3
204	.align 5
205	bzt2: // loop over 16-byte chunks on 64-bit processor
206	std r6,0(r5)
207	std r6,8(r5)
208	addi r5,r5,16
209	bdnz bzt2
210	bf 28,bzt4 // 8-byte chunk?
211	std r6,0(r5)
212	addi r5,r5,8
213	b bzt4
214	bzt3:
215	bf 28,bzt4 // 8-byte chunk?
216	stw r6,0(r5)
217	stw r6,4(r5)
218	addi r5,r5,8
219	bzt4:
220	bf 29,bzt5 // word?
221	stw r6,0(r5)
222	addi r5,r5,4
223	bzt5:
224	bf 30,bzt6 // halfword?
225	sth r6,0(r5)
226	addi r5,r5,2
227	bzt6:
228	bflr 31 // byte?
229	stb r6,0(r5)
230	blr
231
232	// Operand length is >=128 and cache line size is 128. We assume that
233	// because the linesize is 128 bytes, this is a 64-bit processor.
234	// r4 = length
235	// r5 = ptr to operand
236	// r6 = 0
237	// r7 = neg(r5)
238	// r9 = #bytes to 128-byte align
239
240	.align 5
241	bzero_128:
242	sub r2,r4,r9 // r2 <- length remaining after cache-line aligning
243	rlwinm r0,r7,0,0xF // r0 <- #bytes to 16-byte align
244	srwi. r8,r2,7 // r8 <- number of cache lines to 0
245	std r6,0(r5) // always store 16 bytes to 16-byte align...
246	std r6,8(r5) // ...even if too short for dcbz128
247	add r5,r5,r0 // 16-byte align ptr
248	sub r4,r4,r0 // adjust count
249	beq bzero_tail // r8==0, not long enough to dcbz128
250	sub. r7,r9,r0 // get #bytes remaining to 128-byte align
251	rlwinm r4,r2,0,0x7F // r4 <- length remaining after dcbz128'ing
252	mtctr r8 // set up dcbz128 loop
253	beq bz_dcbz128 // already 128-byte aligned
254	b bz_align // enter loop over 16-byte chunks
255
256	// 128-byte align by looping over 16-byte chunks.
257
258	.align 5
259	bz_align: // loop over 16-byte chunks
260	subic. r7,r7,16 // more to go?
261	std r6,0(r5)
262	std r6,8(r5)
263	addi r5,r5,16
264	bgt bz_align
265
266	b bz_dcbz128 // enter dcbz128 loop
267
268	// Loop over 128-byte cache lines.
269	// r4 = length remaining after cache lines (0..127)
270	// r5 = ptr (128-byte aligned)
271	// r6 = 0
272	// ctr = count of cache lines to 0
273
274	.align 5
275	bz_dcbz128:
276	dcbz128 0,r5 // zero a 128-byte cache line
277	addi r5,r5,128
278	bdnz bz_dcbz128
279
280	b bzero_tail // handle leftovers
281
282
283	// Handle memset() for nonzero values. This case is relatively infrequent;
284	// the large majority of memset() calls are for 0.
285	// r3 = ptr
286	// r4 = count
287	// r6 = value in lower byte (nonzero)
288
289	memset1:
290	cmplwi r4,16 // too short to bother aligning?
291	rlwimi r6,r6,8,16,23 // replicate value to low 2 bytes
292	mr r5,r3 // make working copy of operand ptr
293	rlwimi r6,r6,16,0,15 // value now in all 4 bytes
294	blt bzero_tail // length<16, we won't be using "std"
295	mfsprg r10,2 // get feature flags
296	neg r7,r5 // start to compute #bytes to align
297	rlwinm r6,r6,0,1,0 // value now in all 8 bytes (if 64-bit)
298	andi. r0,r7,7 // r6 <- #bytes to doubleword align
299	stw r6,0(r5) // store 8 bytes to avoid a loop
300	stw r6,4(r5)
301	mtcrf 0x02,r10 // get pf64Bit flag etc in cr6
302	sub r4,r4,r0 // adjust count
303	add r5,r5,r0 // doubleword align ptr
304	b bzero_tail
305
306
9bccf70c	307