[apple/xnu.git] / osfmk / i386 / commpage / longcopy_sse3x_64.s

/*
 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
        
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
      

/*
 * The bcopy/memcpy loops for very long operands, tuned for 64-bit
 * Pentium-M class processors with Supplemental SSE3 and 64-byte cache lines.
 * This is the 64-bit version.
 *
 * The following #defines are tightly coupled to the u-architecture:
 */

#define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands


// Very long forward moves.  These are at least several pages, so we loop over big
// chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
// it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
// so the copy loop reads from L2 and writes directly to memory (with write combining.)
// This minimizes bus turnaround and maintains good DRAM page locality.
// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
// size.  Otherwise, it is counter-productive to bypass L2 on the stores.
//
// We are called from the commpage bcopy loops when they encounter very long
// operands, with the standard ABI:
//      rdi = dest ptr
//      rsi = source ptr
//      rdx = length (>= 8kb, probably much bigger)

// void longcopy(const void *dest, void *sou, size_t len)
			
COMMPAGE_FUNCTION_START(longcopy_sse3x_64, 64, 5)
	pushq	%rbp			// set up a frame for backtraces
	movq	%rsp,%rbp
	movl	%edi,%eax		// copy dest ptr
	negl	%eax
	andl	$63,%eax		// get #bytes to cache line align destination
	jz	LBigChunkLoop		// already aligned
	
// Cache line align destination, so temporal stores in copy loops work right.
// The recursive call returns with the source and dest ptrs properly updated.

	subq	%rax,%rdx		// get length remaining after dest is aligned
	pushq	%rdx			// save length remaining
	movl	%eax,%edx		// #bytes to copy to align destination
	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
	call	*%rax
	popq	%rdx			// recover adjusted length
	
// Loop over big chunks.
//      rdx = length remaining (>= 4096)
//      rdi = dest (64-byte aligned)
//      rsi = source (may be unaligned)
	
LBigChunkLoop:
        movl    $(kBigChunk),%r8d       // assume we can do a full chunk
	cmpq	%r8,%rdx		// do we have a full chunk left to do?
	cmovbl	%edx,%r8d		// if not, only move what we have left
	andl	$-4096,%r8d		// we work in page multiples
	xorl	%eax,%eax		// initialize chunk offset
	jmp	LTouchLoop

// Touch in the next chunk.  We try to keep the prefetch unit in "kick-start" mode,
// by touching two adjacent cache lines every 8 lines of each page, in four slices.
// Because the source may be unaligned, we use byte loads to touch.
//      rdx = length remaining (including this chunk)
//      rdi = ptr to start of dest chunk
//      rsi = ptr to start of source chunk
//	r8d = chunk length (multiples of pages, less than  2**32)
//	ecx = scratch reg used to read a byte of each cache line
//	eax = chunk offset

	.align	4,0x90			// 16-byte align inner loops
LTouchLoop:
	movzb	(%rsi,%rax),%ecx	// touch line 0, 2, 4, or 6 of page
	movzb	1*64(%rsi,%rax),%ecx	// touch line 1, 3, 5, or 7
	movzb	8*64(%rsi,%rax),%ecx	// touch line 8, 10, 12, or 14
	movzb	9*64(%rsi,%rax),%ecx	// etc
	
	movzb	16*64(%rsi,%rax),%ecx
	movzb	17*64(%rsi,%rax),%ecx
	movzb	24*64(%rsi,%rax),%ecx
	movzb	25*64(%rsi,%rax),%ecx

	movzb	32*64(%rsi,%rax),%ecx
	movzb	33*64(%rsi,%rax),%ecx
	movzb	40*64(%rsi,%rax),%ecx
	movzb	41*64(%rsi,%rax),%ecx

	movzb	48*64(%rsi,%rax),%ecx
	movzb	49*64(%rsi,%rax),%ecx
	movzb	56*64(%rsi,%rax),%ecx
	movzb	57*64(%rsi,%rax),%ecx
	
	subl	$-128,%eax		// next slice of page (adding 128 w 8-bit immediate)
	testl	$512,%eax		// done with this page?
	jz	LTouchLoop		// no, next of four slices
	addl	$(4096-512),%eax	// move on to next page
	cmpl	%eax,%r8d		// done with this chunk?
	jnz	LTouchLoop		// no, do next page
	
// The chunk has been pre-fetched, now copy it using non-temporal stores.
// There are two copy loops, depending on whether the source is 16-byte aligned
// or not.

	movl	%r8d,%ecx		// copy chunk size to a reg that doesn't use REX prefix
	addq	%rcx,%rsi		// increment ptrs by chunk length
	addq	%rcx,%rdi
	subq	%rcx,%rdx		// adjust remaining length
	negq	%rcx			// prepare loop index (counts up to 0)
	testl	$15,%esi		// is source 16-byte aligned?
	jnz	LVeryLongUnaligned	// no
	jmp	LVeryLongAligned

	.align	4,0x90			// 16-byte align inner loops
LVeryLongAligned:			// aligned loop over 128-bytes
        movdqa  (%rsi,%rcx),%xmm0
        movdqa  16(%rsi,%rcx),%xmm1
        movdqa  32(%rsi,%rcx),%xmm2
        movdqa  48(%rsi,%rcx),%xmm3
        movdqa  64(%rsi,%rcx),%xmm4
        movdqa  80(%rsi,%rcx),%xmm5
        movdqa  96(%rsi,%rcx),%xmm6
        movdqa  112(%rsi,%rcx),%xmm7

        movntdq %xmm0,(%rdi,%rcx)
        movntdq %xmm1,16(%rdi,%rcx)
        movntdq %xmm2,32(%rdi,%rcx)
        movntdq %xmm3,48(%rdi,%rcx)
        movntdq %xmm4,64(%rdi,%rcx)
        movntdq %xmm5,80(%rdi,%rcx)
        movntdq %xmm6,96(%rdi,%rcx)
        movntdq %xmm7,112(%rdi,%rcx)
        
        subq    $-128,%rcx		// add 128 with an 8-bit immediate
        jnz	LVeryLongAligned
	jmp	LVeryLongChunkEnd

	.align	4,0x90			// 16-byte align inner loops
LVeryLongUnaligned:			// unaligned loop over 128-bytes
        movdqu  (%rsi,%rcx),%xmm0
        movdqu  16(%rsi,%rcx),%xmm1
        movdqu  32(%rsi,%rcx),%xmm2
        movdqu  48(%rsi,%rcx),%xmm3
        movdqu  64(%rsi,%rcx),%xmm4
        movdqu  80(%rsi,%rcx),%xmm5
        movdqu  96(%rsi,%rcx),%xmm6
        movdqu  112(%rsi,%rcx),%xmm7

        movntdq %xmm0,(%rdi,%rcx)
        movntdq %xmm1,16(%rdi,%rcx)
        movntdq %xmm2,32(%rdi,%rcx)
        movntdq %xmm3,48(%rdi,%rcx)
        movntdq %xmm4,64(%rdi,%rcx)
        movntdq %xmm5,80(%rdi,%rcx)
        movntdq %xmm6,96(%rdi,%rcx)
        movntdq %xmm7,112(%rdi,%rcx)
        
        subq    $-128,%rcx		// add 128 with an 8-bit immediate
        jnz     LVeryLongUnaligned

LVeryLongChunkEnd:
	cmpq	$4096,%rdx		// at least another page to go?
	jae	LBigChunkLoop		// yes
	
// Done.  Call memcpy() again to handle the 0-4095 bytes at the end.
// We still have the args in the right registers:
//	rdi = destination ptr
//	rsi = source ptr
//	rdx = length remaining (0..4095)

	sfence				// required by non-temporal stores
	testl	%edx,%edx		// anything left to copy?
	jz	1f
	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
	call	*%rax
1:
	popq	%rbp			// restore frame ptr
	ret

/* always match for now, as commpage_stuff_routine() will panic if no match */
COMMPAGE_DESCRIPTOR(longcopy_sse3x_64, _COMM_PAGE_LONGCOPY, 0 ,0)
Commit	Line	Data
0c530ab8 A	1	/*
	2	* Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
	3	*
2d21ac55 A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
0c530ab8 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8 A	27	*/
	28
	29	#include <machine/cpu_capabilities.h>
	30	#include <machine/commpage.h>
	31
	32
	33	/*
	34	* The bcopy/memcpy loops for very long operands, tuned for 64-bit
2d21ac55	35	* Pentium-M class processors with Supplemental SSE3 and 64-byte cache lines.
0c530ab8 A	36	* This is the 64-bit version.
	37	*
	38	* The following #defines are tightly coupled to the u-architecture:
	39	*/
	40
	41	#define kBigChunk (256*1024) // outer loop chunk size for kVeryLong sized operands
	42
	43
	44	// Very long forward moves. These are at least several pages, so we loop over big
	45	// chunks of memory (kBigChunk in size.) We first prefetch the chunk, and then copy
	46	// it using non-temporal stores. Hopefully all the reads occur in the prefetch loop,
	47	// so the copy loop reads from L2 and writes directly to memory (with write combining.)
	48	// This minimizes bus turnaround and maintains good DRAM page locality.
	49	// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
	50	// size. Otherwise, it is counter-productive to bypass L2 on the stores.
	51	//
	52	// We are called from the commpage bcopy loops when they encounter very long
	53	// operands, with the standard ABI:
	54	// rdi = dest ptr
	55	// rsi = source ptr
	56	// rdx = length (>= 8kb, probably much bigger)
b0d623f7 A	57
	58	// void longcopy(const void dest, void sou, size_t len)
	59
	60	COMMPAGE_FUNCTION_START(longcopy_sse3x_64, 64, 5)
0c530ab8 A	61	pushq %rbp // set up a frame for backtraces
	62	movq %rsp,%rbp
	63	movl %edi,%eax // copy dest ptr
	64	negl %eax
	65	andl $63,%eax // get #bytes to cache line align destination
	66	jz LBigChunkLoop // already aligned
	67
	68	// Cache line align destination, so temporal stores in copy loops work right.
	69	// The recursive call returns with the source and dest ptrs properly updated.
	70
	71	subq %rax,%rdx // get length remaining after dest is aligned
	72	pushq %rdx // save length remaining
	73	movl %eax,%edx // #bytes to copy to align destination
	74	movq $_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
	75	call *%rax
	76	popq %rdx // recover adjusted length
	77
	78	// Loop over big chunks.
	79	// rdx = length remaining (>= 4096)
	80	// rdi = dest (64-byte aligned)
	81	// rsi = source (may be unaligned)
	82
	83	LBigChunkLoop:
	84	movl $(kBigChunk),%r8d // assume we can do a full chunk
	85	cmpq %r8,%rdx // do we have a full chunk left to do?
	86	cmovbl %edx,%r8d // if not, only move what we have left
	87	andl $-4096,%r8d // we work in page multiples
	88	xorl %eax,%eax // initialize chunk offset
	89	jmp LTouchLoop
	90
	91	// Touch in the next chunk. We try to keep the prefetch unit in "kick-start" mode,
	92	// by touching two adjacent cache lines every 8 lines of each page, in four slices.
	93	// Because the source may be unaligned, we use byte loads to touch.
	94	// rdx = length remaining (including this chunk)
	95	// rdi = ptr to start of dest chunk
	96	// rsi = ptr to start of source chunk
	97	// r8d = chunk length (multiples of pages, less than 2**32)
	98	// ecx = scratch reg used to read a byte of each cache line
	99	// eax = chunk offset
	100
	101	.align 4,0x90 // 16-byte align inner loops
	102	LTouchLoop:
	103	movzb (%rsi,%rax),%ecx // touch line 0, 2, 4, or 6 of page
	104	movzb 1*64(%rsi,%rax),%ecx // touch line 1, 3, 5, or 7
	105	movzb 8*64(%rsi,%rax),%ecx // touch line 8, 10, 12, or 14
	106	movzb 9*64(%rsi,%rax),%ecx // etc
	107
	108	movzb 16*64(%rsi,%rax),%ecx
	109	movzb 17*64(%rsi,%rax),%ecx
	110	movzb 24*64(%rsi,%rax),%ecx
	111	movzb 25*64(%rsi,%rax),%ecx
	112
	113	movzb 32*64(%rsi,%rax),%ecx
	114	movzb 33*64(%rsi,%rax),%ecx
	115	movzb 40*64(%rsi,%rax),%ecx
	116	movzb 41*64(%rsi,%rax),%ecx
	117
	118	movzb 48*64(%rsi,%rax),%ecx
	119	movzb 49*64(%rsi,%rax),%ecx
	120	movzb 56*64(%rsi,%rax),%ecx
	121	movzb 57*64(%rsi,%rax),%ecx
	122
	123	subl $-128,%eax // next slice of page (adding 128 w 8-bit immediate)
	124	testl $512,%eax // done with this page?
125	jz LTouchLoop // no, next of four slices
126	addl $(4096-512),%eax // move on to next page
127	cmpl %eax,%r8d // done with this chunk?
128	jnz LTouchLoop // no, do next page
129
130	// The chunk has been pre-fetched, now copy it using non-temporal stores.
131	// There are two copy loops, depending on whether the source is 16-byte aligned
132	// or not.
133
134	movl %r8d,%ecx // copy chunk size to a reg that doesn't use REX prefix
135	addq %rcx,%rsi // increment ptrs by chunk length
136	addq %rcx,%rdi
137	subq %rcx,%rdx // adjust remaining length
138	negq %rcx // prepare loop index (counts up to 0)
139	testl $15,%esi // is source 16-byte aligned?
140	jnz LVeryLongUnaligned // no
141	jmp LVeryLongAligned
142
143	.align 4,0x90 // 16-byte align inner loops
144	LVeryLongAligned: // aligned loop over 128-bytes
145	movdqa (%rsi,%rcx),%xmm0
146	movdqa 16(%rsi,%rcx),%xmm1
147	movdqa 32(%rsi,%rcx),%xmm2
148	movdqa 48(%rsi,%rcx),%xmm3
149	movdqa 64(%rsi,%rcx),%xmm4
150	movdqa 80(%rsi,%rcx),%xmm5
151	movdqa 96(%rsi,%rcx),%xmm6
152	movdqa 112(%rsi,%rcx),%xmm7
153
154	movntdq %xmm0,(%rdi,%rcx)
155	movntdq %xmm1,16(%rdi,%rcx)
156	movntdq %xmm2,32(%rdi,%rcx)
157	movntdq %xmm3,48(%rdi,%rcx)
158	movntdq %xmm4,64(%rdi,%rcx)
159	movntdq %xmm5,80(%rdi,%rcx)
160	movntdq %xmm6,96(%rdi,%rcx)
161	movntdq %xmm7,112(%rdi,%rcx)
162
163	subq $-128,%rcx // add 128 with an 8-bit immediate
164	jnz LVeryLongAligned
165	jmp LVeryLongChunkEnd
166
167	.align 4,0x90 // 16-byte align inner loops
168	LVeryLongUnaligned: // unaligned loop over 128-bytes
169	movdqu (%rsi,%rcx),%xmm0
170	movdqu 16(%rsi,%rcx),%xmm1
171	movdqu 32(%rsi,%rcx),%xmm2
172	movdqu 48(%rsi,%rcx),%xmm3
173	movdqu 64(%rsi,%rcx),%xmm4
174	movdqu 80(%rsi,%rcx),%xmm5
175	movdqu 96(%rsi,%rcx),%xmm6
176	movdqu 112(%rsi,%rcx),%xmm7
177
178	movntdq %xmm0,(%rdi,%rcx)
179	movntdq %xmm1,16(%rdi,%rcx)
180	movntdq %xmm2,32(%rdi,%rcx)
181	movntdq %xmm3,48(%rdi,%rcx)
182	movntdq %xmm4,64(%rdi,%rcx)
183	movntdq %xmm5,80(%rdi,%rcx)
184	movntdq %xmm6,96(%rdi,%rcx)
185	movntdq %xmm7,112(%rdi,%rcx)
186
187	subq $-128,%rcx // add 128 with an 8-bit immediate
188	jnz LVeryLongUnaligned
189
190	LVeryLongChunkEnd:
191	cmpq $4096,%rdx // at least another page to go?
192	jae LBigChunkLoop // yes
193
194	// Done. Call memcpy() again to handle the 0-4095 bytes at the end.
195	// We still have the args in the right registers:
196	// rdi = destination ptr
197	// rsi = source ptr
198	// rdx = length remaining (0..4095)
199
200	sfence // required by non-temporal stores
201	testl %edx,%edx // anything left to copy?
202	jz 1f
203	movq $_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
204	call *%rax
205	1:
206	popq %rbp // restore frame ptr
207	ret
208
b0d623f7 A	209	/* always match for now, as commpage_stuff_routine() will panic if no match */
b0d623f7 A	210	COMMPAGE_DESCRIPTOR(longcopy_sse3x_64, _COMM_PAGE_LONGCOPY, 0 ,0)