[apple/xnu.git] / osfmk / i386 / commpage / longcopy_sse3x_64.s

/*
 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
        
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
      

/*
 * The bcopy/memcpy loops for very long operands, tuned for 64-bit
 * Pentium-M class processors with Supplemental SSE3 and 64-byte cache lines.
 * This is the 64-bit version.
 *
 * The following #defines are tightly coupled to the u-architecture:
 */

#define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands


// Very long forward moves.  These are at least several pages, so we loop over big
// chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
// it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
// so the copy loop reads from L2 and writes directly to memory (with write combining.)
// This minimizes bus turnaround and maintains good DRAM page locality.
// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
// size.  Otherwise, it is counter-productive to bypass L2 on the stores.
//
// We are called from the commpage bcopy loops when they encounter very long
// operands, with the standard ABI:
//      rdi = dest ptr
//      rsi = source ptr
//      rdx = length (>= 8kb, probably much bigger)
 
        .text
	.code64
        .align 5, 0x90
Llongcopy_sse3x_64:			// void longcopy(const void *dest, void *sou, size_t len)
	pushq	%rbp			// set up a frame for backtraces
	movq	%rsp,%rbp
	movl	%edi,%eax		// copy dest ptr
	negl	%eax
	andl	$63,%eax		// get #bytes to cache line align destination
	jz	LBigChunkLoop		// already aligned
	
// Cache line align destination, so temporal stores in copy loops work right.
// The recursive call returns with the source and dest ptrs properly updated.

	subq	%rax,%rdx		// get length remaining after dest is aligned
	pushq	%rdx			// save length remaining
	movl	%eax,%edx		// #bytes to copy to align destination
	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
	call	*%rax
	popq	%rdx			// recover adjusted length
	
// Loop over big chunks.
//      rdx = length remaining (>= 4096)
//      rdi = dest (64-byte aligned)
//      rsi = source (may be unaligned)
	
LBigChunkLoop:
        movl    $(kBigChunk),%r8d       // assume we can do a full chunk
	cmpq	%r8,%rdx		// do we have a full chunk left to do?
	cmovbl	%edx,%r8d		// if not, only move what we have left
	andl	$-4096,%r8d		// we work in page multiples
	xorl	%eax,%eax		// initialize chunk offset
	jmp	LTouchLoop

// Touch in the next chunk.  We try to keep the prefetch unit in "kick-start" mode,
// by touching two adjacent cache lines every 8 lines of each page, in four slices.
// Because the source may be unaligned, we use byte loads to touch.
//      rdx = length remaining (including this chunk)
//      rdi = ptr to start of dest chunk
//      rsi = ptr to start of source chunk
//	r8d = chunk length (multiples of pages, less than  2**32)
//	ecx = scratch reg used to read a byte of each cache line
//	eax = chunk offset

	.align	4,0x90			// 16-byte align inner loops
LTouchLoop:
	movzb	(%rsi,%rax),%ecx	// touch line 0, 2, 4, or 6 of page
	movzb	1*64(%rsi,%rax),%ecx	// touch line 1, 3, 5, or 7
	movzb	8*64(%rsi,%rax),%ecx	// touch line 8, 10, 12, or 14
	movzb	9*64(%rsi,%rax),%ecx	// etc
	
	movzb	16*64(%rsi,%rax),%ecx
	movzb	17*64(%rsi,%rax),%ecx
	movzb	24*64(%rsi,%rax),%ecx
	movzb	25*64(%rsi,%rax),%ecx

	movzb	32*64(%rsi,%rax),%ecx
	movzb	33*64(%rsi,%rax),%ecx
	movzb	40*64(%rsi,%rax),%ecx
	movzb	41*64(%rsi,%rax),%ecx

	movzb	48*64(%rsi,%rax),%ecx
	movzb	49*64(%rsi,%rax),%ecx
	movzb	56*64(%rsi,%rax),%ecx
	movzb	57*64(%rsi,%rax),%ecx
	
	subl	$-128,%eax		// next slice of page (adding 128 w 8-bit immediate)
	testl	$512,%eax		// done with this page?
	jz	LTouchLoop		// no, next of four slices
	addl	$(4096-512),%eax	// move on to next page
	cmpl	%eax,%r8d		// done with this chunk?
	jnz	LTouchLoop		// no, do next page
	
// The chunk has been pre-fetched, now copy it using non-temporal stores.
// There are two copy loops, depending on whether the source is 16-byte aligned
// or not.

	movl	%r8d,%ecx		// copy chunk size to a reg that doesn't use REX prefix
	addq	%rcx,%rsi		// increment ptrs by chunk length
	addq	%rcx,%rdi
	subq	%rcx,%rdx		// adjust remaining length
	negq	%rcx			// prepare loop index (counts up to 0)
	testl	$15,%esi		// is source 16-byte aligned?
	jnz	LVeryLongUnaligned	// no
	jmp	LVeryLongAligned

	.align	4,0x90			// 16-byte align inner loops
LVeryLongAligned:			// aligned loop over 128-bytes
        movdqa  (%rsi,%rcx),%xmm0
        movdqa  16(%rsi,%rcx),%xmm1
        movdqa  32(%rsi,%rcx),%xmm2
        movdqa  48(%rsi,%rcx),%xmm3
        movdqa  64(%rsi,%rcx),%xmm4
        movdqa  80(%rsi,%rcx),%xmm5
        movdqa  96(%rsi,%rcx),%xmm6
        movdqa  112(%rsi,%rcx),%xmm7

        movntdq %xmm0,(%rdi,%rcx)
        movntdq %xmm1,16(%rdi,%rcx)
        movntdq %xmm2,32(%rdi,%rcx)
        movntdq %xmm3,48(%rdi,%rcx)
        movntdq %xmm4,64(%rdi,%rcx)
        movntdq %xmm5,80(%rdi,%rcx)
        movntdq %xmm6,96(%rdi,%rcx)
        movntdq %xmm7,112(%rdi,%rcx)
        
        subq    $-128,%rcx		// add 128 with an 8-bit immediate
        jnz	LVeryLongAligned
	jmp	LVeryLongChunkEnd

	.align	4,0x90			// 16-byte align inner loops
LVeryLongUnaligned:			// unaligned loop over 128-bytes
        movdqu  (%rsi,%rcx),%xmm0
        movdqu  16(%rsi,%rcx),%xmm1
        movdqu  32(%rsi,%rcx),%xmm2
        movdqu  48(%rsi,%rcx),%xmm3
        movdqu  64(%rsi,%rcx),%xmm4
        movdqu  80(%rsi,%rcx),%xmm5
        movdqu  96(%rsi,%rcx),%xmm6
        movdqu  112(%rsi,%rcx),%xmm7

        movntdq %xmm0,(%rdi,%rcx)
        movntdq %xmm1,16(%rdi,%rcx)
        movntdq %xmm2,32(%rdi,%rcx)
        movntdq %xmm3,48(%rdi,%rcx)
        movntdq %xmm4,64(%rdi,%rcx)
        movntdq %xmm5,80(%rdi,%rcx)
        movntdq %xmm6,96(%rdi,%rcx)
        movntdq %xmm7,112(%rdi,%rcx)
        
        subq    $-128,%rcx		// add 128 with an 8-bit immediate
        jnz     LVeryLongUnaligned

LVeryLongChunkEnd:
	cmpq	$4096,%rdx		// at least another page to go?
	jae	LBigChunkLoop		// yes
	
// Done.  Call memcpy() again to handle the 0-4095 bytes at the end.
// We still have the args in the right registers:
//	rdi = destination ptr
//	rsi = source ptr
//	rdx = length remaining (0..4095)

	sfence				// required by non-temporal stores
	testl	%edx,%edx		// anything left to copy?
	jz	1f
	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
	call	*%rax
1:
	popq	%rbp			// restore frame ptr
	ret

	/* always match for now, as commpage_stuff_routine() will panic if no match */
	COMMPAGE_DESCRIPTOR(longcopy_sse3x_64, _COMM_PAGE_LONGCOPY, 0 ,0)
Commit	Line	Data
0c530ab8 A	1	/*
	2	* Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
	3	*
2d21ac55 A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
0c530ab8 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8 A	27	*/
	28
	29	#include <machine/cpu_capabilities.h>
	30	#include <machine/commpage.h>
	31
	32
	33	/*
	34	* The bcopy/memcpy loops for very long operands, tuned for 64-bit
2d21ac55	35	* Pentium-M class processors with Supplemental SSE3 and 64-byte cache lines.
0c530ab8 A	36	* This is the 64-bit version.
	37	*
	38	* The following #defines are tightly coupled to the u-architecture:
	39	*/
	40
	41	#define kBigChunk (256*1024) // outer loop chunk size for kVeryLong sized operands
	42
	43
	44	// Very long forward moves. These are at least several pages, so we loop over big
	45	// chunks of memory (kBigChunk in size.) We first prefetch the chunk, and then copy
	46	// it using non-temporal stores. Hopefully all the reads occur in the prefetch loop,
	47	// so the copy loop reads from L2 and writes directly to memory (with write combining.)
	48	// This minimizes bus turnaround and maintains good DRAM page locality.
	49	// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
	50	// size. Otherwise, it is counter-productive to bypass L2 on the stores.
	51	//
	52	// We are called from the commpage bcopy loops when they encounter very long
	53	// operands, with the standard ABI:
	54	// rdi = dest ptr
	55	// rsi = source ptr
	56	// rdx = length (>= 8kb, probably much bigger)
	57
	58	.text
	59	.code64
	60	.align 5, 0x90
2d21ac55	61	Llongcopy_sse3x_64: // void longcopy(const void dest, void sou, size_t len)
0c530ab8 A	62	pushq %rbp // set up a frame for backtraces
	63	movq %rsp,%rbp
	64	movl %edi,%eax // copy dest ptr
	65	negl %eax
	66	andl $63,%eax // get #bytes to cache line align destination
	67	jz LBigChunkLoop // already aligned
	68
	69	// Cache line align destination, so temporal stores in copy loops work right.
	70	// The recursive call returns with the source and dest ptrs properly updated.
	71
	72	subq %rax,%rdx // get length remaining after dest is aligned
	73	pushq %rdx // save length remaining
	74	movl %eax,%edx // #bytes to copy to align destination
	75	movq $_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
	76	call *%rax
	77	popq %rdx // recover adjusted length
	78
	79	// Loop over big chunks.
	80	// rdx = length remaining (>= 4096)
	81	// rdi = dest (64-byte aligned)
	82	// rsi = source (may be unaligned)
	83
	84	LBigChunkLoop:
	85	movl $(kBigChunk),%r8d // assume we can do a full chunk
	86	cmpq %r8,%rdx // do we have a full chunk left to do?
	87	cmovbl %edx,%r8d // if not, only move what we have left
	88	andl $-4096,%r8d // we work in page multiples
	89	xorl %eax,%eax // initialize chunk offset
	90	jmp LTouchLoop
	91
	92	// Touch in the next chunk. We try to keep the prefetch unit in "kick-start" mode,
	93	// by touching two adjacent cache lines every 8 lines of each page, in four slices.
	94	// Because the source may be unaligned, we use byte loads to touch.
	95	// rdx = length remaining (including this chunk)
	96	// rdi = ptr to start of dest chunk
	97	// rsi = ptr to start of source chunk
	98	// r8d = chunk length (multiples of pages, less than 2**32)
	99	// ecx = scratch reg used to read a byte of each cache line
	100	// eax = chunk offset
	101
	102	.align 4,0x90 // 16-byte align inner loops
	103	LTouchLoop:
	104	movzb (%rsi,%rax),%ecx // touch line 0, 2, 4, or 6 of page
	105	movzb 1*64(%rsi,%rax),%ecx // touch line 1, 3, 5, or 7
	106	movzb 8*64(%rsi,%rax),%ecx // touch line 8, 10, 12, or 14
	107	movzb 9*64(%rsi,%rax),%ecx // etc
	108
	109	movzb 16*64(%rsi,%rax),%ecx
	110	movzb 17*64(%rsi,%rax),%ecx
	111	movzb 24*64(%rsi,%rax),%ecx
	112	movzb 25*64(%rsi,%rax),%ecx
	113
	114	movzb 32*64(%rsi,%rax),%ecx
	115	movzb 33*64(%rsi,%rax),%ecx
	116	movzb 40*64(%rsi,%rax),%ecx
	117	movzb 41*64(%rsi,%rax),%ecx
	118
	119	movzb 48*64(%rsi,%rax),%ecx
	120	movzb 49*64(%rsi,%rax),%ecx
	121	movzb 56*64(%rsi,%rax),%ecx
	122	movzb 57*64(%rsi,%rax),%ecx
	123
	124	subl $-128,%eax // next slice of page (adding 128 w 8-bit immediate)
	125	testl $512,%eax // done with this page?
126	jz LTouchLoop // no, next of four slices
127	addl $(4096-512),%eax // move on to next page
128	cmpl %eax,%r8d // done with this chunk?
129	jnz LTouchLoop // no, do next page
130
131	// The chunk has been pre-fetched, now copy it using non-temporal stores.
132	// There are two copy loops, depending on whether the source is 16-byte aligned
133	// or not.
134
135	movl %r8d,%ecx // copy chunk size to a reg that doesn't use REX prefix
136	addq %rcx,%rsi // increment ptrs by chunk length
137	addq %rcx,%rdi
138	subq %rcx,%rdx // adjust remaining length
139	negq %rcx // prepare loop index (counts up to 0)
140	testl $15,%esi // is source 16-byte aligned?
141	jnz LVeryLongUnaligned // no
142	jmp LVeryLongAligned
143
144	.align 4,0x90 // 16-byte align inner loops
145	LVeryLongAligned: // aligned loop over 128-bytes
146	movdqa (%rsi,%rcx),%xmm0
147	movdqa 16(%rsi,%rcx),%xmm1
148	movdqa 32(%rsi,%rcx),%xmm2
149	movdqa 48(%rsi,%rcx),%xmm3
150	movdqa 64(%rsi,%rcx),%xmm4
151	movdqa 80(%rsi,%rcx),%xmm5
152	movdqa 96(%rsi,%rcx),%xmm6
153	movdqa 112(%rsi,%rcx),%xmm7
154
155	movntdq %xmm0,(%rdi,%rcx)
156	movntdq %xmm1,16(%rdi,%rcx)
157	movntdq %xmm2,32(%rdi,%rcx)
158	movntdq %xmm3,48(%rdi,%rcx)
159	movntdq %xmm4,64(%rdi,%rcx)
160	movntdq %xmm5,80(%rdi,%rcx)
161	movntdq %xmm6,96(%rdi,%rcx)
162	movntdq %xmm7,112(%rdi,%rcx)
163
164	subq $-128,%rcx // add 128 with an 8-bit immediate
165	jnz LVeryLongAligned
166	jmp LVeryLongChunkEnd
167
168	.align 4,0x90 // 16-byte align inner loops
169	LVeryLongUnaligned: // unaligned loop over 128-bytes
170	movdqu (%rsi,%rcx),%xmm0
171	movdqu 16(%rsi,%rcx),%xmm1
172	movdqu 32(%rsi,%rcx),%xmm2
173	movdqu 48(%rsi,%rcx),%xmm3
174	movdqu 64(%rsi,%rcx),%xmm4
175	movdqu 80(%rsi,%rcx),%xmm5
176	movdqu 96(%rsi,%rcx),%xmm6
177	movdqu 112(%rsi,%rcx),%xmm7
178
179	movntdq %xmm0,(%rdi,%rcx)
180	movntdq %xmm1,16(%rdi,%rcx)
181	movntdq %xmm2,32(%rdi,%rcx)
182	movntdq %xmm3,48(%rdi,%rcx)
183	movntdq %xmm4,64(%rdi,%rcx)
184	movntdq %xmm5,80(%rdi,%rcx)
185	movntdq %xmm6,96(%rdi,%rcx)
186	movntdq %xmm7,112(%rdi,%rcx)
187
188	subq $-128,%rcx // add 128 with an 8-bit immediate
189	jnz LVeryLongUnaligned
190
191	LVeryLongChunkEnd:
192	cmpq $4096,%rdx // at least another page to go?
193	jae LBigChunkLoop // yes
194
195	// Done. Call memcpy() again to handle the 0-4095 bytes at the end.
196	// We still have the args in the right registers:
197	// rdi = destination ptr
198	// rsi = source ptr
199	// rdx = length remaining (0..4095)
200
201	sfence // required by non-temporal stores
202	testl %edx,%edx // anything left to copy?
203	jz 1f
204	movq $_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
205	call *%rax
206	1:
207	popq %rbp // restore frame ptr
208	ret
209
210	/* always match for now, as commpage_stuff_routine() will panic if no match */
2d21ac55	211	COMMPAGE_DESCRIPTOR(longcopy_sse3x_64, _COMM_PAGE_LONGCOPY, 0 ,0)