Libc-825.26.tar.gz

[apple/libc.git] / x86_64 / string / bcopy_sse3x.s

/*
 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <machine/cpu_capabilities.h>
#include "platfunc.h"

/*
 * The bcopy/memcpy loops, tuned for 64-bit Pentium-M class processors with 
 * Supplemental SSE3 and 64-byte cache lines.  This is the 64-bit version.
 *
 * The following #defines are tightly coupled to the u-architecture:
 */

#define kShort  80			// too short to bother with SSE (must be >=80)
#define kVeryLong   (500*1024)          // large enough for non-temporal stores (>=8192 and <2GB)
#define kFastUCode  ((16*1024)-15)	// cutoff for microcode fastpath for "rep/movsl"

// void bcopy(const void *src, void *dst, size_t len);

PLATFUNC_FUNCTION_START_GENERIC(bcopy, sse3x, 64, 5)
LZero:
	pushq	%rbp			// set up a frame for backtraces
	movq	%rsp,%rbp
	movq	%rsi,%rax		// copy dest ptr
	movq	%rdi,%rsi		// xchange source and dest ptrs
	movq	%rax,%rdi
	subq    %rsi,%rax               // (dest - source)
	cmpq    %rdx,%rax               // must move in reverse if (dest - source) < length
	jb      LReverseIsland
	cmpq    $(kShort),%rdx          // long enough to bother with SSE?
	jbe     LShort			// no
	jmp	LNotShort

//
// void *memcpy(void *dst, const void *src, size_t len);
// void *memmove(void *dst, const void *src, size_t len);
//

PLATFUNC_FUNCTION_START_GENERIC(memcpy, sse3x, 64, 0)		// void *memcpy(void *dst, const void *src, size_t len)
PLATFUNC_FUNCTION_START_GENERIC(memmove, sse3x, 64, 0)	// void *memmove(void *dst, const void *src, size_t len)
	pushq	%rbp			// set up a frame for backtraces
	movq	%rsp,%rbp
	movq	%rdi,%r11		// save return value here        
	movq    %rdi,%rax
	subq    %rsi,%rax               // (dest - source)
	cmpq    %rdx,%rax               // must move in reverse if (dest - source) < length
	jb      LReverseIsland
	cmpq    $(kShort),%rdx          // long enough to bother with SSE?
	ja      LNotShort               // yes

// Handle short forward copies.  As the most common case, this is the fall-through path.
//      rdx = length (<= kShort)
//      rsi = source ptr
//      rdi = dest ptr

LShort:
	movl    %edx,%ecx		// copy length using 32-bit operation
	shrl	$2,%ecx			// get #doublewords
	jz	LLeftovers
2:					// loop copying doublewords
	movl	(%rsi),%eax
	addq	$4,%rsi
	movl	%eax,(%rdi)
	addq	$4,%rdi
	decl	%ecx
	jnz	2b
LLeftovers:				// handle leftover bytes (0..3) in last word
	andl	$3,%edx			// any leftover bytes?
	jz	5f
4:					// loop copying bytes
	movb	(%rsi),%al
	incq	%rsi
	movb	%al,(%rdi)
	incq	%rdi
	decl	%edx
	jnz	4b
5:
	movq	%r11,%rax		// get return value (dst ptr) for memcpy/memmove
	popq	%rbp
	ret


LReverseIsland:				// keep the "jb" above a short branch...
	jmp	LReverse		// ...because reverse moves are uncommon


// Handle forward moves that are long enough to justify use of SSE.
// First, 16-byte align the destination.
//      rdx = length (> kShort)
//      rsi = source ptr
//      rdi = dest ptr

LNotShort:
	cmpq    $(kVeryLong),%rdx       // long enough to justify heavyweight loops?
	jae     LVeryLong		// use very-long-operand path
	movl    %edi,%ecx               // copy low half of destination ptr
	negl    %ecx
	andl    $15,%ecx                // get #bytes to align destination
	jz	LDestAligned		// already aligned
	subl    %ecx,%edx               // decrement length
	rep				// align destination
	movsb


// Destination is now aligned.  Dispatch to one of sixteen loops over 64-byte chunks,
// based on the alignment of the source.  All vector loads and stores are aligned.
// Even though this means we have to shift and repack vectors, doing so is much faster
// than unaligned loads.  Since kShort>=80 and we've moved at most 15 bytes already,
// there is at least one chunk.  When we enter the copy loops, the following registers
// are set up:
//      rdx = residual length (0..63)
//	rcx = -(length to move), a multiple of 64 less than 2GB
//      rsi = ptr to 1st source byte not to move (unaligned)
//      rdi = ptr to 1st dest byte not to move (aligned)

LDestAligned:
	movq    %rdx,%rcx               // copy length
	movl	%esi,%eax		// copy low half of source address
	andl    $63,%edx                // get remaining bytes for LShort
	andl	$15,%eax		// mask to low 4 bits of source address
	andq    $-64,%rcx               // get number of bytes we will copy in inner loop
	leaq	LTable(%rip), %r8
	addq    %rcx,%rsi               // point to 1st byte not copied
	addq    %rcx,%rdi
	movl	(%r8,%rax,4),%eax	// get offset of routine
	negq    %rcx                    // now generate offset to 1st byte to be copied
	addq	%r8,%rax		// generate address of copy loop
	jmp	*%rax			// enter copy loop, selected by source alignment

	.align	2
LTable:					// table of copy loop addresses
// force generation of assembly-time constants. Otherwise assembler
// creates subtractor relocations relative to first external symbol,
// and this file has none
	.set LMod0Offset, LMod0 - LTable
	.set LMod1Offset, LMod1 - LTable
	.set LMod2Offset, LMod2 - LTable
	.set LMod3Offset, LMod3 - LTable
	.set LMod4Offset, LMod4 - LTable
	.set LMod5Offset, LMod5 - LTable
	.set LMod6Offset, LMod6 - LTable
	.set LMod7Offset, LMod7 - LTable
	.set LMod8Offset, LMod8 - LTable
	.set LMod9Offset, LMod9 - LTable
	.set LMod10Offset, LMod10 - LTable
	.set LMod11Offset, LMod11 - LTable
	.set LMod12Offset, LMod12 - LTable
	.set LMod13Offset, LMod13 - LTable
	.set LMod14Offset, LMod14 - LTable
	.set LMod15Offset, LMod15 - LTable
	.long LMod0Offset
	.long LMod1Offset
	.long LMod2Offset
	.long LMod3Offset
	.long LMod4Offset
	.long LMod5Offset
	.long LMod6Offset
	.long LMod7Offset
	.long LMod8Offset
	.long LMod9Offset
	.long LMod10Offset
	.long LMod11Offset
	.long LMod12Offset
	.long LMod13Offset
	.long LMod14Offset
	.long LMod15Offset


// Very long forward moves.  These are at least several pages.  They are special cased
// and aggressively optimized, not so much because they are common or useful, but
// because they are subject to benchmark.  There isn't enough room for them in the
// area reserved on the platfunc for bcopy, so we put them elsewhere.  We call
// the longcopy routine using the normal ABI:
//      rdi = dest
//      rsi = source
//      rdx = length (>= kVeryLong bytes)

LVeryLong:
	pushq	%r11			// save return value
	call	_longcopy		// call very long operand routine
	popq	%rax			// pop return value
	popq	%rbp
	ret	


// On Pentium-M, the microcode for "rep/movsl" is faster than SSE for 16-byte
// aligned operands from about 32KB up to kVeryLong for the hot cache case, and from
// about 256 bytes up to kVeryLong for cold caches.  This is because the microcode
// avoids having to read destination cache lines that will be completely overwritten.
// The cutoff we use (ie, kFastUCode) must somehow balance the two cases, since
// we do not know if the destination is in cache or not.

Lfastpath:
	addq    %rcx,%rsi               // restore ptrs to 1st byte of source and dest
	addq    %rcx,%rdi
	negl	%ecx			// make length positive (known to be < 2GB)
	orl	%edx,%ecx		// restore total #bytes remaining to move
	cld				// we'll move forward
	shrl	$2,%ecx			// compute #words to move
	rep				// the u-code will optimize this
	movsl
	jmp	LLeftovers		// handle 0..3 leftover bytes


// Forward loop for medium length operands in which low four bits of %rsi == 0000

LMod0:
	cmpl	$(-kFastUCode),%ecx	// %rcx == -length, where (length < kVeryLong)
	jle	Lfastpath		// long enough for fastpath in microcode
	jmp	1f
	.align	4,0x90			// 16-byte align inner loops
1:					// loop over 64-byte chunks
	movdqa  (%rsi,%rcx),%xmm0
	movdqa  16(%rsi,%rcx),%xmm1
	movdqa  32(%rsi,%rcx),%xmm2
	movdqa  48(%rsi,%rcx),%xmm3

	movdqa  %xmm0,(%rdi,%rcx)
	movdqa  %xmm1,16(%rdi,%rcx)
	movdqa  %xmm2,32(%rdi,%rcx)
	movdqa  %xmm3,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0001

LMod1:
	movdqa	-1(%rsi,%rcx),%xmm0	// prime the loop by loading 1st quadword
1:					// loop over 64-byte chunks
	movdqa  15(%rsi,%rcx),%xmm1
	movdqa  31(%rsi,%rcx),%xmm2
	movdqa  47(%rsi,%rcx),%xmm3
	movdqa  63(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$1,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$1,%xmm2,%xmm3
	palignr	$1,%xmm1,%xmm2
	palignr	$1,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0010

LMod2:
	movdqa	-2(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  14(%rsi,%rcx),%xmm1
	movdqa  30(%rsi,%rcx),%xmm2
	movdqa  46(%rsi,%rcx),%xmm3
	movdqa  62(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$2,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$2,%xmm2,%xmm3
	palignr	$2,%xmm1,%xmm2
	palignr	$2,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0011

LMod3:
	movdqa	-3(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  13(%rsi,%rcx),%xmm1
	movdqa  29(%rsi,%rcx),%xmm2
	movdqa  45(%rsi,%rcx),%xmm3
	movdqa  61(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$3,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$3,%xmm2,%xmm3
	palignr	$3,%xmm1,%xmm2
	palignr	$3,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0100
// We use the float single data type in order to use "movss" to merge vectors.

LMod4:
	movaps	-4(%rsi,%rcx),%xmm0	// 4-byte aligned: prime the loop
	jmp	1f
	.align	4,0x90
1:					// loop over 64-byte chunks
	movaps  12(%rsi,%rcx),%xmm1
	movaps  28(%rsi,%rcx),%xmm2
	movss	%xmm1,%xmm0		// copy low 4 bytes of source into destination
	pshufd	$(0x39),%xmm0,%xmm0	// rotate right 4 bytes (mask -- 00 11 10 01)
	movaps  44(%rsi,%rcx),%xmm3
	movss	%xmm2,%xmm1
	pshufd	$(0x39),%xmm1,%xmm1
	movaps	60(%rsi,%rcx),%xmm4
	movss	%xmm3,%xmm2
	pshufd	$(0x39),%xmm2,%xmm2

	movaps  %xmm0,(%rdi,%rcx)
	movss	%xmm4,%xmm3
	pshufd	$(0x39),%xmm3,%xmm3
	movaps  %xmm1,16(%rdi,%rcx)
	movaps  %xmm2,32(%rdi,%rcx)
	movaps	%xmm4,%xmm0
	movaps  %xmm3,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0101

LMod5:
	movdqa	-5(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  11(%rsi,%rcx),%xmm1
	movdqa  27(%rsi,%rcx),%xmm2
	movdqa  43(%rsi,%rcx),%xmm3
	movdqa  59(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$5,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$5,%xmm2,%xmm3
	palignr	$5,%xmm1,%xmm2
	palignr	$5,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0110

LMod6:
	movdqa	-6(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  10(%rsi,%rcx),%xmm1
	movdqa  26(%rsi,%rcx),%xmm2
	movdqa  42(%rsi,%rcx),%xmm3
	movdqa  58(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$6,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$6,%xmm2,%xmm3
	palignr	$6,%xmm1,%xmm2
	palignr	$6,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 0111

LMod7:
	movdqa	-7(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  9(%rsi,%rcx),%xmm1
	movdqa  25(%rsi,%rcx),%xmm2
	movdqa  41(%rsi,%rcx),%xmm3
	movdqa  57(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$7,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$7,%xmm2,%xmm3
	palignr	$7,%xmm1,%xmm2
	palignr	$7,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1000
// We use the float double data type in order to use "shufpd" to shift by 8 bytes.

LMod8:
	cmpl	$(-kFastUCode),%ecx	// %rcx == -length, where (length < kVeryLong)
	jle	Lfastpath		// long enough for fastpath in microcode
	movapd	-8(%rsi,%rcx),%xmm0	// 8-byte aligned: prime the loop
	jmp	1f
	.align	4,0x90
1:					// loop over 64-byte chunks
	movapd  8(%rsi,%rcx),%xmm1
	movapd  24(%rsi,%rcx),%xmm2
	shufpd	$01,%xmm1,%xmm0		// %xmm0 <- shr( %xmm0 || %xmm1, 8 bytes)
	movapd  40(%rsi,%rcx),%xmm3
	shufpd	$01,%xmm2,%xmm1
	movapd	56(%rsi,%rcx),%xmm4
	shufpd	$01,%xmm3,%xmm2

	movapd  %xmm0,(%rdi,%rcx)
	shufpd	$01,%xmm4,%xmm3
	movapd  %xmm1,16(%rdi,%rcx)
	movapd  %xmm2,32(%rdi,%rcx)
	movapd	%xmm4,%xmm0
	movapd  %xmm3,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1001

LMod9:
	movdqa	-9(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  7(%rsi,%rcx),%xmm1
	movdqa  23(%rsi,%rcx),%xmm2
	movdqa  39(%rsi,%rcx),%xmm3
	movdqa  55(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$9,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$9,%xmm2,%xmm3
	palignr	$9,%xmm1,%xmm2
	palignr	$9,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1010

LMod10:
	movdqa	-10(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  6(%rsi,%rcx),%xmm1
	movdqa  22(%rsi,%rcx),%xmm2
	movdqa  38(%rsi,%rcx),%xmm3
	movdqa  54(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$10,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$10,%xmm2,%xmm3
	palignr	$10,%xmm1,%xmm2
	palignr	$10,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1011

LMod11:
	movdqa	-11(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  5(%rsi,%rcx),%xmm1
	movdqa  21(%rsi,%rcx),%xmm2
	movdqa  37(%rsi,%rcx),%xmm3
	movdqa  53(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$11,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$11,%xmm2,%xmm3
	palignr	$11,%xmm1,%xmm2
	palignr	$11,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1100
// We use the float single data type in order to use "movss" to merge vectors.

LMod12:
	movss	(%rsi,%rcx),%xmm0	// prefetch 1st four bytes of source, right justified
	jmp	1f
	.align	4,0x90
1:					// loop over 64-byte chunks
	pshufd	$(0x93),4(%rsi,%rcx),%xmm1 // load and rotate right 12 bytes (mask -- 10 01 00 11)
	pshufd	$(0x93),20(%rsi,%rcx),%xmm2
	pshufd	$(0x93),36(%rsi,%rcx),%xmm3
	pshufd	$(0x93),52(%rsi,%rcx),%xmm4

	movaps	%xmm4,%xmm5
	movss	%xmm3,%xmm4		// copy low 4 bytes of source into destination
	movss	%xmm2,%xmm3
	movss	%xmm1,%xmm2
	movss	%xmm0,%xmm1

	movaps  %xmm1,(%rdi,%rcx)
	movaps  %xmm2,16(%rdi,%rcx)
	movaps	%xmm5,%xmm0
	movaps  %xmm3,32(%rdi,%rcx)
	movaps  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1101

LMod13:
	movdqa	-13(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  3(%rsi,%rcx),%xmm1
	movdqa  19(%rsi,%rcx),%xmm2
	movdqa  35(%rsi,%rcx),%xmm3
	movdqa  51(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$13,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$13,%xmm2,%xmm3
	palignr	$13,%xmm1,%xmm2
	palignr	$13,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1110

LMod14:
	movdqa	-14(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  2(%rsi,%rcx),%xmm1
	movdqa  18(%rsi,%rcx),%xmm2
	movdqa  34(%rsi,%rcx),%xmm3
	movdqa  50(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$14,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$14,%xmm2,%xmm3
	palignr	$14,%xmm1,%xmm2
	palignr	$14,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Forward loop for medium length operands in which low four bits of %rsi == 1111

LMod15:
	movdqa	-15(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
1:					// loop over 64-byte chunks
	movdqa  1(%rsi,%rcx),%xmm1
	movdqa  17(%rsi,%rcx),%xmm2
	movdqa  33(%rsi,%rcx),%xmm3
	movdqa  49(%rsi,%rcx),%xmm4

	movdqa	%xmm0,%xmm5
	movdqa	%xmm4,%xmm0

	palignr	$15,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
	palignr	$15,%xmm2,%xmm3
	palignr	$15,%xmm1,%xmm2
	palignr	$15,%xmm5,%xmm1

	movdqa  %xmm1,(%rdi,%rcx)
	movdqa  %xmm2,16(%rdi,%rcx)
	movdqa  %xmm3,32(%rdi,%rcx)
	movdqa  %xmm4,48(%rdi,%rcx)

	addq    $64,%rcx
	jnz     1b

	jmp     LShort                  // copy remaining 0..63 bytes and done


// Reverse moves.  These are not optimized as aggressively as their forward
// counterparts, as they are only used with destructive overlap.
//      rdx = length
//      rsi = source ptr
//      rdi = dest ptr

LReverse:
	addq    %rdx,%rsi               // point to end of strings
	addq    %rdx,%rdi
	cmpq    $(kShort),%rdx          // long enough to bother with SSE?
	ja      LReverseNotShort        // yes

// Handle reverse short copies.
//      edx = length (<= kShort)
//      rsi = one byte past end of source
//      rdi = one byte past end of dest

LReverseShort:
	movl    %edx,%ecx		// copy length
	shrl	$3,%ecx			// #quadwords
	jz	3f
1:
	subq	$8,%rsi
	movq	(%rsi),%rax
	subq	$8,%rdi
	movq	%rax,(%rdi)
	decl	%ecx
	jnz	1b
3:
	andl	$7,%edx			// bytes?
	jz	5f
4:
	decq	%rsi
	movb	(%rsi),%al
	decq	%rdi
	movb	%al,(%rdi)
	decl	%edx
	jnz	4b
5:
	movq	%r11,%rax		// get return value (dst ptr) for memcpy/memmove
	popq	%rbp
	ret

// Handle a reverse move long enough to justify using SSE.
//      rdx = length (> kShort)
//      rsi = one byte past end of source
//      rdi = one byte past end of dest

LReverseNotShort:
	movl    %edi,%ecx               // copy destination
	andl    $15,%ecx                // get #bytes to align destination
	je      LReverseDestAligned     // already aligned
	subq	%rcx,%rdx		// adjust length
1:					// loop copying 1..15 bytes
	decq	%rsi
	movb	(%rsi),%al
	decq	%rdi
	movb	%al,(%rdi)
	decl	%ecx
	jnz	1b

// Destination is now aligned.  Prepare for reverse loops.

LReverseDestAligned:
	movq    %rdx,%rcx               // copy length
	andl    $63,%edx                // get remaining bytes for LReverseShort
	andq    $-64,%rcx               // get number of bytes we will copy in inner loop
	subq    %rcx,%rsi               // point to endpoint of copy
	subq    %rcx,%rdi
	testl	$15,%esi		// is source aligned too?
	jnz     LReverseUnalignedLoop   // no

LReverseAlignedLoop:                    // loop over 64-byte chunks
	movdqa  -16(%rsi,%rcx),%xmm0
	movdqa  -32(%rsi,%rcx),%xmm1
	movdqa  -48(%rsi,%rcx),%xmm2
	movdqa  -64(%rsi,%rcx),%xmm3

	movdqa  %xmm0,-16(%rdi,%rcx)
	movdqa  %xmm1,-32(%rdi,%rcx)
	movdqa  %xmm2,-48(%rdi,%rcx)
	movdqa  %xmm3,-64(%rdi,%rcx)

	subq    $64,%rcx
	jne     LReverseAlignedLoop

	jmp     LReverseShort           // copy remaining 0..63 bytes and done


// Reverse, unaligned loop.  LDDQU==MOVDQU on these machines.

LReverseUnalignedLoop:                  // loop over 64-byte chunks
	movdqu  -16(%rsi,%rcx),%xmm0
	movdqu  -32(%rsi,%rcx),%xmm1
	movdqu  -48(%rsi,%rcx),%xmm2
	movdqu  -64(%rsi,%rcx),%xmm3

	movdqa  %xmm0,-16(%rdi,%rcx)
	movdqa  %xmm1,-32(%rdi,%rcx)
	movdqa  %xmm2,-48(%rdi,%rcx)
	movdqa  %xmm3,-64(%rdi,%rcx)

	subq    $64,%rcx
	jne     LReverseUnalignedLoop

	jmp     LReverseShort           // copy remaining 0..63 bytes and done

PLATFUNC_DESCRIPTOR(bcopy,sse3x,kHasSSE2|kHasSupplementalSSE3|kCache64,kHasSSE4_2)
PLATFUNC_DESCRIPTOR(memcpy,sse3x,kHasSSE2|kHasSupplementalSSE3|kCache64,kHasSSE4_2)
PLATFUNC_DESCRIPTOR(memmove,sse3x,kHasSSE2|kHasSupplementalSSE3|kCache64,kHasSSE4_2)