-/*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#include <machine/cpu_capabilities.h>
-
-
-/* This file contains the following functions:
- *
- * void *memset(void *b, int c, size_t len);
- * void memset_pattern4(void *b, const void *c4, size_t len);
- * void memset_pattern8(void *b, const void *c8, size_t len);
- * void memset_pattern16(void *b, const void *c16, size_t len);
- *
- * Calls of memset() with c==0 are routed to the bzero() routine. Most of the
- * others go to _COMM_PAGE_MEMSET_PATTERN, which is entered as follows:
- * %edi = ptr to memory to set (aligned)
- * %edx = length (which can be short, though we bias in favor of long operands)
- * %xmm0 = the pattern to store
- * Return conditions:
- * %eax, %edi, %esi, %ecx, and %edx all trashed
- *
- * NB: we avoid "stos" family of instructions (stosl, stosb), as they are very slow
- * on P4s and probably other processors.
- */
-
- #define kShort 255 // for nonzero memset(), too short for commpage
-
-
- .text
- .globl _memset
- .align 2
-_memset: // void *memset(void *b, int c, size_t len);
- movl 8(%esp),%eax // get 1-byte pattern
- movl 12(%esp),%edx // get length
- andl $0xFF,%eax // (c==0) ?
- jnz LNonzero // not a bzero
-
- movl $(_COMM_PAGE_BZERO),%eax// map memset(p,0,n) into bzero(p,n)
- movl %edx,8(%esp) // put count where bzero() expects it
- jmp %eax // enter commpage
-
-
- // Handle memset of a nonzero value.
-
-LNonzero:
- pushl %edi // save a few nonvolatiles
- pushl %esi
- movl %eax,%esi // replicate byte in %al into all four bytes
- movl 12(%esp),%edi // point to operand
- shll $8,%esi
- orl %esi,%eax
- movl %eax,%esi
- shll $16,%esi
- orl %esi,%eax // now %eax has "c" in all 4 bytes
- cmpl $(kShort),%edx // is operand too short for SSE?
- ja LCallCommpage // no
-
-// Nonzero memset() too short to call commpage.
-// %eax = replicated 4-byte pattern
-// %edi = ptr
-// %edx = length (<= kShort)
-
- cmpl $16,%edx // long enough to word align?
- jge 3f // yes
- test %edx,%edx // length==0?
- jz 6f
-1:
- movb %al,(%edi) // pack in a byte
- inc %edi
- dec %edx
- jnz 1b
- jmp 6f
-2:
- movb %al,(%edi) // pack in a byte
- inc %edi
- dec %edx
-3:
- test $3,%edi // is ptr doubleword aligned?
- jnz 2b // no
- movl %edx,%ecx // copy length
- shrl $2,%edx // #doublewords to store
-4:
- movl %eax,(%edi) // store aligned doubleword
- addl $4,%edi
- dec %edx
- jnz 4b
- andl $3,%ecx // any leftover bytes?
- jz 6f // no
-5:
- movb %al,(%edi) // pack in a byte
- inc %edi
- dec %ecx
- jnz 5b
-6:
- movl 12(%esp),%eax // get return value (ie, original ptr)
- popl %esi
- popl %edi
- ret
-
-// Nonzero memset() is long enough to call commpage.
-// %eax = replicated 4-byte pattern
-// %edi = ptr
-// %edx = length (> kShort)
-
-LCallCommpage:
- movd %eax,%xmm0 // move %eax to low 4 bytes of %xmm0
- pshufd $(0x00),%xmm0,%xmm0 // replicate across the vector
- movl %edi,%ecx // copy dest ptr
- negl %ecx
- andl $15,%ecx // get #bytes to align ptr
- jz 2f // skip if already aligned
- subl %ecx,%edx // decrement length
-1:
- movb %al,(%edi) // pack in a byte
- inc %edi
- dec %ecx
- jnz 1b
-2: // ptr aligned, length long enough to justify
- movl $(_COMM_PAGE_MEMSET_PATTERN),%eax
- call %eax // call commpage to do the heavy lifting
- movl 12(%esp),%eax // get return value (ie, original ptr)
- popl %esi
- popl %edi
- ret
-
-
-// Handle memset of a 16-byte pattern.
-
- .globl _memset_pattern16
- .align 2, 0x90
-_memset_pattern16: // void memset_pattern16(void *b, const void *c16, size_t len);
- pushl %edi
- pushl %esi
- movl 20(%esp),%edx // get length
- movl 16(%esp),%esi // get ptr to 16-byte pattern
- movl 12(%esp),%edi // point to operand
- movdqu (%esi),%xmm0 // load the pattern
- jmp LAlignPtr
-
-
-// Handle memset of an 8-byte pattern.
-
- .globl _memset_pattern8
- .align 2, 0x90
-_memset_pattern8: // void memset_pattern8(void *b, const void *c8, size_t len);
- pushl %edi
- pushl %esi
- movl 20(%esp),%edx // get length
- movl 16(%esp),%esi // get ptr to 8-byte pattern
- movl 12(%esp),%edi // point to operand
- movq (%esi),%xmm0 // load pattern into low 8 bytes
- punpcklqdq %xmm0,%xmm0 // replicate into all 16
- jmp LAlignPtr
-
-// Handle memset of a 4-byte pattern.
-
- .globl _memset_pattern4
- .align 2, 0x90
-_memset_pattern4: // void memset_pattern4(void *b, const void *c4, size_t len);
- pushl %edi
- pushl %esi
- movl 20(%esp),%edx // get length
- movl 16(%esp),%esi // get ptr to 4-byte pattern
- movl 12(%esp),%edi // point to operand
- movd (%esi),%xmm0 // load pattern into low 4 bytes
- pshufd $(0x00),%xmm0,%xmm0 // replicate the 4 bytes across the vector
-
-
-// Align ptr if necessary. We must rotate the pattern right for each byte we
-// store while aligning the ptr. Since there is no rotate instruction in SSE3,
-// we have to synthesize the rotates.
-// %edi = ptr
-// %edx = length
-// %xmm0 = pattern
-
-LAlignPtr: // NB: can drop down to here!
- cmpl $100,%edx // long enough to bother aligning ptr?
- movl %edi,%ecx // copy ptr
- jb LReady // not long enough
- negl %ecx
- andl $15,%ecx // get #bytes to align ptr
- jz LReady // already aligned
- subl %ecx,%edx // adjust length
-
- test $1,%cl // 1-byte store required?
- movd %xmm0,%eax // get 4 low bytes in %eax
- jz 2f // no
- movdqa %xmm0,%xmm1 // copy pattern so we can shift in both directions
- movb %al,(%edi) // pack in the low-order byte
- psrldq $1,%xmm0 // shift pattern right 1 byte
- inc %edi
- pslldq $15,%xmm1 // shift pattern left 15 bytes
- shrl $8,%eax // in case 2-byte store is required
- por %xmm1,%xmm0 // complete right rotate of pattern by 1 byte
-2:
- test $2,%cl // 2-byte store required?
- jz 4f // no
- psrldq $2,%xmm0 // shift pattern down 2 bytes
- movw %ax,(%edi) // pack in next two bytes
- pinsrw $7,%eax,%xmm0 // insert low word of %eax into high word of %xmm0
- addl $2,%edi // adjust ptr
-4:
- test $4,%cl // 4-byte store required?
- jz 8f // no
- movd %xmm0,(%edi) // store low 4 bytes of %xmm0
- pshufd $(0x39),%xmm0,%xmm0 // rotate %xmm0 right 4 bytes (mask == 00 11 10 01)
- addl $4,%edi // adjust ptr
-8:
- test $8,%cl // 8-byte store required?
- jz LReady // no
- movq %xmm0,(%edi) // store low 8 bytes of %xmm0
- pshufd $(0x4e),%xmm0,%xmm0 // rotate %xmm0 right 8 bytes (mask == 01 00 11 10)
- addl $8,%edi // adjust ptr
-
-// Ptr is aligned if practical, we're ready to call commpage to do the heavy lifting.
-
-LReady:
- movl $(_COMM_PAGE_MEMSET_PATTERN),%eax
- call %eax // call commpage to do the heavy lifting
- popl %esi
- popl %edi
- ret
projects / apple / libc.git / commitdiff
