[apple/xnu.git] / osfmk / arm64 / bzero.s

/*
 * Copyright (c) 2012 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 *
 * This file implements the following functions for the arm64 architecture:
 *
 *  void bzero(void *buffer, size_t length);
 *  void __bzero(void *buffer, size_t length);
 *  void *memset(void *buffer, int value, size_t length);
 *
 * The first two zero-fill a buffer.  The third fills the buffer with the low
 * byte of its second argument.
 */

#include "asm.h"

.globl _bzero
.globl ___bzero
.globl _memset
.globl _secure_memset

/*****************************************************************************
 *  bzero entrypoint                                                         *
 *****************************************************************************/

.text
.align 4
_bzero:
___bzero:
    ARM64_STACK_PROLOG
    PUSH_FRAME
    mov     x2,      x1
    eor     x1,      x1, x1
    mov     x3,      x0
    cmp     x2,      #128
    b.cc    L_memsetSmall

/*****************************************************************************
 *  Large buffer zero engine                                                 *
 *****************************************************************************/

L_bzeroLarge:
//  Write the first 64 bytes of the buffer without regard to alignment, then
//  advance x3 to point to a cacheline-aligned location within the buffer, and
//  decrement the length accordingly.
    stp     x1, x1, [x0]
    stp     x1, x1, [x0, #16]
    stp     x1, x1, [x0, #32]
    stp     x1, x1, [x0, #48]
    add     x3,      x0, #64
    and     x3,      x3, #-64
    add     x2,      x2, x0   // end of buffer
    add     x4,      x3, #64  // end of first cacheline to zero
    subs    x2,      x2, x4   // if the end of the buffer comes first, jump
    b.ls    1f                //    directly to the cleanup pass.
0:  dc      zva,     x3       // zero cacheline
    add     x3,      x3, #64  // increment pointer
    subs    x2,      x2, #64  // decrement length
    b.hi    0b
1:  add     x3,      x3, x2   // back up pointer to (end of buffer) - 64.
    stp     x1, x1, [x3]      // and store 64 bytes to reach end of buffer.
    stp     x1, x1, [x3, #16]
    stp     x1, x1, [x3, #32]
    stp     x1, x1, [x3, #48]
    POP_FRAME
    ARM64_STACK_EPILOG

/*****************************************************************************
 *  memset entrypoint                                                        *
 *****************************************************************************/

.align 4
/*
 * It is important that secure_memset remains defined in assembly to avoid
 * compiler optimizations.
 */
_secure_memset:
_memset:
    ARM64_STACK_PROLOG
    PUSH_FRAME
    and     x1,      x1, #0xff
    orr     x3,      xzr,#0x0101010101010101
    mul     x1,      x1, x3
    mov     x3,      x0
    cmp     x2,      #64
    b.cc    L_memsetSmall

/*****************************************************************************
 *  Large buffer store engine                                                *
 *****************************************************************************/

L_memsetLarge:
//  Write the first 64 bytes of the buffer without regard to alignment, then
//  advance x3 to point to an aligned location within the buffer, and
//  decrement the length accordingly.
    stp     x1, x1, [x0]
    add     x3,      x0, #16
    and     x3,      x3, #-16
    add     x2,      x2, x0   // end of buffer
    add     x4,      x3, #64  // end of first aligned 64-byte store
    subs    x2,      x2, x4   // if the end of the buffer comes first, jump
    b.ls    1f                //    directly to the cleanup store.
0:  stnp    x1, x1, [x3]
    stnp    x1, x1, [x3, #16]
    stnp    x1, x1, [x3, #32]
    stnp    x1, x1, [x3, #48]
    add     x3,      x3, #64
    subs    x2,      x2, #64
    b.hi    0b
1:  add     x3,      x3, x2   // back up pointer to (end of buffer) - 64.
    stp     x1, x1, [x3]
    stp     x1, x1, [x3, #16]
    stp     x1, x1, [x3, #32]
    stp     x1, x1, [x3, #48]
    POP_FRAME
    ARM64_STACK_EPILOG

/*****************************************************************************
 *  Small buffer store engine                                                *
 *****************************************************************************/

0:  str     x1,     [x3],#8
L_memsetSmall:
    subs    x2,      x2, #8
    b.cs    0b
    adds    x2,      x2, #8
    b.eq    2f
1:  strb    w1,     [x3],#1
    subs    x2,      x2, #1
    b.ne    1b
2:  POP_FRAME
    ARM64_STACK_EPILOG
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2012 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*
	28	* This file implements the following functions for the arm64 architecture:
	29	*
	30	* void bzero(void *buffer, size_t length);
	31	* void __bzero(void *buffer, size_t length);
	32	* void memset(void buffer, int value, size_t length);
	33	*
	34	* The first two zero-fill a buffer. The third fills the buffer with the low
	35	* byte of its second argument.
	36	*/
	37
	38	#include "asm.h"
	39
	40	.globl _bzero
	41	.globl ___bzero
	42	.globl _memset
	43	.globl _secure_memset
	44
	45	/*****************************************************************************
	46	* bzero entrypoint *
	47	*****************************************************************************/
	48
	49	.text
	50	.align 4
	51	_bzero:
	52	___bzero:
d9a64523	53	ARM64_STACK_PROLOG
5ba3f43e A	54	PUSH_FRAME
	55	mov x2, x1
	56	eor x1, x1, x1
	57	mov x3, x0
	58	cmp x2, #128
	59	b.cc L_memsetSmall
	60
	61	/*****************************************************************************
	62	* Large buffer zero engine *
	63	*****************************************************************************/
	64
	65	L_bzeroLarge:
	66	// Write the first 64 bytes of the buffer without regard to alignment, then
	67	// advance x3 to point to a cacheline-aligned location within the buffer, and
	68	// decrement the length accordingly.
	69	stp x1, x1, [x0]
	70	stp x1, x1, [x0, #16]
	71	stp x1, x1, [x0, #32]
	72	stp x1, x1, [x0, #48]
	73	add x3, x0, #64
	74	and x3, x3, #-64
	75	add x2, x2, x0 // end of buffer
	76	add x4, x3, #64 // end of first cacheline to zero
	77	subs x2, x2, x4 // if the end of the buffer comes first, jump
	78	b.ls 1f // directly to the cleanup pass.
	79	0: dc zva, x3 // zero cacheline
	80	add x3, x3, #64 // increment pointer
	81	subs x2, x2, #64 // decrement length
	82	b.hi 0b
	83	1: add x3, x3, x2 // back up pointer to (end of buffer) - 64.
	84	stp x1, x1, [x3] // and store 64 bytes to reach end of buffer.
	85	stp x1, x1, [x3, #16]
	86	stp x1, x1, [x3, #32]
	87	stp x1, x1, [x3, #48]
	88	POP_FRAME
d9a64523	89	ARM64_STACK_EPILOG
5ba3f43e A	90
	91	/*****************************************************************************
	92	* memset entrypoint *
	93	*****************************************************************************/
	94
	95	.align 4
	96	/*
	97	* It is important that secure_memset remains defined in assembly to avoid
	98	* compiler optimizations.
	99	*/
	100	_secure_memset:
	101	_memset:
d9a64523	102	ARM64_STACK_PROLOG
5ba3f43e A	103	PUSH_FRAME
	104	and x1, x1, #0xff
	105	orr x3, xzr,#0x0101010101010101
	106	mul x1, x1, x3
	107	mov x3, x0
	108	cmp x2, #64
	109	b.cc L_memsetSmall
	110
	111	/*****************************************************************************
	112	* Large buffer store engine *
	113	*****************************************************************************/
	114
	115	L_memsetLarge:
	116	// Write the first 64 bytes of the buffer without regard to alignment, then
	117	// advance x3 to point to an aligned location within the buffer, and
	118	// decrement the length accordingly.
	119	stp x1, x1, [x0]
	120	add x3, x0, #16
	121	and x3, x3, #-16
	122	add x2, x2, x0 // end of buffer
	123	add x4, x3, #64 // end of first aligned 64-byte store
	124	subs x2, x2, x4 // if the end of the buffer comes first, jump
	125	b.ls 1f // directly to the cleanup store.
	126	0: stnp x1, x1, [x3]
	127	stnp x1, x1, [x3, #16]
	128	stnp x1, x1, [x3, #32]
	129	stnp x1, x1, [x3, #48]
	130	add x3, x3, #64
	131	subs x2, x2, #64
	132	b.hi 0b
	133	1: add x3, x3, x2 // back up pointer to (end of buffer) - 64.
	134	stp x1, x1, [x3]
	135	stp x1, x1, [x3, #16]
	136	stp x1, x1, [x3, #32]
	137	stp x1, x1, [x3, #48]
	138	POP_FRAME
d9a64523	139	ARM64_STACK_EPILOG
5ba3f43e A	140
	141	/*****************************************************************************
	142	* Small buffer store engine *
	143	*****************************************************************************/
	144
	145	0: str x1, [x3],#8
	146	L_memsetSmall:
	147	subs x2, x2, #8
	148	b.cs 0b
	149	adds x2, x2, #8
	150	b.eq 2f
	151	1: strb w1, [x3],#1
	152	subs x2, x2, #1
	153	b.ne 1b
	154	2: POP_FRAME
d9a64523 A	155	ARM64_STACK_EPILOG
d9a64523 A	156