osfmk/arm64/bzero.s

   1 /*
   2  * Copyright (c) 2012 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  *
  28  * This file implements the following functions for the arm64 architecture:
  29  *
  30  *  void bzero(void *buffer, size_t length);
  31  *  void __bzero(void *buffer, size_t length);
  32  *  void *memset(void *buffer, int value, size_t length);
  33  *
  34  * The first two zero-fill a buffer.  The third fills the buffer with the low
  35  * byte of its second argument.
  36  */
  37
  38 #include "asm.h"
  39
  40 .globl _bzero
  41 .globl ___bzero
  42 .globl _memset
  43 .globl _secure_memset
  44
  45 /*****************************************************************************
  46  *  bzero entrypoint                                                         *
  47  *****************************************************************************/
  48
  49 .text
  50 .align 4
  51 _bzero:
  52 ___bzero:
  53     PUSH_FRAME
  54     mov     x2,      x1
  55     eor     x1,      x1, x1
  56     mov     x3,      x0
  57     cmp     x2,      #128
  58     b.cc    L_memsetSmall
  59
  60 /*****************************************************************************
  61  *  Large buffer zero engine                                                 *
  62  *****************************************************************************/
  63
  64 L_bzeroLarge:
  65 //  Write the first 64 bytes of the buffer without regard to alignment, then
  66 //  advance x3 to point to a cacheline-aligned location within the buffer, and
  67 //  decrement the length accordingly.
  68     stp     x1, x1, [x0]
  69     stp     x1, x1, [x0, #16]
  70     stp     x1, x1, [x0, #32]
  71     stp     x1, x1, [x0, #48]
  72     add     x3,      x0, #64
  73     and     x3,      x3, #-64
  74     add     x2,      x2, x0   // end of buffer
  75     add     x4,      x3, #64  // end of first cacheline to zero
  76     subs    x2,      x2, x4   // if the end of the buffer comes first, jump
  77     b.ls    1f                //    directly to the cleanup pass.
  78 0:  dc      zva,     x3       // zero cacheline
  79     add     x3,      x3, #64  // increment pointer
  80     subs    x2,      x2, #64  // decrement length
  81     b.hi    0b
  82 1:  add     x3,      x3, x2   // back up pointer to (end of buffer) - 64.
  83     stp     x1, x1, [x3]      // and store 64 bytes to reach end of buffer.
  84     stp     x1, x1, [x3, #16]
  85     stp     x1, x1, [x3, #32]
  86     stp     x1, x1, [x3, #48]
  87     POP_FRAME
  88         ret
  89
  90 /*****************************************************************************
  91  *  memset entrypoint                                                        *
  92  *****************************************************************************/
  93
  94 .align 4
  95 /*
  96  * It is important that secure_memset remains defined in assembly to avoid
  97  * compiler optimizations.
  98  */
  99 _secure_memset:
 100 _memset:
 101     PUSH_FRAME
 102     and     x1,      x1, #0xff
 103     orr     x3,      xzr,#0x0101010101010101
 104     mul     x1,      x1, x3
 105     mov     x3,      x0
 106     cmp     x2,      #64
 107     b.cc    L_memsetSmall
 108
 109 /*****************************************************************************
 110  *  Large buffer store engine                                                *
 111  *****************************************************************************/
 112
 113 L_memsetLarge:
 114 //  Write the first 64 bytes of the buffer without regard to alignment, then
 115 //  advance x3 to point to an aligned location within the buffer, and
 116 //  decrement the length accordingly.
 117     stp     x1, x1, [x0]
 118     add     x3,      x0, #16
 119     and     x3,      x3, #-16
 120     add     x2,      x2, x0   // end of buffer
 121     add     x4,      x3, #64  // end of first aligned 64-byte store
 122     subs    x2,      x2, x4   // if the end of the buffer comes first, jump
 123     b.ls    1f                //    directly to the cleanup store.
 124 0:  stnp    x1, x1, [x3]
 125     stnp    x1, x1, [x3, #16]
 126     stnp    x1, x1, [x3, #32]
 127     stnp    x1, x1, [x3, #48]
 128     add     x3,      x3, #64
 129     subs    x2,      x2, #64
 130     b.hi    0b
 131 1:  add     x3,      x3, x2   // back up pointer to (end of buffer) - 64.
 132     stp     x1, x1, [x3]
 133     stp     x1, x1, [x3, #16]
 134     stp     x1, x1, [x3, #32]
 135     stp     x1, x1, [x3, #48]
 136     POP_FRAME
 137         ret
 138
 139 /*****************************************************************************
 140  *  Small buffer store engine                                                *
 141  *****************************************************************************/
 142
 143 0:  str     x1,     [x3],#8
 144 L_memsetSmall:
 145     subs    x2,      x2, #8
 146     b.cs    0b
 147     adds    x2,      x2, #8
 148     b.eq    2f
 149 1:  strb    w1,     [x3],#1
 150     subs    x2,      x2, #1
 151     b.ne    1b
 152 2:  POP_FRAME
 153         ret