osfmk/ppc/bzero.s

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25  /*
  26  * @OSF_FREE_COPYRIGHT@
  27  */
  28
  29 #include <ppc/asm.h>
  30 #include <ppc/proc_reg.h>       /* For CACHE_LINE_SIZE */
  31
  32 /*
  33  *       void   bzero(char *addr, unsigned int length)
  34  *
  35  * bzero implementation for PowerPC
  36  *   - assumes cacheable memory (i.e. uses DCBZ)
  37  *   - assumes non-pic code
  38  *
  39  * returns start address in r3, as per memset (called by memset)
  40  */
  41
  42 ENTRY(bzero, TAG_NO_FRAME_USED)
  43
  44         cmpwi   cr0,    r4,     0 /* no bytes to zero? */
  45         mr      r7,     r3
  46         mr      r8,     r3      /* use r8 as counter to where we are */
  47         beqlr-
  48         cmpwi   cr0,    r4,     CACHE_LINE_SIZE /* clear less than a block? */
  49         li      r0,     0        /* use r0 as source of zeros */
  50         blt     .L_bzeroEndWord
  51
  52 /* first, clear bytes up to the next word boundary */
  53         addis   r6,     0,      HIGH_CADDR(.L_bzeroBeginWord)
  54         addi    r6,     r6,     LOW_ADDR(.L_bzeroBeginWord)
  55                  /* extract byte offset as word offset */
  56         rlwinm. r5,     r8,     2,      28,     29
  57         addi    r8,     r8,     -1 /* adjust for update */
  58         beq     .L_bzeroBeginWord /* no bytes to zero */
  59         subfic  r5,     r5,     16 /* compute the number of instructions */
  60         sub     r6,     r6,     r5 /* back from word clear to execute */
  61         mtctr   r6
  62         bctr
  63
  64         stbu    r0,     1(r8)
  65         stbu    r0,     1(r8)
  66         stbu    r0,     1(r8)
  67
  68 /* clear words up to the next block boundary */
  69 .L_bzeroBeginWord:
  70         addis   r6,     0,      HIGH_CADDR(.L_bzeroBlock)
  71         addi    r6,     r6,     LOW_ADDR(.L_bzeroBlock)
  72         addi    r8,     r8,     1
  73         rlwinm. r5,     r8,     0,      27,     29 /* extract word offset */
  74         addi    r8,     r8,     -4              /* adjust for update */
  75         beq     .L_bzeroBlock                   /* no words to zero */
  76                 /* compute the number of instructions */
  77         subfic  r5,     r5,     CACHE_LINE_SIZE
  78         sub     r6,     r6,     r5 /* back from word clear to execute */
  79         mtctr   r6
  80         bctr
  81
  82         stwu    r0,     4(r8)
  83         stwu    r0,     4(r8)
  84         stwu    r0,     4(r8)
  85         stwu    r0,     4(r8)
  86         stwu    r0,     4(r8)
  87         stwu    r0,     4(r8)
  88         stwu    r0,     4(r8)
  89
  90  /* clear cache blocks */
  91 .L_bzeroBlock:
  92         addi    r8,     r8,     4 /* remove update adjust */
  93         sub     r5,     r8,     r7 /* bytes zeroed */
  94         sub     r4,     r4,     r5
  95         srwi.   r5,     r4,     CACHE_LINE_POW2 /* blocks to zero */
  96         beq     .L_bzeroEndWord
  97         mtctr   r5
  98
  99 .L_bzeroBlock1:
 100         dcbz    0,      r8
 101         addi    r8,     r8,     CACHE_LINE_SIZE
 102         bdnz    .L_bzeroBlock1
 103
 104  /* clear remaining words */
 105 .L_bzeroEndWord:
 106         addis   r6,     0,      HIGH_CADDR(.L_bzeroEndByte)
 107         addi    r6,     r6,     LOW_ADDR(.L_bzeroEndByte)
 108         rlwinm. r5,     r4,     0,      27,     29 /* extract word offset */
 109         addi    r8,     r8,     -4                 /* adjust for update */
 110         beq     .L_bzeroEndByte                    /* no words to zero */
 111         sub     r6,     r6,     r5 /* back from word clear to execute */
 112         mtctr   r6
 113         bctr
 114
 115         stwu    r0,     4(r8)
 116         stwu    r0,     4(r8)
 117         stwu    r0,     4(r8)
 118         stwu    r0,     4(r8)
 119         stwu    r0,     4(r8)
 120         stwu    r0,     4(r8)
 121         stwu    r0,     4(r8)
 122
 123  /* clear remaining bytes */
 124 .L_bzeroEndByte:
 125         addis   r6,     0,      HIGH_CADDR(.L_bzeroEnd)
 126         addi    r6,     r6,     LOW_ADDR(.L_bzeroEnd)
 127                 /* extract byte offset as word offset */
 128         rlwinm. r5,     r4,     2,      28,     29
 129         addi    r8,     r8,     3 /* adjust for update */
 130         beqlr
 131         sub     r6,     r6,     r5 /* back from word clear to execute */
 132         mtctr   r6
 133         bctr
 134
 135         stbu    r0,     1(r8)
 136         stbu    r0,     1(r8)
 137         stbu    r0,     1(r8)
 138
 139 .L_bzeroEnd:
 140         blr
 141
 142 /*
 143  * void *memset(void *from, int c, vm_size_t nbytes)
 144  *
 145  * almost everywhere in the kernel
 146  * this appears to be called with argument c==0. We optimise for those
 147  * cases and call bzero if we can.
 148  *
 149  */
 150
 151 ENTRY(memset, TAG_NO_FRAME_USED)
 152
 153         mr.     ARG3,   ARG1
 154         mr      ARG1,   ARG2
 155         /* optimised case - do a bzero */
 156         beq+    EXT(bzero)
 157
 158         /* If count is zero, return straight away */
 159         cmpi    cr0,    ARG1,   0
 160         beqlr-
 161
 162         /* Now, ARG0 = addr, ARG1=len, ARG3=value */
 163
 164         subi    ARG2,   ARG0,   1       /* use ARG2 as our counter */
 165
 166 0:
 167         subi    ARG1,   ARG1,   1
 168         cmpi    cr0,    ARG1,   0
 169         stbu    ARG3,   1(ARG2)
 170         bne+    0b
 171
 172         /* Return original address in ARG0 */
 173
 174         blr
 175
 176 /*
 177  *       void   bzero_nc(char *addr, unsigned int length)
 178  *
 179  * bzero implementation for PowerPC
 180  *   - assumes non-pic code
 181  *
 182  * returns start address in r3, as per memset (called by memset)
 183  */
 184
 185 ENTRY(bzero_nc, TAG_NO_FRAME_USED)
 186
 187         cmpwi   cr0,    r4,     0 /* no bytes to zero? */
 188         mr      r7,     r3
 189         mr      r8,     r3      /* use r8 as counter to where we are */
 190         beqlr-
 191         cmpwi   cr0,    r4,     CACHE_LINE_SIZE /* clear less than a block? */
 192         li      r0,     0        /* use r0 as source of zeros */
 193         blt     .L_bzeroNCEndWord
 194
 195 /* first, clear bytes up to the next word boundary */
 196         addis   r6,     0,      HIGH_CADDR(.L_bzeroNCBeginWord)
 197         addi    r6,     r6,     LOW_ADDR(.L_bzeroNCBeginWord)
 198                  /* extract byte offset as word offset */
 199         rlwinm. r5,     r8,     2,      28,     29
 200         addi    r8,     r8,     -1 /* adjust for update */
 201         beq     .L_bzeroNCBeginWord /* no bytes to zero */
 202         subfic  r5,     r5,     16 /* compute the number of instructions */
 203         sub     r6,     r6,     r5 /* back from word clear to execute */
 204         mtctr   r6
 205         bctr
 206
 207         stbu    r0,     1(r8)
 208         stbu    r0,     1(r8)
 209         stbu    r0,     1(r8)
 210
 211 /* clear words up to the next block boundary */
 212 .L_bzeroNCBeginWord:
 213         addis   r6,     0,      HIGH_CADDR(.L_bzeroNCBlock)
 214         addi    r6,     r6,     LOW_ADDR(.L_bzeroNCBlock)
 215         addi    r8,     r8,     1
 216         rlwinm. r5,     r8,     0,      27,     29 /* extract word offset */
 217         addi    r8,     r8,     -4              /* adjust for update */
 218         beq     .L_bzeroNCBlock                 /* no words to zero */
 219                 /* compute the number of instructions */
 220         subfic  r5,     r5,     CACHE_LINE_SIZE
 221         sub     r6,     r6,     r5 /* back from word clear to execute */
 222         mtctr   r6
 223         bctr
 224
 225         stwu    r0,     4(r8)
 226         stwu    r0,     4(r8)
 227         stwu    r0,     4(r8)
 228         stwu    r0,     4(r8)
 229         stwu    r0,     4(r8)
 230         stwu    r0,     4(r8)
 231         stwu    r0,     4(r8)
 232
 233  /* clear cache blocks */
 234 .L_bzeroNCBlock:
 235         addi    r8,     r8,     4 /* remove update adjust */
 236         sub     r5,     r8,     r7 /* bytes zeroed */
 237         sub     r4,     r4,     r5
 238         srwi.   r5,     r4,     CACHE_LINE_POW2 /* blocks to zero */
 239         beq     .L_bzeroNCEndWord
 240         mtctr   r5
 241
 242 .L_bzeroNCBlock1:
 243         stw     r0,     0(r8)
 244         stw     r0,     4(r8)
 245         stw     r0,     8(r8)
 246         stw     r0,     12(r8)
 247         stw     r0,     16(r8)
 248         stw     r0,     20(r8)
 249         stw     r0,     24(r8)
 250         stw     r0,     28(r8)
 251         addi    r8,     r8,     CACHE_LINE_SIZE
 252         bdnz    .L_bzeroNCBlock1
 253
 254  /* clear remaining words */
 255 .L_bzeroNCEndWord:
 256         addis   r6,     0,      HIGH_CADDR(.L_bzeroNCEndByte)
 257         addi    r6,     r6,     LOW_ADDR(.L_bzeroNCEndByte)
 258         rlwinm. r5,     r4,     0,      27,     29 /* extract word offset */
 259         addi    r8,     r8,     -4                 /* adjust for update */
 260         beq     .L_bzeroNCEndByte                          /* no words to zero */
 261         sub     r6,     r6,     r5 /* back from word clear to execute */
 262         mtctr   r6
 263         bctr
 264
 265         stwu    r0,     4(r8)
 266         stwu    r0,     4(r8)
 267         stwu    r0,     4(r8)
 268         stwu    r0,     4(r8)
 269         stwu    r0,     4(r8)
 270         stwu    r0,     4(r8)
 271         stwu    r0,     4(r8)
 272
 273  /* clear remaining bytes */
 274 .L_bzeroNCEndByte:
 275         addis   r6,     0,      HIGH_CADDR(.L_bzeroNCEnd)
 276         addi    r6,     r6,     LOW_ADDR(.L_bzeroNCEnd)
 277                 /* extract byte offset as word offset */
 278         rlwinm. r5,     r4,     2,      28,     29
 279         addi    r8,     r8,     3 /* adjust for update */
 280         beqlr
 281         sub     r6,     r6,     r5 /* back from word clear to execute */
 282         mtctr   r6
 283         bctr
 284
 285         stbu    r0,     1(r8)
 286         stbu    r0,     1(r8)
 287         stbu    r0,     1(r8)
 288
 289 .L_bzeroNCEnd:
 290         blr