osfmk/ppc/commpage/bzero_128.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #define ASSEMBLER
  24 #include <sys/appleapiopts.h>
  25 #include <ppc/asm.h>
  26 #include <machine/cpu_capabilities.h>
  27 #include <machine/commpage.h>
  28
  29         .text
  30         .align  2
  31         .globl  EXT(bzero_128)
  32
  33
  34 // *********************
  35 // * B Z E R O _ 1 2 8 *
  36 // *********************
  37 //
  38 // For 64-bit processors with a 128-byte cache line.
  39 //
  40 // Register use:
  41 //              r0 = zero
  42 //              r3 = original ptr, not changed since memset returns it
  43 //              r4 = count of bytes to set
  44 //              r9 = working operand ptr
  45 // We do not touch r2 and r10-r12, which some callers depend on.
  46
  47         .align  5
  48 bzero_128:                                              // void bzero(void *b, size_t len);
  49         cmplwi  cr7,r4,128              // too short for DCBZ128?
  50         li              r0,0                    // get a 0
  51         neg             r5,r3                   // start to compute #bytes to align
  52         mr              r9,r3                   // make copy of operand ptr (can't change r3)
  53         blt             cr7,Ltail               // length < 128, too short for DCBZ
  54
  55 // At least 128 bytes long, so compute alignment and #cache blocks.
  56
  57         andi.   r5,r5,0x7F              // r5 <-  #bytes to 128-byte align
  58         sub             r4,r4,r5                // adjust length
  59         srwi    r8,r4,7                 // r8 <- 128-byte chunks
  60         rlwinm  r4,r4,0,0x7F    // mask length down to remaining bytes
  61         mtctr   r8                              // set up loop count
  62         beq             Ldcbz                   // skip if already aligned (r8!=0)
  63
  64 // 128-byte align
  65
  66         mtcrf   0x01,r5                 // start to move #bytes to align to cr6 and cr7
  67         cmpwi   cr1,r8,0                // any 128-byte cache lines to 0?
  68         mtcrf   0x02,r5
  69
  70         bf              31,1f                   // byte?
  71         stb             r0,0(r9)
  72         addi    r9,r9,1
  73 1:
  74         bf              30,2f                   // halfword?
  75         sth             r0,0(r9)
  76         addi    r9,r9,2
  77 2:
  78         bf              29,3f                   // word?
  79         stw             r0,0(r9)
  80         addi    r9,r9,4
  81 3:
  82         bf              28,4f                   // doubleword?
  83         std             r0,0(r9)
  84         addi    r9,r9,8
  85 4:
  86         bf              27,5f                   // quadword?
  87         std             r0,0(r9)
  88         std             r0,8(r9)
  89         addi    r9,r9,16
  90 5:
  91         bf              26,6f                   // 32-byte chunk?
  92         std             r0,0(r9)
  93         std             r0,8(r9)
  94         std             r0,16(r9)
  95         std             r0,24(r9)
  96         addi    r9,r9,32
  97 6:
  98         bf              25,7f                   // 64-byte chunk?
  99         std             r0,0(r9)
 100         std             r0,8(r9)
 101         std             r0,16(r9)
 102         std             r0,24(r9)
 103         std             r0,32(r9)
 104         std             r0,40(r9)
 105         std             r0,48(r9)
 106         std             r0,56(r9)
 107         addi    r9,r9,64
 108 7:
 109         beq             cr1,Ltail               // no chunks to dcbz128
 110
 111 // Loop doing 128-byte version of DCBZ instruction.
 112 // NB: if the memory is cache-inhibited, the kernel will clear cr7
 113 // when it emulates the alignment exception.  Eventually, we may want
 114 // to check for this case.
 115
 116 Ldcbz:
 117         dcbz128 0,r9                    // zero another 32 bytes
 118         addi    r9,r9,128
 119         bdnz    Ldcbz
 120
 121 // Store trailing bytes.
 122 //              r0 = 0
 123 //              r4 = count
 124 //              r9 = ptr
 125
 126 Ltail:
 127         srwi.   r5,r4,4                 // r5 <- 16-byte chunks to 0
 128         mtcrf   0x01,r4                 // remaining byte count to cr7
 129         mtctr   r5
 130         beq             2f                              // skip if no 16-byte chunks
 131 1:                                                              // loop over 16-byte chunks
 132         std             r0,0(r9)
 133         std             r0,8(r9)
 134         addi    r9,r9,16
 135         bdnz    1b
 136 2:
 137         bf              28,4f                   // 8-byte chunk?
 138         std             r0,0(r9)
 139         addi    r9,r9,8
 140 4:
 141         bf              29,5f                   // word?
 142         stw             r0,0(r9)
 143         addi    r9,r9,4
 144 5:
 145         bf              30,6f                   // halfword?
 146         sth             r0,0(r9)
 147         addi    r9,r9,2
 148 6:
 149         bflr    31                              // byte?
 150         stb             r0,0(r9)
 151         blr
 152
 153         COMMPAGE_DESCRIPTOR(bzero_128,_COMM_PAGE_BZERO,kCache128+k64Bit,0,kCommPageMTCRF)