osfmk/ppc/commpage/bzero_128.s

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25
  26 #define ASSEMBLER
  27 #include <sys/appleapiopts.h>
  28 #include <ppc/asm.h>
  29 #include <machine/cpu_capabilities.h>
  30 #include <machine/commpage.h>
  31
  32         .text
  33         .align  2
  34         .globl  EXT(bzero_128)
  35
  36
  37 // *********************
  38 // * B Z E R O _ 1 2 8 *
  39 // *********************
  40 //
  41 // For 64-bit processors with a 128-byte cache line.
  42 //
  43 // Register use:
  44 //              r0 = zero
  45 //              r3 = original ptr, not changed since memset returns it
  46 //              r4 = count of bytes to set
  47 //              r9 = working operand ptr
  48 // We do not touch r2 and r10-r12, which some callers depend on.
  49
  50         .align  5
  51 bzero_128:                                              // void bzero(void *b, size_t len);
  52         cmplwi  cr7,r4,128              // too short for DCBZ128?
  53         li              r0,0                    // get a 0
  54         neg             r5,r3                   // start to compute #bytes to align
  55         mr              r9,r3                   // make copy of operand ptr (can't change r3)
  56         blt             cr7,Ltail               // length < 128, too short for DCBZ
  57
  58 // At least 128 bytes long, so compute alignment and #cache blocks.
  59
  60         andi.   r5,r5,0x7F              // r5 <-  #bytes to 128-byte align
  61         sub             r4,r4,r5                // adjust length
  62         srwi    r8,r4,7                 // r8 <- 128-byte chunks
  63         rlwinm  r4,r4,0,0x7F    // mask length down to remaining bytes
  64         mtctr   r8                              // set up loop count
  65         beq             Ldcbz                   // skip if already aligned (r8!=0)
  66
  67 // 128-byte align
  68
  69         mtcrf   0x01,r5                 // start to move #bytes to align to cr6 and cr7
  70         cmpwi   cr1,r8,0                // any 128-byte cache lines to 0?
  71         mtcrf   0x02,r5
  72
  73         bf              31,1f                   // byte?
  74         stb             r0,0(r9)
  75         addi    r9,r9,1
  76 1:
  77         bf              30,2f                   // halfword?
  78         sth             r0,0(r9)
  79         addi    r9,r9,2
  80 2:
  81         bf              29,3f                   // word?
  82         stw             r0,0(r9)
  83         addi    r9,r9,4
  84 3:
  85         bf              28,4f                   // doubleword?
  86         std             r0,0(r9)
  87         addi    r9,r9,8
  88 4:
  89         bf              27,5f                   // quadword?
  90         std             r0,0(r9)
  91         std             r0,8(r9)
  92         addi    r9,r9,16
  93 5:
  94         bf              26,6f                   // 32-byte chunk?
  95         std             r0,0(r9)
  96         std             r0,8(r9)
  97         std             r0,16(r9)
  98         std             r0,24(r9)
  99         addi    r9,r9,32
 100 6:
 101         bf              25,7f                   // 64-byte chunk?
 102         std             r0,0(r9)
 103         std             r0,8(r9)
 104         std             r0,16(r9)
 105         std             r0,24(r9)
 106         std             r0,32(r9)
 107         std             r0,40(r9)
 108         std             r0,48(r9)
 109         std             r0,56(r9)
 110         addi    r9,r9,64
 111 7:
 112         beq             cr1,Ltail               // no chunks to dcbz128
 113
 114 // Loop doing 128-byte version of DCBZ instruction.
 115 // NB: if the memory is cache-inhibited, the kernel will clear cr7
 116 // when it emulates the alignment exception.  Eventually, we may want
 117 // to check for this case.
 118
 119 Ldcbz:
 120         dcbz128 0,r9                    // zero another 32 bytes
 121         addi    r9,r9,128
 122         bdnz    Ldcbz
 123
 124 // Store trailing bytes.
 125 //              r0 = 0
 126 //              r4 = count
 127 //              r9 = ptr
 128
 129 Ltail:
 130         srwi.   r5,r4,4                 // r5 <- 16-byte chunks to 0
 131         mtcrf   0x01,r4                 // remaining byte count to cr7
 132         mtctr   r5
 133         beq             2f                              // skip if no 16-byte chunks
 134 1:                                                              // loop over 16-byte chunks
 135         std             r0,0(r9)
 136         std             r0,8(r9)
 137         addi    r9,r9,16
 138         bdnz    1b
 139 2:
 140         bf              28,4f                   // 8-byte chunk?
 141         std             r0,0(r9)
 142         addi    r9,r9,8
 143 4:
 144         bf              29,5f                   // word?
 145         stw             r0,0(r9)
 146         addi    r9,r9,4
 147 5:
 148         bf              30,6f                   // halfword?
 149         sth             r0,0(r9)
 150         addi    r9,r9,2
 151 6:
 152         bflr    31                              // byte?
 153         stb             r0,0(r9)
 154         blr
 155
 156         COMMPAGE_DESCRIPTOR(bzero_128,_COMM_PAGE_BZERO,kCache128+k64Bit,0,kCommPageMTCRF)