xnu-1228.3.13.tar.gz

[apple/xnu.git] / osfmk / ppc / commpage / bigcopy_970.s

diff --git a/osfmk/ppc/commpage/bigcopy_970.s b/osfmk/ppc/commpage/bigcopy_970.s
index befd85ba72a39211a2887d58c0760ab16685584d..add093ea38ec875c2c1600e857401ae9cc9478cc 100644 (file)
--- a/osfmk/ppc/commpage/bigcopy_970.s
+++ b/osfmk/ppc/commpage/bigcopy_970.s
@@ -1,30 +1,37 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /* ====================================
  * Very Long Operand BCOPY for Mac OS X
  * ====================================
  *
  */
 /* ====================================
  * Very Long Operand BCOPY for Mac OS X
  * ====================================
  *

- * Version of 6/11/2003, tuned for the IBM 970.  This is for operands at
- * least several pages long.  It is called from bcopy()/memcpy()/memmove().
+ * Version of 2/21/2004, tuned for the IBM 970.  This is for operands at
+ * least several pages long.  It is called from bcopy()/memcpy()/memmove(),
+ * and runs both in 32 and 64-bit mode.

  *
  * We use the following additional strategies not used by the shorter
  * operand paths.  Mostly, we try to optimize for memory bandwidth:
  *
  * We use the following additional strategies not used by the shorter
  * operand paths.  Mostly, we try to optimize for memory bandwidth:


@@ -36,58 +43,40 @@
  *     which is amortized across the very long operand.
  *     2. Copy larger chunks per iteration to minimize R/W bus turnaround
  *     and maximize DRAM page locality (opening a new page is expensive.)
  *     which is amortized across the very long operand.
  *     2. Copy larger chunks per iteration to minimize R/W bus turnaround
  *     and maximize DRAM page locality (opening a new page is expensive.)

+ *     We use 256-byte chunks.

  *  3. Touch in one source chunk ahead with DCBT.  This is probably the
  *     least important change, and probably only helps restart the
  *     hardware stream at the start of each source page.
  *  3. Touch in one source chunk ahead with DCBT.  This is probably the
  *     least important change, and probably only helps restart the
  *     hardware stream at the start of each source page.

- *
- * Register usage.  Note the rather delicate way we assign multiple uses
- * to the same register.  Beware.
- *   r0  = temp (NB: cannot use r0 for any constant such as "c16")
- *   r3  = not used, as memcpy and memmove return 1st parameter as a value
- *   r4  = source ptr ("rs")
- *   r5  = count of bytes to move ("rc")
- *   r6  = constant 16 ("c16")
- *   r7  = constant 32 (""c32")
- *   r8  = constant 48 (""c48")
- *   r9  = constant 128 (""c128")
- *   r10 = vrsave ("rv")
- *   r11 = constant 256 (""c256")
- *   r12 = destination ptr ("rd")
- *      r13 = constant 384 (""c384")
- *      r14 = temp ("rx")
- *      r15 = temp ("rt")

  */
  */

-#define rs     r4
-#define rd     r12
-#define rc     r5
-#define        rv      r10
-#define        rx      r14
-#define        rt      r15
-
-#define c16    r6
-#define c32    r7
-#define c48    r8
-#define        c128    r9
-#define        c256    r11
-#define        c384    r13
+ 
+#define rs     r13
+#define rd     r14
+#define rc     r15
+#define rx  r16
+
+#define c16     r3
+#define c32     r4
+#define c48     r5
+#define c64     r6
+#define c80     r7
+#define c96     r8
+#define c112    r9
+#define        c256    r10
+#define        c384    r11
+#define rv      r12     // vrsave

 
 // Offsets within the "red zone" (which is 224 bytes long):
 
 
 // Offsets within the "red zone" (which is 224 bytes long):
 

-#define rzR13  -8
-#define rzR14  -12
-#define rzR15  -16
-#define rzV20  -32
-#define rzV21  -48
-#define rzV22  -64
-#define rzV23  -80
-#define rzV24  -96
-#define rzV25  -112
-#define rzV26  -128
-#define rzV27  -144
-#define rzV28  -160
-#define rzV29  -176
-#define rzV30  -192
-#define rzV31  -208
+#define rzR3    -8
+#define rzR13  -16
+#define rzR14  -24
+#define rzR15   -32
+#define rzR16   -40
+
+#define rzV20  -64
+#define rzV21  -80
+#define rzV22  -96
+#define rzV23  -112

 
 
 #include <sys/appleapiopts.h>
 
 
 #include <sys/appleapiopts.h>


@@ -96,401 +85,247 @@
 #include <machine/commpage.h>
 
         .text
 #include <machine/commpage.h>
 
         .text

-        .globl EXT(bigcopy_970)
-
+/*
+ * WARNING: this code is written for 32-bit mode, and ported by the kernel if necessary
+ * to 64-bit mode for use in the 64-bit commpage.  This "port" consists of the following
+ * simple transformations:
+ *      - all word compares are changed to doubleword
+ *      - all "srwi[.]" opcodes are changed to "srdi[.]"                      
+ * Nothing else is done.  For this to work, the following rules must be
+ * carefully followed:
+ *      - do not use carry or overflow
+ *      - only use record mode if you are sure the results are mode-invariant
+ *        for example, all "andi." and almost all "rlwinm." are fine
+ *      - do not use "slwi", "slw", or "srw"
+ * An imaginative programmer could break the porting model in other ways, but the above
+ * are the most likely problem areas.  It is perhaps surprising how well in practice
+ * this simple method works.
+ */

 
 // Entry point.  This is a subroutine of bcopy().  When called:
 
 // Entry point.  This is a subroutine of bcopy().  When called:

-//     r4 = source ptr (aka "rs")
-// r12 = dest ptr (aka "rd")
-//     r5 = length (>= 16K bytes) (aka "rc")
+//  r0 = return address (also stored in caller's SF)
+//     r4 = source ptr
+//     r5 = length (at least several pages)
+// r12 = dest ptr

 // 
 // 

-// We only do "forward" moves, ie non-overlapping or toward 0.
-//
-// We return with non-volatiles and r3 preserved.
+// We only do "forward" moves, ie non-overlapping or toward 0.  We return with non-volatiles
+// and r3 preserved.

 
         .align         5
 bigcopy_970:
 
         .align         5
 bigcopy_970:

-        stw            r13,rzR13(r1)           // spill non-volatile regs we use to redzone
-        stw            r14,rzR14(r1)
-        stw            r15,rzR15(r1)
-        li             r0,rzV20
-        neg            rt,rd                           // start to cache-line-align destination
-        stvx   v20,r1,r0                       // we use all 32 VRs
-        li             r0,rzV21
-        stvx   v21,r1,r0
-        li             r0,rzV22
-        stvx   v22,r1,r0
-        li             r0,rzV23
-        stvx   v23,r1,r0
-        li             r0,rzV24
-        andi.  rt,rt,127                       // get #bytes to 128-byte align
-        stvx   v24,r1,r0
-        li             r0,rzV25
-        stvx   v25,r1,r0
-        li             r0,rzV26
-        sub            rc,rc,rt                        // adjust length by #bytes to align destination
-        stvx   v26,r1,r0
-        li             r0,rzV27
-        stvx   v27,r1,r0
-        li             r0,rzV28
-        mtctr  rt                                      // #bytes to align destination
-        stvx   v28,r1,r0
-        li             r0,rzV29
-        stvx   v29,r1,r0
-        li             r0,rzV30
-        stvx   v30,r1,r0
-        li             r0,rzV31
-        stvx   v31,r1,r0
-        beq            2f                                      // dest already 128-byte aligned
-        b              1f
-
+        neg     r2,r12              // is destination cache-line-aligned?
+        std     r3,rzR3(r1)         // save caller's r3, which must be preserved for memcpy()
+        std            r13,rzR13(r1)           // spill non-volatile regs we use to redzone
+        std            r14,rzR14(r1)
+        std            r15,rzR15(r1)
+        andi.   r2,r2,0x7F          // #bytes to align
+        std     r16,rzR16(r1)
+        mr      rs,r4               // copy parameters into nonvolatile registers
+        mr      rd,r12
+        mr      rc,r5
+        mr      rx,r0               // also save return address
+        beq     1f                  // skip if already aligned

 
 // Cache-line-align destination.
 
 // Cache-line-align destination.

-
-        .align 5
-1:
-        lbz            r0,0(rs)
-        addi   rs,rs,1
-        stb            r0,0(rd)
-        addi   rd,rd,1
-        bdnz   1b
+        
+        mr      r3,rd               // set up dest ptr for memcpy()
+        mr      r5,r2               // number of bytes to copy
+        add     rs,rs,r2            // then bump our parameters past initial copy
+        add     rd,rd,r2
+        sub     rc,rc,r2
+        bla     _COMM_PAGE_MEMCPY   // 128-byte-align destination

 
 
 
 

-// Is source 16-byte aligned?  Load constant offsets.
+// Load constant offsets and check whether source is 16-byte aligned.
+// NB: the kernel clears cr7 if it emulates a dcbz128 on the commpage,
+// and we dcbz only if cr7 beq is set.

 
 

-2:
+1:
+        dcbt    0,rs                // touch in 1st line of source

         andi.  r0,rs,15                        // check source alignment
         mfspr  rv,vrsave                       // save caller's bitmask
         andi.  r0,rs,15                        // check source alignment
         mfspr  rv,vrsave                       // save caller's bitmask

-        li             r0,-1                           // we use all 32 VRs

         li             c16,16                          // load the constant offsets for x-form ops
         li             c32,32
         li             c16,16                          // load the constant offsets for x-form ops
         li             c32,32

+        srwi    r2,rc,8             // get number of 256-byte chunks to xfer
+        li             r0,-256                         // we use 24 VRs (ie, 0-23)

         li             c48,48
         li             c48,48

-        li             c128,128
+        li      c64,64
+        li      c80,80
+        or      r0,r0,rv            // add our bits to caller's
+        li      c96,96
+        mtctr   r2                  // set up loop count
+        li      c112,112
+        cmpd    cr7,r2,r2           // initialize cr7_eq to "on", so we dcbz128
+        mtspr  vrsave,r0           // say we use vr0..vr23

         li             c256,256
         li             c384,384
         li             c256,256
         li             c384,384

-        mtspr  vrsave,r0
-
-// NB: the kernel clears cr7 if it emulates a dcbz128 on the commpage,
-// and we dcbz only if cr7 beq is set.  We check to be sure the dcbz's
-// won't zero source bytes before we load them, since we zero before
-// loading as this is faster than zeroing after loading and before storing.
+        beq            LalignedLoop            // handle aligned sources

 
 

-        cmpw   cr7,r0,r0                       // initialize cr7 beq to use dcbz128
-        sub            rt,rs,rd                        // get (rs-rd)
-        cmplwi cr1,rt,512                      // are we moving down less than 512 bytes?

         
         

-// Start fetching in source cache lines.
+// Set up for unaligned loop.

 
 

-        dcbt   c128,rs                         // first line already touched in
-        dcbt   c256,rs
-        dcbt   c384,rs
-        
-        bge++  cr1,3f                          // skip if not moving down less than 512 bytes
-        cmpw   cr7,c16,c32                     // cannot dcbz since it would zero source bytes
-3:
-        beq            LalignedLoop            // handle aligned sources

         lvsl   v0,0,rs                         // get permute vector for left shift
         lvxl   v1,0,rs                         // prime the loop
         lvsl   v0,0,rs                         // get permute vector for left shift
         lvxl   v1,0,rs                         // prime the loop

+        li             r0,rzV20            // save non-volatile VRs in redzone
+        stvx   v20,r1,r0
+        li             r0,rzV21
+        stvx   v21,r1,r0
+        li             r0,rzV22
+        stvx   v22,r1,r0
+        li             r0,rzV23
+        stvx   v23,r1,r0

         b              LunalignedLoop          // enter unaligned loop
 
 
         b              LunalignedLoop          // enter unaligned loop
 
 

-// Main loop for unaligned operands.  We loop over 384-byte chunks (3 cache lines)
-// since we need a few VRs for permuted destination QWs and the permute vector.
+// Main loop for unaligned operands.  We loop over 256-byte chunks (2 cache lines).
+// Destination is 128-byte aligned, source is unaligned.

 
         .align 5
 LunalignedLoop:
 
         .align 5
 LunalignedLoop:

-        subi   rc,rc,384                       // decrement byte count
-        addi   rx,rs,384                       // get address of next chunk
+        dcbt   c256,rs             // touch in next chunk
+        dcbt   c384,rs
+        addi    r2,rs,128           // point to 2nd 128 bytes of source

         lvxl   v2,c16,rs
         lvxl   v3,c32,rs
         lvxl   v2,c16,rs
         lvxl   v3,c32,rs

+        lvxl   v4,c48,rs
+        lvxl    v5,c64,rs
+        lvxl    v6,c80,rs
+        lvxl    v7,c96,rs
+        lvxl    v8,c112,rs
+        lvxl    v9,0,r2
+        addi    rs,rs,256           // point to next source chunk
+        lvxl    v10,c16,r2
+        lvxl    v11,c32,r2
+        vperm   v17,v1,v2,v0
+        lvxl    v12,c48,r2
+        lvxl    v13,c64,r2
+        vperm   v18,v2,v3,v0
+        lvxl    v14,c80,r2
+        lvxl    v15,c96,r2
+        vperm   v19,v3,v4,v0
+        lvxl    v16,c112,r2
+        lvxl   v1,0,rs             // peek ahead at first source quad in next chunk
+        vperm   v20,v4,v5,v0
+        addi    r2,rd,128           // point to 2nd 128 bytes of dest 

         bne--  cr7,1f                          // skip dcbz's if cr7 beq has been turned off by kernel
         bne--  cr7,1f                          // skip dcbz's if cr7 beq has been turned off by kernel

-        dcbz128        0,rd                            // (also skip if moving down less than 512 bytes)
-        bne--  cr7,1f                          // catch it first time through
-        dcbz128        c128,rd
-        dcbz128        c256,rd
+        dcbz128        0,rd
+        dcbz128        0,r2

 1:
 1:

-        addi   rt,rs,64
-        dcbt   0,rx                            // touch in next chunk
-        dcbt   c128,rx
-        dcbt   c256,rx
-        lvxl   v4,c48,rs
-        addi   rs,rs,128
-        lvxl   v5,0,rt
-        cmplwi rc,384                          // another chunk to go?
-        lvxl   v6,c16,rt
-        lvxl   v7,c32,rt
-        lvxl   v8,c48,rt
-        addi   rt,rs,64
-        vperm  v25,v1,v2,v0
-        lvxl   v9,0,rs
-        lvxl   v10,c16,rs
-        vperm  v26,v2,v3,v0
-        lvxl   v11,c32,rs
-        lvxl   v12,c48,rs
-        vperm  v27,v3,v4,v0
-        addi   rs,rs,128
-        lvxl   v13,0,rt
-        lvxl   v14,c16,rt
-        vperm  v28,v4,v5,v0
-        lvxl   v15,c32,rt
-        lvxl   v16,c48,rt
-        vperm  v29,v5,v6,v0
-        addi   rt,rs,64
-        lvxl   v17,0,rs
-        lvxl   v18,c16,rs
-        vperm  v30,v6,v7,v0
-        lvxl   v19,c32,rs
-        lvxl   v20,c48,rs
-        vperm  v31,v7,v8,v0
-        addi   rs,rs,128
-        lvxl   v21,0,rt
-        lvxl   v22,c16,rt
-        vperm  v2,v8,v9,v0
-        lvxl   v23,c32,rt
-        lvxl   v24,c48,rt
-        vperm  v3,v9,v10,v0
-        lvx            v1,0,rs                         // get 1st qw of next chunk
-        vperm  v4,v10,v11,v0
-        
-        addi   rt,rd,64
-        stvxl  v25,0,rd
-        stvxl  v26,c16,rd
-        vperm  v5,v11,v12,v0
-        stvxl  v27,c32,rd
-        stvxl  v28,c48,rd
-        vperm  v6,v12,v13,v0
-        addi   rd,rd,128
-        stvxl  v29,0,rt
-        stvxl  v30,c16,rt
-        vperm  v7,v13,v14,v0
-        stvxl  v31,c32,rt
-        stvxl  v2,c48,rt
-        vperm  v8,v14,v15,v0
-        addi   rt,rd,64
-        stvxl  v3,0,rd
-        stvxl  v4,c16,rd
-        vperm  v9,v15,v16,v0
-        stvxl  v5,c32,rd
-        stvxl  v6,c48,rd
-        vperm  v10,v16,v17,v0
-        addi   rd,rd,128
-        stvxl  v7,0,rt
-        vperm  v11,v17,v18,v0
-        stvxl  v8,c16,rt
-        stvxl  v9,c32,rt
-        vperm  v12,v18,v19,v0
-        stvxl  v10,c48,rt
-        addi   rt,rd,64
-        vperm  v13,v19,v20,v0
-        stvxl  v11,0,rd
-        stvxl  v12,c16,rd
-        vperm  v14,v20,v21,v0
-        stvxl  v13,c32,rd
-        vperm  v15,v21,v22,v0
-        stvxl  v14,c48,rd
-        vperm  v16,v22,v23,v0
-        addi   rd,rd,128
-        stvxl  v15,0,rt
-        vperm  v17,v23,v24,v0
-        stvxl  v16,c16,rt
-        vperm  v18,v24,v1,v0
-        stvxl  v17,c32,rt
-        stvxl  v18,c48,rt
-        bge++  LunalignedLoop          // loop if another 384 bytes to go
-
-// End of unaligned main loop.  Handle up to 384 leftover bytes.
-
-        srwi.  r0,rc,5                         // get count of 32-byte chunks remaining
-        beq            Ldone                           // none
-        rlwinm rc,rc,0,0x1F            // mask count down to 0..31 leftover bytes
-        mtctr  r0
-1:                                                                     // loop over 32-byte chunks
-        lvx            v2,c16,rs
-        lvx            v3,c32,rs
-        addi   rs,rs,32
-        vperm  v8,v1,v2,v0
-        vperm  v9,v2,v3,v0
-        vor            v1,v3,v3                        // v1 <- v3
-        stvx   v8,0,rd
-        stvx   v9,c16,rd
-        addi   rd,rd,32
-        bdnz   1b
-        
-        b              Ldone
+        vperm   v21,v5,v6,v0
+        stvxl  v17,0,rd
+        vperm   v22,v6,v7,v0
+        stvxl  v18,c16,rd
+        vperm   v23,v7,v8,v0
+        stvxl  v19,c32,rd
+        vperm   v17,v8,v9,v0
+        stvxl  v20,c48,rd
+        vperm   v18,v9,v10,v0
+        stvxl  v21,c64,rd
+        vperm   v19,v10,v11,v0
+        stvxl  v22,c80,rd
+        vperm   v20,v11,v12,v0
+        stvxl  v23,c96,rd
+        vperm   v21,v12,v13,v0
+        stvxl  v17,c112,rd
+        vperm   v22,v13,v14,v0
+        addi   rd,rd,256           // point to next dest chunk
+        stvxl  v18,0,r2
+        vperm   v23,v14,v15,v0
+        stvxl  v19,c16,r2
+        vperm   v17,v15,v16,v0
+        stvxl  v20,c32,r2
+        vperm   v18,v16,v1,v0
+        stvxl  v21,c48,r2
+        stvxl  v22,c64,r2
+        stvxl  v23,c80,r2
+        stvxl  v17,c96,r2
+        stvxl  v18,c112,r2
+        bdnz++ LunalignedLoop      // loop if another 256 bytes to go
+
+        li             r6,rzV20            // restore non-volatile VRs
+        li             r7,rzV21
+        li             r8,rzV22
+        li             r9,rzV23
+        lvx            v20,r1,r6
+        lvx            v21,r1,r7
+        lvx            v22,r1,r8
+        lvx            v23,r1,r9
+        b       Ldone

         
         
 // Aligned loop.  Destination is 128-byte aligned, and source is 16-byte
         
         
 // Aligned loop.  Destination is 128-byte aligned, and source is 16-byte

-// aligned.  Loop over 512-byte chunks (4 cache lines.)
+// aligned.  Loop over 256-byte chunks (2 cache lines.)

 
         .align 5
 LalignedLoop:
 
         .align 5
 LalignedLoop:

-        subi   rc,rc,512                       // decrement count
-        addi   rx,rs,512                       // address of next chunk
+        dcbt   c256,rs             // touch in next chunk
+        dcbt   c384,rs
+        addi    r2,rs,128           // point to 2nd 128 bytes of source

         lvxl   v1,0,rs
         lvxl   v2,c16,rs
         lvxl   v1,0,rs
         lvxl   v2,c16,rs

-        bne--  cr7,1f                          // skip dcbz's if cr7 beq has been turned off by kernel
-        dcbz128        0,rd                            // (also skip if moving down less than 512 bytes)
-        bne--  cr7,1f                          // catch it first time through
-        dcbz128        c128,rd
-        dcbz128        c256,rd
-        dcbz128        c384,rd
-1:
-        addi   rt,rs,64
-        dcbt   0,rx                            // touch in next chunk
-        dcbt   c128,rx
-        dcbt   c256,rx
-        dcbt   c384,rx

         lvxl   v3,c32,rs
         lvxl   v4,c48,rs
         lvxl   v3,c32,rs
         lvxl   v4,c48,rs

-        addi   rs,rs,128
-        lvxl   v5,0,rt
-        cmplwi rc,512                          // another chunk to go?
-        lvxl   v6,c16,rt
-        lvxl   v7,c32,rt
-        lvxl   v8,c48,rt
-        addi   rt,rs,64
-        lvxl   v9,0,rs
-        lvxl   v10,c16,rs
-        lvxl   v11,c32,rs
-        lvxl   v12,c48,rs
-        addi   rs,rs,128
-        lvxl   v13,0,rt
-        lvxl   v14,c16,rt
-        lvxl   v15,c32,rt
-        lvxl   v16,c48,rt
-        addi   rt,rs,64
-        lvxl   v17,0,rs
-        lvxl   v18,c16,rs
-        lvxl   v19,c32,rs
-        lvxl   v20,c48,rs
-        addi   rs,rs,128
-        lvxl   v21,0,rt
-        lvxl   v22,c16,rt
-        lvxl   v23,c32,rt
-        lvxl   v24,c48,rt
-        addi   rt,rs,64
-        lvxl   v25,0,rs
-        lvxl   v26,c16,rs
-        lvxl   v27,c32,rs
-        lvxl   v28,c48,rs
-        addi   rs,rs,128
-        lvxl   v29,0,rt
-        lvxl   v30,c16,rt
-        lvxl   v31,c32,rt
-        lvxl   v0,c48,rt
-
-        addi   rt,rd,64
+        lvxl    v5,c64,rs
+        lvxl    v6,c80,rs
+        lvxl    v7,c96,rs
+        lvxl    v8,c112,rs
+        lvxl    v9,0,r2
+        lvxl    v10,c16,r2
+        lvxl    v11,c32,r2
+        lvxl    v12,c48,r2
+        lvxl    v13,c64,r2
+        lvxl    v14,c80,r2
+        lvxl    v15,c96,r2
+        lvxl    v16,c112,r2
+        addi    r2,rd,128           // point to 2nd 128 bytes of dest 
+        bne--  cr7,1f                          // skip dcbz's if cr7 beq has been turned off by kernel
+        dcbz128        0,rd
+        dcbz128        0,r2
+1:
+        addi    rs,rs,256           // point to next source chunk

         stvxl  v1,0,rd
         stvxl  v2,c16,rd
         stvxl  v3,c32,rd
         stvxl  v4,c48,rd
         stvxl  v1,0,rd
         stvxl  v2,c16,rd
         stvxl  v3,c32,rd
         stvxl  v4,c48,rd

-        addi   rd,rd,128
-        stvxl  v5,0,rt
-        stvxl  v6,c16,rt
-        stvxl  v7,c32,rt
-        stvxl  v8,c48,rt
-        addi   rt,rd,64
-        stvxl  v9,0,rd
-        stvxl  v10,c16,rd
-        stvxl  v11,c32,rd
-        stvxl  v12,c48,rd
-        addi   rd,rd,128
-        stvxl  v13,0,rt
-        stvxl  v14,c16,rt
-        stvxl  v15,c32,rt
-        stvxl  v16,c48,rt
-        addi   rt,rd,64
-        stvxl  v17,0,rd
-        stvxl  v18,c16,rd
-        stvxl  v19,c32,rd
-        stvxl  v20,c48,rd
-        addi   rd,rd,128
-        stvxl  v21,0,rt
-        stvxl  v22,c16,rt
-        stvxl  v23,c32,rt
-        stvxl  v24,c48,rt
-        addi   rt,rd,64
-        stvxl  v25,0,rd
-        stvxl  v26,c16,rd
-        stvxl  v27,c32,rd
-        stvxl  v28,c48,rd
-        addi   rd,rd,128
-        stvxl  v29,0,rt
-        stvxl  v30,c16,rt
-        stvxl  v31,c32,rt
-        stvxl  v0,c48,rt
-        bge++  LalignedLoop            // loop if another 512 bytes to go
-
-// End of aligned main loop.  Handle up to 511 leftover bytes.
-
-        srwi.  r0,rc,5                         // get count of 32-byte chunks remaining
-        beq            Ldone                           // none
-        rlwinm rc,rc,0,0x1F            // mask count down to 0..31 leftover bytes
-        mtctr  r0
-1:                                                                     // loop over 32-byte chunks
-        lvx            v1,0,rs
-        lvx            v2,c16,rs
-        addi   rs,rs,32
-        stvx   v1,0,rd
-        stvx   v2,c16,rd
-        addi   rd,rd,32
-        bdnz   1b
-
-
-// Done, except for 0..31 leftovers at end.  Restore non-volatiles.
+        stvxl  v5,c64,rd
+        stvxl  v6,c80,rd
+        stvxl  v7,c96,rd
+        stvxl  v8,c112,rd
+        addi   rd,rd,256           // point to next dest chunk
+        stvxl  v9,0,r2
+        stvxl  v10,c16,r2
+        stvxl  v11,c32,r2
+        stvxl  v12,c48,r2
+        stvxl  v13,c64,r2
+        stvxl  v14,c80,r2
+        stvxl  v15,c96,r2
+        stvxl  v16,c112,r2
+        bdnz++ LalignedLoop            // loop if another 256 bytes to go
+
+
+// Done, except for 0..255 leftover bytes at end.

 //     rs = source ptr
 //     rd = dest ptr
 //     rs = source ptr
 //     rd = dest ptr

-//     rc = count (0..31)
+//     rc = remaining count in low 7 bits

 //     rv = caller's vrsave
 //     rv = caller's vrsave

+//  rx = caller's return address

 
 Ldone:
 
 Ldone:

-        cmpwi  rc,0                            // any leftover bytes?
-        lwz            r13,rzR13(r1)           // restore non-volatiles from redzone
-        lwz            r14,rzR14(r1)
-        lwz            r15,rzR15(r1)
-        li             r0,rzV20
-        lvx            v20,r1,r0
-        li             r0,rzV21
-        lvx            v21,r1,r0
-        li             r0,rzV22
-        lvx            v22,r1,r0
-        li             r0,rzV23
-        lvx            v23,r1,r0
-        li             r0,rzV24
-        lvx            v24,r1,r0
-        li             r0,rzV25
-        lvx            v25,r1,r0
-        li             r0,rzV26
-        lvx            v26,r1,r0
-        li             r0,rzV27
-        lvx            v27,r1,r0
-        li             r0,rzV28
-        lvx            v28,r1,r0
-        li             r0,rzV29
-        lvx            v29,r1,r0
-        li             r0,rzV30
-        lvx            v30,r1,r0
-        li             r0,rzV31
-        lvx            v31,r1,r0
-        mtspr  vrsave,rv                       // restore caller's bitmask
-        beqlr                                          // done if no leftover bytes
-        
-
-// Handle 1..31 leftover bytes at end.
-
-        mtctr  rc                                      // set up loop count
-        b              1f
-        
-        .align 5
-1:
-        lbz            r0,0(rs)
-        addi   rs,rs,1
-        stb            r0,0(rd)
-        addi   rd,rd,1
-        bdnz   1b
+        andi.   r5,rc,0xFF          // any leftover bytes? (0..255)
+        mtspr  vrsave,rv                       // restore bitmap of live vr's

         
         

+        mr      r3,rd
+        mr      r4,rs
+        bnela   _COMM_PAGE_MEMCPY   // copy leftover bytes
+
+        mtlr    rx                  // restore return address
+        ld      r3,rzR3(r1)         // restore non-volatile GPRs from redzone
+        ld             r13,rzR13(r1)
+        ld             r14,rzR14(r1)
+        ld             r15,rzR15(r1)
+        ld      r16,rzR16(r1)

         blr
 
 
         blr
 
 

-        COMMPAGE_DESCRIPTOR(bigcopy_970,_COMM_PAGE_BIGCOPY,0,0,0) // load on all machines for now
+        COMMPAGE_DESCRIPTOR(bigcopy_970,_COMM_PAGE_BIGCOPY,0,0,kPort32to64+kCommPageBoth)