i386/string/bcopy_sse2.s

   1 /*
   2  * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <machine/cpu_capabilities.h>
  30 #include <platfunc.h>
  31
  32 /*
  33  * The bcopy/memcpy loops, tuned for Pentium-M class processors with SSE2
  34  * and 64-byte cache lines, such as Core and Core 2.
  35  *
  36  * The following #defines are tightly coupled to the u-architecture:
  37  */
  38
  39 #define kShort  80                      // too short to bother with SSE (must be >=80)
  40 #define kVeryLong   (500*1024)          // large enough for non-temporal stores (must be >= 8192)
  41 #define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands
  42 #define kFastUCode  (16*1024)           // cutoff for microcode fastpath for "rep/movsl"
  43
  44
  45 // void bcopy(const void *src, void *dst, size_t len);
  46
  47 PLATFUNC_FUNCTION_START(bcopy, sse2, 32, 5)
  48         pushl   %ebp                    // set up a frame for backtraces
  49         movl    %esp,%ebp
  50         pushl   %esi
  51         pushl   %edi
  52         movl    8(%ebp),%esi            // get source ptr
  53         movl    12(%ebp),%edi           // get dest ptr
  54         jmp     Ljoin
  55
  56 //
  57 // void *memcpy(void *dst, const void *src, size_t len);
  58 // void *memmove(void *dst, const void *src, size_t len);
  59 //
  60
  61 PLATFUNC_FUNCTION_START(memcpy, sse2, 32, 0)    // void *memcpy(void *dst, const void *src, size_t len)
  62 PLATFUNC_FUNCTION_START(memmove, sse2, 32, 0)   // void *memmove(void *dst, const void *src, size_t len)
  63 Lmemcpy_sse2:
  64         pushl   %ebp                    // set up a frame for backtraces
  65         movl    %esp,%ebp
  66         pushl   %esi
  67         pushl   %edi
  68         movl    8(%ebp),%edi            // get dest ptr
  69         movl    12(%ebp),%esi           // get source ptr
  70
  71 Ljoin:                                  // here from bcopy() with esi and edi loaded
  72         movl    16(%ebp),%ecx           // get length
  73         movl    %edi,%edx
  74         subl    %esi,%edx               // (dest - source)
  75         cmpl    %ecx,%edx               // must move in reverse if (dest - source) < length
  76         jb      LReverseIsland
  77 Lrejoin:                                // here from very-long-operand copies
  78         cmpl    $(kShort),%ecx          // long enough to bother with SSE?
  79         ja      LNotShort               // yes
  80
  81 // Handle short forward copies.  As the most common case, this is the fall-through path.
  82 //      ecx = length (<= kShort)
  83 //      esi = source ptr
  84 //      edi = dest ptr
  85
  86 Lshort:
  87         movl    %ecx,%edx               // copy length
  88         shrl    $2,%ecx                 // get #doublewords
  89         jz      LLeftovers
  90 2:                                      // loop copying doublewords
  91         movl    (%esi),%eax
  92         addl    $4,%esi
  93         movl    %eax,(%edi)
  94         addl    $4,%edi
  95         dec     %ecx
  96         jnz     2b
  97 LLeftovers:                             // handle leftover bytes (0..3) in last word
  98         andl    $3,%edx                 // any leftover bytes?
  99         jz      5f
 100 4:                                      // loop copying bytes
 101         movb    (%esi),%al
 102         inc     %esi
 103 movb    %al,(%edi)
 104         inc     %edi
 105         dec     %edx
 106         jnz     4b
 107 5:
 108         movl    8(%ebp),%eax            // get return value (dst ptr) for memcpy/memmove
 109         popl    %edi
 110         popl    %esi
 111         popl    %ebp
 112         ret
 113
 114
 115 LReverseIsland:                         // keep the "jb" above a short branch...
 116         jmp     LReverse                // ...because reverse moves are uncommon
 117
 118
 119 // Handle forward moves that are long enough to justify use of SSE3.
 120 // First, 16-byte align the destination.
 121 //      ecx = length (> kShort)
 122 //      esi = source ptr
 123 //      edi = dest ptr
 124
 125 LNotShort:
 126         cmpl    $(kVeryLong),%ecx       // long enough to justify heavyweight loops?
 127         movl    %edi,%edx               // copy destination
 128         jae     LVeryLong               // use very-long-operand path
 129         negl    %edx
 130         andl    $15,%edx                // get #bytes to align destination
 131         jz      LDestAligned            // already aligned
 132         subl    %edx,%ecx               // decrement length
 133 1:                                      // loop copying 1..15 bytes
 134         movb    (%esi),%al
 135         inc     %esi
 136         movb    %al,(%edi)
 137         inc     %edi
 138         dec     %edx
 139         jnz     1b
 140
 141 // Destination is now aligned.  Prepare for forward loops over 64-byte chunks.
 142 // Since kShort>=80 and we've moved at most 15 bytes already, there is at least one chunk.
 143
 144 LDestAligned:
 145         movl    %ecx,%edx               // copy length
 146         movl    %ecx,%eax               // twice
 147         andl    $63,%ecx                // get remaining bytes for Lshort
 148         andl    $-64,%edx               // get number of bytes we will copy in inner loop
 149         addl    %edx,%esi               // point to 1st byte not copied
 150         addl    %edx,%edi
 151         negl    %edx                    // now generate offset to 1st byte to be copied
 152         testl   $15,%esi                // is source aligned too?
 153         jnz     LUnalignedLoop          // no
 154
 155         cmpl    $(kFastUCode),%eax      // long enough for the fastpath in microcode?
 156         jb  LAlignedLoop                // no, use SSE
 157         cld                             // we'll move forward
 158         movl    %eax,%ecx               // copy length again
 159         shrl    $2,%ecx                 // compute #words to move
 160         addl    %edx,%esi               // restore ptrs to 1st byte of source and dest
 161         addl    %edx,%edi
 162         rep                             // the u-code will optimize this
 163         movsl
 164         movl    %eax,%edx               // original length
 165         jmp     LLeftovers              // handle 0..3 leftover bytes
 166
 167
 168 // Forward aligned loop for medium length operands (kShort < n < kVeryLong).
 169
 170         .align  4,0x90              // 16-byte align inner loops
 171 LAlignedLoop:                       // loop over 64-byte chunks
 172         movdqa  (%esi,%edx),%xmm0
 173         movdqa  16(%esi,%edx),%xmm1
 174         movdqa  32(%esi,%edx),%xmm2
 175         movdqa  48(%esi,%edx),%xmm3
 176
 177         movdqa  %xmm0,(%edi,%edx)
 178         movdqa  %xmm1,16(%edi,%edx)
 179         movdqa  %xmm2,32(%edi,%edx)
 180         movdqa  %xmm3,48(%edi,%edx)
 181
 182         addl    $64,%edx
 183         jnz     LAlignedLoop
 184
 185         jmp     Lshort              // copy remaining 0..15 bytes and done
 186
 187
 188 // Forward unaligned loop for medium length operands (kShort < n < kVeryLong).
 189 // Note that LDDQU==MOVDQU on these machines, ie we don't care when we cross
 190 // source cache lines.
 191
 192         .align  4,0x90              // 16-byte align inner loops
 193 LUnalignedLoop:                     // loop over 64-byte chunks
 194         movdqu  (%esi,%edx),%xmm0   // the loads are unaligned
 195         movdqu  16(%esi,%edx),%xmm1
 196         movdqu  32(%esi,%edx),%xmm2
 197         movdqu  48(%esi,%edx),%xmm3
 198
 199         movdqa  %xmm0,(%edi,%edx)   // we can use aligned stores
 200         movdqa  %xmm1,16(%edi,%edx)
 201         movdqa  %xmm2,32(%edi,%edx)
 202         movdqa  %xmm3,48(%edi,%edx)
 203
 204         addl    $64,%edx
 205         jnz     LUnalignedLoop
 206
 207         jmp     Lshort                  // copy remaining 0..63 bytes and done
 208
 209
 210 // Very long forward moves.  These are at least several pages, so we loop over big
 211 // chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
 212 // it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
 213 // so the copy loop reads from L2 and writes directly to memory (with write combining.)
 214 // This minimizes bus turnaround and maintains good DRAM page locality.
 215 // Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
 216 // size.  Otherwise, it is counter-productive to bypass L2 on the stores.
 217 //      ecx = length (>= kVeryLong bytes)
 218 //      edi = dest (aligned)
 219 //      esi = source
 220
 221 LVeryLong:
 222         pushl   %ebx            // we'll need to use this
 223         movl    %edi,%ebx       // copy dest ptr
 224         negl    %ebx
 225         andl    $63,%ebx        // get #bytes to cache line align destination
 226         jz      LBigChunkLoop   // already aligned
 227
 228 // Cache line align destination, so temporal stores in copy loops work right.
 229
 230         pushl   %ecx            // save total length remaining
 231         pushl   %ebx            // arg3 - #bytes to align destination (1..63)
 232         pushl   %esi            // arg2 - source
 233         pushl   %edi            // arg1 - dest
 234         call    Lmemcpy_sse2    // align the destination
 235         movl    12(%esp),%ecx   // recover total length
 236         addl    $16,%esp
 237         addl    %ebx,%esi       // adjust ptrs and lengths past copy
 238         addl    %ebx,%edi
 239         subl    %ebx,%ecx
 240
 241 // Loop over big chunks.
 242 //      ecx = length remaining (>= 4096)
 243 //      edi = dest (64-byte aligned)
 244 //      esi = source (may be unaligned)
 245
 246 LBigChunkLoop:
 247         movl    $(kBigChunk),%edx       // assume we can do a full chunk
 248         cmpl    %edx,%ecx               // do we have a full chunk left to do?
 249         cmovbl  %ecx,%edx               // if not, only move what we have left
 250         andl    $-4096,%edx             // we work in page multiples
 251         xor     %eax,%eax               // initialize chunk offset
 252         jmp     LTouchLoop
 253
 254 // Because the source may be unaligned, we use byte loads to touch.
 255 //      ecx = length remaining (including this chunk)
 256 //      edi = ptr to start of dest chunk
 257 //      esi = ptr to start of source chunk
 258 //      edx = chunk length (multiples of pages)
 259 //      ebx = scratch reg used to read a byte of each cache line
 260 //      eax = chunk offset
 261
 262         .align  4,0x90                  // 16-byte align inner loops
 263 LTouchLoop:
 264         movzb   (%esi,%eax),%ebx        // touch line 0, 2, 4, or 6 of page
 265         movzb   1*64(%esi,%eax),%ebx    // touch line 1, 3, 5, or 7
 266         movzb   8*64(%esi,%eax),%ebx    // touch line 8, 10, 12, or 14
 267         movzb   9*64(%esi,%eax),%ebx    // etc
 268
 269         movzb   16*64(%esi,%eax),%ebx
 270         movzb   17*64(%esi,%eax),%ebx
 271         movzb   24*64(%esi,%eax),%ebx
 272         movzb   25*64(%esi,%eax),%ebx
 273
 274         movzb   32*64(%esi,%eax),%ebx
 275         movzb   33*64(%esi,%eax),%ebx
 276         movzb   40*64(%esi,%eax),%ebx
 277         movzb   41*64(%esi,%eax),%ebx
 278
 279         movzb   48*64(%esi,%eax),%ebx
 280         movzb   49*64(%esi,%eax),%ebx
 281         movzb   56*64(%esi,%eax),%ebx
 282         movzb   57*64(%esi,%eax),%ebx
 283
 284         subl    $-128,%eax          // next slice of page (adding 128 w 8-bit immediate)
 285         testl   $512,%eax           // done with this page?
 286         jz      LTouchLoop          // no, next of four slices
 287         addl    $(4096-512),%eax    // move on to next page
 288         cmpl    %eax,%edx           // done with this chunk?
 289         jnz     LTouchLoop          // no, do next page
 290
 291 // The chunk has been pre-fetched, now copy it using non-temporal stores.
 292 // There are two copy loops, depending on whether the source is 16-byte aligned
 293 // or not.
 294
 295         addl    %edx,%esi           // increment ptrs by chunk length
 296         addl    %edx,%edi
 297         subl    %edx,%ecx           // adjust remaining length
 298         negl    %edx                // prepare loop index (counts up to 0)
 299         testl   $15,%esi            // is source 16-byte aligned?
 300         jnz     LVeryLongUnaligned  // source is not aligned
 301         jmp     LVeryLongAligned
 302
 303         .align  4,0x90              // 16-byte align inner loops
 304 LVeryLongAligned:                   // aligned loop over 128-bytes
 305         movdqa  (%esi,%edx),%xmm0
 306         movdqa  16(%esi,%edx),%xmm1
 307         movdqa  32(%esi,%edx),%xmm2
 308         movdqa  48(%esi,%edx),%xmm3
 309         movdqa  64(%esi,%edx),%xmm4
 310         movdqa  80(%esi,%edx),%xmm5
 311         movdqa  96(%esi,%edx),%xmm6
 312         movdqa  112(%esi,%edx),%xmm7
 313
 314         movntdq %xmm0,(%edi,%edx)
 315         movntdq %xmm1,16(%edi,%edx)
 316         movntdq %xmm2,32(%edi,%edx)
 317         movntdq %xmm3,48(%edi,%edx)
 318         movntdq %xmm4,64(%edi,%edx)
 319         movntdq %xmm5,80(%edi,%edx)
 320         movntdq %xmm6,96(%edi,%edx)
 321         movntdq %xmm7,112(%edi,%edx)
 322
 323         subl    $-128,%edx      // add 128 with an 8-bit immediate
 324         jnz     LVeryLongAligned
 325         jmp     LVeryLongChunkEnd
 326
 327         .align  4,0x90          // 16-byte align inner loops
 328 LVeryLongUnaligned:             // unaligned loop over 128-bytes
 329         movdqu  (%esi,%edx),%xmm0
 330         movdqu  16(%esi,%edx),%xmm1
 331         movdqu  32(%esi,%edx),%xmm2
 332         movdqu  48(%esi,%edx),%xmm3
 333         movdqu  64(%esi,%edx),%xmm4
 334         movdqu  80(%esi,%edx),%xmm5
 335         movdqu  96(%esi,%edx),%xmm6
 336         movdqu  112(%esi,%edx),%xmm7
 337
 338         movntdq %xmm0,(%edi,%edx)
 339         movntdq %xmm1,16(%edi,%edx)
 340         movntdq %xmm2,32(%edi,%edx)
 341         movntdq %xmm3,48(%edi,%edx)
 342         movntdq %xmm4,64(%edi,%edx)
 343         movntdq %xmm5,80(%edi,%edx)
 344         movntdq %xmm6,96(%edi,%edx)
 345         movntdq %xmm7,112(%edi,%edx)
 346
 347         subl    $-128,%edx              // add 128 with an 8-bit immediate
 348         jnz     LVeryLongUnaligned
 349
 350 LVeryLongChunkEnd:
 351         cmpl    $4096,%ecx              // at least another page to go?
 352         jae     LBigChunkLoop           // yes
 353
 354         sfence                          // required by non-temporal stores
 355         popl    %ebx
 356         jmp     Lrejoin                 // handle remaining (0..4095) bytes
 357
 358
 359 // Reverse moves.
 360 //      ecx = length
 361 //      esi = source ptr
 362 //      edi = dest ptr
 363
 364 LReverse:
 365         addl    %ecx,%esi               // point to end of strings
 366         addl    %ecx,%edi
 367         cmpl    $(kShort),%ecx          // long enough to bother with SSE?
 368         ja      LReverseNotShort        // yes
 369
 370 // Handle reverse short copies.
 371 //      ecx = length
 372 //      esi = one byte past end of source
 373 //      edi = one byte past end of dest
 374
 375 LReverseShort:
 376         movl    %ecx,%edx       // copy length
 377         shrl    $2,%ecx         // #words
 378         jz      3f
 379 1:
 380         subl    $4,%esi
 381         movl    (%esi),%eax
 382         subl    $4,%edi
 383         movl    %eax,(%edi)
 384         dec     %ecx
 385         jnz     1b
 386 3:
 387         andl    $3,%edx         // bytes?
 388         jz      5f
 389 4:
 390         dec     %esi
 391         movb    (%esi),%al
 392         dec     %edi
 393         movb    %al,(%edi)
 394         dec     %edx
 395         jnz     4b
 396 5:
 397         movl    8(%ebp),%eax    // get return value (dst ptr) for memcpy/memmove
 398         popl    %edi
 399         popl    %esi
 400         popl    %ebp
 401         ret
 402
 403 // Handle a reverse move long enough to justify using SSE.
 404 //      ecx = length
 405 //      esi = one byte past end of source
 406 //      edi = one byte past end of dest
 407
 408 LReverseNotShort:
 409         movl    %edi,%edx               // copy destination
 410         andl    $15,%edx                // get #bytes to align destination
 411         je      LReverseDestAligned     // already aligned
 412         subl    %edx,%ecx               // adjust length
 413 1:                                      // loop copying 1..15 bytes
 414         dec     %esi
 415         movb    (%esi),%al
 416         dec     %edi
 417         movb    %al,(%edi)
 418         dec     %edx
 419         jnz     1b
 420
 421 // Destination is now aligned.  Prepare for reverse loops.
 422
 423 LReverseDestAligned:
 424         movl    %ecx,%edx               // copy length
 425         andl    $63,%ecx                // get remaining bytes for Lshort
 426         andl    $-64,%edx               // get number of bytes we will copy in inner loop
 427         subl    %edx,%esi               // point to endpoint of copy
 428         subl    %edx,%edi
 429         testl   $15,%esi                // is source aligned too?
 430         jnz     LReverseUnalignedLoop   // no
 431         jmp     LReverseAlignedLoop     // use aligned loop
 432
 433         .align  4,0x90                  // 16-byte align inner loops
 434 LReverseAlignedLoop:                    // loop over 64-byte chunks
 435         movdqa  -16(%esi,%edx),%xmm0
 436         movdqa  -32(%esi,%edx),%xmm1
 437         movdqa  -48(%esi,%edx),%xmm2
 438         movdqa  -64(%esi,%edx),%xmm3
 439
 440         movdqa  %xmm0,-16(%edi,%edx)
 441         movdqa  %xmm1,-32(%edi,%edx)
 442         movdqa  %xmm2,-48(%edi,%edx)
 443         movdqa  %xmm3,-64(%edi,%edx)
 444
 445         subl    $64,%edx
 446         jne     LReverseAlignedLoop
 447
 448         jmp     LReverseShort           // copy remaining 0..63 bytes and done
 449
 450
 451 // Reverse, unaligned loop.  LDDQU==MOVDQU on these machines.
 452
 453         .align  4,0x90                  // 16-byte align inner loops
 454 LReverseUnalignedLoop:                  // loop over 64-byte chunks
 455         movdqu  -16(%esi,%edx),%xmm0
 456         movdqu  -32(%esi,%edx),%xmm1
 457         movdqu  -48(%esi,%edx),%xmm2
 458         movdqu  -64(%esi,%edx),%xmm3
 459
 460         movdqa  %xmm0,-16(%edi,%edx)
 461         movdqa  %xmm1,-32(%edi,%edx)
 462         movdqa  %xmm2,-48(%edi,%edx)
 463         movdqa  %xmm3,-64(%edi,%edx)
 464
 465         subl    $64,%edx
 466         jne     LReverseUnalignedLoop
 467
 468         jmp     LReverseShort           // copy remaining 0..63 bytes and done
 469
 470 PLATFUNC_DESCRIPTOR(bcopy,sse2,kHasSSE2|kCache64,kHasSupplementalSSE3)
 471 PLATFUNC_DESCRIPTOR(memcpy,sse2,kHasSSE2|kCache64,kHasSupplementalSSE3)
 472 PLATFUNC_DESCRIPTOR(memmove,sse2,kHasSSE2|kCache64,kHasSupplementalSSE3)