src/cachecontrol/arm64/cache.s

   1 /*
   2  * Copyright (c) 2011-2017 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include <mach/arm/syscall_sw.h>
  25 #include <mach/arm64/asm.h>
  26 #include <machine/cpu_capabilities.h>
  27
  28 #define MMU_I_CLINE     6               // cache line size as 1<<MMU_I_CLINE (64)
  29
  30 /* void sys_icache_invalidate(void *start, size_t length) */
  31 .globl  _sys_icache_invalidate
  32 .p2align        2
  33 _sys_icache_invalidate:
  34         // see InvalidatePoU_IcacheRegion() in xnu/osfmk/arm64/caches_asm.s
  35         cbz             x1, 2f                                                  // length > 0 ?
  36         MOV64   x8, _COMM_PAGE_CPU_CAPABILITIES
  37         ldr             w8, [x8]
  38         and             x9, x0, #~((1<<MMU_I_CLINE)-1)  // cacheline align address
  39         and             x10, x0, #((1<<MMU_I_CLINE)-1)  // extend length by alignment
  40         add             x10, x1, x10
  41         sub             x10, x10, #1
  42         mov             x11, #-1
  43         eor             x10, x11, x10, lsr #MMU_I_CLINE // compute cacheline counter
  44         dsb             ish
  45 1:
  46         ic              ivau, x9                                                // invalidate icache line
  47         add             x9, x9, #1<<MMU_I_CLINE                 // next cacheline address
  48         adds    x10, x10, #1                                    // decrement cacheline counter
  49         b.ne    1b
  50         dsb             ish
  51         tbnz    w8, kHasICDSBShift, 2f
  52         isb
  53 2:
  54         ret
  55
  56 /* void sys_dcache_flush(void *start, size_t length) */
  57 .globl  _sys_dcache_flush
  58 .p2align        2
  59 _sys_dcache_flush:
  60         // see FlushPoC_DcacheRegion() in xnu/osfmk/arm64/caches_asm.s
  61         dsb             ish                                                             // noop, we are fully coherent
  62         ret
  63
  64 #if 0
  65 // Above based on output generated by clang from:
  66 static void __attribute((used))
  67 sys_icache_invalidate(uintptr_t start, size_t length)
  68 {
  69         if (!length) return;
  70         boolean_t hasICDSB = (*(uint32_t*)(uintptr_t)_COMM_PAGE_CPU_CAPABILITIES) & kHasICDSB;
  71         uintptr_t addr = start & ~((1 << MMU_I_CLINE) - 1);
  72         length += start & ((1 << MMU_I_CLINE) - 1);
  73         size_t count = ((length - 1) >> MMU_I_CLINE) + 1;
  74         asm volatile("dsb ish" ::: "memory");
  75         while (count--) {
  76                 asm("ic ivau, %[addr]" :: [addr] "r" (addr) : "memory");
  77                 addr += (1 << MMU_I_CLINE);
  78         }
  79         if (hasICDSB) {
  80                 asm volatile("dsb ish" ::: "memory");
  81         } else {
  82                 asm volatile("dsb ish" ::: "memory");
  83                 asm volatile("isb" ::: "memory");
  84         }
  85 }
  86
  87 cbz    x1, 0x44
  88 mov    x8, #0xfffff0000
  89 movk   x8, #0xc020
  90 ldr    w8, [x8]
  91 and    x9, x0, #0xffffffffffffffc0
  92 and    x10, x0, #0x3f
  93 add    x10, x1, x10
  94 sub    x10, x10, #0x1
  95 mov    x11, #-0x1
  96 eor    x10, x11, x10, lsr #6
  97 ic     ivau, x9
  98 add    x9, x9, #0x40
  99 adds   x10, x10, #0x1
 100 b.ne   0x28
 101 dsb    ish
 102 tbnz   w8, #0x2, 0x44
 103 isb
 104 ret
 105
 106 #endif
 107