src/cachecontrol/arm64/cache.s

   1 /*
   2  * Copyright (c) 2011-2017 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include <mach/arm/syscall_sw.h>
  25
  26 #define MMU_I_CLINE     6               // cache line size as 1<<MMU_I_CLINE (64)
  27
  28 /* void sys_icache_invalidate(void *start, size_t length) */
  29 .globl  _sys_icache_invalidate
  30 .p2align        2
  31 _sys_icache_invalidate:
  32         // see InvalidatePoU_IcacheRegion() in xnu/osfmk/arm64/caches_asm.s
  33         cbz             x1, 2f                                                  // length > 0 ?
  34         and             x8, x0, #~((1<<MMU_I_CLINE)-1)  // cacheline align address
  35         and             x9, x0, #((1<<MMU_I_CLINE)-1)   // extend length by alignment
  36         add             x9, x1, x9
  37         sub             x9, x9, #1
  38         mov             x10, #-1
  39         eor             x9, x10, x9, lsr #MMU_I_CLINE   // compute cacheline counter
  40 1:
  41         ic              ivau, x8                                                // invalidate icache line
  42         add             x8, x8, #1<<MMU_I_CLINE                 // next cacheline address
  43         add             x9, x9, #1                                              // decrement cacheline counter
  44         cbnz    x9, 1b
  45         dsb             ish
  46         isb
  47 2:
  48         ret
  49
  50 /* void sys_dcache_flush(void *start, size_t length) */
  51 .globl  _sys_dcache_flush
  52 .p2align        2
  53 _sys_dcache_flush:
  54         // see FlushPoC_DcacheRegion() in xnu/osfmk/arm64/caches_asm.s
  55         dsb             ish                                                             // noop, we are fully coherent
  56         ret
  57
  58 #if 0
  59 // Above generated by clang from:
  60 static void __attribute((used))
  61 sys_icache_invalidate(uintptr_t start, size_t length)
  62 {
  63         if (!length) return;
  64         uintptr_t addr = start & ~((1 << MMU_I_CLINE) - 1);
  65         length += start & ((1 << MMU_I_CLINE) - 1);
  66         size_t count = ((length - 1) >> MMU_I_CLINE) + 1;
  67         while (count--) {
  68                 asm("ic ivau, %[addr]" :: [addr] "r" (addr) : "memory");
  69                 addr += (1 << MMU_I_CLINE);
  70         }
  71         asm volatile("dsb ish\n\tisb" ::: "memory");
  72 }
  73 #endif
  74