libsyscall/wrappers/mach_continuous_time.c

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 #include <stddef.h>
  24 #include <stdbool.h>
  25 #include <sys/types.h>
  26 #include <machine/cpu_capabilities.h>
  27 #include <mach/mach_time.h>
  28
  29 __attribute__((visibility("hidden")))
  30 uint64_t
  31 _mach_continuous_time_base(void)
  32 {
  33 #if !defined(__x86_64__) && !defined(__arm64__)
  34         // Deal with the lack of 64-bit loads on arm32 (see mach_approximate_time.s)
  35         while (1) {
  36                 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  37                 uint64_t read1, read2;
  38                 read1 = *base_ptr;
  39 #if defined(__arm__)
  40                 __asm__ volatile ("dsb sy" ::: "memory");
  41 #elif defined(__i386__)
  42                 __asm__ volatile ("lfence" ::: "memory");
  43 #else
  44 #error "unsupported arch"
  45 #endif
  46                 read2 = *base_ptr;
  47
  48                 if (__builtin_expect((read1 == read2), 1)) {
  49                         return read1;
  50                 }
  51         }
  52 #else // 64-bit
  53         return *(volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  54 #endif // 64-bit
  55 }
  56
  57 __attribute__((visibility("hidden")))
  58 kern_return_t
  59 _mach_continuous_hwclock(uint64_t *cont_time __unused)
  60 {
  61 #if defined(__arm64__)
  62         uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK);
  63         uint64_t timebase;
  64         if (cont_hwclock) {
  65                 __asm__ volatile ("isb\n" "mrs %0, CNTPCT_EL0" : "=r"(timebase));
  66                 *cont_time = timebase;
  67                 return KERN_SUCCESS;
  68         }
  69 #endif
  70         return KERN_NOT_SUPPORTED;
  71 }
  72
  73 __attribute__((visibility("hidden")))
  74 kern_return_t
  75 _mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time)
  76 {
  77         volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  78         volatile uint64_t read1, read2;
  79         volatile uint64_t absolute;
  80
  81         do {
  82                 read1 = *base_ptr;
  83                 absolute = mach_absolute_time();
  84 #if     defined(__arm__) || defined(__arm64__)
  85                 /*
  86                  * mach_absolute_time() contains an instruction barrier which will
  87                  * prevent the speculation of read2 above this point, so we don't
  88                  * need another barrier here.
  89                  */
  90 #endif
  91                 read2 = *base_ptr;
  92         } while (__builtin_expect((read1 != read2), 0));
  93
  94         if (absolute_time) {
  95                 *absolute_time = absolute;
  96         }
  97         if (cont_time) {
  98                 *cont_time = absolute + read1;
  99         }
 100
 101         return KERN_SUCCESS;
 102 }
 103
 104 uint64_t
 105 mach_continuous_time(void)
 106 {
 107         uint64_t cont_time;
 108         if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS) {
 109                 _mach_continuous_time(NULL, &cont_time);
 110         }
 111         return cont_time;
 112 }
 113
 114 uint64_t
 115 mach_continuous_approximate_time(void)
 116 {
 117         /*
 118          * No retry loop here because if we use a slightly too old timebase that's
 119          * okay, we are approximate time anyway.
 120          */
 121         volatile register uint64_t time_base = _mach_continuous_time_base();
 122         return time_base + mach_approximate_time();
 123 }