libsyscall/wrappers/mach_continuous_time.c

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 #include <stddef.h>
  24 #include <stdbool.h>
  25 #include <sys/types.h>
  26 #include <machine/cpu_capabilities.h>
  27 #include <mach/mach_time.h>
  28
  29 __attribute__((visibility("hidden")))
  30 uint64_t
  31 _mach_continuous_time_base(void)
  32 {
  33 #if !defined(__x86_64__) && !defined(__arm64__)
  34         // Deal with the lack of 64-bit loads on arm32 (see mach_approximate_time.s)
  35         while (1) {
  36                 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  37                 uint64_t read1, read2;
  38                 read1 = *base_ptr;
  39 #if defined(__arm__)
  40                 __asm__ volatile ("dsb sy" ::: "memory");
  41 #elif defined(__i386__)
  42                 __asm__ volatile ("lfence" ::: "memory");
  43 #else
  44 #error "unsupported arch"
  45 #endif
  46                 read2 = *base_ptr;
  47
  48                 if (__builtin_expect((read1 == read2), 1)) {
  49                         return read1;
  50                 }
  51         }
  52 #else // 64-bit
  53         return *(volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  54 #endif // 64-bit
  55 }
  56
  57 __attribute__((visibility("hidden")))
  58 kern_return_t
  59 _mach_continuous_hwclock(uint64_t *cont_time __unused)
  60 {
  61 #if defined(__arm64__)
  62 #define ISB_SY          0xf
  63         uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK);
  64         if (cont_hwclock) {
  65                 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_HW_TIMEBASE;
  66                 __builtin_arm_isb(ISB_SY);
  67                 *cont_time = __builtin_arm_rsr64("CNTVCT_EL0") + *base_ptr;
  68                 return KERN_SUCCESS;
  69         }
  70 #endif
  71         return KERN_NOT_SUPPORTED;
  72 }
  73
  74 __attribute__((visibility("hidden")))
  75 kern_return_t
  76 _mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time)
  77 {
  78         volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  79         volatile uint64_t read1, read2;
  80         volatile uint64_t absolute;
  81
  82         do {
  83                 read1 = *base_ptr;
  84                 absolute = mach_absolute_time();
  85 #if     defined(__arm__) || defined(__arm64__)
  86                 /*
  87                  * mach_absolute_time() contains an instruction barrier which will
  88                  * prevent the speculation of read2 above this point, so we don't
  89                  * need another barrier here.
  90                  */
  91 #endif
  92                 read2 = *base_ptr;
  93         } while (__builtin_expect((read1 != read2), 0));
  94
  95         if (absolute_time) {
  96                 *absolute_time = absolute;
  97         }
  98         if (cont_time) {
  99                 *cont_time = absolute + read1;
 100         }
 101
 102         return KERN_SUCCESS;
 103 }
 104
 105 uint64_t
 106 mach_continuous_time(void)
 107 {
 108         uint64_t cont_time;
 109         if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS) {
 110                 _mach_continuous_time(NULL, &cont_time);
 111         }
 112         return cont_time;
 113 }
 114
 115 uint64_t
 116 mach_continuous_approximate_time(void)
 117 {
 118         /*
 119          * No retry loop here because if we use a slightly too old timebase that's
 120          * okay, we are approximate time anyway.
 121          */
 122         volatile register uint64_t time_base = _mach_continuous_time_base();
 123         return time_base + mach_approximate_time();
 124 }