libsyscall/wrappers/mach_continuous_time.c

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 #include <stddef.h>
  24 #include <stdbool.h>
  25 #include <sys/types.h>
  26 #include <machine/cpu_capabilities.h>
  27 #include <mach/mach_time.h>
  28
  29 __attribute__((visibility("hidden")))
  30 uint64_t
  31 _mach_continuous_time_base(void)
  32 {
  33 #if !defined(__x86_64__) && !defined(__arm64__)
  34         // Deal with the lack of 64-bit loads on arm32 (see mach_approximate_time.s)
  35         while(1) {
  36                 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  37                 uint64_t read1, read2;
  38                 read1 = *base_ptr;
  39 #if defined(__arm__)
  40                 __asm__ volatile("dsb sy" ::: "memory");
  41 #elif defined(__i386__)
  42                 __asm__ volatile("lfence" ::: "memory");
  43 #else
  44 #error "unsupported arch"
  45 #endif
  46                 read2 = *base_ptr;
  47
  48                 if(__builtin_expect((read1 == read2), 1))
  49                         return read1;
  50         }
  51 #else // 64-bit
  52         return *(volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  53 #endif // 64-bit
  54 }
  55
  56 __attribute__((visibility("hidden")))
  57 kern_return_t
  58 _mach_continuous_hwclock(uint64_t *cont_time __unused)
  59 {
  60 #if defined(__arm64__)
  61         uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK);
  62         uint64_t timebase;
  63         if (cont_hwclock) {
  64                 __asm__ volatile("isb\n" "mrs %0, CNTPCT_EL0" : "=r"(timebase));
  65                 *cont_time = timebase;
  66                 return KERN_SUCCESS;
  67         }
  68 #endif
  69         return KERN_NOT_SUPPORTED;
  70 }
  71
  72 __attribute__((visibility("hidden")))
  73 kern_return_t
  74 _mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time)
  75 {
  76         volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
  77         volatile uint64_t read1, read2;
  78         volatile uint64_t absolute;
  79
  80         do {
  81         read1 = *base_ptr;
  82         absolute = mach_absolute_time();
  83 #if     defined(__arm__) || defined(__arm64__)
  84             /*
  85              * mach_absolute_time() contains an instruction barrier which will
  86              * prevent the speculation of read2 above this point, so we don't
  87              * need another barrier here.
  88              */
  89 #endif
  90                 read2 = *base_ptr;
  91         } while (__builtin_expect((read1 != read2), 0));
  92
  93         if (absolute_time) *absolute_time = absolute;
  94         if (cont_time) *cont_time = absolute + read1;
  95
  96         return KERN_SUCCESS;
  97 }
  98
  99 uint64_t
 100 mach_continuous_time(void)
 101 {
 102         uint64_t cont_time;
 103         if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS)
 104                 _mach_continuous_time(NULL, &cont_time);
 105         return cont_time;
 106 }
 107
 108 uint64_t
 109 mach_continuous_approximate_time(void)
 110 {
 111         /*
 112          * No retry loop here because if we use a slightly too old timebase that's
 113          * okay, we are approximate time anyway.
 114         */
 115         volatile register uint64_t time_base = _mach_continuous_time_base();
 116         return time_base + mach_approximate_time();
 117 }