osfmk/arm/rtclock.c

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * @APPLE_FREE_COPYRIGHT@
  33  */
  34 /*
  35  * File: arm/rtclock.c
  36  * Purpose: Routines for handling the machine dependent
  37  *   real-time clock.
  38  */
  39
  40 #include <mach/mach_types.h>
  41
  42 #include <kern/clock.h>
  43 #include <kern/thread.h>
  44 #include <kern/macro_help.h>
  45 #include <kern/spl.h>
  46 #include <kern/timer_queue.h>
  47
  48 #include <kern/host_notify.h>
  49
  50 #include <machine/commpage.h>
  51 #include <machine/machine_routines.h>
  52 #include <arm/exception.h>
  53 #include <arm/cpu_data_internal.h>
  54 #if __arm64__
  55 #include <arm64/proc_reg.h>
  56 #elif __arm__
  57 #include <arm/proc_reg.h>
  58 #else
  59 #error Unsupported arch
  60 #endif
  61 #include <arm/rtclock.h>
  62
  63 #include <IOKit/IOPlatformExpert.h>
  64 #include <libkern/OSAtomic.h>
  65
  66 #include <sys/kdebug.h>
  67
  68 #define MAX_TIMEBASE_TRIES 10
  69
  70 int rtclock_init(void);
  71
  72 static int
  73 deadline_to_decrementer(uint64_t deadline,
  74                         uint64_t now);
  75 static void
  76 timebase_callback(struct timebase_freq_t * freq);
  77
  78 #if DEVELOPMENT || DEBUG
  79 uint32_t absolute_time_validation = 1;
  80 #endif
  81
  82 /*
  83  * Configure the real-time clock device at boot
  84  */
  85 void
  86 rtclock_early_init(void)
  87 {
  88         PE_register_timebase_callback(timebase_callback);
  89 #if DEVELOPMENT || DEBUG
  90         uint32_t tmp_mv = 1;
  91         if (kern_feature_override(KF_MATV_OVRD)) {
  92                 absolute_time_validation = 0;
  93         }
  94         if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
  95                 if (tmp_mv == 0) {
  96                         absolute_time_validation = 0;
  97                 }
  98         }
  99 #endif
 100 }
 101
 102 static void
 103 timebase_callback(struct timebase_freq_t * freq)
 104 {
 105         unsigned long numer, denom;
 106         uint64_t      t64_1, t64_2;
 107         uint32_t      divisor;
 108
 109         if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
 110             freq->timebase_num < freq->timebase_den)
 111                 panic("rtclock timebase_callback: invalid constant %ld / %ld",
 112                       freq->timebase_num, freq->timebase_den);
 113
 114         denom = freq->timebase_num;
 115         numer = freq->timebase_den * NSEC_PER_SEC;
 116         // reduce by the greatest common denominator to minimize overflow
 117         if (numer > denom) {
 118                 t64_1 = numer;
 119                 t64_2 = denom;
 120         } else {
 121                 t64_1 = denom;
 122                 t64_2 = numer;
 123         }
 124         while (t64_2 != 0) {
 125                 uint64_t temp = t64_2;
 126                 t64_2 = t64_1 % t64_2;
 127                 t64_1 = temp;
 128         }
 129         numer /= t64_1;
 130         denom /= t64_1;
 131
 132         rtclock_timebase_const.numer = (uint32_t)numer;
 133         rtclock_timebase_const.denom = (uint32_t)denom;
 134         divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
 135
 136         rtclock_sec_divisor = divisor;
 137         rtclock_usec_divisor = divisor / USEC_PER_SEC;
 138 }
 139
 140 /*
 141  * Initialize the system clock device for the current cpu
 142  */
 143 int
 144 rtclock_init(void)
 145 {
 146         uint64_t     abstime;
 147         cpu_data_t * cdp;
 148
 149         clock_timebase_init();
 150         ml_init_lock_timeout();
 151
 152         cdp = getCpuDatap();
 153
 154         abstime = mach_absolute_time();
 155         cdp->rtcPop = EndOfAllTime;                                     /* Init Pop time */
 156         timer_resync_deadlines();                                       /* Start the timers going */
 157
 158         return (1);
 159 }
 160
 161 uint64_t
 162 mach_absolute_time(void)
 163 {
 164 #if DEVELOPMENT || DEBUG
 165         if (__improbable(absolute_time_validation == 1)) {
 166                 static volatile uint64_t s_last_absolute_time = 0;
 167                 uint64_t                 new_absolute_time, old_absolute_time;
 168                 int                      attempts = 0;
 169
 170                 /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
 171                  * completes before the timebase read. Were the load to complete after the
 172                  * timebase read, there would be a window for another CPU to update
 173                  * s_last_absolute_time and leave us in an inconsistent state. Consider the
 174                  * following interleaving:
 175                  *
 176                  *   Let s_last_absolute_time = t0
 177                  *   CPU0: Read timebase at t1
 178                  *   CPU1: Read timebase at t2
 179                  *   CPU1: Update s_last_absolute_time to t2
 180                  *   CPU0: Load completes
 181                  *   CPU0: Update s_last_absolute_time to t1
 182                  *
 183                  * This would cause the assertion to fail even though time did not go
 184                  * backwards. Thus, we use a dsb to guarantee completion of the load before
 185                  * the timebase read.
 186                  */
 187                 do {
 188                         attempts++;
 189                         old_absolute_time = s_last_absolute_time;
 190
 191 #if __arm64__
 192                         __asm__ volatile("dsb ld" ::: "memory");
 193 #else
 194                         OSSynchronizeIO(); // See osfmk/arm64/rtclock.c
 195 #endif
 196
 197                         new_absolute_time = ml_get_timebase();
 198                 } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
 199
 200                 if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
 201                         panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n",
 202                             new_absolute_time, old_absolute_time);
 203                 }
 204                 return new_absolute_time;
 205         } else {
 206                 return ml_get_timebase();
 207         }
 208 #else
 209         return ml_get_timebase();
 210 #endif
 211 }
 212
 213 uint64_t
 214 mach_approximate_time(void)
 215 {
 216 #if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
 217         /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
 218         return ml_get_timebase();
 219 #else
 220         processor_t processor;
 221         uint64_t    approx_time;
 222
 223         disable_preemption();
 224         processor = current_processor();
 225         approx_time = processor->last_dispatch;
 226         enable_preemption();
 227
 228         return approx_time;
 229 #endif
 230 }
 231
 232 void
 233 clock_get_system_microtime(clock_sec_t *  secs,
 234                            clock_usec_t * microsecs)
 235 {
 236         absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
 237 }
 238
 239 void
 240 clock_get_system_nanotime(clock_sec_t *  secs,
 241                           clock_nsec_t * nanosecs)
 242 {
 243         uint64_t abstime;
 244         uint64_t t64;
 245
 246         abstime = mach_absolute_time();
 247         *secs = (t64 = abstime / rtclock_sec_divisor);
 248         abstime -= (t64 * rtclock_sec_divisor);
 249
 250         *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
 251 }
 252
 253 void
 254 clock_gettimeofday_set_commpage(uint64_t abstime,
 255                                 uint64_t sec,
 256                                 uint64_t frac,
 257                                 uint64_t scale,
 258                                 uint64_t tick_per_sec)
 259 {
 260         commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
 261 }
 262
 263 void
 264 clock_timebase_info(mach_timebase_info_t info)
 265 {
 266         *info = rtclock_timebase_const;
 267 }
 268
 269 /*
 270  * Real-time clock device interrupt.
 271  */
 272 void
 273 rtclock_intr(__unused unsigned int is_user_context)
 274 {
 275         uint64_t                 abstime;
 276         cpu_data_t *             cdp;
 277         struct arm_saved_state * regs;
 278         unsigned int             user_mode;
 279         uintptr_t                pc;
 280
 281         cdp = getCpuDatap();
 282
 283         cdp->cpu_stat.timer_cnt++;
 284         cdp->cpu_stat.timer_cnt_wake++;
 285         SCHED_STATS_TIMER_POP(current_processor());
 286
 287         assert(!ml_get_interrupts_enabled());
 288
 289         abstime = mach_absolute_time();
 290
 291         if (cdp->cpu_idle_pop != 0x0ULL) {
 292                 if (( cdp->rtcPop-abstime) < cdp->cpu_idle_latency) {
 293                         cdp->cpu_idle_pop = 0x0ULL;
 294                         while (abstime < cdp->rtcPop)
 295                                 abstime = mach_absolute_time();
 296                 } else {
 297                         ClearIdlePop(FALSE);
 298                 }
 299         }
 300
 301         if ((regs = cdp->cpu_int_state)) {
 302                 pc = get_saved_state_pc(regs);
 303
 304 #if __arm64__
 305                 user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
 306 #else
 307                 user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE;
 308 #endif
 309         } else {
 310                 pc = 0;
 311                 user_mode = 0;
 312         }
 313         if (abstime >= cdp->rtcPop) {
 314                 /* Log the interrupt service latency (-ve value expected by tool) */
 315                 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 316                                           MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
 317                                           -(abstime - cdp->rtcPop),
 318                                           user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0);
 319         }
 320
 321         /* call the generic etimer */
 322         timer_intr(user_mode, pc);
 323 }
 324
 325 static int
 326 deadline_to_decrementer(uint64_t deadline,
 327                         uint64_t now)
 328 {
 329         uint64_t delt;
 330
 331         if (deadline <= now)
 332                 return DECREMENTER_MIN;
 333         else {
 334                 delt = deadline - now;
 335
 336                 return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
 337         }
 338 }
 339
 340 /*
 341  *      Request a decrementer pop
 342  */
 343 int
 344 setPop(uint64_t time)
 345 {
 346         int          delay_time;
 347         uint64_t     current_time;
 348         cpu_data_t * cdp;
 349
 350         cdp = getCpuDatap();
 351         current_time = mach_absolute_time();
 352
 353         delay_time = deadline_to_decrementer(time, current_time);
 354         cdp->rtcPop = delay_time + current_time;
 355
 356         ml_set_decrementer((uint32_t) delay_time);
 357
 358         return (delay_time);
 359 }
 360
 361 /*
 362  *      Request decrementer Idle Pop. Return true if set
 363  */
 364 boolean_t
 365 SetIdlePop(void)
 366 {
 367         int          delay_time;
 368         uint64_t     time;
 369         uint64_t     current_time;
 370         cpu_data_t * cdp;
 371
 372         cdp = getCpuDatap();
 373         current_time = mach_absolute_time();
 374
 375         if (((cdp->rtcPop < current_time) ||
 376             (cdp->rtcPop - current_time) < cdp->cpu_idle_latency))
 377                 return FALSE;
 378
 379         time = cdp->rtcPop - cdp->cpu_idle_latency;
 380
 381         delay_time = deadline_to_decrementer(time, current_time);
 382         cdp->cpu_idle_pop = delay_time + current_time;
 383         ml_set_decrementer((uint32_t) delay_time);
 384
 385         return TRUE;
 386 }
 387
 388 /*
 389  *      Clear decrementer Idle Pop
 390  */
 391 void
 392 ClearIdlePop(
 393              boolean_t wfi)
 394 {
 395 #if !__arm64__
 396 #pragma unused(wfi)
 397 #endif
 398         cpu_data_t * cdp;
 399
 400         cdp = getCpuDatap();
 401         cdp->cpu_idle_pop = 0x0ULL;
 402
 403 #if __arm64__
 404         /*
 405          * Don't update the HW timer if there's a pending
 406          * interrupt (we can lose interrupt assertion);
 407          * we want to take the interrupt right now and update
 408          * the deadline from the handler).
 409          *
 410          * ARM64_TODO: consider this more carefully.
 411          */
 412         if (!(wfi && ml_get_timer_pending()))
 413 #endif
 414         {
 415                 setPop(cdp->rtcPop);
 416         }
 417 }
 418
 419 void
 420 absolutetime_to_microtime(uint64_t       abstime,
 421                           clock_sec_t *  secs,
 422                           clock_usec_t * microsecs)
 423 {
 424         uint64_t t64;
 425
 426         *secs = t64 = abstime / rtclock_sec_divisor;
 427         abstime -= (t64 * rtclock_sec_divisor);
 428
 429         *microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
 430 }
 431
 432 void
 433 absolutetime_to_nanoseconds(uint64_t   abstime,
 434                             uint64_t * result)
 435 {
 436         uint64_t        t64;
 437
 438         *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
 439         abstime -= (t64 * rtclock_sec_divisor);
 440         *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
 441 }
 442
 443 void
 444 nanoseconds_to_absolutetime(uint64_t   nanosecs,
 445                             uint64_t * result)
 446 {
 447         uint64_t        t64;
 448
 449         *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
 450         nanosecs -= (t64 * NSEC_PER_SEC);
 451         *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
 452 }
 453
 454 void
 455 nanotime_to_absolutetime(clock_sec_t  secs,
 456                          clock_nsec_t nanosecs,
 457                          uint64_t *   result)
 458 {
 459         *result = ((uint64_t) secs * rtclock_sec_divisor) +
 460         ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
 461 }
 462
 463 void
 464 clock_interval_to_absolutetime_interval(uint32_t   interval,
 465                                         uint32_t   scale_factor,
 466                                         uint64_t * result)
 467 {
 468         uint64_t nanosecs = (uint64_t) interval * scale_factor;
 469         uint64_t t64;
 470
 471         *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
 472         nanosecs -= (t64 * NSEC_PER_SEC);
 473         *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
 474 }
 475
 476 void
 477 machine_delay_until(uint64_t interval,
 478                     uint64_t deadline)
 479 {
 480 #pragma unused(interval)
 481         uint64_t now;
 482
 483         do {
 484 #if     __ARM_ENABLE_WFE_
 485 #if __arm64__
 486                 if (arm64_wfe_allowed())
 487 #endif /* __arm64__ */
 488                 {
 489                         __builtin_arm_wfe();
 490                 }
 491 #endif /* __ARM_ENABLE_WFE_ */
 492
 493                 now = mach_absolute_time();
 494         } while (now < deadline);
 495 }