X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/060df5ea7c632b1ac8cc8aac1fb59758165c2084..527f99514973766e9c0382a4d8550dfb00f54939:/osfmk/i386/rtclock.c?ds=sidebyside diff --git a/osfmk/i386/rtclock.c b/osfmk/i386/rtclock.c index 72b1f556f..c8abc4b1e 100644 --- a/osfmk/i386/rtclock.c +++ b/osfmk/i386/rtclock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2010 Apple Inc. All rights reserved. + * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -39,8 +39,6 @@ * the cpu clock counted by the timestamp MSR. */ -#include -#include #include @@ -52,291 +50,61 @@ #include #include #include -#include +#include #include #include #include /* for kernel_map */ -#include #include #include #include #include #include #include +#include #include #include -#include #include #include #include #include #include -#include - +#include #define UI_CPUFREQ_ROUNDING_FACTOR 10000000 -int rtclock_config(void); - int rtclock_init(void); uint64_t tsc_rebase_abs_time = 0; -void rtclock_intr(x86_saved_state_t *regs); - static void rtc_set_timescale(uint64_t cycles); static uint64_t rtc_export_speed(uint64_t cycles); -rtc_nanotime_t rtc_nanotime_info = {0,0,0,0,1,0}; - -static uint64_t rtc_decrementer_min; -static uint64_t rtc_decrementer_max; - -static uint64_t -deadline_to_decrementer( - uint64_t deadline, - uint64_t now) -{ - uint64_t delta; - - if (deadline <= now) - return rtc_decrementer_min; - else { - delta = deadline - now; - return MIN(MAX(rtc_decrementer_min,delta),rtc_decrementer_max); - } -} - -static inline uint64_t -_absolutetime_to_tsc(uint64_t ns) -{ - uint32_t generation; - uint64_t tsc; - - do { - generation = rtc_nanotime_info.generation; - tsc = tmrCvt(ns - rtc_nanotime_info.ns_base, tscFCvtn2t) - + rtc_nanotime_info.tsc_base; - } while (generation == 0 || - generation != rtc_nanotime_info.generation); - - return tsc; -} - -/* - * Regular local APIC timer case: - */ -static void -rtc_lapic_config_timer(void) -{ - lapic_config_timer(TRUE, one_shot, divide_by_1); -} -static uint64_t -rtc_lapic_set_timer(uint64_t deadline, uint64_t now) -{ - uint64_t count; - uint64_t set = 0; - - if (deadline > 0) { - /* - * Convert delta to bus ticks - * - time now is not relevant - */ - count = deadline_to_decrementer(deadline, now); - set = now + count; - lapic_set_timer_fast((uint32_t) tmrCvt(count, busFCvtn2t)); - } else { - lapic_set_timer(FALSE, one_shot, divide_by_1, 0); - } - return set; -} - -/* - * TSC-deadline timer case: - */ -static void -rtc_lapic_config_tsc_deadline_timer(void) -{ - lapic_config_tsc_deadline_timer(); -} -static uint64_t -rtc_lapic_set_tsc_deadline_timer(uint64_t deadline, uint64_t now) -{ - uint64_t set = 0; - - if (deadline > 0) { - /* - * Convert to TSC - */ - set = now + deadline_to_decrementer(deadline, now); - lapic_set_tsc_deadline_timer(_absolutetime_to_tsc(set)); - } else { - lapic_set_tsc_deadline_timer(0); - } - return set; -} - -/* - * Definitions for timer operations table - */ -typedef struct { - void (*config)(void); - uint64_t (*set) (uint64_t, uint64_t); -} rtc_timer_t; - -rtc_timer_t rtc_timer_lapic = { - rtc_lapic_config_timer, - rtc_lapic_set_timer -}; - -rtc_timer_t rtc_timer_tsc_deadline = { - rtc_lapic_config_tsc_deadline_timer, - rtc_lapic_set_tsc_deadline_timer -}; - -rtc_timer_t *rtc_timer = &rtc_timer_lapic; /* defaults to LAPIC timer */ - -/* - * rtc_timer_init() is called at startup on the boot processor only. - */ -static void -rtc_timer_init(void) -{ - int TSC_deadline_timer = 0; - - /* See whether we can use the local apic in TSC-deadline mode */ - if ((cpuid_features() & CPUID_FEATURE_TSCTMR)) { - TSC_deadline_timer = 1; - PE_parse_boot_argn("TSC_deadline_timer", &TSC_deadline_timer, - sizeof(TSC_deadline_timer)); - printf("TSC Deadline Timer supported %s enabled\n", - TSC_deadline_timer ? "and" : "but not"); - } - - if (TSC_deadline_timer) { - rtc_timer = &rtc_timer_tsc_deadline; - rtc_decrementer_max = UINT64_MAX; /* effectively none */ - /* - * The min could be as low as 1nsec, - * but we're being conservative for now and making it the same - * as for the local apic timer. - */ - rtc_decrementer_min = 1*NSEC_PER_USEC; /* 1 usec */ - } else { - /* - * Compute the longest interval using LAPIC timer. - */ - rtc_decrementer_max = tmrCvt(0x7fffffffULL, busFCvtt2n); - kprintf("maxDec: %lld\n", rtc_decrementer_max); - rtc_decrementer_min = 1*NSEC_PER_USEC; /* 1 usec */ - } - - /* Point LAPIC interrupts to hardclock() */ - lapic_set_timer_func((i386_intr_func_t) rtclock_intr); -} - -static inline uint64_t -rtc_timer_set(uint64_t deadline, uint64_t now) -{ - return rtc_timer->set(deadline, now); -} - void rtc_timer_start(void) { /* * Force a complete re-evaluation of timer deadlines. */ - etimer_resync_deadlines(); -} - -/* - * tsc_to_nanoseconds: - * - * Basic routine to convert a raw 64 bit TSC value to a - * 64 bit nanosecond value. The conversion is implemented - * based on the scale factor and an implicit 32 bit shift. - */ -static inline uint64_t -_tsc_to_nanoseconds(uint64_t value) -{ -#if defined(__i386__) - asm volatile("movl %%edx,%%esi ;" - "mull %%ecx ;" - "movl %%edx,%%edi ;" - "movl %%esi,%%eax ;" - "mull %%ecx ;" - "addl %%edi,%%eax ;" - "adcl $0,%%edx " - : "+A" (value) - : "c" (rtc_nanotime_info.scale) - : "esi", "edi"); -#elif defined(__x86_64__) - asm volatile("mul %%rcx;" - "shrq $32, %%rax;" - "shlq $32, %%rdx;" - "orq %%rdx, %%rax;" - : "=a"(value) - : "a"(value), "c"(rtc_nanotime_info.scale) - : "rdx", "cc" ); -#else -#error Unsupported architecture -#endif - - return (value); + x86_lcpu()->rtcDeadline = EndOfAllTime; + timer_resync_deadlines(); } static inline uint32_t _absolutetime_to_microtime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs) { uint32_t remain; -#if defined(__i386__) - asm volatile( - "divl %3" - : "=a" (*secs), "=d" (remain) - : "A" (abstime), "r" (NSEC_PER_SEC)); - asm volatile( - "divl %3" - : "=a" (*microsecs) - : "0" (remain), "d" (0), "r" (NSEC_PER_USEC)); -#elif defined(__x86_64__) *secs = abstime / (uint64_t)NSEC_PER_SEC; remain = (uint32_t)(abstime % (uint64_t)NSEC_PER_SEC); *microsecs = remain / NSEC_PER_USEC; -#else -#error Unsupported architecture -#endif return remain; } static inline void _absolutetime_to_nanotime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *nanosecs) { -#if defined(__i386__) - asm volatile( - "divl %3" - : "=a" (*secs), "=d" (*nanosecs) - : "A" (abstime), "r" (NSEC_PER_SEC)); -#elif defined(__x86_64__) *secs = abstime / (uint64_t)NSEC_PER_SEC; *nanosecs = (clock_usec_t)(abstime % (uint64_t)NSEC_PER_SEC); -#else -#error Unsupported architecture -#endif -} - -/* - * Configure the real-time clock device. Return success (1) - * or failure (0). - */ - -int -rtclock_config(void) -{ - /* nothing to do */ - return (1); } - /* * Nanotime/mach_absolutime_time * ----------------------------- @@ -359,7 +127,7 @@ rtclock_config(void) * be guaranteed by the caller. */ static inline void -rtc_nanotime_set_commpage(rtc_nanotime_t *rntp) +rtc_nanotime_set_commpage(pal_rtc_nanotime_t *rntp) { commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift); } @@ -370,18 +138,18 @@ rtc_nanotime_set_commpage(rtc_nanotime_t *rntp) * Intialize the nanotime info from the base time. */ static inline void -_rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base) +_rtc_nanotime_init(pal_rtc_nanotime_t *rntp, uint64_t base) { uint64_t tsc = rdtsc64(); - _rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp); + _pal_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp); } -static void +void rtc_nanotime_init(uint64_t base) { - _rtc_nanotime_init(&rtc_nanotime_info, base); - rtc_nanotime_set_commpage(&rtc_nanotime_info); + _rtc_nanotime_init(&pal_rtc_nanotime_info, base); + rtc_nanotime_set_commpage(&pal_rtc_nanotime_info); } /* @@ -396,8 +164,7 @@ rtc_nanotime_init_commpage(void) { spl_t s = splclock(); - rtc_nanotime_set_commpage(&rtc_nanotime_info); - + rtc_nanotime_set_commpage(&pal_rtc_nanotime_info); splx(s); } @@ -410,13 +177,7 @@ rtc_nanotime_init_commpage(void) static inline uint64_t rtc_nanotime_read(void) { - -#if CONFIG_EMBEDDED - if (gPEClockFrequencyInfo.timebase_frequency_hz > SLOW_TSC_THRESHOLD) - return _rtc_nanotime_read(&rtc_nanotime_info, 1); /* slow processor */ - else -#endif - return _rtc_nanotime_read(&rtc_nanotime_info, 0); /* assume fast processor */ + return _rtc_nanotime_read(&pal_rtc_nanotime_info); } /* @@ -429,45 +190,43 @@ rtc_nanotime_read(void) void rtc_clock_napped(uint64_t base, uint64_t tsc_base) { - rtc_nanotime_t *rntp = &rtc_nanotime_info; + pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; uint64_t oldnsecs; uint64_t newnsecs; uint64_t tsc; assert(!ml_get_interrupts_enabled()); tsc = rdtsc64(); - oldnsecs = rntp->ns_base + _tsc_to_nanoseconds(tsc - rntp->tsc_base); - newnsecs = base + _tsc_to_nanoseconds(tsc - tsc_base); + oldnsecs = rntp->ns_base + _rtc_tsc_to_nanoseconds(tsc - rntp->tsc_base, rntp); + newnsecs = base + _rtc_tsc_to_nanoseconds(tsc - tsc_base, rntp); /* * Only update the base values if time using the new base values * is later than the time using the old base values. */ if (oldnsecs < newnsecs) { - _rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp); + _pal_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp); rtc_nanotime_set_commpage(rntp); } } - /* * Invoked from power management to correct the SFLM TSC entry drift problem: - * a small delta is added to the tsc_base. This is equivalent to nudging time - * backwards. We require this of the order of a TSC quantum which won't cause - * callers of mach_absolute_time() to see time going backwards! + * a small delta is added to the tsc_base. This is equivalent to nudgin time + * backwards. We require this to be on the order of a TSC quantum which won't + * cause callers of mach_absolute_time() to see time going backwards! */ void rtc_clock_adjust(uint64_t tsc_base_delta) { - rtc_nanotime_t *rntp = &rtc_nanotime_info; + pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; - assert(!ml_get_interrupts_enabled()); - assert(tsc_base_delta < 100ULL); /* i.e. it's small */ - _rtc_nanotime_adjust(tsc_base_delta, rntp); - rtc_nanotime_set_commpage(rntp); + assert(!ml_get_interrupts_enabled()); + assert(tsc_base_delta < 100ULL); /* i.e. it's small */ + _rtc_nanotime_adjust(tsc_base_delta, rntp); + rtc_nanotime_set_commpage(rntp); } - void rtc_clock_stepping(__unused uint32_t new_frequency, __unused uint32_t old_frequency) @@ -485,9 +244,9 @@ rtc_clock_stepped(__unused uint32_t new_frequency, /* * rtc_sleep_wakeup: * - * Invoked from power manageent when we have awoken from a sleep (S3) - * and the TSC has been reset. The nanotime data is updated based on - * the passed in value. + * Invoked from power management when we have awoken from a sleep (S3) + * and the TSC has been reset, or from Deep Idle (S0) sleep when the TSC + * has progressed. The nanotime data is updated based on the passed-in value. * * The caller must guarantee non-reentrancy. */ @@ -496,7 +255,7 @@ rtc_sleep_wakeup( uint64_t base) { /* Set fixed configuration for lapic timers */ - rtc_timer->config(); + rtc_timer->rtc_config(); /* * Reset nanotime. @@ -506,6 +265,21 @@ rtc_sleep_wakeup( rtc_nanotime_init(base); } +void +rtc_decrementer_configure(void) { + rtc_timer->rtc_config(); +} +/* + * rtclock_early_init() is called very early at boot to + * establish mach_absolute_time() and set it to zero. + */ +void +rtclock_early_init(void) +{ + assert(tscFreq); + rtc_set_timescale(tscFreq); +} + /* * Initialize the real-time clock device. * In addition, various variables used to support the clock are initialized. @@ -520,7 +294,6 @@ rtclock_init(void) if (cpu_number() == master_cpu) { assert(tscFreq); - rtc_set_timescale(tscFreq); /* * Adjust and set the exported cpu speed. @@ -537,11 +310,11 @@ rtclock_init(void) rtc_timer_init(); clock_timebase_init(); ml_init_lock_timeout(); + ml_init_delay_spin_threshold(10); } - /* Set fixed configuration for lapic timers */ - rtc_timer->config(); - + /* Set fixed configuration for lapic timers */ + rtc_timer->rtc_config(); rtc_timer_start(); return (1); @@ -553,16 +326,29 @@ rtclock_init(void) static void rtc_set_timescale(uint64_t cycles) { - rtc_nanotime_t *rntp = &rtc_nanotime_info; + pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; + uint32_t shift = 0; + + /* the "scale" factor will overflow unless cycles>SLOW_TSC_THRESHOLD */ + + while ( cycles <= SLOW_TSC_THRESHOLD) { + shift++; + cycles <<= 1; + } + rntp->scale = (uint32_t)(((uint64_t)NSEC_PER_SEC << 32) / cycles); - if (cycles <= SLOW_TSC_THRESHOLD) - rntp->shift = (uint32_t)cycles; - else - rntp->shift = 32; + rntp->shift = shift; + /* + * On some platforms, the TSC is not reset at warm boot. But the + * rebase time must be relative to the current boot so we can't use + * mach_absolute_time(). Instead, we convert the TSC delta since boot + * to nanoseconds. + */ if (tsc_rebase_abs_time == 0) - tsc_rebase_abs_time = mach_absolute_time(); + tsc_rebase_abs_time = _rtc_tsc_to_nanoseconds( + rdtsc64() - tsc_at_boot, rntp); rtc_nanotime_init(0); } @@ -570,8 +356,12 @@ rtc_set_timescale(uint64_t cycles) static uint64_t rtc_export_speed(uint64_t cyc_per_sec) { + pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; uint64_t cycles; + if (rntp->shift != 0 ) + printf("Slow TSC, rtc_nanotime.shift == %d\n", rntp->shift); + /* Round: */ cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2)) / UI_CPUFREQ_ROUNDING_FACTOR) @@ -612,21 +402,9 @@ clock_get_system_nanotime( } void -clock_gettimeofday_set_commpage( - uint64_t abstime, - uint64_t epoch, - uint64_t offset, - clock_sec_t *secs, - clock_usec_t *microsecs) +clock_gettimeofday_set_commpage(uint64_t abstime, uint64_t sec, uint64_t frac, uint64_t scale, uint64_t tick_per_sec) { - uint64_t now = abstime + offset; - uint32_t remain; - - remain = _absolutetime_to_microtime(now, secs, microsecs); - - *secs += (clock_sec_t)epoch; - - commpage_set_timestamp(abstime - remain, *secs); + commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec); } void @@ -668,7 +446,7 @@ rtclock_intr( } /* call the generic etimer */ - etimer_intr(user_mode, rip); + timer_intr(user_mode, rip); } @@ -677,29 +455,25 @@ rtclock_intr( */ uint64_t -setPop( - uint64_t time) +setPop(uint64_t time) { - uint64_t now; - uint64_t pop; + uint64_t now; + uint64_t pop; /* 0 and EndOfAllTime are special-cases for "clear the timer" */ - if (time == 0 || time == EndOfAllTime) { + if (time == 0 || time == EndOfAllTime ) { time = EndOfAllTime; now = 0; - pop = rtc_timer_set(0, 0); + pop = rtc_timer->rtc_set(0, 0); } else { - now = rtc_nanotime_read(); - pop = rtc_timer_set(time, now); + now = rtc_nanotime_read(); /* The time in nanoseconds */ + pop = rtc_timer->rtc_set(time, now); } - /* Record actual deadline set */ + /* Record requested and actual deadlines set */ x86_lcpu()->rtcDeadline = time; - x86_lcpu()->rtcPop = pop; + x86_lcpu()->rtcPop = pop; - /* - * Pass back the delta we set - */ return pop - now; } @@ -709,6 +483,12 @@ mach_absolute_time(void) return rtc_nanotime_read(); } +uint64_t +mach_approximate_time(void) +{ + return rtc_nanotime_read(); +} + void clock_interval_to_absolutetime_interval( uint32_t interval, @@ -727,15 +507,6 @@ absolutetime_to_microtime( _absolutetime_to_microtime(abstime, secs, microsecs); } -void -absolutetime_to_nanotime( - uint64_t abstime, - clock_sec_t *secs, - clock_nsec_t *nanosecs) -{ - _absolutetime_to_nanotime(abstime, secs, nanosecs); -} - void nanotime_to_absolutetime( clock_sec_t secs, @@ -763,12 +534,11 @@ nanoseconds_to_absolutetime( void machine_delay_until( + uint64_t interval, uint64_t deadline) { - uint64_t now; - - do { + (void)interval; + while (mach_absolute_time() < deadline) { cpu_pause(); - now = mach_absolute_time(); - } while (now < deadline); + } }