/*
- * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
+ *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
- *
+ *
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
+ *
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* the cpu clock counted by the timestamp MSR.
*/
-#include <platforms.h>
-#include <mach_kdb.h>
#include <mach/mach_types.h>
#include <kern/misc_protos.h>
#include <kern/spl.h>
#include <kern/assert.h>
+#include <kern/timer_queue.h>
#include <mach/vm_prot.h>
#include <vm/pmap.h>
-#include <vm/vm_kern.h> /* for kernel_map */
-#include <i386/ipl.h>
+#include <vm/vm_kern.h> /* for kernel_map */
#include <architecture/i386/pio.h>
-#include <i386/misc_protos.h>
-#include <i386/proc_reg.h>
#include <i386/machine_cpu.h>
-#include <i386/lapic.h>
#include <i386/cpuid.h>
-#include <i386/cpu_data.h>
#include <i386/cpu_threads.h>
-#include <i386/perfmon.h>
+#include <i386/mp.h>
#include <i386/machine_routines.h>
+#include <i386/pal_routines.h>
+#include <i386/proc_reg.h>
+#include <i386/misc_protos.h>
#include <pexpert/pexpert.h>
#include <machine/limits.h>
#include <machine/commpage.h>
#include <sys/kdebug.h>
#include <i386/tsc.h>
-#include <i386/rtclock.h>
-
-#define NSEC_PER_HZ (NSEC_PER_SEC / 100) /* nsec per tick */
-
-#define UI_CPUFREQ_ROUNDING_FACTOR 10000000
-
-int rtclock_config(void);
-
-int rtclock_init(void);
-
-uint64_t rtc_decrementer_min;
+#include <i386/rtclock_protos.h>
+#define UI_CPUFREQ_ROUNDING_FACTOR 10000000
-void rtclock_intr(x86_saved_state_t *regs);
-static uint64_t maxDec; /* longest interval our hardware timer can handle (nsec) */
+int rtclock_init(void);
-/* XXX this should really be in a header somewhere */
-extern clock_timer_func_t rtclock_timer_expire;
+uint64_t tsc_rebase_abs_time = 0;
-static void rtc_set_timescale(uint64_t cycles);
-static uint64_t rtc_export_speed(uint64_t cycles);
-
-rtc_nanotime_t rtc_nanotime_info = {0,0,0,0,1,0};
-
-/*
- * tsc_to_nanoseconds:
- *
- * Basic routine to convert a raw 64 bit TSC value to a
- * 64 bit nanosecond value. The conversion is implemented
- * based on the scale factor and an implicit 32 bit shift.
- */
-static inline uint64_t
-_tsc_to_nanoseconds(uint64_t value)
-{
- asm volatile("movl %%edx,%%esi ;"
- "mull %%ecx ;"
- "movl %%edx,%%edi ;"
- "movl %%esi,%%eax ;"
- "mull %%ecx ;"
- "addl %%edi,%%eax ;"
- "adcl $0,%%edx "
- : "+A" (value)
- : "c" (current_cpu_datap()->cpu_nanotime->scale)
- : "esi", "edi");
-
- return (value);
-}
-
-static uint32_t
-deadline_to_decrementer(
- uint64_t deadline,
- uint64_t now)
-{
- uint64_t delta;
-
- if (deadline <= now)
- return rtc_decrementer_min;
- else {
- delta = deadline - now;
- return MIN(MAX(rtc_decrementer_min,delta),maxDec);
- }
-}
+static void rtc_set_timescale(uint64_t cycles);
+static uint64_t rtc_export_speed(uint64_t cycles);
void
-rtc_lapic_start_ticking(void)
+rtc_timer_start(void)
{
- x86_lcpu_t *lcpu = x86_lcpu();
-
/*
* Force a complete re-evaluation of timer deadlines.
*/
- lcpu->rtcPop = EndOfAllTime;
- etimer_resync_deadlines();
+ x86_lcpu()->rtcDeadline = EndOfAllTime;
+ timer_resync_deadlines();
}
-/*
- * Configure the real-time clock device. Return success (1)
- * or failure (0).
- */
-
-int
-rtclock_config(void)
+static inline uint32_t
+_absolutetime_to_microtime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs)
{
- /* nothing to do */
- return (1);
+ uint32_t remain;
+ *secs = abstime / (uint64_t)NSEC_PER_SEC;
+ remain = (uint32_t)(abstime % (uint64_t)NSEC_PER_SEC);
+ *microsecs = remain / NSEC_PER_USEC;
+ return remain;
}
+static inline void
+_absolutetime_to_nanotime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *nanosecs)
+{
+ *secs = abstime / (uint64_t)NSEC_PER_SEC;
+ *nanosecs = (clock_usec_t)(abstime % (uint64_t)NSEC_PER_SEC);
+}
/*
* Nanotime/mach_absolutime_time
* used to maintain a monotonic clock, adjusted from an outside reference as needed.
*
* The kernel maintains nanotime information recording:
- * - the ratio of tsc to nanoseconds
+ * - the ratio of tsc to nanoseconds
* with this ratio expressed as a 32-bit scale and shift
* (power of 2 divider);
* - { tsc_base, ns_base } pair of corresponding timestamps.
*
- * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage
+ * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage
* for the userspace nanotime routine to read.
*
* All of the routines which update the nanotime data are non-reentrant. This must
* be guaranteed by the caller.
*/
static inline void
-rtc_nanotime_set_commpage(rtc_nanotime_t *rntp)
+rtc_nanotime_set_commpage(pal_rtc_nanotime_t *rntp)
{
commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift);
}
* Intialize the nanotime info from the base time.
*/
static inline void
-_rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base)
+_rtc_nanotime_init(pal_rtc_nanotime_t *rntp, uint64_t base)
{
- uint64_t tsc = rdtsc64();
+ uint64_t tsc = rdtsc64();
- _rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
+ _pal_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
}
-static void
+void
rtc_nanotime_init(uint64_t base)
{
- rtc_nanotime_t *rntp = current_cpu_datap()->cpu_nanotime;
-
- _rtc_nanotime_init(rntp, base);
- rtc_nanotime_set_commpage(rntp);
+ _rtc_nanotime_init(&pal_rtc_nanotime_info, base);
+ rtc_nanotime_set_commpage(&pal_rtc_nanotime_info);
}
/*
void
rtc_nanotime_init_commpage(void)
{
- spl_t s = splclock();
-
- rtc_nanotime_set_commpage(current_cpu_datap()->cpu_nanotime);
+ spl_t s = splclock();
+ rtc_nanotime_set_commpage(&pal_rtc_nanotime_info);
splx(s);
}
static inline uint64_t
rtc_nanotime_read(void)
{
-
-#if CONFIG_EMBEDDED
- if (gPEClockFrequencyInfo.timebase_frequency_hz > SLOW_TSC_THRESHOLD)
- return _rtc_nanotime_read(current_cpu_datap()->cpu_nanotime, 1); /* slow processor */
- else
-#endif
- return _rtc_nanotime_read(current_cpu_datap()->cpu_nanotime, 0); /* assume fast processor */
+ return _rtc_nanotime_read(&pal_rtc_nanotime_info);
}
/*
void
rtc_clock_napped(uint64_t base, uint64_t tsc_base)
{
- rtc_nanotime_t *rntp = current_cpu_datap()->cpu_nanotime;
- uint64_t oldnsecs;
- uint64_t newnsecs;
- uint64_t tsc;
+ pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info;
+ uint64_t oldnsecs;
+ uint64_t newnsecs;
+ uint64_t tsc;
assert(!ml_get_interrupts_enabled());
tsc = rdtsc64();
- oldnsecs = rntp->ns_base + _tsc_to_nanoseconds(tsc - rntp->tsc_base);
- newnsecs = base + _tsc_to_nanoseconds(tsc - tsc_base);
-
+ oldnsecs = rntp->ns_base + _rtc_tsc_to_nanoseconds(tsc - rntp->tsc_base, rntp);
+ newnsecs = base + _rtc_tsc_to_nanoseconds(tsc - tsc_base, rntp);
+
/*
* Only update the base values if time using the new base values
* is later than the time using the old base values.
*/
if (oldnsecs < newnsecs) {
- _rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp);
- rtc_nanotime_set_commpage(rntp);
+ _pal_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp);
+ rtc_nanotime_set_commpage(rntp);
}
}
+/*
+ * Invoked from power management to correct the SFLM TSC entry drift problem:
+ * a small delta is added to the tsc_base. This is equivalent to nudgin time
+ * backwards. We require this to be on the order of a TSC quantum which won't
+ * cause callers of mach_absolute_time() to see time going backwards!
+ */
void
-rtc_clock_stepping(__unused uint32_t new_frequency,
- __unused uint32_t old_frequency)
+rtc_clock_adjust(uint64_t tsc_base_delta)
{
- panic("rtc_clock_stepping unsupported");
-}
+ pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info;
-void
-rtc_clock_stepped(__unused uint32_t new_frequency,
- __unused uint32_t old_frequency)
-{
- panic("rtc_clock_stepped unsupported");
+ assert(!ml_get_interrupts_enabled());
+ assert(tsc_base_delta < 100ULL); /* i.e. it's small */
+ _rtc_nanotime_adjust(tsc_base_delta, rntp);
+ rtc_nanotime_set_commpage(rntp);
}
/*
* rtc_sleep_wakeup:
*
- * Invoked from power manageent when we have awoken from a sleep (S3)
- * and the TSC has been reset. The nanotime data is updated based on
- * the passed in value.
+ * Invoked from power management when we have awoken from a sleep (S3)
+ * and the TSC has been reset, or from Deep Idle (S0) sleep when the TSC
+ * has progressed. The nanotime data is updated based on the passed-in value.
*
* The caller must guarantee non-reentrancy.
*/
void
rtc_sleep_wakeup(
- uint64_t base)
+ uint64_t base)
{
+ /* Set fixed configuration for lapic timers */
+ rtc_timer->rtc_config();
+
/*
* Reset nanotime.
* The timestamp counter will have been reset
rtc_nanotime_init(base);
}
+void
+rtc_decrementer_configure(void)
+{
+ rtc_timer->rtc_config();
+}
+/*
+ * rtclock_early_init() is called very early at boot to
+ * establish mach_absolute_time() and set it to zero.
+ */
+void
+rtclock_early_init(void)
+{
+ assert(tscFreq);
+ rtc_set_timescale(tscFreq);
+}
+
/*
* Initialize the real-time clock device.
* In addition, various variables used to support the clock are initialized.
int
rtclock_init(void)
{
- uint64_t cycles;
+ uint64_t cycles;
assert(!ml_get_interrupts_enabled());
if (cpu_number() == master_cpu) {
-
assert(tscFreq);
- rtc_set_timescale(tscFreq);
/*
* Adjust and set the exported cpu speed.
gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles;
gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles;
- /*
- * Compute the longest interval we can represent.
- */
- maxDec = tmrCvt(0x7fffffffULL, busFCvtt2n);
- kprintf("maxDec: %lld\n", maxDec);
-
- /* Minimum interval is 1usec */
- rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC, 0ULL);
- /* Point LAPIC interrupts to hardclock() */
- lapic_set_timer_func((i386_intr_func_t) rtclock_intr);
-
+ rtc_timer_init();
clock_timebase_init();
ml_init_lock_timeout();
+ ml_init_delay_spin_threshold(10);
}
- rtc_lapic_start_ticking();
+ /* Set fixed configuration for lapic timers */
+ rtc_timer->rtc_config();
+ rtc_timer_start();
- return (1);
+ return 1;
}
-// utility routine
+// utility routine
// Code to calculate how many processor cycles are in a second...
static void
rtc_set_timescale(uint64_t cycles)
{
- rtc_nanotime_t *rntp = current_cpu_datap()->cpu_nanotime;
- rntp->scale = ((uint64_t)NSEC_PER_SEC << 32) / cycles;
+ pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info;
+ uint32_t shift = 0;
+
+ /* the "scale" factor will overflow unless cycles>SLOW_TSC_THRESHOLD */
+
+ while (cycles <= SLOW_TSC_THRESHOLD) {
+ shift++;
+ cycles <<= 1;
+ }
+
+ rntp->scale = (uint32_t)(((uint64_t)NSEC_PER_SEC << 32) / cycles);
- if (cycles <= SLOW_TSC_THRESHOLD)
- rntp->shift = cycles;
- else
- rntp->shift = 32;
+ rntp->shift = shift;
+
+ /*
+ * On some platforms, the TSC is not reset at warm boot. But the
+ * rebase time must be relative to the current boot so we can't use
+ * mach_absolute_time(). Instead, we convert the TSC delta since boot
+ * to nanoseconds.
+ */
+ if (tsc_rebase_abs_time == 0) {
+ tsc_rebase_abs_time = _rtc_tsc_to_nanoseconds(
+ rdtsc64() - tsc_at_boot, rntp);
+ }
rtc_nanotime_init(0);
}
static uint64_t
rtc_export_speed(uint64_t cyc_per_sec)
{
- uint64_t cycles;
+ pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info;
+ uint64_t cycles;
+
+ if (rntp->shift != 0) {
+ printf("Slow TSC, rtc_nanotime.shift == %d\n", rntp->shift);
+ }
/* Round: */
- cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2))
- / UI_CPUFREQ_ROUNDING_FACTOR)
- * UI_CPUFREQ_ROUNDING_FACTOR;
+ cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR / 2))
+ / UI_CPUFREQ_ROUNDING_FACTOR)
+ * UI_CPUFREQ_ROUNDING_FACTOR;
/*
* Set current measured speed.
*/
- if (cycles >= 0x100000000ULL) {
- gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL;
- } else {
- gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles;
- }
- gPEClockFrequencyInfo.cpu_frequency_hz = cycles;
+ if (cycles >= 0x100000000ULL) {
+ gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL;
+ } else {
+ gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles;
+ }
+ gPEClockFrequencyInfo.cpu_frequency_hz = cycles;
kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec);
- return(cycles);
+ return cycles;
}
void
clock_get_system_microtime(
- uint32_t *secs,
- uint32_t *microsecs)
+ clock_sec_t *secs,
+ clock_usec_t *microsecs)
{
- uint64_t now = rtc_nanotime_read();
- uint32_t remain;
-
- asm volatile(
- "divl %3"
- : "=a" (*secs), "=d" (remain)
- : "A" (now), "r" (NSEC_PER_SEC));
- asm volatile(
- "divl %3"
- : "=a" (*microsecs)
- : "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
+ uint64_t now = rtc_nanotime_read();
+
+ _absolutetime_to_microtime(now, secs, microsecs);
}
void
clock_get_system_nanotime(
- uint32_t *secs,
- uint32_t *nanosecs)
+ clock_sec_t *secs,
+ clock_nsec_t *nanosecs)
{
- uint64_t now = rtc_nanotime_read();
+ uint64_t now = rtc_nanotime_read();
- asm volatile(
- "divl %3"
- : "=a" (*secs), "=d" (*nanosecs)
- : "A" (now), "r" (NSEC_PER_SEC));
+ _absolutetime_to_nanotime(now, secs, nanosecs);
}
void
-clock_gettimeofday_set_commpage(
- uint64_t abstime,
- uint64_t epoch,
- uint64_t offset,
- uint32_t *secs,
- uint32_t *microsecs)
+clock_gettimeofday_set_commpage(uint64_t abstime, uint64_t sec, uint64_t frac, uint64_t scale, uint64_t tick_per_sec)
{
- uint64_t now = abstime;
- uint32_t remain;
-
- now += offset;
-
- asm volatile(
- "divl %3"
- : "=a" (*secs), "=d" (remain)
- : "A" (now), "r" (NSEC_PER_SEC));
- asm volatile(
- "divl %3"
- : "=a" (*microsecs)
- : "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
-
- *secs += epoch;
-
- commpage_set_timestamp(abstime - remain, *secs);
+ commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
}
void
clock_timebase_info(
- mach_timebase_info_t info)
+ mach_timebase_info_t info)
{
info->numer = info->denom = 1;
-}
-
-void
-clock_set_timer_func(
- clock_timer_func_t func)
-{
- if (rtclock_timer_expire == NULL)
- rtclock_timer_expire = func;
}
/*
*/
void
rtclock_intr(
- x86_saved_state_t *tregs)
+ x86_saved_state_t *tregs)
{
- uint64_t rip;
- boolean_t user_mode = FALSE;
- uint64_t abstime;
- uint32_t latency;
- x86_lcpu_t *lcpu = x86_lcpu();
+ uint64_t rip;
+ boolean_t user_mode = FALSE;
assert(get_preemption_level() > 0);
assert(!ml_get_interrupts_enabled());
- abstime = rtc_nanotime_read();
- latency = (uint32_t)(abstime - lcpu->rtcDeadline);
- if (abstime < lcpu->rtcDeadline)
- latency = 1;
-
if (is_saved_state64(tregs) == TRUE) {
- x86_saved_state64_t *regs;
-
+ x86_saved_state64_t *regs;
+
regs = saved_state64(tregs);
- user_mode = TRUE;
+ if (regs->isf.cs & 0x03) {
+ user_mode = TRUE;
+ }
rip = regs->isf.rip;
} else {
- x86_saved_state32_t *regs;
+ x86_saved_state32_t *regs;
regs = saved_state32(tregs);
- if (regs->cs & 0x03)
- user_mode = TRUE;
+ if (regs->cs & 0x03) {
+ user_mode = TRUE;
+ }
rip = regs->eip;
}
- /* Log the interrupt service latency (-ve value expected by tool) */
- KERNEL_DEBUG_CONSTANT(
- MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
- -latency, (uint32_t)rip, user_mode, 0, 0);
-
/* call the generic etimer */
- etimer_intr(user_mode, rip);
+ timer_intr(user_mode, rip);
}
+
/*
- * Request timer pop from the hardware
+ * Request timer pop from the hardware
*/
-int
-setPop(
- uint64_t time)
+uint64_t
+setPop(uint64_t time)
{
- uint64_t now;
- uint32_t decr;
- uint64_t count;
-
- now = rtc_nanotime_read(); /* The time in nanoseconds */
- decr = deadline_to_decrementer(time, now);
+ uint64_t now;
+ uint64_t pop;
+
+ /* 0 and EndOfAllTime are special-cases for "clear the timer" */
+ if (time == 0 || time == EndOfAllTime) {
+ time = EndOfAllTime;
+ now = 0;
+ pop = rtc_timer->rtc_set(0, 0);
+ } else {
+ now = rtc_nanotime_read(); /* The time in nanoseconds */
+ pop = rtc_timer->rtc_set(time, now);
+ }
- count = tmrCvt(decr, busFCvtn2t);
- lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count);
+ /* Record requested and actual deadlines set */
+ x86_lcpu()->rtcDeadline = time;
+ x86_lcpu()->rtcPop = pop;
- return decr; /* Pass back what we set */
+ return pop - now;
}
-
uint64_t
mach_absolute_time(void)
{
return rtc_nanotime_read();
}
-void
-clock_interval_to_absolutetime_interval(
- uint32_t interval,
- uint32_t scale_factor,
- uint64_t *result)
+uint64_t
+mach_approximate_time(void)
{
- *result = (uint64_t)interval * scale_factor;
+ return rtc_nanotime_read();
}
void
-absolutetime_to_microtime(
- uint64_t abstime,
- uint32_t *secs,
- uint32_t *microsecs)
+clock_interval_to_absolutetime_interval(
+ uint32_t interval,
+ uint32_t scale_factor,
+ uint64_t *result)
{
- uint32_t remain;
-
- asm volatile(
- "divl %3"
- : "=a" (*secs), "=d" (remain)
- : "A" (abstime), "r" (NSEC_PER_SEC));
- asm volatile(
- "divl %3"
- : "=a" (*microsecs)
- : "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
+ *result = (uint64_t)interval * scale_factor;
}
void
-absolutetime_to_nanotime(
- uint64_t abstime,
- uint32_t *secs,
- uint32_t *nanosecs)
+absolutetime_to_microtime(
+ uint64_t abstime,
+ clock_sec_t *secs,
+ clock_usec_t *microsecs)
{
- asm volatile(
- "divl %3"
- : "=a" (*secs), "=d" (*nanosecs)
- : "A" (abstime), "r" (NSEC_PER_SEC));
+ _absolutetime_to_microtime(abstime, secs, microsecs);
}
void
nanotime_to_absolutetime(
- uint32_t secs,
- uint32_t nanosecs,
- uint64_t *result)
+ clock_sec_t secs,
+ clock_nsec_t nanosecs,
+ uint64_t *result)
{
*result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs;
}
void
absolutetime_to_nanoseconds(
- uint64_t abstime,
- uint64_t *result)
+ uint64_t abstime,
+ uint64_t *result)
{
*result = abstime;
}
void
nanoseconds_to_absolutetime(
- uint64_t nanoseconds,
- uint64_t *result)
+ uint64_t nanoseconds,
+ uint64_t *result)
{
*result = nanoseconds;
}
void
machine_delay_until(
- uint64_t deadline)
+ uint64_t interval,
+ uint64_t deadline)
{
- uint64_t now;
-
- do {
+ (void)interval;
+ while (mach_absolute_time() < deadline) {
cpu_pause();
- now = mach_absolute_time();
- } while (now < deadline);
+ }
}
projects / apple / xnu.git / blobdiff