/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
*
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
/*
* File: i386/rtclock.c
* Purpose: Routines for handling the machine dependent
- * real-time clock. This clock is generated by
- * the Intel 8254 Programmable Interval Timer.
+ * real-time clock. Historically, this clock is
+ * generated by the Intel 8254 Programmable Interval
+ * Timer, but local apic timers are now used for
+ * this purpose with the master time reference being
+ * the cpu clock counted by the timestamp MSR.
*/
-#include <cpus.h>
#include <platforms.h>
-#include <mp_v1_1.h>
#include <mach_kdb.h>
-#include <kern/cpu_number.h>
+
+#include <mach/mach_types.h>
+
#include <kern/cpu_data.h>
+#include <kern/cpu_number.h>
#include <kern/clock.h>
+#include <kern/host_notify.h>
#include <kern/macro_help.h>
#include <kern/misc_protos.h>
#include <kern/spl.h>
-#include <machine/mach_param.h> /* HZ */
+#include <kern/assert.h>
#include <mach/vm_prot.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h> /* for kernel_map */
#include <i386/ipl.h>
-#include <i386/pit.h>
-#include <i386/pio.h>
+#include <architecture/i386/pio.h>
#include <i386/misc_protos.h>
-#include <i386/rtclock_entries.h>
-#include <i386/hardclock_entries.h>
+#include <i386/proc_reg.h>
+#include <i386/machine_cpu.h>
+#include <i386/mp.h>
+#include <i386/cpuid.h>
+#include <i386/cpu_data.h>
+#include <i386/cpu_threads.h>
+#include <i386/perfmon.h>
+#include <i386/machine_routines.h>
+#include <pexpert/pexpert.h>
+#include <machine/limits.h>
+#include <machine/commpage.h>
+#include <sys/kdebug.h>
+#include <i386/tsc.h>
+#include <i386/hpet.h>
+#include <i386/rtclock.h>
-int sysclk_config(void);
+#define NSEC_PER_HZ (NSEC_PER_SEC / 100) /* nsec per tick */
-int sysclk_init(void);
+#define UI_CPUFREQ_ROUNDING_FACTOR 10000000
-kern_return_t sysclk_gettime(
- mach_timespec_t *cur_time);
+int rtclock_config(void);
-kern_return_t sysclk_getattr(
- clock_flavor_t flavor,
- clock_attr_t attr,
- mach_msg_type_number_t *count);
+int rtclock_init(void);
-kern_return_t sysclk_setattr(
- clock_flavor_t flavor,
- clock_attr_t attr,
- mach_msg_type_number_t count);
+uint64_t rtc_decrementer_min;
-void sysclk_setalarm(
- mach_timespec_t *alarm_time);
+void rtclock_intr(x86_saved_state_t *regs);
+static uint64_t maxDec; /* longest interval our hardware timer can handle (nsec) */
-extern void (*IOKitRegisterInterruptHook)(void *, int irq, int isclock);
+/* XXX this should really be in a header somewhere */
+extern clock_timer_func_t rtclock_timer_expire;
-/*
- * Inlines to get timestamp counter value.
- */
+static void rtc_set_timescale(uint64_t cycles);
+static uint64_t rtc_export_speed(uint64_t cycles);
-static inline void rdtsc_hilo(uint32_t *hi, uint32_t *lo) {
- asm volatile("rdtsc": "=a" (*lo), "=d" (*hi));
-}
+extern void _rtc_nanotime_store(
+ uint64_t tsc,
+ uint64_t nsec,
+ uint32_t scale,
+ uint32_t shift,
+ rtc_nanotime_t *dst);
-static inline uint64_t rdtsc_64(void) {
- uint64_t result;
- asm volatile("rdtsc": "=A" (result));
- return result;
-}
+extern uint64_t _rtc_nanotime_read(
+ rtc_nanotime_t *rntp,
+ int slow );
-/*
- * Lists of clock routines.
- */
-struct clock_ops sysclk_ops = {
- sysclk_config, sysclk_init,
- sysclk_gettime, 0,
- sysclk_getattr, sysclk_setattr,
- sysclk_setalarm,
-};
-
-int calend_config(void);
-
-int calend_init(void);
-
-kern_return_t calend_gettime(
- mach_timespec_t *cur_time);
-
-kern_return_t calend_settime(
- mach_timespec_t *cur_time);
-
-kern_return_t calend_getattr(
- clock_flavor_t flavor,
- clock_attr_t attr,
- mach_msg_type_number_t *count);
-
-struct clock_ops calend_ops = {
- calend_config, calend_init,
- calend_gettime, calend_settime,
- calend_getattr, 0,
- 0,
-};
-
-/* local data declarations */
-mach_timespec_t *RtcTime = (mach_timespec_t *)0;
-mach_timespec_t *RtcAlrm;
-clock_res_t RtcDelt;
-
-/* global data declarations */
-struct {
- uint64_t abstime;
-
- mach_timespec_t time;
- mach_timespec_t alarm_time; /* time of next alarm */
-
- mach_timespec_t calend_offset;
- boolean_t calend_is_set;
-
- uint64_t timer_deadline;
- boolean_t timer_is_set;
- clock_timer_func_t timer_expire;
-
- clock_res_t new_ires; /* pending new resolution (nano ) */
- clock_res_t intr_nsec; /* interrupt resolution (nano) */
-
- decl_simple_lock_data(,lock) /* real-time clock device lock */
-} rtclock;
-
-unsigned int clknum; /* clks per second */
-unsigned int new_clknum; /* pending clknum */
-unsigned int time_per_clk; /* time per clk in ZHZ */
-unsigned int clks_per_int; /* clks per interrupt */
-unsigned int clks_per_int_99;
-int rtc_intr_count; /* interrupt counter */
-int rtc_intr_hertz; /* interrupts per HZ */
-int rtc_intr_freq; /* interrupt frequency */
-int rtc_print_lost_tick; /* print lost tick */
-
-uint32_t rtc_cyc_per_sec; /* processor cycles per seconds */
-uint32_t rtc_last_int_tsc_lo; /* tsc values saved per interupt */
-uint32_t rtc_last_int_tsc_hi;
+rtc_nanotime_t rtc_nanotime_info = {0,0,0,0,1,0};
-/*
- * Macros to lock/unlock real-time clock device.
- */
-#define LOCK_RTC(s) \
-MACRO_BEGIN \
- (s) = splclock(); \
- simple_lock(&rtclock.lock); \
-MACRO_END
-
-#define UNLOCK_RTC(s) \
-MACRO_BEGIN \
- simple_unlock(&rtclock.lock); \
- splx(s); \
-MACRO_END
-/*
- * i8254 control. ** MONUMENT **
- *
- * The i8254 is a traditional PC device with some arbitrary characteristics.
- * Basically, it is a register that counts at a fixed rate and can be
- * programmed to generate an interrupt every N counts. The count rate is
- * clknum counts per second (see pit.h), historically 1193167 we believe.
- * Various constants are computed based on this value, and we calculate
- * them at init time for execution efficiency. To obtain sufficient
- * accuracy, some of the calculation are most easily done in floating
- * point and then converted to int.
- *
- * We want an interrupt every 10 milliseconds, approximately. The count
- * which will do that is clks_per_int. However, that many counts is not
- * *exactly* 10 milliseconds; it is a bit more or less depending on
- * roundoff. The actual time per tick is calculated and saved in
- * rtclock.intr_nsec, and it is that value which is added to the time
- * register on each tick.
- *
- * The i8254 counter can be read between interrupts in order to determine
- * the time more accurately. The counter counts down from the preset value
- * toward 0, and we have to handle the case where the counter has been
- * reset just before being read and before the interrupt has been serviced.
- * Given a count since the last interrupt, the time since then is given
- * by (count * time_per_clk). In order to minimize integer truncation,
- * we perform this calculation in an arbitrary unit of time which maintains
- * the maximum precision, i.e. such that one tick is 1.0e9 of these units,
- * or close to the precision of a 32-bit int. We then divide by this unit
- * (which doesn't lose precision) to get nanoseconds. For notation
- * purposes, this unit is defined as ZHZ = zanoseconds per nanosecond.
- *
- * This sequence to do all this is in sysclk_gettime. For efficiency, this
- * sequence also needs the value that the counter will have if it has just
- * overflowed, so we precompute that also. ALSO, certain platforms
- * (specifically the DEC XL5100) have been observed to have problem
- * with latching the counter, and they occasionally (say, one out of
- * 100,000 times) return a bogus value. Hence, the present code reads
- * the counter twice and checks for a consistent pair of values.
- *
- * Some attributes of the rt clock can be changed, including the
- * interrupt resolution. We default to the minimum resolution (10 ms),
- * but allow a finer resolution to be requested. The assumed frequency
- * of the clock can also be set since it appears that the actual
- * frequency of real-world hardware can vary from the nominal by
- * 200 ppm or more. When the frequency is set, the values above are
- * recomputed and we continue without resetting or changing anything else.
- */
-#define RTC_MINRES (NSEC_PER_SEC / HZ) /* nsec per tick */
-#define RTC_MAXRES (RTC_MINRES / 20) /* nsec per tick */
-#define ZANO (1000000000)
-#define ZHZ (ZANO / (NSEC_PER_SEC / HZ))
-#define READ_8254(val) { \
- outb(PITCTL_PORT, PIT_C0); \
- (val) = inb(PITCTR0_PORT); \
- (val) |= inb(PITCTR0_PORT) << 8 ; }
-
-/*
- * Calibration delay counts.
- */
-unsigned int delaycount = 100;
-unsigned int microdata = 50;
-
-/*
- * Forward decl.
- */
-
-extern int measure_delay(int us);
-void rtc_setvals( unsigned int, clock_res_t );
+static uint32_t
+deadline_to_decrementer(
+ uint64_t deadline,
+ uint64_t now)
+{
+ uint64_t delta;
-static void rtc_set_cyc_per_sec();
+ if (deadline <= now)
+ return rtc_decrementer_min;
+ else {
+ delta = deadline - now;
+ return MIN(MAX(rtc_decrementer_min,delta),maxDec);
+ }
+}
-/*
- * Initialize non-zero clock structure values.
- */
void
-rtc_setvals(
- unsigned int new_clknum,
- clock_res_t new_ires
- )
+rtc_lapic_start_ticking(void)
{
- unsigned int timeperclk;
- unsigned int scale0;
- unsigned int scale1;
- unsigned int res;
-
- clknum = new_clknum;
- rtc_intr_freq = (NSEC_PER_SEC / new_ires);
- rtc_intr_hertz = rtc_intr_freq / HZ;
- clks_per_int = (clknum + (rtc_intr_freq / 2)) / rtc_intr_freq;
- clks_per_int_99 = clks_per_int - clks_per_int/100;
-
- /*
- * The following calculations are done with scaling integer operations
- * in order that the integer results are accurate to the lsb.
- */
- timeperclk = div_scale(ZANO, clknum, &scale0); /* 838.105647 nsec */
-
- time_per_clk = mul_scale(ZHZ, timeperclk, &scale1); /* 83810 */
- if (scale0 > scale1)
- time_per_clk >>= (scale0 - scale1);
- else if (scale0 < scale1)
- panic("rtc_clock: time_per_clk overflow\n");
-
- /*
- * Notice that rtclock.intr_nsec is signed ==> use unsigned int res
- */
- res = mul_scale(clks_per_int, timeperclk, &scale1); /* 10000276 */
- if (scale0 > scale1)
- rtclock.intr_nsec = res >> (scale0 - scale1);
- else
- panic("rtc_clock: rtclock.intr_nsec overflow\n");
-
- rtc_intr_count = 1;
- RtcDelt = rtclock.intr_nsec/2;
+ x86_lcpu_t *lcpu = x86_lcpu();
+
+ /*
+ * Force a complete re-evaluation of timer deadlines.
+ */
+ lcpu->rtcPop = EndOfAllTime;
+ etimer_resync_deadlines();
}
/*
*/
int
-sysclk_config(void)
+rtclock_config(void)
{
- int RtcFlag;
- int pic;
-
-#if NCPUS > 1
- mp_disable_preemption();
- if (cpu_number() != master_cpu) {
- mp_enable_preemption();
- return(1);
- }
- mp_enable_preemption();
-#endif
- /*
- * Setup device.
- */
-#if MP_V1_1
- {
- extern boolean_t mp_v1_1_initialized;
- if (mp_v1_1_initialized)
- pic = 2;
- else
- pic = 0;
- }
-#else
- pic = 0; /* FIXME .. interrupt registration moved to AppleIntelClock */
-#endif
-
-
- /*
- * We should attempt to test the real-time clock
- * device here. If it were to fail, we should panic
- * the system.
- */
- RtcFlag = /* test device */1;
- printf("realtime clock configured\n");
-
- simple_lock_init(&rtclock.lock, ETAP_NO_TRACE);
- return (RtcFlag);
+ /* nothing to do */
+ return (1);
}
+
/*
- * Initialize the real-time clock device. Return success (1)
- * or failure (0). Since the real-time clock is required to
- * provide canonical mapped time, we allocate a page to keep
- * the clock time value. In addition, various variables used
- * to support the clock are initialized. Note: the clock is
- * not started until rtclock_reset is called.
+ * Nanotime/mach_absolutime_time
+ * -----------------------------
+ * The timestamp counter (TSC) - which counts cpu clock cycles and can be read
+ * efficiently by the kernel and in userspace - is the reference for all timing.
+ * The cpu clock rate is platform-dependent and may stop or be reset when the
+ * processor is napped/slept. As a result, nanotime is the software abstraction
+ * used to maintain a monotonic clock, adjusted from an outside reference as needed.
+ *
+ * The kernel maintains nanotime information recording:
+ * - the ratio of tsc to nanoseconds
+ * with this ratio expressed as a 32-bit scale and shift
+ * (power of 2 divider);
+ * - { tsc_base, ns_base } pair of corresponding timestamps.
+ *
+ * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage
+ * for the userspace nanotime routine to read.
+ *
+ * All of the routines which update the nanotime data are non-reentrant. This must
+ * be guaranteed by the caller.
*/
-int
-sysclk_init(void)
+static inline void
+rtc_nanotime_set_commpage(rtc_nanotime_t *rntp)
{
- vm_offset_t *vp;
-#if NCPUS > 1
- mp_disable_preemption();
- if (cpu_number() != master_cpu) {
- mp_enable_preemption();
- return(1);
- }
- mp_enable_preemption();
-#endif
-
- RtcTime = &rtclock.time;
- rtc_setvals( CLKNUM, RTC_MINRES ); /* compute constants */
- rtc_set_cyc_per_sec(); /* compute number of tsc beats per second */
- return (1);
+ commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift);
}
-static volatile unsigned int last_ival = 0;
-
/*
- * Get the clock device time. This routine is responsible
- * for converting the device's machine dependent time value
- * into a canonical mach_timespec_t value.
+ * rtc_nanotime_init:
+ *
+ * Intialize the nanotime info from the base time.
*/
-kern_return_t
-sysclk_gettime(
- mach_timespec_t *cur_time) /* OUT */
+static inline void
+_rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base)
{
- mach_timespec_t itime = {0, 0};
- unsigned int val, val2;
- int s;
-
- if (!RtcTime) {
- /* Uninitialized */
- cur_time->tv_nsec = 0;
- cur_time->tv_sec = 0;
- return (KERN_SUCCESS);
- }
+ uint64_t tsc = rdtsc64();
- /*
- * Inhibit interrupts. Determine the incremental
- * time since the last interrupt. (This could be
- * done in assembler for a bit more speed).
- */
- LOCK_RTC(s);
- do {
- READ_8254(val); /* read clock */
- READ_8254(val2); /* read clock */
- } while ( val2 > val || val2 < val - 10 );
- if ( val > clks_per_int_99 ) {
- outb( 0x0a, 0x20 ); /* see if interrupt pending */
- if ( inb( 0x20 ) & 1 )
- itime.tv_nsec = rtclock.intr_nsec; /* yes, add a tick */
- }
- itime.tv_nsec += ((clks_per_int - val) * time_per_clk) / ZHZ;
- if ( itime.tv_nsec < last_ival ) {
- if (rtc_print_lost_tick)
- printf( "rtclock: missed clock interrupt.\n" );
- }
- last_ival = itime.tv_nsec;
- cur_time->tv_sec = rtclock.time.tv_sec;
- cur_time->tv_nsec = rtclock.time.tv_nsec;
- UNLOCK_RTC(s);
- ADD_MACH_TIMESPEC(cur_time, ((mach_timespec_t *)&itime));
- return (KERN_SUCCESS);
+ _rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
}
-kern_return_t
-sysclk_gettime_internal(
- mach_timespec_t *cur_time) /* OUT */
+static void
+rtc_nanotime_init(uint64_t base)
{
- mach_timespec_t itime = {0, 0};
- unsigned int val, val2;
-
- if (!RtcTime) {
- /* Uninitialized */
- cur_time->tv_nsec = 0;
- cur_time->tv_sec = 0;
- return (KERN_SUCCESS);
- }
+ rtc_nanotime_t *rntp = &rtc_nanotime_info;
- /*
- * Inhibit interrupts. Determine the incremental
- * time since the last interrupt. (This could be
- * done in assembler for a bit more speed).
- */
- do {
- READ_8254(val); /* read clock */
- READ_8254(val2); /* read clock */
- } while ( val2 > val || val2 < val - 10 );
- if ( val > clks_per_int_99 ) {
- outb( 0x0a, 0x20 ); /* see if interrupt pending */
- if ( inb( 0x20 ) & 1 )
- itime.tv_nsec = rtclock.intr_nsec; /* yes, add a tick */
- }
- itime.tv_nsec += ((clks_per_int - val) * time_per_clk) / ZHZ;
- if ( itime.tv_nsec < last_ival ) {
- if (rtc_print_lost_tick)
- printf( "rtclock: missed clock interrupt.\n" );
- }
- last_ival = itime.tv_nsec;
- cur_time->tv_sec = rtclock.time.tv_sec;
- cur_time->tv_nsec = rtclock.time.tv_nsec;
- ADD_MACH_TIMESPEC(cur_time, ((mach_timespec_t *)&itime));
- return (KERN_SUCCESS);
+ _rtc_nanotime_init(rntp, base);
+ rtc_nanotime_set_commpage(rntp);
}
/*
- * Get the clock device time when ALL interrupts are already disabled.
- * Same as above except for turning interrupts off and on.
- * This routine is responsible for converting the device's machine dependent
- * time value into a canonical mach_timespec_t value.
+ * rtc_nanotime_init_commpage:
+ *
+ * Call back from the commpage initialization to
+ * cause the commpage data to be filled in once the
+ * commpages have been created.
*/
void
-sysclk_gettime_interrupts_disabled(
- mach_timespec_t *cur_time) /* OUT */
+rtc_nanotime_init_commpage(void)
{
- mach_timespec_t itime = {0, 0};
- unsigned int val;
-
- if (!RtcTime) {
- /* Uninitialized */
- cur_time->tv_nsec = 0;
- cur_time->tv_sec = 0;
- return;
- }
+ spl_t s = splclock();
- simple_lock(&rtclock.lock);
+ rtc_nanotime_set_commpage(&rtc_nanotime_info);
- /*
- * Copy the current time knowing that we cant be interrupted
- * between the two longwords and so dont need to use MTS_TO_TS
- */
- READ_8254(val); /* read clock */
- if ( val > clks_per_int_99 ) {
- outb( 0x0a, 0x20 ); /* see if interrupt pending */
- if ( inb( 0x20 ) & 1 )
- itime.tv_nsec = rtclock.intr_nsec; /* yes, add a tick */
- }
- itime.tv_nsec += ((clks_per_int - val) * time_per_clk) / ZHZ;
- if ( itime.tv_nsec < last_ival ) {
- if (rtc_print_lost_tick)
- printf( "rtclock: missed clock interrupt.\n" );
- }
- last_ival = itime.tv_nsec;
- cur_time->tv_sec = rtclock.time.tv_sec;
- cur_time->tv_nsec = rtclock.time.tv_nsec;
- ADD_MACH_TIMESPEC(cur_time, ((mach_timespec_t *)&itime));
-
- simple_unlock(&rtclock.lock);
+ splx(s);
}
-// utility routine
-// Code to calculate how many processor cycles are in a second...
-
-static void
-rtc_set_cyc_per_sec()
+/*
+ * rtc_nanotime_read:
+ *
+ * Returns the current nanotime value, accessable from any
+ * context.
+ */
+static inline uint64_t
+rtc_nanotime_read(void)
{
+
+#if CONFIG_EMBEDDED
+ if (gPEClockFrequencyInfo.timebase_frequency_hz > SLOW_TSC_THRESHOLD)
+ return _rtc_nanotime_read( &rtc_nanotime_info, 1 ); /* slow processor */
+ else
+#endif
+ return _rtc_nanotime_read( &rtc_nanotime_info, 0 ); /* assume fast processor */
+}
- int x, y;
- uint64_t cycles;
- uint32_t c[15]; // array for holding sampled cycle counts
- mach_timespec_t tst[15]; // array for holding time values. NOTE for some reason tv_sec not work
+/*
+ * rtc_clock_napped:
+ *
+ * Invoked from power manangement when we have awoken from a nap (C3/C4)
+ * during which the TSC lost counts. The nanotime data is updated according
+ * to the provided value which indicates the number of nanoseconds that the
+ * TSC was not counting.
+ *
+ * The caller must guarantee non-reentrancy.
+ */
+void
+rtc_clock_napped(
+ uint64_t delta)
+{
+ rtc_nanotime_t *rntp = &rtc_nanotime_info;
+ uint32_t generation;
- for (x=0; x<15; x++) { // quick sample 15 times
- tst[x].tv_sec = 0;
- tst[x].tv_nsec = 0;
- sysclk_gettime_internal(&tst[x]);
- rdtsc_hilo(&y, &c[x]);
- }
- y = 0;
- cycles = 0;
- for (x=0; x<14; x++) {
- // simple formula really. calculate the numerator as the number of elapsed processor
- // cycles * 1000 to adjust for the resolution we want. The denominator is the
- // elapsed "real" time in nano-seconds. The result will be the processor speed in
- // Mhz. any overflows will be discarded before they are added
- if ((c[x+1] > c[x]) && (tst[x+1].tv_nsec > tst[x].tv_nsec)) {
- cycles += ((uint64_t)(c[x+1]-c[x]) * NSEC_PER_SEC ) / (uint64_t)(tst[x+1].tv_nsec - tst[x].tv_nsec); // elapsed nsecs
- y +=1;
- }
- }
- if (y>0) { // we got more than 1 valid sample. This also takes care of the case of if the clock isn't running
- cycles = cycles / y; // calc our average
- }
- rtc_cyc_per_sec = cycles;
- rdtsc_hilo(&rtc_last_int_tsc_hi, &rtc_last_int_tsc_lo);
+ assert(!ml_get_interrupts_enabled());
+ generation = rntp->generation;
+ rntp->generation = 0;
+ rntp->ns_base += delta;
+ rntp->generation = ((generation + 1) != 0) ? (generation + 1) : 1;
+ rtc_nanotime_set_commpage(rntp);
}
-static
-natural_t
-get_uptime_cycles(void)
+void
+rtc_clock_stepping(__unused uint32_t new_frequency,
+ __unused uint32_t old_frequency)
{
- // get the time since the last interupt based on the processors TSC ignoring the
- // RTC for speed
-
- uint32_t a,d,intermediate_lo,intermediate_hi,result;
- uint64_t newTime;
-
- rdtsc_hilo(&d, &a);
- if (d != rtc_last_int_tsc_hi) {
- newTime = d-rtc_last_int_tsc_hi;
- newTime = (newTime<<32) + (a-rtc_last_int_tsc_lo);
- result = newTime;
- } else {
- result = a-rtc_last_int_tsc_lo;
- }
- __asm__ volatile ( " mul %3 ": "=eax" (intermediate_lo), "=edx" (intermediate_hi): "a"(result), "d"(NSEC_PER_SEC) );
- __asm__ volatile ( " div %3": "=eax" (result): "eax"(intermediate_lo), "edx" (intermediate_hi), "ecx" (rtc_cyc_per_sec) );
- return result;
+ panic("rtc_clock_stepping unsupported");
}
+void
+rtc_clock_stepped(__unused uint32_t new_frequency,
+ __unused uint32_t old_frequency)
+{
+ panic("rtc_clock_stepped unsupported");
+}
/*
- * Get clock device attributes.
+ * rtc_sleep_wakeup:
+ *
+ * Invoked from power manageent when we have awoken from a sleep (S3)
+ * and the TSC has been reset. The nanotime data is updated based on
+ * the passed in value.
+ *
+ * The caller must guarantee non-reentrancy.
*/
-kern_return_t
-sysclk_getattr(
- clock_flavor_t flavor,
- clock_attr_t attr, /* OUT */
- mach_msg_type_number_t *count) /* IN/OUT */
+void
+rtc_sleep_wakeup(
+ uint64_t base)
{
- spl_t s;
-
- if (*count != 1)
- return (KERN_FAILURE);
- switch (flavor) {
-
- case CLOCK_GET_TIME_RES: /* >0 res */
-#if (NCPUS == 1 || (MP_V1_1 && 0))
- LOCK_RTC(s);
- *(clock_res_t *) attr = 1000;
- UNLOCK_RTC(s);
- break;
-#endif /* (NCPUS == 1 || (MP_V1_1 && 0)) && AT386 */
- case CLOCK_ALARM_CURRES: /* =0 no alarm */
- LOCK_RTC(s);
- *(clock_res_t *) attr = rtclock.intr_nsec;
- UNLOCK_RTC(s);
- break;
-
- case CLOCK_ALARM_MAXRES:
- *(clock_res_t *) attr = RTC_MAXRES;
- break;
-
- case CLOCK_ALARM_MINRES:
- *(clock_res_t *) attr = RTC_MINRES;
- break;
-
- default:
- return (KERN_INVALID_VALUE);
- }
- return (KERN_SUCCESS);
+ /*
+ * Reset nanotime.
+ * The timestamp counter will have been reset
+ * but nanotime (uptime) marches onward.
+ */
+ rtc_nanotime_init(base);
}
/*
- * Set clock device attributes.
+ * Initialize the real-time clock device.
+ * In addition, various variables used to support the clock are initialized.
*/
-kern_return_t
-sysclk_setattr(
- clock_flavor_t flavor,
- clock_attr_t attr, /* IN */
- mach_msg_type_number_t count) /* IN */
+int
+rtclock_init(void)
{
- spl_t s;
- int freq;
- int adj;
- clock_res_t new_ires;
+ uint64_t cycles;
- if (count != 1)
- return (KERN_FAILURE);
- switch (flavor) {
+ assert(!ml_get_interrupts_enabled());
- case CLOCK_GET_TIME_RES:
- case CLOCK_ALARM_MAXRES:
- case CLOCK_ALARM_MINRES:
- return (KERN_FAILURE);
+ if (cpu_number() == master_cpu) {
- case CLOCK_ALARM_CURRES:
- new_ires = *(clock_res_t *) attr;
+ assert(tscFreq);
+ rtc_set_timescale(tscFreq);
/*
- * The new resolution must be within the predetermined
- * range. If the desired resolution cannot be achieved
- * to within 0.1%, an error is returned.
+ * Adjust and set the exported cpu speed.
*/
- if (new_ires < RTC_MAXRES || new_ires > RTC_MINRES)
- return (KERN_INVALID_VALUE);
- freq = (NSEC_PER_SEC / new_ires);
- adj = (((clknum % freq) * new_ires) / clknum);
- if (adj > (new_ires / 1000))
- return (KERN_INVALID_VALUE);
+ cycles = rtc_export_speed(tscFreq);
+
/*
- * Record the new alarm resolution which will take effect
- * on the next HZ aligned clock tick.
+ * Set min/max to actual.
+ * ACPI may update these later if speed-stepping is detected.
*/
- LOCK_RTC(s);
- if ( freq != rtc_intr_freq ) {
- rtclock.new_ires = new_ires;
- new_clknum = clknum;
- }
- UNLOCK_RTC(s);
- return (KERN_SUCCESS);
-
- default:
- return (KERN_INVALID_VALUE);
- }
-}
+ gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles;
+ gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles;
-/*
- * Set next alarm time for the clock device. This call
- * always resets the time to deliver an alarm for the
- * clock.
- */
-void
-sysclk_setalarm(
- mach_timespec_t *alarm_time)
-{
- spl_t s;
+ /*
+ * Compute the longest interval we can represent.
+ */
+ maxDec = tmrCvt(0x7fffffffULL, busFCvtt2n);
+ kprintf("maxDec: %lld\n", maxDec);
- LOCK_RTC(s);
- rtclock.alarm_time = *alarm_time;
- RtcAlrm = &rtclock.alarm_time;
- UNLOCK_RTC(s);
-}
+ /* Minimum interval is 1usec */
+ rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC, 0ULL);
+ /* Point LAPIC interrupts to hardclock() */
+ lapic_set_timer_func((i386_intr_func_t) rtclock_intr);
-/*
- * Configure the calendar clock.
- */
-int
-calend_config(void)
-{
- return bbc_config();
-}
+ clock_timebase_init();
+ ml_init_lock_timeout();
+ }
+
+ rtc_lapic_start_ticking();
-/*
- * Initialize calendar clock.
- */
-int
-calend_init(void)
-{
return (1);
}
-/*
- * Get the current clock time.
- */
-kern_return_t
-calend_gettime(
- mach_timespec_t *cur_time) /* OUT */
-{
- spl_t s;
+// utility routine
+// Code to calculate how many processor cycles are in a second...
- LOCK_RTC(s);
- if (!rtclock.calend_is_set) {
- UNLOCK_RTC(s);
- return (KERN_FAILURE);
- }
+static void
+rtc_set_timescale(uint64_t cycles)
+{
+ rtc_nanotime_info.scale = ((uint64_t)NSEC_PER_SEC << 32) / cycles;
- (void) sysclk_gettime_internal(cur_time);
- ADD_MACH_TIMESPEC(cur_time, &rtclock.calend_offset);
- UNLOCK_RTC(s);
+ if (cycles <= SLOW_TSC_THRESHOLD)
+ rtc_nanotime_info.shift = cycles;
+ else
+ rtc_nanotime_info.shift = 32;
- return (KERN_SUCCESS);
+ rtc_nanotime_init(0);
}
-/*
- * Set the current clock time.
- */
-kern_return_t
-calend_settime(
- mach_timespec_t *new_time)
+static uint64_t
+rtc_export_speed(uint64_t cyc_per_sec)
{
- mach_timespec_t curr_time;
- spl_t s;
+ uint64_t cycles;
- LOCK_RTC(s);
- (void) sysclk_gettime_internal(&curr_time);
- rtclock.calend_offset = *new_time;
- SUB_MACH_TIMESPEC(&rtclock.calend_offset, &curr_time);
- rtclock.calend_is_set = TRUE;
- UNLOCK_RTC(s);
+ /* Round: */
+ cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2))
+ / UI_CPUFREQ_ROUNDING_FACTOR)
+ * UI_CPUFREQ_ROUNDING_FACTOR;
- (void) bbc_settime(new_time);
+ /*
+ * Set current measured speed.
+ */
+ if (cycles >= 0x100000000ULL) {
+ gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL;
+ } else {
+ gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles;
+ }
+ gPEClockFrequencyInfo.cpu_frequency_hz = cycles;
- return (KERN_SUCCESS);
+ kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec);
+ return(cycles);
}
-/*
- * Get clock device attributes.
- */
-kern_return_t
-calend_getattr(
- clock_flavor_t flavor,
- clock_attr_t attr, /* OUT */
- mach_msg_type_number_t *count) /* IN/OUT */
+void
+clock_get_system_microtime(
+ uint32_t *secs,
+ uint32_t *microsecs)
{
- spl_t s;
-
- if (*count != 1)
- return (KERN_FAILURE);
- switch (flavor) {
-
- case CLOCK_GET_TIME_RES: /* >0 res */
-#if (NCPUS == 1 || (MP_V1_1 && 0))
- LOCK_RTC(s);
- *(clock_res_t *) attr = 1000;
- UNLOCK_RTC(s);
- break;
-#else /* (NCPUS == 1 || (MP_V1_1 && 0)) && AT386 */
- LOCK_RTC(s);
- *(clock_res_t *) attr = rtclock.intr_nsec;
- UNLOCK_RTC(s);
- break;
-#endif /* (NCPUS == 1 || (MP_V1_1 && 0)) && AT386 */
-
- case CLOCK_ALARM_CURRES: /* =0 no alarm */
- case CLOCK_ALARM_MINRES:
- case CLOCK_ALARM_MAXRES:
- *(clock_res_t *) attr = 0;
- break;
-
- default:
- return (KERN_INVALID_VALUE);
- }
- return (KERN_SUCCESS);
+ uint64_t now = rtc_nanotime_read();
+ uint32_t remain;
+
+ asm volatile(
+ "divl %3"
+ : "=a" (*secs), "=d" (remain)
+ : "A" (now), "r" (NSEC_PER_SEC));
+ asm volatile(
+ "divl %3"
+ : "=a" (*microsecs)
+ : "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
}
void
-clock_adjust_calendar(
- clock_res_t nsec)
+clock_get_system_nanotime(
+ uint32_t *secs,
+ uint32_t *nanosecs)
{
- spl_t s;
+ uint64_t now = rtc_nanotime_read();
- LOCK_RTC(s);
- if (rtclock.calend_is_set)
- ADD_MACH_TIMESPEC_NSEC(&rtclock.calend_offset, nsec);
- UNLOCK_RTC(s);
+ asm volatile(
+ "divl %3"
+ : "=a" (*secs), "=d" (*nanosecs)
+ : "A" (now), "r" (NSEC_PER_SEC));
}
void
-clock_initialize_calendar(void)
+clock_gettimeofday_set_commpage(
+ uint64_t abstime,
+ uint64_t epoch,
+ uint64_t offset,
+ uint32_t *secs,
+ uint32_t *microsecs)
{
- mach_timespec_t bbc_time, curr_time;
- spl_t s;
-
- if (bbc_gettime(&bbc_time) != KERN_SUCCESS)
- return;
-
- LOCK_RTC(s);
- if (!rtclock.calend_is_set) {
- (void) sysclk_gettime_internal(&curr_time);
- rtclock.calend_offset = bbc_time;
- SUB_MACH_TIMESPEC(&rtclock.calend_offset, &curr_time);
- rtclock.calend_is_set = TRUE;
- }
- UNLOCK_RTC(s);
-}
+ uint64_t now = abstime;
+ uint32_t remain;
-mach_timespec_t
-clock_get_calendar_offset(void)
-{
- mach_timespec_t result = MACH_TIMESPEC_ZERO;
- spl_t s;
+ now += offset;
- LOCK_RTC(s);
- if (rtclock.calend_is_set)
- result = rtclock.calend_offset;
- UNLOCK_RTC(s);
+ asm volatile(
+ "divl %3"
+ : "=a" (*secs), "=d" (remain)
+ : "A" (now), "r" (NSEC_PER_SEC));
+ asm volatile(
+ "divl %3"
+ : "=a" (*microsecs)
+ : "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
- return (result);
+ *secs += epoch;
+
+ commpage_set_timestamp(abstime - remain, *secs);
}
void
clock_timebase_info(
mach_timebase_info_t info)
{
- spl_t s;
-
- LOCK_RTC(s);
- info->numer = info->denom = 1;
- UNLOCK_RTC(s);
+ info->numer = info->denom = 1;
}
-void
-clock_set_timer_deadline(
- uint64_t deadline)
-{
- spl_t s;
-
- LOCK_RTC(s);
- rtclock.timer_deadline = deadline;
- rtclock.timer_is_set = TRUE;
- UNLOCK_RTC(s);
-}
-
void
clock_set_timer_func(
clock_timer_func_t func)
{
- spl_t s;
-
- LOCK_RTC(s);
- if (rtclock.timer_expire == NULL)
- rtclock.timer_expire = func;
- UNLOCK_RTC(s);
-}
-
-\f
-
-/*
- * Load the count register and start the clock.
- */
-#define RTCLOCK_RESET() { \
- outb(PITCTL_PORT, PIT_C0|PIT_NDIVMODE|PIT_READMODE); \
- outb(PITCTR0_PORT, (clks_per_int & 0xff)); \
- outb(PITCTR0_PORT, (clks_per_int >> 8)); \
+ if (rtclock_timer_expire == NULL)
+ rtclock_timer_expire = func;
}
/*
- * Reset the clock device. This causes the realtime clock
- * device to reload its mode and count value (frequency).
- * Note: the CPU should be calibrated
- * before starting the clock for the first time.
+ * Real-time clock device interrupt.
*/
-
void
-rtclock_reset(void)
-{
- int s;
-
-#if NCPUS > 1 && !(MP_V1_1 && 0)
- mp_disable_preemption();
- if (cpu_number() != master_cpu) {
- mp_enable_preemption();
- return;
+rtclock_intr(
+ x86_saved_state_t *tregs)
+{
+ uint64_t rip;
+ boolean_t user_mode = FALSE;
+ uint64_t abstime;
+ uint32_t latency;
+ x86_lcpu_t *lcpu = x86_lcpu();
+
+ assert(get_preemption_level() > 0);
+ assert(!ml_get_interrupts_enabled());
+
+ abstime = rtc_nanotime_read();
+ latency = (uint32_t)(abstime - lcpu->rtcDeadline);
+ if (abstime < lcpu->rtcDeadline)
+ latency = 1;
+
+ if (is_saved_state64(tregs) == TRUE) {
+ x86_saved_state64_t *regs;
+
+ regs = saved_state64(tregs);
+
+ user_mode = TRUE;
+ rip = regs->isf.rip;
+ } else {
+ x86_saved_state32_t *regs;
+
+ regs = saved_state32(tregs);
+
+ if (regs->cs & 0x03)
+ user_mode = TRUE;
+ rip = regs->eip;
}
- mp_enable_preemption();
-#endif /* NCPUS > 1 && AT386 && !MP_V1_1 */
- LOCK_RTC(s);
- RTCLOCK_RESET();
- UNLOCK_RTC(s);
+
+ /* Log the interrupt service latency (-ve value expected by tool) */
+ KERNEL_DEBUG_CONSTANT(
+ MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
+ -latency, (uint32_t)rip, user_mode, 0, 0);
+
+ /* call the generic etimer */
+ etimer_intr(user_mode, rip);
}
/*
- * Real-time clock device interrupt. Called only on the
- * master processor. Updates the clock time and upcalls
- * into the higher level clock code to deliver alarms.
+ * Request timer pop from the hardware
*/
+
int
-rtclock_intr(void)
+setPop(
+ uint64_t time)
{
- uint64_t abstime;
- mach_timespec_t clock_time;
- int i;
- spl_t s;
+ uint64_t now;
+ uint32_t decr;
+ uint64_t count;
+
+ now = rtc_nanotime_read(); /* The time in nanoseconds */
+ decr = deadline_to_decrementer(time, now);
- /*
- * Update clock time. Do the update so that the macro
- * MTS_TO_TS() for reading the mapped time works (e.g.
- * update in order: mtv_csec, mtv_time.tv_nsec, mtv_time.tv_sec).
- */
- LOCK_RTC(s);
- rdtsc_hilo(&rtc_last_int_tsc_hi, &rtc_last_int_tsc_lo);
- i = rtclock.time.tv_nsec + rtclock.intr_nsec;
- if (i < NSEC_PER_SEC)
- rtclock.time.tv_nsec = i;
- else {
- rtclock.time.tv_nsec = i - NSEC_PER_SEC;
- rtclock.time.tv_sec++;
- }
- /* note time now up to date */
- last_ival = 0;
-
- rtclock.abstime += rtclock.intr_nsec;
- abstime = rtclock.abstime;
- if ( rtclock.timer_is_set &&
- rtclock.timer_deadline <= abstime ) {
- rtclock.timer_is_set = FALSE;
- UNLOCK_RTC(s);
+ count = tmrCvt(decr, busFCvtn2t);
+ lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count);
- (*rtclock.timer_expire)(abstime);
-
- LOCK_RTC(s);
- }
-
- /*
- * Perform alarm clock processing if needed. The time
- * passed up is incremented by a half-interrupt tick
- * to trigger alarms closest to their desired times.
- * The clock_alarm_intr() routine calls sysclk_setalrm()
- * before returning if later alarms are pending.
- */
-
- if (RtcAlrm && (RtcAlrm->tv_sec < RtcTime->tv_sec ||
- (RtcAlrm->tv_sec == RtcTime->tv_sec &&
- RtcDelt >= RtcAlrm->tv_nsec - RtcTime->tv_nsec))) {
- clock_time.tv_sec = 0;
- clock_time.tv_nsec = RtcDelt;
- ADD_MACH_TIMESPEC (&clock_time, RtcTime);
- RtcAlrm = 0;
- UNLOCK_RTC(s);
- /*
- * Call clock_alarm_intr() without RTC-lock.
- * The lock ordering is always CLOCK-lock
- * before RTC-lock.
- */
- clock_alarm_intr(SYSTEM_CLOCK, &clock_time);
- LOCK_RTC(s);
- }
-
- /*
- * On a HZ-tick boundary: return 0 and adjust the clock
- * alarm resolution (if requested). Otherwise return a
- * non-zero value.
- */
- if ((i = --rtc_intr_count) == 0) {
- if (rtclock.new_ires) {
- rtc_setvals(new_clknum, rtclock.new_ires);
- RTCLOCK_RESET(); /* lock clock register */
- rtclock.new_ires = 0;
- }
- rtc_intr_count = rtc_intr_hertz;
- }
- UNLOCK_RTC(s);
- return (i);
+ return decr; /* Pass back what we set */
}
-void
-clock_get_uptime(
- uint64_t *result)
-{
- uint32_t ticks;
- spl_t s;
-
- LOCK_RTC(s);
- ticks = get_uptime_cycles();
- *result = rtclock.abstime;
- UNLOCK_RTC(s);
- *result += ticks;
+uint64_t
+mach_absolute_time(void)
+{
+ return rtc_nanotime_read();
}
void
-clock_interval_to_deadline(
+clock_interval_to_absolutetime_interval(
uint32_t interval,
uint32_t scale_factor,
uint64_t *result)
{
- uint64_t abstime;
-
- clock_get_uptime(result);
+ *result = (uint64_t)interval * scale_factor;
+}
- clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
+void
+absolutetime_to_microtime(
+ uint64_t abstime,
+ uint32_t *secs,
+ uint32_t *microsecs)
+{
+ uint32_t remain;
- *result += abstime;
+ asm volatile(
+ "divl %3"
+ : "=a" (*secs), "=d" (remain)
+ : "A" (abstime), "r" (NSEC_PER_SEC));
+ asm volatile(
+ "divl %3"
+ : "=a" (*microsecs)
+ : "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
}
void
-clock_interval_to_absolutetime_interval(
- uint32_t interval,
- uint32_t scale_factor,
- uint64_t *result)
+absolutetime_to_nanotime(
+ uint64_t abstime,
+ uint32_t *secs,
+ uint32_t *nanosecs)
{
- *result = (uint64_t)interval * scale_factor;
+ asm volatile(
+ "divl %3"
+ : "=a" (*secs), "=d" (*nanosecs)
+ : "A" (abstime), "r" (NSEC_PER_SEC));
}
void
-clock_absolutetime_interval_to_deadline(
- uint64_t abstime,
- uint64_t *result)
+nanotime_to_absolutetime(
+ uint32_t secs,
+ uint32_t nanosecs,
+ uint64_t *result)
{
- clock_get_uptime(result);
-
- *result += abstime;
+ *result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs;
}
void
*result = nanoseconds;
}
-/*
- * measure_delay(microseconds)
- *
- * Measure elapsed time for delay calls
- * Returns microseconds.
- *
- * Microseconds must not be too large since the counter (short)
- * will roll over. Max is about 13 ms. Values smaller than 1 ms are ok.
- * This uses the assumed frequency of the rt clock which is emperically
- * accurate to only about 200 ppm.
- */
-
-int
-measure_delay(
- int us)
-{
- unsigned int lsb, val;
-
- outb(PITCTL_PORT, PIT_C0|PIT_NDIVMODE|PIT_READMODE);
- outb(PITCTR0_PORT, 0xff); /* set counter to max value */
- outb(PITCTR0_PORT, 0xff);
- delay(us);
- outb(PITCTL_PORT, PIT_C0);
- lsb = inb(PITCTR0_PORT);
- val = (inb(PITCTR0_PORT) << 8) | lsb;
- val = 0xffff - val;
- val *= 1000000;
- val /= CLKNUM;
- return(val);
-}
-
-/*
- * calibrate_delay(void)
- *
- * Adjust delaycount. Called from startup before clock is started
- * for normal interrupt generation.
- */
-
-void
-calibrate_delay(void)
-{
- unsigned val;
- int prev = 0;
- register int i;
-
- printf("adjusting delay count: %d", delaycount);
- for (i=0; i<10; i++) {
- prev = delaycount;
- /*
- * microdata must not be too large since measure_timer
- * will not return accurate values if the counter (short)
- * rolls over
- */
- val = measure_delay(microdata);
- if (val == 0) {
- delaycount *= 2;
- } else {
- delaycount *= microdata;
- delaycount += val-1; /* round up to upper us */
- delaycount /= val;
- }
- if (delaycount <= 0)
- delaycount = 1;
- if (delaycount != prev)
- printf(" %d", delaycount);
- }
- printf("\n");
-}
-
-#if MACH_KDB
void
-test_delay(void);
-
-void
-test_delay(void)
+machine_delay_until(
+ uint64_t deadline)
{
- register i;
+ uint64_t now;
- for (i = 0; i < 10; i++)
- printf("%d, %d\n", i, measure_delay(i));
- for (i = 10; i <= 100; i+=10)
- printf("%d, %d\n", i, measure_delay(i));
+ do {
+ cpu_pause();
+ now = mach_absolute_time();
+ } while (now < deadline);
}
-#endif /* MACH_KDB */
projects / apple / xnu.git / blobdiff