X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..3903760236c30e3b5ace7a4eefac3a269d68957c:/osfmk/kern/clock.c diff --git a/osfmk/kern/clock.c b/osfmk/kern/clock.c index e9c487ad6..6173d89b6 100644 --- a/osfmk/kern/clock.c +++ b/osfmk/kern/clock.c @@ -33,12 +33,13 @@ #include -#include #include #include #include #include #include +#include +#include #include @@ -47,6 +48,8 @@ #include #include +#include + uint32_t hz_tick_interval = 1; @@ -61,6 +64,12 @@ decl_simple_lock_data(,clock_lock) #define clock_lock_init() \ simple_lock_init(&clock_lock, 0) +#ifdef kdp_simple_lock_is_acquired +boolean_t kdp_clock_is_locked() +{ + return kdp_simple_lock_is_acquired(&clock_lock); +} +#endif /* * Time of day (calendar) variables. @@ -74,6 +83,7 @@ decl_simple_lock_data(,clock_lock) static struct clock_calend { uint64_t epoch; uint64_t offset; + uint64_t epoch_absolute; int32_t adjdelta; /* Nanosecond time delta for this adjustment period */ uint64_t adjstart; /* Absolute time value for start of this adjustment period */ @@ -122,11 +132,15 @@ static uint32_t calend_set_adjustment( static void calend_adjust_call(void); static uint32_t calend_adjust(void); -static thread_call_data_t calend_wakecall; - -extern void IOKitResetTime(void); +void _clock_delay_until_deadline(uint64_t interval, + uint64_t deadline); +void _clock_delay_until_deadline_with_leeway(uint64_t interval, + uint64_t deadline, + uint64_t leeway); -static uint64_t clock_boottime; /* Seconds boottime epoch */ +/* Seconds boottime epoch */ +static uint64_t clock_boottime; +static uint32_t clock_boottime_usec; #define TIME_ADD(rsecs, secs, rfrac, frac, unit) \ MACRO_BEGIN \ @@ -157,7 +171,6 @@ clock_config(void) clock_lock_init(); timer_call_setup(&calend_adjcall, (timer_call_func_t)calend_adjust_call, NULL); - thread_call_setup(&calend_wakecall, (thread_call_func_t)IOKitResetTime, NULL); clock_oldconfig(); } @@ -229,18 +242,23 @@ clock_get_calendar_microtime( clock_sec_t *secs, clock_usec_t *microsecs) { - uint64_t now; - spl_t s; - - s = splclock(); - clock_lock(); + clock_get_calendar_absolute_and_microtime(secs, microsecs, NULL); +} - now = mach_absolute_time(); +static void +clock_get_calendar_absolute_and_microtime_locked( + clock_sec_t *secs, + clock_usec_t *microsecs, + uint64_t *abstime) +{ + uint64_t now = mach_absolute_time(); + if (abstime) + *abstime = now; if (clock_calend.adjdelta < 0) { uint32_t t32; - /* + /* * Since offset is decremented during a negative adjustment, * ensure that time increases monotonically without going * temporarily backwards. @@ -264,6 +282,28 @@ clock_get_calendar_microtime( absolutetime_to_microtime(now, secs, microsecs); *secs += (clock_sec_t)clock_calend.epoch; +} + +/* + * clock_get_calendar_absolute_and_microtime: + * + * Returns the current calendar value, + * microseconds as the fraction. Also + * returns mach_absolute_time if abstime + * is not NULL. + */ +void +clock_get_calendar_absolute_and_microtime( + clock_sec_t *secs, + clock_usec_t *microsecs, + uint64_t *abstime) +{ + spl_t s; + + s = splclock(); + clock_lock(); + + clock_get_calendar_absolute_and_microtime_locked(secs, microsecs, abstime); clock_unlock(); splx(s); @@ -284,35 +324,15 @@ clock_get_calendar_nanotime( clock_sec_t *secs, clock_nsec_t *nanosecs) { - uint64_t now; spl_t s; s = splclock(); clock_lock(); - now = mach_absolute_time(); - - if (clock_calend.adjdelta < 0) { - uint32_t t32; - - if (now > clock_calend.adjstart) { - t32 = (uint32_t)(now - clock_calend.adjstart); - - if (t32 > clock_calend.adjoffset) - now -= clock_calend.adjoffset; - else - now = clock_calend.adjstart; - } - } - - now += clock_calend.offset; - - absolutetime_to_microtime(now, secs, nanosecs); + clock_get_calendar_absolute_and_microtime_locked(secs, nanosecs, NULL); *nanosecs *= NSEC_PER_USEC; - *secs += (clock_sec_t)clock_calend.epoch; - clock_unlock(); splx(s); } @@ -332,6 +352,15 @@ void clock_gettimeofday( clock_sec_t *secs, clock_usec_t *microsecs) +{ + clock_gettimeofday_and_absolute_time(secs, microsecs, NULL); +} + +void +clock_gettimeofday_and_absolute_time( + clock_sec_t *secs, + clock_usec_t *microsecs, + uint64_t *mach_time) { uint64_t now; spl_t s; @@ -365,6 +394,10 @@ clock_gettimeofday( clock_unlock(); splx(s); + + if (mach_time) { + *mach_time = now; + } } /* @@ -386,10 +419,16 @@ clock_set_calendar_microtime( { clock_sec_t sys; clock_usec_t microsys; + uint64_t absolutesys; clock_sec_t newsecs; + clock_sec_t oldsecs; + clock_usec_t newmicrosecs; + clock_usec_t oldmicrosecs; + uint64_t commpage_value; spl_t s; - newsecs = (microsecs < 500*USEC_PER_SEC)? secs: secs + 1; + newsecs = secs; + newmicrosecs = microsecs; s = splclock(); clock_lock(); @@ -397,16 +436,28 @@ clock_set_calendar_microtime( commpage_disable_timestamp(); /* - * Calculate the new calendar epoch based on - * the new value and the system clock. + * Adjust the boottime based on the delta. */ - clock_get_system_microtime(&sys, µsys); - TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC); + clock_get_calendar_absolute_and_microtime_locked(&oldsecs, &oldmicrosecs, &absolutesys); + if (oldsecs < secs || (oldsecs == secs && oldmicrosecs < microsecs)){ + // moving forwards + long deltasecs = secs, deltamicrosecs = microsecs; + TIME_SUB(deltasecs, oldsecs, deltamicrosecs, oldmicrosecs, USEC_PER_SEC); + TIME_ADD(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC); + } else { + // moving backwards + long deltasecs = oldsecs, deltamicrosecs = oldmicrosecs; + TIME_SUB(deltasecs, secs, deltamicrosecs, microsecs, USEC_PER_SEC); + TIME_SUB(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC); + } + commpage_value = clock_boottime * USEC_PER_SEC + clock_boottime_usec; /* - * Adjust the boottime based on the delta. + * Calculate the new calendar epoch based on + * the new value and the system clock. */ - clock_boottime += secs - clock_calend.epoch; + absolutetime_to_microtime(absolutesys, &sys, µsys); + TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC); /* * Set the new calendar epoch. @@ -415,6 +466,9 @@ clock_set_calendar_microtime( nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset); + clock_interval_to_absolutetime_interval((uint32_t) secs, NSEC_PER_SEC, &clock_calend.epoch_absolute); + clock_calend.epoch_absolute += clock_calend.offset; + /* * Cancel any adjustment in progress. */ @@ -425,15 +479,18 @@ clock_set_calendar_microtime( /* * Set the new value for the platform clock. */ - PESetGMTTimeOfDay(newsecs); + PESetUTCTimeOfDay(newsecs, newmicrosecs); splx(s); + commpage_update_boottime(commpage_value); + /* * Send host notifications. */ host_notify_calendar_change(); - + host_notify_calendar_set(); + #if CONFIG_DTRACE clock_track_calend_nowait(); #endif @@ -448,13 +505,24 @@ clock_set_calendar_microtime( * * Also sends host notifications. */ + +uint64_t mach_absolutetime_asleep; +uint64_t mach_absolutetime_last_sleep; + void clock_initialize_calendar(void) { - clock_sec_t sys, secs = PEGetGMTTimeOfDay(); - clock_usec_t microsys, microsecs = 0; + clock_sec_t sys; // sleepless time since boot in seconds + clock_sec_t secs; // Current UTC time + clock_sec_t utc_offset_secs; // Difference in current UTC time and sleepless time since boot + clock_usec_t microsys; + clock_usec_t microsecs; + clock_usec_t utc_offset_microsecs; + uint64_t new_epoch; // utc_offset_secs in mach absolute time units spl_t s; + PEGetUTCTimeOfDay(&secs, µsecs); + s = splclock(); clock_lock(); @@ -464,22 +532,59 @@ clock_initialize_calendar(void) /* * Initialize the boot time based on the platform clock. */ - if (clock_boottime == 0) + if (clock_boottime == 0){ clock_boottime = secs; + clock_boottime_usec = microsecs; + commpage_update_boottime(clock_boottime * USEC_PER_SEC + clock_boottime_usec); + } /* * Calculate the new calendar epoch based on * the platform clock and the system clock. */ clock_get_system_microtime(&sys, µsys); - TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC); + utc_offset_secs = secs; + utc_offset_microsecs = microsecs; + + // This macro mutates utc_offset_secs and micro_utc_offset + TIME_SUB(utc_offset_secs, sys, utc_offset_microsecs, microsys, USEC_PER_SEC); /* * Set the new calendar epoch. */ - clock_calend.epoch = secs; - nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset); + clock_calend.epoch = utc_offset_secs; + + nanoseconds_to_absolutetime((uint64_t)utc_offset_microsecs * NSEC_PER_USEC, &clock_calend.offset); + + clock_interval_to_absolutetime_interval((uint32_t) utc_offset_secs, NSEC_PER_SEC, &new_epoch); + new_epoch += clock_calend.offset; + + if (clock_calend.epoch_absolute) + { + /* new_epoch is the difference between absolute_time and utc_time + * this value will remain constant until the system sleeps. + * Then, difference between values would go up by the time the system sleeps. + * epoch_absolute is the last difference between the two values + * so the difference in the differences would be the time of the last sleep + */ + + if(new_epoch > clock_calend.epoch_absolute) { + mach_absolutetime_last_sleep = new_epoch - clock_calend.epoch_absolute; + } + else { + mach_absolutetime_last_sleep = 0; + } + mach_absolutetime_asleep += mach_absolutetime_last_sleep; + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_CLOCK,MACH_EPOCH_CHANGE) | DBG_FUNC_NONE, + (uintptr_t) mach_absolutetime_last_sleep, + (uintptr_t) mach_absolutetime_asleep, + (uintptr_t) (mach_absolutetime_last_sleep >> 32), + (uintptr_t) (mach_absolutetime_asleep >> 32), + 0); + } + clock_calend.epoch_absolute = new_epoch; /* * Cancel any adjustment in progress. @@ -487,6 +592,9 @@ clock_initialize_calendar(void) calend_adjtotal = clock_calend.adjdelta = 0; } + commpage_update_mach_continuous_time(mach_absolutetime_asleep); + adjust_cont_time_thread_calls(); + clock_unlock(); splx(s); @@ -516,7 +624,29 @@ clock_get_boottime_nanotime( clock_lock(); *secs = (clock_sec_t)clock_boottime; - *nanosecs = 0; + *nanosecs = (clock_nsec_t)clock_boottime_usec * NSEC_PER_USEC; + + clock_unlock(); + splx(s); +} + +/* + * clock_get_boottime_nanotime: + * + * Return the boottime, used by sysctl. + */ +void +clock_get_boottime_microtime( + clock_sec_t *secs, + clock_usec_t *microsecs) +{ + spl_t s; + + s = splclock(); + clock_lock(); + + *secs = (clock_sec_t)clock_boottime; + *microsecs = (clock_nsec_t)clock_boottime_usec; clock_unlock(); splx(s); @@ -544,7 +674,7 @@ clock_adjtime( interval = calend_set_adjustment(secs, microsecs); if (interval != 0) { calend_adjdeadline = mach_absolute_time() + interval; - if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_CRITICAL)) + if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_SYS_CRITICAL)) calend_adjactive++; } else @@ -567,7 +697,7 @@ calend_set_adjustment( /* * Compute the total adjustment time in nanoseconds. */ - total = (int64_t)*secs * NSEC_PER_SEC + *microsecs * NSEC_PER_USEC; + total = ((int64_t)*secs * (int64_t)NSEC_PER_SEC) + (*microsecs * (int64_t)NSEC_PER_USEC); /* * Disable commpage gettimeofday(). @@ -598,7 +728,7 @@ calend_set_adjustment( * Positive adjustment. If greater than the preset 'big' * threshold, slew at a faster rate, capping if necessary. */ - if (total > calend_adjbig) + if (total > (int64_t) calend_adjbig) delta *= 10; if (delta > total) delta = (int32_t)total; @@ -615,7 +745,7 @@ calend_set_adjustment( * greater than the preset 'big' threshold, slew at a faster * rate, capping if necessary. */ - if (total < -calend_adjbig) + if (total < (int64_t) -calend_adjbig) delta *= 10; delta = -delta; if (delta < total) @@ -662,8 +792,8 @@ calend_set_adjustment( * remaining uncorrected time from it. */ if (ototal != 0) { - *secs = (long)(ototal / NSEC_PER_SEC); - *microsecs = (int)((ototal % NSEC_PER_SEC) / NSEC_PER_USEC); + *secs = (long)(ototal / (long)NSEC_PER_SEC); + *microsecs = (int)((ototal % (int)NSEC_PER_SEC) / (int)NSEC_PER_USEC); } else *secs = *microsecs = 0; @@ -689,7 +819,7 @@ calend_adjust_call(void) if (interval != 0) { clock_deadline_for_periodic_event(interval, mach_absolute_time(), &calend_adjdeadline); - if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_CRITICAL)) + if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_SYS_CRITICAL)) calend_adjactive++; } } @@ -748,19 +878,6 @@ calend_adjust(void) return (interval); } -/* - * clock_wakeup_calendar: - * - * Interface to power management, used - * to initiate the reset of the calendar - * on wake from sleep event. - */ -void -clock_wakeup_calendar(void) -{ - thread_call_enter(&calend_wakecall); -} - /* * Wait / delay routines. */ @@ -773,6 +890,15 @@ mach_wait_until_continue( /*NOTREACHED*/ } +/* + * mach_wait_until_trap: Suspend execution of calling thread until the specified time has passed + * + * Parameters: args->deadline Amount of time to wait + * + * Returns: 0 Success + * !0 Not success + * + */ kern_return_t mach_wait_until_trap( struct mach_wait_until_trap_args *args) @@ -780,7 +906,8 @@ mach_wait_until_trap( uint64_t deadline = args->deadline; wait_result_t wresult; - wresult = assert_wait_deadline((event_t)mach_wait_until_trap, THREAD_ABORTSAFE, deadline); + wresult = assert_wait_deadline_with_leeway((event_t)mach_wait_until_trap, THREAD_ABORTSAFE, + TIMEOUT_URGENCY_USER_NORMAL, deadline, 0); if (wresult == THREAD_WAITING) wresult = thread_block(mach_wait_until_continue); @@ -796,12 +923,50 @@ clock_delay_until( if (now >= deadline) return; - if ( (deadline - now) < (8 * sched_cswtime) || + _clock_delay_until_deadline(deadline - now, deadline); +} + +/* + * Preserve the original precise interval that the client + * requested for comparison to the spin threshold. + */ +void +_clock_delay_until_deadline( + uint64_t interval, + uint64_t deadline) +{ + _clock_delay_until_deadline_with_leeway(interval, deadline, 0); +} + +/* + * Like _clock_delay_until_deadline, but it accepts a + * leeway value. + */ +void +_clock_delay_until_deadline_with_leeway( + uint64_t interval, + uint64_t deadline, + uint64_t leeway) +{ + + if (interval == 0) + return; + + if ( ml_delay_should_spin(interval) || get_preemption_level() != 0 || - ml_get_interrupts_enabled() == FALSE ) - machine_delay_until(deadline); - else { - assert_wait_deadline((event_t)clock_delay_until, THREAD_UNINT, deadline - sched_cswtime); + ml_get_interrupts_enabled() == FALSE ) { + machine_delay_until(interval, deadline); + } else { + /* + * For now, assume a leeway request of 0 means the client does not want a leeway + * value. We may want to change this interpretation in the future. + */ + + if (leeway) { + assert_wait_deadline_with_leeway((event_t)clock_delay_until, THREAD_UNINT, TIMEOUT_URGENCY_LEEWAY, deadline, leeway); + } else { + assert_wait_deadline((event_t)clock_delay_until, THREAD_UNINT, deadline); + } thread_block(THREAD_CONTINUE_NULL); } @@ -812,11 +977,26 @@ delay_for_interval( uint32_t interval, uint32_t scale_factor) { - uint64_t end; + uint64_t abstime; + + clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime); + + _clock_delay_until_deadline(abstime, mach_absolute_time() + abstime); +} + +void +delay_for_interval_with_leeway( + uint32_t interval, + uint32_t leeway, + uint32_t scale_factor) +{ + uint64_t abstime_interval; + uint64_t abstime_leeway; - clock_interval_to_deadline(interval, scale_factor, &end); + clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime_interval); + clock_interval_to_absolutetime_interval(leeway, scale_factor, &abstime_leeway); - clock_delay_until(end); + _clock_delay_until_deadline_with_leeway(abstime_interval, mach_absolute_time() + abstime_interval, abstime_leeway); } void @@ -850,6 +1030,14 @@ clock_absolutetime_interval_to_deadline( *result = mach_absolute_time() + abstime; } +void +clock_continuoustime_interval_to_deadline( + uint64_t conttime, + uint64_t *result) +{ + *result = mach_continuous_time() + conttime; +} + void clock_get_uptime( uint64_t *result) @@ -876,6 +1064,61 @@ clock_deadline_for_periodic_event( } } +uint64_t +mach_continuous_time(void) +{ + while(1) { + uint64_t read1 = mach_absolutetime_asleep; + uint64_t absolute = mach_absolute_time(); + OSMemoryBarrier(); + uint64_t read2 = mach_absolutetime_asleep; + + if(__builtin_expect(read1 == read2, 1)) { + return absolute + read1; + } + } +} + +uint64_t +mach_continuous_approximate_time(void) +{ + while(1) { + uint64_t read1 = mach_absolutetime_asleep; + uint64_t absolute = mach_approximate_time(); + OSMemoryBarrier(); + uint64_t read2 = mach_absolutetime_asleep; + + if(__builtin_expect(read1 == read2, 1)) { + return absolute + read1; + } + } +} + +/* + * continuoustime_to_absolutetime + * Must be called with interrupts disabled + * Returned value is only valid until the next update to + * mach_continuous_time + */ +uint64_t +continuoustime_to_absolutetime(uint64_t conttime) { + if (conttime <= mach_absolutetime_asleep) + return 0; + else + return conttime - mach_absolutetime_asleep; +} + +/* + * absolutetime_to_continuoustime + * Must be called with interrupts disabled + * Returned value is only valid until the next update to + * mach_continuous_time + */ +uint64_t +absolutetime_to_continuoustime(uint64_t abstime) { + return abstime + mach_absolutetime_asleep; +} + #if CONFIG_DTRACE /* @@ -974,3 +1217,4 @@ clock_track_calend_nowait(void) } #endif /* CONFIG_DTRACE */ +