xnu-2782.40.9.tar.gz

[apple/xnu.git] / osfmk / kern / clock.c

diff --git a/osfmk/kern/clock.c b/osfmk/kern/clock.c
index fd2e29797517ebd34a5d246cdc3f7bd09c8e04e6..1bd578496eebf317883bf4ded4492c61e978fc90 100644 (file)
--- a/osfmk/kern/clock.c
+++ b/osfmk/kern/clock.c
@@ -33,7 +33,6 @@
 
 #include <mach/mach_types.h>
 
 
 #include <mach/mach_types.h>
 

-#include <kern/lock.h>

 #include <kern/spl.h>
 #include <kern/sched_prim.h>
 #include <kern/thread.h>
 #include <kern/spl.h>
 #include <kern/sched_prim.h>
 #include <kern/thread.h>


@@ -50,7 +49,7 @@
 uint32_t       hz_tick_interval = 1;
 
 
 uint32_t       hz_tick_interval = 1;
 
 

-decl_simple_lock_data(static,clock_lock)
+decl_simple_lock_data(,clock_lock)

 
 #define clock_lock()   \
        simple_lock(&clock_lock)
 
 #define clock_lock()   \
        simple_lock(&clock_lock)


@@ -72,7 +71,6 @@ decl_simple_lock_data(static,clock_lock)
  *     where CONV converts absolute time units into seconds and a fraction.
  */
 static struct clock_calend {
  *     where CONV converts absolute time units into seconds and a fraction.
  */
 static struct clock_calend {

-

        uint64_t        epoch;
        uint64_t        offset;
 
        uint64_t        epoch;
        uint64_t        offset;
 


@@ -127,6 +125,9 @@ static thread_call_data_t   calend_wakecall;
 
 extern void    IOKitResetTime(void);
 
 
 extern void    IOKitResetTime(void);
 

+void _clock_delay_until_deadline(uint64_t              interval,
+                                                                uint64_t               deadline);
+

 static uint64_t                clock_boottime;                         /* Seconds boottime epoch */
 
 #define TIME_ADD(rsecs, secs, rfrac, frac, unit)       \
 static uint64_t                clock_boottime;                         /* Seconds boottime epoch */
 
 #define TIME_ADD(rsecs, secs, rfrac, frac, unit)       \


@@ -161,11 +162,6 @@ clock_config(void)
        thread_call_setup(&calend_wakecall, (thread_call_func_t)IOKitResetTime, NULL);
 
        clock_oldconfig();
        thread_call_setup(&calend_wakecall, (thread_call_func_t)IOKitResetTime, NULL);
 
        clock_oldconfig();

-
-       /*
-        * Initialize the timer callouts.
-        */
-       timer_call_initialize();

 }
 
 /*
 }
 
 /*


@@ -234,6 +230,23 @@ void
 clock_get_calendar_microtime(
        clock_sec_t                     *secs,
        clock_usec_t            *microsecs)
 clock_get_calendar_microtime(
        clock_sec_t                     *secs,
        clock_usec_t            *microsecs)

+{
+       clock_get_calendar_absolute_and_microtime(secs, microsecs, NULL);
+}
+
+/*
+ *     clock_get_calendar_absolute_and_microtime:
+ *
+ *     Returns the current calendar value,
+ *     microseconds as the fraction. Also
+ *     returns mach_absolute_time if abstime
+ *     is not NULL.
+ */
+void
+clock_get_calendar_absolute_and_microtime(
+       clock_sec_t                     *secs,
+       clock_usec_t            *microsecs,
+       uint64_t                *abstime)

 {
        uint64_t                now;
        spl_t                   s;
 {
        uint64_t                now;
        spl_t                   s;


@@ -242,10 +255,21 @@ clock_get_calendar_microtime(
        clock_lock();
 
        now = mach_absolute_time();
        clock_lock();
 
        now = mach_absolute_time();

+       if (abstime)
+               *abstime = now;

 
        if (clock_calend.adjdelta < 0) {
                uint32_t        t32;
 
 
        if (clock_calend.adjdelta < 0) {
                uint32_t        t32;
 

+               /* 
+                * Since offset is decremented during a negative adjustment,
+                * ensure that time increases monotonically without going
+                * temporarily backwards.
+                * If the delta has not yet passed, now is set to the start
+                * of the current adjustment period; otherwise, we're between
+                * the expiry of the delta and the next call to calend_adjust(),
+                * and we offset accordingly.
+                */

                if (now > clock_calend.adjstart) {
                        t32 = (uint32_t)(now - clock_calend.adjstart);
 
                if (now > clock_calend.adjstart) {
                        t32 = (uint32_t)(now - clock_calend.adjstart);
 


@@ -305,6 +329,7 @@ clock_get_calendar_nanotime(
        now += clock_calend.offset;
 
        absolutetime_to_microtime(now, secs, nanosecs);
        now += clock_calend.offset;
 
        absolutetime_to_microtime(now, secs, nanosecs);

+

        *nanosecs *= NSEC_PER_USEC;
 
        *secs += (clock_sec_t)clock_calend.epoch;
        *nanosecs *= NSEC_PER_USEC;
 
        *secs += (clock_sec_t)clock_calend.epoch;


@@ -383,9 +408,11 @@ clock_set_calendar_microtime(
        clock_sec_t                     sys;
        clock_usec_t            microsys;
        clock_sec_t                     newsecs;
        clock_sec_t                     sys;
        clock_usec_t            microsys;
        clock_sec_t                     newsecs;

+    clock_usec_t        newmicrosecs;

        spl_t                           s;
 
        spl_t                           s;
 

-       newsecs = (microsecs < 500*USEC_PER_SEC)? secs: secs + 1;
+    newsecs = secs;
+    newmicrosecs = microsecs;

 
        s = splclock();
        clock_lock();
 
        s = splclock();
        clock_lock();


@@ -408,6 +435,7 @@ clock_set_calendar_microtime(
         *      Set the new calendar epoch.
         */
        clock_calend.epoch = secs;
         *      Set the new calendar epoch.
         */
        clock_calend.epoch = secs;

+

        nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset);
 
        /*
        nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset);
 
        /*


@@ -420,7 +448,7 @@ clock_set_calendar_microtime(
        /*
         *      Set the new value for the platform clock.
         */
        /*
         *      Set the new value for the platform clock.
         */

-       PESetGMTTimeOfDay(newsecs);
+       PESetUTCTimeOfDay(newsecs, newmicrosecs);

 
        splx(s);
 
 
        splx(s);
 


@@ -446,10 +474,12 @@ clock_set_calendar_microtime(
 void
 clock_initialize_calendar(void)
 {
 void
 clock_initialize_calendar(void)
 {

-       clock_sec_t                     sys, secs = PEGetGMTTimeOfDay();
-       clock_usec_t            microsys, microsecs = 0;
+       clock_sec_t                     sys, secs;
+       clock_usec_t            microsys, microsecs;

        spl_t                           s;
 
        spl_t                           s;
 

+    PEGetUTCTimeOfDay(&secs, &microsecs);
+

        s = splclock();
        clock_lock();
 
        s = splclock();
        clock_lock();
 


@@ -473,6 +503,7 @@ clock_initialize_calendar(void)
                 *      Set the new calendar epoch.
                 */
                clock_calend.epoch = secs;
                 *      Set the new calendar epoch.
                 */
                clock_calend.epoch = secs;

+

                nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset);
 
                /*
                nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset);
 
                /*


@@ -538,7 +569,7 @@ clock_adjtime(
        interval = calend_set_adjustment(secs, microsecs);
        if (interval != 0) {
                calend_adjdeadline = mach_absolute_time() + interval;
        interval = calend_set_adjustment(secs, microsecs);
        if (interval != 0) {
                calend_adjdeadline = mach_absolute_time() + interval;

-               if (!timer_call_enter(&calend_adjcall, calend_adjdeadline))
+               if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_SYS_CRITICAL))

                        calend_adjactive++;
        }
        else
                        calend_adjactive++;
        }
        else


@@ -558,50 +589,106 @@ calend_set_adjustment(
        int64_t                 total, ototal;
        uint32_t                interval = 0;
 
        int64_t                 total, ototal;
        uint32_t                interval = 0;
 

-       total = (int64_t)*secs * NSEC_PER_SEC + *microsecs * NSEC_PER_USEC;
+       /* 
+        * Compute the total adjustment time in nanoseconds.
+        */
+       total = ((int64_t)*secs * (int64_t)NSEC_PER_SEC) + (*microsecs * (int64_t)NSEC_PER_USEC);

 
 

+       /* 
+        * Disable commpage gettimeofday().
+        */

        commpage_disable_timestamp();
 
        commpage_disable_timestamp();
 

+       /* 
+        * Get current absolute time.
+        */

        now = mach_absolute_time();
 
        now = mach_absolute_time();
 

+       /* 
+        * Save the old adjustment total for later return.
+        */

        ototal = calend_adjtotal;
 
        ototal = calend_adjtotal;
 

+       /*
+        * Is a new correction specified?
+        */

        if (total != 0) {
        if (total != 0) {

+               /*
+                * Set delta to the standard, small, adjustment skew.
+                */

                int32_t         delta = calend_adjskew;
 
                if (total > 0) {
                int32_t         delta = calend_adjskew;
 
                if (total > 0) {

-                       if (total > calend_adjbig)
+                       /*
+                        * Positive adjustment. If greater than the preset 'big' 
+                        * threshold, slew at a faster rate, capping if necessary.
+                        */
+                       if (total > (int64_t) calend_adjbig)

                                delta *= 10;
                        if (delta > total)
                                delta = (int32_t)total;
 
                                delta *= 10;
                        if (delta > total)
                                delta = (int32_t)total;
 

+                       /* 
+                        * Convert the delta back from ns to absolute time and store in adjoffset.
+                        */

                        nanoseconds_to_absolutetime((uint64_t)delta, &t64);
                        clock_calend.adjoffset = (uint32_t)t64;
                }
                else {
                        nanoseconds_to_absolutetime((uint64_t)delta, &t64);
                        clock_calend.adjoffset = (uint32_t)t64;
                }
                else {

-                       if (total < -calend_adjbig)
+                       /*
+                        * Negative adjustment; therefore, negate the delta. If 
+                        * greater than the preset 'big' threshold, slew at a faster 
+                        * rate, capping if necessary.
+                        */
+                       if (total < (int64_t) -calend_adjbig)

                                delta *= 10;
                        delta = -delta;
                        if (delta < total)
                                delta = (int32_t)total;
 
                                delta *= 10;
                        delta = -delta;
                        if (delta < total)
                                delta = (int32_t)total;
 

+                       /* 
+                        * Save the current absolute time. Subsequent time operations occuring
+                        * during this negative correction can make use of this value to ensure 
+                        * that time increases monotonically.
+                        */

                        clock_calend.adjstart = now;
 
                        clock_calend.adjstart = now;
 

+                       /* 
+                        * Convert the delta back from ns to absolute time and store in adjoffset.
+                        */

                        nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
                        clock_calend.adjoffset = (uint32_t)t64;
                }
 
                        nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
                        clock_calend.adjoffset = (uint32_t)t64;
                }
 

+               /* 
+                * Store the total adjustment time in ns. 
+                */

                calend_adjtotal = total;
                calend_adjtotal = total;

+               
+               /* 
+                * Store the delta for this adjustment period in ns. 
+                */

                clock_calend.adjdelta = delta;
 
                clock_calend.adjdelta = delta;
 

+               /* 
+                * Set the interval in absolute time for later return. 
+                */

                interval = calend_adjinterval;
        }
                interval = calend_adjinterval;
        }

-       else
+       else {
+               /* 
+                * No change; clear any prior adjustment.
+                */

                calend_adjtotal = clock_calend.adjdelta = 0;
                calend_adjtotal = clock_calend.adjdelta = 0;

+       }

 
 

+       /* 
+        * If an prior correction was in progress, return the
+        * remaining uncorrected time from it. 
+        */

        if (ototal != 0) {
        if (ototal != 0) {

-               *secs = (long)(ototal / NSEC_PER_SEC);
-               *microsecs = (int)((ototal % NSEC_PER_SEC) / NSEC_PER_USEC);
+               *secs = (long)(ototal / (long)NSEC_PER_SEC);
+               *microsecs = (int)((ototal % (int)NSEC_PER_SEC) / (int)NSEC_PER_USEC);

        }
        else
                *secs = *microsecs = 0;
        }
        else
                *secs = *microsecs = 0;


@@ -627,7 +714,7 @@ calend_adjust_call(void)
                if (interval != 0) {
                        clock_deadline_for_periodic_event(interval, mach_absolute_time(), &calend_adjdeadline);
 
                if (interval != 0) {
                        clock_deadline_for_periodic_event(interval, mach_absolute_time(), &calend_adjdeadline);
 

-                       if (!timer_call_enter(&calend_adjcall, calend_adjdeadline))
+                       if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_SYS_CRITICAL))

                                calend_adjactive++;
                }
        }
                                calend_adjactive++;
                }
        }


@@ -661,21 +748,21 @@ calend_adjust(void)
                }
        }
        else
                }
        }
        else

-       if (delta < 0) {
-               clock_calend.offset -= clock_calend.adjoffset;
+               if (delta < 0) {
+                       clock_calend.offset -= clock_calend.adjoffset;

 
 

-               calend_adjtotal -= delta;
-               if (delta < calend_adjtotal) {
-                       clock_calend.adjdelta = delta = (int32_t)calend_adjtotal;
+                       calend_adjtotal -= delta;
+                       if (delta < calend_adjtotal) {
+                               clock_calend.adjdelta = delta = (int32_t)calend_adjtotal;

 
 

-                       nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
-                       clock_calend.adjoffset = (uint32_t)t64;
+                               nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
+                               clock_calend.adjoffset = (uint32_t)t64;
+                       }
+
+                       if (clock_calend.adjdelta != 0)
+                               clock_calend.adjstart = now;

                }
 
                }
 

-               if (clock_calend.adjdelta != 0)
-                       clock_calend.adjstart = now;
-       }
-       

        if (clock_calend.adjdelta != 0)
                interval = calend_adjinterval;
 
        if (clock_calend.adjdelta != 0)
                interval = calend_adjinterval;
 


@@ -711,6 +798,15 @@ mach_wait_until_continue(
        /*NOTREACHED*/
 }
 
        /*NOTREACHED*/
 }
 

+/*
+ * mach_wait_until_trap: Suspend execution of calling thread until the specified time has passed
+ *
+ * Parameters:    args->deadline          Amount of time to wait
+ *
+ * Returns:        0                      Success
+ *                !0                      Not success           
+ *
+ */

 kern_return_t
 mach_wait_until_trap(
        struct mach_wait_until_trap_args        *args)
 kern_return_t
 mach_wait_until_trap(
        struct mach_wait_until_trap_args        *args)


@@ -718,7 +814,8 @@ mach_wait_until_trap(
        uint64_t                deadline = args->deadline;
        wait_result_t   wresult;
 
        uint64_t                deadline = args->deadline;
        wait_result_t   wresult;
 

-       wresult = assert_wait_deadline((event_t)mach_wait_until_trap, THREAD_ABORTSAFE, deadline);
+       wresult = assert_wait_deadline_with_leeway((event_t)mach_wait_until_trap, THREAD_ABORTSAFE,
+                                                  TIMEOUT_URGENCY_USER_NORMAL, deadline, 0);

        if (wresult == THREAD_WAITING)
                wresult = thread_block(mach_wait_until_continue);
 
        if (wresult == THREAD_WAITING)
                wresult = thread_block(mach_wait_until_continue);
 


@@ -734,27 +831,44 @@ clock_delay_until(
        if (now >= deadline)
                return;
 
        if (now >= deadline)
                return;
 

-       if (    (deadline - now) < (8 * sched_cswtime)  ||
+       _clock_delay_until_deadline(deadline - now, deadline);
+}
+
+/*
+ * Preserve the original precise interval that the client
+ * requested for comparison to the spin threshold.
+ */
+void
+_clock_delay_until_deadline(
+       uint64_t                interval,
+       uint64_t                deadline)
+{
+
+       if (interval == 0)
+               return;
+
+       if (    ml_delay_should_spin(interval)  ||

                        get_preemption_level() != 0                             ||
                        get_preemption_level() != 0                             ||

-                       ml_get_interrupts_enabled() == FALSE    )
-               machine_delay_until(deadline);
-       else {
-               assert_wait_deadline((event_t)clock_delay_until, THREAD_UNINT, deadline - sched_cswtime);
+                       ml_get_interrupts_enabled() == FALSE    ) {
+               machine_delay_until(interval, deadline);
+       } else {
+               assert_wait_deadline((event_t)clock_delay_until, THREAD_UNINT, deadline);

 
                thread_block(THREAD_CONTINUE_NULL);
        }
 }
 
 
                thread_block(THREAD_CONTINUE_NULL);
        }
 }
 

+

 void
 delay_for_interval(
        uint32_t                interval,
        uint32_t                scale_factor)
 {
 void
 delay_for_interval(
        uint32_t                interval,
        uint32_t                scale_factor)
 {

-       uint64_t                end;
+       uint64_t                abstime;

 
 

-       clock_interval_to_deadline(interval, scale_factor, &end);
+       clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);

 
 

-       clock_delay_until(end);
+       _clock_delay_until_deadline(abstime, mach_absolute_time() + abstime);

 }
 
 void
 }
 
 void


@@ -912,3 +1026,4 @@ clock_track_calend_nowait(void)
 }
 
 #endif /* CONFIG_DTRACE */
 }
 
 #endif /* CONFIG_DTRACE */

+