X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..ff6e181ae92fc6f1e89841290f461d1f2f9badd9:/osfmk/kern/timer.c diff --git a/osfmk/kern/timer.c b/osfmk/kern/timer.c index 996bc6fe6..845314fbf 100644 --- a/osfmk/kern/timer.c +++ b/osfmk/kern/timer.c @@ -3,19 +3,20 @@ * * @APPLE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ */ @@ -50,542 +51,129 @@ /* */ -#include #include +#include #include #include #include +#include #include #include -#include #include -#include - -#include -#include - -timer_t current_timer[NCPUS]; -timer_data_t kernel_timer[NCPUS]; - -/* Forwards */ -void timer_grab( - timer_t timer, - timer_save_t save); - -void db_timer_grab( - timer_t timer, - timer_save_t save); - -void db_thread_read_times( - thread_t thread, - time_value_t *user_time_p, - time_value_t *system_time_p); /* - * init_timers initializes all non-thread timers and puts the - * service routine on the callout queue. All timers must be - * serviced by the callout routine once an hour. - */ -void -init_timers(void) -{ - register int i; - register timer_t this_timer; - - /* - * Initialize all the kernel timers and start the one - * for this cpu (master) slaves start theirs later. - */ - this_timer = &kernel_timer[0]; - for ( i=0 ; ilow_bits = 0; - this_timer->high_bits = 0; - this_timer->tstamp = 0; - this_timer->high_bits_check = 0; -} - -#if STAT_TIME -#else /* STAT_TIME */ - -#ifdef MACHINE_TIMER_ROUTINES - -/* - * Machine-dependent code implements the timer routines. - */ - -#else /* MACHINE_TIMER_ROUTINES */ - -/* - * start_timer starts the given timer for this cpu. It is called - * exactly once for each cpu during the boot sequence. - */ -void -start_timer( - register timer_t timer) -{ - timer->tstamp = get_timestamp(); - mp_disable_preemption(); - current_timer[cpu_number()] = timer; - mp_enable_preemption(); -} - -/* - * time_trap_uentry does trap entry timing. Caller must lock out - * interrupts and take a timestamp. ts is a timestamp taken after - * interrupts were locked out. Must only be called if trap was - * from user mode. - */ -void -time_trap_uentry( - unsigned ts) -{ - int elapsed; - int mycpu; - timer_t mytimer; - - mp_disable_preemption(); - - /* - * Calculate elapsed time. - */ - mycpu = cpu_number(); - mytimer = current_timer[mycpu]; - elapsed = ts - mytimer->tstamp; -#ifdef TIMER_MAX - if (elapsed < 0) elapsed += TIMER_MAX; -#endif /* TIMER_MAX */ - - /* - * Update current timer. - */ - mytimer->low_bits += elapsed; - mytimer->tstamp = 0; - - if (mytimer->low_bits & TIMER_LOW_FULL) { - timer_normalize(mytimer); - } - - /* - * Record new timer. - */ - mytimer = &(current_thread()->system_timer); - current_timer[mycpu] = mytimer; - mytimer->tstamp = ts; - - mp_enable_preemption(); -} - -/* - * time_trap_uexit does trap exit timing. Caller must lock out - * interrupts and take a timestamp. ts is a timestamp taken after - * interrupts were locked out. Must only be called if returning to - * user mode. - */ -void -time_trap_uexit( - unsigned ts) -{ - int elapsed; - int mycpu; - timer_t mytimer; - - mp_disable_preemption(); - - /* - * Calculate elapsed time. - */ - mycpu = cpu_number(); - mytimer = current_timer[mycpu]; - elapsed = ts - mytimer->tstamp; -#ifdef TIMER_MAX - if (elapsed < 0) elapsed += TIMER_MAX; -#endif /* TIMER_MAX */ - - /* - * Update current timer. - */ - mytimer->low_bits += elapsed; - mytimer->tstamp = 0; - - if (mytimer->low_bits & TIMER_LOW_FULL) { - timer_normalize(mytimer); /* SYSTEMMODE */ - } - - mytimer = &(current_thread()->user_timer); - - /* - * Record new timer. - */ - current_timer[mycpu] = mytimer; - mytimer->tstamp = ts; - - mp_enable_preemption(); + timer->low_bits = 0; + timer->high_bits = 0; + timer->high_bits_check = 0; +#if !STAT_TIME + timer->tstamp = 0; +#endif /* STAT_TIME */ } /* - * time_int_entry does interrupt entry timing. Caller must lock out - * interrupts and take a timestamp. ts is a timestamp taken after - * interrupts were locked out. new_timer is the new timer to - * switch to. This routine returns the currently running timer, - * which MUST be pushed onto the stack by the caller, or otherwise - * saved for time_int_exit. + * Calculate the difference between a timer + * and saved value, and update the saved value. */ -timer_t -time_int_entry( - unsigned ts, - timer_t new_timer) +uint64_t +timer_delta( + timer_t timer, + uint64_t *save) { - int elapsed; - int mycpu; - timer_t mytimer; - - mp_disable_preemption(); - - /* - * Calculate elapsed time. - */ - mycpu = cpu_number(); - mytimer = current_timer[mycpu]; - - elapsed = ts - mytimer->tstamp; -#ifdef TIMER_MAX - if (elapsed < 0) elapsed += TIMER_MAX; -#endif /* TIMER_MAX */ + uint64_t new, old = *save; - /* - * Update current timer. - */ - mytimer->low_bits += elapsed; - mytimer->tstamp = 0; - - /* - * Switch to new timer, and save old one on stack. - */ - new_timer->tstamp = ts; - current_timer[mycpu] = new_timer; + *save = new = timer_grab(timer); - mp_enable_preemption(); - - return(mytimer); + return (new - old); } -/* - * time_int_exit does interrupt exit timing. Caller must lock out - * interrupts and take a timestamp. ts is a timestamp taken after - * interrupts were locked out. old_timer is the timer value pushed - * onto the stack or otherwise saved after time_int_entry returned - * it. - */ -void -time_int_exit( - unsigned ts, - timer_t old_timer) -{ - int elapsed; - int mycpu; - timer_t mytimer; - - mp_disable_preemption(); - - /* - * Calculate elapsed time. - */ - mycpu = cpu_number(); - mytimer = current_timer[mycpu]; - elapsed = ts - mytimer->tstamp; -#ifdef TIMER_MAX - if (elapsed < 0) elapsed += TIMER_MAX; -#endif /* TIMER_MAX */ - - /* - * Update current timer. - */ - mytimer->low_bits += elapsed; - mytimer->tstamp = 0; - - /* - * If normalization requested, do it. - */ - if (mytimer->low_bits & TIMER_LOW_FULL) { - timer_normalize(mytimer); - } - if (old_timer->low_bits & TIMER_LOW_FULL) { - timer_normalize(old_timer); - } - - /* - * Start timer that was running before interrupt. - */ - old_timer->tstamp = ts; - current_timer[mycpu] = old_timer; - - mp_enable_preemption(); -} +#if !STAT_TIME /* - * timer_switch switches to a new timer. The machine - * dependent routine/macro get_timestamp must return a timestamp. - * Caller must lock out interrupts. + * Update the current timer (if any) + * and start the new timer, which + * could be either the same or NULL. + * + * Called with interrupts disabled. */ void timer_switch( - timer_t new_timer) + uint32_t tstamp, + timer_t new_timer) { - int elapsed; - int mycpu; - timer_t mytimer; - unsigned ts; - - mp_disable_preemption(); - - /* - * Calculate elapsed time. - */ - mycpu = cpu_number(); - mytimer = current_timer[mycpu]; - ts = get_timestamp(); - elapsed = ts - mytimer->tstamp; -#ifdef TIMER_MAX - if (elapsed < 0) elapsed += TIMER_MAX; -#endif /* TIMER_MAX */ + processor_t processor = current_processor(); + timer_t timer; + uint32_t old_low, low; /* * Update current timer. */ - mytimer->low_bits += elapsed; - mytimer->tstamp = 0; - - /* - * Normalization check - */ - if (mytimer->low_bits & TIMER_LOW_FULL) { - timer_normalize(mytimer); + timer = PROCESSOR_DATA(processor, current_timer); + if (timer != NULL) { + old_low = timer->low_bits; + low = old_low + tstamp - timer->tstamp; + if (low < old_low) + timer_update(timer, timer->high_bits + 1, low); + else + timer->low_bits = low; } /* - * Record new timer. + * Start new timer. */ - current_timer[mycpu] = new_timer; - new_timer->tstamp = ts; - - mp_enable_preemption(); + PROCESSOR_DATA(processor, current_timer) = new_timer; + if (new_timer != NULL) + new_timer->tstamp = tstamp; } -#endif /* MACHINE_TIMER_ROUTINES */ -#endif /* STAT_TIME */ +#if MACHINE_TIMER_ROUTINES /* - * timer_normalize normalizes the value of a timer. It is - * called only rarely, to make sure low_bits never overflows. + * Machine-dependent code implements the timer event routine. */ -void -timer_normalize( - register timer_t timer) -{ - unsigned int high_increment; - - /* - * Calculate high_increment, then write high check field first - * followed by low and high. timer_grab() reads these fields in - * reverse order so if high and high check match, we know - * that the values read are ok. - */ - - high_increment = timer->low_bits/TIMER_HIGH_UNIT; - timer->high_bits_check += high_increment; - timer->low_bits %= TIMER_HIGH_UNIT; - timer->high_bits += high_increment; -} +#else /* MACHINE_TIMER_ROUTINES */ /* - * timer_grab() retrieves the value of a timer. + * Update the current timer and start + * the new timer. Requires a current + * and new timer. * - * Critical scheduling code uses TIMER_DELTA macro in timer.h - * (called from thread_timer_delta in sched.h). - * - * Keep coherent with db_time_grab below. + * Called with interrupts disabled. */ - void -timer_grab( - timer_t timer, - timer_save_t save) +timer_event( + uint32_t tstamp, + timer_t new_timer) { -#if MACH_ASSERT - unsigned int passes=0; -#endif - do { - (save)->high = (timer)->high_bits; - (save)->low = (timer)->low_bits; + processor_t processor = current_processor(); + timer_t timer; + uint32_t old_low, low; + /* - * If the timer was normalized while we were doing this, - * the high_bits value read above and the high_bits check - * value will not match because high_bits_check is the first - * field touched by the normalization procedure, and - * high_bits is the last. - * - * Additions to timer only touch low bits and - * are therefore atomic with respect to this. + * Update current timer. */ -#if MACH_ASSERT - passes++; - assert(passes < 10000); -#endif - } while ( (save)->high != (timer)->high_bits_check); -} + timer = PROCESSOR_DATA(processor, current_timer); + old_low = timer->low_bits; + low = old_low + tstamp - timer->tstamp; + if (low < old_low) + timer_update(timer, timer->high_bits + 1, low); + else + timer->low_bits = low; -/* - * - * Db_timer_grab(): used by db_thread_read_times. An nonblocking - * version of db_thread_get_times. Keep coherent with timer_grab - * above. - * - */ -void -db_timer_grab( - timer_t timer, - timer_save_t save) -{ - /* Don't worry about coherency */ - - (save)->high = (timer)->high_bits; - (save)->low = (timer)->low_bits; -} - - -/* - * timer_read reads the value of a timer into a time_value_t. If the - * timer was modified during the read, retry. The value returned - * is accurate to the last update; time accumulated by a running - * timer since its last timestamp is not included. - */ - -void -timer_read( - timer_t timer, - register time_value_t *tv) -{ - timer_save_data_t temp; - - timer_grab(timer,&temp); /* - * Normalize the result + * Start new timer. */ -#ifdef TIMER_ADJUST - TIMER_ADJUST(&temp); -#endif /* TIMER_ADJUST */ - tv->seconds = temp.high + temp.low/1000000; - tv->microseconds = temp.low%1000000; -} - -/* - * thread_read_times reads the user and system times from a thread. - * Time accumulated since last timestamp is not included. Should - * be called at splsched() to avoid having user and system times - * be out of step. Doesn't care if caller locked thread. - * - * Needs to be kept coherent with thread_read_times ahead. - */ -void -thread_read_times( - thread_t thread, - time_value_t *user_time_p, - time_value_t *system_time_p) -{ - timer_save_data_t temp; - register timer_t timer; - - timer = &thread->user_timer; - timer_grab(timer, &temp); - -#ifdef TIMER_ADJUST - TIMER_ADJUST(&temp); -#endif /* TIMER_ADJUST */ - user_time_p->seconds = temp.high + temp.low/1000000; - user_time_p->microseconds = temp.low % 1000000; - - timer = &thread->system_timer; - timer_grab(timer, &temp); - -#ifdef TIMER_ADJUST - TIMER_ADJUST(&temp); -#endif /* TIMER_ADJUST */ - system_time_p->seconds = temp.high + temp.low/1000000; - system_time_p->microseconds = temp.low % 1000000; -} - -/* - * Db_thread_read_times: A version of thread_read_times that - * can be called by the debugger. This version does not call - * timer_grab, which can block. Please keep it up to date with - * thread_read_times above. - * - */ -void -db_thread_read_times( - thread_t thread, - time_value_t *user_time_p, - time_value_t *system_time_p) -{ - timer_save_data_t temp; - register timer_t timer; - - timer = &thread->user_timer; - db_timer_grab(timer, &temp); - -#ifdef TIMER_ADJUST - TIMER_ADJUST(&temp); -#endif /* TIMER_ADJUST */ - user_time_p->seconds = temp.high + temp.low/1000000; - user_time_p->microseconds = temp.low % 1000000; - - timer = &thread->system_timer; - timer_grab(timer, &temp); - -#ifdef TIMER_ADJUST - TIMER_ADJUST(&temp); -#endif /* TIMER_ADJUST */ - system_time_p->seconds = temp.high + temp.low/1000000; - system_time_p->microseconds = temp.low % 1000000; + PROCESSOR_DATA(processor, current_timer) = new_timer; + new_timer->tstamp = tstamp; } -/* - * timer_delta takes the difference of a saved timer value - * and the current one, and updates the saved value to current. - * The difference is returned as a function value. See - * TIMER_DELTA macro (timer.h) for optimization to this. - */ - -unsigned -timer_delta( - register timer_t timer, - timer_save_t save) -{ - timer_save_data_t new_save; - register unsigned result; +#endif /* MACHINE_TIMER_ROUTINES */ - timer_grab(timer,&new_save); - result = (new_save.high - save->high) * TIMER_HIGH_UNIT + - new_save.low - save->low; - save->high = new_save.high; - save->low = new_save.low; - return(result); -} +#endif /* STAT_TIME */