osfmk/i386/i386_timer.c

   1 /*
   2  * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * @APPLE_FREE_COPYRIGHT@
  33  */
  34 /*
  35  *      File:           timer.c
  36  *      Purpose:        Routines for handling the machine independent timer.
  37  */
  38
  39 #include <mach/mach_types.h>
  40
  41 #include <kern/timer_queue.h>
  42 #include <kern/timer_call.h>
  43 #include <kern/clock.h>
  44 #include <kern/thread.h>
  45 #include <kern/processor.h>
  46 #include <kern/macro_help.h>
  47 #include <kern/spl.h>
  48 #include <kern/timer_queue.h>
  49 #include <kern/pms.h>
  50
  51 #include <machine/commpage.h>
  52 #include <machine/machine_routines.h>
  53
  54 #include <sys/kdebug.h>
  55 #include <i386/cpu_data.h>
  56 #include <i386/cpu_topology.h>
  57 #include <i386/cpu_threads.h>
  58
  59 uint32_t spurious_timers;
  60
  61 /*
  62  *      Event timer interrupt.
  63  *
  64  * XXX a drawback of this implementation is that events serviced earlier must not set deadlines
  65  *     that occur before the entire chain completes.
  66  *
  67  * XXX a better implementation would use a set of generic callouts and iterate over them
  68  */
  69 void
  70 timer_intr(int          user_mode,
  71     uint64_t    rip)
  72 {
  73         uint64_t                abstime;
  74         rtclock_timer_t         *mytimer;
  75         cpu_data_t              *pp;
  76         int64_t                 latency;
  77         uint64_t                pmdeadline;
  78         boolean_t               timer_processed = FALSE;
  79
  80         pp = current_cpu_datap();
  81
  82         SCHED_STATS_TIMER_POP(current_processor());
  83
  84         abstime = mach_absolute_time();         /* Get the time now */
  85
  86         /* has a pending clock timer expired? */
  87         mytimer = &pp->rtclock_timer;           /* Point to the event timer */
  88
  89         if ((timer_processed = ((mytimer->deadline <= abstime) ||
  90             (abstime >= (mytimer->queue.earliest_soft_deadline))))) {
  91                 /*
  92                  * Log interrupt service latency (-ve value expected by tool)
  93                  * a non-PM event is expected next.
  94                  * The requested deadline may be earlier than when it was set
  95                  * - use MAX to avoid reporting bogus latencies.
  96                  */
  97                 latency = (int64_t) (abstime - MAX(mytimer->deadline,
  98                     mytimer->when_set));
  99                 /* Log zero timer latencies when opportunistically processing
 100                  * coalesced timers.
 101                  */
 102                 if (latency < 0) {
 103                         TCOAL_DEBUG(0xEEEE0000, abstime, mytimer->queue.earliest_soft_deadline, abstime - mytimer->queue.earliest_soft_deadline, 0, 0);
 104                         latency = 0;
 105                 }
 106
 107                 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 108                     DECR_TRAP_LATENCY | DBG_FUNC_NONE,
 109                     -latency,
 110                     ((user_mode != 0) ? rip : VM_KERNEL_UNSLIDE(rip)),
 111                     user_mode, 0, 0);
 112
 113                 mytimer->has_expired = TRUE;    /* Remember that we popped */
 114                 mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
 115                 mytimer->has_expired = FALSE;
 116
 117                 /* Get the time again since we ran a bit */
 118                 abstime = mach_absolute_time();
 119                 mytimer->when_set = abstime;
 120         }
 121
 122         /* is it time for power management state change? */
 123         if ((pmdeadline = pmCPUGetDeadline(pp)) && (pmdeadline <= abstime)) {
 124                 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 125                     DECR_PM_DEADLINE | DBG_FUNC_START,
 126                     0, 0, 0, 0, 0);
 127                 pmCPUDeadline(pp);
 128                 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 129                     DECR_PM_DEADLINE | DBG_FUNC_END,
 130                     0, 0, 0, 0, 0);
 131                 timer_processed = TRUE;
 132                 abstime = mach_absolute_time(); /* Get the time again since we ran a bit */
 133         }
 134
 135         uint64_t quantum_deadline = pp->quantum_timer_deadline;
 136         /* is it the quantum timer expiration? */
 137         if ((quantum_deadline <= abstime) && (quantum_deadline > 0)) {
 138                 pp->quantum_timer_deadline = 0;
 139                 quantum_timer_expire(abstime);
 140         }
 141
 142         /* schedule our next deadline */
 143         x86_lcpu()->rtcDeadline = EndOfAllTime;
 144         timer_resync_deadlines();
 145
 146         if (__improbable(timer_processed == FALSE)) {
 147                 spurious_timers++;
 148         }
 149 }
 150
 151 /*
 152  * Set the clock deadline.
 153  */
 154 void
 155 timer_set_deadline(uint64_t deadline)
 156 {
 157         rtclock_timer_t         *mytimer;
 158         spl_t                   s;
 159         cpu_data_t              *pp;
 160
 161         s = splclock();                         /* no interruptions */
 162         pp = current_cpu_datap();
 163
 164         mytimer = &pp->rtclock_timer;           /* Point to the timer itself */
 165         mytimer->deadline = deadline;           /* Set new expiration time */
 166         mytimer->when_set = mach_absolute_time();
 167
 168         timer_resync_deadlines();
 169
 170         splx(s);
 171 }
 172
 173 void
 174 quantum_timer_set_deadline(uint64_t deadline)
 175 {
 176         cpu_data_t              *pp;
 177         /* We should've only come into this path with interrupts disabled */
 178         assert(ml_get_interrupts_enabled() == FALSE);
 179
 180         pp = current_cpu_datap();
 181         pp->quantum_timer_deadline = deadline;
 182         timer_resync_deadlines();
 183 }
 184
 185 /*
 186  * Re-evaluate the outstanding deadlines and select the most proximate.
 187  *
 188  * Should be called at splclock.
 189  */
 190 void
 191 timer_resync_deadlines(void)
 192 {
 193         uint64_t                deadline = EndOfAllTime;
 194         uint64_t                pmdeadline;
 195         uint64_t                quantum_deadline;
 196         rtclock_timer_t         *mytimer;
 197         spl_t                   s = splclock();
 198         cpu_data_t              *pp;
 199         uint32_t                decr;
 200
 201         pp = current_cpu_datap();
 202         if (!pp->cpu_running) {
 203                 /* There's really nothing to do if this processor is down */
 204                 return;
 205         }
 206
 207         /*
 208          * If we have a clock timer set, pick that.
 209          */
 210         mytimer = &pp->rtclock_timer;
 211         if (!mytimer->has_expired &&
 212             0 < mytimer->deadline && mytimer->deadline < EndOfAllTime) {
 213                 deadline = mytimer->deadline;
 214         }
 215
 216         /*
 217          * If we have a power management deadline, see if that's earlier.
 218          */
 219         pmdeadline = pmCPUGetDeadline(pp);
 220         if (0 < pmdeadline && pmdeadline < deadline) {
 221                 deadline = pmdeadline;
 222         }
 223
 224         /* If we have the quantum timer setup, check that */
 225         quantum_deadline = pp->quantum_timer_deadline;
 226         if ((quantum_deadline > 0) &&
 227             (quantum_deadline < deadline)) {
 228                 deadline = quantum_deadline;
 229         }
 230
 231
 232         /*
 233          * Go and set the "pop" event.
 234          */
 235         decr = (uint32_t) setPop(deadline);
 236
 237         /* Record non-PM deadline for latency tool */
 238         if (decr != 0 && deadline != pmdeadline) {
 239                 uint64_t queue_count = 0;
 240                 if (deadline != quantum_deadline) {
 241                         /*
 242                          * For non-quantum timer put the queue count
 243                          * in the tracepoint.
 244                          */
 245                         queue_count = mytimer->queue.count;
 246                 }
 247                 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 248                     DECR_SET_DEADLINE | DBG_FUNC_NONE,
 249                     decr, 2,
 250                     deadline,
 251                     queue_count, 0);
 252         }
 253         splx(s);
 254 }
 255
 256 void
 257 timer_queue_expire_local(
 258         __unused void                   *arg)
 259 {
 260         rtclock_timer_t         *mytimer;
 261         uint64_t                        abstime;
 262         cpu_data_t                      *pp;
 263
 264         pp = current_cpu_datap();
 265
 266         mytimer = &pp->rtclock_timer;
 267         abstime = mach_absolute_time();
 268
 269         mytimer->has_expired = TRUE;
 270         mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
 271         mytimer->has_expired = FALSE;
 272         mytimer->when_set = mach_absolute_time();
 273
 274         timer_resync_deadlines();
 275 }
 276
 277 void
 278 timer_queue_expire_rescan(
 279         __unused void                   *arg)
 280 {
 281         rtclock_timer_t         *mytimer;
 282         uint64_t                abstime;
 283         cpu_data_t              *pp;
 284
 285         assert(ml_get_interrupts_enabled() == FALSE);
 286         pp = current_cpu_datap();
 287
 288         mytimer = &pp->rtclock_timer;
 289         abstime = mach_absolute_time();
 290
 291         mytimer->has_expired = TRUE;
 292         mytimer->deadline = timer_queue_expire_with_options(&mytimer->queue, abstime, TRUE);
 293         mytimer->has_expired = FALSE;
 294         mytimer->when_set = mach_absolute_time();
 295
 296         timer_resync_deadlines();
 297 }
 298
 299 #define TIMER_RESORT_THRESHOLD_ABSTIME (50 * NSEC_PER_MSEC)
 300
 301 #if TCOAL_PRIO_STATS
 302 int32_t nc_tcl, rt_tcl, bg_tcl, kt_tcl, fp_tcl, ts_tcl, qos_tcl;
 303 #define TCOAL_PRIO_STAT(x) (x++)
 304 #else
 305 #define TCOAL_PRIO_STAT(x)
 306 #endif
 307
 308 boolean_t
 309 timer_resort_threshold(uint64_t skew)
 310 {
 311         if (skew >= TIMER_RESORT_THRESHOLD_ABSTIME) {
 312                 return TRUE;
 313         } else {
 314                 return FALSE;
 315         }
 316 }
 317
 318 /*
 319  * Return the local timer queue for a running processor
 320  * else return the boot processor's timer queue.
 321  */
 322 mpqueue_head_t *
 323 timer_queue_assign(
 324         uint64_t        deadline)
 325 {
 326         cpu_data_t              *cdp = current_cpu_datap();
 327         mpqueue_head_t          *queue;
 328
 329         if (cdp->cpu_running) {
 330                 queue = &cdp->rtclock_timer.queue;
 331
 332                 if (deadline < cdp->rtclock_timer.deadline) {
 333                         timer_set_deadline(deadline);
 334                 }
 335         } else {
 336                 queue = &cpu_datap(master_cpu)->rtclock_timer.queue;
 337         }
 338
 339         return queue;
 340 }
 341
 342 void
 343 timer_queue_cancel(
 344         mpqueue_head_t  *queue,
 345         uint64_t        deadline,
 346         uint64_t        new_deadline)
 347 {
 348         if (queue == &current_cpu_datap()->rtclock_timer.queue) {
 349                 if (deadline < new_deadline) {
 350                         timer_set_deadline(new_deadline);
 351                 }
 352         }
 353 }
 354
 355 /*
 356  * timer_queue_migrate_cpu() is called from the Power-Management kext
 357  * when a logical processor goes idle (in a deep C-state) with a distant
 358  * deadline so that it's timer queue can be moved to another processor.
 359  * This target processor should be the least idle (most busy) --
 360  * currently this is the primary processor for the calling thread's package.
 361  * Locking restrictions demand that the target cpu must be the boot cpu.
 362  */
 363 uint32_t
 364 timer_queue_migrate_cpu(int target_cpu)
 365 {
 366         cpu_data_t      *target_cdp = cpu_datap(target_cpu);
 367         cpu_data_t      *cdp = current_cpu_datap();
 368         int             ntimers_moved;
 369
 370         assert(!ml_get_interrupts_enabled());
 371         assert(target_cpu != cdp->cpu_number);
 372         assert(target_cpu == master_cpu);
 373
 374         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 375             DECR_TIMER_MIGRATE | DBG_FUNC_START,
 376             target_cpu,
 377             cdp->rtclock_timer.deadline, (cdp->rtclock_timer.deadline >> 32),
 378             0, 0);
 379
 380         /*
 381          * Move timer requests from the local queue to the target processor's.
 382          * The return value is the number of requests moved. If this is 0,
 383          * it indicates that the first (i.e. earliest) timer is earlier than
 384          * the earliest for the target processor. Since this would force a
 385          * resync, the move of this and all later requests is aborted.
 386          */
 387         ntimers_moved = timer_queue_migrate(&cdp->rtclock_timer.queue,
 388             &target_cdp->rtclock_timer.queue);
 389
 390         /*
 391          * Assuming we moved stuff, clear local deadline.
 392          */
 393         if (ntimers_moved > 0) {
 394                 cdp->rtclock_timer.deadline = EndOfAllTime;
 395                 setPop(EndOfAllTime);
 396         }
 397
 398         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 399             DECR_TIMER_MIGRATE | DBG_FUNC_END,
 400             target_cpu, ntimers_moved, 0, 0, 0);
 401
 402         return ntimers_moved;
 403 }
 404
 405 mpqueue_head_t *
 406 timer_queue_cpu(int cpu)
 407 {
 408         return &cpu_datap(cpu)->rtclock_timer.queue;
 409 }
 410
 411 void
 412 timer_call_cpu(int cpu, void (*fn)(void *), void *arg)
 413 {
 414         mp_cpus_call(cpu_to_cpumask(cpu), SYNC, fn, arg);
 415 }
 416
 417 void
 418 timer_call_nosync_cpu(int cpu, void (*fn)(void *), void *arg)
 419 {
 420         /* XXX Needs error checking and retry */
 421         mp_cpus_call(cpu_to_cpumask(cpu), NOSYNC, fn, arg);
 422 }
 423
 424
 425 static timer_coalescing_priority_params_ns_t tcoal_prio_params_init =
 426 {
 427         .idle_entry_timer_processing_hdeadline_threshold_ns = 5000ULL * NSEC_PER_USEC,
 428         .interrupt_timer_coalescing_ilat_threshold_ns = 30ULL * NSEC_PER_USEC,
 429         .timer_resort_threshold_ns = 50 * NSEC_PER_MSEC,
 430         .timer_coalesce_rt_shift = 0,
 431         .timer_coalesce_bg_shift = -5,
 432         .timer_coalesce_kt_shift = 3,
 433         .timer_coalesce_fp_shift = 3,
 434         .timer_coalesce_ts_shift = 3,
 435         .timer_coalesce_rt_ns_max = 0ULL,
 436         .timer_coalesce_bg_ns_max = 100 * NSEC_PER_MSEC,
 437         .timer_coalesce_kt_ns_max = 1 * NSEC_PER_MSEC,
 438         .timer_coalesce_fp_ns_max = 1 * NSEC_PER_MSEC,
 439         .timer_coalesce_ts_ns_max = 1 * NSEC_PER_MSEC,
 440         .latency_qos_scale = {3, 2, 1, -2, -15, -15},
 441         .latency_qos_ns_max = {1 * NSEC_PER_MSEC, 5 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC,
 442                                75 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC},
 443         .latency_tier_rate_limited = {FALSE, FALSE, FALSE, FALSE, TRUE, TRUE},
 444 };
 445
 446 timer_coalescing_priority_params_ns_t *
 447 timer_call_get_priority_params(void)
 448 {
 449         return &tcoal_prio_params_init;
 450 }