+ /* Note comma operator in while expression re-locking each iteration */
+ while ((void)timer_queue_lock_spin(queue), !queue_empty(&queue->head)) {
+ call = TIMER_CALL(queue_first(&queue->head));
+
+ if (!simple_lock_try(&call->lock, LCK_GRP_NULL)) {
+ /*
+ * case (2b) lock order inversion, dequeue and skip
+ * Don't change the call_entry queue back-pointer
+ * but set the async_dequeue field.
+ */
+ timer_queue_shutdown_lock_skips++;
+ timer_call_entry_dequeue_async(call);
+#if TIMER_ASSERT
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_ASYNC_DEQ | DBG_FUNC_NONE,
+ VM_KERNEL_UNSLIDE_OR_PERM(call),
+ call->async_dequeue,
+ VM_KERNEL_UNSLIDE_OR_PERM(TCE(call)->queue),
+ 0x2b, 0);
+#endif
+ timer_queue_unlock(queue);
+ continue;
+ }
+
+ boolean_t call_local = ((call->flags & TIMER_CALL_LOCAL) != 0);
+
+ /* remove entry from old queue */
+ timer_call_entry_dequeue(call);
+ timer_queue_unlock(queue);
+
+ if (call_local == FALSE) {
+ /* and queue it on new, discarding LOCAL timers */
+ new_queue = timer_queue_assign(TCE(call)->deadline);
+ timer_queue_lock_spin(new_queue);
+ timer_call_entry_enqueue_deadline(
+ call, new_queue, TCE(call)->deadline);
+ timer_queue_unlock(new_queue);
+ } else {
+ timer_queue_shutdown_discarded++;
+ }
+
+ assert(call_local == FALSE);
+ simple_unlock(&call->lock);
+ }
+
+ timer_queue_unlock(queue);
+ splx(s);
+}
+
+
+void
+quantum_timer_expire(
+ uint64_t deadline)
+{
+ processor_t processor = current_processor();
+ timer_call_t call = TIMER_CALL(&(processor->quantum_timer));
+
+ if (__improbable(TCE(call)->deadline > deadline)) {
+ panic("CPU quantum timer deadlin out of sync with timer call deadline");
+ }
+
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_EXPIRE | DBG_FUNC_NONE,
+ VM_KERNEL_UNSLIDE_OR_PERM(call),
+ TCE(call)->deadline,
+ TCE(call)->deadline,
+ TCE(call)->entry_time, 0);
+
+ timer_call_func_t func = TCE(call)->func;
+ timer_call_param_t param0 = TCE(call)->param0;
+ timer_call_param_t param1 = TCE(call)->param1;
+
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_CALLOUT | DBG_FUNC_START,
+ VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func),
+ VM_KERNEL_ADDRHIDE(param0),
+ VM_KERNEL_ADDRHIDE(param1),
+ 0);
+
+#if CONFIG_DTRACE
+ DTRACE_TMR7(callout__start, timer_call_func_t, func,
+ timer_call_param_t, param0, unsigned, call->flags,
+ 0, (call->ttd >> 32),
+ (unsigned) (call->ttd & 0xFFFFFFFF), call);
+#endif
+ (*func)(param0, param1);
+
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_CALLOUT | DBG_FUNC_END,
+ VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func),
+ VM_KERNEL_ADDRHIDE(param0),
+ VM_KERNEL_ADDRHIDE(param1),
+ 0);
+}
+
+static uint32_t timer_queue_expire_lock_skips;
+uint64_t
+timer_queue_expire_with_options(
+ mpqueue_head_t *queue,
+ uint64_t deadline,
+ boolean_t rescan)
+{
+ timer_call_t call = NULL;
+ uint32_t tc_iterations = 0;
+ DBG("timer_queue_expire(%p,)\n", queue);
+
+ uint64_t cur_deadline = deadline;
+ timer_queue_lock_spin(queue);
+
+ while (!queue_empty(&queue->head)) {
+ /* Upon processing one or more timer calls, refresh the
+ * deadline to account for time elapsed in the callout
+ */
+ if (++tc_iterations > 1) {
+ cur_deadline = mach_absolute_time();
+ }
+
+ if (call == NULL) {
+ call = TIMER_CALL(queue_first(&queue->head));
+ }
+
+ if (call->soft_deadline <= cur_deadline) {
+ timer_call_func_t func;
+ timer_call_param_t param0, param1;
+
+ TCOAL_DEBUG(0xDDDD0000, queue->earliest_soft_deadline, call->soft_deadline, 0, 0, 0);
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_EXPIRE | DBG_FUNC_NONE,
+ VM_KERNEL_UNSLIDE_OR_PERM(call),
+ call->soft_deadline,
+ TCE(call)->deadline,
+ TCE(call)->entry_time, 0);
+
+ if ((call->flags & TIMER_CALL_RATELIMITED) &&
+ (TCE(call)->deadline > cur_deadline)) {
+ if (rescan == FALSE) {
+ break;
+ }
+ }
+
+ if (!simple_lock_try(&call->lock, LCK_GRP_NULL)) {
+ /* case (2b) lock inversion, dequeue and skip */
+ timer_queue_expire_lock_skips++;
+ timer_call_entry_dequeue_async(call);
+ call = NULL;
+ continue;
+ }
+
+ timer_call_entry_dequeue(call);
+
+ func = TCE(call)->func;
+ param0 = TCE(call)->param0;
+ param1 = TCE(call)->param1;
+
+ simple_unlock(&call->lock);
+ timer_queue_unlock(queue);
+
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_CALLOUT | DBG_FUNC_START,
+ VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func),
+ VM_KERNEL_ADDRHIDE(param0),
+ VM_KERNEL_ADDRHIDE(param1),
+ 0);
+
+#if CONFIG_DTRACE
+ DTRACE_TMR7(callout__start, timer_call_func_t, func,
+ timer_call_param_t, param0, unsigned, call->flags,
+ 0, (call->ttd >> 32),
+ (unsigned) (call->ttd & 0xFFFFFFFF), call);
+#endif
+ /* Maintain time-to-deadline in per-processor data
+ * structure for thread wakeup deadline statistics.
+ */
+ uint64_t *ttdp = &(PROCESSOR_DATA(current_processor(), timer_call_ttd));
+ *ttdp = call->ttd;
+ (*func)(param0, param1);
+ *ttdp = 0;
+#if CONFIG_DTRACE
+ DTRACE_TMR4(callout__end, timer_call_func_t, func,
+ param0, param1, call);
+#endif
+
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_CALLOUT | DBG_FUNC_END,
+ VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func),
+ VM_KERNEL_ADDRHIDE(param0),
+ VM_KERNEL_ADDRHIDE(param1),
+ 0);
+ call = NULL;
+ timer_queue_lock_spin(queue);
+ } else {
+ if (__probable(rescan == FALSE)) {
+ break;
+ } else {
+ int64_t skew = TCE(call)->deadline - call->soft_deadline;
+ assert(TCE(call)->deadline >= call->soft_deadline);
+
+ /* DRK: On a latency quality-of-service level change,
+ * re-sort potentially rate-limited timers. The platform
+ * layer determines which timers require
+ * this. In the absence of the per-callout
+ * synchronization requirement, a global resort could
+ * be more efficient. The re-sort effectively
+ * annuls all timer adjustments, i.e. the "soft
+ * deadline" is the sort key.
+ */
+
+ if (timer_resort_threshold(skew)) {
+ if (__probable(simple_lock_try(&call->lock, LCK_GRP_NULL))) {
+ timer_call_entry_dequeue(call);
+ timer_call_entry_enqueue_deadline(call, queue, call->soft_deadline);
+ simple_unlock(&call->lock);
+ call = NULL;
+ }
+ }
+ if (call) {
+ call = TIMER_CALL(queue_next(qe(call)));
+ if (queue_end(&queue->head, qe(call))) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (!queue_empty(&queue->head)) {
+ call = TIMER_CALL(queue_first(&queue->head));
+ cur_deadline = TCE(call)->deadline;
+ queue->earliest_soft_deadline = (call->flags & TIMER_CALL_RATELIMITED) ? TCE(call)->deadline: call->soft_deadline;
+ } else {
+ queue->earliest_soft_deadline = cur_deadline = UINT64_MAX;
+ }
+
+ timer_queue_unlock(queue);
+
+ return cur_deadline;
+}
+
+uint64_t
+timer_queue_expire(
+ mpqueue_head_t *queue,
+ uint64_t deadline)
+{
+ return timer_queue_expire_with_options(queue, deadline, FALSE);
+}
+
+extern int serverperfmode;
+static uint32_t timer_queue_migrate_lock_skips;
+/*
+ * timer_queue_migrate() is called by timer_queue_migrate_cpu()
+ * to move timer requests from the local processor (queue_from)
+ * to a target processor's (queue_to).
+ */
+int
+timer_queue_migrate(mpqueue_head_t *queue_from, mpqueue_head_t *queue_to)
+{
+ timer_call_t call;
+ timer_call_t head_to;
+ int timers_migrated = 0;
+
+ DBG("timer_queue_migrate(%p,%p)\n", queue_from, queue_to);
+
+ assert(!ml_get_interrupts_enabled());
+ assert(queue_from != queue_to);
+
+ if (serverperfmode) {
+ /*
+ * if we're running a high end server
+ * avoid migrations... they add latency
+ * and don't save us power under typical
+ * server workloads
+ */
+ return -4;
+ }
+
+ /*
+ * Take both local (from) and target (to) timer queue locks while
+ * moving the timers from the local queue to the target processor.
+ * We assume that the target is always the boot processor.
+ * But only move if all of the following is true:
+ * - the target queue is non-empty
+ * - the local queue is non-empty
+ * - the local queue's first deadline is later than the target's
+ * - the local queue contains no non-migrateable "local" call
+ * so that we need not have the target resync.
+ */
+
+ timer_queue_lock_spin(queue_to);
+
+ head_to = TIMER_CALL(queue_first(&queue_to->head));
+ if (queue_empty(&queue_to->head)) {
+ timers_migrated = -1;
+ goto abort1;
+ }
+
+ timer_queue_lock_spin(queue_from);
+
+ if (queue_empty(&queue_from->head)) {
+ timers_migrated = -2;
+ goto abort2;
+ }
+
+ call = TIMER_CALL(queue_first(&queue_from->head));
+ if (TCE(call)->deadline < TCE(head_to)->deadline) {
+ timers_migrated = 0;
+ goto abort2;
+ }
+
+ /* perform scan for non-migratable timers */
+ do {
+ if (call->flags & TIMER_CALL_LOCAL) {
+ timers_migrated = -3;
+ goto abort2;
+ }
+ call = TIMER_CALL(queue_next(qe(call)));
+ } while (!queue_end(&queue_from->head, qe(call)));
+
+ /* migration loop itself -- both queues are locked */
+ while (!queue_empty(&queue_from->head)) {
+ call = TIMER_CALL(queue_first(&queue_from->head));
+ if (!simple_lock_try(&call->lock, LCK_GRP_NULL)) {
+ /* case (2b) lock order inversion, dequeue only */
+#ifdef TIMER_ASSERT
+ TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
+ DECR_TIMER_ASYNC_DEQ | DBG_FUNC_NONE,
+ VM_KERNEL_UNSLIDE_OR_PERM(call),
+ VM_KERNEL_UNSLIDE_OR_PERM(TCE(call)->queue),
+ VM_KERNEL_UNSLIDE_OR_PERM(call->lock.interlock.lock_data),
+ 0x2b, 0);
+#endif
+ timer_queue_migrate_lock_skips++;
+ timer_call_entry_dequeue_async(call);
+ continue;
+ }
+ timer_call_entry_dequeue(call);
+ timer_call_entry_enqueue_deadline(
+ call, queue_to, TCE(call)->deadline);
+ timers_migrated++;
+ simple_unlock(&call->lock);
+ }
+ queue_from->earliest_soft_deadline = UINT64_MAX;
+abort2:
+ timer_queue_unlock(queue_from);
+abort1:
+ timer_queue_unlock(queue_to);
+
+ return timers_migrated;
+}
+
+void
+timer_queue_trace_cpu(int ncpu)
+{
+ timer_call_nosync_cpu(
+ ncpu,
+ (void (*)(void *))timer_queue_trace,
+ (void*) timer_queue_cpu(ncpu));
+}
+
+void
+timer_queue_trace(
+ mpqueue_head_t *queue)
+{
+ timer_call_t call;
+ spl_t s;
+
+ if (!kdebug_enable) {
+ return;