+/*
+ * thread_bind_internal:
+ *
+ * If the specified thread is not the current thread, and it is currently
+ * running on another CPU, a remote AST must be sent to that CPU to cause
+ * the thread to migrate to its bound processor. Otherwise, the migration
+ * will occur at the next quantum expiration or blocking point.
+ *
+ * When the thread is the current thread, and explicit thread_block() should
+ * be used to force the current processor to context switch away and
+ * let the thread migrate to the bound processor.
+ *
+ * Thread must be locked, and at splsched.
+ */
+
+static processor_t
+thread_bind_internal(
+ thread_t thread,
+ processor_t processor)
+{
+ processor_t prev;
+
+ /* <rdar://problem/15102234> */
+ assert(thread->sched_pri < BASEPRI_RTQUEUES);
+ /* A thread can't be bound if it's sitting on a (potentially incorrect) runqueue */
+ assert(thread->runq == PROCESSOR_NULL);
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_THREAD_BIND), thread_tid(thread), processor ? (uintptr_t)processor->cpu_id : (uintptr_t)-1, 0, 0, 0);
+
+ prev = thread->bound_processor;
+ thread->bound_processor = processor;
+
+ return (prev);
+}
+
+/*
+ * thread_vm_bind_group_add:
+ *
+ * The "VM bind group" is a special mechanism to mark a collection
+ * of threads from the VM subsystem that, in general, should be scheduled
+ * with only one CPU of parallelism. To accomplish this, we initially
+ * bind all the threads to the master processor, which has the effect
+ * that only one of the threads in the group can execute at once, including
+ * preempting threads in the group that are a lower priority. Future
+ * mechanisms may use more dynamic mechanisms to prevent the collection
+ * of VM threads from using more CPU time than desired.
+ *
+ * The current implementation can result in priority inversions where
+ * compute-bound priority 95 or realtime threads that happen to have
+ * landed on the master processor prevent the VM threads from running.
+ * When this situation is detected, we unbind the threads for one
+ * scheduler tick to allow the scheduler to run the threads an
+ * additional CPUs, before restoring the binding (assuming high latency
+ * is no longer a problem).
+ */
+
+/*
+ * The current max is provisioned for:
+ * vm_compressor_swap_trigger_thread (92)
+ * 2 x vm_pageout_iothread_internal (92) when vm_restricted_to_single_processor==TRUE
+ * vm_pageout_continue (92)
+ * memorystatus_thread (95)
+ */
+#define MAX_VM_BIND_GROUP_COUNT (5)
+decl_simple_lock_data(static,sched_vm_group_list_lock);
+static thread_t sched_vm_group_thread_list[MAX_VM_BIND_GROUP_COUNT];
+static int sched_vm_group_thread_count;
+static boolean_t sched_vm_group_temporarily_unbound = FALSE;
+
+void
+thread_vm_bind_group_add(void)
+{
+ thread_t self = current_thread();
+
+ thread_reference_internal(self);
+ self->options |= TH_OPT_SCHED_VM_GROUP;
+
+ simple_lock(&sched_vm_group_list_lock);
+ assert(sched_vm_group_thread_count < MAX_VM_BIND_GROUP_COUNT);
+ sched_vm_group_thread_list[sched_vm_group_thread_count++] = self;
+ simple_unlock(&sched_vm_group_list_lock);
+
+ thread_bind(master_processor);
+
+ /* Switch to bound processor if not already there */
+ thread_block(THREAD_CONTINUE_NULL);
+}
+
+static void
+sched_vm_group_maintenance(void)
+{
+ uint64_t ctime = mach_absolute_time();
+ uint64_t longtime = ctime - sched_tick_interval;
+ int i;
+ spl_t s;
+ boolean_t high_latency_observed = FALSE;
+ boolean_t runnable_and_not_on_runq_observed = FALSE;
+ boolean_t bind_target_changed = FALSE;
+ processor_t bind_target = PROCESSOR_NULL;
+
+ /* Make sure nobody attempts to add new threads while we are enumerating them */
+ simple_lock(&sched_vm_group_list_lock);
+
+ s = splsched();
+
+ for (i=0; i < sched_vm_group_thread_count; i++) {
+ thread_t thread = sched_vm_group_thread_list[i];
+ assert(thread != THREAD_NULL);
+ thread_lock(thread);
+ if ((thread->state & (TH_RUN|TH_WAIT)) == TH_RUN) {
+ if (thread->runq != PROCESSOR_NULL && thread->last_made_runnable_time < longtime) {
+ high_latency_observed = TRUE;
+ } else if (thread->runq == PROCESSOR_NULL) {
+ /* There are some cases where a thread be transitiong that also fall into this case */
+ runnable_and_not_on_runq_observed = TRUE;
+ }
+ }
+ thread_unlock(thread);
+
+ if (high_latency_observed && runnable_and_not_on_runq_observed) {
+ /* All the things we are looking for are true, stop looking */
+ break;
+ }
+ }
+
+ splx(s);
+
+ if (sched_vm_group_temporarily_unbound) {
+ /* If we turned off binding, make sure everything is OK before rebinding */
+ if (!high_latency_observed) {
+ /* rebind */
+ bind_target_changed = TRUE;
+ bind_target = master_processor;
+ sched_vm_group_temporarily_unbound = FALSE; /* might be reset to TRUE if change cannot be completed */
+ }
+ } else {
+ /*
+ * Check if we're in a bad state, which is defined by high
+ * latency with no core currently executing a thread. If a
+ * single thread is making progress on a CPU, that means the
+ * binding concept to reduce parallelism is working as
+ * designed.
+ */
+ if (high_latency_observed && !runnable_and_not_on_runq_observed) {
+ /* unbind */
+ bind_target_changed = TRUE;
+ bind_target = PROCESSOR_NULL;
+ sched_vm_group_temporarily_unbound = TRUE;
+ }
+ }
+
+ if (bind_target_changed) {
+ s = splsched();
+ for (i=0; i < sched_vm_group_thread_count; i++) {
+ thread_t thread = sched_vm_group_thread_list[i];
+ boolean_t removed;
+ assert(thread != THREAD_NULL);
+
+ thread_lock(thread);
+ removed = thread_run_queue_remove(thread);
+ if (removed || ((thread->state & (TH_RUN | TH_WAIT)) == TH_WAIT)) {
+ thread_bind_internal(thread, bind_target);
+ } else {
+ /*
+ * Thread was in the middle of being context-switched-to,
+ * or was in the process of blocking. To avoid switching the bind
+ * state out mid-flight, defer the change if possible.
+ */
+ if (bind_target == PROCESSOR_NULL) {
+ thread_bind_internal(thread, bind_target);
+ } else {
+ sched_vm_group_temporarily_unbound = TRUE; /* next pass will try again */
+ }
+ }
+
+ if (removed) {
+ thread_run_queue_reinsert(thread, SCHED_PREEMPT | SCHED_TAILQ);
+ }
+ thread_unlock(thread);
+ }
+ splx(s);
+ }
+
+ simple_unlock(&sched_vm_group_list_lock);
+}
+
+/* Invoked prior to idle entry to determine if, on SMT capable processors, an SMT
+ * rebalancing opportunity exists when a core is (instantaneously) idle, but
+ * other SMT-capable cores may be over-committed. TODO: some possible negatives:
+ * IPI thrash if this core does not remain idle following the load balancing ASTs
+ * Idle "thrash", when IPI issue is followed by idle entry/core power down
+ * followed by a wakeup shortly thereafter.
+ */
+
+#if (DEVELOPMENT || DEBUG)
+int sched_smt_balance = 1;
+#endif
+
+#if __SMP__
+/* Invoked with pset locked, returns with pset unlocked */
+static void
+sched_SMT_balance(processor_t cprocessor, processor_set_t cpset) {
+ processor_t ast_processor = NULL;
+
+#if (DEVELOPMENT || DEBUG)
+ if (__improbable(sched_smt_balance == 0))
+ goto smt_balance_exit;
+#endif
+
+ assert(cprocessor == current_processor());
+ if (cprocessor->is_SMT == FALSE)
+ goto smt_balance_exit;
+
+ processor_t sib_processor = cprocessor->processor_secondary ? cprocessor->processor_secondary : cprocessor->processor_primary;
+
+ /* Determine if both this processor and its sibling are idle,
+ * indicating an SMT rebalancing opportunity.
+ */
+ if (sib_processor->state != PROCESSOR_IDLE)
+ goto smt_balance_exit;
+
+ processor_t sprocessor;
+
+ qe_foreach_element(sprocessor, &cpset->active_queue, processor_queue) {
+ if ((sprocessor->state == PROCESSOR_RUNNING) &&
+ (sprocessor->processor_primary != sprocessor) &&
+ (sprocessor->processor_primary->state == PROCESSOR_RUNNING) &&
+ (sprocessor->current_pri < BASEPRI_RTQUEUES) &&
+ ((cpset->pending_AST_cpu_mask & (1ULL << sprocessor->cpu_id)) == 0)) {
+ assert(sprocessor != cprocessor);
+ ast_processor = sprocessor;
+ break;
+ }
+ }
+
+smt_balance_exit:
+ pset_unlock(cpset);
+
+ if (ast_processor) {
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_SMT_BALANCE), ast_processor->cpu_id, ast_processor->state, ast_processor->processor_primary->state, 0, 0);
+ cause_ast_check(ast_processor);
+ }
+}
+#endif /* __SMP__ */
+
projects / apple / xnu.git / blobdiff