+
+uint32_t
+sched_run_decr(thread_t thread)
+{
+ assert((thread->state & (TH_RUN|TH_IDLE)) != TH_RUN);
+
+ sched_decr_bucket(thread->th_sched_bucket);
+
+ uint32_t new_count = hw_atomic_sub(&sched_run_buckets[TH_BUCKET_RUN], 1);
+
+ return new_count;
+}
+
+static void
+sched_update_thread_bucket(thread_t thread)
+{
+ sched_bucket_t old_bucket = thread->th_sched_bucket;
+ sched_bucket_t new_bucket = TH_BUCKET_RUN;
+
+ switch (thread->sched_mode) {
+ case TH_MODE_FIXED:
+ case TH_MODE_REALTIME:
+ new_bucket = TH_BUCKET_FIXPRI;
+ break;
+
+ case TH_MODE_TIMESHARE:
+ if (thread->base_pri > BASEPRI_DEFAULT)
+ new_bucket = TH_BUCKET_SHARE_FG;
+ else if (thread->base_pri > BASEPRI_UTILITY)
+ new_bucket = TH_BUCKET_SHARE_DF;
+ else if (thread->base_pri > MAXPRI_THROTTLE)
+ new_bucket = TH_BUCKET_SHARE_UT;
+ else
+ new_bucket = TH_BUCKET_SHARE_BG;
+ break;
+
+ default:
+ panic("unexpected mode: %d", thread->sched_mode);
+ break;
+ }
+
+ if (old_bucket != new_bucket) {
+ thread->th_sched_bucket = new_bucket;
+ thread->pri_shift = sched_pri_shifts[new_bucket];
+
+ if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN) {
+ sched_decr_bucket(old_bucket);
+ sched_incr_bucket(new_bucket);
+ }
+ }
+}
+
+/*
+ * Set the thread's true scheduling mode
+ * Called with thread mutex and thread locked
+ * The thread has already been removed from the runqueue.
+ *
+ * (saved_mode is handled before this point)
+ */
+void
+sched_set_thread_mode(thread_t thread, sched_mode_t new_mode)
+{
+ assert(thread->runq == PROCESSOR_NULL);
+
+ switch (new_mode) {
+ case TH_MODE_FIXED:
+ case TH_MODE_REALTIME:
+ case TH_MODE_TIMESHARE:
+ break;
+
+ default:
+ panic("unexpected mode: %d", new_mode);
+ break;
+ }
+
+ thread->sched_mode = new_mode;
+
+ sched_update_thread_bucket(thread);
+}
+
+/*
+ * Demote the true scheduler mode to timeshare (called with the thread locked)
+ */
+void
+sched_thread_mode_demote(thread_t thread, uint32_t reason)
+{
+ assert(reason & TH_SFLAG_DEMOTED_MASK);
+ assert((thread->sched_flags & reason) != reason);
+
+ if (thread->policy_reset)
+ return;
+
+ if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {
+ /* Another demotion reason is already active */
+ thread->sched_flags |= reason;
+ return;
+ }
+
+ assert(thread->saved_mode == TH_MODE_NONE);
+
+ boolean_t removed = thread_run_queue_remove(thread);
+
+ thread->sched_flags |= reason;
+
+ thread->saved_mode = thread->sched_mode;
+
+ sched_set_thread_mode(thread, TH_MODE_TIMESHARE);
+
+ thread_recompute_priority(thread);
+
+ if (removed)
+ thread_run_queue_reinsert(thread, SCHED_TAILQ);
+}
+
+/*
+ * Un-demote the true scheduler mode back to the saved mode (called with the thread locked)
+ */
+void
+sched_thread_mode_undemote(thread_t thread, uint32_t reason)
+{
+ assert(reason & TH_SFLAG_DEMOTED_MASK);
+ assert((thread->sched_flags & reason) == reason);
+ assert(thread->saved_mode != TH_MODE_NONE);
+ assert(thread->sched_mode == TH_MODE_TIMESHARE);
+ assert(thread->policy_reset == 0);
+
+ thread->sched_flags &= ~reason;
+
+ if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {
+ /* Another demotion reason is still active */
+ return;
+ }
+
+ boolean_t removed = thread_run_queue_remove(thread);
+
+ sched_set_thread_mode(thread, thread->saved_mode);
+
+ thread->saved_mode = TH_MODE_NONE;
+
+ thread_recompute_priority(thread);
+
+ if (removed)
+ thread_run_queue_reinsert(thread, SCHED_TAILQ);
+}
+
+/*
+ * Promote thread to a specific priority
+ *
+ * Promotion must not last past syscall boundary
+ * Clients must always pair promote and unpromote 1:1
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_promote_to_pri(thread_t thread,
+ int priority,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) != TH_SFLAG_PROMOTED);
+ assert(thread->promotion_priority == 0);
+ assert(priority <= MAXPRI_PROMOTE);
+ assert(priority > 0);
+
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_PROMOTED),
+ thread_tid(thread), trace_obj, priority);
+
+ thread->sched_flags |= TH_SFLAG_PROMOTED;
+ thread->promotion_priority = priority;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+
+/*
+ * Update a pre-existing priority promotion to have a higher priority floor
+ * Priority can only go up from the previous value
+ * Update must occur while a promotion is active
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_update_promotion_to_pri(thread_t thread,
+ int priority,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert(thread->promotions > 0);
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) == TH_SFLAG_PROMOTED);
+ assert(thread->promotion_priority > 0);
+ assert(priority <= MAXPRI_PROMOTE);
+
+ if (thread->promotion_priority < priority) {
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_PROMOTED_UPDATE),
+ thread_tid(thread), trace_obj, priority);
+
+ thread->promotion_priority = priority;
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+ }
+}
+
+/*
+ * End a priority promotion
+ * Demotes a thread back to its expected priority without the promotion in place
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_unpromote(thread_t thread,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) == TH_SFLAG_PROMOTED);
+ assert(thread->promotion_priority > 0);
+
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_UNPROMOTED),
+ thread_tid(thread), trace_obj, 0);
+
+ thread->sched_flags &= ~TH_SFLAG_PROMOTED;
+ thread->promotion_priority = 0;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+/* called with thread locked */
+void
+assert_promotions_invariant(thread_t thread)
+{
+ if (thread->promotions > 0)
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) == TH_SFLAG_PROMOTED);
+
+ if (thread->promotions == 0)
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) != TH_SFLAG_PROMOTED);
+}
+
+/*
+ * Promote thread to have a sched pri floor for a specific reason
+ *
+ * Promotion must not last past syscall boundary
+ * Clients must always pair promote and demote 1:1,
+ * Handling nesting of the same promote reason is the client's responsibility
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_promote_reason(thread_t thread,
+ uint32_t reason,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert(reason & TH_SFLAG_PROMOTE_REASON_MASK);
+ assert((thread->sched_flags & reason) != reason);
+
+ switch (reason) {
+ case TH_SFLAG_RW_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_RW_PROMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_WAITQ_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAITQ_PROMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_EXEC_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_PROMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ }
+
+ thread->sched_flags |= reason;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+/*
+ * End a specific promotion reason
+ * Demotes a thread back to its expected priority without the promotion in place
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_unpromote_reason(thread_t thread,
+ uint32_t reason,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert(reason & TH_SFLAG_PROMOTE_REASON_MASK);
+ assert((thread->sched_flags & reason) == reason);
+
+ switch (reason) {
+ case TH_SFLAG_RW_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_RW_DEMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_WAITQ_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAITQ_DEMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_EXEC_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_DEMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ }
+
+ thread->sched_flags &= ~reason;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+
projects / apple / xnu.git / blobdiff