+
+/*
+ * Set the thread's true scheduling mode
+ * Called with thread mutex and thread locked
+ * The thread has already been removed from the runqueue.
+ *
+ * (saved_mode is handled before this point)
+ */
+void
+sched_set_thread_mode(thread_t thread, sched_mode_t new_mode)
+{
+ assert(thread->runq == PROCESSOR_NULL);
+
+ switch (new_mode) {
+ case TH_MODE_FIXED:
+ case TH_MODE_REALTIME:
+ case TH_MODE_TIMESHARE:
+ break;
+
+ default:
+ panic("unexpected mode: %d", new_mode);
+ break;
+ }
+
+ thread->sched_mode = new_mode;
+
+ sched_update_thread_bucket(thread);
+}
+
+/*
+ * Demote the true scheduler mode to timeshare (called with the thread locked)
+ */
+void
+sched_thread_mode_demote(thread_t thread, uint32_t reason)
+{
+ assert(reason & TH_SFLAG_DEMOTED_MASK);
+ assert((thread->sched_flags & reason) != reason);
+
+ if (thread->policy_reset)
+ return;
+
+ if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {
+ /* Another demotion reason is already active */
+ thread->sched_flags |= reason;
+ return;
+ }
+
+ assert(thread->saved_mode == TH_MODE_NONE);
+
+ boolean_t removed = thread_run_queue_remove(thread);
+
+ thread->sched_flags |= reason;
+
+ thread->saved_mode = thread->sched_mode;
+
+ sched_set_thread_mode(thread, TH_MODE_TIMESHARE);
+
+ thread_recompute_priority(thread);
+
+ if (removed)
+ thread_run_queue_reinsert(thread, SCHED_TAILQ);
+}
+
+/*
+ * Un-demote the true scheduler mode back to the saved mode (called with the thread locked)
+ */
+void
+sched_thread_mode_undemote(thread_t thread, uint32_t reason)
+{
+ assert(reason & TH_SFLAG_DEMOTED_MASK);
+ assert((thread->sched_flags & reason) == reason);
+ assert(thread->saved_mode != TH_MODE_NONE);
+ assert(thread->sched_mode == TH_MODE_TIMESHARE);
+ assert(thread->policy_reset == 0);
+
+ thread->sched_flags &= ~reason;
+
+ if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {
+ /* Another demotion reason is still active */
+ return;
+ }
+
+ boolean_t removed = thread_run_queue_remove(thread);
+
+ sched_set_thread_mode(thread, thread->saved_mode);
+
+ thread->saved_mode = TH_MODE_NONE;
+
+ thread_recompute_priority(thread);
+
+ if (removed)
+ thread_run_queue_reinsert(thread, SCHED_TAILQ);
+}
+
+/*
+ * Promote thread to a specific priority
+ *
+ * Promotion must not last past syscall boundary
+ * Clients must always pair promote and unpromote 1:1
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_promote_to_pri(thread_t thread,
+ int priority,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) != TH_SFLAG_PROMOTED);
+ assert(thread->promotion_priority == 0);
+ assert(priority <= MAXPRI_PROMOTE);
+ assert(priority > 0);
+
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_PROMOTED),
+ thread_tid(thread), trace_obj, priority);
+
+ thread->sched_flags |= TH_SFLAG_PROMOTED;
+ thread->promotion_priority = priority;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+
+/*
+ * Update a pre-existing priority promotion to have a higher priority floor
+ * Priority can only go up from the previous value
+ * Update must occur while a promotion is active
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_update_promotion_to_pri(thread_t thread,
+ int priority,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert(thread->promotions > 0);
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) == TH_SFLAG_PROMOTED);
+ assert(thread->promotion_priority > 0);
+ assert(priority <= MAXPRI_PROMOTE);
+
+ if (thread->promotion_priority < priority) {
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_PROMOTED_UPDATE),
+ thread_tid(thread), trace_obj, priority);
+
+ thread->promotion_priority = priority;
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+ }
+}
+
+/*
+ * End a priority promotion
+ * Demotes a thread back to its expected priority without the promotion in place
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_unpromote(thread_t thread,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) == TH_SFLAG_PROMOTED);
+ assert(thread->promotion_priority > 0);
+
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_UNPROMOTED),
+ thread_tid(thread), trace_obj, 0);
+
+ thread->sched_flags &= ~TH_SFLAG_PROMOTED;
+ thread->promotion_priority = 0;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+/* called with thread locked */
+void
+assert_promotions_invariant(thread_t thread)
+{
+ if (thread->promotions > 0)
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) == TH_SFLAG_PROMOTED);
+
+ if (thread->promotions == 0)
+ assert((thread->sched_flags & TH_SFLAG_PROMOTED) != TH_SFLAG_PROMOTED);
+}
+
+/*
+ * Promote thread to have a sched pri floor for a specific reason
+ *
+ * Promotion must not last past syscall boundary
+ * Clients must always pair promote and demote 1:1,
+ * Handling nesting of the same promote reason is the client's responsibility
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_promote_reason(thread_t thread,
+ uint32_t reason,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert(reason & TH_SFLAG_PROMOTE_REASON_MASK);
+ assert((thread->sched_flags & reason) != reason);
+
+ switch (reason) {
+ case TH_SFLAG_RW_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_RW_PROMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_WAITQ_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAITQ_PROMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_EXEC_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_PROMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ }
+
+ thread->sched_flags |= reason;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+/*
+ * End a specific promotion reason
+ * Demotes a thread back to its expected priority without the promotion in place
+ *
+ * Called at splsched with thread locked
+ */
+void
+sched_thread_unpromote_reason(thread_t thread,
+ uint32_t reason,
+ __kdebug_only uintptr_t trace_obj /* already unslid */)
+{
+ assert(reason & TH_SFLAG_PROMOTE_REASON_MASK);
+ assert((thread->sched_flags & reason) == reason);
+
+ switch (reason) {
+ case TH_SFLAG_RW_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_RW_DEMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_WAITQ_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAITQ_DEMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ case TH_SFLAG_EXEC_PROMOTED:
+ KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_DEMOTE),
+ thread_tid(thread), thread->sched_pri,
+ thread->base_pri, trace_obj);
+ break;
+ }
+
+ thread->sched_flags &= ~reason;
+
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+}
+
+
projects / apple / xnu.git / blobdiff