KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_QUANTUM_EXPIRED) | DBG_FUNC_START, 0, 0, 0, 0, 0);
- SCHED_STATS_QUANTUM_TIMER_EXPIRATION(processor);
+ SCHED_STATS_INC(quantum_timer_expirations);
/*
* We bill CPU time to both the individual thread and its task.
(thread->quantum_remaining - thread->t_deduct_bank_ledger_time));
}
thread->t_deduct_bank_ledger_time = 0;
-
ctime = mach_absolute_time();
#ifdef CONFIG_MACH_APPROXIMATE_TIME
commpage_update_mach_approximate_time(ctime);
#endif
+ sched_update_pset_avg_execution_time(processor->processor_set, thread->quantum_remaining, ctime, thread->th_sched_bucket);
#if MONOTONIC
mt_sched_update(thread);
* during privilege transitions, synthesize an event now.
*/
if (!thread->precise_user_kernel_time) {
- timer_update(PROCESSOR_DATA(processor, current_state), ctime);
- timer_update(PROCESSOR_DATA(processor, thread_timer), ctime);
+ timer_update(processor->current_state, ctime);
+ timer_update(processor->thread_timer, ctime);
timer_update(&thread->runnable_timer, ctime);
}
ast_propagate(thread);
thread_unlock(thread);
-
- timer_call_quantum_timer_enter(&processor->quantum_timer, thread,
+ running_timer_enter(processor, RUNNING_TIMER_QUANTUM, thread,
processor->quantum_end, ctime);
/* Tell platform layer that we are still running this thread */
}
int old_base_pri = thread->base_pri;
- thread->req_base_pri = priority;
+ thread->req_base_pri = (int16_t)priority;
if (thread->sched_flags & TH_SFLAG_BASE_PRI_FROZEN) {
priority = MAX(priority, old_base_pri);
}
- thread->base_pri = priority;
+ thread->base_pri = (int16_t)priority;
if ((thread->state & TH_RUN) == TH_RUN) {
assert(thread->last_made_runnable_time != THREAD_NOT_RUNNABLE);
if (new_priority > thread->max_priority) {
new_priority = thread->max_priority;
}
-#if CONFIG_EMBEDDED
+#if !defined(XNU_TARGET_OS_OSX)
if (new_priority < MAXPRI_THROTTLE) {
new_priority = MAXPRI_THROTTLE;
}
-#endif /* CONFIG_EMBEDDED */
+#endif /* !defined(XNU_TARGET_OS_OSX) */
thread->importance = new_priority - thread->task_priority;
uint32_t sched_flags = thread->sched_flags;
sched_mode_t sched_mode = thread->sched_mode;
- int priority = thread->base_pri;
+ int16_t priority = thread->base_pri;
if (sched_mode == TH_MODE_TIMESHARE) {
- priority = SCHED(compute_timeshare_priority)(thread);
+ priority = (int16_t)SCHED(compute_timeshare_priority)(thread);
}
if (sched_flags & TH_SFLAG_DEPRESS) {
*/
}
+int smt_timeshare_enabled = 1;
+int smt_sched_bonus_16ths = 8;
+
#if defined(CONFIG_SCHED_TIMESHARE_CORE)
/*
* resources.
*/
if (thread->pri_shift < INT8_MAX) {
- thread->sched_usage += delta;
+ if (thread_no_smt(thread) && smt_timeshare_enabled) {
+ thread->sched_usage += (delta + ((delta * smt_sched_bonus_16ths) >> 4));
+ } else {
+ thread->sched_usage += delta;
+ }
}
thread->cpu_delta += delta;
extern int sched_pri_decay_band_limit;
-/* Only use the decay floor logic on embedded non-clutch schedulers */
-#if CONFIG_EMBEDDED && !CONFIG_SCHED_CLUTCH
+/* Only use the decay floor logic on non-macOS and non-clutch schedulers */
+#if !defined(XNU_TARGET_OS_OSX) && !CONFIG_SCHED_CLUTCH
int
sched_compute_timeshare_priority(thread_t thread)
{
- int decay_amount = (thread->sched_usage >> thread->pri_shift);
+ int decay_amount;
int decay_limit = sched_pri_decay_band_limit;
if (thread->base_pri > BASEPRI_FOREGROUND) {
decay_limit += (thread->base_pri - BASEPRI_FOREGROUND);
}
+ if (thread->pri_shift == INT8_MAX) {
+ decay_amount = 0;
+ } else {
+ decay_amount = (thread->sched_usage >> thread->pri_shift);
+ }
+
if (decay_amount > decay_limit) {
decay_amount = decay_limit;
}
return priority;
}
-#else /* CONFIG_EMBEDDED && !CONFIG_SCHED_CLUTCH */
+#else /* !defined(XNU_TARGET_OS_OSX) && !CONFIG_SCHED_CLUTCH */
int
sched_compute_timeshare_priority(thread_t thread)
{
/* start with base priority */
- int priority = thread->base_pri - (thread->sched_usage >> thread->pri_shift);
+ int priority = thread->base_pri;
+
+ if (thread->pri_shift != INT8_MAX) {
+ priority -= (thread->sched_usage >> thread->pri_shift);
+ }
if (priority < MINPRI_USER) {
priority = MINPRI_USER;
return priority;
}
-#endif /* CONFIG_EMBEDDED && !CONFIG_SCHED_CLUTCH */
+#endif /* !defined(XNU_TARGET_OS_OSX) && !CONFIG_SCHED_CLUTCH */
/*
* can_update_priority
* determine if the system was in a contended state.
*/
if (thread->pri_shift < INT8_MAX) {
- thread->sched_usage += delta;
+ if (thread_no_smt(thread) && smt_timeshare_enabled) {
+ thread->sched_usage += (delta + ((delta * smt_sched_bonus_16ths) >> 4));
+ } else {
+ thread->sched_usage += delta;
+ }
}
thread->cpu_usage += delta + thread->cpu_delta;
os_atomic_dec(&sched_run_buckets[bucket], relaxed);
}
+static void
+sched_add_bucket(sched_bucket_t bucket, uint8_t run_weight)
+{
+ assert(bucket >= TH_BUCKET_FIXPRI &&
+ bucket <= TH_BUCKET_SHARE_BG);
+
+ os_atomic_add(&sched_run_buckets[bucket], run_weight, relaxed);
+}
+
+static void
+sched_sub_bucket(sched_bucket_t bucket, uint8_t run_weight)
+{
+ assert(bucket >= TH_BUCKET_FIXPRI &&
+ bucket <= TH_BUCKET_SHARE_BG);
+
+ assert(os_atomic_load(&sched_run_buckets[bucket], relaxed) > 0);
+
+ os_atomic_sub(&sched_run_buckets[bucket], run_weight, relaxed);
+}
+
uint32_t
sched_run_incr(thread_t thread)
{
return new_count;
}
+uint32_t
+sched_smt_run_incr(thread_t thread)
+{
+ assert((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN);
+
+ uint8_t run_weight = (thread_no_smt(thread) && smt_timeshare_enabled) ? 2 : 1;
+ thread->sched_saved_run_weight = run_weight;
+
+ uint32_t new_count = os_atomic_add(&sched_run_buckets[TH_BUCKET_RUN], run_weight, relaxed);
+
+ sched_add_bucket(thread->th_sched_bucket, run_weight);
+
+ return new_count;
+}
+
+uint32_t
+sched_smt_run_decr(thread_t thread)
+{
+ assert((thread->state & (TH_RUN | TH_IDLE)) != TH_RUN);
+
+ uint8_t run_weight = thread->sched_saved_run_weight;
+
+ sched_sub_bucket(thread->th_sched_bucket, run_weight);
+
+ uint32_t new_count = os_atomic_sub(&sched_run_buckets[TH_BUCKET_RUN], run_weight, relaxed);
+
+ return new_count;
+}
+
void
sched_update_thread_bucket(thread_t thread)
{
}
}
+void
+sched_smt_update_thread_bucket(thread_t thread)
+{
+ sched_bucket_t old_bucket = thread->th_sched_bucket;
+ sched_bucket_t new_bucket = TH_BUCKET_RUN;
+
+ switch (thread->sched_mode) {
+ case TH_MODE_FIXED:
+ case TH_MODE_REALTIME:
+ new_bucket = TH_BUCKET_FIXPRI;
+ break;
+
+ case TH_MODE_TIMESHARE:
+ if (thread->base_pri > BASEPRI_DEFAULT) {
+ new_bucket = TH_BUCKET_SHARE_FG;
+ } else if (thread->base_pri > BASEPRI_UTILITY) {
+ new_bucket = TH_BUCKET_SHARE_DF;
+ } else if (thread->base_pri > MAXPRI_THROTTLE) {
+ new_bucket = TH_BUCKET_SHARE_UT;
+ } else {
+ new_bucket = TH_BUCKET_SHARE_BG;
+ }
+ break;
+
+ default:
+ panic("unexpected mode: %d", thread->sched_mode);
+ break;
+ }
+
+ if (old_bucket != new_bucket) {
+ thread->th_sched_bucket = new_bucket;
+ thread->pri_shift = sched_pri_shifts[new_bucket];
+
+ if ((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN) {
+ sched_sub_bucket(old_bucket, thread->sched_saved_run_weight);
+ sched_add_bucket(new_bucket, thread->sched_saved_run_weight);
+ }
+ }
+}
+
/*
* Set the thread's true scheduling mode
* Called with thread mutex and thread locked
break;
}
+#if CONFIG_SCHED_AUTO_JOIN
+ /*
+ * Realtime threads might have auto-joined a work interval based on
+ * make runnable relationships. If such an RT thread is now being demoted
+ * to non-RT, unjoin the thread from the work interval.
+ */
+ if ((thread->sched_flags & TH_SFLAG_THREAD_GROUP_AUTO_JOIN) && (new_mode != TH_MODE_REALTIME)) {
+ assert((thread->sched_mode == TH_MODE_REALTIME) || (thread->th_work_interval_flags & TH_WORK_INTERVAL_FLAGS_AUTO_JOIN_LEAK));
+ work_interval_auto_join_demote(thread);
+ }
+#endif /* CONFIG_SCHED_AUTO_JOIN */
+
thread->sched_mode = new_mode;
SCHED(update_thread_bucket)(thread);
projects / apple / xnu.git / blobdiff