+ return KERN_INVALID_ARGUMENT;
+ }
+
+ /*
+ * Translate the port name if supplied.
+ */
+ if (thread_name != MACH_PORT_NULL) {
+ thread = port_name_to_thread(thread_name, ptt_options);
+ }
+
+ if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
+ if (thread != THREAD_NULL) {
+ /*
+ * Attempt to kick the lock owner up to our same IO throttling tier.
+ * If the thread is currently blocked in throttle_lowpri_io(),
+ * it will immediately break out.
+ *
+ * TODO: SFI break out?
+ */
+ int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
+
+ set_thread_iotier_override(thread, new_policy);
+ }
+ }
+
+ /*
+ * Try to handoff if supplied.
+ */
+ if (thread != THREAD_NULL) {
+ spl_t s = splsched();
+
+ /* This may return a different thread if the target is pushing on something */
+ thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_THREAD_SWITCH) | DBG_FUNC_NONE,
+ thread_tid(thread), thread->state,
+ pulled_thread ? TRUE : FALSE, 0, 0);
+
+ if (pulled_thread != THREAD_NULL) {
+ /* We can't be dropping the last ref here */
+ thread_deallocate_safe(thread);
+
+ if (wait_option) {
+ assert_wait_timeout((event_t)assert_wait_timeout, interruptible,
+ option_time, scale_factor);
+ } else if (depress_option) {
+ thread_depress_ms(option_time);
+ }
+
+ thread_run(self, thread_switch_continue, (void *)(intptr_t)option, pulled_thread);
+ __builtin_unreachable();
+ }
+
+ splx(s);
+
+ thread_deallocate(thread);
+ }
+
+ if (wait_option) {
+ assert_wait_timeout((event_t)assert_wait_timeout, interruptible, option_time, scale_factor);
+ } else {
+ disable_preemption();
+ bool should_yield = SCHED(thread_should_yield)(current_processor(), current_thread());
+ enable_preemption();
+
+ if (should_yield == false) {
+ /* Early-return if yielding to the scheduler will not be beneficial */
+ return KERN_SUCCESS;
+ }
+
+ if (depress_option) {
+ thread_depress_ms(option_time);
+ }
+ }
+
+ thread_yield_with_continuation(thread_switch_continue, (void *)(intptr_t)option);
+ __builtin_unreachable();
+}
+
+void
+thread_yield_with_continuation(
+ thread_continue_t continuation,
+ void *parameter)
+{
+ assert(continuation);
+ thread_block_reason(continuation, parameter, AST_YIELD);
+ __builtin_unreachable();
+}
+
+
+/* This function is called after an assert_wait(), therefore it must not
+ * cause another wait until after the thread_run() or thread_block()
+ *
+ *
+ * When called with a NULL continuation, the thread ref is consumed
+ * (thread_handoff_deallocate calling convention) else it is up to the
+ * continuation to do the cleanup (thread_handoff_parameter calling convention)
+ * and it instead doesn't return.
+ */
+static wait_result_t
+thread_handoff_internal(thread_t thread, thread_continue_t continuation,
+ void *parameter)
+{
+ thread_t deallocate_thread = THREAD_NULL;
+ thread_t self = current_thread();
+
+ /*
+ * Try to handoff if supplied.
+ */
+ if (thread != THREAD_NULL) {
+ spl_t s = splsched();
+
+ thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_THREAD_SWITCH) | DBG_FUNC_NONE,
+ thread_tid(thread), thread->state,
+ pulled_thread ? TRUE : FALSE, 0, 0);
+
+ if (pulled_thread != THREAD_NULL) {
+ if (continuation == NULL) {
+ /* We can't be dropping the last ref here */
+ thread_deallocate_safe(thread);
+ }
+
+ int result = thread_run(self, continuation, parameter, pulled_thread);
+
+ splx(s);
+ return result;
+ }
+
+ splx(s);
+
+ deallocate_thread = thread;
+ thread = THREAD_NULL;
+ }
+
+ int result = thread_block_parameter(continuation, parameter);
+ if (deallocate_thread != THREAD_NULL) {
+ thread_deallocate(deallocate_thread);
+ }
+
+ return result;
+}
+
+void
+thread_handoff_parameter(thread_t thread, thread_continue_t continuation,
+ void *parameter)
+{
+ thread_handoff_internal(thread, continuation, parameter);
+ panic("NULL continuation passed to %s", __func__);
+ __builtin_unreachable();
+}
+
+wait_result_t
+thread_handoff_deallocate(thread_t thread)
+{
+ return thread_handoff_internal(thread, NULL, NULL);
+}
+
+/*
+ * Thread depression
+ *
+ * This mechanism drops a thread to priority 0 in order for it to yield to
+ * all other runnnable threads on the system. It can be canceled or timed out,
+ * whereupon the thread goes back to where it was.
+ *
+ * Note that TH_SFLAG_DEPRESS and TH_SFLAG_POLLDEPRESS are never set at the
+ * same time. DEPRESS always defers to POLLDEPRESS.
+ *
+ * DEPRESS only lasts across a single thread_block call, and never returns
+ * to userspace.
+ * POLLDEPRESS can be active anywhere up until thread termination.
+ */
+
+/*
+ * Depress thread's priority to lowest possible for the specified interval,
+ * with an interval of zero resulting in no timeout being scheduled.
+ *
+ * Must block with AST_YIELD afterwards to take effect
+ */
+void
+thread_depress_abstime(uint64_t interval)
+{
+ thread_t self = current_thread();
+
+ spl_t s = splsched();
+ thread_lock(self);
+
+ assert((self->sched_flags & TH_SFLAG_DEPRESS) == 0);
+
+ if ((self->sched_flags & TH_SFLAG_POLLDEPRESS) == 0) {
+ self->sched_flags |= TH_SFLAG_DEPRESS;
+ thread_recompute_sched_pri(self, SETPRI_LAZY);
+
+ if (interval != 0) {
+ uint64_t deadline;
+
+ clock_absolutetime_interval_to_deadline(interval, &deadline);
+ if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL)) {
+ self->depress_timer_active++;
+ }
+ }
+ }
+
+ thread_unlock(self);
+ splx(s);
+}
+
+void
+thread_depress_ms(mach_msg_timeout_t interval)
+{
+ uint64_t abstime;
+
+ clock_interval_to_absolutetime_interval(interval, NSEC_PER_MSEC, &abstime);
+ thread_depress_abstime(abstime);
+}
+
+/*
+ * Priority depression expiration.
+ */
+void
+thread_depress_expire(void *p0,
+ __unused void *p1)
+{
+ thread_t thread = (thread_t)p0;
+
+ spl_t s = splsched();
+ thread_lock(thread);
+
+ assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
+
+ if (--thread->depress_timer_active == 0) {
+ thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
+ if ((thread->state & TH_RUN) == TH_RUN) {
+ thread->last_basepri_change_time = mach_absolute_time();
+ }
+ thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
+ }
+
+ thread_unlock(thread);
+ splx(s);
+}
+
+/*
+ * Prematurely abort priority depression if there is one.
+ */
+kern_return_t
+thread_depress_abort(thread_t thread)
+{
+ kern_return_t result = KERN_NOT_DEPRESSED;
+
+ spl_t s = splsched();
+ thread_lock(thread);
+
+ assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
+
+ /*
+ * User-triggered depress-aborts should not get out
+ * of the poll-depress, but they should cancel a regular depress.
+ */
+ if ((thread->sched_flags & TH_SFLAG_POLLDEPRESS) == 0) {
+ result = thread_depress_abort_locked(thread);
+ }
+
+ thread_unlock(thread);
+ splx(s);
+
+ return result;
+}
+
+/*
+ * Prematurely abort priority depression or poll depression if one is active.
+ * Called with the thread locked.
+ */
+kern_return_t
+thread_depress_abort_locked(thread_t thread)
+{
+ if ((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0) {
+ return KERN_NOT_DEPRESSED;
+ }
+
+ assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
+
+ thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
+ if ((thread->state & TH_RUN) == TH_RUN) {
+ thread->last_basepri_change_time = mach_absolute_time();
+ }
+
+ thread_recompute_sched_pri(thread, SETPRI_LAZY);
+
+ if (timer_call_cancel(&thread->depress_timer)) {
+ thread->depress_timer_active--;
+ }
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * Invoked as part of a polling operation like a no-timeout port receive
+ *
+ * Forces a fixpri thread to yield if it is detected polling without blocking for too long.
+ */
+void
+thread_poll_yield(thread_t self)
+{
+ assert(self == current_thread());
+ assert((self->sched_flags & TH_SFLAG_DEPRESS) == 0);
+
+ if (self->sched_mode != TH_MODE_FIXED) {
+ return;
+ }
+
+ spl_t s = splsched();
projects / apple / xnu.git / blobdiff