#include <kern/syscall_subr.h>
#include <mach/mach_host_server.h>
#include <mach/mach_syscalls.h>
-
+#include <sys/kdebug.h>
#ifdef MACH_BSD
extern void workqueue_thread_yielded(void);
+extern sched_call_t workqueue_get_sched_callback(void);
#endif /* MACH_BSD */
* swtch and swtch_pri both attempt to context switch (logic in
* thread_block no-ops the context switch if nothing would happen).
* A boolean is returned that indicates whether there is anything
- * else runnable.
- *
- * This boolean can be used by a thread waiting on a
- * lock or condition: If FALSE is returned, the thread is justified
- * in becoming a resource hog by continuing to spin because there's
- * nothing else useful that the processor could do. If TRUE is
- * returned, the thread should make one more check on the
- * lock and then be a good citizen and really suspend.
+ * else runnable. That's no excuse to spin, though.
*/
static void
disable_preemption();
myprocessor = current_processor();
- result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+ result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
enable_preemption();
thread_syscall_return(result);
disable_preemption();
myprocessor = current_processor();
- if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
+ if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
mp_enable_preemption();
return (FALSE);
disable_preemption();
myprocessor = current_processor();
- result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+ result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
enable_preemption();
return (result);
disable_preemption();
myprocessor = current_processor();
- result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+ result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
mp_enable_preemption();
thread_syscall_return(result);
disable_preemption();
myprocessor = current_processor();
- if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
+ if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
mp_enable_preemption();
return (FALSE);
counter(c_swtch_pri_block++);
- thread_depress_abstime(std_quantum);
+ thread_depress_abstime(thread_depress_time);
thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD);
disable_preemption();
myprocessor = current_processor();
- result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+ result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
enable_preemption();
return (result);
}
+static int
+thread_switch_disable_workqueue_sched_callback(void)
+{
+ sched_call_t callback = workqueue_get_sched_callback();
+ thread_t self = current_thread();
+ if (!callback || self->sched_call != callback) {
+ return FALSE;
+ }
+ spl_t s = splsched();
+ thread_lock(self);
+ thread_sched_call(self, NULL);
+ thread_unlock(self);
+ splx(s);
+ return TRUE;
+}
+
+static void
+thread_switch_enable_workqueue_sched_callback(void)
+{
+ sched_call_t callback = workqueue_get_sched_callback();
+ thread_t self = current_thread();
+ spl_t s = splsched();
+ thread_lock(self);
+ thread_sched_call(self, callback);
+ thread_unlock(self);
+ splx(s);
+}
+
static void
thread_switch_continue(void)
{
register thread_t self = current_thread();
int option = self->saved.swtch.option;
+ boolean_t reenable_workq_callback = self->saved.swtch.reenable_workq_callback;
- if (option == SWITCH_OPTION_DEPRESS)
+
+ if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS)
thread_depress_abort_internal(self);
+ if (reenable_workq_callback)
+ thread_switch_enable_workqueue_sched_callback();
+
thread_syscall_return(KERN_SUCCESS);
/*NOTREACHED*/
}
thread_switch(
struct thread_switch_args *args)
{
- register thread_t thread, self = current_thread();
+ thread_t thread = THREAD_NULL;
+ thread_t self = current_thread();
mach_port_name_t thread_name = args->thread_name;
int option = args->option;
mach_msg_timeout_t option_time = args->option_time;
+ uint32_t scale_factor = NSEC_PER_MSEC;
+ boolean_t reenable_workq_callback = FALSE;
+ boolean_t depress_option = FALSE;
+ boolean_t wait_option = FALSE;
/*
- * Process option.
+ * Validate and process option.
*/
switch (option) {
case SWITCH_OPTION_NONE:
- case SWITCH_OPTION_DEPRESS:
+ workqueue_thread_yielded();
+ break;
case SWITCH_OPTION_WAIT:
- break;
-
+ wait_option = TRUE;
+ workqueue_thread_yielded();
+ break;
+ case SWITCH_OPTION_DEPRESS:
+ depress_option = TRUE;
+ workqueue_thread_yielded();
+ break;
+ case SWITCH_OPTION_DISPATCH_CONTENTION:
+ scale_factor = NSEC_PER_USEC;
+ wait_option = TRUE;
+ if (thread_switch_disable_workqueue_sched_callback())
+ reenable_workq_callback = TRUE;
+ break;
+ case SWITCH_OPTION_OSLOCK_DEPRESS:
+ depress_option = TRUE;
+ if (thread_switch_disable_workqueue_sched_callback())
+ reenable_workq_callback = TRUE;
+ break;
+ case SWITCH_OPTION_OSLOCK_WAIT:
+ wait_option = TRUE;
+ if (thread_switch_disable_workqueue_sched_callback())
+ reenable_workq_callback = TRUE;
+ break;
default:
return (KERN_INVALID_ARGUMENT);
}
- workqueue_thread_yielded();
-
/*
* Translate the port name if supplied.
*/
- if (thread_name != MACH_PORT_NULL) {
- ipc_port_t port;
+ if (thread_name != MACH_PORT_NULL) {
+ ipc_port_t port;
if (ipc_port_translate_send(self->task->itk_space,
- thread_name, &port) == KERN_SUCCESS) {
+ thread_name, &port) == KERN_SUCCESS) {
ip_reference(port);
ip_unlock(port);
thread = convert_port_to_thread(port);
- ipc_port_release(port);
+ ip_release(port);
if (thread == self) {
- (void)thread_deallocate_internal(thread);
+ thread_deallocate(thread);
thread = THREAD_NULL;
}
}
- else
- thread = THREAD_NULL;
}
- else
- thread = THREAD_NULL;
+
+ if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
+ if (thread != THREAD_NULL) {
+
+ if (thread->task != self->task) {
+ /*
+ * OSLock boosting only applies to other threads
+ * in your same task (even if you have a port for
+ * a thread in another task)
+ */
+
+ thread_deallocate(thread);
+ thread = THREAD_NULL;
+ } else {
+ /*
+ * Attempt to kick the lock owner up to our same IO throttling tier.
+ * If the thread is currently blocked in throttle_lowpri_io(),
+ * it will immediately break out.
+ *
+ * TODO: SFI break out?
+ */
+ int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
+
+ set_thread_iotier_override(thread, new_policy);
+ }
+ }
+ }
/*
* Try to handoff if supplied.
*/
if (thread != THREAD_NULL) {
- processor_t processor;
- spl_t s;
-
- s = splsched();
- thread_lock(thread);
-
- /*
- * Check that the thread is not bound
- * to a different processor, and that realtime
- * is not involved.
- *
- * Next, pull it off its run queue. If it
- * doesn't come, it's not eligible.
- */
- processor = current_processor();
- if (processor->current_pri < BASEPRI_RTQUEUES &&
- thread->sched_pri < BASEPRI_RTQUEUES &&
- (thread->bound_processor == PROCESSOR_NULL ||
- thread->bound_processor == processor) &&
- run_queue_remove(thread) ) {
- /*
- * Hah, got it!!
- */
- thread_unlock(thread);
-
- (void)thread_deallocate_internal(thread);
-
- if (option == SWITCH_OPTION_WAIT)
+ spl_t s = splsched();
+
+ /* This may return a different thread if the target is pushing on something */
+ thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_THREAD_SWITCH)|DBG_FUNC_NONE,
+ thread_tid(thread), thread->state,
+ pulled_thread ? TRUE : FALSE, 0, 0);
+
+ if (pulled_thread != THREAD_NULL) {
+ /* We can't be dropping the last ref here */
+ thread_deallocate_safe(thread);
+
+ if (wait_option)
assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
- option_time, 1000*NSEC_PER_USEC);
- else
- if (option == SWITCH_OPTION_DEPRESS)
+ option_time, scale_factor);
+ else if (depress_option)
thread_depress_ms(option_time);
self->saved.swtch.option = option;
+ self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
- thread_run(self, (thread_continue_t)thread_switch_continue, NULL, thread);
+ thread_run(self, (thread_continue_t)thread_switch_continue, NULL, pulled_thread);
/* NOTREACHED */
+ panic("returned from thread_run!");
}
- thread_unlock(thread);
splx(s);
thread_deallocate(thread);
}
-
- if (option == SWITCH_OPTION_WAIT)
- assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, 1000*NSEC_PER_USEC);
- else
- if (option == SWITCH_OPTION_DEPRESS)
+
+ if (wait_option)
+ assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor);
+ else if (depress_option)
thread_depress_ms(option_time);
-
+
self->saved.swtch.option = option;
+ self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
- if (option == SWITCH_OPTION_DEPRESS)
+ if (depress_option)
thread_depress_abort_internal(self);
+ if (reenable_workq_callback)
+ thread_switch_enable_workqueue_sched_callback();
+
return (KERN_SUCCESS);
}
s = splsched();
thread_lock(self);
- if (!(self->sched_mode & TH_MODE_ISDEPRESSED)) {
+ if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
processor_t myprocessor = self->last_processor;
self->sched_pri = DEPRESSPRI;
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY),
+ (uintptr_t)thread_tid(self),
+ self->base_pri,
+ self->sched_pri,
+ 0, /* eventually, 'reason' */
+ 0);
+
myprocessor->current_pri = self->sched_pri;
- self->sched_mode |= TH_MODE_DEPRESS;
+ self->sched_flags |= TH_SFLAG_DEPRESS;
if (interval != 0) {
clock_absolutetime_interval_to_deadline(interval, &deadline);
- if (!timer_call_enter(&self->depress_timer, deadline))
+ if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL))
self->depress_timer_active++;
}
}
uint64_t abstime;
clock_interval_to_absolutetime_interval(
- interval, 1000*NSEC_PER_USEC, &abstime);
+ interval, NSEC_PER_MSEC, &abstime);
thread_depress_abstime(abstime);
}
s = splsched();
thread_lock(thread);
if (--thread->depress_timer_active == 0) {
- thread->sched_mode &= ~TH_MODE_ISDEPRESSED;
- compute_priority(thread, FALSE);
+ thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
+ thread_recompute_sched_pri(thread, FALSE);
}
thread_unlock(thread);
splx(s);
s = splsched();
thread_lock(thread);
- if (!(thread->sched_mode & TH_MODE_POLLDEPRESS)) {
- if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
- thread->sched_mode &= ~TH_MODE_ISDEPRESSED;
- compute_priority(thread, FALSE);
+ if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) {
+ if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
+ thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
+ thread_recompute_sched_pri(thread, FALSE);
result = KERN_SUCCESS;
}
assert(self == current_thread());
s = splsched();
- if (!(self->sched_mode & (TH_MODE_REALTIME|TH_MODE_TIMESHARE))) {
+ if (self->sched_mode == TH_MODE_FIXED) {
uint64_t total_computation, abstime;
abstime = mach_absolute_time();
ast_t preempt;
thread_lock(self);
- if (!(self->sched_mode & TH_MODE_ISDEPRESSED)) {
+ if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
self->sched_pri = DEPRESSPRI;
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY),
+ (uintptr_t)thread_tid(self),
+ self->base_pri,
+ self->sched_pri,
+ 0, /* eventually, 'reason' */
+ 0);
+
myprocessor->current_pri = self->sched_pri;
}
self->computation_epoch = abstime;
self->computation_metered = 0;
- self->sched_mode |= TH_MODE_POLLDEPRESS;
+ self->sched_flags |= TH_SFLAG_POLLDEPRESS;
abstime += (total_computation >> sched_poll_yield_shift);
- if (!timer_call_enter(&self->depress_timer, abstime))
+ if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_USER_CRITICAL))
self->depress_timer_active++;
- thread_unlock(self);
- if ((preempt = csw_check(myprocessor)) != AST_NONE)
+ if ((preempt = csw_check(myprocessor, AST_NONE)) != AST_NONE)
ast_on(preempt);
+
+ thread_unlock(self);
}
}
splx(s);
disable_preemption();
myprocessor = current_processor();
- if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
+ if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
mp_enable_preemption();
return;
projects / apple / xnu.git / blobdiff