X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..15129b1c8dbb3650c63b70adb1cad9af601c6c17:/osfmk/kern/syscall_subr.c?ds=sidebyside diff --git a/osfmk/kern/syscall_subr.c b/osfmk/kern/syscall_subr.c index 3daf1ec38..da2fe9e74 100644 --- a/osfmk/kern/syscall_subr.c +++ b/osfmk/kern/syscall_subr.c @@ -77,6 +77,7 @@ #ifdef MACH_BSD extern void workqueue_thread_yielded(void); +extern sched_call_t workqueue_get_sched_callback(void); #endif /* MACH_BSD */ @@ -115,7 +116,7 @@ swtch_continue(void) disable_preemption(); myprocessor = current_processor(); - result = myprocessor->runq.count > 0 || rt_runq.count > 0; + result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); thread_syscall_return(result); @@ -131,7 +132,7 @@ swtch( disable_preemption(); myprocessor = current_processor(); - if (myprocessor->runq.count == 0 && rt_runq.count == 0) { + if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { mp_enable_preemption(); return (FALSE); @@ -144,7 +145,7 @@ swtch( disable_preemption(); myprocessor = current_processor(); - result = myprocessor->runq.count > 0 || rt_runq.count > 0; + result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); return (result); @@ -160,7 +161,7 @@ swtch_pri_continue(void) disable_preemption(); myprocessor = current_processor(); - result = myprocessor->runq.count > 0 || rt_runq.count > 0; + result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; mp_enable_preemption(); thread_syscall_return(result); @@ -176,7 +177,7 @@ __unused struct swtch_pri_args *args) disable_preemption(); myprocessor = current_processor(); - if (myprocessor->runq.count == 0 && rt_runq.count == 0) { + if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { mp_enable_preemption(); return (FALSE); @@ -185,7 +186,7 @@ __unused struct swtch_pri_args *args) counter(c_swtch_pri_block++); - thread_depress_abstime(std_quantum); + thread_depress_abstime(thread_depress_time); thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD); @@ -193,21 +194,54 @@ __unused struct swtch_pri_args *args) disable_preemption(); myprocessor = current_processor(); - result = myprocessor->runq.count > 0 || rt_runq.count > 0; + result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); return (result); } +static int +thread_switch_disable_workqueue_sched_callback(void) +{ + sched_call_t callback = workqueue_get_sched_callback(); + thread_t self = current_thread(); + if (!callback || self->sched_call != callback) { + return FALSE; + } + spl_t s = splsched(); + thread_lock(self); + thread_sched_call(self, NULL); + thread_unlock(self); + splx(s); + return TRUE; +} + +static void +thread_switch_enable_workqueue_sched_callback(void) +{ + sched_call_t callback = workqueue_get_sched_callback(); + thread_t self = current_thread(); + spl_t s = splsched(); + thread_lock(self); + thread_sched_call(self, callback); + thread_unlock(self); + splx(s); +} + static void thread_switch_continue(void) { register thread_t self = current_thread(); int option = self->saved.swtch.option; + boolean_t reenable_workq_callback = self->saved.swtch.reenable_workq_callback; + - if (option == SWITCH_OPTION_DEPRESS) + if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS) thread_depress_abort_internal(self); + if (reenable_workq_callback) + thread_switch_enable_workqueue_sched_callback(); + thread_syscall_return(KERN_SUCCESS); /*NOTREACHED*/ } @@ -225,23 +259,47 @@ thread_switch( mach_port_name_t thread_name = args->thread_name; int option = args->option; mach_msg_timeout_t option_time = args->option_time; + uint32_t scale_factor = NSEC_PER_MSEC; + boolean_t reenable_workq_callback = FALSE; + boolean_t depress_option = FALSE; + boolean_t wait_option = FALSE; /* - * Process option. + * Validate and process option. */ switch (option) { case SWITCH_OPTION_NONE: - case SWITCH_OPTION_DEPRESS: + workqueue_thread_yielded(); + break; case SWITCH_OPTION_WAIT: - break; - + wait_option = TRUE; + workqueue_thread_yielded(); + break; + case SWITCH_OPTION_DEPRESS: + depress_option = TRUE; + workqueue_thread_yielded(); + break; + case SWITCH_OPTION_DISPATCH_CONTENTION: + scale_factor = NSEC_PER_USEC; + wait_option = TRUE; + if (thread_switch_disable_workqueue_sched_callback()) + reenable_workq_callback = TRUE; + break; + case SWITCH_OPTION_OSLOCK_DEPRESS: + depress_option = TRUE; + if (thread_switch_disable_workqueue_sched_callback()) + reenable_workq_callback = TRUE; + break; + case SWITCH_OPTION_OSLOCK_WAIT: + wait_option = TRUE; + if (thread_switch_disable_workqueue_sched_callback()) + reenable_workq_callback = TRUE; + break; default: return (KERN_INVALID_ARGUMENT); } - workqueue_thread_yielded(); - /* * Translate the port name if supplied. */ @@ -254,7 +312,7 @@ thread_switch( ip_unlock(port); thread = convert_port_to_thread(port); - ipc_port_release(port); + ip_release(port); if (thread == self) { (void)thread_deallocate_internal(thread); @@ -267,6 +325,32 @@ thread_switch( else thread = THREAD_NULL; + + if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) { + if (thread != THREAD_NULL) { + + if (thread->task != self->task) { + /* + * OSLock boosting only applies to other threads + * in your same task (even if you have a port for + * a thread in another task) + */ + + (void)thread_deallocate_internal(thread); + thread = THREAD_NULL; + } else { + /* + * Attempt to kick the lock owner up to our same IO throttling tier. + * If the thread is currently blocked in throttle_lowpri_io(), + * it will immediately break out. + */ + int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO); + + set_thread_iotier_override(thread, new_policy); + } + } + } + /* * Try to handoff if supplied. */ @@ -290,7 +374,7 @@ thread_switch( thread->sched_pri < BASEPRI_RTQUEUES && (thread->bound_processor == PROCESSOR_NULL || thread->bound_processor == processor) && - run_queue_remove(thread) ) { + thread_run_queue_remove(thread) ) { /* * Hah, got it!! */ @@ -298,14 +382,15 @@ thread_switch( (void)thread_deallocate_internal(thread); - if (option == SWITCH_OPTION_WAIT) + if (wait_option) assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, - option_time, 1000*NSEC_PER_USEC); + option_time, scale_factor); else - if (option == SWITCH_OPTION_DEPRESS) + if (depress_option) thread_depress_ms(option_time); self->saved.swtch.option = option; + self->saved.swtch.reenable_workq_callback = reenable_workq_callback; thread_run(self, (thread_continue_t)thread_switch_continue, NULL, thread); /* NOTREACHED */ @@ -317,19 +402,23 @@ thread_switch( thread_deallocate(thread); } - if (option == SWITCH_OPTION_WAIT) - assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, 1000*NSEC_PER_USEC); + if (wait_option) + assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor); else - if (option == SWITCH_OPTION_DEPRESS) + if (depress_option) thread_depress_ms(option_time); self->saved.swtch.option = option; + self->saved.swtch.reenable_workq_callback = reenable_workq_callback; thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD); - if (option == SWITCH_OPTION_DEPRESS) + if (depress_option) thread_depress_abort_internal(self); + if (reenable_workq_callback) + thread_switch_enable_workqueue_sched_callback(); + return (KERN_SUCCESS); } @@ -347,16 +436,16 @@ thread_depress_abstime( s = splsched(); thread_lock(self); - if (!(self->sched_mode & TH_MODE_ISDEPRESSED)) { + if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) { processor_t myprocessor = self->last_processor; self->sched_pri = DEPRESSPRI; myprocessor->current_pri = self->sched_pri; - self->sched_mode |= TH_MODE_DEPRESS; + self->sched_flags |= TH_SFLAG_DEPRESS; if (interval != 0) { clock_absolutetime_interval_to_deadline(interval, &deadline); - if (!timer_call_enter(&self->depress_timer, deadline)) + if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL)) self->depress_timer_active++; } } @@ -371,7 +460,7 @@ thread_depress_ms( uint64_t abstime; clock_interval_to_absolutetime_interval( - interval, 1000*NSEC_PER_USEC, &abstime); + interval, NSEC_PER_MSEC, &abstime); thread_depress_abstime(abstime); } @@ -389,8 +478,8 @@ thread_depress_expire( s = splsched(); thread_lock(thread); if (--thread->depress_timer_active == 0) { - thread->sched_mode &= ~TH_MODE_ISDEPRESSED; - compute_priority(thread, FALSE); + thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK; + SCHED(compute_priority)(thread, FALSE); } thread_unlock(thread); splx(s); @@ -408,10 +497,10 @@ thread_depress_abort_internal( s = splsched(); thread_lock(thread); - if (!(thread->sched_mode & TH_MODE_POLLDEPRESS)) { - if (thread->sched_mode & TH_MODE_ISDEPRESSED) { - thread->sched_mode &= ~TH_MODE_ISDEPRESSED; - compute_priority(thread, FALSE); + if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) { + if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) { + thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK; + SCHED(compute_priority)(thread, FALSE); result = KERN_SUCCESS; } @@ -433,7 +522,7 @@ thread_poll_yield( assert(self == current_thread()); s = splsched(); - if (!(self->sched_mode & (TH_MODE_REALTIME|TH_MODE_TIMESHARE))) { + if (self->sched_mode == TH_MODE_FIXED) { uint64_t total_computation, abstime; abstime = mach_absolute_time(); @@ -444,16 +533,16 @@ thread_poll_yield( ast_t preempt; thread_lock(self); - if (!(self->sched_mode & TH_MODE_ISDEPRESSED)) { + if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) { self->sched_pri = DEPRESSPRI; myprocessor->current_pri = self->sched_pri; } self->computation_epoch = abstime; self->computation_metered = 0; - self->sched_mode |= TH_MODE_POLLDEPRESS; + self->sched_flags |= TH_SFLAG_POLLDEPRESS; abstime += (total_computation >> sched_poll_yield_shift); - if (!timer_call_enter(&self->depress_timer, abstime)) + if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_USER_CRITICAL)) self->depress_timer_active++; thread_unlock(self); @@ -473,7 +562,7 @@ thread_yield_internal( disable_preemption(); myprocessor = current_processor(); - if (myprocessor->runq.count == 0 && rt_runq.count == 0) { + if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { mp_enable_preemption(); return;