xnu-3248.40.184.tar.gz

[apple/xnu.git] / osfmk / kern / syscall_subr.c

diff --git a/osfmk/kern/syscall_subr.c b/osfmk/kern/syscall_subr.c
index 3daf1ec3874eccb8884bcb60508ff74ba14ffe8b..1ebf39e4f9d4f405b1a9e9a0f41f3624b0595bb9 100644 (file)
--- a/osfmk/kern/syscall_subr.c
+++ b/osfmk/kern/syscall_subr.c
@@ -73,10 +73,11 @@
 #include <kern/syscall_subr.h>
 #include <mach/mach_host_server.h>
 #include <mach/mach_syscalls.h>
 #include <kern/syscall_subr.h>
 #include <mach/mach_host_server.h>
 #include <mach/mach_syscalls.h>

-
+#include <sys/kdebug.h>

 
 #ifdef MACH_BSD
 extern void workqueue_thread_yielded(void);
 
 #ifdef MACH_BSD
 extern void workqueue_thread_yielded(void);

+extern sched_call_t workqueue_get_sched_callback(void);

 #endif /* MACH_BSD */
 
 
 #endif /* MACH_BSD */
 
 


@@ -97,14 +98,7 @@ __unused     struct pfz_exit_args *args)
  *     swtch and swtch_pri both attempt to context switch (logic in
  *     thread_block no-ops the context switch if nothing would happen).
  *     A boolean is returned that indicates whether there is anything
  *     swtch and swtch_pri both attempt to context switch (logic in
  *     thread_block no-ops the context switch if nothing would happen).
  *     A boolean is returned that indicates whether there is anything

- *     else runnable.
- *
- *     This boolean can be used by a thread waiting on a
- *     lock or condition:  If FALSE is returned, the thread is justified
- *     in becoming a resource hog by continuing to spin because there's
- *     nothing else useful that the processor could do.  If TRUE is
- *     returned, the thread should make one more check on the
- *     lock and then be a good citizen and really suspend.
+ *     else runnable.  That's no excuse to spin, though.

  */
 
 static void
  */
 
 static void


@@ -115,7 +109,7 @@ swtch_continue(void)
 
     disable_preemption();
        myprocessor = current_processor();
 
     disable_preemption();
        myprocessor = current_processor();

-       result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+       result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;

        enable_preemption();
 
        thread_syscall_return(result);
        enable_preemption();
 
        thread_syscall_return(result);


@@ -131,7 +125,7 @@ swtch(
 
        disable_preemption();
        myprocessor = current_processor();
 
        disable_preemption();
        myprocessor = current_processor();

-       if (myprocessor->runq.count == 0 &&     rt_runq.count == 0) {
+       if (SCHED(processor_queue_empty)(myprocessor) &&        rt_runq.count == 0) {

                mp_enable_preemption();
 
                return (FALSE);
                mp_enable_preemption();
 
                return (FALSE);


@@ -144,7 +138,7 @@ swtch(
 
        disable_preemption();
        myprocessor = current_processor();
 
        disable_preemption();
        myprocessor = current_processor();

-       result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+       result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;

        enable_preemption();
 
        return (result);
        enable_preemption();
 
        return (result);


@@ -160,7 +154,7 @@ swtch_pri_continue(void)
 
     disable_preemption();
        myprocessor = current_processor();
 
     disable_preemption();
        myprocessor = current_processor();

-       result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+       result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;

        mp_enable_preemption();
 
        thread_syscall_return(result);
        mp_enable_preemption();
 
        thread_syscall_return(result);


@@ -176,7 +170,7 @@ __unused    struct swtch_pri_args *args)
 
        disable_preemption();
        myprocessor = current_processor();
 
        disable_preemption();
        myprocessor = current_processor();

-       if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
+       if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {

                mp_enable_preemption();
 
                return (FALSE);
                mp_enable_preemption();
 
                return (FALSE);


@@ -185,7 +179,7 @@ __unused    struct swtch_pri_args *args)
 
        counter(c_swtch_pri_block++);
 
 
        counter(c_swtch_pri_block++);
 

-       thread_depress_abstime(std_quantum);
+       thread_depress_abstime(thread_depress_time);

 
        thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD);
 
 
        thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD);
 


@@ -193,21 +187,54 @@ __unused  struct swtch_pri_args *args)
 
        disable_preemption();
        myprocessor = current_processor();
 
        disable_preemption();
        myprocessor = current_processor();

-       result = myprocessor->runq.count > 0 || rt_runq.count > 0;
+       result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;

        enable_preemption();
 
        return (result);
 }
 
        enable_preemption();
 
        return (result);
 }
 

+static int
+thread_switch_disable_workqueue_sched_callback(void)
+{
+       sched_call_t callback = workqueue_get_sched_callback();
+       thread_t self = current_thread();
+       if (!callback || self->sched_call != callback) {
+               return FALSE;
+       }
+       spl_t s = splsched();
+       thread_lock(self);
+       thread_sched_call(self, NULL);
+       thread_unlock(self);
+       splx(s);
+       return TRUE;
+}
+
+static void
+thread_switch_enable_workqueue_sched_callback(void)
+{
+       sched_call_t callback = workqueue_get_sched_callback();
+       thread_t self = current_thread();
+       spl_t s = splsched();
+       thread_lock(self);
+       thread_sched_call(self, callback);
+       thread_unlock(self);
+       splx(s);
+}
+

 static void
 thread_switch_continue(void)
 {
        register thread_t       self = current_thread();
        int                                     option = self->saved.swtch.option;
 static void
 thread_switch_continue(void)
 {
        register thread_t       self = current_thread();
        int                                     option = self->saved.swtch.option;

+       boolean_t                       reenable_workq_callback = self->saved.swtch.reenable_workq_callback;

 
 

-       if (option == SWITCH_OPTION_DEPRESS)
+
+       if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS)

                thread_depress_abort_internal(self);
 
                thread_depress_abort_internal(self);
 

+       if (reenable_workq_callback)
+               thread_switch_enable_workqueue_sched_callback();
+

        thread_syscall_return(KERN_SUCCESS);
        /*NOTREACHED*/
 }
        thread_syscall_return(KERN_SUCCESS);
        /*NOTREACHED*/
 }


@@ -221,115 +248,152 @@ kern_return_t
 thread_switch(
        struct thread_switch_args *args)
 {
 thread_switch(
        struct thread_switch_args *args)
 {

-       register thread_t               thread, self = current_thread();
+       thread_t                        thread = THREAD_NULL;
+       thread_t                        self = current_thread();

        mach_port_name_t                thread_name = args->thread_name;
        int                                             option = args->option;
        mach_msg_timeout_t              option_time = args->option_time;
        mach_port_name_t                thread_name = args->thread_name;
        int                                             option = args->option;
        mach_msg_timeout_t              option_time = args->option_time;

+       uint32_t                                scale_factor = NSEC_PER_MSEC;
+       boolean_t                               reenable_workq_callback = FALSE;
+       boolean_t                               depress_option = FALSE;
+       boolean_t                               wait_option = FALSE;

 
     /*
 
     /*

-     * Process option.
+     * Validate and process option.

      */
     switch (option) {
 
        case SWITCH_OPTION_NONE:
      */
     switch (option) {
 
        case SWITCH_OPTION_NONE:

-       case SWITCH_OPTION_DEPRESS:
+               workqueue_thread_yielded();
+               break;

        case SWITCH_OPTION_WAIT:
        case SWITCH_OPTION_WAIT:

-           break;
-
+               wait_option = TRUE;
+               workqueue_thread_yielded();
+               break;
+       case SWITCH_OPTION_DEPRESS:
+               depress_option = TRUE;
+               workqueue_thread_yielded();
+               break;
+       case SWITCH_OPTION_DISPATCH_CONTENTION:
+               scale_factor = NSEC_PER_USEC;
+               wait_option = TRUE;
+               if (thread_switch_disable_workqueue_sched_callback())
+                       reenable_workq_callback = TRUE;
+               break;
+       case SWITCH_OPTION_OSLOCK_DEPRESS:
+               depress_option = TRUE;
+               if (thread_switch_disable_workqueue_sched_callback())
+                       reenable_workq_callback = TRUE;
+               break;
+       case SWITCH_OPTION_OSLOCK_WAIT:
+               wait_option = TRUE;
+               if (thread_switch_disable_workqueue_sched_callback())
+                       reenable_workq_callback = TRUE;
+               break;

        default:
            return (KERN_INVALID_ARGUMENT);
     }
 
        default:
            return (KERN_INVALID_ARGUMENT);
     }
 

-    workqueue_thread_yielded();
-

        /*
         * Translate the port name if supplied.
         */
        /*
         * Translate the port name if supplied.
         */

-    if (thread_name != MACH_PORT_NULL) {
-               ipc_port_t                      port;
+       if (thread_name != MACH_PORT_NULL) {
+               ipc_port_t port;

 
                if (ipc_port_translate_send(self->task->itk_space,
 
                if (ipc_port_translate_send(self->task->itk_space,

-                                                                       thread_name, &port) == KERN_SUCCESS) {
+                                           thread_name, &port) == KERN_SUCCESS) {

                        ip_reference(port);
                        ip_unlock(port);
 
                        thread = convert_port_to_thread(port);
                        ip_reference(port);
                        ip_unlock(port);
 
                        thread = convert_port_to_thread(port);

-                       ipc_port_release(port);
+                       ip_release(port);

 
                        if (thread == self) {
 
                        if (thread == self) {

-                               (void)thread_deallocate_internal(thread);
+                               thread_deallocate(thread);

                                thread = THREAD_NULL;
                        }
                }
                                thread = THREAD_NULL;
                        }
                }

-               else
-                       thread = THREAD_NULL;

        }
        }

-       else
-               thread = THREAD_NULL;
+
+       if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
+               if (thread != THREAD_NULL) {
+
+                       if (thread->task != self->task) {
+                               /*
+                                * OSLock boosting only applies to other threads
+                                * in your same task (even if you have a port for
+                                * a thread in another task)
+                                */
+
+                               thread_deallocate(thread);
+                               thread = THREAD_NULL;
+                       } else {
+                               /*
+                                * Attempt to kick the lock owner up to our same IO throttling tier.
+                                * If the thread is currently blocked in throttle_lowpri_io(),
+                                * it will immediately break out.
+                                *
+                                * TODO: SFI break out?
+                                */
+                               int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
+
+                               set_thread_iotier_override(thread, new_policy);
+                       }
+               }
+       }

 
        /*
         * Try to handoff if supplied.
         */
        if (thread != THREAD_NULL) {
 
        /*
         * Try to handoff if supplied.
         */
        if (thread != THREAD_NULL) {

-               processor_t             processor;
-               spl_t                   s;
-
-               s = splsched();
-               thread_lock(thread);
-
-               /*
-                *      Check that the thread is not bound
-                *      to a different processor, and that realtime
-                *      is not involved.
-                *
-                *      Next, pull it off its run queue.  If it
-                *      doesn't come, it's not eligible.
-                */
-               processor = current_processor();
-               if (processor->current_pri < BASEPRI_RTQUEUES                   &&
-                       thread->sched_pri < BASEPRI_RTQUEUES                            &&
-                       (thread->bound_processor == PROCESSOR_NULL      ||
-                        thread->bound_processor == processor)                          &&
-                               run_queue_remove(thread)                                                        ) {
-                       /*
-                        *      Hah, got it!!
-                        */
-                       thread_unlock(thread);
-
-                       (void)thread_deallocate_internal(thread);
-
-                       if (option == SWITCH_OPTION_WAIT)
+               spl_t s = splsched();
+
+               /* This may return a different thread if the target is pushing on something */
+               thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
+
+               KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_THREAD_SWITCH)|DBG_FUNC_NONE,
+                                     thread_tid(thread), thread->state,
+                                     pulled_thread ? TRUE : FALSE, 0, 0);
+
+               if (pulled_thread != THREAD_NULL) {
+                       /* We can't be dropping the last ref here */
+                       thread_deallocate_safe(thread);
+
+                       if (wait_option)

                                assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
                                assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,

-                                                                                                               option_time, 1000*NSEC_PER_USEC);
-                       else
-                       if (option == SWITCH_OPTION_DEPRESS)
+                                                   option_time, scale_factor);
+                       else if (depress_option)

                                thread_depress_ms(option_time);
 
                        self->saved.swtch.option = option;
                                thread_depress_ms(option_time);
 
                        self->saved.swtch.option = option;

+                       self->saved.swtch.reenable_workq_callback = reenable_workq_callback;

 
 

-                       thread_run(self, (thread_continue_t)thread_switch_continue, NULL, thread);
+                       thread_run(self, (thread_continue_t)thread_switch_continue, NULL, pulled_thread);

                        /* NOTREACHED */
                        /* NOTREACHED */

+                       panic("returned from thread_run!");

                }
 
                }
 

-               thread_unlock(thread);

                splx(s);
 
                thread_deallocate(thread);
        }
                splx(s);
 
                thread_deallocate(thread);
        }

-               
-       if (option == SWITCH_OPTION_WAIT)
-               assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, 1000*NSEC_PER_USEC);
-       else
-       if (option == SWITCH_OPTION_DEPRESS)
+
+       if (wait_option)
+               assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor);
+       else if (depress_option)

                thread_depress_ms(option_time);
                thread_depress_ms(option_time);

-         
+

        self->saved.swtch.option = option;
        self->saved.swtch.option = option;

+       self->saved.swtch.reenable_workq_callback = reenable_workq_callback;

 
        thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
 
 
        thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
 

-       if (option == SWITCH_OPTION_DEPRESS)
+       if (depress_option)

                thread_depress_abort_internal(self);
 
                thread_depress_abort_internal(self);
 

+       if (reenable_workq_callback)
+               thread_switch_enable_workqueue_sched_callback();
+

     return (KERN_SUCCESS);
 }
 
     return (KERN_SUCCESS);
 }
 


@@ -347,16 +411,24 @@ thread_depress_abstime(
 
     s = splsched();
     thread_lock(self);
 
     s = splsched();
     thread_lock(self);

-       if (!(self->sched_mode & TH_MODE_ISDEPRESSED)) {
+       if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {

                processor_t             myprocessor = self->last_processor;
 
                self->sched_pri = DEPRESSPRI;
                processor_t             myprocessor = self->last_processor;
 
                self->sched_pri = DEPRESSPRI;

+
+               KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY),
+                                     (uintptr_t)thread_tid(self),
+                                     self->base_pri,
+                                     self->sched_pri,
+                                     0, /* eventually, 'reason' */
+                                     0);
+

                myprocessor->current_pri = self->sched_pri;
                myprocessor->current_pri = self->sched_pri;

-               self->sched_mode |= TH_MODE_DEPRESS;
+               self->sched_flags |= TH_SFLAG_DEPRESS;

 
                if (interval != 0) {
                        clock_absolutetime_interval_to_deadline(interval, &deadline);
 
                if (interval != 0) {
                        clock_absolutetime_interval_to_deadline(interval, &deadline);

-                       if (!timer_call_enter(&self->depress_timer, deadline))
+                       if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL))

                                self->depress_timer_active++;
                }
        }
                                self->depress_timer_active++;
                }
        }


@@ -371,7 +443,7 @@ thread_depress_ms(
        uint64_t                abstime;
 
        clock_interval_to_absolutetime_interval(
        uint64_t                abstime;
 
        clock_interval_to_absolutetime_interval(

-                                                       interval, 1000*NSEC_PER_USEC, &abstime);
+                                                       interval, NSEC_PER_MSEC, &abstime);

        thread_depress_abstime(abstime);
 }
 
        thread_depress_abstime(abstime);
 }
 


@@ -389,8 +461,8 @@ thread_depress_expire(
     s = splsched();
     thread_lock(thread);
        if (--thread->depress_timer_active == 0) {
     s = splsched();
     thread_lock(thread);
        if (--thread->depress_timer_active == 0) {

-               thread->sched_mode &= ~TH_MODE_ISDEPRESSED;
-               compute_priority(thread, FALSE);
+               thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
+               thread_recompute_sched_pri(thread, FALSE);

        }
     thread_unlock(thread);
     splx(s);
        }
     thread_unlock(thread);
     splx(s);


@@ -408,10 +480,10 @@ thread_depress_abort_internal(
 
     s = splsched();
     thread_lock(thread);
 
     s = splsched();
     thread_lock(thread);

-       if (!(thread->sched_mode & TH_MODE_POLLDEPRESS)) {
-               if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
-                       thread->sched_mode &= ~TH_MODE_ISDEPRESSED;
-                       compute_priority(thread, FALSE);
+       if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) {
+               if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
+                       thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
+                       thread_recompute_sched_pri(thread, FALSE);

                        result = KERN_SUCCESS;
                }
 
                        result = KERN_SUCCESS;
                }
 


@@ -433,7 +505,7 @@ thread_poll_yield(
        assert(self == current_thread());
 
        s = splsched();
        assert(self == current_thread());
 
        s = splsched();

-       if (!(self->sched_mode & (TH_MODE_REALTIME|TH_MODE_TIMESHARE))) {
+       if (self->sched_mode == TH_MODE_FIXED) {

                uint64_t                        total_computation, abstime;
 
                abstime = mach_absolute_time();
                uint64_t                        total_computation, abstime;
 
                abstime = mach_absolute_time();


@@ -444,21 +516,30 @@ thread_poll_yield(
                        ast_t                   preempt;
 
                        thread_lock(self);
                        ast_t                   preempt;
 
                        thread_lock(self);

-                       if (!(self->sched_mode & TH_MODE_ISDEPRESSED)) {
+                       if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {

                                self->sched_pri = DEPRESSPRI;
                                self->sched_pri = DEPRESSPRI;

+
+                               KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY),
+                                                     (uintptr_t)thread_tid(self),
+                                                     self->base_pri,
+                                                     self->sched_pri,
+                                                     0, /* eventually, 'reason' */
+                                                     0);
+

                                myprocessor->current_pri = self->sched_pri;
                        }
                        self->computation_epoch = abstime;
                        self->computation_metered = 0;
                                myprocessor->current_pri = self->sched_pri;
                        }
                        self->computation_epoch = abstime;
                        self->computation_metered = 0;

-                       self->sched_mode |= TH_MODE_POLLDEPRESS;
+                       self->sched_flags |= TH_SFLAG_POLLDEPRESS;

 
                        abstime += (total_computation >> sched_poll_yield_shift);
 
                        abstime += (total_computation >> sched_poll_yield_shift);

-                       if (!timer_call_enter(&self->depress_timer, abstime))
+                       if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_USER_CRITICAL))

                                self->depress_timer_active++;
                                self->depress_timer_active++;

-                       thread_unlock(self);

 
 

-                       if ((preempt = csw_check(myprocessor)) != AST_NONE)
+                       if ((preempt = csw_check(myprocessor, AST_NONE)) != AST_NONE)

                                ast_on(preempt);
                                ast_on(preempt);

+
+                       thread_unlock(self);

                }
        }
        splx(s);
                }
        }
        splx(s);


@@ -473,7 +554,7 @@ thread_yield_internal(
 
        disable_preemption();
        myprocessor = current_processor();
 
        disable_preemption();
        myprocessor = current_processor();

-       if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
+       if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {

                mp_enable_preemption();
 
                return;
                mp_enable_preemption();
 
                return;