]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/kern/sched_prim.h
projects / apple / xnu.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
xnu-6153.11.26.tar.gz
[apple/xnu.git] / osfmk / kern / sched_prim.h
diff --git a/osfmk/kern/sched_prim.h b/osfmk/kern/sched_prim.h
index c0014e86bfc61d37d0dab28a50ad5f4f730ee494..2f806bdd0ce1c01b4cc78a2431efa3bc47548f4c 100644 (file)
--- a/osfmk/kern/sched_prim.h
+++ b/osfmk/kern/sched_prim.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
-/* 
+/*
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
- * 
+ *
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
- * 
+ *
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
+ *
  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to
- * 
+ *
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
- * 
+ *
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
@@ -63,7 +63,7 @@
  *
  */
 
  *
  */
 
-#ifndef        _KERN_SCHED_PRIM_H_
+#ifndef _KERN_SCHED_PRIM_H_
 #define _KERN_SCHED_PRIM_H_
 
 #include <mach/boolean.h>
 #define _KERN_SCHED_PRIM_H_
 
 #include <mach/boolean.h>
@@ -75,222 +75,251 @@
 #include <sys/cdefs.h>
 #include <kern/block_hint.h>
 
 #include <sys/cdefs.h>
 #include <kern/block_hint.h>
 
-#ifdef MACH_KERNEL_PRIVATE
+#ifdef  MACH_KERNEL_PRIVATE
 
 
-#include <mach/branch_predicates.h>
+#include <kern/sched_urgency.h>
+#include <kern/thread_group.h>
 
 /* Initialization */
 
 /* Initialization */
-extern void            sched_init(void);
+extern void             sched_init(void);
 
 
-extern void            sched_startup(void);
+extern void             sched_startup(void);
 
 
-extern void            sched_timebase_init(void);
+extern void             sched_timebase_init(void);
 
 
-extern void            pset_rt_init(processor_set_t pset);
+extern void             pset_rt_init(processor_set_t pset);
 
 
-extern void            sched_rtglobal_init(processor_set_t pset);
+extern void             sched_rtglobal_init(processor_set_t pset);
 
 
-extern rt_queue_t      sched_rtglobal_runq(processor_set_t pset);
+extern rt_queue_t       sched_rtglobal_runq(processor_set_t pset);
 
 
-extern void            sched_rtglobal_queue_shutdown(processor_t processor);
+extern void             sched_rtglobal_queue_shutdown(processor_t processor);
 
 
-extern int64_t         sched_rtglobal_runq_count_sum(void);
+extern int64_t          sched_rtglobal_runq_count_sum(void);
 
 
-extern void            sched_check_spill(processor_set_t pset, thread_t thread);
+extern void             sched_check_spill(processor_set_t pset, thread_t thread);
 
 extern bool             sched_thread_should_yield(processor_t processor, thread_t thread);
 
 
 extern bool             sched_thread_should_yield(processor_t processor, thread_t thread);
 
+extern bool             sched_steal_thread_DISABLED(processor_set_t pset);
+extern bool             sched_steal_thread_enabled(processor_set_t pset);
+
 /* Force a preemption point for a thread and wait for it to stop running */
 /* Force a preemption point for a thread and wait for it to stop running */
-extern boolean_t       thread_stop( 
-                                               thread_t        thread,
-                                               boolean_t       until_not_runnable);
+extern boolean_t        thread_stop(
+       thread_t        thread,
+       boolean_t       until_not_runnable);
 
 /* Release a previous stop request */
 
 /* Release a previous stop request */
-extern void                    thread_unstop(
-                                               thread_t        thread);
+extern void                     thread_unstop(
+       thread_t        thread);
 
 /* Wait for a thread to stop running */
 
 /* Wait for a thread to stop running */
-extern void                    thread_wait(
-                                               thread_t        thread,
-                                               boolean_t       until_not_runnable);
+extern void                     thread_wait(
+       thread_t        thread,
+       boolean_t       until_not_runnable);
 
 /* Unblock thread on wake up */
 
 /* Unblock thread on wake up */
-extern boolean_t       thread_unblock(
-                                               thread_t                thread,
-                                               wait_result_t   wresult);
+extern boolean_t        thread_unblock(
+       thread_t                thread,
+       wait_result_t   wresult);
 
 /* Unblock and dispatch thread */
 
 /* Unblock and dispatch thread */
-extern kern_return_t   thread_go(
-                                                       thread_t                thread,
-                                                       wait_result_t   wresult);
+extern kern_return_t    thread_go(
+       thread_t                thread,
+       wait_result_t   wresult);
 
 /* Handle threads at context switch */
 
 /* Handle threads at context switch */
-extern void                    thread_dispatch(
-                                               thread_t                old_thread,
-                                               thread_t                new_thread);
+extern void                     thread_dispatch(
+       thread_t                old_thread,
+       thread_t                new_thread);
 
 /* Switch directly to a particular thread */
 
 /* Switch directly to a particular thread */
-extern int                     thread_run(
-                                               thread_t                        self,
-                                               thread_continue_t       continuation,
-                                               void                            *parameter,
-                                               thread_t                        new_thread);
+extern int                      thread_run(
+       thread_t                        self,
+       thread_continue_t       continuation,
+       void                            *parameter,
+       thread_t                        new_thread);
 
 /* Resume thread with new stack */
 
 /* Resume thread with new stack */
-extern void                    thread_continue(
-                                               thread_t                old_thread);
+extern __dead2 void     thread_continue(thread_t old_thread);
 
 /* Invoke continuation */
 
 /* Invoke continuation */
-extern void            call_continuation(
-                                       thread_continue_t       continuation,
-                                       void                            *parameter,
-                                       wait_result_t           wresult);
+extern __dead2 void     call_continuation(
+       thread_continue_t       continuation,
+       void                    *parameter,
+       wait_result_t           wresult,
+       boolean_t               enable_interrupts);
+
+/*
+ * Flags that can be passed to set_sched_pri
+ * to skip side effects
+ */
+__options_decl(set_sched_pri_options_t, uint32_t, {
+       SETPRI_DEFAULT  = 0x0,
+       SETPRI_LAZY     = 0x1,  /* Avoid setting AST flags or sending IPIs */
+});
 
 /* Set the current scheduled priority */
 
 /* Set the current scheduled priority */
-extern void            set_sched_pri(
-                                       thread_t                thread,
-                                       int                             priority);
+extern void set_sched_pri(
+       thread_t      thread,
+       int           priority,
+       set_sched_pri_options_t options);
 
 /* Set base priority of the specified thread */
 
 /* Set base priority of the specified thread */
-extern void            sched_set_thread_base_priority(
-                                       thread_t                thread,
-                                       int                             priority);
+extern void             sched_set_thread_base_priority(
+       thread_t                thread,
+       int                             priority);
+
+/* Set absolute base priority of the specified thread */
+extern void             sched_set_kernel_thread_priority(
+       thread_t                thread,
+       int                             priority);
+
 
 /* Set the thread's true scheduling mode */
 extern void             sched_set_thread_mode(thread_t thread,
 
 /* Set the thread's true scheduling mode */
 extern void             sched_set_thread_mode(thread_t thread,
-                                              sched_mode_t mode);
+    sched_mode_t mode);
 /* Demote the true scheduler mode */
 extern void             sched_thread_mode_demote(thread_t thread,
 /* Demote the true scheduler mode */
 extern void             sched_thread_mode_demote(thread_t thread,
-                                                 uint32_t reason);
+    uint32_t reason);
 /* Un-demote the true scheduler mode */
 extern void             sched_thread_mode_undemote(thread_t thread,
 /* Un-demote the true scheduler mode */
 extern void             sched_thread_mode_undemote(thread_t thread,
-                                                   uint32_t reason);
+    uint32_t reason);
+
+extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj);
+extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj);
 
 /* Re-evaluate base priority of thread (thread locked) */
 void thread_recompute_priority(thread_t thread);
 
 
 /* Re-evaluate base priority of thread (thread locked) */
 void thread_recompute_priority(thread_t thread);
 
-/* Re-evaluate base priority of thread (thread unlocked) */
-void thread_recompute_qos(thread_t thread);
-
-/* Reset scheduled priority of thread */
-extern void            thread_recompute_sched_pri(
-                                       thread_t                thread,
-                                       boolean_t               override_depress);
+/* Re-evaluate scheduled priority of thread (thread locked) */
+extern void thread_recompute_sched_pri(
+       thread_t thread,
+       set_sched_pri_options_t options);
 
 /* Periodic scheduler activity */
 
 /* Periodic scheduler activity */
-extern void            sched_init_thread(void (*)(void));
+extern void             sched_init_thread(void (*)(void));
 
 /* Perform sched_tick housekeeping activities */
 
 /* Perform sched_tick housekeeping activities */
-extern boolean_t               can_update_priority(
-                                       thread_t                thread);
+extern boolean_t                can_update_priority(
+       thread_t                thread);
 
 
-extern void            update_priority(
-                                                                                       thread_t                thread);
+extern void             update_priority(
+       thread_t                thread);
 
 
-extern void            lightweight_update_priority(
-                                                               thread_t                thread);
+extern void             lightweight_update_priority(
+       thread_t                thread);
 
 extern void             sched_default_quantum_expire(thread_t thread);
 
 
 extern void             sched_default_quantum_expire(thread_t thread);
 
-/* Idle processor thread */
-extern void            idle_thread(void);
+/* Idle processor thread continuation */
+extern void             idle_thread(
+       void*           parameter,
+       wait_result_t   result);
 
 
-extern kern_return_t   idle_thread_create(
-                                                       processor_t             processor);
+extern kern_return_t    idle_thread_create(
+       processor_t             processor);
 
 /* Continuation return from syscall */
 extern void     thread_syscall_return(
 
 /* Continuation return from syscall */
 extern void     thread_syscall_return(
-                        kern_return_t   ret);
+       kern_return_t   ret);
 
 /* Context switch */
 
 /* Context switch */
-extern wait_result_t   thread_block_reason(
-                                                       thread_continue_t       continuation,
-                                                       void                            *parameter,
-                                                       ast_t                           reason);
+extern wait_result_t    thread_block_reason(
+       thread_continue_t       continuation,
+       void                            *parameter,
+       ast_t                           reason);
 
 
-/* Reschedule thread for execution */
-extern void            thread_setrun(
-                                       thread_t        thread,
-                                       integer_t       options);
-
-typedef enum {
+__options_decl(sched_options_t, uint32_t, {
        SCHED_NONE      = 0x0,
        SCHED_TAILQ     = 0x1,
        SCHED_HEADQ     = 0x2,
        SCHED_PREEMPT   = 0x4,
        SCHED_REBALANCE = 0x8,
        SCHED_NONE      = 0x0,
        SCHED_TAILQ     = 0x1,
        SCHED_HEADQ     = 0x2,
        SCHED_PREEMPT   = 0x4,
        SCHED_REBALANCE = 0x8,
-} sched_options_t;
+});
 
 
-extern processor_set_t task_choose_pset(
-                                                       task_t                  task);
+/* Reschedule thread for execution */
+extern void             thread_setrun(
+       thread_t        thread,
+       sched_options_t options);
+
+extern processor_set_t  task_choose_pset(
+       task_t                  task);
 
 /* Bind the current thread to a particular processor */
 
 /* Bind the current thread to a particular processor */
-extern processor_t             thread_bind(
-                                                       processor_t             processor);
+extern processor_t              thread_bind(
+       processor_t             processor);
 
 /* Choose the best processor to run a thread */
 
 /* Choose the best processor to run a thread */
-extern processor_t     choose_processor(
-                                                                        processor_set_t                pset,
-                                                                        processor_t                    processor,
-                                                                        thread_t                       thread);
+extern processor_t      choose_processor(
+       processor_set_t                pset,
+       processor_t                    processor,
+       thread_t                       thread);
 
 extern void sched_SMT_balance(
 
 extern void sched_SMT_balance(
-                             processor_t processor,
-                             processor_set_t pset);
+       processor_t processor,
+       processor_set_t pset);
 
 extern void thread_quantum_init(
 
 extern void thread_quantum_init(
-                                                               thread_t thread);
-
-extern void            run_queue_init(
-                                       run_queue_t             runq);
-
-extern thread_t        run_queue_dequeue(
-                                                         run_queue_t           runq,
-                                                         integer_t             options);
-
-extern boolean_t       run_queue_enqueue(
-                                                         run_queue_t           runq,
-                                                         thread_t                      thread,
-                                                         integer_t             options);
-
-extern void    run_queue_remove(
-                                                                        run_queue_t            runq,
-                                                                        thread_t                       thread);
-                                                                         
-struct sched_update_scan_context
-{
-       uint64_t        earliest_bg_make_runnable_time;
-       uint64_t        earliest_normal_make_runnable_time;
-       uint64_t        earliest_rt_make_runnable_time;
+       thread_t thread);
+
+extern void             run_queue_init(
+       run_queue_t             runq);
+
+extern thread_t run_queue_dequeue(
+       run_queue_t           runq,
+       sched_options_t       options);
+
+extern boolean_t        run_queue_enqueue(
+       run_queue_t           runq,
+       thread_t              thread,
+       sched_options_t       options);
+
+extern void     run_queue_remove(
+       run_queue_t            runq,
+       thread_t                       thread);
+
+extern thread_t run_queue_peek(
+       run_queue_t            runq);
+
+struct sched_update_scan_context {
+       uint64_t        earliest_bg_make_runnable_time;
+       uint64_t        earliest_normal_make_runnable_time;
+       uint64_t        earliest_rt_make_runnable_time;
 };
 typedef struct sched_update_scan_context *sched_update_scan_context_t;
 
 };
 typedef struct sched_update_scan_context *sched_update_scan_context_t;
 
-extern void            sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context);
+extern void             sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context);
 
 
-/* 
- * Enum to define various events which need IPIs. The IPI policy 
- * engine decides what kind of IPI to use based on destination 
+extern void sched_pset_made_schedulable(
+       processor_t processor,
+       processor_set_t pset,
+       boolean_t drop_lock);
+
+/*
+ * Enum to define various events which need IPIs. The IPI policy
+ * engine decides what kind of IPI to use based on destination
  * processor state, thread and one of the following scheduling events.
  */
 typedef enum {
        SCHED_IPI_EVENT_BOUND_THR   = 0x1,
  * processor state, thread and one of the following scheduling events.
  */
 typedef enum {
        SCHED_IPI_EVENT_BOUND_THR   = 0x1,
-       SCHED_IPI_EVENT_PREEMPT     = 0x2,
+       SCHED_IPI_EVENT_PREEMPT     = 0x2,
        SCHED_IPI_EVENT_SMT_REBAL   = 0x3,
        SCHED_IPI_EVENT_SMT_REBAL   = 0x3,
-       SCHED_IPI_EVENT_SPILL       = 0x4,
+       SCHED_IPI_EVENT_SPILL       = 0x4,
        SCHED_IPI_EVENT_REBALANCE   = 0x5,
 } sched_ipi_event_t;
 
 
 /* Enum to define various IPI types used by the scheduler */
 typedef enum {
        SCHED_IPI_EVENT_REBALANCE   = 0x5,
 } sched_ipi_event_t;
 
 
 /* Enum to define various IPI types used by the scheduler */
 typedef enum {
-       SCHED_IPI_NONE              = 0x0,
-       SCHED_IPI_IMMEDIATE         = 0x1,
-       SCHED_IPI_IDLE              = 0x2,
-       SCHED_IPI_DEFERRED          = 0x3,
+       SCHED_IPI_NONE              = 0x0,
+       SCHED_IPI_IMMEDIATE         = 0x1,
+       SCHED_IPI_IDLE              = 0x2,
+       SCHED_IPI_DEFERRED          = 0x3,
 } sched_ipi_type_t;
 
 /* The IPI policy engine behaves in the following manner:
 } sched_ipi_type_t;
 
 /* The IPI policy engine behaves in the following manner:
- * - All scheduler events which need an IPI invoke sched_ipi_action() with  
+ * - All scheduler events which need an IPI invoke sched_ipi_action() with
  *   the appropriate destination processor, thread and event.
  * - sched_ipi_action() performs basic checks, invokes the scheduler specific
  *   ipi_policy routine and sets pending_AST bits based on the result.
  *   the appropriate destination processor, thread and event.
  * - sched_ipi_action() performs basic checks, invokes the scheduler specific
  *   ipi_policy routine and sets pending_AST bits based on the result.
@@ -298,16 +327,16 @@ typedef enum {
  *   routine which actually sends the appropriate IPI to the destination core.
  */
 extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread,
  *   routine which actually sends the appropriate IPI to the destination core.
  */
 extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread,
-         boolean_t dst_idle, sched_ipi_event_t event);
+    boolean_t dst_idle, sched_ipi_event_t event);
 extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi);
 
 /* sched_ipi_policy() is the global default IPI policy for all schedulers */
 extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread,
 extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi);
 
 /* sched_ipi_policy() is the global default IPI policy for all schedulers */
 extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread,
-         boolean_t dst_idle, sched_ipi_event_t event);
+    boolean_t dst_idle, sched_ipi_event_t event);
 
 /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */
 extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset,
 
 /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */
 extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset,
-         processor_t dst, sched_ipi_event_t event);
+    processor_t dst, sched_ipi_event_t event);
 
 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
 
 
 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
 
@@ -327,63 +356,63 @@ extern int sched_compute_timeshare_priority(thread_t thread);
 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
 
 /* Remove thread from its run queue */
 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
 
 /* Remove thread from its run queue */
-extern boolean_t       thread_run_queue_remove(thread_t thread);
+extern boolean_t        thread_run_queue_remove(thread_t thread);
 thread_t thread_run_queue_remove_for_handoff(thread_t thread);
 
 /* Put a thread back in the run queue after being yanked */
 thread_t thread_run_queue_remove_for_handoff(thread_t thread);
 
 /* Put a thread back in the run queue after being yanked */
-extern void thread_run_queue_reinsert(thread_t thread, integer_t options);
+extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options);
 
 
-extern void            thread_timer_expire(
-                                       void                    *thread,
-                                       void                    *p1);
+extern void             thread_timer_expire(
+       void                    *thread,
+       void                    *p1);
 
 
-extern boolean_t       thread_eager_preemption(
-                                               thread_t thread);
+extern boolean_t        thread_eager_preemption(
+       thread_t thread);
 
 extern boolean_t sched_generic_direct_dispatch_to_idle_processors;
 
 /* Set the maximum interrupt level for the thread */
 __private_extern__ wait_interrupt_t thread_interrupt_level(
 
 extern boolean_t sched_generic_direct_dispatch_to_idle_processors;
 
 /* Set the maximum interrupt level for the thread */
 __private_extern__ wait_interrupt_t thread_interrupt_level(
-                                               wait_interrupt_t interruptible);
+       wait_interrupt_t interruptible);
 
 __private_extern__ wait_result_t thread_mark_wait_locked(
 
 __private_extern__ wait_result_t thread_mark_wait_locked(
-                                               thread_t                 thread,
-                                               wait_interrupt_t interruptible);
+       thread_t                 thread,
+       wait_interrupt_t interruptible);
 
 /* Wake up locked thread directly, passing result */
 __private_extern__ kern_return_t clear_wait_internal(
 
 /* Wake up locked thread directly, passing result */
 __private_extern__ kern_return_t clear_wait_internal(
-                                               thread_t                thread,
-                                               wait_result_t   result);
+       thread_t                thread,
+       wait_result_t   result);
 
 extern void sched_stats_handle_csw(
 
 extern void sched_stats_handle_csw(
-                                                       processor_t processor, 
-                                                       int reasons, 
-                                                       int selfpri, 
-                                                       int otherpri);
+       processor_t processor,
+       int reasons,
+       int selfpri,
+       int otherpri);
 
 extern void sched_stats_handle_runq_change(
 
 extern void sched_stats_handle_runq_change(
-                                                                       struct runq_stats *stats, 
-                                                                       int old_count);
+       struct runq_stats *stats,
+       int old_count);
 
 
 #if DEBUG
 
 
 
 #if DEBUG
 
-#define        SCHED_STATS_CSW(processor, reasons, selfpri, otherpri)          \
-do {                                                           \
-       if (__builtin_expect(sched_stats_active, 0)) {  \
-               sched_stats_handle_csw((processor),             \
-                               (reasons), (selfpri), (otherpri));      \
-       }                                                       \
-} while (0) 
+#define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri)          \
+do {                                                            \
+       if (__builtin_expect(sched_stats_active, 0)) {  \
+               sched_stats_handle_csw((processor),             \
+                               (reasons), (selfpri), (otherpri));      \
+       }                                                       \
+} while (0)
 
 
 
 
-#define SCHED_STATS_RUNQ_CHANGE(stats, old_count)              \
-do {                                                           \
-       if (__builtin_expect(sched_stats_active, 0)) {  \
-               sched_stats_handle_runq_change((stats),         \
-                                                               (old_count));           \
-       }                                                       \
-} while (0) 
+#define SCHED_STATS_RUNQ_CHANGE(stats, old_count)               \
+do {                                                            \
+       if (__builtin_expect(sched_stats_active, 0)) {  \
+               sched_stats_handle_runq_change((stats),         \
+                                                               (old_count));           \
+       }                                                       \
+} while (0)
 
 #else /* DEBUG */
 
 
 #else /* DEBUG */
 
@@ -393,53 +422,28 @@ do {                                                              \
 #endif /* DEBUG */
 
 extern uint32_t sched_debug_flags;
 #endif /* DEBUG */
 
 extern uint32_t sched_debug_flags;
-#define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS  0x00000001
-#define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS  0x00000002
+#define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS   0x00000001
+#define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS   0x00000002
 
 
-#define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) do {                                           \
-               if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \
-                       KERNEL_DEBUG_CONSTANT(__VA_ARGS__);                                                     \
-               }                                                                                                                               \
+#define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) do {                                            \
+               if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \
+                       KERNEL_DEBUG_CONSTANT(__VA_ARGS__);                                                     \
+               }                                                                                                                               \
        } while(0)
 
        } while(0)
 
-#define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) do {                                           \
-               if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \
-                       KERNEL_DEBUG_CONSTANT(__VA_ARGS__);                                                     \
-               }                                                                                                                               \
+#define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) do {                                            \
+               if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \
+                       KERNEL_DEBUG_CONSTANT(__VA_ARGS__);                                                     \
+               }                                                                                                                               \
        } while(0)
 
        } while(0)
 
-#define THREAD_URGENCY_NONE            0       /* indicates that there is no currently runnable */
-#define THREAD_URGENCY_BACKGROUND      1       /* indicates that the thread is marked as a "background" thread */
-#define THREAD_URGENCY_NORMAL          2       /* indicates that the thread is marked as a "normal" thread */
-#define THREAD_URGENCY_REAL_TIME       3       /* indicates that the thread is marked as a "real-time" or urgent thread */
-#define        THREAD_URGENCY_MAX              4       /* Marker */
-/* Returns the "urgency" of a thread (provided by scheduler) */
-extern int     thread_get_urgency(
-                                       thread_t        thread,
-                                       uint64_t        *rt_period,
-                                       uint64_t        *rt_deadline);
-
-/* Tells the "urgency" of the just scheduled thread (provided by CPU PM) */
-extern void    thread_tell_urgency(
-                                       int             urgency,
-                                       uint64_t        rt_period,
-                                       uint64_t        rt_deadline,
-                                       uint64_t        sched_latency,
-                                   thread_t nthread);
-
 /* Tells if there are "active" RT threads in the system (provided by CPU PM) */
 /* Tells if there are "active" RT threads in the system (provided by CPU PM) */
-extern void    active_rt_threads(
-                                       boolean_t       active);
+extern void     active_rt_threads(
+       boolean_t       active);
 
 /* Returns the perfcontrol attribute for the thread */
 extern perfcontrol_class_t thread_get_perfcontrol_class(
 
 /* Returns the perfcontrol attribute for the thread */
 extern perfcontrol_class_t thread_get_perfcontrol_class(
-                                       thread_t        thread);
-
-#define PSET_LOAD_NUMERATOR_SHIFT   16
-#define PSET_LOAD_FRACTIONAL_SHIFT   4
-
-extern int sched_get_pset_load_average(processor_set_t pset);
-extern void sched_update_pset_load_average(processor_set_t pset);
+       thread_t        thread);
 
 /* Generic routine for Non-AMP schedulers to calculate parallelism */
 extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options);
 
 /* Generic routine for Non-AMP schedulers to calculate parallelism */
 extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options);
@@ -448,113 +452,118 @@ extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options);
 
 __BEGIN_DECLS
 
 
 __BEGIN_DECLS
 
-#ifdef XNU_KERNEL_PRIVATE
+#ifdef  XNU_KERNEL_PRIVATE
 
 /* Toggles a global override to turn off CPU Throttling */
 
 /* Toggles a global override to turn off CPU Throttling */
-#define CPU_THROTTLE_DISABLE   0
-#define CPU_THROTTLE_ENABLE    1
-extern void    sys_override_cpu_throttle(int flag);
+extern void     sys_override_cpu_throttle(boolean_t enable_override);
 
 /*
  ****************** Only exported until BSD stops using ********************
  */
 
 
 /*
  ****************** Only exported until BSD stops using ********************
  */
 
-extern void                    thread_vm_bind_group_add(void);
+extern void                     thread_vm_bind_group_add(void);
 
 /* Wake up thread directly, passing result */
 extern kern_return_t clear_wait(
 
 /* Wake up thread directly, passing result */
 extern kern_return_t clear_wait(
-                                               thread_t                thread,
-                                               wait_result_t   result);
+       thread_t                thread,
+       wait_result_t   result);
 
 /* Start thread running */
 
 /* Start thread running */
-extern void            thread_bootstrap_return(void);
+extern void             thread_bootstrap_return(void) __attribute__((noreturn));
 
 /* Return from exception (BSD-visible interface) */
 
 /* Return from exception (BSD-visible interface) */
-extern void            thread_exception_return(void) __dead2;
+extern void             thread_exception_return(void) __dead2;
 
 #define SCHED_STRING_MAX_LENGTH (48)
 /* String declaring the name of the current scheduler */
 extern char sched_string[SCHED_STRING_MAX_LENGTH];
 
 
 #define SCHED_STRING_MAX_LENGTH (48)
 /* String declaring the name of the current scheduler */
 extern char sched_string[SCHED_STRING_MAX_LENGTH];
 
-extern thread_t port_name_to_thread_for_ulock(mach_port_name_t thread_name);
-
 /* Attempt to context switch to a specific runnable thread */
 /* Attempt to context switch to a specific runnable thread */
-extern wait_result_t thread_handoff(thread_t thread);
+extern wait_result_t thread_handoff_deallocate(thread_t thread);
+
+__attribute__((nonnull(1, 2)))
+extern void thread_handoff_parameter(thread_t thread,
+    thread_continue_t continuation, void *parameter) __dead2;
 
 
-extern struct waitq    *assert_wait_queue(event_t event);
+extern struct waitq     *assert_wait_queue(event_t event);
 
 extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority);
 
 extern thread_t thread_wakeup_identify(event_t event, int priority);
 
 
 extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority);
 
 extern thread_t thread_wakeup_identify(event_t event, int priority);
 
-#endif /* XNU_KERNEL_PRIVATE */
+#endif  /* XNU_KERNEL_PRIVATE */
 
 #ifdef KERNEL_PRIVATE
 /* Set pending block hint for a particular object before we go into a wait state */
 
 #ifdef KERNEL_PRIVATE
 /* Set pending block hint for a particular object before we go into a wait state */
-extern void            thread_set_pending_block_hint(
-                                                       thread_t                        thread,
-                                                       block_hint_t                    block_hint);
+extern void             thread_set_pending_block_hint(
+       thread_t                        thread,
+       block_hint_t                    block_hint);
 
 #define QOS_PARALLELISM_COUNT_LOGICAL   0x1
 #define QOS_PARALLELISM_REALTIME        0x2
 extern uint32_t qos_max_parallelism(int qos, uint64_t options);
 
 #define QOS_PARALLELISM_COUNT_LOGICAL   0x1
 #define QOS_PARALLELISM_REALTIME        0x2
 extern uint32_t qos_max_parallelism(int qos, uint64_t options);
-
 #endif /* KERNEL_PRIVATE */
 
 #endif /* KERNEL_PRIVATE */
 
+#if XNU_KERNEL_PRIVATE
+extern void             thread_yield_with_continuation(
+       thread_continue_t       continuation,
+       void                            *parameter) __dead2;
+#endif
+
 /* Context switch */
 /* Context switch */
-extern wait_result_t   thread_block(
-                                                       thread_continue_t       continuation);
+extern wait_result_t    thread_block(
+       thread_continue_t       continuation);
 
 
-extern wait_result_t   thread_block_parameter(
-                                                       thread_continue_t       continuation,
-                                                       void                            *parameter);
+extern wait_result_t    thread_block_parameter(
+       thread_continue_t       continuation,
+       void                            *parameter);
 
 /* Declare thread will wait on a particular event */
 
 /* Declare thread will wait on a particular event */
-extern wait_result_t   assert_wait(
-                                                       event_t                         event,
-                                                       wait_interrupt_t        interruptible);
+extern wait_result_t    assert_wait(
+       event_t                         event,
+       wait_interrupt_t        interruptible);
 
 /* Assert that the thread intends to wait with a timeout */
 
 /* Assert that the thread intends to wait with a timeout */
-extern wait_result_t   assert_wait_timeout(
-                                                       event_t                         event,
-                                                       wait_interrupt_t        interruptible,
-                                                       uint32_t                        interval,
-                                                       uint32_t                        scale_factor);
+extern wait_result_t    assert_wait_timeout(
+       event_t                         event,
+       wait_interrupt_t        interruptible,
+       uint32_t                        interval,
+       uint32_t                        scale_factor);
 
 /* Assert that the thread intends to wait with an urgency, timeout and leeway */
 
 /* Assert that the thread intends to wait with an urgency, timeout and leeway */
-extern wait_result_t   assert_wait_timeout_with_leeway(
-                                                       event_t                         event,
-                                                       wait_interrupt_t        interruptible,
-                                                       wait_timeout_urgency_t  urgency,
-                                                       uint32_t                        interval,
-                                                       uint32_t                        leeway,
-                                                       uint32_t                        scale_factor);
-
-extern wait_result_t   assert_wait_deadline(
-                                                       event_t                         event,
-                                                       wait_interrupt_t        interruptible,
-                                                       uint64_t                        deadline);
+extern wait_result_t    assert_wait_timeout_with_leeway(
+       event_t                         event,
+       wait_interrupt_t        interruptible,
+       wait_timeout_urgency_t  urgency,
+       uint32_t                        interval,
+       uint32_t                        leeway,
+       uint32_t                        scale_factor);
+
+extern wait_result_t    assert_wait_deadline(
+       event_t                         event,
+       wait_interrupt_t        interruptible,
+       uint64_t                        deadline);
 
 /* Assert that the thread intends to wait with an urgency, deadline, and leeway */
 
 /* Assert that the thread intends to wait with an urgency, deadline, and leeway */
-extern wait_result_t   assert_wait_deadline_with_leeway(
-                                                       event_t                         event,
-                                                       wait_interrupt_t        interruptible,
-                                                       wait_timeout_urgency_t  urgency,
-                                                       uint64_t                        deadline,
-                                                       uint64_t                        leeway);
+extern wait_result_t    assert_wait_deadline_with_leeway(
+       event_t                         event,
+       wait_interrupt_t        interruptible,
+       wait_timeout_urgency_t  urgency,
+       uint64_t                        deadline,
+       uint64_t                        leeway);
 
 /* Wake up thread (or threads) waiting on a particular event */
 
 /* Wake up thread (or threads) waiting on a particular event */
-extern kern_return_t   thread_wakeup_prim(
-                                                       event_t                         event,
-                                                       boolean_t                       one_thread,
-                                                       wait_result_t                   result);
-
-#define thread_wakeup(x)                                       \
-                       thread_wakeup_prim((x), FALSE, THREAD_AWAKENED)
-#define thread_wakeup_with_result(x, z)                \
-                       thread_wakeup_prim((x), FALSE, (z))
-#define thread_wakeup_one(x)                           \
-                       thread_wakeup_prim((x), TRUE, THREAD_AWAKENED)
+extern kern_return_t    thread_wakeup_prim(
+       event_t                         event,
+       boolean_t                       one_thread,
+       wait_result_t                   result);
+
+#define thread_wakeup(x)                                        \
+                       thread_wakeup_prim((x), FALSE, THREAD_AWAKENED)
+#define thread_wakeup_with_result(x, z)         \
+                       thread_wakeup_prim((x), FALSE, (z))
+#define thread_wakeup_one(x)                            \
+                       thread_wakeup_prim((x), TRUE, THREAD_AWAKENED)
 
 /* Wakeup the specified thread if it is waiting on this event */
 extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread);
 
 /* Wakeup the specified thread if it is waiting on this event */
 extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread);
@@ -570,50 +579,47 @@ extern boolean_t preemption_enabled(void);
  * a function pointer table.
  */
 
  * a function pointer table.
  */
 
-#if   !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ)
+#if   !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) && !defined(CONFIG_SCHED_CLUTCH)
 #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX
 #endif
 
 #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX
 #endif
 
-#if DEBUG
-#define SCHED(f) (sched_current_dispatch->f)
-#else /* DEBUG */
 
 
-/* 
- * For DEV & REL kernels, use a static dispatch table instead of 
- * using the indirect function table.
- */
-extern const struct sched_dispatch_table sched_multiq_dispatch;
-#define SCHED(f) (sched_multiq_dispatch.f)
+#if CONFIG_SCHED_CLUTCH
+extern const struct sched_dispatch_table sched_clutch_dispatch;
+#define SCHED(f) (sched_clutch_dispatch.f)
+#else /* CONFIG_SCHED_CLUTCH */
+extern const struct sched_dispatch_table sched_dualq_dispatch;
+#define SCHED(f) (sched_dualq_dispatch.f)
+#endif /* CONFIG_SCHED_CLUTCH */
 
 
-#endif /* DEBUG */
 
 struct sched_dispatch_table {
        const char *sched_name;
 
 struct sched_dispatch_table {
        const char *sched_name;
-       void    (*init)(void);                          /* Init global state */
-       void    (*timebase_init)(void);         /* Timebase-dependent initialization */
-       void    (*processor_init)(processor_t processor);       /* Per-processor scheduler init */
-       void    (*pset_init)(processor_set_t pset);     /* Per-processor set scheduler init */
+       void    (*init)(void);                          /* Init global state */
+       void    (*timebase_init)(void);         /* Timebase-dependent initialization */
+       void    (*processor_init)(processor_t processor);       /* Per-processor scheduler init */
+       void    (*pset_init)(processor_set_t pset);     /* Per-processor set scheduler init */
 
 
-       void    (*maintenance_continuation)(void);      /* Function called regularly */
+       void    (*maintenance_continuation)(void);      /* Function called regularly */
 
        /*
         * Choose a thread of greater or equal priority from the per-processor
         * runqueue for timeshare/fixed threads
         */
 
        /*
         * Choose a thread of greater or equal priority from the per-processor
         * runqueue for timeshare/fixed threads
         */
-       thread_t        (*choose_thread)(
-                                                                 processor_t           processor,
-                                                                 int                           priority,
-                                                                 ast_t reason);
+       thread_t        (*choose_thread)(
+               processor_t           processor,
+               int                           priority,
+               ast_t reason);
 
 
-       /* True if scheduler supports stealing threads */
-       boolean_t   steal_thread_enabled;
+       /* True if scheduler supports stealing threads for this pset */
+       bool    (*steal_thread_enabled)(processor_set_t pset);
 
        /*
         * Steal a thread from another processor in the pset so that it can run
         * immediately
         */
 
        /*
         * Steal a thread from another processor in the pset so that it can run
         * immediately
         */
-       thread_t        (*steal_thread)(
-                                                               processor_set_t         pset);
+       thread_t        (*steal_thread)(
+               processor_set_t         pset);
 
        /*
         * Compute priority for a timeshare thread based on base priority.
 
        /*
         * Compute priority for a timeshare thread based on base priority.
@@ -623,100 +629,93 @@ struct sched_dispatch_table {
        /*
         * Pick the best processor for a thread (any kind of thread) to run on.
         */
        /*
         * Pick the best processor for a thread (any kind of thread) to run on.
         */
-       processor_t     (*choose_processor)(
-                                                                                processor_set_t                pset,
-                                                                                processor_t                    processor,
-                                                                                thread_t                       thread);
+       processor_t     (*choose_processor)(
+               processor_set_t                pset,
+               processor_t                    processor,
+               thread_t                       thread);
        /*
         * Enqueue a timeshare or fixed priority thread onto the per-processor
         * runqueue
         */
        boolean_t (*processor_enqueue)(
        /*
         * Enqueue a timeshare or fixed priority thread onto the per-processor
         * runqueue
         */
        boolean_t (*processor_enqueue)(
-                                                                processor_t                    processor,
-                                                                thread_t                       thread,
-                                                                integer_t                      options);
+               processor_t                    processor,
+               thread_t                       thread,
+               sched_options_t                options);
 
        /* Migrate threads away in preparation for processor shutdown */
        void (*processor_queue_shutdown)(
 
        /* Migrate threads away in preparation for processor shutdown */
        void (*processor_queue_shutdown)(
-                                                                        processor_t                    processor);
+               processor_t                    processor);
 
        /* Remove the specific thread from the per-processor runqueue */
 
        /* Remove the specific thread from the per-processor runqueue */
-       boolean_t       (*processor_queue_remove)(
-                                                                       processor_t             processor,
-                                                                       thread_t                thread);
+       boolean_t       (*processor_queue_remove)(
+               processor_t             processor,
+               thread_t                thread);
 
        /*
         * Does the per-processor runqueue have any timeshare or fixed priority
         * threads on it? Called without pset lock held, so should
         * not assume immutability while executing.
         */
 
        /*
         * Does the per-processor runqueue have any timeshare or fixed priority
         * threads on it? Called without pset lock held, so should
         * not assume immutability while executing.
         */
-       boolean_t       (*processor_queue_empty)(processor_t            processor);
+       boolean_t       (*processor_queue_empty)(processor_t            processor);
 
        /*
         * Would this priority trigger an urgent preemption if it's sitting
         * on the per-processor runqueue?
         */
 
        /*
         * Would this priority trigger an urgent preemption if it's sitting
         * on the per-processor runqueue?
         */
-       boolean_t       (*priority_is_urgent)(int priority);
+       boolean_t       (*priority_is_urgent)(int priority);
 
        /*
         * Does the per-processor runqueue contain runnable threads that
         * should cause the currently-running thread to be preempted?
         */
 
        /*
         * Does the per-processor runqueue contain runnable threads that
         * should cause the currently-running thread to be preempted?
         */
-       ast_t           (*processor_csw_check)(processor_t processor);
+       ast_t           (*processor_csw_check)(processor_t processor);
 
        /*
         * Does the per-processor runqueue contain a runnable thread
         * of > or >= priority, as a preflight for choose_thread() or other
         * thread selection
         */
 
        /*
         * Does the per-processor runqueue contain a runnable thread
         * of > or >= priority, as a preflight for choose_thread() or other
         * thread selection
         */
-       boolean_t       (*processor_queue_has_priority)(processor_t             processor,
-                                                                                               int                             priority,
-                                                                                               boolean_t               gte);
+       boolean_t       (*processor_queue_has_priority)(processor_t             processor,
+           int                             priority,
+           boolean_t               gte);
 
        /* Quantum size for the specified non-realtime thread. */
 
        /* Quantum size for the specified non-realtime thread. */
-       uint32_t        (*initial_quantum_size)(thread_t thread);
-       
+       uint32_t        (*initial_quantum_size)(thread_t thread);
+
        /* Scheduler mode for a new thread */
        /* Scheduler mode for a new thread */
-       sched_mode_t    (*initial_thread_sched_mode)(task_t parent_task);
+       sched_mode_t    (*initial_thread_sched_mode)(task_t parent_task);
 
        /*
         * Is it safe to call update_priority, which may change a thread's
         * runqueue or other state. This can be used to throttle changes
         * to dynamic priority.
         */
 
        /*
         * Is it safe to call update_priority, which may change a thread's
         * runqueue or other state. This can be used to throttle changes
         * to dynamic priority.
         */
-       boolean_t       (*can_update_priority)(thread_t thread);
+       boolean_t       (*can_update_priority)(thread_t thread);
 
        /*
         * Update both scheduled priority and other persistent state.
         * Side effects may including migration to another processor's runqueue.
         */
 
        /*
         * Update both scheduled priority and other persistent state.
         * Side effects may including migration to another processor's runqueue.
         */
-       void            (*update_priority)(thread_t thread);
+       void            (*update_priority)(thread_t thread);
 
        /* Lower overhead update to scheduled priority and state. */
 
        /* Lower overhead update to scheduled priority and state. */
-       void            (*lightweight_update_priority)(thread_t thread);
+       void            (*lightweight_update_priority)(thread_t thread);
 
        /* Callback for non-realtime threads when the quantum timer fires */
 
        /* Callback for non-realtime threads when the quantum timer fires */
-       void            (*quantum_expire)(thread_t thread);
+       void            (*quantum_expire)(thread_t thread);
 
        /*
         * Runnable threads on per-processor runqueue. Should only
         * be used for relative comparisons of load between processors.
         */
 
        /*
         * Runnable threads on per-processor runqueue. Should only
         * be used for relative comparisons of load between processors.
         */
-       int                     (*processor_runq_count)(processor_t     processor);
+       int                     (*processor_runq_count)(processor_t     processor);
 
        /* Aggregate runcount statistics for per-processor runqueue */
        uint64_t    (*processor_runq_stats_count_sum)(processor_t   processor);
 
 
        /* Aggregate runcount statistics for per-processor runqueue */
        uint64_t    (*processor_runq_stats_count_sum)(processor_t   processor);
 
-       boolean_t       (*processor_bound_count)(processor_t processor);
-
-       void            (*thread_update_scan)(sched_update_scan_context_t scan_context);
+       boolean_t       (*processor_bound_count)(processor_t processor);
 
 
-       /*
-       * Use processor->next_thread to pin a thread to an idle
-       * processor. If FALSE, threads are enqueued and can
-       * be stolen by other processors.
-       */
-       boolean_t   direct_dispatch_to_idle_processors;
+       void            (*thread_update_scan)(sched_update_scan_context_t scan_context);
 
        /* Supports more than one pset */
        boolean_t   multiple_psets_enabled;
 
        /* Supports more than one pset */
        boolean_t   multiple_psets_enabled;
@@ -735,16 +734,26 @@ struct sched_dispatch_table {
         * Called with pset lock held, returns pset lock unlocked.
         */
        void    (*processor_balance)(processor_t processor, processor_set_t pset);
         * Called with pset lock held, returns pset lock unlocked.
         */
        void    (*processor_balance)(processor_t processor, processor_set_t pset);
-       rt_queue_t      (*rt_runq)(processor_set_t pset);
-       void    (*rt_init)(processor_set_t pset);
-       void    (*rt_queue_shutdown)(processor_t processor);
-       void    (*rt_runq_scan)(sched_update_scan_context_t scan_context);
-       int64_t (*rt_runq_count_sum)(void);
+       rt_queue_t      (*rt_runq)(processor_set_t pset);
+       void    (*rt_init)(processor_set_t pset);
+       void    (*rt_queue_shutdown)(processor_t processor);
+       void    (*rt_runq_scan)(sched_update_scan_context_t scan_context);
+       int64_t (*rt_runq_count_sum)(void);
 
        uint32_t (*qos_max_parallelism)(int qos, uint64_t options);
 
        uint32_t (*qos_max_parallelism)(int qos, uint64_t options);
-       void    (*check_spill)(processor_set_t pset, thread_t thread);
+       void    (*check_spill)(processor_set_t pset, thread_t thread);
        sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event);
        bool    (*thread_should_yield)(processor_t processor, thread_t thread);
        sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event);
        bool    (*thread_should_yield)(processor_t processor, thread_t thread);
+
+       /* Routine to update run counts */
+       uint32_t (*run_count_incr)(thread_t thread);
+       uint32_t (*run_count_decr)(thread_t thread);
+
+       /* Routine to update scheduling bucket for a thread */
+       void (*update_thread_bucket)(thread_t thread);
+
+       /* Routine to inform the scheduler when a new pset becomes schedulable */
+       void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock);
 };
 
 #if defined(CONFIG_SCHED_TRADITIONAL)
 };
 
 #if defined(CONFIG_SCHED_TRADITIONAL)
@@ -765,14 +774,12 @@ extern const struct sched_dispatch_table sched_proto_dispatch;
 extern const struct sched_dispatch_table sched_grrr_dispatch;
 #endif
 
 extern const struct sched_dispatch_table sched_grrr_dispatch;
 #endif
 
-/*
- * It is an error to invoke any scheduler-related code
- * before this is set up
- */
-extern const struct sched_dispatch_table *sched_current_dispatch;
+#if defined(CONFIG_SCHED_CLUTCH)
+extern const struct sched_dispatch_table sched_clutch_dispatch;
+#endif
 
 
-#endif /* MACH_KERNEL_PRIVATE */
+#endif  /* MACH_KERNEL_PRIVATE */
 
 __END_DECLS
 
 
 __END_DECLS
 
-#endif /* _KERN_SCHED_PRIM_H_ */
+#endif  /* _KERN_SCHED_PRIM_H_ */
mirror of XNU