X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5ba3f43ea354af8ad55bea84372a2bc834d8757c..cb3231590a3c94ab4375e2228bd5e86b0cf1ad7e:/osfmk/kern/sched_prim.h?ds=sidebyside diff --git a/osfmk/kern/sched_prim.h b/osfmk/kern/sched_prim.h index c0014e86b..2f806bdd0 100644 --- a/osfmk/kern/sched_prim.h +++ b/osfmk/kern/sched_prim.h @@ -2,7 +2,7 @@ * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -63,7 +63,7 @@ * */ -#ifndef _KERN_SCHED_PRIM_H_ +#ifndef _KERN_SCHED_PRIM_H_ #define _KERN_SCHED_PRIM_H_ #include @@ -75,222 +75,251 @@ #include #include -#ifdef MACH_KERNEL_PRIVATE +#ifdef MACH_KERNEL_PRIVATE -#include +#include +#include /* Initialization */ -extern void sched_init(void); +extern void sched_init(void); -extern void sched_startup(void); +extern void sched_startup(void); -extern void sched_timebase_init(void); +extern void sched_timebase_init(void); -extern void pset_rt_init(processor_set_t pset); +extern void pset_rt_init(processor_set_t pset); -extern void sched_rtglobal_init(processor_set_t pset); +extern void sched_rtglobal_init(processor_set_t pset); -extern rt_queue_t sched_rtglobal_runq(processor_set_t pset); +extern rt_queue_t sched_rtglobal_runq(processor_set_t pset); -extern void sched_rtglobal_queue_shutdown(processor_t processor); +extern void sched_rtglobal_queue_shutdown(processor_t processor); -extern int64_t sched_rtglobal_runq_count_sum(void); +extern int64_t sched_rtglobal_runq_count_sum(void); -extern void sched_check_spill(processor_set_t pset, thread_t thread); +extern void sched_check_spill(processor_set_t pset, thread_t thread); extern bool sched_thread_should_yield(processor_t processor, thread_t thread); +extern bool sched_steal_thread_DISABLED(processor_set_t pset); +extern bool sched_steal_thread_enabled(processor_set_t pset); + /* Force a preemption point for a thread and wait for it to stop running */ -extern boolean_t thread_stop( - thread_t thread, - boolean_t until_not_runnable); +extern boolean_t thread_stop( + thread_t thread, + boolean_t until_not_runnable); /* Release a previous stop request */ -extern void thread_unstop( - thread_t thread); +extern void thread_unstop( + thread_t thread); /* Wait for a thread to stop running */ -extern void thread_wait( - thread_t thread, - boolean_t until_not_runnable); +extern void thread_wait( + thread_t thread, + boolean_t until_not_runnable); /* Unblock thread on wake up */ -extern boolean_t thread_unblock( - thread_t thread, - wait_result_t wresult); +extern boolean_t thread_unblock( + thread_t thread, + wait_result_t wresult); /* Unblock and dispatch thread */ -extern kern_return_t thread_go( - thread_t thread, - wait_result_t wresult); +extern kern_return_t thread_go( + thread_t thread, + wait_result_t wresult); /* Handle threads at context switch */ -extern void thread_dispatch( - thread_t old_thread, - thread_t new_thread); +extern void thread_dispatch( + thread_t old_thread, + thread_t new_thread); /* Switch directly to a particular thread */ -extern int thread_run( - thread_t self, - thread_continue_t continuation, - void *parameter, - thread_t new_thread); +extern int thread_run( + thread_t self, + thread_continue_t continuation, + void *parameter, + thread_t new_thread); /* Resume thread with new stack */ -extern void thread_continue( - thread_t old_thread); +extern __dead2 void thread_continue(thread_t old_thread); /* Invoke continuation */ -extern void call_continuation( - thread_continue_t continuation, - void *parameter, - wait_result_t wresult); +extern __dead2 void call_continuation( + thread_continue_t continuation, + void *parameter, + wait_result_t wresult, + boolean_t enable_interrupts); + +/* + * Flags that can be passed to set_sched_pri + * to skip side effects + */ +__options_decl(set_sched_pri_options_t, uint32_t, { + SETPRI_DEFAULT = 0x0, + SETPRI_LAZY = 0x1, /* Avoid setting AST flags or sending IPIs */ +}); /* Set the current scheduled priority */ -extern void set_sched_pri( - thread_t thread, - int priority); +extern void set_sched_pri( + thread_t thread, + int priority, + set_sched_pri_options_t options); /* Set base priority of the specified thread */ -extern void sched_set_thread_base_priority( - thread_t thread, - int priority); +extern void sched_set_thread_base_priority( + thread_t thread, + int priority); + +/* Set absolute base priority of the specified thread */ +extern void sched_set_kernel_thread_priority( + thread_t thread, + int priority); + /* Set the thread's true scheduling mode */ extern void sched_set_thread_mode(thread_t thread, - sched_mode_t mode); + sched_mode_t mode); /* Demote the true scheduler mode */ extern void sched_thread_mode_demote(thread_t thread, - uint32_t reason); + uint32_t reason); /* Un-demote the true scheduler mode */ extern void sched_thread_mode_undemote(thread_t thread, - uint32_t reason); + uint32_t reason); + +extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); +extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); /* Re-evaluate base priority of thread (thread locked) */ void thread_recompute_priority(thread_t thread); -/* Re-evaluate base priority of thread (thread unlocked) */ -void thread_recompute_qos(thread_t thread); - -/* Reset scheduled priority of thread */ -extern void thread_recompute_sched_pri( - thread_t thread, - boolean_t override_depress); +/* Re-evaluate scheduled priority of thread (thread locked) */ +extern void thread_recompute_sched_pri( + thread_t thread, + set_sched_pri_options_t options); /* Periodic scheduler activity */ -extern void sched_init_thread(void (*)(void)); +extern void sched_init_thread(void (*)(void)); /* Perform sched_tick housekeeping activities */ -extern boolean_t can_update_priority( - thread_t thread); +extern boolean_t can_update_priority( + thread_t thread); -extern void update_priority( - thread_t thread); +extern void update_priority( + thread_t thread); -extern void lightweight_update_priority( - thread_t thread); +extern void lightweight_update_priority( + thread_t thread); extern void sched_default_quantum_expire(thread_t thread); -/* Idle processor thread */ -extern void idle_thread(void); +/* Idle processor thread continuation */ +extern void idle_thread( + void* parameter, + wait_result_t result); -extern kern_return_t idle_thread_create( - processor_t processor); +extern kern_return_t idle_thread_create( + processor_t processor); /* Continuation return from syscall */ extern void thread_syscall_return( - kern_return_t ret); + kern_return_t ret); /* Context switch */ -extern wait_result_t thread_block_reason( - thread_continue_t continuation, - void *parameter, - ast_t reason); +extern wait_result_t thread_block_reason( + thread_continue_t continuation, + void *parameter, + ast_t reason); -/* Reschedule thread for execution */ -extern void thread_setrun( - thread_t thread, - integer_t options); - -typedef enum { +__options_decl(sched_options_t, uint32_t, { SCHED_NONE = 0x0, SCHED_TAILQ = 0x1, SCHED_HEADQ = 0x2, SCHED_PREEMPT = 0x4, SCHED_REBALANCE = 0x8, -} sched_options_t; +}); -extern processor_set_t task_choose_pset( - task_t task); +/* Reschedule thread for execution */ +extern void thread_setrun( + thread_t thread, + sched_options_t options); + +extern processor_set_t task_choose_pset( + task_t task); /* Bind the current thread to a particular processor */ -extern processor_t thread_bind( - processor_t processor); +extern processor_t thread_bind( + processor_t processor); /* Choose the best processor to run a thread */ -extern processor_t choose_processor( - processor_set_t pset, - processor_t processor, - thread_t thread); +extern processor_t choose_processor( + processor_set_t pset, + processor_t processor, + thread_t thread); extern void sched_SMT_balance( - processor_t processor, - processor_set_t pset); + processor_t processor, + processor_set_t pset); extern void thread_quantum_init( - thread_t thread); - -extern void run_queue_init( - run_queue_t runq); - -extern thread_t run_queue_dequeue( - run_queue_t runq, - integer_t options); - -extern boolean_t run_queue_enqueue( - run_queue_t runq, - thread_t thread, - integer_t options); - -extern void run_queue_remove( - run_queue_t runq, - thread_t thread); - -struct sched_update_scan_context -{ - uint64_t earliest_bg_make_runnable_time; - uint64_t earliest_normal_make_runnable_time; - uint64_t earliest_rt_make_runnable_time; + thread_t thread); + +extern void run_queue_init( + run_queue_t runq); + +extern thread_t run_queue_dequeue( + run_queue_t runq, + sched_options_t options); + +extern boolean_t run_queue_enqueue( + run_queue_t runq, + thread_t thread, + sched_options_t options); + +extern void run_queue_remove( + run_queue_t runq, + thread_t thread); + +extern thread_t run_queue_peek( + run_queue_t runq); + +struct sched_update_scan_context { + uint64_t earliest_bg_make_runnable_time; + uint64_t earliest_normal_make_runnable_time; + uint64_t earliest_rt_make_runnable_time; }; typedef struct sched_update_scan_context *sched_update_scan_context_t; -extern void sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context); +extern void sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context); -/* - * Enum to define various events which need IPIs. The IPI policy - * engine decides what kind of IPI to use based on destination +extern void sched_pset_made_schedulable( + processor_t processor, + processor_set_t pset, + boolean_t drop_lock); + +/* + * Enum to define various events which need IPIs. The IPI policy + * engine decides what kind of IPI to use based on destination * processor state, thread and one of the following scheduling events. */ typedef enum { SCHED_IPI_EVENT_BOUND_THR = 0x1, - SCHED_IPI_EVENT_PREEMPT = 0x2, + SCHED_IPI_EVENT_PREEMPT = 0x2, SCHED_IPI_EVENT_SMT_REBAL = 0x3, - SCHED_IPI_EVENT_SPILL = 0x4, + SCHED_IPI_EVENT_SPILL = 0x4, SCHED_IPI_EVENT_REBALANCE = 0x5, } sched_ipi_event_t; /* Enum to define various IPI types used by the scheduler */ typedef enum { - SCHED_IPI_NONE = 0x0, - SCHED_IPI_IMMEDIATE = 0x1, - SCHED_IPI_IDLE = 0x2, - SCHED_IPI_DEFERRED = 0x3, + SCHED_IPI_NONE = 0x0, + SCHED_IPI_IMMEDIATE = 0x1, + SCHED_IPI_IDLE = 0x2, + SCHED_IPI_DEFERRED = 0x3, } sched_ipi_type_t; /* The IPI policy engine behaves in the following manner: - * - All scheduler events which need an IPI invoke sched_ipi_action() with + * - All scheduler events which need an IPI invoke sched_ipi_action() with * the appropriate destination processor, thread and event. * - sched_ipi_action() performs basic checks, invokes the scheduler specific * ipi_policy routine and sets pending_AST bits based on the result. @@ -298,16 +327,16 @@ typedef enum { * routine which actually sends the appropriate IPI to the destination core. */ extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, - boolean_t dst_idle, sched_ipi_event_t event); + boolean_t dst_idle, sched_ipi_event_t event); extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); /* sched_ipi_policy() is the global default IPI policy for all schedulers */ extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, - boolean_t dst_idle, sched_ipi_event_t event); + boolean_t dst_idle, sched_ipi_event_t event); /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, - processor_t dst, sched_ipi_event_t event); + processor_t dst, sched_ipi_event_t event); #if defined(CONFIG_SCHED_TIMESHARE_CORE) @@ -327,63 +356,63 @@ extern int sched_compute_timeshare_priority(thread_t thread); #endif /* CONFIG_SCHED_TIMESHARE_CORE */ /* Remove thread from its run queue */ -extern boolean_t thread_run_queue_remove(thread_t thread); +extern boolean_t thread_run_queue_remove(thread_t thread); thread_t thread_run_queue_remove_for_handoff(thread_t thread); /* Put a thread back in the run queue after being yanked */ -extern void thread_run_queue_reinsert(thread_t thread, integer_t options); +extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options); -extern void thread_timer_expire( - void *thread, - void *p1); +extern void thread_timer_expire( + void *thread, + void *p1); -extern boolean_t thread_eager_preemption( - thread_t thread); +extern boolean_t thread_eager_preemption( + thread_t thread); extern boolean_t sched_generic_direct_dispatch_to_idle_processors; /* Set the maximum interrupt level for the thread */ __private_extern__ wait_interrupt_t thread_interrupt_level( - wait_interrupt_t interruptible); + wait_interrupt_t interruptible); __private_extern__ wait_result_t thread_mark_wait_locked( - thread_t thread, - wait_interrupt_t interruptible); + thread_t thread, + wait_interrupt_t interruptible); /* Wake up locked thread directly, passing result */ __private_extern__ kern_return_t clear_wait_internal( - thread_t thread, - wait_result_t result); + thread_t thread, + wait_result_t result); extern void sched_stats_handle_csw( - processor_t processor, - int reasons, - int selfpri, - int otherpri); + processor_t processor, + int reasons, + int selfpri, + int otherpri); extern void sched_stats_handle_runq_change( - struct runq_stats *stats, - int old_count); + struct runq_stats *stats, + int old_count); #if DEBUG -#define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ -do { \ - if (__builtin_expect(sched_stats_active, 0)) { \ - sched_stats_handle_csw((processor), \ - (reasons), (selfpri), (otherpri)); \ - } \ -} while (0) +#define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ +do { \ + if (__builtin_expect(sched_stats_active, 0)) { \ + sched_stats_handle_csw((processor), \ + (reasons), (selfpri), (otherpri)); \ + } \ +} while (0) -#define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ -do { \ - if (__builtin_expect(sched_stats_active, 0)) { \ - sched_stats_handle_runq_change((stats), \ - (old_count)); \ - } \ -} while (0) +#define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ +do { \ + if (__builtin_expect(sched_stats_active, 0)) { \ + sched_stats_handle_runq_change((stats), \ + (old_count)); \ + } \ +} while (0) #else /* DEBUG */ @@ -393,53 +422,28 @@ do { \ #endif /* DEBUG */ extern uint32_t sched_debug_flags; -#define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 -#define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 +#define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 +#define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 -#define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) do { \ - if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ - KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ - } \ +#define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) do { \ + if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ + KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ + } \ } while(0) -#define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) do { \ - if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ - KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ - } \ +#define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) do { \ + if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ + KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ + } \ } while(0) -#define THREAD_URGENCY_NONE 0 /* indicates that there is no currently runnable */ -#define THREAD_URGENCY_BACKGROUND 1 /* indicates that the thread is marked as a "background" thread */ -#define THREAD_URGENCY_NORMAL 2 /* indicates that the thread is marked as a "normal" thread */ -#define THREAD_URGENCY_REAL_TIME 3 /* indicates that the thread is marked as a "real-time" or urgent thread */ -#define THREAD_URGENCY_MAX 4 /* Marker */ -/* Returns the "urgency" of a thread (provided by scheduler) */ -extern int thread_get_urgency( - thread_t thread, - uint64_t *rt_period, - uint64_t *rt_deadline); - -/* Tells the "urgency" of the just scheduled thread (provided by CPU PM) */ -extern void thread_tell_urgency( - int urgency, - uint64_t rt_period, - uint64_t rt_deadline, - uint64_t sched_latency, - thread_t nthread); - /* Tells if there are "active" RT threads in the system (provided by CPU PM) */ -extern void active_rt_threads( - boolean_t active); +extern void active_rt_threads( + boolean_t active); /* Returns the perfcontrol attribute for the thread */ extern perfcontrol_class_t thread_get_perfcontrol_class( - thread_t thread); - -#define PSET_LOAD_NUMERATOR_SHIFT 16 -#define PSET_LOAD_FRACTIONAL_SHIFT 4 - -extern int sched_get_pset_load_average(processor_set_t pset); -extern void sched_update_pset_load_average(processor_set_t pset); + thread_t thread); /* Generic routine for Non-AMP schedulers to calculate parallelism */ extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); @@ -448,113 +452,118 @@ extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); __BEGIN_DECLS -#ifdef XNU_KERNEL_PRIVATE +#ifdef XNU_KERNEL_PRIVATE /* Toggles a global override to turn off CPU Throttling */ -#define CPU_THROTTLE_DISABLE 0 -#define CPU_THROTTLE_ENABLE 1 -extern void sys_override_cpu_throttle(int flag); +extern void sys_override_cpu_throttle(boolean_t enable_override); /* ****************** Only exported until BSD stops using ******************** */ -extern void thread_vm_bind_group_add(void); +extern void thread_vm_bind_group_add(void); /* Wake up thread directly, passing result */ extern kern_return_t clear_wait( - thread_t thread, - wait_result_t result); + thread_t thread, + wait_result_t result); /* Start thread running */ -extern void thread_bootstrap_return(void); +extern void thread_bootstrap_return(void) __attribute__((noreturn)); /* Return from exception (BSD-visible interface) */ -extern void thread_exception_return(void) __dead2; +extern void thread_exception_return(void) __dead2; #define SCHED_STRING_MAX_LENGTH (48) /* String declaring the name of the current scheduler */ extern char sched_string[SCHED_STRING_MAX_LENGTH]; -extern thread_t port_name_to_thread_for_ulock(mach_port_name_t thread_name); - /* Attempt to context switch to a specific runnable thread */ -extern wait_result_t thread_handoff(thread_t thread); +extern wait_result_t thread_handoff_deallocate(thread_t thread); + +__attribute__((nonnull(1, 2))) +extern void thread_handoff_parameter(thread_t thread, + thread_continue_t continuation, void *parameter) __dead2; -extern struct waitq *assert_wait_queue(event_t event); +extern struct waitq *assert_wait_queue(event_t event); extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority); extern thread_t thread_wakeup_identify(event_t event, int priority); -#endif /* XNU_KERNEL_PRIVATE */ +#endif /* XNU_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE /* Set pending block hint for a particular object before we go into a wait state */ -extern void thread_set_pending_block_hint( - thread_t thread, - block_hint_t block_hint); +extern void thread_set_pending_block_hint( + thread_t thread, + block_hint_t block_hint); #define QOS_PARALLELISM_COUNT_LOGICAL 0x1 #define QOS_PARALLELISM_REALTIME 0x2 extern uint32_t qos_max_parallelism(int qos, uint64_t options); - #endif /* KERNEL_PRIVATE */ +#if XNU_KERNEL_PRIVATE +extern void thread_yield_with_continuation( + thread_continue_t continuation, + void *parameter) __dead2; +#endif + /* Context switch */ -extern wait_result_t thread_block( - thread_continue_t continuation); +extern wait_result_t thread_block( + thread_continue_t continuation); -extern wait_result_t thread_block_parameter( - thread_continue_t continuation, - void *parameter); +extern wait_result_t thread_block_parameter( + thread_continue_t continuation, + void *parameter); /* Declare thread will wait on a particular event */ -extern wait_result_t assert_wait( - event_t event, - wait_interrupt_t interruptible); +extern wait_result_t assert_wait( + event_t event, + wait_interrupt_t interruptible); /* Assert that the thread intends to wait with a timeout */ -extern wait_result_t assert_wait_timeout( - event_t event, - wait_interrupt_t interruptible, - uint32_t interval, - uint32_t scale_factor); +extern wait_result_t assert_wait_timeout( + event_t event, + wait_interrupt_t interruptible, + uint32_t interval, + uint32_t scale_factor); /* Assert that the thread intends to wait with an urgency, timeout and leeway */ -extern wait_result_t assert_wait_timeout_with_leeway( - event_t event, - wait_interrupt_t interruptible, - wait_timeout_urgency_t urgency, - uint32_t interval, - uint32_t leeway, - uint32_t scale_factor); - -extern wait_result_t assert_wait_deadline( - event_t event, - wait_interrupt_t interruptible, - uint64_t deadline); +extern wait_result_t assert_wait_timeout_with_leeway( + event_t event, + wait_interrupt_t interruptible, + wait_timeout_urgency_t urgency, + uint32_t interval, + uint32_t leeway, + uint32_t scale_factor); + +extern wait_result_t assert_wait_deadline( + event_t event, + wait_interrupt_t interruptible, + uint64_t deadline); /* Assert that the thread intends to wait with an urgency, deadline, and leeway */ -extern wait_result_t assert_wait_deadline_with_leeway( - event_t event, - wait_interrupt_t interruptible, - wait_timeout_urgency_t urgency, - uint64_t deadline, - uint64_t leeway); +extern wait_result_t assert_wait_deadline_with_leeway( + event_t event, + wait_interrupt_t interruptible, + wait_timeout_urgency_t urgency, + uint64_t deadline, + uint64_t leeway); /* Wake up thread (or threads) waiting on a particular event */ -extern kern_return_t thread_wakeup_prim( - event_t event, - boolean_t one_thread, - wait_result_t result); - -#define thread_wakeup(x) \ - thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) -#define thread_wakeup_with_result(x, z) \ - thread_wakeup_prim((x), FALSE, (z)) -#define thread_wakeup_one(x) \ - thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) +extern kern_return_t thread_wakeup_prim( + event_t event, + boolean_t one_thread, + wait_result_t result); + +#define thread_wakeup(x) \ + thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) +#define thread_wakeup_with_result(x, z) \ + thread_wakeup_prim((x), FALSE, (z)) +#define thread_wakeup_one(x) \ + thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) /* Wakeup the specified thread if it is waiting on this event */ extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread); @@ -570,50 +579,47 @@ extern boolean_t preemption_enabled(void); * a function pointer table. */ -#if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) +#if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) && !defined(CONFIG_SCHED_CLUTCH) #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX #endif -#if DEBUG -#define SCHED(f) (sched_current_dispatch->f) -#else /* DEBUG */ -/* - * For DEV & REL kernels, use a static dispatch table instead of - * using the indirect function table. - */ -extern const struct sched_dispatch_table sched_multiq_dispatch; -#define SCHED(f) (sched_multiq_dispatch.f) +#if CONFIG_SCHED_CLUTCH +extern const struct sched_dispatch_table sched_clutch_dispatch; +#define SCHED(f) (sched_clutch_dispatch.f) +#else /* CONFIG_SCHED_CLUTCH */ +extern const struct sched_dispatch_table sched_dualq_dispatch; +#define SCHED(f) (sched_dualq_dispatch.f) +#endif /* CONFIG_SCHED_CLUTCH */ -#endif /* DEBUG */ struct sched_dispatch_table { const char *sched_name; - void (*init)(void); /* Init global state */ - void (*timebase_init)(void); /* Timebase-dependent initialization */ - void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ - void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ + void (*init)(void); /* Init global state */ + void (*timebase_init)(void); /* Timebase-dependent initialization */ + void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ + void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ - void (*maintenance_continuation)(void); /* Function called regularly */ + void (*maintenance_continuation)(void); /* Function called regularly */ /* * Choose a thread of greater or equal priority from the per-processor * runqueue for timeshare/fixed threads */ - thread_t (*choose_thread)( - processor_t processor, - int priority, - ast_t reason); + thread_t (*choose_thread)( + processor_t processor, + int priority, + ast_t reason); - /* True if scheduler supports stealing threads */ - boolean_t steal_thread_enabled; + /* True if scheduler supports stealing threads for this pset */ + bool (*steal_thread_enabled)(processor_set_t pset); /* * Steal a thread from another processor in the pset so that it can run * immediately */ - thread_t (*steal_thread)( - processor_set_t pset); + thread_t (*steal_thread)( + processor_set_t pset); /* * Compute priority for a timeshare thread based on base priority. @@ -623,100 +629,93 @@ struct sched_dispatch_table { /* * Pick the best processor for a thread (any kind of thread) to run on. */ - processor_t (*choose_processor)( - processor_set_t pset, - processor_t processor, - thread_t thread); + processor_t (*choose_processor)( + processor_set_t pset, + processor_t processor, + thread_t thread); /* * Enqueue a timeshare or fixed priority thread onto the per-processor * runqueue */ boolean_t (*processor_enqueue)( - processor_t processor, - thread_t thread, - integer_t options); + processor_t processor, + thread_t thread, + sched_options_t options); /* Migrate threads away in preparation for processor shutdown */ void (*processor_queue_shutdown)( - processor_t processor); + processor_t processor); /* Remove the specific thread from the per-processor runqueue */ - boolean_t (*processor_queue_remove)( - processor_t processor, - thread_t thread); + boolean_t (*processor_queue_remove)( + processor_t processor, + thread_t thread); /* * Does the per-processor runqueue have any timeshare or fixed priority * threads on it? Called without pset lock held, so should * not assume immutability while executing. */ - boolean_t (*processor_queue_empty)(processor_t processor); + boolean_t (*processor_queue_empty)(processor_t processor); /* * Would this priority trigger an urgent preemption if it's sitting * on the per-processor runqueue? */ - boolean_t (*priority_is_urgent)(int priority); + boolean_t (*priority_is_urgent)(int priority); /* * Does the per-processor runqueue contain runnable threads that * should cause the currently-running thread to be preempted? */ - ast_t (*processor_csw_check)(processor_t processor); + ast_t (*processor_csw_check)(processor_t processor); /* * Does the per-processor runqueue contain a runnable thread * of > or >= priority, as a preflight for choose_thread() or other * thread selection */ - boolean_t (*processor_queue_has_priority)(processor_t processor, - int priority, - boolean_t gte); + boolean_t (*processor_queue_has_priority)(processor_t processor, + int priority, + boolean_t gte); /* Quantum size for the specified non-realtime thread. */ - uint32_t (*initial_quantum_size)(thread_t thread); - + uint32_t (*initial_quantum_size)(thread_t thread); + /* Scheduler mode for a new thread */ - sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); + sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); /* * Is it safe to call update_priority, which may change a thread's * runqueue or other state. This can be used to throttle changes * to dynamic priority. */ - boolean_t (*can_update_priority)(thread_t thread); + boolean_t (*can_update_priority)(thread_t thread); /* * Update both scheduled priority and other persistent state. * Side effects may including migration to another processor's runqueue. */ - void (*update_priority)(thread_t thread); + void (*update_priority)(thread_t thread); /* Lower overhead update to scheduled priority and state. */ - void (*lightweight_update_priority)(thread_t thread); + void (*lightweight_update_priority)(thread_t thread); /* Callback for non-realtime threads when the quantum timer fires */ - void (*quantum_expire)(thread_t thread); + void (*quantum_expire)(thread_t thread); /* * Runnable threads on per-processor runqueue. Should only * be used for relative comparisons of load between processors. */ - int (*processor_runq_count)(processor_t processor); + int (*processor_runq_count)(processor_t processor); /* Aggregate runcount statistics for per-processor runqueue */ uint64_t (*processor_runq_stats_count_sum)(processor_t processor); - boolean_t (*processor_bound_count)(processor_t processor); - - void (*thread_update_scan)(sched_update_scan_context_t scan_context); + boolean_t (*processor_bound_count)(processor_t processor); - /* - * Use processor->next_thread to pin a thread to an idle - * processor. If FALSE, threads are enqueued and can - * be stolen by other processors. - */ - boolean_t direct_dispatch_to_idle_processors; + void (*thread_update_scan)(sched_update_scan_context_t scan_context); /* Supports more than one pset */ boolean_t multiple_psets_enabled; @@ -735,16 +734,26 @@ struct sched_dispatch_table { * Called with pset lock held, returns pset lock unlocked. */ void (*processor_balance)(processor_t processor, processor_set_t pset); - rt_queue_t (*rt_runq)(processor_set_t pset); - void (*rt_init)(processor_set_t pset); - void (*rt_queue_shutdown)(processor_t processor); - void (*rt_runq_scan)(sched_update_scan_context_t scan_context); - int64_t (*rt_runq_count_sum)(void); + rt_queue_t (*rt_runq)(processor_set_t pset); + void (*rt_init)(processor_set_t pset); + void (*rt_queue_shutdown)(processor_t processor); + void (*rt_runq_scan)(sched_update_scan_context_t scan_context); + int64_t (*rt_runq_count_sum)(void); uint32_t (*qos_max_parallelism)(int qos, uint64_t options); - void (*check_spill)(processor_set_t pset, thread_t thread); + void (*check_spill)(processor_set_t pset, thread_t thread); sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); bool (*thread_should_yield)(processor_t processor, thread_t thread); + + /* Routine to update run counts */ + uint32_t (*run_count_incr)(thread_t thread); + uint32_t (*run_count_decr)(thread_t thread); + + /* Routine to update scheduling bucket for a thread */ + void (*update_thread_bucket)(thread_t thread); + + /* Routine to inform the scheduler when a new pset becomes schedulable */ + void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock); }; #if defined(CONFIG_SCHED_TRADITIONAL) @@ -765,14 +774,12 @@ extern const struct sched_dispatch_table sched_proto_dispatch; extern const struct sched_dispatch_table sched_grrr_dispatch; #endif -/* - * It is an error to invoke any scheduler-related code - * before this is set up - */ -extern const struct sched_dispatch_table *sched_current_dispatch; +#if defined(CONFIG_SCHED_CLUTCH) +extern const struct sched_dispatch_table sched_clutch_dispatch; +#endif -#endif /* MACH_KERNEL_PRIVATE */ +#endif /* MACH_KERNEL_PRIVATE */ __END_DECLS -#endif /* _KERN_SCHED_PRIM_H_ */ +#endif /* _KERN_SCHED_PRIM_H_ */