X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..527f99514973766e9c0382a4d8550dfb00f54939:/osfmk/kern/processor.h diff --git a/osfmk/kern/processor.h b/osfmk/kern/processor.h index 922e20aa2..09caf6a7f 100644 --- a/osfmk/kern/processor.h +++ b/osfmk/kern/processor.h @@ -74,6 +74,7 @@ #include <mach/mach_types.h> #include <kern/ast.h> #include <kern/cpu_number.h> +#include <kern/smp.h> #include <kern/simple_lock.h> #include <kern/locks.h> #include <kern/queue.h> @@ -81,23 +82,33 @@ #include <mach/sfi_class.h> #include <kern/processor_data.h> -#include <machine/ast_types.h> +typedef enum { + PSET_SMP, +} pset_cluster_type_t; struct processor_set { queue_head_t active_queue; /* active processors */ queue_head_t idle_queue; /* idle processors */ - queue_head_t idle_secondary_queue; /* idle secondary processors */ + queue_head_t idle_secondary_queue; /* idle secondary processors */ + queue_head_t unused_queue; /* processors not recommended by CLPC */ int online_processor_count; + int active_processor_count; + int load_average; int cpu_set_low, cpu_set_hi; int cpu_set_count; + uint64_t cpu_bitmask; + uint64_t recommended_bitmask; +#if __SMP__ decl_simple_lock_data(,sched_lock) /* lock for above */ +#endif #if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) struct run_queue pset_runq; /* runq for this processor set */ #endif + struct rt_queue rt_runq; /* realtime runq for this processor set */ #if defined(CONFIG_SCHED_TRADITIONAL) int pset_runq_bound_count; @@ -105,13 +116,30 @@ struct processor_set { #endif /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */ - uint32_t pending_AST_cpu_mask; + uint64_t pending_AST_cpu_mask; +#if defined(CONFIG_SCHED_DEFERRED_AST) + /* + * A separate mask, for ASTs that we may be able to cancel. This is dependent on + * some level of support for requesting an AST on a processor, and then quashing + * that request later. + * + * The purpose of this field (and the associated codepaths) is to infer when we + * no longer need a processor that is DISPATCHING to come up, and to prevent it + * from coming out of IDLE if possible. This should serve to decrease the number + * of spurious ASTs in the system, and let processors spend longer periods in + * IDLE. + */ + uint64_t pending_deferred_AST_cpu_mask; +#endif + uint64_t pending_spill_cpu_mask; struct ipc_port * pset_self; /* port for operations */ struct ipc_port * pset_name_self; /* port for information */ processor_set_t pset_list; /* chain of associated psets */ - pset_node_t node; + pset_node_t node; + uint32_t pset_cluster_id; + pset_cluster_type_t pset_cluster_type; }; extern struct processor_set pset0; @@ -127,15 +155,17 @@ struct pset_node { extern struct pset_node pset_node0; -extern queue_head_t tasks, terminated_tasks, threads; /* Terminated tasks are ONLY for stackshot */ +extern queue_head_t tasks, terminated_tasks, threads, corpse_tasks; /* Terminated tasks are ONLY for stackshot */ extern int tasks_count, terminated_tasks_count, threads_count; decl_lck_mtx_data(extern,tasks_threads_lock) +decl_lck_mtx_data(extern,tasks_corpse_lock) struct processor { queue_chain_t processor_queue;/* idle/active queue link, * MUST remain the first element */ int state; /* See below */ boolean_t is_SMT; + boolean_t is_recommended; struct thread *active_thread, /* thread running on processor */ *next_thread, /* next thread when dispatched */ @@ -143,17 +173,19 @@ struct processor { processor_set_t processor_set; /* assigned set */ - int current_pri; /* priority of current thread */ - sched_mode_t current_thmode; /* sched mode of current thread */ + int current_pri; /* priority of current thread */ sfi_class_id_t current_sfi_class; /* SFI class of current thread */ - int cpu_id; /* platform numeric id */ + perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */ + int starting_pri; /* priority of current thread as it was when scheduled */ + pset_cluster_type_t current_recommended_pset_type; /* Cluster type recommended for current thread */ + int cpu_id; /* platform numeric id */ timer_call_data_t quantum_timer; /* timer for quantum expiration */ uint64_t quantum_end; /* time when current quantum ends */ uint64_t last_dispatch; /* time of last dispatch */ uint64_t deadline; /* current deadline */ - int timeslice; /* quanta before timeslice ends */ + boolean_t first_timeslice; /* has the quantum expired since context switch */ #if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) struct run_queue runq; /* runq for this processor */ @@ -177,9 +209,11 @@ struct processor { }; extern processor_t processor_list; -extern unsigned int processor_count; decl_simple_lock_data(extern,processor_list_lock) +#define MAX_SCHED_CPUS 64 /* Maximum number of CPUs supported by the scheduler. bits.h:bitmap_*() macros need to be used to support greater than 64 */ +extern processor_t processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */ + extern uint32_t processor_avail_count; extern processor_t master_processor; @@ -211,7 +245,28 @@ extern boolean_t sched_stats_active; * When a processor is in DISPATCHING or RUNNING state, the current_pri, * current_thmode, and deadline fields should be set, so that other * processors can evaluate if it is an appropriate candidate for preemption. -*/ + */ +#if defined(CONFIG_SCHED_DEFERRED_AST) +/* + * -------------------- SHUTDOWN + * / ^ ^ + * _/ | \ + * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING + * \_________________^ ^ ^______/ ^_____ / / + * \__________________/ + * + * A DISPATCHING processor may be put back into IDLE, if another + * processor determines that the target processor will have nothing to do + * upon reaching the RUNNING state. This is racy, but if the target + * responds and becomes RUNNING, it will not break the processor state + * machine. + * + * This change allows us to cancel an outstanding signal/AST on a processor + * (if such an operation is supported through hardware or software), and + * push the processor back into the IDLE state as a power optimization. + */ +#endif + #define PROCESSOR_OFF_LINE 0 /* Not available */ #define PROCESSOR_SHUTDOWN 1 /* Going off-line */ #define PROCESSOR_START 2 /* Being started */ @@ -222,12 +277,26 @@ extern boolean_t sched_stats_active; extern processor_t current_processor(void); -/* Lock macros */ +/* Lock macros, always acquired and released with interrupts disabled (splsched()) */ +#if __SMP__ #define pset_lock(p) simple_lock(&(p)->sched_lock) #define pset_unlock(p) simple_unlock(&(p)->sched_lock) #define pset_lock_init(p) simple_lock_init(&(p)->sched_lock, 0) +#define rt_lock_lock(p) simple_lock(&SCHED(rt_runq)(p)->rt_lock) +#define rt_lock_unlock(p) simple_unlock(&SCHED(rt_runq)(p)->rt_lock) +#define rt_lock_init(p) simple_lock_init(&SCHED(rt_runq)(p)->rt_lock, 0) +#else +#define pset_lock(p) do { (void)p; } while(0) +#define pset_unlock(p) do { (void)p; } while(0) +#define pset_lock_init(p) do { (void)p; } while(0) + +#define rt_lock_lock(p) do { (void)p; } while(0) +#define rt_lock_unlock(p) do { (void)p; } while(0) +#define rt_lock_init(p) do { (void)p; } while(0) +#endif + extern void processor_bootstrap(void); extern void processor_init( @@ -257,6 +326,10 @@ extern void pset_init( processor_set_t pset, pset_node_t node); +extern processor_set_t pset_find( + uint32_t cluster_id, + processor_set_t default_pset); + extern kern_return_t processor_info_count( processor_flavor_t flavor, mach_msg_type_number_t *count); @@ -273,6 +346,29 @@ extern processor_t machine_choose_processor( #define next_pset(p) (((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets) +#define PSET_THING_TASK 0 +#define PSET_THING_THREAD 1 + +extern kern_return_t processor_set_things( + processor_set_t pset, + void **thing_list, + mach_msg_type_number_t *count, + int type); + +extern pset_cluster_type_t recommended_pset_type(thread_t thread); + +inline static bool +pset_is_recommended(processor_set_t pset) +{ + return ((pset->recommended_bitmask & pset->cpu_bitmask) != 0); +} + +extern void processor_state_update_idle(processor_t processor); +extern void processor_state_update_from_thread(processor_t processor, thread_t thread); +extern void processor_state_update_explicit(processor_t processor, int pri, + sfi_class_id_t sfi_class, pset_cluster_type_t pset_type, + perfcontrol_class_t perfctl_class); + #else /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -289,6 +385,7 @@ __END_DECLS #ifdef KERNEL_PRIVATE __BEGIN_DECLS +extern unsigned int processor_count; extern processor_t cpu_to_processor(int cpu); __END_DECLS