X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..527f99514973766e9c0382a4d8550dfb00f54939:/osfmk/kern/processor.h diff --git a/osfmk/kern/processor.h b/osfmk/kern/processor.h index c7fcd9020..09caf6a7f 100644 --- a/osfmk/kern/processor.h +++ b/osfmk/kern/processor.h @@ -1,23 +1,29 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2009 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -51,216 +57,338 @@ */ /* - * processor.h: Processor and processor-set definitions. + * processor.h: Processor and processor-related definitions. */ #ifndef _KERN_PROCESSOR_H_ #define _KERN_PROCESSOR_H_ -/* - * Data structures for managing processors and sets of processors. - */ #include #include #include -extern struct processor_set default_pset; -extern processor_t master_processor; +#include -#ifdef MACH_KERNEL_PRIVATE - -#include -#include +#ifdef MACH_KERNEL_PRIVATE #include +#include #include -#include +#include +#include +#include #include #include +#include +#include -#if NCPUS > 1 -#include -#endif /* NCPUS > 1 */ +typedef enum { + PSET_SMP, +} pset_cluster_type_t; struct processor_set { - struct run_queue runq; /* runq for this set */ + queue_head_t active_queue; /* active processors */ queue_head_t idle_queue; /* idle processors */ - int idle_count; /* how many ? */ - decl_simple_lock_data(,idle_lock) /* lock for above */ - queue_head_t processors; /* all processors here */ - int processor_count;/* how many ? */ - decl_simple_lock_data(,processors_lock) /* lock for above */ - queue_head_t tasks; /* tasks assigned */ - int task_count; /* how many */ - queue_head_t threads; /* threads in this set */ - int thread_count; /* how many */ - int ref_count; /* structure ref count */ - boolean_t active; /* is pset in use */ - decl_mutex_data(, lock) /* lock for everything else */ + queue_head_t idle_secondary_queue; /* idle secondary processors */ + queue_head_t unused_queue; /* processors not recommended by CLPC */ + + int online_processor_count; + int active_processor_count; + int load_average; + + int cpu_set_low, cpu_set_hi; + int cpu_set_count; + uint64_t cpu_bitmask; + uint64_t recommended_bitmask; + +#if __SMP__ + decl_simple_lock_data(,sched_lock) /* lock for above */ +#endif + +#if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) + struct run_queue pset_runq; /* runq for this processor set */ +#endif + struct rt_queue rt_runq; /* realtime runq for this processor set */ + +#if defined(CONFIG_SCHED_TRADITIONAL) + int pset_runq_bound_count; + /* # of threads in runq bound to any processor in pset */ +#endif + + /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */ + uint64_t pending_AST_cpu_mask; +#if defined(CONFIG_SCHED_DEFERRED_AST) + /* + * A separate mask, for ASTs that we may be able to cancel. This is dependent on + * some level of support for requesting an AST on a processor, and then quashing + * that request later. + * + * The purpose of this field (and the associated codepaths) is to infer when we + * no longer need a processor that is DISPATCHING to come up, and to prevent it + * from coming out of IDLE if possible. This should serve to decrease the number + * of spurious ASTs in the system, and let processors spend longer periods in + * IDLE. + */ + uint64_t pending_deferred_AST_cpu_mask; +#endif + uint64_t pending_spill_cpu_mask; + struct ipc_port * pset_self; /* port for operations */ struct ipc_port * pset_name_self; /* port for information */ - int max_priority; /* maximum priority */ - int policies; /* bit vector for policies */ - int set_quantum; /* current default quantum */ -#if NCPUS > 1 - int quantum_adj_index; /* runtime quantum adj. */ - decl_simple_lock_data(,quantum_adj_lock) /* lock for above */ - int machine_quantum[NCPUS+1]; /* ditto */ -#endif /* NCPUS > 1 */ - integer_t mach_factor; /* mach_factor */ - integer_t load_average; /* load_average */ - long sched_load; /* load avg for scheduler */ - policy_t policy_default; /* per set default */ - policy_base_data_t policy_base; /* base attributes */ - policy_limit_data_t policy_limit; /* limit attributes */ + + processor_set_t pset_list; /* chain of associated psets */ + pset_node_t node; + uint32_t pset_cluster_id; + pset_cluster_type_t pset_cluster_type; }; +extern struct processor_set pset0; + +struct pset_node { + processor_set_t psets; /* list of associated psets */ + + pset_node_t nodes; /* list of associated subnodes */ + pset_node_t node_list; /* chain of associated nodes */ + + pset_node_t parent; +}; + +extern struct pset_node pset_node0; + +extern queue_head_t tasks, terminated_tasks, threads, corpse_tasks; /* Terminated tasks are ONLY for stackshot */ +extern int tasks_count, terminated_tasks_count, threads_count; +decl_lck_mtx_data(extern,tasks_threads_lock) +decl_lck_mtx_data(extern,tasks_corpse_lock) + struct processor { - struct run_queue runq; /* local runq for this processor */ - queue_chain_t processor_queue;/* idle/assign/shutdown queue link */ + queue_chain_t processor_queue;/* idle/active queue link, + * MUST remain the first element */ int state; /* See below */ - struct thread_shuttle - *next_thread, /* next thread to run if dispatched */ + boolean_t is_SMT; + boolean_t is_recommended; + struct thread + *active_thread, /* thread running on processor */ + *next_thread, /* next thread when dispatched */ *idle_thread; /* this processor's idle thread. */ - int quantum; /* quantum for current thread */ - boolean_t first_quantum; /* first quantum in succession */ - int last_quantum; /* last quantum assigned */ - - processor_set_t processor_set; /* processor set I belong to */ - processor_set_t processor_set_next; /* set I will belong to */ - queue_chain_t processors; /* all processors in set */ - decl_simple_lock_data(,lock) - struct ipc_port *processor_self;/* port for operations */ - int slot_num; /* machine-indep slot number */ -#if NCPUS > 1 - ast_check_t ast_check_data; /* for remote ast_check invocation */ - queue_chain_t softclock_queue;/* cpus handling softclocks */ -#endif /* NCPUS > 1 */ - /* punt id data temporarily */ + + processor_set_t processor_set; /* assigned set */ + + int current_pri; /* priority of current thread */ + sfi_class_id_t current_sfi_class; /* SFI class of current thread */ + perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */ + int starting_pri; /* priority of current thread as it was when scheduled */ + pset_cluster_type_t current_recommended_pset_type; /* Cluster type recommended for current thread */ + int cpu_id; /* platform numeric id */ + + timer_call_data_t quantum_timer; /* timer for quantum expiration */ + uint64_t quantum_end; /* time when current quantum ends */ + uint64_t last_dispatch; /* time of last dispatch */ + + uint64_t deadline; /* current deadline */ + boolean_t first_timeslice; /* has the quantum expired since context switch */ + +#if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) + struct run_queue runq; /* runq for this processor */ +#endif + +#if defined(CONFIG_SCHED_TRADITIONAL) + int runq_bound_count; /* # of threads bound to this processor */ +#endif +#if defined(CONFIG_SCHED_GRRR) + struct grrr_run_queue grrr_runq; /* Group Ratio Round-Robin runq */ +#endif + + processor_t processor_primary; /* pointer to primary processor for + * secondary SMT processors, or a pointer + * to ourselves for primaries or non-SMT */ + processor_t processor_secondary; + struct ipc_port * processor_self; /* port for operations */ + + processor_t processor_list; /* all existing processors */ + processor_data_t processor_data; /* per-processor data */ }; -extern struct processor processor_array[NCPUS]; +extern processor_t processor_list; +decl_simple_lock_data(extern,processor_list_lock) + +#define MAX_SCHED_CPUS 64 /* Maximum number of CPUs supported by the scheduler. bits.h:bitmap_*() macros need to be used to support greater than 64 */ +extern processor_t processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */ + +extern uint32_t processor_avail_count; + +extern processor_t master_processor; + +extern boolean_t sched_stats_active; /* - * NOTE: The processor->processor_set link is needed in one of the - * scheduler's critical paths. [Figure out where to look for another - * thread to run on this processor.] It is accessed without locking. - * The following access protocol controls this field. + * Processor state is accessed by locking the scheduling lock + * for the assigned processor set. + * + * -------------------- SHUTDOWN + * / ^ ^ + * _/ | \ + * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING + * \_________________^ ^ ^______/ / + * \__________________/ + * + * Most of these state transitions are externally driven as a + * a directive (for instance telling an IDLE processor to start + * coming out of the idle state to run a thread). However these + * are typically paired with a handshake by the processor itself + * to indicate that it has completed a transition of indeterminate + * length (for example, the DISPATCHING->RUNNING or START->RUNNING + * transitions must occur on the processor itself). * - * Read from own processor - just read. - * Read from another processor - lock processor structure during read. - * Write from own processor - lock processor structure during write. - * Write from another processor - NOT PERMITTED. + * The boot processor has some special cases, and skips the START state, + * since it has already bootstrapped and is ready to context switch threads. * + * When a processor is in DISPATCHING or RUNNING state, the current_pri, + * current_thmode, and deadline fields should be set, so that other + * processors can evaluate if it is an appropriate candidate for preemption. */ - +#if defined(CONFIG_SCHED_DEFERRED_AST) /* - * Processor state locking: + * -------------------- SHUTDOWN + * / ^ ^ + * _/ | \ + * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING + * \_________________^ ^ ^______/ ^_____ / / + * \__________________/ + * + * A DISPATCHING processor may be put back into IDLE, if another + * processor determines that the target processor will have nothing to do + * upon reaching the RUNNING state. This is racy, but if the target + * responds and becomes RUNNING, it will not break the processor state + * machine. * - * Values for the processor state are defined below. If the processor - * is off-line or being shutdown, then it is only necessary to lock - * the processor to change its state. Otherwise it is only necessary - * to lock its processor set's idle_lock. Scheduler code will - * typically lock only the idle_lock, but processor manipulation code - * will often lock both. + * This change allows us to cancel an outstanding signal/AST on a processor + * (if such an operation is supported through hardware or software), and + * push the processor back into the IDLE state as a power optimization. */ +#endif -#define PROCESSOR_OFF_LINE 0 /* Not in system */ -#define PROCESSOR_RUNNING 1 /* Running a normal thread */ -#define PROCESSOR_IDLE 2 /* idle */ -#define PROCESSOR_DISPATCHING 3 /* dispatching (idle -> running) */ -#define PROCESSOR_ASSIGN 4 /* Assignment is changing */ -#define PROCESSOR_SHUTDOWN 5 /* Being shutdown */ -#define PROCESSOR_START 6 /* Being start */ +#define PROCESSOR_OFF_LINE 0 /* Not available */ +#define PROCESSOR_SHUTDOWN 1 /* Going off-line */ +#define PROCESSOR_START 2 /* Being started */ +/* 3 Formerly Inactive (unavailable) */ +#define PROCESSOR_IDLE 4 /* Idle (available) */ +#define PROCESSOR_DISPATCHING 5 /* Dispatching (idle -> active) */ +#define PROCESSOR_RUNNING 6 /* Normal execution */ -/* - * Use processor ptr array to find current processor's data structure. - * This replaces a multiplication (index into processor_array) with - * an array lookup and a memory reference. It also allows us to save - * space if processor numbering gets too sparse. - */ +extern processor_t current_processor(void); -extern processor_t processor_ptr[NCPUS]; +/* Lock macros, always acquired and released with interrupts disabled (splsched()) */ -#define cpu_to_processor(i) (processor_ptr[i]) +#if __SMP__ +#define pset_lock(p) simple_lock(&(p)->sched_lock) +#define pset_unlock(p) simple_unlock(&(p)->sched_lock) +#define pset_lock_init(p) simple_lock_init(&(p)->sched_lock, 0) -#define current_processor() (processor_ptr[cpu_number()]) -#define current_processor_set() (current_processor()->processor_set) +#define rt_lock_lock(p) simple_lock(&SCHED(rt_runq)(p)->rt_lock) +#define rt_lock_unlock(p) simple_unlock(&SCHED(rt_runq)(p)->rt_lock) +#define rt_lock_init(p) simple_lock_init(&SCHED(rt_runq)(p)->rt_lock, 0) +#else +#define pset_lock(p) do { (void)p; } while(0) +#define pset_unlock(p) do { (void)p; } while(0) +#define pset_lock_init(p) do { (void)p; } while(0) -/* Compatibility -- will go away */ +#define rt_lock_lock(p) do { (void)p; } while(0) +#define rt_lock_unlock(p) do { (void)p; } while(0) +#define rt_lock_init(p) do { (void)p; } while(0) +#endif -#define cpu_state(slot_num) (processor_ptr[slot_num]->state) -#define cpu_idle(slot_num) (cpu_state(slot_num) == PROCESSOR_IDLE) +extern void processor_bootstrap(void); -/* Useful lock macros */ +extern void processor_init( + processor_t processor, + int cpu_id, + processor_set_t processor_set); -#define pset_lock(pset) mutex_lock(&(pset)->lock) -#define pset_lock_try(pset) mutex_try(&(pset)->lock) -#define pset_unlock(pset) mutex_unlock(&(pset)->lock) +extern void processor_set_primary( + processor_t processor, + processor_t primary); -#define processor_lock(pr) simple_lock(&(pr)->lock) -#define processor_unlock(pr) simple_unlock(&(pr)->lock) +extern kern_return_t processor_shutdown( + processor_t processor); +extern void processor_queue_shutdown( + processor_t processor); -extern void pset_sys_bootstrap(void); +extern processor_set_t processor_pset( + processor_t processor); -/* Implemented by MD layer */ +extern pset_node_t pset_node_root(void); -extern void cpu_up( - int cpu); +extern processor_set_t pset_create( + pset_node_t node); -extern kern_return_t processor_shutdown( - processor_t processor); +extern void pset_init( + processor_set_t pset, + pset_node_t node); -extern void pset_remove_processor( - processor_set_t pset, - processor_t processor); +extern processor_set_t pset_find( + uint32_t cluster_id, + processor_set_t default_pset); -extern void pset_add_processor( - processor_set_t pset, - processor_t processor); +extern kern_return_t processor_info_count( + processor_flavor_t flavor, + mach_msg_type_number_t *count); + +#define pset_deallocate(x) +#define pset_reference(x) + +extern void machine_run_count( + uint32_t count); + +extern processor_t machine_choose_processor( + processor_set_t pset, + processor_t processor); -extern void pset_remove_task( - processor_set_t pset, - task_t task); +#define next_pset(p) (((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets) -extern void pset_add_task( - processor_set_t pset, - task_t task); +#define PSET_THING_TASK 0 +#define PSET_THING_THREAD 1 -extern void pset_remove_thread( - processor_set_t pset, - thread_t thread); +extern kern_return_t processor_set_things( + processor_set_t pset, + void **thing_list, + mach_msg_type_number_t *count, + int type); -extern void pset_add_thread( - processor_set_t pset, - thread_t thread); +extern pset_cluster_type_t recommended_pset_type(thread_t thread); -extern void thread_change_psets( - thread_t thread, - processor_set_t old_pset, - processor_set_t new_pset); +inline static bool +pset_is_recommended(processor_set_t pset) +{ + return ((pset->recommended_bitmask & pset->cpu_bitmask) != 0); +} + +extern void processor_state_update_idle(processor_t processor); +extern void processor_state_update_from_thread(processor_t processor, thread_t thread); +extern void processor_state_update_explicit(processor_t processor, int pri, + sfi_class_id_t sfi_class, pset_cluster_type_t pset_type, + perfcontrol_class_t perfctl_class); + +#else /* MACH_KERNEL_PRIVATE */ + +__BEGIN_DECLS extern void pset_deallocate( - processor_set_t pset); + processor_set_t pset); extern void pset_reference( - processor_set_t pset); + processor_set_t pset); -extern kern_return_t processor_assign( - processor_t processor, - processor_set_t new_pset, - boolean_t wait); +__END_DECLS -extern kern_return_t processor_info_count( - processor_flavor_t flavor, - mach_msg_type_number_t *count); -#endif /* MACH_KERNEL_PRIVATE */ +#endif /* MACH_KERNEL_PRIVATE */ -extern kern_return_t processor_start( - processor_t processor); +#ifdef KERNEL_PRIVATE +__BEGIN_DECLS +extern unsigned int processor_count; +extern processor_t cpu_to_processor(int cpu); +__END_DECLS -extern kern_return_t processor_exit( - processor_t processor); +#endif /* KERNEL_PRIVATE */ #endif /* _KERN_PROCESSOR_H_ */