xnu-4570.20.62.tar.gz

[apple/xnu.git] / osfmk / kern / processor.h

diff --git a/osfmk/kern/processor.h b/osfmk/kern/processor.h
index 6c238db71a2d3df50326814efe2c6f07f6f12eef..09caf6a7fbe0b3c1a980656720073ef39803dd82 100644 (file)
--- a/osfmk/kern/processor.h
+++ b/osfmk/kern/processor.h
@@ -1,16 +1,19 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2009 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in

- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.

  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER


@@ -20,7 +23,7 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
 /*
  * @OSF_COPYRIGHT@


@@ -54,241 +57,338 @@
  */
 
 /*
  */
 
 /*

- *     processor.h:    Processor and processor-set definitions.
+ *     processor.h:    Processor and processor-related definitions.

  */
 
 #ifndef        _KERN_PROCESSOR_H_
 #define        _KERN_PROCESSOR_H_
 
  */
 
 #ifndef        _KERN_PROCESSOR_H_
 #define        _KERN_PROCESSOR_H_
 

-/*
- *     Data structures for managing processors and sets of processors.
- */

 #include <mach/boolean.h>
 #include <mach/kern_return.h>
 #include <kern/kern_types.h>
 
 #include <mach/boolean.h>
 #include <mach/kern_return.h>
 #include <kern/kern_types.h>
 

-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_PRIVATE
+#include <sys/cdefs.h>

 
 #ifdef MACH_KERNEL_PRIVATE
 
 
 #ifdef MACH_KERNEL_PRIVATE
 

-#include <cpus.h>
-

 #include <mach/mach_types.h>
 #include <mach/mach_types.h>

+#include <kern/ast.h>

 #include <kern/cpu_number.h>
 #include <kern/cpu_number.h>

-#include <kern/lock.h>
+#include <kern/smp.h>
+#include <kern/simple_lock.h>
+#include <kern/locks.h>

 #include <kern/queue.h>
 #include <kern/sched.h>
 #include <kern/queue.h>
 #include <kern/sched.h>

+#include <mach/sfi_class.h>
+#include <kern/processor_data.h>

 
 

-#include <machine/ast_types.h>
+typedef enum {
+       PSET_SMP,
+} pset_cluster_type_t;

 
 struct processor_set {
 
 struct processor_set {

-       queue_head_t            idle_queue;             /* idle processors */
-       int                                     idle_count;             /* how many ? */

        queue_head_t            active_queue;   /* active processors */
        queue_head_t            active_queue;   /* active processors */

+       queue_head_t            idle_queue;             /* idle processors */
+       queue_head_t            idle_secondary_queue;   /* idle secondary processors */
+       queue_head_t            unused_queue;           /* processors not recommended by CLPC */

 
 

-       queue_head_t            processors;             /* all processors here */
-       int                                     processor_count;/* how many ? */
-       decl_simple_lock_data(,sched_lock)      /* lock for above */
-
-       struct  run_queue       runq;                   /* runq for this set */
+       int                                     online_processor_count;
+       int                                     active_processor_count;
+       int                                     load_average;

 
 

-       queue_head_t            tasks;                  /* tasks assigned */
-       int                                     task_count;             /* how many */
-       queue_head_t            threads;                /* threads in this set */
-       int                                     thread_count;   /* how many */
-       int                                     ref_count;              /* structure ref count */
-       boolean_t                       active;                 /* is pset in use */
-       decl_mutex_data(,       lock)                   /* lock for above */
+       int                                     cpu_set_low, cpu_set_hi;
+       int                                     cpu_set_count;
+       uint64_t                                cpu_bitmask;
+       uint64_t                                recommended_bitmask;

 
 

-       int                                     timeshare_quanta;       /* timeshare quantum factor */
-       int                                     quantum_factors[NCPUS+1];
+#if __SMP__
+       decl_simple_lock_data(,sched_lock)      /* lock for above */
+#endif
+
+#if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ)
+       struct run_queue        pset_runq;      /* runq for this processor set */
+#endif
+       struct rt_queue         rt_runq;        /* realtime runq for this processor set */
+
+#if defined(CONFIG_SCHED_TRADITIONAL)
+       int                                     pset_runq_bound_count;
+               /* # of threads in runq bound to any processor in pset */
+#endif
+
+       /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */
+       uint64_t                        pending_AST_cpu_mask;
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+       /*
+        * A separate mask, for ASTs that we may be able to cancel.  This is dependent on
+        * some level of support for requesting an AST on a processor, and then quashing
+        * that request later.
+        *
+        * The purpose of this field (and the associated codepaths) is to infer when we
+        * no longer need a processor that is DISPATCHING to come up, and to prevent it
+        * from coming out of IDLE if possible.  This should serve to decrease the number
+        * of spurious ASTs in the system, and let processors spend longer periods in
+        * IDLE.
+        */
+       uint64_t                        pending_deferred_AST_cpu_mask;
+#endif
+       uint64_t                        pending_spill_cpu_mask;

 
        struct ipc_port *       pset_self;              /* port for operations */
        struct ipc_port *       pset_name_self; /* port for information */
 
 
        struct ipc_port *       pset_self;              /* port for operations */
        struct ipc_port *       pset_name_self; /* port for information */
 

-       uint32_t                        run_count;              /* threads running in set */
-       uint32_t                        share_count;    /* timeshare threads running in set */
+       processor_set_t         pset_list;              /* chain of associated psets */
+       pset_node_t             node;
+       uint32_t                pset_cluster_id;
+       pset_cluster_type_t     pset_cluster_type;
+};
+
+extern struct processor_set    pset0;
+
+struct pset_node {
+       processor_set_t         psets;                  /* list of associated psets */

 
 

-       integer_t                       mach_factor;    /* mach_factor */
-       integer_t                       load_average;   /* load_average */
-       uint32_t                        sched_load;             /* load avg for scheduler */
+       pset_node_t                     nodes;                  /* list of associated subnodes */
+       pset_node_t                     node_list;              /* chain of associated nodes */
+
+       pset_node_t                     parent;

 };
 
 };
 

+extern struct pset_node        pset_node0;
+
+extern queue_head_t            tasks, terminated_tasks, threads, corpse_tasks; /* Terminated tasks are ONLY for stackshot */
+extern int                             tasks_count, terminated_tasks_count, threads_count;
+decl_lck_mtx_data(extern,tasks_threads_lock)
+decl_lck_mtx_data(extern,tasks_corpse_lock)
+

 struct processor {
 struct processor {

-       queue_chain_t           processor_queue;/* idle/active/action queue link,
+       queue_chain_t           processor_queue;/* idle/active queue link,

                                                                                 * MUST remain the first element */
        int                                     state;                  /* See below */
                                                                                 * MUST remain the first element */
        int                                     state;                  /* See below */

+       boolean_t               is_SMT;
+       boolean_t               is_recommended;

        struct thread
                                                *active_thread, /* thread running on processor */
        struct thread
                                                *active_thread, /* thread running on processor */

-                                               *next_thread,   /* next thread to run if dispatched */
+                                               *next_thread,   /* next thread when dispatched */

                                                *idle_thread;   /* this processor's idle thread. */
 
                                                *idle_thread;   /* this processor's idle thread. */
 

-       processor_set_t         processor_set;  /* current membership */
+       processor_set_t         processor_set;  /* assigned set */

 
 

-       int                                     current_pri;    /* priority of current thread */
+       int                     current_pri;    /* priority of current thread */
+       sfi_class_id_t          current_sfi_class;      /* SFI class of current thread */
+       perfcontrol_class_t     current_perfctl_class;  /* Perfcontrol class for current thread */
+       int                     starting_pri;       /* priority of current thread as it was when scheduled */
+       pset_cluster_type_t     current_recommended_pset_type;  /* Cluster type recommended for current thread */
+       int                     cpu_id;                 /* platform numeric id */

 
        timer_call_data_t       quantum_timer;  /* timer for quantum expiration */
        uint64_t                        quantum_end;    /* time when current quantum ends */
        uint64_t                        last_dispatch;  /* time of last dispatch */
 
 
        timer_call_data_t       quantum_timer;  /* timer for quantum expiration */
        uint64_t                        quantum_end;    /* time when current quantum ends */
        uint64_t                        last_dispatch;  /* time of last dispatch */
 

-       int                                     timeslice;              /* quanta before timeslice ends */

        uint64_t                        deadline;               /* current deadline */
        uint64_t                        deadline;               /* current deadline */

+       boolean_t               first_timeslice;                /* has the quantum expired since context switch */
+
+#if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ)
+       struct run_queue        runq;                   /* runq for this processor */
+#endif
+
+#if defined(CONFIG_SCHED_TRADITIONAL)
+       int                                     runq_bound_count; /* # of threads bound to this processor */
+#endif
+#if defined(CONFIG_SCHED_GRRR)
+       struct grrr_run_queue   grrr_runq;      /* Group Ratio Round-Robin runq */
+#endif
+
+       processor_t                     processor_primary;      /* pointer to primary processor for
+                                                                                        * secondary SMT processors, or a pointer
+                                                                                        * to ourselves for primaries or non-SMT */
+       processor_t             processor_secondary;
+       struct ipc_port *       processor_self; /* port for operations */
+
+       processor_t                     processor_list; /* all existing processors */
+       processor_data_t        processor_data; /* per-processor data */
+};

 
 

-       struct run_queue        runq;                   /* local runq for this processor */
+extern processor_t             processor_list;
+decl_simple_lock_data(extern,processor_list_lock)

 
 

-       queue_chain_t           processors;             /* all processors in set */
-       decl_simple_lock_data(,lock)
-       struct ipc_port         *processor_self;/* port for operations */
-       int                                     slot_num;               /* machine-indep slot number */
-};
+#define MAX_SCHED_CPUS          64 /* Maximum number of CPUs supported by the scheduler.  bits.h:bitmap_*() macros need to be used to support greater than 64 */
+extern processor_t              processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */
+
+extern uint32_t                        processor_avail_count;

 
 

-extern struct processor_set    default_pset;
-extern processor_t     master_processor;
+extern processor_t             master_processor;

 
 

-extern struct processor        processor_array[NCPUS];
+extern boolean_t               sched_stats_active;

 
 /*
 
 /*

- *     NOTE: The processor->processor_set link is needed in one of the
- *     scheduler's critical paths.  [Figure out where to look for another
- *     thread to run on this processor.]  It is accessed without locking.
- *     The following access protocol controls this field.
+ *     Processor state is accessed by locking the scheduling lock
+ *     for the assigned processor set.

  *
  *

- *     Read from own processor - just read.
- *     Read from another processor - lock processor structure during read.
- *     Write from own processor - lock processor structure during write.
- *     Write from another processor - NOT PERMITTED.
+ *           -------------------- SHUTDOWN
+ *          /                     ^     ^
+ *        _/                      |      \
+ *  OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING
+ *         \_________________^   ^ ^______/           /
+ *                                \__________________/

  *
  *

+ *  Most of these state transitions are externally driven as a
+ *  a directive (for instance telling an IDLE processor to start
+ *  coming out of the idle state to run a thread). However these
+ *  are typically paired with a handshake by the processor itself
+ *  to indicate that it has completed a transition of indeterminate
+ *  length (for example, the DISPATCHING->RUNNING or START->RUNNING
+ *  transitions must occur on the processor itself).
+ *
+ *  The boot processor has some special cases, and skips the START state,
+ *  since it has already bootstrapped and is ready to context switch threads.
+ *
+ *  When a processor is in DISPATCHING or RUNNING state, the current_pri,
+ *  current_thmode, and deadline fields should be set, so that other
+ *  processors can evaluate if it is an appropriate candidate for preemption.

  */
  */

-
+#if defined(CONFIG_SCHED_DEFERRED_AST)

 /*
 /*

- *     Processor state locking:
+ *           -------------------- SHUTDOWN
+ *          /                     ^     ^
+ *        _/                      |      \
+ *  OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING
+ *         \_________________^   ^ ^______/ ^_____ /  /
+ *                                \__________________/
+ *
+ *  A DISPATCHING processor may be put back into IDLE, if another
+ *  processor determines that the target processor will have nothing to do
+ *  upon reaching the RUNNING state.  This is racy, but if the target
+ *  responds and becomes RUNNING, it will not break the processor state
+ *  machine.

  *
  *

- *     Values for the processor state are defined below.  If the processor
- *     is off-line or being shutdown, then it is only necessary to lock
- *     the processor to change its state.  Otherwise it is only necessary
- *     to lock its processor set's sched_lock.  Scheduler code will
- *     typically lock only the sched_lock, but processor manipulation code
- *     will often lock both.
+ *  This change allows us to cancel an outstanding signal/AST on a processor
+ *  (if such an operation is supported through hardware or software), and
+ *  push the processor back into the IDLE state as a power optimization.

  */
  */

+#endif

 
 #define PROCESSOR_OFF_LINE             0       /* Not available */
 
 #define PROCESSOR_OFF_LINE             0       /* Not available */

-#define        PROCESSOR_RUNNING               1       /* Normal execution */
-#define        PROCESSOR_IDLE                  2       /* Idle */
-#define PROCESSOR_DISPATCHING  3       /* Dispatching (idle -> running) */
-#define PROCESSOR_SHUTDOWN             4       /* Going off-line */
-#define PROCESSOR_START                        5       /* Being started */
+#define PROCESSOR_SHUTDOWN             1       /* Going off-line */
+#define PROCESSOR_START                        2       /* Being started */
+/*                                             3          Formerly Inactive (unavailable) */
+#define        PROCESSOR_IDLE                  4       /* Idle (available) */
+#define PROCESSOR_DISPATCHING  5       /* Dispatching (idle -> active) */
+#define        PROCESSOR_RUNNING               6       /* Normal execution */
+
+extern processor_t     current_processor(void);
+
+/* Lock macros, always acquired and released with interrupts disabled (splsched()) */
+
+#if __SMP__
+#define pset_lock(p)                   simple_lock(&(p)->sched_lock)
+#define pset_unlock(p)                 simple_unlock(&(p)->sched_lock)
+#define pset_lock_init(p)              simple_lock_init(&(p)->sched_lock, 0)
+
+#define rt_lock_lock(p)                        simple_lock(&SCHED(rt_runq)(p)->rt_lock)
+#define rt_lock_unlock(p)              simple_unlock(&SCHED(rt_runq)(p)->rt_lock)
+#define rt_lock_init(p)                        simple_lock_init(&SCHED(rt_runq)(p)->rt_lock, 0)
+#else
+#define pset_lock(p)                   do { (void)p; } while(0)
+#define pset_unlock(p)                 do { (void)p; } while(0)
+#define pset_lock_init(p)              do { (void)p; } while(0)
+
+#define rt_lock_lock(p)                        do { (void)p; } while(0)
+#define rt_lock_unlock(p)              do { (void)p; } while(0)
+#define rt_lock_init(p)                        do { (void)p; } while(0)
+#endif
+
+extern void            processor_bootstrap(void);
+
+extern void            processor_init(
+                                       processor_t             processor,
+                                       int                             cpu_id,
+                                       processor_set_t processor_set);
+
+extern void            processor_set_primary(
+                                       processor_t             processor,
+                                       processor_t             primary);

 
 

-/*
- *     Use processor ptr array to find current processor's data structure.
- *     This replaces a multiplication (index into processor_array) with
- *     an array lookup and a memory reference.  It also allows us to save
- *     space if processor numbering gets too sparse.
- */
-
-extern processor_t     processor_ptr[NCPUS];
-
-#define cpu_to_processor(i)    (processor_ptr[i])
-
-#define current_processor()    (processor_ptr[cpu_number()])
-
-/* Compatibility -- will go away */
+extern kern_return_t   processor_shutdown(
+                                                       processor_t             processor);

 
 

-#define cpu_state(slot_num)    (processor_ptr[slot_num]->state)
-#define cpu_idle(slot_num)     (cpu_state(slot_num) == PROCESSOR_IDLE)
+extern void            processor_queue_shutdown(
+                                       processor_t             processor);

 
 

-/* Useful lock macros */
+extern processor_set_t processor_pset(
+                                                       processor_t             processor);

 
 

-#define        pset_lock(pset)         mutex_lock(&(pset)->lock)
-#define        pset_lock_try(pset)     mutex_try(&(pset)->lock)
-#define pset_unlock(pset)      mutex_unlock(&(pset)->lock)
+extern pset_node_t             pset_node_root(void);

 
 

-#define processor_lock(pr)     simple_lock(&(pr)->lock)
-#define processor_unlock(pr)   simple_unlock(&(pr)->lock)
+extern processor_set_t pset_create(
+                                                       pset_node_t             node);

 
 

-extern void            pset_sys_bootstrap(void);
+extern void            pset_init(
+                                       processor_set_t         pset,
+                                       pset_node_t                     node);

 
 

-#define timeshare_quanta_update(pset)                                  \
-MACRO_BEGIN                                                                                            \
-       int             proc_count = (pset)->processor_count;           \
-       int             runq_count = (pset)->runq.count;                        \
-                                                                                                               \
-       (pset)->timeshare_quanta = (pset)->quantum_factors[     \
-                                       (runq_count > proc_count)?                      \
-                                                       proc_count: runq_count];        \
-MACRO_END
+extern processor_set_t pset_find(
+                                       uint32_t cluster_id,
+                                       processor_set_t default_pset);

 
 

-#define pset_run_incr(pset)                                    \
-       hw_atomic_add(&(pset)->run_count, 1)
+extern kern_return_t   processor_info_count(
+                                                       processor_flavor_t              flavor,
+                                                       mach_msg_type_number_t  *count);

 
 

-#define pset_run_decr(pset)                                    \
-       hw_atomic_sub(&(pset)->run_count, 1)
+#define pset_deallocate(x)
+#define pset_reference(x)

 
 

-#define pset_share_incr(pset)                          \
-       hw_atomic_add(&(pset)->share_count, 1)
+extern void                            machine_run_count(
+                                                       uint32_t        count);

 
 

-#define pset_share_decr(pset)                          \
-       hw_atomic_sub(&(pset)->share_count, 1)
+extern processor_t             machine_choose_processor(
+                                                       processor_set_t         pset,
+                                                       processor_t                     processor);

 
 

-extern void            cpu_up(
-                                       int             cpu);
+#define next_pset(p)   (((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets)

 
 

-extern kern_return_t   processor_shutdown(
-                                                       processor_t             processor);
+#define PSET_THING_TASK                0
+#define PSET_THING_THREAD      1

 
 

-extern void            pset_remove_processor(
-                                       processor_set_t         pset,
-                                       processor_t                     processor);
+extern kern_return_t   processor_set_things(
+                       processor_set_t pset,
+                       void **thing_list,
+                       mach_msg_type_number_t *count,
+                       int type);

 
 

-extern void            pset_add_processor(
-                                       processor_set_t         pset,
-                                       processor_t                     processor);
+extern pset_cluster_type_t recommended_pset_type(thread_t thread);

 
 

-extern void            pset_remove_task(
-                                       processor_set_t         pset,
-                                       task_t                          task);
+inline static bool
+pset_is_recommended(processor_set_t pset)
+{
+       return ((pset->recommended_bitmask & pset->cpu_bitmask) != 0);
+}

 
 

-extern void            pset_add_task(
-                                       processor_set_t         pset,
-                                       task_t                          task);
+extern void processor_state_update_idle(processor_t processor);
+extern void processor_state_update_from_thread(processor_t processor, thread_t thread);
+extern void processor_state_update_explicit(processor_t processor, int pri,
+       sfi_class_id_t sfi_class, pset_cluster_type_t pset_type, 
+       perfcontrol_class_t perfctl_class);

 
 

-extern void            pset_remove_thread(
-                                       processor_set_t         pset,
-                                       thread_t                        thread);
+#else  /* MACH_KERNEL_PRIVATE */

 
 

-extern void            pset_add_thread(
-                                       processor_set_t         pset,
-                                       thread_t                        thread);
+__BEGIN_DECLS

 
 

-extern void            thread_change_psets(
-                                       thread_t                        thread,
-                                       processor_set_t         old_pset,
-                                       processor_set_t         new_pset);
+extern void            pset_deallocate(
+                                       processor_set_t pset);

 
 

-extern kern_return_t   processor_assign(
-                                                       processor_t                     processor,
-                                                       processor_set_t         new_pset,
-                                                       boolean_t                       wait);
+extern void            pset_reference(
+                                       processor_set_t pset);

 
 

-extern kern_return_t   processor_info_count(
-                                                       processor_flavor_t              flavor,
-                                                       mach_msg_type_number_t  *count);
+__END_DECLS

 
 #endif /* MACH_KERNEL_PRIVATE */
 
 
 #endif /* MACH_KERNEL_PRIVATE */
 

-extern kern_return_t   processor_start(
-                                                       processor_t             processor);
+#ifdef KERNEL_PRIVATE
+__BEGIN_DECLS
+extern unsigned int            processor_count;
+extern processor_t     cpu_to_processor(int cpu);
+__END_DECLS

 
 

-extern kern_return_t   processor_exit(
-                                                       processor_t             processor);
-
-#endif /* __APPLE_API_PRIVATE */
-
-extern void            pset_deallocate(
-                                       processor_set_t pset);
-
-extern void            pset_reference(
-                                       processor_set_t pset);
+#endif /* KERNEL_PRIVATE */

 
 #endif /* _KERN_PROCESSOR_H_ */
 
 #endif /* _KERN_PROCESSOR_H_ */