/*
- * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
processor_t runq; /* run queue assignment */
wait_queue_t wait_queue; /* wait queue we are currently on */
event64_t wait_event; /* wait queue event */
- integer_t options; /* options set by thread itself */
-#define TH_OPT_INTMASK 0x03 /* interrupt / abort level */
-#define TH_OPT_VMPRIV 0x04 /* may allocate reserved memory */
-#define TH_OPT_DTRACE 0x08 /* executing under dtrace_probe */
-
/* Data updated during assert_wait/thread_wakeup */
decl_simple_lock_data(,sched_lock) /* scheduling lock (thread_lock()) */
decl_simple_lock_data(,wake_lock) /* for thread stop / wait (wake_lock()) */
+ integer_t options; /* options set by thread itself */
+#define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */
+#define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */
+#define TH_OPT_DTRACE 0x0008 /* executing under dtrace_probe */
+#define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */
+#define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */
+#define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */
+#define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */
+
boolean_t wake_active; /* wake event on stop */
int at_safe_point; /* thread_abort_safely allowed */
ast_t reason; /* why we blocked */
+ thread_continue_t continuation; /* continue here next dispatch */
+ void *parameter; /* continuation parameter */
wait_result_t wait_result; /* outcome of wait -
* may be examined by this thread
* WITHOUT locking */
- thread_continue_t continuation; /* continue here next dispatch */
- void *parameter; /* continuation parameter */
/* Data updated/used in thread_invoke */
- struct funnel_lock *funnel_lock; /* Non-reentrancy funnel */
int funnel_state;
+ struct funnel_lock *funnel_lock; /* Non-reentrancy funnel */
#define TH_FN_OWNED 0x1 /* we own the funnel */
#define TH_FN_REFUNNEL 0x2 /* re-acquire funnel on dispatch */
#define TH_IDLE 0x80 /* idling processor */
/* Scheduling information */
- integer_t sched_mode; /* scheduling mode bits */
-#define TH_MODE_REALTIME 0x0001 /* time constraints supplied */
-#define TH_MODE_TIMESHARE 0x0002 /* use timesharing algorithm */
-#define TH_MODE_FAILSAFE 0x0004 /* fail-safe has tripped */
-#define TH_MODE_PROMOTED 0x0008 /* sched pri has been promoted */
-#define TH_MODE_ABORT 0x0010 /* abort interruptible waits */
-#define TH_MODE_ABORTSAFELY 0x0020 /* ... but only those at safe point */
-#define TH_MODE_ISABORTED (TH_MODE_ABORT | TH_MODE_ABORTSAFELY)
-#define TH_MODE_DEPRESS 0x0040 /* normal depress yield */
-#define TH_MODE_POLLDEPRESS 0x0080 /* polled depress yield */
-#define TH_MODE_ISDEPRESSED (TH_MODE_DEPRESS | TH_MODE_POLLDEPRESS)
-
- integer_t sched_pri; /* scheduled (current) priority */
- integer_t priority; /* base priority */
- integer_t max_priority; /* max base priority */
- integer_t task_priority; /* copy of task base priority */
-
- integer_t promotions; /* level of promotion */
- integer_t pending_promoter_index;
+ sched_mode_t sched_mode; /* scheduling mode */
+ sched_mode_t saved_mode; /* saved mode during forced mode demotion */
+
+ unsigned int sched_flags; /* current flag bits */
+#define TH_SFLAG_FAIRSHARE_TRIPPED 0x0001 /* fairshare scheduling activated */
+#define TH_SFLAG_FAILSAFE 0x0002 /* fail-safe has tripped */
+#define TH_SFLAG_THROTTLED 0x0004 /* owner task in throttled state */
+#define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE | TH_SFLAG_FAIRSHARE_TRIPPED)
+
+#define TH_SFLAG_PROMOTED 0x0008 /* sched pri has been promoted */
+#define TH_SFLAG_ABORT 0x0010 /* abort interruptible waits */
+#define TH_SFLAG_ABORTSAFELY 0x0020 /* ... but only those at safe point */
+#define TH_SFLAG_ABORTED_MASK (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
+#define TH_SFLAG_DEPRESS 0x0040 /* normal depress yield */
+#define TH_SFLAG_POLLDEPRESS 0x0080 /* polled depress yield */
+#define TH_SFLAG_DEPRESSED_MASK (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
+#define TH_SFLAG_PRI_UPDATE 0x0100 /* Updating priority */
+#define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */
+#define TH_SFLAG_RW_PROMOTED 0x0400 /* sched pri has been promoted due to blocking with RW lock held */
+#define TH_SFLAG_PROMOTED_MASK (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED)
+
+#define TH_SFLAG_RW_PROMOTED_BIT (10) /* 0x400 */
+
+/*
+ * A thread can either be completely unthrottled, about to be throttled,
+ * throttled (TH_SFLAG_THROTTLED), or about to be unthrottled
+ */
+#define TH_SFLAG_PENDING_THROTTLE_DEMOTION 0x1000 /* Pending sched_mode demotion */
+#define TH_SFLAG_PENDING_THROTTLE_PROMOTION 0x2000 /* Pending sched_mode promition */
+#define TH_SFLAG_PENDING_THROTTLE_MASK (TH_SFLAG_PENDING_THROTTLE_DEMOTION | TH_SFLAG_PENDING_THROTTLE_PROMOTION)
+
+ int16_t sched_pri; /* scheduled (current) priority */
+ int16_t priority; /* base priority */
+ int16_t max_priority; /* max base priority */
+ int16_t task_priority; /* copy of task base priority */
+#if defined(CONFIG_SCHED_GRRR)
+#if 0
+ uint16_t grrr_deficit; /* fixed point (1/1000th quantum) fractional deficit */
+#endif
+#endif
+
+ int16_t promotions; /* level of promotion */
+ int16_t pending_promoter_index;
+ uint32_t ref_count; /* number of references to me */
void *pending_promoter[2];
- integer_t importance; /* task-relative importance */
+ uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */
- /* real-time parameters */
+ integer_t importance; /* task-relative importance */
+ /* Priority depression expiration */
+ integer_t depress_timer_active;
+ timer_call_data_t depress_timer;
+ /* real-time parameters */
struct { /* see mach/thread_policy.h */
uint32_t period;
uint32_t computation;
uint32_t constraint;
boolean_t preemptible;
-
uint64_t deadline;
} realtime;
+ uint32_t was_promoted_on_wakeup;
uint32_t current_quantum; /* duration of current quantum */
+ uint64_t last_run_time; /* time when thread was switched away from */
+ uint64_t last_quantum_refill_time; /* time when current_quantum was refilled after expiration */
/* Data used during setrun/dispatch */
timer_data_t system_timer; /* system mode timer */
processor_t bound_processor; /* bound to a processor? */
processor_t last_processor; /* processor last dispatched on */
+ processor_t chosen_processor; /* Where we want to run this thread */
/* Fail-safe computation since last unblock or qualifying yield */
uint64_t computation_metered;
uint64_t computation_epoch;
- integer_t safe_mode; /* saved mode during fail-safe */
- natural_t safe_release; /* when to release fail-safe */
+ uint64_t safe_release; /* when to release fail-safe */
/* Call out from scheduler */
void (*sched_call)(
int type,
thread_t thread);
-
+#if defined(CONFIG_SCHED_PROTO)
+ uint32_t runqueue_generation; /* last time runqueue was drained */
+#endif
+
/* Statistics and timesharing calculations */
+#if defined(CONFIG_SCHED_TRADITIONAL)
natural_t sched_stamp; /* last scheduler tick */
natural_t sched_usage; /* timesharing cpu usage [sched] */
natural_t pri_shift; /* usage -> priority from pset */
natural_t cpu_usage; /* instrumented cpu usage [%cpu] */
natural_t cpu_delta; /* accumulated cpu_usage delta */
+#endif
uint32_t c_switch; /* total context switches */
uint32_t p_switch; /* total processor switches */
uint32_t ps_switch; /* total pset switches */
/* Timing data structures */
+ int precise_user_kernel_time; /* precise user/kernel enabled for this thread */
timer_data_t user_timer; /* user mode timer */
uint64_t user_timer_save; /* saved user timer value */
uint64_t system_timer_save; /* saved system timer value */
integer_t wait_timer_active;
boolean_t wait_timer_is_set;
- /* Priority depression expiration */
- timer_call_data_t depress_timer;
- integer_t depress_timer_active;
/*
* Processor/cache affinity
union {
struct {
mach_msg_return_t state; /* receive state */
+ mach_port_seqno_t seqno; /* seqno of recvd message */
ipc_object_t object; /* object received on */
mach_vm_address_t msg_addr; /* receive buffer pointer */
mach_msg_size_t msize; /* max size for recvd msg */
mach_msg_size_t slist_size; /* scatter list size */
mach_port_name_t receiver_name; /* the receive port name */
struct ipc_kmsg *kmsg; /* received message */
- mach_port_seqno_t seqno; /* seqno of recvd message */
mach_msg_continue_t continuation;
} receive;
struct {
} sema;
struct {
int option; /* switch option */
+ boolean_t reenable_workq_callback; /* on entry, callbacks were suspended */
} swtch;
int misc; /* catch-all for other state */
} saved;
+ /* Structure to save information about guard exception */
+ struct {
+ unsigned type; /* EXC_GUARD reason/type */
+ mach_exception_data_type_t code; /* Exception code */
+ mach_exception_data_type_t subcode; /* Exception sub-code */
+ } guard_exc_info;
+
+
/* IPC data structures */
- struct ipc_kmsg_queue ith_messages;
+#if IMPORTANCE_INHERITANCE
+ natural_t ith_assertions; /* assertions pending drop */
+#endif
+ struct ipc_kmsg_queue ith_messages; /* messages to reap */
mach_port_t ith_rpc_reply; /* reply port for kernel RPCs */
/* Ast/Halt data structures */
vm_offset_t recover; /* page fault recover(copyin/out) */
- uint32_t ref_count; /* number of references to me */
queue_chain_t threads; /* global list of all threads */
decl_lck_mtx_data(,mutex)
- /* Kernel holds on this thread */
- int suspend_count;
-
- /* User level suspensions */
- int user_stop_count;
/* Pending thread ast(s) */
ast_t ast;
/* Ports associated with this thread */
struct ipc_port *ith_self; /* not a right, doesn't hold ref */
struct ipc_port *ith_sself; /* a send right */
- struct exception_action exc_actions[EXC_TYPES_COUNT];
-
- /* Owned ulocks (a lock set element) */
- queue_head_t held_ulocks;
+ struct exception_action *exc_actions;
#ifdef MACH_BSD
void *uthread;
#endif
#if CONFIG_DTRACE
+ uint32_t t_dtrace_flags; /* DTrace thread states */
+#define TH_DTRACE_EXECSUCCESS 0x01
uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
int64_t t_dtrace_tracing; /* Thread time under dtrace_probe() */
int64_t t_dtrace_vtime;
#endif
-#define T_CHUD_MARKED 0x1 /* this thread is marked by CHUD */
-#define T_IN_CHUD 0x2 /* this thread is already in a CHUD handler */
-#define THREAD_PMC_FLAG 0x4 /* Bit in "t_chud" signifying PMC interest */
- uint32_t t_page_creation_count;
clock_sec_t t_page_creation_time;
+ uint32_t t_page_creation_count;
+
+#define T_CHUD_MARKED 0x01 /* this thread is marked by CHUD */
+#define T_IN_CHUD 0x02 /* this thread is already in a CHUD handler */
+#define THREAD_PMC_FLAG 0x04 /* Bit in "t_chud" signifying PMC interest */
+#define T_AST_CALLSTACK 0x08 /* Thread scheduled to dump a
+ * callstack on its next
+ * AST */
+#define T_AST_NAME 0x10 /* Thread scheduled to dump
+ * its name on its next
+ * AST */
+#define T_NAME_DONE 0x20 /* Thread has previously
+ * recorded its name */
+#define T_KPC_ALLOC 0x40 /* Thread needs a kpc_buf */
uint32_t t_chud; /* CHUD flags, used for Shark */
+ uint32_t chud_c_switch; /* last dispatch detection */
+
+ integer_t mutex_count; /* total count of locks held */
+
+#ifdef KPC
+ /* accumulated performance counters for this thread */
+ uint64_t *kpc_buf;
+#endif
+
+#ifdef KPERF
+ /* count of how many times a thread has been sampled since it was last scheduled */
+ uint64_t kperf_pet_cnt;
+#endif
+
uint64_t thread_id; /*system wide unique thread-id*/
+
+ /* Statistics accumulated per-thread and aggregated per-task */
+ uint32_t syscalls_unix;
+ uint32_t syscalls_mach;
+ ledger_t t_ledger;
+ ledger_t t_threadledger; /* per thread ledger */
+
+ /* policy is protected by the task lock */
+ struct task_requested_policy requested_policy;
+ struct task_effective_policy effective_policy;
+ struct task_pended_policy pended_policy;
+
+ int iotier_override; /* atomic operations to set, cleared on ret to user */
+
+
+ integer_t saved_importance; /* saved task-relative importance */
+
+ uint32_t thread_callout_interrupt_wakeups;
+ uint32_t thread_callout_platform_idle_wakeups;
+ uint32_t thread_timer_wakeups_bin_1;
+ uint32_t thread_timer_wakeups_bin_2;
+ uint16_t thread_tag;
+ uint16_t callout_woken_from_icontext:1,
+ callout_woken_from_platform_idle:1,
+ callout_woke_thread:1,
+ thread_bitfield_unused:13;
+ /* Kernel holds on this thread */
+ int16_t suspend_count;
+ /* User level suspensions */
+ int16_t user_stop_count;
};
#define ith_state saved.receive.state
#define sth_result saved.sema.result
#define sth_continuation saved.sema.continuation
-extern void thread_bootstrap(void) __attribute__((section("__TEXT, initcode")));
+extern void thread_bootstrap(void);
-extern void thread_init(void) __attribute__((section("__TEXT, initcode")));
+extern void thread_init(void);
extern void thread_daemon_init(void);
extern void stack_alloc(
thread_t thread);
+extern void stack_handoff(
+ thread_t from,
+ thread_t to);
+
extern void stack_free(
thread_t thread);
-extern void stack_free_stack(
- vm_offset_t stack);
+extern void stack_free_reserved(
+ thread_t thread);
extern boolean_t stack_alloc_try(
thread_t thread);
extern void stack_collect(void);
-extern void stack_init(void) __attribute__((section("__TEXT, initcode")));
-
-extern kern_return_t thread_state_initialize(
- thread_t thread);
-
-extern kern_return_t thread_setstatus(
- thread_t thread,
- int flavor,
- thread_state_t tstate,
- mach_msg_type_number_t count);
+extern void stack_init(void);
-extern kern_return_t thread_getstatus(
- thread_t thread,
- int flavor,
- thread_state_t tstate,
- mach_msg_type_number_t *count);
extern kern_return_t thread_info_internal(
thread_t thread,
extern void machine_set_current_thread(
thread_t thread);
-extern void machine_thread_terminate_self(void);
-
extern kern_return_t machine_thread_get_kern_state(
thread_t thread,
thread_flavor_t flavor,
extern void funnel_unlock(
struct funnel_lock *lock);
+
+static inline uint16_t thread_set_tag_internal(thread_t thread, uint16_t tag) {
+ return __sync_fetch_and_or(&thread->thread_tag, tag);
+}
+
+static inline uint16_t thread_get_tag_internal(thread_t thread) {
+ return thread->thread_tag;
+}
+
#else /* MACH_KERNEL_PRIVATE */
__BEGIN_DECLS
__BEGIN_DECLS
-#ifndef __LP64__
-
-extern thread_t kernel_thread(
- task_t task,
- void (*start)(void));
-
-#endif /* __LP64__ */
extern uint64_t thread_tid(
thread_t thread);
#ifdef XNU_KERNEL_PRIVATE
+/*
+ * Thread tags; for easy identification.
+ */
+#define THREAD_TAG_MAINTHREAD 0x1
+#define THREAD_TAG_CALLOUT 0x2
+#define THREAD_TAG_IOWORKLOOP 0x4
+
+uint16_t thread_set_tag(thread_t, uint16_t);
+uint16_t thread_get_tag(thread_t);
+
+
+extern kern_return_t thread_state_initialize(
+ thread_t thread);
+
+extern kern_return_t thread_setstatus(
+ thread_t thread,
+ int flavor,
+ thread_state_t tstate,
+ mach_msg_type_number_t count);
+
+extern kern_return_t thread_getstatus(
+ thread_t thread,
+ int flavor,
+ thread_state_t tstate,
+ mach_msg_type_number_t *count);
+
extern kern_return_t thread_create_workq(
task_t task,
thread_continue_t thread_return,
extern void thread_yield_internal(
mach_msg_timeout_t interval);
+/*
+ * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
+ *
+ * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
+ * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
+ * 3) Disable. Remove any existing CPU limit.
+ */
+#define THREAD_CPULIMIT_BLOCK 0x1
+#define THREAD_CPULIMIT_EXCEPTION 0x2
+#define THREAD_CPULIMIT_DISABLE 0x3
+
+struct _thread_ledger_indices {
+ int cpu_time;
+};
+
+extern struct _thread_ledger_indices thread_ledgers;
+
+extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
+extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
+
typedef struct funnel_lock funnel_t;
#define THR_FUNNEL_NULL (funnel_t *)0
thread_t thread,
int on);
+extern kern_return_t thread_userstack(
+ thread_t,
+ int,
+ thread_state_t,
+ unsigned int,
+ mach_vm_offset_t *,
+ int *);
+
+extern kern_return_t thread_entrypoint(
+ thread_t,
+ int,
+ thread_state_t,
+ unsigned int,
+ mach_vm_offset_t *);
+
+extern kern_return_t thread_userstackdefault(
+ thread_t,
+ mach_vm_offset_t *);
+
extern kern_return_t thread_wire_internal(
host_priv_t host_priv,
thread_t thread,
extern boolean_t thread_should_halt(
thread_t thread);
+extern boolean_t thread_should_abort(
+ thread_t);
+
extern int is_64signalregset(void);
void act_set_apc(thread_t);
+void act_set_kperf(thread_t);
extern uint32_t dtrace_get_thread_predcache(thread_t);
extern int64_t dtrace_get_thread_vtime(thread_t);
extern void dtrace_set_thread_reentering(thread_t, boolean_t);
extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
extern void dtrace_thread_bootstrap(void);
+extern void dtrace_thread_didexec(thread_t);
extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
-extern void thread_set_wq_state32(
+extern kern_return_t thread_set_wq_state32(
thread_t thread,
thread_state_t tstate);
-extern void thread_set_wq_state64(
+extern kern_return_t thread_set_wq_state64(
thread_t thread,
thread_state_t tstate);
extern vm_offset_t kernel_stack_size;
extern vm_offset_t kernel_stack_depth_max;
+void guard_ast(thread_t thread);
+extern void fd_guard_ast(thread_t thread);
+extern void mach_port_guard_ast(thread_t thread);
+extern void thread_guard_violation(thread_t thread, unsigned type);
+
#endif /* XNU_KERNEL_PRIVATE */
/*! @function kernel_thread_start
thread_continue_t continuation,
void *parameter,
thread_t *new_thread);
+#ifdef KERNEL_PRIVATE
+void thread_set_eager_preempt(thread_t thread);
+void thread_clear_eager_preempt(thread_t thread);
+extern ipc_port_t convert_thread_to_port(thread_t);
+#endif /* KERNEL_PRIVATE */
__END_DECLS
projects / apple / xnu.git / blobdiff