#include <kern/debug.h>
#include <kern/block_hint.h>
#include <kern/turnstile.h>
+#include <kern/mpsc_queue.h>
#include <kern/waitq.h>
#include <san/kasan.h>
#include <ipc/ipc_kmsg.h>
+#include <machine/atomic.h>
#include <machine/cpu_data.h>
#include <machine/thread.h>
union {
queue_chain_t runq_links; /* run queue links */
queue_chain_t wait_links; /* wait queue links */
+ struct mpsc_queue_chain mpsc_links; /* thread daemon mpsc links */
struct priority_queue_entry wait_prioq_links; /* priority ordered waitq links */
};
- processor_t runq; /* run queue assignment */
-
event64_t wait_event; /* wait queue event */
+ processor_t runq; /* run queue assignment */
struct waitq *waitq; /* wait queue this thread is enqueued on */
struct turnstile *turnstile; /* thread's turnstile, protected by primitives interlock */
void *inheritor; /* inheritor of the primitive the thread will block on */
- struct priority_queue inheritor_queue; /* Inheritor queue */
+ struct priority_queue sched_inheritor_queue; /* Inheritor queue for kernel promotion */
+ struct priority_queue base_inheritor_queue; /* Inheritor queue for user promotion */
+
+#if CONFIG_SCHED_CLUTCH
+ /*
+ * In the clutch scheduler, the threads are maintained in runqs at the clutch_bucket
+ * level (clutch_bucket defines a unique thread group and scheduling bucket pair). In
+ * order to determine the priority of the clutch bucket as a whole, it is necessary to
+ * find the highest thread in it. The thread could be present in the clutch bucket due
+ * to its base_pri or its promoted pri. This link is used to maintain that queue.
+ */
+ struct priority_queue_entry sched_clutchpri_link;
+
+#endif /* CONFIG_SCHED_CLUTCH */
/* Data updated during assert_wait/thread_wakeup */
#if __SMP__
- decl_simple_lock_data(, sched_lock) /* scheduling lock (thread_lock()) */
- decl_simple_lock_data(, wake_lock) /* for thread stop / wait (wake_lock()) */
+ decl_simple_lock_data(, sched_lock); /* scheduling lock (thread_lock()) */
+ decl_simple_lock_data(, wake_lock); /* for thread stop / wait (wake_lock()) */
#endif
- integer_t options; /* options set by thread itself */
+ uint16_t options; /* options set by thread itself */
#define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */
#define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */
-#define TH_OPT_DTRACE 0x0008 /* executing under dtrace_probe */
#define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */
#define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */
#define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */
#define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */
#define TH_OPT_ZONE_GC 0x1000 /* zone_gc() called on this thread */
- boolean_t wake_active; /* wake event on stop */
- int at_safe_point; /* thread_abort_safely allowed */
+ bool wake_active; /* wake event on stop */
+ bool at_safe_point; /* thread_abort_safely allowed */
ast_t reason; /* why we blocked */
uint32_t quantum_remaining;
wait_result_t wait_result; /* outcome of wait -
struct kasan_thread_data kasan_data;
#endif
+#if CONFIG_KSANCOV
+ void *ksancov_data;
+#endif
+
/* Thread state: */
int state;
/*
/* unused TH_SFLAG_PRI_UPDATE 0x0100 */
#define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */
#define TH_SFLAG_RW_PROMOTED 0x0400 /* promote reason: blocking with RW lock held */
-/* unused TH_SFLAG_THROTTLE_DEMOTED 0x0800 */
+#define TH_SFLAG_BASE_PRI_FROZEN 0x0800 /* (effective) base_pri is frozen */
#define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* promote reason: waitq wakeup (generally for IPC receive) */
#define TH_SFLAG_RW_PROMOTED_BIT (10) /* 0x400 */
int16_t sched_pri; /* scheduled (current) priority */
- int16_t base_pri; /* base priority */
+ int16_t base_pri; /* effective base priority (equal to req_base_pri unless TH_SFLAG_BASE_PRI_FROZEN) */
+ int16_t req_base_pri; /* requested base priority */
int16_t max_priority; /* copy of max base priority */
int16_t task_priority; /* copy of task base priority */
int16_t promotion_priority; /* priority thread is currently promoted to */
#endif
#endif
- int16_t promotions; /* level of promotion */
int iotier_override; /* atomic operations to set, cleared on ret to user */
- struct os_refcnt ref_count; /* number of references to me */
+ os_refcnt_t ref_count; /* number of references to me */
lck_mtx_t* waiting_for_mutex; /* points to mutex we're waiting for until we acquire it */
uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */
integer_t importance; /* task-relative importance */
- uint32_t was_promoted_on_wakeup; /* thread promoted on wakeup to acquire mutex */
/* Priority depression expiration */
integer_t depress_timer_active;
uint64_t safe_release; /* when to release fail-safe */
/* Call out from scheduler */
- void (*sched_call)(
- int type,
- thread_t thread);
+ void (*sched_call)(int type, thread_t thread);
+
#if defined(CONFIG_SCHED_PROTO)
uint32_t runqueue_generation; /* last time runqueue was drained */
#endif
uint64_t wait_sfi_begin_time; /* start time for thread waiting in SFI */
#endif
- /* Timed wait expiration */
- timer_call_data_t wait_timer;
- integer_t wait_timer_active;
- boolean_t wait_timer_is_set;
-
-
/*
* Processor/cache affinity
* - affinity_threads links task threads with the same affinity set
*/
- affinity_set_t affinity_set;
queue_chain_t affinity_threads;
+ affinity_set_t affinity_set;
+
+#if CONFIG_EMBEDDED
+ task_watch_t * taskwatch; /* task watch */
+#endif /* CONFIG_EMBEDDED */
/* Various bits of state to stash across a continuation, exclusive to the current thread block point */
union {
mach_msg_return_t state; /* receive state */
mach_port_seqno_t seqno; /* seqno of recvd message */
ipc_object_t object; /* object received on */
- mach_vm_address_t msg_addr; /* receive buffer pointer */
+ vm_address_t msg_addr; /* receive buffer pointer */
mach_msg_size_t rsize; /* max size for recvd msg */
mach_msg_size_t msize; /* actual size for recvd msg */
mach_msg_option_t option; /* options for receive */
kern_return_t result; /* primary result */
mach_msg_continue_t continuation;
} sema;
+ struct {
+#define THREAD_SAVE_IOKIT_TLS_COUNT 8
+ void *tls[THREAD_SAVE_IOKIT_TLS_COUNT];
+ } iokit;
} saved;
/* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
struct ipc_kmsg_queue ith_messages; /* messages to reap */
mach_port_t ith_rpc_reply; /* reply port for kernel RPCs */
+ /* Pending thread ast(s) */
+ ast_t ast;
+
/* Ast/Halt data structures */
- vm_offset_t recover; /* page fault recover(copyin/out) */
+ vm_offset_t recover; /* page fault recover(copyin/out) */
queue_chain_t threads; /* global list of all threads */
/* Activation */
- queue_chain_t task_threads;
+ queue_chain_t task_threads;
/* Task membership */
struct task *task;
vm_map_t map;
#if DEVELOPMENT || DEBUG
- boolean_t pmap_footprint_suspended;
+ bool pmap_footprint_suspended;
#endif /* DEVELOPMENT || DEBUG */
- decl_lck_mtx_data(, mutex)
-
-
- /* Pending thread ast(s) */
- ast_t ast;
+ /* Timed wait expiration */
+ timer_call_data_t wait_timer;
+ uint16_t wait_timer_active;
+ bool wait_timer_is_set;
/* Miscellaneous bits guarded by mutex */
uint32_t
corpse_dup:1, /* TRUE when thread is an inactive duplicate in a corpse */
:0;
+ decl_lck_mtx_data(, mutex);
+
/* Ports associated with this thread */
struct ipc_port *ith_self; /* not a right, doesn't hold ref */
struct ipc_port *ith_sself; /* a send right */
#endif
#if CONFIG_DTRACE
- uint32_t t_dtrace_flags; /* DTrace thread states */
+ uint16_t t_dtrace_flags; /* DTrace thread states */
#define TH_DTRACE_EXECSUCCESS 0x01
+ uint16_t t_dtrace_inprobe; /* Executing under dtrace_probe */
uint32_t t_dtrace_predcache; /* DTrace per thread predicate value hint */
int64_t t_dtrace_tracing; /* Thread time under dtrace_probe() */
int64_t t_dtrace_vtime;
uint64_t t_page_creation_throttled_hard;
uint64_t t_page_creation_throttled_soft;
#endif /* DEVELOPMENT || DEBUG */
+ int t_pagein_error; /* for vm_fault(), holds error from vnop_pagein() */
#ifdef KPERF
-/* The high 7 bits are the number of frames to sample of a user callstack. */
-#define T_KPERF_CALLSTACK_DEPTH_OFFSET (25)
+/* The high 8 bits are the number of frames to sample of a user callstack. */
+#define T_KPERF_CALLSTACK_DEPTH_OFFSET (24)
#define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
#define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#define T_KPERF_ACTIONID_OFFSET (18)
+#define T_KPERF_SET_ACTIONID(AID) (((uint32_t)(AID)) << T_KPERF_ACTIONID_OFFSET)
+#define T_KPERF_GET_ACTIONID(FLAGS) ((FLAGS) >> T_KPERF_ACTIONID_OFFSET)
#endif
-#define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
-#define T_KPERF_AST_DISPATCH (1U << 1) /* dump a name on thread's next AST */
-#define T_KPC_ALLOC (1U << 2) /* thread needs a kpc_buf allocated */
-/* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
+#define T_KPERF_AST_CALLSTACK 0x1 /* dump a callstack on thread's next AST */
+#define T_KPERF_AST_DISPATCH 0x2 /* dump a name on thread's next AST */
+#define T_KPC_ALLOC 0x4 /* thread needs a kpc_buf allocated */
+
+#define T_KPERF_AST_ALL \
+ (T_KPERF_AST_CALLSTACK | T_KPERF_AST_DISPATCH | T_KPC_ALLOC)
+/* only go up to T_KPERF_ACTIONID_OFFSET - 1 */
#ifdef KPERF
- uint32_t kperf_flags;
+ uint32_t kperf_ast;
uint32_t kperf_pet_gen; /* last generation of PET that sampled this thread*/
uint32_t kperf_c_switch; /* last dispatch detection */
uint32_t kperf_pet_cnt; /* how many times a thread has been sampled by PET */
void *hv_thread_target;
#endif /* HYPERVISOR */
- uint64_t thread_id; /*system wide unique thread-id*/
-
/* Statistics accumulated per-thread and aggregated per-task */
uint32_t syscalls_unix;
uint32_t syscalls_mach;
uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
uint64_t t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
+ uint64_t thread_id; /*system wide unique thread-id*/
+
#if MONOTONIC
struct mt_thread t_monotonic;
#endif /* MONOTONIC */
user_addr_t override_resource;
} *overrides;
- uint32_t ipc_overrides;
- _Atomic uint32_t kqwl_owning_count;
- uint32_t sync_ipc_overrides;
- uint16_t user_promotion_basepri;
+ uint32_t kevent_overrides;
+ uint8_t user_promotion_basepri;
+ uint8_t kern_promotion_schedpri;
_Atomic uint16_t kevent_ast_bits;
io_stat_info_t thread_io_stats; /* per-thread I/O statistics */
-#if CONFIG_EMBEDDED
- task_watch_t * taskwatch; /* task watch */
-#endif /* CONFIG_EMBEDDED */
-
uint32_t thread_callout_interrupt_wakeups;
uint32_t thread_callout_platform_idle_wakeups;
uint32_t thread_timer_wakeups_bin_1;
uint32_t thread_timer_wakeups_bin_2;
uint16_t thread_tag;
+ /*
+ * callout_* fields are only set for thread call threads whereas guard_exc_fatal is set
+ * by user threads on themselves while taking a guard exception. So it's okay for them to
+ * share this bitfield.
+ */
uint16_t callout_woken_from_icontext:1,
callout_woken_from_platform_idle:1,
callout_woke_thread:1,
- thread_bitfield_unused:13;
+ guard_exc_fatal:1,
+ thread_bitfield_unused:12;
mach_port_name_t ith_voucher_name;
ipc_voucher_t ith_voucher;
turnstile_update_flags_t inheritor_flags; /* inheritor flags for inheritor field */
block_hint_t pending_block_hint;
block_hint_t block_hint; /* What type of primitive last caused us to block. */
+ integer_t decompressions; /* Per-thread decompressions counter to be added to per-task decompressions counter */
};
#define ith_state saved.receive.state
extern void thread_deallocate(
thread_t thread);
-extern void thread_deallocate_safe(
- thread_t thread);
-
extern void thread_inspect_deallocate(
thread_inspect_t thread);
extern void thread_terminate_crashed_threads(void);
-extern void turnstile_deallocate_enqueue(
- struct turnstile *turnstile);
-
extern void thread_stack_enqueue(
thread_t thread);
extern void thread_release(
thread_t thread);
-extern void thread_corpse_continue(void);
+extern void thread_corpse_continue(void) __dead2;
extern boolean_t thread_is_active(thread_t thread);
extern void machine_load_context(
thread_t thread) __attribute__((noreturn));
-
extern kern_return_t machine_thread_state_initialize(
thread_t thread);
thread_state_t state,
mach_msg_type_number_t count);
+extern mach_vm_address_t machine_thread_pc(
+ thread_t thread);
+
+extern void machine_thread_reset_pc(
+ thread_t thread,
+ mach_vm_address_t pc);
+
+extern boolean_t machine_thread_on_core(
+ thread_t thread);
+
extern kern_return_t machine_thread_get_state(
thread_t thread,
thread_flavor_t flavor,
static inline uint16_t
thread_set_tag_internal(thread_t thread, uint16_t tag)
{
- return __sync_fetch_and_or(&thread->thread_tag, tag);
+ return os_atomic_or_orig(&thread->thread_tag, tag, relaxed);
}
static inline uint16_t
extern void thread_mtx_unlock(thread_t thread);
-extern thread_t current_thread(void);
+extern thread_t current_thread(void) __attribute__((const));
extern void thread_reference(
thread_t thread);
extern void thread_deallocate(
thread_t thread);
+#if BSD_KERNEL_PRIVATE
+/* Duplicated from osfmk/kern/ipc_tt.h */
+__options_decl(port_to_thread_options_t, uint32_t, {
+ PORT_TO_THREAD_NONE = 0x0000,
+ PORT_TO_THREAD_IN_CURRENT_TASK = 0x0001,
+ PORT_TO_THREAD_NOT_CURRENT_THREAD = 0x0002,
+});
+
+extern thread_t port_name_to_thread(
+ mach_port_name_t port_name,
+ port_to_thread_options_t options);
+#endif /* BSD_KERNEL_PRIVATE */
+
__END_DECLS
#endif /* MACH_KERNEL_PRIVATE */
__BEGIN_DECLS
-extern void thread_starts_owning_workloop(
- thread_t thread);
-
-extern void thread_ends_owning_workloop(
- thread_t thread);
-
-extern uint32_t thread_owned_workloops_count(
+extern void thread_deallocate_safe(
thread_t thread);
-
extern uint64_t thread_dispatchqaddr(
thread_t thread);
extern uint64_t thread_rettokern_addr(
thread_t thread);
+extern integer_t thread_kern_get_pri(thread_t thr) __attribute__((const));
+
+extern void thread_kern_set_pri(thread_t thr, integer_t pri);
+
+extern integer_t thread_kern_get_kernel_maxpri(void) __attribute__((const));
+
__END_DECLS
#endif /* KERNEL_PRIVATE */
#endif
extern void *get_bsdthread_info(thread_t);
extern void set_bsdthread_info(thread_t, void *);
+extern void set_thread_pagein_error(thread_t, int);
extern void *uthread_alloc(task_t, thread_t, int);
extern event_t workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/pthread/
extern void uthread_cleanup_name(void *uthread);
extern uint32_t dtrace_get_thread_predcache(thread_t);
extern int64_t dtrace_get_thread_vtime(thread_t);
extern int64_t dtrace_get_thread_tracing(thread_t);
-extern boolean_t dtrace_get_thread_reentering(thread_t);
+extern uint16_t dtrace_get_thread_inprobe(thread_t);
extern int dtrace_get_thread_last_cpu_id(thread_t);
extern vm_offset_t dtrace_get_kernel_stack(thread_t);
extern void dtrace_set_thread_predcache(thread_t, uint32_t);
extern void dtrace_set_thread_vtime(thread_t, int64_t);
extern void dtrace_set_thread_tracing(thread_t, int64_t);
-extern void dtrace_set_thread_reentering(thread_t, boolean_t);
+extern void dtrace_set_thread_inprobe(thread_t, uint16_t);
extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
extern vm_offset_t dtrace_sign_and_set_thread_recover(thread_t, vm_offset_t);
extern void dtrace_thread_bootstrap(void);
extern void virt_memory_guard_ast(thread_t,
mach_exception_code_t, mach_exception_subcode_t);
extern void thread_guard_violation(thread_t,
- mach_exception_code_t, mach_exception_subcode_t);
+ mach_exception_code_t, mach_exception_subcode_t, boolean_t);
extern void thread_update_io_stats(thread_t, int size, int io_flags);
extern kern_return_t thread_set_voucher_name(mach_port_name_t name);
extern void set_thread_rwlock_boost(void);
extern void clear_thread_rwlock_boost(void);
-/*! @function thread_has_thread_name
- * @abstract Checks if a thread has a name.
- * @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
- * @param th The thread to inspect.
- * @result TRUE if the thread has a name, FALSE otherwise.
- */
-extern boolean_t thread_has_thread_name(thread_t th);
-
-/*! @function thread_set_thread_name
- * @abstract Set a thread's name.
- * @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread. The name will be attached to the thread in order to better identify the thread.
- * @param th The thread to be named.
- * @param name The name to apply to the thread.
- */
-extern void thread_set_thread_name(thread_t th, const char* name);
-
extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
/*
#endif /* XNU_KERNEL_PRIVATE */
+/*! @function thread_has_thread_name
+ * @abstract Checks if a thread has a name.
+ * @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
+ * @param th The thread to inspect.
+ * @result TRUE if the thread has a name, FALSE otherwise.
+ */
+extern boolean_t thread_has_thread_name(thread_t th);
+
+/*! @function thread_set_thread_name
+ * @abstract Set a thread's name.
+ * @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread. The name will be copied over to the thread in order to better identify the thread. If the name is longer than MAXTHREADNAMESIZE - 1, it will be truncated.
+ * @param th The thread to be named.
+ * @param name The name to apply to the thread.
+ */
+extern void thread_set_thread_name(thread_t th, const char* name);
/*! @function kernel_thread_start
* @abstract Create a kernel thread.
extern boolean_t is_vm_privileged(void);
extern boolean_t set_vm_privilege(boolean_t);
extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
+extern void *thread_iokit_tls_get(uint32_t index);
+extern void thread_iokit_tls_set(uint32_t index, void * data);
#endif /* KERNEL_PRIVATE */
__END_DECLS
projects / apple / xnu.git / blobdiff