/*
- * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <mach/exception_types.h>
#include <kern/kern_types.h>
+#include <vm/vm_kern.h>
#include <sys/cdefs.h>
#include <mach/port.h>
#include <kern/cpu_number.h>
+#include <kern/smp.h>
#include <kern/queue.h>
#include <kern/timer.h>
#include <kern/simple_lock.h>
#include <kern/sched_prim.h>
#include <mach/sfi_class.h>
#include <kern/thread_call.h>
+#include <kern/thread_group.h>
#include <kern/timer_call.h>
#include <kern/task.h>
#include <kern/exception.h>
#include <kern/affinity.h>
+#include <kern/debug.h>
+#include <kern/block_hint.h>
+
+#include <kern/waitq.h>
+#include <san/kasan.h>
#include <ipc/ipc_kmsg.h>
#include <machine/cpu_data.h>
#include <machine/thread.h>
+#if MONOTONIC
+#include <stdatomic.h>
+#include <machine/monotonic.h>
+#endif /* MONOTONIC */
+
+#if CONFIG_EMBEDDED
+/* Taskwatch related. TODO: find this a better home */
+typedef struct task_watcher task_watch_t;
+#endif /* CONFIG_EMBEDDED */
+
struct thread {
+
+#if MACH_ASSERT
+#define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
+ /* Ensure nothing uses &thread as a queue entry */
+ uint64_t thread_magic;
+#endif /* MACH_ASSERT */
+
/*
* NOTE: The runq field in the thread structure has an unusual
* locking protocol. If its value is PROCESSOR_NULL, then it is
* transition from PROCESSOR_NULL to non-null must be done
* under the thread lock and the run queue lock.
*
- * When the thread is on a wait queue, these first three fields
- * are treated as an unofficial union with a wait_queue_element.
- * If you change these, you must change that definition as well
- * (kern/wait_queue.h).
+ * New waitq APIs allow the 'links' and 'runq' fields to be
+ * anywhere in the thread structure.
*/
- /* Items examined often, modified infrequently */
- queue_chain_t links; /* run/wait queue links */
- processor_t runq; /* run queue assignment */
- wait_queue_t wait_queue; /* wait queue we are currently on */
- event64_t wait_event; /* wait queue event */
+ union {
+ queue_chain_t runq_links; /* run queue links */
+ queue_chain_t wait_links; /* wait queue links */
+ };
+
+ processor_t runq; /* run queue assignment */
+
+ event64_t wait_event; /* wait queue event */
+ struct waitq *waitq; /* wait queue this thread is enqueued on */
+
/* Data updated during assert_wait/thread_wakeup */
+#if __SMP__
decl_simple_lock_data(,sched_lock) /* scheduling lock (thread_lock()) */
decl_simple_lock_data(,wake_lock) /* for thread stop / wait (wake_lock()) */
+#endif
integer_t options; /* options set by thread itself */
#define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */
#define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */
#define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */
#define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */
#define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */
-#define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */
+#define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */
+#define TH_OPT_GLOBAL_FORCED_IDLE 0x0100 /* Thread performs forced idle for thermal control */
+#define TH_OPT_SCHED_VM_GROUP 0x0200 /* Thread belongs to special scheduler VM group */
+#define TH_OPT_HONOR_QLIMIT 0x0400 /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
+#define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */
+#define TH_OPT_ZONE_GC 0x1000 /* zone_gc() called on this thread */
boolean_t wake_active; /* wake event on stop */
int at_safe_point; /* thread_abort_safely allowed */
ast_t reason; /* why we blocked */
+ uint32_t quantum_remaining;
+ wait_result_t wait_result; /* outcome of wait -
+ * may be examined by this thread
+ * WITHOUT locking */
thread_continue_t continuation; /* continue here next dispatch */
void *parameter; /* continuation parameter */
- wait_result_t wait_result; /* outcome of wait -
- * may be examined by this thread
- * WITHOUT locking */
/* Data updated/used in thread_invoke */
vm_offset_t kernel_stack; /* current kernel stack */
vm_offset_t reserved_stack; /* reserved kernel stack */
+#if KASAN
+ struct kasan_thread_data kasan_data;
+#endif
+
/* Thread state: */
int state;
/*
#define TH_SUSP 0x02 /* stopped or requested to stop */
#define TH_RUN 0x04 /* running or on runq */
#define TH_UNINT 0x08 /* waiting uninteruptibly */
-#define TH_TERMINATE 0x10 /* halted at termination */
-#define TH_TERMINATE2 0x20 /* added to termination queue */
+#define TH_TERMINATE 0x10 /* halted at termination */
+#define TH_TERMINATE2 0x20 /* added to termination queue */
#define TH_IDLE 0x80 /* idling processor */
sched_mode_t sched_mode; /* scheduling mode */
sched_mode_t saved_mode; /* saved mode during forced mode demotion */
+ /* This thread's contribution to global sched counters */
+ sched_bucket_t th_sched_bucket;
+
sfi_class_id_t sfi_class; /* SFI class (XXX Updated on CSW/QE/AST) */
sfi_class_id_t sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
-
+
+
uint32_t sched_flags; /* current flag bits */
-#define TH_SFLAG_FAIRSHARE_TRIPPED 0x0001 /* fairshare scheduling activated */
+/* TH_SFLAG_FAIRSHARE_TRIPPED (unused) 0x0001 */
#define TH_SFLAG_FAILSAFE 0x0002 /* fail-safe has tripped */
-#define TH_SFLAG_THROTTLED 0x0004 /* thread treated as background for scheduler decay purposes */
-#define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLE_DEMOTED | TH_SFLAG_FAILSAFE | TH_SFLAG_FAIRSHARE_TRIPPED) /* saved_mode contains previous sched_mode */
+#define TH_SFLAG_THROTTLED 0x0004 /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
+#define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE) /* saved_mode contains previous sched_mode */
#define TH_SFLAG_PROMOTED 0x0008 /* sched pri has been promoted */
#define TH_SFLAG_ABORT 0x0010 /* abort interruptible waits */
#define TH_SFLAG_DEPRESS 0x0040 /* normal depress yield */
#define TH_SFLAG_POLLDEPRESS 0x0080 /* polled depress yield */
#define TH_SFLAG_DEPRESSED_MASK (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
-#define TH_SFLAG_PRI_UPDATE 0x0100 /* Updating priority */
+/* unused TH_SFLAG_PRI_UPDATE 0x0100 */
#define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */
#define TH_SFLAG_RW_PROMOTED 0x0400 /* sched pri has been promoted due to blocking with RW lock held */
-#define TH_SFLAG_PROMOTED_MASK (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED)
-#define TH_SFLAG_THROTTLE_DEMOTED 0x0800 /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
+/* unused TH_SFLAG_THROTTLE_DEMOTED 0x0800 */
+#define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* sched pri promoted from waitq wakeup (generally for IPC receive) */
+
+
+#define TH_SFLAG_EXEC_PROMOTED 0x8000 /* sched pri has been promoted since thread is in an exec */
+#define TH_SFLAG_PROMOTED_MASK (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
#define TH_SFLAG_RW_PROMOTED_BIT (10) /* 0x400 */
- int16_t sched_pri; /* scheduled (current) priority */
- int16_t priority; /* base priority */
- int16_t max_priority; /* copy of max base priority */
- int16_t task_priority; /* copy of task base priority */
+ int16_t sched_pri; /* scheduled (current) priority */
+ int16_t base_pri; /* base priority */
+ int16_t max_priority; /* copy of max base priority */
+ int16_t task_priority; /* copy of task base priority */
+
#if defined(CONFIG_SCHED_GRRR)
#if 0
uint16_t grrr_deficit; /* fixed point (1/1000th quantum) fractional deficit */
int16_t promotions; /* level of promotion */
int16_t pending_promoter_index;
- uint32_t ref_count; /* number of references to me */
+ _Atomic uint32_t ref_count; /* number of references to me */
void *pending_promoter[2];
uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */
-#if MACH_ASSERT
- uint32_t SHARE_COUNT, BG_COUNT; /* This thread's contribution to global sched counters (temporary debugging) */
-#endif /* MACH_ASSERT */
-
integer_t importance; /* task-relative importance */
+ uint32_t was_promoted_on_wakeup;
/* Priority depression expiration */
integer_t depress_timer_active;
uint64_t deadline;
} realtime;
- uint32_t was_promoted_on_wakeup;
uint64_t last_run_time; /* time when thread was switched away from */
- uint32_t quantum_remaining; /* duration of current quantum remaining */
+ uint64_t last_made_runnable_time; /* time when thread was unblocked or preempted */
+ uint64_t last_basepri_change_time; /* time when thread was last changed in basepri while runnable */
+ uint64_t same_pri_latency;
+#define THREAD_NOT_RUNNABLE (~0ULL)
+
#if defined(CONFIG_SCHED_MULTIQ)
sched_group_t sched_group;
uint32_t p_switch; /* total processor switches */
uint32_t ps_switch; /* total pset switches */
+ integer_t mutex_count; /* total count of locks held */
/* Timing data structures */
int precise_user_kernel_time; /* precise user/kernel enabled for this thread */
timer_data_t user_timer; /* user mode timer */
uint64_t vtimer_user_save; /* saved values for vtimers */
uint64_t vtimer_prof_save;
uint64_t vtimer_rlim_save;
+ uint64_t vtimer_qos_save;
+
+ timer_data_t ptime; /* time executing in P mode */
+#if CONFIG_SCHED_SFI
/* Timing for wait state */
uint64_t wait_sfi_begin_time; /* start time for thread waiting in SFI */
+#endif
/* Timed wait expiration */
timer_call_data_t wait_timer;
affinity_set_t affinity_set;
queue_chain_t affinity_threads;
- /* Various bits of stashed state */
+ /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
union {
struct {
mach_msg_return_t state; /* receive state */
mach_port_seqno_t seqno; /* seqno of recvd message */
ipc_object_t object; /* object received on */
mach_vm_address_t msg_addr; /* receive buffer pointer */
- mach_msg_size_t msize; /* max size for recvd msg */
+ mach_msg_size_t rsize; /* max size for recvd msg */
+ mach_msg_size_t msize; /* actual size for recvd msg */
mach_msg_option_t option; /* options for receive */
- mach_msg_size_t slist_size; /* scatter list size */
mach_port_name_t receiver_name; /* the receive port name */
- struct ipc_kmsg *kmsg; /* received message */
+ struct knote *knote; /* knote fired for rcv */
+ union {
+ struct ipc_kmsg *kmsg; /* received message */
+ struct ipc_mqueue *peekq; /* mqueue to peek at */
+ struct {
+ mach_msg_priority_t qos; /* received message qos */
+ mach_msg_priority_t oqos; /* override qos for message */
+ } received_qos;
+ };
mach_msg_continue_t continuation;
} receive;
struct {
int option; /* switch option */
boolean_t reenable_workq_callback; /* on entry, callbacks were suspended */
} swtch;
- int misc; /* catch-all for other state */
} saved;
- /* Structure to save information about guard exception */
- struct {
- unsigned type; /* EXC_GUARD reason/type */
- mach_exception_data_type_t code; /* Exception code */
- mach_exception_data_type_t subcode; /* Exception sub-code */
- } guard_exc_info;
+ /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
+ union {
+ /* Group and call this thread is working on behalf of */
+ struct {
+ struct thread_call_group * thc_group;
+ struct thread_call * thc_call; /* debug only, may be deallocated */
+ } thc_state;
+ /* Structure to save information about guard exception */
+ struct {
+ mach_exception_code_t code;
+ mach_exception_subcode_t subcode;
+ } guard_exc_info;
+ };
+
+ /* Kernel holds on this thread */
+ int16_t suspend_count;
+ /* User level suspensions */
+ int16_t user_stop_count;
/* IPC data structures */
#if IMPORTANCE_INHERITANCE
/* Activation */
queue_chain_t task_threads;
- /*** Machine-dependent state ***/
- struct machine_thread machine;
-
/* Task membership */
struct task *task;
vm_map_t map;
/* Miscellaneous bits guarded by mutex */
uint32_t
- active:1, /* Thread is active and has not been terminated */
- started:1, /* Thread has been started after creation */
- static_param:1, /* Disallow policy parameter changes */
- policy_reset:1, /* Disallow policy parameter changes on terminating threads */
+ active:1, /* Thread is active and has not been terminated */
+ started:1, /* Thread has been started after creation */
+ static_param:1, /* Disallow policy parameter changes */
+ inspection:1, /* TRUE when task is being inspected by crash reporter */
+ policy_reset:1, /* Disallow policy parameter changes on terminating threads */
+ suspend_parked:1, /* thread parked in thread_suspended */
+ corpse_dup:1, /* TRUE when thread is an inactive duplicate in a corpse */
:0;
- /* Return Handers */
- struct ReturnHandler {
- struct ReturnHandler *next;
- void (*handler)(
- struct ReturnHandler *rh,
- struct thread *thread);
- } *handlers, special_handler;
-
/* Ports associated with this thread */
struct ipc_port *ith_self; /* not a right, doesn't hold ref */
struct ipc_port *ith_sself; /* a send right */
+ struct ipc_port *ith_special_reply_port; /* ref to special reply port */
struct exception_action *exc_actions;
#ifdef MACH_BSD
clock_sec_t t_page_creation_time;
uint32_t t_page_creation_count;
+ uint32_t t_page_creation_throttled;
+#if (DEVELOPMENT || DEBUG)
+ uint64_t t_page_creation_throttled_hard;
+ uint64_t t_page_creation_throttled_soft;
+#endif /* DEVELOPMENT || DEBUG */
-#define T_CHUD_MARKED 0x01 /* this thread is marked by CHUD */
-#define T_IN_CHUD 0x02 /* this thread is already in a CHUD handler */
-#define THREAD_PMC_FLAG 0x04 /* Bit in "t_chud" signifying PMC interest */
-#define T_AST_CALLSTACK 0x08 /* Thread scheduled to dump a
- * callstack on its next
- * AST */
-#define T_AST_NAME 0x10 /* Thread scheduled to dump
- * its name on its next
- * AST */
-#define T_NAME_DONE 0x20 /* Thread has previously
- * recorded its name */
-#define T_KPC_ALLOC 0x40 /* Thread needs a kpc_buf */
+#ifdef KPERF
+/* The high 7 bits are the number of frames to sample of a user callstack. */
+#define T_KPERF_CALLSTACK_DEPTH_OFFSET (25)
+#define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#endif
- uint32_t t_chud; /* CHUD flags, used for Shark */
- uint32_t chud_c_switch; /* last dispatch detection */
+#define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
+#define T_KPERF_AST_DISPATCH (1U << 1) /* dump a name on thread's next AST */
+#define T_KPC_ALLOC (1U << 2) /* thread needs a kpc_buf allocated */
+/* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
- integer_t mutex_count; /* total count of locks held */
+#ifdef KPERF
+ uint32_t kperf_flags;
+ uint32_t kperf_pet_gen; /* last generation of PET that sampled this thread*/
+ uint32_t kperf_c_switch; /* last dispatch detection */
+ uint32_t kperf_pet_cnt; /* how many times a thread has been sampled by PET */
+#endif
#ifdef KPC
/* accumulated performance counters for this thread */
uint64_t *kpc_buf;
#endif
-#ifdef KPERF
- /* count of how many times a thread has been sampled since it was last scheduled */
- uint64_t kperf_pet_cnt;
-#endif
-
#if HYPERVISOR
/* hypervisor virtual CPU object associated with this thread */
void *hv_thread_target;
uint32_t syscalls_mach;
ledger_t t_ledger;
ledger_t t_threadledger; /* per thread ledger */
- uint64_t cpu_time_last_qos;
-#ifdef CONFIG_BANK
- ledger_t t_bankledger; /* ledger to charge someone */
- uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
-#endif
+ ledger_t t_bankledger; /* ledger to charge someone */
+ uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
+ uint64_t t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
- /* policy is protected by the task lock */
- struct task_requested_policy requested_policy;
- struct task_effective_policy effective_policy;
- struct task_pended_policy pended_policy;
+#if MONOTONIC
+ struct mt_thread t_monotonic;
+#endif /* MONOTONIC */
+
+ /*** Machine-dependent state ***/
+ struct machine_thread machine;
+
+ /* policy is protected by the thread mutex */
+ struct thread_requested_policy requested_policy;
+ struct thread_effective_policy effective_policy;
/* usynch override is protected by the task lock, eventually will be thread mutex */
struct thread_qos_override {
user_addr_t override_resource;
} *overrides;
+ _Atomic uint32_t kqwl_owning_count;
+ uint32_t ipc_overrides;
+ uint32_t sync_ipc_overrides;
+ uint32_t user_promotions;
+ uint16_t user_promotion_basepri;
+ _Atomic uint16_t kevent_ast_bits;
+
+ block_hint_t pending_block_hint;
+ block_hint_t block_hint; /* What type of primitive last caused us to block. */
+
int iotier_override; /* atomic operations to set, cleared on ret to user */
io_stat_info_t thread_io_stats; /* per-thread I/O statistics */
-
- integer_t saved_importance; /* saved task-relative importance */
+#if CONFIG_EMBEDDED
+ task_watch_t * taskwatch; /* task watch */
+#endif /* CONFIG_EMBEDDED */
uint32_t thread_callout_interrupt_wakeups;
uint32_t thread_callout_platform_idle_wakeups;
callout_woken_from_platform_idle:1,
callout_woke_thread:1,
thread_bitfield_unused:13;
- /* Kernel holds on this thread */
- int16_t suspend_count;
- /* User level suspensions */
- int16_t user_stop_count;
mach_port_name_t ith_voucher_name;
ipc_voucher_t ith_voucher;
#if CONFIG_IOSCHED
void *decmp_upl;
#endif /* CONFIG_IOSCHED */
+
+ /* work interval (if any) associated with the thread. Uses thread mutex */
+ struct work_interval *th_work_interval;
+
+#if SCHED_TRACE_THREAD_WAKEUPS
+ uintptr_t thread_wakeup_bt[64];
+#endif
};
-#define ith_state saved.receive.state
-#define ith_object saved.receive.object
-#define ith_msg_addr saved.receive.msg_addr
-#define ith_msize saved.receive.msize
-#define ith_option saved.receive.option
-#define ith_scatter_list_size saved.receive.slist_size
-#define ith_receiver_name saved.receive.receiver_name
-#define ith_continuation saved.receive.continuation
-#define ith_kmsg saved.receive.kmsg
-#define ith_seqno saved.receive.seqno
-
-#define sth_waitsemaphore saved.sema.waitsemaphore
-#define sth_signalsemaphore saved.sema.signalsemaphore
-#define sth_options saved.sema.options
-#define sth_result saved.sema.result
-#define sth_continuation saved.sema.continuation
+#define ith_state saved.receive.state
+#define ith_object saved.receive.object
+#define ith_msg_addr saved.receive.msg_addr
+#define ith_rsize saved.receive.rsize
+#define ith_msize saved.receive.msize
+#define ith_option saved.receive.option
+#define ith_receiver_name saved.receive.receiver_name
+#define ith_continuation saved.receive.continuation
+#define ith_kmsg saved.receive.kmsg
+#define ith_peekq saved.receive.peekq
+#define ith_knote saved.receive.knote
+#define ith_qos saved.receive.received_qos.qos
+#define ith_qos_override saved.receive.received_qos.oqos
+#define ith_seqno saved.receive.seqno
+
+#define sth_waitsemaphore saved.sema.waitsemaphore
+#define sth_signalsemaphore saved.sema.signalsemaphore
+#define sth_options saved.sema.options
+#define sth_result saved.sema.result
+#define sth_continuation saved.sema.continuation
+
+#define ITH_KNOTE_NULL ((void *)NULL)
+#define ITH_KNOTE_PSEUDO ((void *)0xdeadbeef)
+#define ITH_KNOTE_VALID(kn) ((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO)
+
+#if MACH_ASSERT
+#define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
+ "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
+ (thread)->thread_magic, (thread), THREAD_MAGIC)
+#else
+#define assert_thread_magic(thread) do { (void)(thread); } while (0)
+#endif
extern void thread_bootstrap(void);
extern void thread_daemon_init(void);
#define thread_reference_internal(thread) \
- (void)hw_atomic_add(&(thread)->ref_count, 1)
-
-#define thread_deallocate_internal(thread) \
- hw_atomic_sub(&(thread)->ref_count, 1)
+ (void)atomic_fetch_add_explicit(&(thread)->ref_count, 1, memory_order_relaxed)
#define thread_reference(thread) \
MACRO_BEGIN \
extern void thread_deallocate(
thread_t thread);
+extern void thread_deallocate_safe(
+ thread_t thread);
+
+extern void thread_inspect_deallocate(
+ thread_inspect_t thread);
+
extern void thread_terminate_self(void);
extern kern_return_t thread_terminate_internal(
thread_t thread);
-extern void thread_start_internal(
+extern void thread_start(
thread_t thread) __attribute__ ((noinline));
+extern void thread_start_in_assert_wait(
+ thread_t thread,
+ event_t event,
+ wait_interrupt_t interruptible) __attribute__ ((noinline));
+
extern void thread_terminate_enqueue(
thread_t thread);
+extern void thread_exception_enqueue(
+ task_t task,
+ thread_t thread,
+ exception_type_t etype);
+
+extern void thread_copy_resource_info(
+ thread_t dst_thread,
+ thread_t src_thread);
+
+extern void thread_terminate_crashed_threads(void);
+
extern void thread_stack_enqueue(
thread_t thread);
extern void thread_release(
thread_t thread);
+extern void thread_corpse_continue(void);
+
+extern boolean_t thread_is_active(thread_t thread);
+/* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
+#if __SMP__
#define thread_lock_init(th) simple_lock_init(&(th)->sched_lock, 0)
#define thread_lock(th) simple_lock(&(th)->sched_lock)
#define thread_unlock(th) simple_unlock(&(th)->sched_lock)
#define wake_lock_init(th) simple_lock_init(&(th)->wake_lock, 0)
#define wake_lock(th) simple_lock(&(th)->wake_lock)
#define wake_unlock(th) simple_unlock(&(th)->wake_lock)
+#else
+#define thread_lock_init(th) do { (void)th; } while(0)
+#define thread_lock(th) do { (void)th; } while(0)
+#define thread_unlock(th) do { (void)th; } while(0)
+
+#define wake_lock_init(th) do { (void)th; } while(0)
+#define wake_lock(th) do { (void)th; } while(0)
+#define wake_unlock(th) do { (void)th; } while(0)
+#endif
#define thread_should_halt_fast(thread) (!(thread)->active)
thread_info_t thread_info_out,
mach_msg_type_number_t *thread_info_count);
-extern void thread_task_priority(
- thread_t thread,
- integer_t priority,
- integer_t max_priority);
-
-extern kern_return_t thread_set_mode_and_absolute_pri(
- thread_t thread,
- integer_t policy,
- integer_t priority);
-extern void thread_policy_reset(
- thread_t thread);
extern kern_return_t kernel_thread_create(
thread_continue_t continuation,
thread_t new_thread);
extern void machine_load_context(
- thread_t thread);
+ thread_t thread) __attribute__((noreturn));
+
extern kern_return_t machine_thread_state_initialize(
thread_t thread);
thread_t thread,
mach_vm_offset_t tsd_base);
-typedef struct ReturnHandler ReturnHandler;
-
#define thread_mtx_lock(thread) lck_mtx_lock(&(thread)->mutex)
#define thread_mtx_try(thread) lck_mtx_try_lock(&(thread)->mutex)
#define thread_mtx_unlock(thread) lck_mtx_unlock(&(thread)->mutex)
-extern void act_execute_returnhandlers(void);
-
-extern void install_special_handler(
- thread_t thread);
-
-extern void special_handler(
- ReturnHandler *rh,
- thread_t thread);
+extern void thread_apc_ast(thread_t thread);
-extern void
-thread_update_qos_cpu_time(
- thread_t thread,
- boolean_t lock_needed);
+extern void thread_update_qos_cpu_time(thread_t thread);
void act_machine_sv_free(thread_t, int);
return thread->thread_tag;
}
-typedef struct {
- int qos_pri[THREAD_QOS_LAST];
- int qos_iotier[THREAD_QOS_LAST];
- uint32_t qos_through_qos[THREAD_QOS_LAST];
- uint32_t qos_latency_qos[THREAD_QOS_LAST];
-} qos_policy_params_t;
+
+extern void thread_set_options(uint32_t thopt);
+
#else /* MACH_KERNEL_PRIVATE */
__BEGIN_DECLS
-extern uint64_t thread_dispatchqaddr(
+extern void thread_starts_owning_workloop(
+ thread_t thread);
+
+extern void thread_ends_owning_workloop(
+ thread_t thread);
+
+extern uint32_t thread_owned_workloops_count(
+ thread_t thread);
+
+
+extern uint64_t thread_dispatchqaddr(
+ thread_t thread);
+
+extern uint64_t thread_rettokern_addr(
thread_t thread);
__END_DECLS
#ifdef KERNEL
__BEGIN_DECLS
-extern uint64_t thread_tid(thread_t thread);
+extern uint64_t thread_tid(thread_t thread);
__END_DECLS
#define THREAD_TAG_CALLOUT 0x2
#define THREAD_TAG_IOWORKLOOP 0x4
+#define THREAD_TAG_PTHREAD 0x10
+#define THREAD_TAG_WORKQUEUE 0x20
+
uint16_t thread_set_tag(thread_t, uint16_t);
uint16_t thread_get_tag(thread_t);
-
extern kern_return_t thread_state_initialize(
thread_t thread);
thread_state_t tstate,
mach_msg_type_number_t *count);
+extern kern_return_t thread_create_with_continuation(
+ task_t task,
+ thread_t *new_thread,
+ thread_continue_t continuation);
+
+extern kern_return_t thread_create_waiting(task_t task,
+ thread_continue_t continuation,
+ event_t event,
+ thread_t *new_thread);
+
extern kern_return_t thread_create_workq(
task_t task,
thread_continue_t thread_return,
thread_t *new_thread);
+extern kern_return_t thread_create_workq_waiting(
+ task_t task,
+ thread_continue_t thread_return,
+ event_t event,
+ thread_t *new_thread);
+
extern void thread_yield_internal(
mach_msg_timeout_t interval);
+extern void thread_yield_to_preemption(void);
+
/*
* Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
*
thread_state_t,
unsigned int,
mach_vm_offset_t *,
- int *);
+ int *,
+ boolean_t);
extern kern_return_t thread_entrypoint(
thread_t,
mach_vm_offset_t *);
extern kern_return_t thread_userstackdefault(
- thread_t,
- mach_vm_offset_t *);
+ mach_vm_offset_t *,
+ boolean_t);
extern kern_return_t thread_wire_internal(
host_priv_t host_priv,
extern kern_return_t thread_dup(thread_t);
+extern kern_return_t thread_dup2(thread_t, thread_t);
+
+#if !defined(_SCHED_CALL_T_DEFINED)
+#define _SCHED_CALL_T_DEFINED
typedef void (*sched_call_t)(
int type,
thread_t thread);
+#endif
#define SCHED_CALL_BLOCK 0x1
#define SCHED_CALL_UNBLOCK 0x2
thread_t thread,
sched_call_t call);
+extern sched_call_t thread_disable_sched_call(
+ thread_t thread,
+ sched_call_t call);
+
+extern void thread_reenable_sched_call(
+ thread_t thread,
+ sched_call_t call);
+
extern void thread_static_param(
thread_t thread,
boolean_t state);
extern boolean_t thread_is_static_param(
thread_t thread);
-extern kern_return_t thread_policy_set_internal(
- thread_t thread,
- thread_policy_flavor_t flavor,
- thread_policy_t policy_info,
- mach_msg_type_number_t count);
-
-extern boolean_t thread_has_qos_policy(thread_t thread);
-
-extern kern_return_t thread_remove_qos_policy(thread_t thread);
-
extern task_t get_threadtask(thread_t);
#define thread_is_64bit(thd) \
task_has_64BitAddr(get_threadtask(thd))
extern void *get_bsdthread_info(thread_t);
extern void set_bsdthread_info(thread_t, void *);
extern void *uthread_alloc(task_t, thread_t, int);
-extern void uthread_cleanup(task_t, void *, void *);
+extern void uthread_cleanup_name(void *uthread);
+extern void uthread_cleanup(task_t, void *, void *);
extern void uthread_zone_free(void *);
-extern void uthread_cred_free(void *);
+extern void uthread_cred_free(void *);
+
+extern void uthread_reset_proc_refcount(void *);
+#if PROC_REF_DEBUG
+extern int uthread_get_proc_refcount(void *);
+extern int proc_ref_tracking_disabled;
+#endif
extern boolean_t thread_should_halt(
thread_t thread);
extern int is_64signalregset(void);
-void act_set_apc(thread_t);
-void act_set_kperf(thread_t);
+extern void act_set_kperf(thread_t);
+extern void act_set_astledger(thread_t thread);
+extern void act_set_astledger_async(thread_t thread);
+extern void act_set_io_telemetry_ast(thread_t);
extern uint32_t dtrace_get_thread_predcache(thread_t);
extern int64_t dtrace_get_thread_vtime(thread_t);
extern int64_t dtrace_get_thread_tracing(thread_t);
extern boolean_t dtrace_get_thread_reentering(thread_t);
+extern int dtrace_get_thread_last_cpu_id(thread_t);
extern vm_offset_t dtrace_get_kernel_stack(thread_t);
extern void dtrace_set_thread_predcache(thread_t, uint32_t);
extern void dtrace_set_thread_vtime(thread_t, int64_t);
extern vm_offset_t kernel_stack_size;
extern vm_offset_t kernel_stack_depth_max;
-void guard_ast(thread_t thread);
-extern void fd_guard_ast(thread_t thread);
-extern void mach_port_guard_ast(thread_t thread);
-extern void thread_guard_violation(thread_t thread, unsigned type);
-extern void thread_update_io_stats(thread_t thread, int size, int io_flags);
+extern void guard_ast(thread_t);
+extern void fd_guard_ast(thread_t,
+ mach_exception_code_t, mach_exception_subcode_t);
+#if CONFIG_VNGUARD
+extern void vn_guard_ast(thread_t,
+ mach_exception_code_t, mach_exception_subcode_t);
+#endif
+extern void mach_port_guard_ast(thread_t,
+ mach_exception_code_t, mach_exception_subcode_t);
+extern void thread_guard_violation(thread_t,
+ mach_exception_code_t, mach_exception_subcode_t);
+extern void thread_update_io_stats(thread_t, int size, int io_flags);
extern kern_return_t thread_set_voucher_name(mach_port_name_t name);
extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
+extern void set_thread_rwlock_boost(void);
+extern void clear_thread_rwlock_boost(void);
+
+/*! @function thread_has_thread_name
+ @abstract Checks if a thread has a name.
+ @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
+ @param th The thread to inspect.
+ @result TRUE if the thread has a name, FALSE otherwise.
+*/
+extern boolean_t thread_has_thread_name(thread_t th);
+
+/*! @function thread_set_thread_name
+ @abstract Set a thread's name.
+ @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread. The name will be attached to the thread in order to better identify the thread.
+ @param th The thread to be named.
+ @param name The name to apply to the thread.
+*/
+extern void thread_set_thread_name(thread_t th, const char* name);
+
+extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
+
+/* Get a backtrace for a threads kernel or user stack (user_p), with pc and optionally
+ * frame pointer (getfp). Returns bytes added to buffer, and kThreadTruncatedBT in
+ * thread_trace_flags if a user page is not present after kdp_lightweight_fault() is
+ * called.
+ */
+
+extern int machine_trace_thread(
+ thread_t thread,
+ char *tracepos,
+ char *tracebound,
+ int nframes,
+ boolean_t user_p,
+ boolean_t getfp,
+ uint32_t *thread_trace_flags);
+
+extern int machine_trace_thread64(thread_t thread,
+ char *tracepos,
+ char *tracebound,
+ int nframes,
+ boolean_t user_p,
+ boolean_t getfp,
+ uint32_t *thread_trace_flags);
+
#endif /* XNU_KERNEL_PRIVATE */
thread_continue_t continuation,
void *parameter,
thread_t *new_thread);
+
#ifdef KERNEL_PRIVATE
void thread_set_eager_preempt(thread_t thread);
void thread_clear_eager_preempt(thread_t thread);
+void thread_set_honor_qlimit(thread_t thread);
+void thread_clear_honor_qlimit(thread_t thread);
extern ipc_port_t convert_thread_to_port(thread_t);
+extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
+extern boolean_t is_vm_privileged(void);
extern boolean_t set_vm_privilege(boolean_t);
+extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
#endif /* KERNEL_PRIVATE */
__END_DECLS
projects / apple / xnu.git / blobdiff