/*
- * Copyright (c) 2000-2015 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <kern/kern_types.h>
#include <kern/kalloc.h>
#include <kern/cpu_data.h>
-#include <kern/counters.h>
#include <kern/extmod_statistics.h>
#include <kern/ipc_mig.h>
#include <kern/ipc_tt.h>
#include <kern/telemetry.h>
#include <kern/policy_internal.h>
#include <kern/turnstile.h>
+#include <kern/sched_clutch.h>
#include <corpses/task_corpse.h>
#if KPC
#include <sys/bsdtask_info.h>
#include <mach/sdt.h>
#include <san/kasan.h>
+#if CONFIG_KSANCOV
+#include <san/ksancov.h>
+#endif
#include <stdatomic.h>
+#if defined(HAS_APPLE_PAC)
+#include <ptrauth.h>
+#include <arm64/proc_reg.h>
+#endif /* defined(HAS_APPLE_PAC) */
/*
* Exported interfaces
#include <mach/mach_voucher_server.h>
#include <kern/policy_internal.h>
-static struct zone *thread_zone;
-static lck_grp_attr_t thread_lck_grp_attr;
-lck_attr_t thread_lck_attr;
-lck_grp_t thread_lck_grp;
-
-struct zone *thread_qos_override_zone;
+#if CONFIG_MACF
+#include <security/mac_mach_internal.h>
+#endif
-decl_simple_lock_data(static, thread_stack_lock)
-static queue_head_t thread_stack_queue;
+LCK_GRP_DECLARE(thread_lck_grp, "thread");
-decl_simple_lock_data(static, thread_terminate_lock)
-static queue_head_t thread_terminate_queue;
+ZONE_DECLARE(thread_zone, "threads", sizeof(struct thread), ZC_ZFREE_CLEARMEM);
-static queue_head_t thread_deallocate_queue;
+ZONE_DECLARE(thread_qos_override_zone, "thread qos override",
+ sizeof(struct thread_qos_override), ZC_NOENCRYPT);
-static queue_head_t turnstile_deallocate_queue;
+static struct mpsc_daemon_queue thread_stack_queue;
+static struct mpsc_daemon_queue thread_terminate_queue;
+static struct mpsc_daemon_queue thread_deallocate_queue;
+static struct mpsc_daemon_queue thread_exception_queue;
+decl_simple_lock_data(static, crashed_threads_lock);
static queue_head_t crashed_threads_queue;
-static queue_head_t workq_deallocate_queue;
-
-decl_simple_lock_data(static, thread_exception_lock)
-static queue_head_t thread_exception_queue;
-
struct thread_exception_elt {
- queue_chain_t elt;
+ struct mpsc_queue_chain link;
exception_type_t exception_type;
task_t exception_task;
thread_t exception_thread;
};
-static struct thread thread_template, init_thread;
+static SECURITY_READ_ONLY_LATE(struct thread) thread_template = {
+#if MACH_ASSERT
+ .thread_magic = THREAD_MAGIC,
+#endif /* MACH_ASSERT */
+ .wait_result = THREAD_WAITING,
+ .options = THREAD_ABORTSAFE,
+ .state = TH_WAIT | TH_UNINT,
+ .th_sched_bucket = TH_BUCKET_RUN,
+ .base_pri = BASEPRI_DEFAULT,
+ .realtime.deadline = UINT64_MAX,
+ .last_made_runnable_time = THREAD_NOT_RUNNABLE,
+ .last_basepri_change_time = THREAD_NOT_RUNNABLE,
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+ .pri_shift = INT8_MAX,
+#endif
+ /* timers are initialized in thread_bootstrap */
+};
+
+static struct thread init_thread;
static void thread_deallocate_enqueue(thread_t thread);
static void thread_deallocate_complete(thread_t thread);
static uint64_t thread_unique_id = 100;
-struct _thread_ledger_indices thread_ledgers = { -1 };
+struct _thread_ledger_indices thread_ledgers = { .cpu_time = -1 };
static ledger_template_t thread_ledger_template = NULL;
static void init_thread_ledgers(void);
*/
#define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
-int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
+/* Percentage. Level at which we start gathering telemetry. */
+static TUNABLE(uint8_t, cpumon_ustackshots_trigger_pct,
+ "cpumon_ustackshots_trigger_pct", CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT);
void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void);
#if DEVELOPMENT || DEBUG
void __attribute__((noinline)) SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t, int);
os_refgrp_decl(static, thread_refgrp, "thread", NULL);
-void
+static inline void
+init_thread_from_template(thread_t thread)
+{
+ /*
+ * In general, struct thread isn't trivially-copyable, since it may
+ * contain pointers to thread-specific state. This may be enforced at
+ * compile time on architectures that store authed + diversified
+ * pointers in machine_thread.
+ *
+ * In this specific case, where we're initializing a new thread from a
+ * thread_template, we know all diversified pointers are NULL; these are
+ * safe to bitwise copy.
+ */
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wnontrivial-memaccess"
+ memcpy(thread, &thread_template, sizeof(*thread));
+#pragma clang diagnostic pop
+}
+
+thread_t
thread_bootstrap(void)
{
/*
* Fill in a template thread for fast initialization.
*/
-
-#if MACH_ASSERT
- thread_template.thread_magic = THREAD_MAGIC;
-#endif /* MACH_ASSERT */
-
- thread_template.runq = PROCESSOR_NULL;
-
- thread_template.reason = AST_NONE;
- thread_template.at_safe_point = FALSE;
- thread_template.wait_event = NO_EVENT64;
- thread_template.waitq = NULL;
- thread_template.wait_result = THREAD_WAITING;
- thread_template.options = THREAD_ABORTSAFE;
- thread_template.state = TH_WAIT | TH_UNINT;
- thread_template.wake_active = FALSE;
- thread_template.continuation = THREAD_CONTINUE_NULL;
- thread_template.parameter = NULL;
-
- thread_template.importance = 0;
- thread_template.sched_mode = TH_MODE_NONE;
- thread_template.sched_flags = 0;
- thread_template.saved_mode = TH_MODE_NONE;
- thread_template.safe_release = 0;
- thread_template.th_sched_bucket = TH_BUCKET_RUN;
-
- thread_template.sfi_class = SFI_CLASS_UNSPECIFIED;
- thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED;
-
- thread_template.active = 0;
- thread_template.started = 0;
- thread_template.static_param = 0;
- thread_template.policy_reset = 0;
-
- thread_template.base_pri = BASEPRI_DEFAULT;
- thread_template.sched_pri = 0;
- thread_template.max_priority = 0;
- thread_template.task_priority = 0;
- thread_template.promotions = 0;
- thread_template.rwlock_count = 0;
- thread_template.waiting_for_mutex = NULL;
-
-
- thread_template.realtime.deadline = UINT64_MAX;
-
- thread_template.quantum_remaining = 0;
- thread_template.last_run_time = 0;
- thread_template.last_made_runnable_time = THREAD_NOT_RUNNABLE;
- thread_template.last_basepri_change_time = THREAD_NOT_RUNNABLE;
- thread_template.same_pri_latency = 0;
-
- thread_template.computation_metered = 0;
- thread_template.computation_epoch = 0;
-
-#if defined(CONFIG_SCHED_TIMESHARE_CORE)
- thread_template.sched_stamp = 0;
- thread_template.pri_shift = INT8_MAX;
- thread_template.sched_usage = 0;
- thread_template.cpu_usage = thread_template.cpu_delta = 0;
-#endif
- thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
-
-#if MONOTONIC
- memset(&thread_template.t_monotonic, 0,
- sizeof(thread_template.t_monotonic));
-#endif /* MONOTONIC */
-
- thread_template.bound_processor = PROCESSOR_NULL;
- thread_template.last_processor = PROCESSOR_NULL;
-
- thread_template.sched_call = NULL;
-
timer_init(&thread_template.user_timer);
timer_init(&thread_template.system_timer);
timer_init(&thread_template.ptime);
timer_init(&thread_template.runnable_timer);
- thread_template.user_timer_save = 0;
- thread_template.system_timer_save = 0;
- thread_template.vtimer_user_save = 0;
- thread_template.vtimer_prof_save = 0;
- thread_template.vtimer_rlim_save = 0;
- thread_template.vtimer_qos_save = 0;
-
-#if CONFIG_SCHED_SFI
- thread_template.wait_sfi_begin_time = 0;
-#endif
-
- thread_template.wait_timer_is_set = FALSE;
- thread_template.wait_timer_active = 0;
-
- thread_template.depress_timer_active = 0;
-
- thread_template.recover = (vm_offset_t)NULL;
-
- thread_template.map = VM_MAP_NULL;
-#if DEVELOPMENT || DEBUG
- thread_template.pmap_footprint_suspended = FALSE;
-#endif /* DEVELOPMENT || DEBUG */
-
-#if CONFIG_DTRACE
- thread_template.t_dtrace_predcache = 0;
- thread_template.t_dtrace_vtime = 0;
- thread_template.t_dtrace_tracing = 0;
-#endif /* CONFIG_DTRACE */
-
-#if KPERF
- thread_template.kperf_flags = 0;
- thread_template.kperf_pet_gen = 0;
- thread_template.kperf_c_switch = 0;
- thread_template.kperf_pet_cnt = 0;
-#endif
-
-#if KPC
- thread_template.kpc_buf = NULL;
-#endif
-
-#if HYPERVISOR
- thread_template.hv_thread_target = NULL;
-#endif /* HYPERVISOR */
-
-#if (DEVELOPMENT || DEBUG)
- thread_template.t_page_creation_throttled_hard = 0;
- thread_template.t_page_creation_throttled_soft = 0;
-#endif /* DEVELOPMENT || DEBUG */
- thread_template.t_page_creation_throttled = 0;
- thread_template.t_page_creation_count = 0;
- thread_template.t_page_creation_time = 0;
-
- thread_template.affinity_set = NULL;
-
- thread_template.syscalls_unix = 0;
- thread_template.syscalls_mach = 0;
-
- thread_template.t_ledger = LEDGER_NULL;
- thread_template.t_threadledger = LEDGER_NULL;
- thread_template.t_bankledger = LEDGER_NULL;
- thread_template.t_deduct_bank_ledger_time = 0;
-
- thread_template.requested_policy = (struct thread_requested_policy) {};
- thread_template.effective_policy = (struct thread_effective_policy) {};
-
- bzero(&thread_template.overrides, sizeof(thread_template.overrides));
- thread_template.sync_ipc_overrides = 0;
-
- thread_template.iotier_override = THROTTLE_LEVEL_NONE;
- thread_template.thread_io_stats = NULL;
-#if CONFIG_EMBEDDED
- thread_template.taskwatch = NULL;
-#endif /* CONFIG_EMBEDDED */
- thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
-
- thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
- thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
-
- thread_template.thread_tag = 0;
-
- thread_template.ith_voucher_name = MACH_PORT_NULL;
- thread_template.ith_voucher = IPC_VOUCHER_NULL;
-
- thread_template.th_work_interval = NULL;
-
- init_thread = thread_template;
+ init_thread_from_template(&init_thread);
/* fiddle with init thread to skip asserts in set_sched_pri */
init_thread.sched_pri = MAXPRI_KERNEL;
+#if DEBUG || DEVELOPMENT
+ queue_init(&init_thread.t_temp_alloc_list);
+#endif /* DEBUG || DEVELOPMENT */
- machine_set_current_thread(&init_thread);
+ return &init_thread;
}
-extern boolean_t allow_qos_policy_set;
+void
+thread_machine_init_template(void)
+{
+ machine_thread_template_init(&thread_template);
+}
void
thread_init(void)
{
- thread_zone = zinit(
- sizeof(struct thread),
- thread_max * sizeof(struct thread),
- THREAD_CHUNK * sizeof(struct thread),
- "threads");
-
- thread_qos_override_zone = zinit(
- sizeof(struct thread_qos_override),
- 4 * thread_max * sizeof(struct thread_qos_override),
- PAGE_SIZE,
- "thread qos override");
- zone_change(thread_qos_override_zone, Z_EXPAND, TRUE);
- zone_change(thread_qos_override_zone, Z_COLLECT, TRUE);
- zone_change(thread_qos_override_zone, Z_CALLERACCT, FALSE);
- zone_change(thread_qos_override_zone, Z_NOENCRYPT, TRUE);
-
- lck_grp_attr_setdefault(&thread_lck_grp_attr);
- lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
- lck_attr_setdefault(&thread_lck_attr);
-
stack_init();
thread_policy_init();
*/
machine_thread_init();
- if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
- sizeof(cpumon_ustackshots_trigger_pct))) {
- cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
- }
-
- PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));
-
init_thread_ledgers();
}
{
thread_t thread = current_thread();
- thread_terminate_internal(thread);
+ thread_terminate_internal(thread, TH_TERMINATE_OPTION_NONE);
/*
* Handle the thread termination directly
/*NOTREACHED*/
}
+__dead2
static void
thread_terminate_continue(void)
{
task_t task;
int threadcnt;
+ if (thread->t_temp_alloc_count) {
+ kheap_temp_leak_panic(thread);
+ }
+
pal_thread_terminate_self(thread);
DTRACE_PROC(lwp__exit);
thread_unlock(thread);
splx(s);
-#if CONFIG_EMBEDDED
+#if CONFIG_TASKWATCH
thead_remove_taskwatch(thread);
-#endif /* CONFIG_EMBEDDED */
+#endif /* CONFIG_TASKWATCH */
work_interval_thread_terminate(thread);
thread_mtx_unlock(thread);
+ assert(thread->th_work_interval == NULL);
+
bank_swap_thread_bank_ledger(thread, NULL);
if (kdebug_enable && bsd_hasthreadname(thread->uthread)) {
long dbg_arg2 = 0;
kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2);
+#if MONOTONIC
+ if (kdebug_debugid_enabled(DBG_MT_INSTRS_CYCLES_THR_EXIT)) {
+ uint64_t counts[MT_CORE_NFIXED];
+ uint64_t thread_user_time;
+ uint64_t thread_system_time;
+ thread_user_time = timer_grab(&thread->user_timer);
+ thread_system_time = timer_grab(&thread->system_timer);
+ mt_fixed_thread_counts(thread, counts);
+ KDBG_RELEASE(DBG_MT_INSTRS_CYCLES_THR_EXIT,
+#ifdef MT_CORE_INSTRS
+ counts[MT_CORE_INSTRS],
+#else /* defined(MT_CORE_INSTRS) */
+ 0,
+#endif/* !defined(MT_CORE_INSTRS) */
+ counts[MT_CORE_CYCLES],
+ thread_system_time, thread_user_time);
+ }
+#endif/* MONOTONIC */
KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE_PID, dbg_arg1, dbg_arg2);
}
/*
* After this subtraction, this thread should never access
- * task->bsd_info unless it got 0 back from the hw_atomic_sub. It
+ * task->bsd_info unless it got 0 back from the os_atomic_dec. It
* could be racing with other threads to be the last thread in the
* process, and the last thread in the process will tear down the proc
* structure and zero-out task->bsd_info.
*/
- threadcnt = hw_atomic_sub(&task->active_thread_count, 1);
+ threadcnt = os_atomic_dec(&task->active_thread_count, relaxed);
/*
* If we are the last thread to terminate and the task is
/* since we're the last thread in this process, trace out the command name too */
long args[4] = {};
kdbg_trace_string(thread->task->bsd_info, &args[0], &args[1], &args[2], &args[3]);
+#if MONOTONIC
+ if (kdebug_debugid_enabled(DBG_MT_INSTRS_CYCLES_PROC_EXIT)) {
+ uint64_t counts[MT_CORE_NFIXED];
+ uint64_t task_user_time;
+ uint64_t task_system_time;
+ mt_fixed_task_counts(task, counts);
+ /* since the thread time is not yet added to the task */
+ task_user_time = task->total_user_time + timer_grab(&thread->user_timer);
+ task_system_time = task->total_system_time + timer_grab(&thread->system_timer);
+ KDBG_RELEASE((DBG_MT_INSTRS_CYCLES_PROC_EXIT),
+#ifdef MT_CORE_INSTRS
+ counts[MT_CORE_INSTRS],
+#else /* defined(MT_CORE_INSTRS) */
+ 0,
+#endif/* !defined(MT_CORE_INSTRS) */
+ counts[MT_CORE_CYCLES],
+ task_system_time, task_user_time);
+ }
+#endif/* MONOTONIC */
KDBG_RELEASE(TRACE_STRING_PROC_EXIT, args[0], args[1], args[2], args[3]);
}
assert((thread->sched_flags & TH_SFLAG_RW_PROMOTED) == 0);
assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0);
assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0);
- assert(thread->promotions == 0);
- assert(thread->was_promoted_on_wakeup == 0);
+ assert((thread->sched_flags & TH_SFLAG_THREAD_GROUP_AUTO_JOIN) == 0);
+ assert(thread->th_work_interval_flags == TH_WORK_INTERVAL_FLAGS_NONE);
+ assert(thread->kern_promotion_schedpri == 0);
assert(thread->waiting_for_mutex == NULL);
assert(thread->rwlock_count == 0);
+ assert(thread->handoff_thread == THREAD_NULL);
+ assert(thread->th_work_interval == NULL);
thread_unlock(thread);
/* splsched */
assert(os_ref_get_count(&thread->ref_count) == 0);
- assert(thread_owned_workloops_count(thread) == 0);
-
if (!(thread->state & TH_TERMINATE2)) {
panic("thread_deallocate: thread not properly terminated\n");
}
}
if (thread->thread_io_stats) {
- kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
+ kheap_free(KHEAP_DATA_BUFFERS, thread->thread_io_stats,
+ sizeof(struct io_stat_info));
}
if (thread->kernel_stack != 0) {
thread->thread_magic = 0;
#endif /* MACH_ASSERT */
- zfree(thread_zone, thread);
-}
-
-void
-thread_starts_owning_workloop(thread_t thread)
-{
- atomic_fetch_add_explicit(&thread->kqwl_owning_count, 1,
- memory_order_relaxed);
-}
-
-void
-thread_ends_owning_workloop(thread_t thread)
-{
- __assert_only uint32_t count;
- count = atomic_fetch_sub_explicit(&thread->kqwl_owning_count, 1,
- memory_order_relaxed);
- assert(count > 0);
-}
+ lck_mtx_lock(&tasks_threads_lock);
+ assert(terminated_threads_count > 0);
+ queue_remove(&terminated_threads, thread, thread_t, threads);
+ terminated_threads_count--;
+ lck_mtx_unlock(&tasks_threads_lock);
-uint32_t
-thread_owned_workloops_count(thread_t thread)
-{
- return atomic_load_explicit(&thread->kqwl_owning_count,
- memory_order_relaxed);
+ zfree(thread_zone, thread);
}
/*
}
/*
- * thread_exception_daemon:
+ * thread_read_deallocate:
+ *
+ * Drop a reference on thread read port.
+ */
+void
+thread_read_deallocate(
+ thread_read_t thread_read)
+{
+ return thread_deallocate((thread_t)thread_read);
+}
+
+
+/*
+ * thread_exception_queue_invoke:
*
* Deliver EXC_{RESOURCE,GUARD} exception
*/
static void
-thread_exception_daemon(void)
+thread_exception_queue_invoke(mpsc_queue_chain_t elm,
+ __assert_only mpsc_daemon_queue_t dq)
{
struct thread_exception_elt *elt;
task_t task;
thread_t thread;
exception_type_t etype;
- simple_lock(&thread_exception_lock, LCK_GRP_NULL);
- while ((elt = (struct thread_exception_elt *)dequeue_head(&thread_exception_queue)) != NULL) {
- simple_unlock(&thread_exception_lock);
-
- etype = elt->exception_type;
- task = elt->exception_task;
- thread = elt->exception_thread;
- assert_thread_magic(thread);
+ assert(dq == &thread_exception_queue);
+ elt = mpsc_queue_element(elm, struct thread_exception_elt, link);
- kfree(elt, sizeof(*elt));
-
- /* wait for all the threads in the task to terminate */
- task_lock(task);
- task_wait_till_threads_terminate_locked(task);
- task_unlock(task);
-
- /* Consumes the task ref returned by task_generate_corpse_internal */
- task_deallocate(task);
- /* Consumes the thread ref returned by task_generate_corpse_internal */
- thread_deallocate(thread);
+ etype = elt->exception_type;
+ task = elt->exception_task;
+ thread = elt->exception_thread;
+ assert_thread_magic(thread);
- /* Deliver the notification, also clears the corpse. */
- task_deliver_crash_notification(task, thread, etype, 0);
+ kfree(elt, sizeof(*elt));
- simple_lock(&thread_exception_lock, LCK_GRP_NULL);
- }
+ /* wait for all the threads in the task to terminate */
+ task_lock(task);
+ task_wait_till_threads_terminate_locked(task);
+ task_unlock(task);
- assert_wait((event_t)&thread_exception_queue, THREAD_UNINT);
- simple_unlock(&thread_exception_lock);
+ /* Consumes the task ref returned by task_generate_corpse_internal */
+ task_deallocate(task);
+ /* Consumes the thread ref returned by task_generate_corpse_internal */
+ thread_deallocate(thread);
- thread_block((thread_continue_t)thread_exception_daemon);
+ /* Deliver the notification, also clears the corpse. */
+ task_deliver_crash_notification(task, thread, etype, 0);
}
/*
elt->exception_task = task;
elt->exception_thread = thread;
- simple_lock(&thread_exception_lock, LCK_GRP_NULL);
- enqueue_tail(&thread_exception_queue, (queue_entry_t)elt);
- simple_unlock(&thread_exception_lock);
-
- thread_wakeup((event_t)&thread_exception_queue);
+ mpsc_daemon_enqueue(&thread_exception_queue, &elt->link,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
}
/*
*dst_thread->thread_io_stats = *src_thread->thread_io_stats;
}
-/*
- * thread_terminate_daemon:
- *
- * Perform final clean up for terminating threads.
- */
static void
-thread_terminate_daemon(void)
+thread_terminate_queue_invoke(mpsc_queue_chain_t e,
+ __assert_only mpsc_daemon_queue_t dq)
{
- thread_t self, thread;
- task_t task;
-
- self = current_thread();
- self->options |= TH_OPT_SYSTEM_CRITICAL;
-
- (void)splsched();
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
-
-thread_terminate_start:
- while ((thread = qe_dequeue_head(&thread_terminate_queue, struct thread, runq_links)) != THREAD_NULL) {
- assert_thread_magic(thread);
-
- /*
- * if marked for crash reporting, skip reaping.
- * The corpse delivery thread will clear bit and enqueue
- * for reaping when done
- */
- if (thread->inspection) {
- enqueue_tail(&crashed_threads_queue, &thread->runq_links);
- continue;
- }
-
- simple_unlock(&thread_terminate_lock);
- (void)spllo();
-
- task = thread->task;
-
- task_lock(task);
- task->total_user_time += timer_grab(&thread->user_timer);
- task->total_ptime += timer_grab(&thread->ptime);
- task->total_runnable_time += timer_grab(&thread->runnable_timer);
- if (thread->precise_user_kernel_time) {
- task->total_system_time += timer_grab(&thread->system_timer);
- } else {
- task->total_user_time += timer_grab(&thread->system_timer);
- }
-
- task->c_switch += thread->c_switch;
- task->p_switch += thread->p_switch;
- task->ps_switch += thread->ps_switch;
-
- task->syscalls_unix += thread->syscalls_unix;
- task->syscalls_mach += thread->syscalls_mach;
-
- task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
- task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
- task->task_gpu_ns += ml_gpu_stat(thread);
- task->task_energy += ml_energy_stat(thread);
-
-#if MONOTONIC
- mt_terminate_update(task, thread);
-#endif /* MONOTONIC */
+ thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
+ task_t task = thread->task;
- thread_update_qos_cpu_time(thread);
+ assert(dq == &thread_terminate_queue);
- queue_remove(&task->threads, thread, thread_t, task_threads);
- task->thread_count--;
-
- /*
- * If the task is being halted, and there is only one thread
- * left in the task after this one, then wakeup that thread.
- */
- if (task->thread_count == 1 && task->halting) {
- thread_wakeup((event_t)&task->halting);
- }
+ task_lock(task);
+ /*
+ * if marked for crash reporting, skip reaping.
+ * The corpse delivery thread will clear bit and enqueue
+ * for reaping when done
+ *
+ * Note: the inspection field is set under the task lock
+ *
+ * FIXME[mad]: why enqueue for termination before `inspection` is false ?
+ */
+ if (__improbable(thread->inspection)) {
+ simple_lock(&crashed_threads_lock, &thread_lck_grp);
task_unlock(task);
- lck_mtx_lock(&tasks_threads_lock);
- queue_remove(&threads, thread, thread_t, threads);
- threads_count--;
- lck_mtx_unlock(&tasks_threads_lock);
-
- thread_deallocate(thread);
-
- (void)splsched();
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
+ enqueue_tail(&crashed_threads_queue, &thread->runq_links);
+ simple_unlock(&crashed_threads_lock);
+ return;
}
- while ((thread = qe_dequeue_head(&thread_deallocate_queue, struct thread, runq_links)) != THREAD_NULL) {
- assert_thread_magic(thread);
- simple_unlock(&thread_terminate_lock);
- (void)spllo();
+ task->total_user_time += timer_grab(&thread->user_timer);
+ task->total_ptime += timer_grab(&thread->ptime);
+ task->total_runnable_time += timer_grab(&thread->runnable_timer);
+ if (thread->precise_user_kernel_time) {
+ task->total_system_time += timer_grab(&thread->system_timer);
+ } else {
+ task->total_user_time += timer_grab(&thread->system_timer);
+ }
- thread_deallocate_complete(thread);
+ task->c_switch += thread->c_switch;
+ task->p_switch += thread->p_switch;
+ task->ps_switch += thread->ps_switch;
- (void)splsched();
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- }
+ task->syscalls_unix += thread->syscalls_unix;
+ task->syscalls_mach += thread->syscalls_mach;
- struct turnstile *turnstile;
- while ((turnstile = qe_dequeue_head(&turnstile_deallocate_queue, struct turnstile, ts_deallocate_link)) != TURNSTILE_NULL) {
- simple_unlock(&thread_terminate_lock);
- (void)spllo();
+ task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
+ task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
+ task->task_gpu_ns += ml_gpu_stat(thread);
+ task->task_energy += ml_energy_stat(thread);
+ task->decompressions += thread->decompressions;
- turnstile_destroy(turnstile);
+#if MONOTONIC
+ mt_terminate_update(task, thread);
+#endif /* MONOTONIC */
- (void)splsched();
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- }
+ thread_update_qos_cpu_time(thread);
- queue_entry_t qe;
+ queue_remove(&task->threads, thread, thread_t, task_threads);
+ task->thread_count--;
/*
- * see workq_deallocate_enqueue: struct workqueue is opaque to thread.c and
- * we just link pieces of memory here
+ * If the task is being halted, and there is only one thread
+ * left in the task after this one, then wakeup that thread.
*/
- while ((qe = dequeue_head(&workq_deallocate_queue))) {
- simple_unlock(&thread_terminate_lock);
- (void)spllo();
+ if (task->thread_count == 1 && task->halting) {
+ thread_wakeup((event_t)&task->halting);
+ }
- workq_destroy((struct workqueue *)qe);
+ task_unlock(task);
- (void)splsched();
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- }
+ lck_mtx_lock(&tasks_threads_lock);
+ queue_remove(&threads, thread, thread_t, threads);
+ threads_count--;
+ queue_enter(&terminated_threads, thread, thread_t, threads);
+ terminated_threads_count++;
+ lck_mtx_unlock(&tasks_threads_lock);
- /*
- * Check if something enqueued in thread terminate/deallocate queue
- * while processing workq deallocate queue
- */
- if (!queue_empty(&thread_terminate_queue) ||
- !queue_empty(&thread_deallocate_queue) ||
- !queue_empty(&turnstile_deallocate_queue)) {
- goto thread_terminate_start;
- }
+ thread_deallocate(thread);
+}
- assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
- simple_unlock(&thread_terminate_lock);
- /* splsched */
+static void
+thread_deallocate_queue_invoke(mpsc_queue_chain_t e,
+ __assert_only mpsc_daemon_queue_t dq)
+{
+ thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
- self->options &= ~TH_OPT_SYSTEM_CRITICAL;
- thread_block((thread_continue_t)thread_terminate_daemon);
- /*NOTREACHED*/
+ assert(dq == &thread_deallocate_queue);
+
+ thread_deallocate_complete(thread);
}
/*
{
KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id);
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- enqueue_tail(&thread_terminate_queue, &thread->runq_links);
- simple_unlock(&thread_terminate_lock);
-
- thread_wakeup((event_t)&thread_terminate_queue);
+ mpsc_daemon_enqueue(&thread_terminate_queue, &thread->mpsc_links,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
}
/*
thread_deallocate_enqueue(
thread_t thread)
{
- spl_t s = splsched();
-
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- enqueue_tail(&thread_deallocate_queue, &thread->runq_links);
- simple_unlock(&thread_terminate_lock);
-
- thread_wakeup((event_t)&thread_terminate_queue);
- splx(s);
-}
-
-/*
- * turnstile_deallocate_enqueue:
- *
- * Enqueue a turnstile for final deallocation.
- */
-void
-turnstile_deallocate_enqueue(
- struct turnstile *turnstile)
-{
- spl_t s = splsched();
-
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- enqueue_tail(&turnstile_deallocate_queue, &turnstile->ts_deallocate_link);
- simple_unlock(&thread_terminate_lock);
-
- thread_wakeup((event_t)&thread_terminate_queue);
- splx(s);
-}
-
-/*
- * workq_deallocate_enqueue:
- *
- * Enqueue a workqueue for final deallocation.
- */
-void
-workq_deallocate_enqueue(
- struct workqueue *wq)
-{
- spl_t s = splsched();
-
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
- /*
- * this is just to delay a zfree(), so we link the memory with no regards
- * for how the struct looks like.
- */
- enqueue_tail(&workq_deallocate_queue, (queue_entry_t)wq);
- simple_unlock(&thread_terminate_lock);
-
- thread_wakeup((event_t)&thread_terminate_queue);
- splx(s);
+ mpsc_daemon_enqueue(&thread_deallocate_queue, &thread->mpsc_links,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
}
/*
* who are no longer being inspected.
*/
void
-thread_terminate_crashed_threads()
+thread_terminate_crashed_threads(void)
{
thread_t th_remove;
- boolean_t should_wake_terminate_queue = FALSE;
- spl_t s = splsched();
- simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
+ simple_lock(&crashed_threads_lock, &thread_lck_grp);
/*
* loop through the crashed threads queue
* to put any threads that are not being inspected anymore
assert(th_remove != current_thread());
if (th_remove->inspection == FALSE) {
- re_queue_tail(&thread_terminate_queue, &th_remove->runq_links);
- should_wake_terminate_queue = TRUE;
+ remqueue(&th_remove->runq_links);
+ mpsc_daemon_enqueue(&thread_terminate_queue, &th_remove->mpsc_links,
+ MPSC_QUEUE_NONE);
}
}
- simple_unlock(&thread_terminate_lock);
- splx(s);
- if (should_wake_terminate_queue == TRUE) {
- thread_wakeup((event_t)&thread_terminate_queue);
- }
+ simple_unlock(&crashed_threads_lock);
}
/*
- * thread_stack_daemon:
+ * thread_stack_queue_invoke:
*
* Perform stack allocation as required due to
* invoke failures.
*/
static void
-thread_stack_daemon(void)
+thread_stack_queue_invoke(mpsc_queue_chain_t elm,
+ __assert_only mpsc_daemon_queue_t dq)
{
- thread_t thread;
- spl_t s;
-
- s = splsched();
- simple_lock(&thread_stack_lock, LCK_GRP_NULL);
-
- while ((thread = qe_dequeue_head(&thread_stack_queue, struct thread, runq_links)) != THREAD_NULL) {
- assert_thread_magic(thread);
-
- simple_unlock(&thread_stack_lock);
- splx(s);
+ thread_t thread = mpsc_queue_element(elm, struct thread, mpsc_links);
- /* allocate stack with interrupts enabled so that we can call into VM */
- stack_alloc(thread);
+ assert(dq == &thread_stack_queue);
- KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
-
- s = splsched();
- thread_lock(thread);
- thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
- thread_unlock(thread);
+ /* allocate stack with interrupts enabled so that we can call into VM */
+ stack_alloc(thread);
- simple_lock(&thread_stack_lock, LCK_GRP_NULL);
- }
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
- assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
- simple_unlock(&thread_stack_lock);
+ spl_t s = splsched();
+ thread_lock(thread);
+ thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
+ thread_unlock(thread);
splx(s);
-
- thread_block((thread_continue_t)thread_stack_daemon);
- /*NOTREACHED*/
}
/*
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);
assert_thread_magic(thread);
- simple_lock(&thread_stack_lock, LCK_GRP_NULL);
- enqueue_tail(&thread_stack_queue, &thread->runq_links);
- simple_unlock(&thread_stack_lock);
-
- thread_wakeup((event_t)&thread_stack_queue);
+ mpsc_daemon_enqueue(&thread_stack_queue, &thread->mpsc_links,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
}
void
thread_daemon_init(void)
{
kern_return_t result;
- thread_t thread = NULL;
- simple_lock_init(&thread_terminate_lock, 0);
- queue_init(&thread_terminate_queue);
- queue_init(&thread_deallocate_queue);
- queue_init(&workq_deallocate_queue);
- queue_init(&turnstile_deallocate_queue);
- queue_init(&crashed_threads_queue);
+ thread_deallocate_daemon_init();
- result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
- if (result != KERN_SUCCESS) {
- panic("thread_daemon_init: thread_terminate_daemon");
- }
+ thread_deallocate_daemon_register_queue(&thread_terminate_queue,
+ thread_terminate_queue_invoke);
- thread_deallocate(thread);
+ thread_deallocate_daemon_register_queue(&thread_deallocate_queue,
+ thread_deallocate_queue_invoke);
- simple_lock_init(&thread_stack_lock, 0);
- queue_init(&thread_stack_queue);
+ simple_lock_init(&crashed_threads_lock, 0);
+ queue_init(&crashed_threads_queue);
- result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT_HIGH, &thread);
+ result = mpsc_daemon_queue_init_with_thread(&thread_stack_queue,
+ thread_stack_queue_invoke, BASEPRI_PREEMPT_HIGH,
+ "daemon.thread-stack");
if (result != KERN_SUCCESS) {
panic("thread_daemon_init: thread_stack_daemon");
}
- thread_deallocate(thread);
-
- simple_lock_init(&thread_exception_lock, 0);
- queue_init(&thread_exception_queue);
-
- result = kernel_thread_start_priority((thread_continue_t)thread_exception_daemon, NULL, MINPRI_KERNEL, &thread);
+ result = mpsc_daemon_queue_init_with_thread(&thread_exception_queue,
+ thread_exception_queue_invoke, MINPRI_KERNEL,
+ "daemon.thread-exception");
if (result != KERN_SUCCESS) {
panic("thread_daemon_init: thread_exception_daemon");
}
-
- thread_deallocate(thread);
}
-#define TH_OPTION_NONE 0x00
-#define TH_OPTION_NOCRED 0x01
-#define TH_OPTION_NOSUSP 0x02
-#define TH_OPTION_WORKQ 0x04
+__options_decl(thread_create_internal_options_t, uint32_t, {
+ TH_OPTION_NONE = 0x00,
+ TH_OPTION_NOCRED = 0x01,
+ TH_OPTION_NOSUSP = 0x02,
+ TH_OPTION_WORKQ = 0x04,
+ TH_OPTION_IMMOVABLE = 0x08,
+ TH_OPTION_PINNED = 0x10,
+});
/*
* Create a new thread.
thread_create_internal(
task_t parent_task,
integer_t priority,
- thread_continue_t continuation,
+ thread_continue_t continuation,
void *parameter,
- int options,
+ thread_create_internal_options_t options,
thread_t *out_thread)
{
thread_t new_thread;
- static thread_t first_thread;
+ static thread_t first_thread;
+ ipc_thread_init_options_t init_options = IPC_THREAD_INIT_NONE;
/*
* Allocate a thread and initialize static fields
}
if (new_thread != first_thread) {
- *new_thread = thread_template;
+ init_thread_from_template(new_thread);
+ }
+
+ if (options & TH_OPTION_PINNED) {
+ init_options |= IPC_THREAD_INIT_PINNED;
+ }
+
+ if (options & TH_OPTION_IMMOVABLE) {
+ init_options |= IPC_THREAD_INIT_IMMOVABLE;
}
os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2);
+#if DEBUG || DEVELOPMENT
+ queue_init(&new_thread->t_temp_alloc_list);
+#endif /* DEBUG || DEVELOPMENT */
#ifdef MACH_BSD
new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
thread_lock_init(new_thread);
wake_lock_init(new_thread);
- lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
+ lck_mtx_init(&new_thread->mutex, &thread_lck_grp, LCK_ATTR_NULL);
- ipc_thread_init(new_thread);
+ ipc_thread_init(new_thread, init_options);
new_thread->continuation = continuation;
new_thread->parameter = parameter;
new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE;
- priority_queue_init(&new_thread->inheritor_queue,
- PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
+ priority_queue_init(&new_thread->sched_inheritor_queue);
+ priority_queue_init(&new_thread->base_inheritor_queue);
+#if CONFIG_SCHED_CLUTCH
+ priority_queue_entry_init(&new_thread->th_clutch_runq_link);
+ priority_queue_entry_init(&new_thread->th_clutch_pri_link);
+#endif /* CONFIG_SCHED_CLUTCH */
+
+#if CONFIG_SCHED_EDGE
+ new_thread->th_bound_cluster_enqueued = false;
+#endif /* CONFIG_SCHED_EDGE */
/* Allocate I/O Statistics structure */
- new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
+ new_thread->thread_io_stats = kheap_alloc(KHEAP_DATA_BUFFERS,
+ sizeof(struct io_stat_info), Z_WAITOK | Z_ZERO);
assert(new_thread->thread_io_stats != NULL);
- bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
- new_thread->sync_ipc_overrides = 0;
#if KASAN
kasan_init_thread(&new_thread->kasan_data);
#endif
+#if CONFIG_KSANCOV
+ new_thread->ksancov_data = NULL;
+#endif
+
#if CONFIG_IOSCHED
/* Clear out the I/O Scheduling info for AppleFSCompression */
new_thread->decmp_upl = NULL;
#endif /* CONFIG_IOSCHED */
+ new_thread->thread_region_page_shift = 0;
+
#if DEVELOPMENT || DEBUG
task_lock(parent_task);
uint16_t thread_limit = parent_task->task_thread_limit;
#endif /* MACH_BSD */
ipc_thread_disable(new_thread);
ipc_thread_terminate(new_thread);
- kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info));
+ kheap_free(KHEAP_DATA_BUFFERS, new_thread->thread_io_stats,
+ sizeof(struct io_stat_info));
lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
machine_thread_destroy(new_thread);
zfree(thread_zone, new_thread);
new_thread->max_priority = parent_task->max_priority;
new_thread->task_priority = parent_task->priority;
+#if CONFIG_THREAD_GROUPS
+ thread_group_init_thread(new_thread, parent_task);
+#endif /* CONFIG_THREAD_GROUPS */
+
int new_priority = (priority < 0) ? parent_task->priority: priority;
new_priority = (priority < 0)? parent_task->priority: priority;
if (new_priority > new_thread->max_priority) {
new_priority = new_thread->max_priority;
}
-#if CONFIG_EMBEDDED
+#if !defined(XNU_TARGET_OS_OSX)
if (new_priority < MAXPRI_THROTTLE) {
new_priority = MAXPRI_THROTTLE;
}
-#endif /* CONFIG_EMBEDDED */
+#endif /* !defined(XNU_TARGET_OS_OSX) */
new_thread->importance = new_priority - new_thread->task_priority;
#if defined(CONFIG_SCHED_TIMESHARE_CORE)
new_thread->sched_stamp = sched_tick;
+#if CONFIG_SCHED_CLUTCH
+ new_thread->pri_shift = sched_clutch_thread_pri_shift(new_thread, new_thread->th_sched_bucket);
+#else /* CONFIG_SCHED_CLUTCH */
new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];
+#endif /* CONFIG_SCHED_CLUTCH */
#endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
-#if CONFIG_EMBEDDED
if (parent_task->max_priority <= MAXPRI_THROTTLE) {
sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED);
}
-#endif /* CONFIG_EMBEDDED */
thread_policy_create(new_thread);
parent_task->thread_count++;
/* So terminating threads don't need to take the task lock to decrement */
- hw_atomic_add(&parent_task->active_thread_count, 1);
-
+ os_atomic_inc(&parent_task->active_thread_count, relaxed);
queue_enter(&threads, new_thread, thread_t, threads);
threads_count++;
}
new_thread->corpse_dup = FALSE;
new_thread->turnstile = turnstile_alloc();
+
+
*out_thread = new_thread;
if (kdebug_enable) {
}
static kern_return_t
-thread_create_internal2(
- task_t task,
- thread_t *new_thread,
- boolean_t from_user,
- thread_continue_t continuation)
+thread_create_with_options_internal(
+ task_t task,
+ thread_t *new_thread,
+ boolean_t from_user,
+ thread_create_internal_options_t options,
+ thread_continue_t continuation)
{
kern_return_t result;
- thread_t thread;
+ thread_t thread;
if (task == TASK_NULL || task == kernel_task) {
return KERN_INVALID_ARGUMENT;
}
- result = thread_create_internal(task, -1, continuation, NULL, TH_OPTION_NONE, &thread);
+#if CONFIG_MACF
+ if (from_user && current_task() != task &&
+ mac_proc_check_remote_thread_create(task, -1, NULL, 0) != 0) {
+ return KERN_DENIED;
+ }
+#endif
+
+ result = thread_create_internal(task, -1, continuation, NULL, options, &thread);
if (result != KERN_SUCCESS) {
return result;
}
task_t task,
thread_t *new_thread)
{
- return thread_create_internal2(task, new_thread, FALSE, (thread_continue_t)thread_bootstrap_return);
+ return thread_create_with_options_internal(task, new_thread, FALSE, TH_OPTION_NONE,
+ (thread_continue_t)thread_bootstrap_return);
+}
+
+/*
+ * Create a thread that has its itk_self pinned
+ * Deprecated, should be cleanup once rdar://70892168 lands
+ */
+kern_return_t
+thread_create_pinned(
+ task_t task,
+ thread_t *new_thread)
+{
+ return thread_create_with_options_internal(task, new_thread, FALSE,
+ TH_OPTION_PINNED | TH_OPTION_IMMOVABLE, (thread_continue_t)thread_bootstrap_return);
+}
+
+kern_return_t
+thread_create_immovable(
+ task_t task,
+ thread_t *new_thread)
+{
+ return thread_create_with_options_internal(task, new_thread, FALSE,
+ TH_OPTION_IMMOVABLE, (thread_continue_t)thread_bootstrap_return);
}
kern_return_t
task_t task,
thread_t *new_thread)
{
- return thread_create_internal2(task, new_thread, TRUE, (thread_continue_t)thread_bootstrap_return);
+ return thread_create_with_options_internal(task, new_thread, TRUE, TH_OPTION_NONE,
+ (thread_continue_t)thread_bootstrap_return);
}
kern_return_t
thread_t *new_thread,
thread_continue_t continuation)
{
- return thread_create_internal2(task, new_thread, FALSE, continuation);
+ return thread_create_with_options_internal(task, new_thread, FALSE, TH_OPTION_NONE, continuation);
}
/*
kern_return_t
thread_create_waiting(
- task_t task,
- thread_continue_t continuation,
- event_t event,
- thread_t *new_thread)
+ task_t task,
+ thread_continue_t continuation,
+ event_t event,
+ th_create_waiting_options_t options,
+ thread_t *new_thread)
{
+ thread_create_internal_options_t ci_options = TH_OPTION_NONE;
+
+ assert((options & ~TH_CREATE_WAITING_OPTION_MASK) == 0);
+ if (options & TH_CREATE_WAITING_OPTION_PINNED) {
+ ci_options |= TH_OPTION_PINNED;
+ }
+ if (options & TH_CREATE_WAITING_OPTION_IMMOVABLE) {
+ ci_options |= TH_OPTION_IMMOVABLE;
+ }
+
return thread_create_waiting_internal(task, continuation, event,
- kThreadWaitNone, TH_OPTION_NONE, new_thread);
+ kThreadWaitNone, ci_options, new_thread);
}
return KERN_INVALID_ARGUMENT;
}
+#if CONFIG_MACF
+ if (from_user && current_task() != task &&
+ mac_proc_check_remote_thread_create(task, flavor, new_state, new_state_count) != 0) {
+ return KERN_DENIED;
+ }
+#endif
+
result = thread_create_internal(task, -1,
(thread_continue_t)thread_bootstrap_return, NULL,
TH_OPTION_NONE, &thread);
thread_continue_t continuation,
thread_t *new_thread)
{
- int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ;
+ /*
+ * Create thread, but don't pin control port just yet, in case someone calls
+ * task_threads() and deallocates pinned port before kernel copyout happens,
+ * which will result in pinned port guard exception. Instead, pin and make
+ * it immovable atomically at copyout during workq_setup_and_run().
+ */
+ int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ | TH_OPTION_IMMOVABLE;
return thread_create_waiting_internal(task, continuation, NULL,
kThreadWaitParkedWorkQueue, options, new_thread);
}
stack_alloc(thread);
assert(thread->kernel_stack != 0);
-#if CONFIG_EMBEDDED
+#if !defined(XNU_TARGET_OS_OSX)
if (priority > BASEPRI_KERNEL)
#endif
thread->reserved_stack = thread->kernel_stack;
return KERN_INVALID_ARGUMENT;
}
- identifier_info = (thread_identifier_info_t) thread_info_out;
+ identifier_info = __IGNORE_WCASTALIGN((thread_identifier_info_t)thread_info_out);
s = splsched();
thread_lock(thread);
return KERN_SUCCESS;
} else if (flavor == THREAD_EXTENDED_INFO) {
thread_basic_info_data_t basic_info;
- thread_extended_info_t extended_info = (thread_extended_info_t) thread_info_out;
+ thread_extended_info_t extended_info = __IGNORE_WCASTALIGN((thread_extended_info_t)thread_info_out);
if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {
return KERN_INVALID_ARGUMENT;
* the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
*/
retrieve_thread_basic_info(thread, &basic_info);
- extended_info->pth_user_time = ((basic_info.user_time.seconds * (integer_t)NSEC_PER_SEC) + (basic_info.user_time.microseconds * (integer_t)NSEC_PER_USEC));
- extended_info->pth_system_time = ((basic_info.system_time.seconds * (integer_t)NSEC_PER_SEC) + (basic_info.system_time.microseconds * (integer_t)NSEC_PER_USEC));
+ extended_info->pth_user_time = (((uint64_t)basic_info.user_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.user_time.microseconds * NSEC_PER_USEC));
+ extended_info->pth_system_time = (((uint64_t)basic_info.system_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.system_time.microseconds * NSEC_PER_USEC));
extended_info->pth_cpu_usage = basic_info.cpu_usage;
extended_info->pth_policy = basic_info.policy;
return KERN_INVALID_ARGUMENT;
}
- dbg_info = (thread_debug_info_internal_t) thread_info_out;
+ dbg_info = __IGNORE_WCASTALIGN((thread_debug_info_internal_t)thread_info_out);
dbg_info->page_creation_count = thread->t_page_creation_count;
*thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;
/* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
interrupt_state = ml_set_interrupts_enabled(FALSE);
processor = current_processor();
- timer_update(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time());
+ timer_update(processor->thread_timer, mach_absolute_time());
runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
ml_set_interrupts_enabled(interrupt_state);
return KERN_INVALID_ARGUMENT;
}
- assert(host_priv == &realhost);
-
if (prev_state) {
*prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
}
}
}
-
/*
* XXX assuming current thread only, for now...
*/
void
thread_guard_violation(thread_t thread,
- mach_exception_data_type_t code, mach_exception_data_type_t subcode)
+ mach_exception_data_type_t code, mach_exception_data_type_t subcode, boolean_t fatal)
{
assert(thread == current_thread());
- /* don't set up the AST for kernel threads */
+ /* Don't set up the AST for kernel threads; this check is needed to ensure
+ * that the guard_exc_* fields in the thread structure are set only by the
+ * current thread and therefore, don't require a lock.
+ */
if (thread->task == kernel_task) {
return;
}
- spl_t s = splsched();
+ assert(EXC_GUARD_DECODE_GUARD_TYPE(code));
+
/*
* Use the saved state area of the thread structure
* to store all info required to handle the AST when
- * returning to userspace
+ * returning to userspace. It's possible that there is
+ * already a pending guard exception. If it's non-fatal,
+ * it can only be over-written by a fatal exception code.
*/
- assert(EXC_GUARD_DECODE_GUARD_TYPE(code));
+ if (thread->guard_exc_info.code && (thread->guard_exc_fatal || !fatal)) {
+ return;
+ }
+
thread->guard_exc_info.code = code;
thread->guard_exc_info.subcode = subcode;
+ thread->guard_exc_fatal = fatal ? 1 : 0;
+
+ spl_t s = splsched();
thread_ast_set(thread, AST_GUARD);
ast_propagate(thread);
-
splx(s);
}
t->guard_exc_info.code = 0;
t->guard_exc_info.subcode = 0;
+ t->guard_exc_fatal = 0;
switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) {
case GUARD_TYPE_NONE:
}
/* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */
- printf("process %s[%d] thread %llu caught burning CPU! "
- "It used more than %d%% CPU over %u seconds "
- "(actual recent usage: %d%% over ~%llu seconds). "
- "Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys) "
- "ledger balance: %lld mabs credit: %lld mabs debit: %lld mabs "
- "limit: %llu mabs period: %llu ns last refill: %llu ns%s.\n",
- procname, pid, tid,
- percentage, interval_sec,
- usage_percent,
- (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC,
+ printf("process %s[%d] thread %llu caught burning CPU! It used more than %d%% CPU over %u seconds\n",
+ procname, pid, tid, percentage, interval_sec);
+ printf(" (actual recent usage: %d%% over ~%llu seconds)\n",
+ usage_percent, (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC);
+ printf(" Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys)\n",
thread_total_time.seconds, thread_total_time.microseconds,
thread_user_time.seconds, thread_user_time.microseconds,
- thread_system_time.seconds, thread_system_time.microseconds,
- lei.lei_balance, lei.lei_credit, lei.lei_debit,
+ thread_system_time.seconds, thread_system_time.microseconds);
+ printf(" Ledger balance: %lld; mabs credit: %lld; mabs debit: %lld\n",
+ lei.lei_balance, lei.lei_credit, lei.lei_debit);
+ printf(" mabs limit: %llu; mabs period: %llu ns; last refill: %llu ns%s.\n",
lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill,
(fatal ? " [fatal violation]" : ""));
* This calculation is the converse to the one in thread_set_cpulimit().
*/
absolutetime_to_nanoseconds(abstime, &limittime);
- *percentage = (limittime * 100ULL) / *interval_ns;
+ *percentage = (uint8_t)((limittime * 100ULL) / *interval_ns);
assert(*percentage <= 100);
if (thread->options & TH_OPT_PROC_CPULIMIT) {
* thread_set_voucher_name - reset the voucher port name bound to this thread
*
* Conditions: nothing locked
- *
- * If we already converted the previous name to a cached voucher
- * reference, then we discard that reference here. The next lookup
- * will cache it again.
*/
kern_return_t
ipc_voucher_t voucher;
ledger_t bankledger = NULL;
struct thread_group *banktg = NULL;
+ uint32_t persona_id = 0;
if (MACH_PORT_DEAD == voucher_name) {
return KERN_INVALID_RIGHT;
return KERN_INVALID_ARGUMENT;
}
}
- bank_get_bank_ledger_and_thread_group(new_voucher, &bankledger, &banktg);
+ bank_get_bank_ledger_thread_group_and_persona(new_voucher, &bankledger, &banktg, &persona_id);
thread_mtx_lock(thread);
voucher = thread->ith_voucher;
thread_mtx_unlock(thread);
bank_swap_thread_bank_ledger(thread, bankledger);
+#if CONFIG_THREAD_GROUPS
+ thread_group_set_bank(thread, banktg);
+#endif /* CONFIG_THREAD_GROUPS */
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
(uintptr_t)thread_tid(thread),
(uintptr_t)voucher_name,
VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
- 1, 0);
+ persona_id, 0);
if (IPC_VOUCHER_NULL != voucher) {
ipc_voucher_release(voucher);
*
* Conditions: nothing locked
*
- * A reference to the voucher may be lazily pending, if someone set the voucher name
- * but nobody has done a lookup yet. In that case, we'll have to do the equivalent
- * lookup here.
- *
* NOTE: At the moment, there is no distinction between the current and effective
* vouchers because we only set them at the thread level currently.
*/
ipc_voucher_t *voucherp)
{
ipc_voucher_t voucher;
- mach_port_name_t voucher_name;
if (THREAD_NULL == thread) {
return KERN_INVALID_ARGUMENT;
thread_mtx_lock(thread);
voucher = thread->ith_voucher;
- /* if already cached, just return a ref */
if (IPC_VOUCHER_NULL != voucher) {
ipc_voucher_reference(voucher);
thread_mtx_unlock(thread);
return KERN_SUCCESS;
}
- voucher_name = thread->ith_voucher_name;
-
- /* convert the name to a port, then voucher reference */
- if (MACH_PORT_VALID(voucher_name)) {
- ipc_port_t port;
-
- if (KERN_SUCCESS !=
- ipc_object_copyin(thread->task->itk_space, voucher_name,
- MACH_MSG_TYPE_COPY_SEND, (ipc_object_t *)&port)) {
- thread->ith_voucher_name = MACH_PORT_NULL;
- thread_mtx_unlock(thread);
- *voucherp = IPC_VOUCHER_NULL;
- return KERN_SUCCESS;
- }
-
- /* convert to a voucher ref to return, and cache a ref on thread */
- voucher = convert_port_to_voucher(port);
- ipc_voucher_reference(voucher);
- thread->ith_voucher = voucher;
- thread_mtx_unlock(thread);
-
- KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
- MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
- (uintptr_t)thread_tid(thread),
- (uintptr_t)port,
- VM_KERNEL_ADDRPERM((uintptr_t)voucher),
- 2, 0);
-
-
- ipc_port_release_send(port);
- } else {
- thread_mtx_unlock(thread);
- }
+ thread_mtx_unlock(thread);
- *voucherp = voucher;
+ *voucherp = IPC_VOUCHER_NULL;
return KERN_SUCCESS;
}
* Conditions: callers holds a reference on the voucher.
* nothing locked.
*
- * We grab another reference to the voucher and bind it to the thread. Any lazy
- * binding is erased. The old voucher reference associated with the thread is
+ * We grab another reference to the voucher and bind it to the thread.
+ * The old voucher reference associated with the thread is
* discarded.
*/
kern_return_t
ipc_voucher_t old_voucher;
ledger_t bankledger = NULL;
struct thread_group *banktg = NULL;
+ uint32_t persona_id = 0;
if (THREAD_NULL == thread) {
return KERN_INVALID_ARGUMENT;
}
+ bank_get_bank_ledger_thread_group_and_persona(voucher, &bankledger, &banktg, &persona_id);
+
+ thread_mtx_lock(thread);
+ /*
+ * Once the thread is started, we will look at `ith_voucher` without
+ * holding any lock.
+ *
+ * Setting the voucher hence can only be done by current_thread() or
+ * before it started. "started" flips under the thread mutex and must be
+ * tested under it too.
+ */
if (thread != current_thread() && thread->started) {
+ thread_mtx_unlock(thread);
return KERN_INVALID_ARGUMENT;
}
ipc_voucher_reference(voucher);
- bank_get_bank_ledger_and_thread_group(voucher, &bankledger, &banktg);
-
- thread_mtx_lock(thread);
old_voucher = thread->ith_voucher;
thread->ith_voucher = voucher;
thread->ith_voucher_name = MACH_PORT_NULL;
thread_mtx_unlock(thread);
bank_swap_thread_bank_ledger(thread, bankledger);
+#if CONFIG_THREAD_GROUPS
+ thread_group_set_bank(thread, banktg);
+#endif /* CONFIG_THREAD_GROUPS */
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
(uintptr_t)thread_tid(thread),
(uintptr_t)MACH_PORT_NULL,
VM_KERNEL_ADDRPERM((uintptr_t)voucher),
- 3, 0);
+ persona_id, 0);
ipc_voucher_release(old_voucher);
return kr;
}
+#if CONFIG_THREAD_GROUPS
+/*
+ * Returns the current thread's voucher-carried thread group
+ *
+ * Reference is borrowed from this being the current voucher, so it does NOT
+ * return a reference to the group.
+ */
+struct thread_group *
+thread_get_current_voucher_thread_group(thread_t thread)
+{
+ assert(thread == current_thread());
+
+ if (thread->ith_voucher == NULL) {
+ return NULL;
+ }
+
+ ledger_t bankledger = NULL;
+ struct thread_group *banktg = NULL;
+
+ bank_get_bank_ledger_thread_group_and_persona(thread->ith_voucher, &bankledger, &banktg, NULL);
+
+ return banktg;
+}
+
+#endif /* CONFIG_THREAD_GROUPS */
boolean_t
thread_has_thread_name(thread_t th)
}
}
+void
+thread_get_thread_name(thread_t th, char* name)
+{
+ if (!name) {
+ return;
+ }
+ if ((th) && (th->uthread)) {
+ bsd_getthreadname(th->uthread, name);
+ } else {
+ name[0] = '\0';
+ }
+}
+
void
thread_set_honor_qlimit(thread_t thread)
{
return ret;
}
+void *
+thread_iokit_tls_get(uint32_t index)
+{
+ assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
+ return current_thread()->saved.iokit.tls[index];
+}
+
+void
+thread_iokit_tls_set(uint32_t index, void * data)
+{
+ assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
+ current_thread()->saved.iokit.tls[index] = data;
+}
+
uint64_t
thread_get_last_wait_duration(thread_t thread)
{
return thread->last_made_runnable_time - thread->last_run_time;
}
+integer_t
+thread_kern_get_pri(thread_t thr)
+{
+ return thr->base_pri;
+}
+
+void
+thread_kern_set_pri(thread_t thr, integer_t pri)
+{
+ sched_set_kernel_thread_priority(thr, pri);
+}
+
+integer_t
+thread_kern_get_kernel_maxpri(void)
+{
+ return MAXPRI_KERNEL;
+}
+/*
+ * thread_port_with_flavor_notify
+ *
+ * Called whenever the Mach port system detects no-senders on
+ * the thread inspect or read port. These ports are allocated lazily and
+ * should be deallocated here when there are no senders remaining.
+ */
+void
+thread_port_with_flavor_notify(mach_msg_header_t *msg)
+{
+ mach_no_senders_notification_t *notification = (void *)msg;
+ ipc_port_t port = notification->not_header.msgh_remote_port;
+ thread_t thread;
+ mach_thread_flavor_t flavor;
+ ipc_kobject_type_t kotype;
+
+ ip_lock(port);
+ if (port->ip_srights > 0) {
+ ip_unlock(port);
+ return;
+ }
+ thread = (thread_t)ipc_kobject_get(port);
+ kotype = ip_kotype(port);
+ if (thread != THREAD_NULL) {
+ assert((IKOT_THREAD_READ == kotype) || (IKOT_THREAD_INSPECT == kotype));
+ thread_reference_internal(thread);
+ }
+ ip_unlock(port);
+
+ if (thread == THREAD_NULL) {
+ /* The thread is exiting or disabled; it will eventually deallocate the port */
+ return;
+ }
+
+ if (kotype == IKOT_THREAD_READ) {
+ flavor = THREAD_FLAVOR_READ;
+ } else {
+ flavor = THREAD_FLAVOR_INSPECT;
+ }
+
+ thread_mtx_lock(thread);
+ ip_lock(port);
+ /*
+ * If the port is no longer active, then ipc_thread_terminate() ran
+ * and destroyed the kobject already. Just deallocate the task
+ * ref we took and go away.
+ *
+ * It is also possible that several nsrequests are in flight,
+ * only one shall NULL-out the port entry, and this is the one
+ * that gets to dealloc the port.
+ *
+ * Check for a stale no-senders notification. A call to any function
+ * that vends out send rights to this port could resurrect it between
+ * this notification being generated and actually being handled here.
+ */
+ if (!ip_active(port) ||
+ thread->ith_thread_ports[flavor] != port ||
+ port->ip_srights > 0) {
+ ip_unlock(port);
+ thread_mtx_unlock(thread);
+ thread_deallocate(thread);
+ return;
+ }
+
+ assert(thread->ith_thread_ports[flavor] == port);
+ thread->ith_thread_ports[flavor] = IP_NULL;
+ ipc_kobject_set_atomically(port, IKO_NULL, IKOT_NONE);
+ ip_unlock(port);
+ thread_mtx_unlock(thread);
+ thread_deallocate(thread);
+
+ ipc_port_dealloc_kernel(port);
+}
+
+/*
+ * The 'thread_region_page_shift' is used by footprint
+ * to specify the page size that it will use to
+ * accomplish its accounting work on the task being
+ * inspected. Since footprint uses a thread for each
+ * task that it works on, we need to keep the page_shift
+ * on a per-thread basis.
+ */
+
+int
+thread_self_region_page_shift(void)
+{
+ /*
+ * Return the page shift that this thread
+ * would like to use for its accounting work.
+ */
+ return current_thread()->thread_region_page_shift;
+}
+
+void
+thread_self_region_page_shift_set(
+ int pgshift)
+{
+ /*
+ * Set the page shift that this thread
+ * would like to use for its accounting work
+ * when dealing with a task.
+ */
+ current_thread()->thread_region_page_shift = pgshift;
+}
+
#if CONFIG_DTRACE
uint32_t
dtrace_get_thread_predcache(thread_t thread)
}
}
-boolean_t
-dtrace_get_thread_reentering(thread_t thread)
+uint16_t
+dtrace_get_thread_inprobe(thread_t thread)
{
if (thread != THREAD_NULL) {
- return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
+ return thread->t_dtrace_inprobe;
} else {
return 0;
}
}
#endif
+#if CONFIG_KSANCOV
+void **
+__sanitizer_get_thread_data(thread_t thread)
+{
+ return &thread->ksancov_data;
+}
+#endif
+
int64_t
dtrace_calc_thread_recent_vtime(thread_t thread)
{
uint64_t abstime = mach_absolute_time();
timer_t timer;
- timer = PROCESSOR_DATA(processor, thread_timer);
+ timer = processor->thread_timer;
return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
(abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
}
void
-dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
+dtrace_set_thread_inprobe(thread_t thread, uint16_t inprobe)
{
if (thread != THREAD_NULL) {
- if (vbool) {
- thread->options |= TH_OPT_DTRACE;
- } else {
- thread->options &= (~TH_OPT_DTRACE);
- }
+ thread->t_dtrace_inprobe = inprobe;
}
}
vm_offset_t
dtrace_sign_and_set_thread_recover(thread_t thread, vm_offset_t recover)
{
+#if defined(HAS_APPLE_PAC)
+ return dtrace_set_thread_recover(thread,
+ (vm_address_t)ptrauth_sign_unauthenticated((void *)recover,
+ ptrauth_key_function_pointer,
+ ptrauth_blend_discriminator(&thread->recover, PAC_DISCRIMINATOR_RECOVER)));
+#else /* defined(HAS_APPLE_PAC) */
return dtrace_set_thread_recover(thread, recover);
+#endif /* defined(HAS_APPLE_PAC) */
}
void
projects / apple / xnu.git / blobdiff