+ ipc_thread_terminate(thread);
+
+ proc_thread_qos_deallocate(thread);
+
+ task = thread->task;
+
+#ifdef MACH_BSD
+ {
+ void *ut = thread->uthread;
+
+ thread->uthread = NULL;
+ uthread_zone_free(ut);
+ }
+#endif /* MACH_BSD */
+
+ if (thread->t_ledger) {
+ ledger_dereference(thread->t_ledger);
+ }
+ if (thread->t_threadledger) {
+ ledger_dereference(thread->t_threadledger);
+ }
+
+ assert(thread->turnstile != TURNSTILE_NULL);
+ if (thread->turnstile) {
+ turnstile_deallocate(thread->turnstile);
+ }
+
+ if (IPC_VOUCHER_NULL != thread->ith_voucher) {
+ ipc_voucher_release(thread->ith_voucher);
+ }
+
+ if (thread->thread_io_stats) {
+ kheap_free(KHEAP_DATA_BUFFERS, thread->thread_io_stats,
+ sizeof(struct io_stat_info));
+ }
+
+ if (thread->kernel_stack != 0) {
+ stack_free(thread);
+ }
+
+ lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
+ machine_thread_destroy(thread);
+
+ task_deallocate(task);
+
+#if MACH_ASSERT
+ assert_thread_magic(thread);
+ thread->thread_magic = 0;
+#endif /* MACH_ASSERT */
+
+ lck_mtx_lock(&tasks_threads_lock);
+ assert(terminated_threads_count > 0);
+ queue_remove(&terminated_threads, thread, thread_t, threads);
+ terminated_threads_count--;
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ zfree(thread_zone, thread);
+}
+
+/*
+ * thread_inspect_deallocate:
+ *
+ * Drop a thread inspection reference.
+ */
+void
+thread_inspect_deallocate(
+ thread_inspect_t thread_inspect)
+{
+ return thread_deallocate((thread_t)thread_inspect);
+}
+
+/*
+ * thread_read_deallocate:
+ *
+ * Drop a reference on thread read port.
+ */
+void
+thread_read_deallocate(
+ thread_read_t thread_read)
+{
+ return thread_deallocate((thread_t)thread_read);
+}
+
+
+/*
+ * thread_exception_queue_invoke:
+ *
+ * Deliver EXC_{RESOURCE,GUARD} exception
+ */
+static void
+thread_exception_queue_invoke(mpsc_queue_chain_t elm,
+ __assert_only mpsc_daemon_queue_t dq)
+{
+ struct thread_exception_elt *elt;
+ task_t task;
+ thread_t thread;
+ exception_type_t etype;
+
+ assert(dq == &thread_exception_queue);
+ elt = mpsc_queue_element(elm, struct thread_exception_elt, link);
+
+ etype = elt->exception_type;
+ task = elt->exception_task;
+ thread = elt->exception_thread;
+ assert_thread_magic(thread);
+
+ kfree(elt, sizeof(*elt));
+
+ /* wait for all the threads in the task to terminate */
+ task_lock(task);
+ task_wait_till_threads_terminate_locked(task);
+ task_unlock(task);
+
+ /* Consumes the task ref returned by task_generate_corpse_internal */
+ task_deallocate(task);
+ /* Consumes the thread ref returned by task_generate_corpse_internal */
+ thread_deallocate(thread);
+
+ /* Deliver the notification, also clears the corpse. */
+ task_deliver_crash_notification(task, thread, etype, 0);
+}
+
+/*
+ * thread_exception_enqueue:
+ *
+ * Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}.
+ */
+void
+thread_exception_enqueue(
+ task_t task,
+ thread_t thread,
+ exception_type_t etype)
+{
+ assert(EXC_RESOURCE == etype || EXC_GUARD == etype);
+ struct thread_exception_elt *elt = kalloc(sizeof(*elt));
+ elt->exception_type = etype;
+ elt->exception_task = task;
+ elt->exception_thread = thread;
+
+ mpsc_daemon_enqueue(&thread_exception_queue, &elt->link,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
+}
+
+/*
+ * thread_copy_resource_info
+ *
+ * Copy the resource info counters from source
+ * thread to destination thread.
+ */
+void
+thread_copy_resource_info(
+ thread_t dst_thread,
+ thread_t src_thread)
+{
+ dst_thread->c_switch = src_thread->c_switch;
+ dst_thread->p_switch = src_thread->p_switch;
+ dst_thread->ps_switch = src_thread->ps_switch;
+ dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time;
+ dst_thread->user_timer = src_thread->user_timer;
+ dst_thread->user_timer_save = src_thread->user_timer_save;
+ dst_thread->system_timer = src_thread->system_timer;
+ dst_thread->system_timer_save = src_thread->system_timer_save;
+ dst_thread->runnable_timer = src_thread->runnable_timer;
+ dst_thread->vtimer_user_save = src_thread->vtimer_user_save;
+ dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save;
+ dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save;
+ dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save;
+ dst_thread->syscalls_unix = src_thread->syscalls_unix;
+ dst_thread->syscalls_mach = src_thread->syscalls_mach;
+ ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger);
+ *dst_thread->thread_io_stats = *src_thread->thread_io_stats;
+}
+
+static void
+thread_terminate_queue_invoke(mpsc_queue_chain_t e,
+ __assert_only mpsc_daemon_queue_t dq)
+{
+ thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
+ task_t task = thread->task;
+
+ assert(dq == &thread_terminate_queue);
+
+ task_lock(task);
+
+ /*
+ * if marked for crash reporting, skip reaping.
+ * The corpse delivery thread will clear bit and enqueue
+ * for reaping when done
+ *
+ * Note: the inspection field is set under the task lock
+ *
+ * FIXME[mad]: why enqueue for termination before `inspection` is false ?
+ */
+ if (__improbable(thread->inspection)) {
+ simple_lock(&crashed_threads_lock, &thread_lck_grp);
+ task_unlock(task);
+
+ enqueue_tail(&crashed_threads_queue, &thread->runq_links);
+ simple_unlock(&crashed_threads_lock);
+ return;
+ }
+
+
+ task->total_user_time += timer_grab(&thread->user_timer);
+ task->total_ptime += timer_grab(&thread->ptime);
+ task->total_runnable_time += timer_grab(&thread->runnable_timer);
+ if (thread->precise_user_kernel_time) {
+ task->total_system_time += timer_grab(&thread->system_timer);
+ } else {
+ task->total_user_time += timer_grab(&thread->system_timer);
+ }
+
+ task->c_switch += thread->c_switch;
+ task->p_switch += thread->p_switch;
+ task->ps_switch += thread->ps_switch;
+
+ task->syscalls_unix += thread->syscalls_unix;
+ task->syscalls_mach += thread->syscalls_mach;
+
+ task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
+ task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
+ task->task_gpu_ns += ml_gpu_stat(thread);
+ task->task_energy += ml_energy_stat(thread);
+ task->decompressions += thread->decompressions;
+
+#if MONOTONIC
+ mt_terminate_update(task, thread);
+#endif /* MONOTONIC */
+
+ thread_update_qos_cpu_time(thread);
+
+ queue_remove(&task->threads, thread, thread_t, task_threads);
+ task->thread_count--;
+
+ /*
+ * If the task is being halted, and there is only one thread
+ * left in the task after this one, then wakeup that thread.
+ */
+ if (task->thread_count == 1 && task->halting) {
+ thread_wakeup((event_t)&task->halting);
+ }
+
+ task_unlock(task);
+
+ lck_mtx_lock(&tasks_threads_lock);
+ queue_remove(&threads, thread, thread_t, threads);
+ threads_count--;
+ queue_enter(&terminated_threads, thread, thread_t, threads);
+ terminated_threads_count++;
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ thread_deallocate(thread);
+}
+
+static void
+thread_deallocate_queue_invoke(mpsc_queue_chain_t e,
+ __assert_only mpsc_daemon_queue_t dq)
+{
+ thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
+
+ assert(dq == &thread_deallocate_queue);
+
+ thread_deallocate_complete(thread);
+}
+
+/*
+ * thread_terminate_enqueue:
+ *
+ * Enqueue a terminating thread for final disposition.
+ *
+ * Called at splsched.
+ */
+void
+thread_terminate_enqueue(
+ thread_t thread)
+{
+ KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id);
+
+ mpsc_daemon_enqueue(&thread_terminate_queue, &thread->mpsc_links,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
+}
+
+/*
+ * thread_deallocate_enqueue:
+ *
+ * Enqueue a thread for final deallocation.
+ */
+static void
+thread_deallocate_enqueue(
+ thread_t thread)
+{
+ mpsc_daemon_enqueue(&thread_deallocate_queue, &thread->mpsc_links,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
+}
+
+/*
+ * thread_terminate_crashed_threads:
+ * walk the list of crashed threads and put back set of threads
+ * who are no longer being inspected.
+ */
+void
+thread_terminate_crashed_threads(void)
+{
+ thread_t th_remove;
+
+ simple_lock(&crashed_threads_lock, &thread_lck_grp);
+ /*
+ * loop through the crashed threads queue
+ * to put any threads that are not being inspected anymore
+ */
+
+ qe_foreach_element_safe(th_remove, &crashed_threads_queue, runq_links) {
+ /* make sure current_thread is never in crashed queue */
+ assert(th_remove != current_thread());
+
+ if (th_remove->inspection == FALSE) {
+ remqueue(&th_remove->runq_links);
+ mpsc_daemon_enqueue(&thread_terminate_queue, &th_remove->mpsc_links,
+ MPSC_QUEUE_NONE);
+ }
+ }
+
+ simple_unlock(&crashed_threads_lock);
+}
+
+/*
+ * thread_stack_queue_invoke:
+ *
+ * Perform stack allocation as required due to
+ * invoke failures.
+ */
+static void
+thread_stack_queue_invoke(mpsc_queue_chain_t elm,
+ __assert_only mpsc_daemon_queue_t dq)
+{
+ thread_t thread = mpsc_queue_element(elm, struct thread, mpsc_links);
+
+ assert(dq == &thread_stack_queue);
+
+ /* allocate stack with interrupts enabled so that we can call into VM */
+ stack_alloc(thread);
+
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
+
+ spl_t s = splsched();
+ thread_lock(thread);
+ thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
+ thread_unlock(thread);
+ splx(s);
+}
+
+/*
+ * thread_stack_enqueue:
+ *
+ * Enqueue a thread for stack allocation.
+ *
+ * Called at splsched.
+ */
+void
+thread_stack_enqueue(
+ thread_t thread)
+{
+ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);
+ assert_thread_magic(thread);
+
+ mpsc_daemon_enqueue(&thread_stack_queue, &thread->mpsc_links,
+ MPSC_QUEUE_DISABLE_PREEMPTION);
+}
+
+void
+thread_daemon_init(void)
+{
+ kern_return_t result;
+
+ thread_deallocate_daemon_init();
+
+ thread_deallocate_daemon_register_queue(&thread_terminate_queue,
+ thread_terminate_queue_invoke);
+
+ thread_deallocate_daemon_register_queue(&thread_deallocate_queue,
+ thread_deallocate_queue_invoke);
+
+ simple_lock_init(&crashed_threads_lock, 0);
+ queue_init(&crashed_threads_queue);
+
+ result = mpsc_daemon_queue_init_with_thread(&thread_stack_queue,
+ thread_stack_queue_invoke, BASEPRI_PREEMPT_HIGH,
+ "daemon.thread-stack");
+ if (result != KERN_SUCCESS) {
+ panic("thread_daemon_init: thread_stack_daemon");
+ }
+
+ result = mpsc_daemon_queue_init_with_thread(&thread_exception_queue,
+ thread_exception_queue_invoke, MINPRI_KERNEL,
+ "daemon.thread-exception");
+ if (result != KERN_SUCCESS) {
+ panic("thread_daemon_init: thread_exception_daemon");
+ }
+}
+
+__options_decl(thread_create_internal_options_t, uint32_t, {
+ TH_OPTION_NONE = 0x00,
+ TH_OPTION_NOCRED = 0x01,
+ TH_OPTION_NOSUSP = 0x02,
+ TH_OPTION_WORKQ = 0x04,
+ TH_OPTION_IMMOVABLE = 0x08,
+ TH_OPTION_PINNED = 0x10,
+});
+
+/*
+ * Create a new thread.
+ * Doesn't start the thread running.
+ *
+ * Task and tasks_threads_lock are returned locked on success.
+ */
+static kern_return_t
+thread_create_internal(
+ task_t parent_task,
+ integer_t priority,
+ thread_continue_t continuation,
+ void *parameter,
+ thread_create_internal_options_t options,
+ thread_t *out_thread)
+{
+ thread_t new_thread;
+ static thread_t first_thread;
+ ipc_thread_init_options_t init_options = IPC_THREAD_INIT_NONE;
+
+ /*
+ * Allocate a thread and initialize static fields
+ */
+ if (first_thread == THREAD_NULL) {
+ new_thread = first_thread = current_thread();
+ } else {
+ new_thread = (thread_t)zalloc(thread_zone);
+ }
+ if (new_thread == THREAD_NULL) {
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ if (new_thread != first_thread) {
+ init_thread_from_template(new_thread);
+ }
+
+ if (options & TH_OPTION_PINNED) {
+ init_options |= IPC_THREAD_INIT_PINNED;
+ }
+
+ if (options & TH_OPTION_IMMOVABLE) {
+ init_options |= IPC_THREAD_INIT_IMMOVABLE;
+ }
+
+ os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2);
+#if DEBUG || DEVELOPMENT
+ queue_init(&new_thread->t_temp_alloc_list);
+#endif /* DEBUG || DEVELOPMENT */
+
+#ifdef MACH_BSD
+ new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
+ if (new_thread->uthread == NULL) {
+#if MACH_ASSERT
+ new_thread->thread_magic = 0;
+#endif /* MACH_ASSERT */
+
+ zfree(thread_zone, new_thread);
+ return KERN_RESOURCE_SHORTAGE;
+ }
+#endif /* MACH_BSD */
+
+ if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
+#ifdef MACH_BSD
+ void *ut = new_thread->uthread;
+
+ new_thread->uthread = NULL;
+ /* cred free may not be necessary */
+ uthread_cleanup(parent_task, ut, parent_task->bsd_info);
+ uthread_cred_free(ut);
+ uthread_zone_free(ut);
+#endif /* MACH_BSD */
+
+#if MACH_ASSERT
+ new_thread->thread_magic = 0;
+#endif /* MACH_ASSERT */
+
+ zfree(thread_zone, new_thread);
+ return KERN_FAILURE;
+ }
+
+ new_thread->task = parent_task;
+
+ thread_lock_init(new_thread);
+ wake_lock_init(new_thread);
+
+ lck_mtx_init(&new_thread->mutex, &thread_lck_grp, LCK_ATTR_NULL);
+
+ ipc_thread_init(new_thread, init_options);
+
+ new_thread->continuation = continuation;
+ new_thread->parameter = parameter;
+ new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE;
+ priority_queue_init(&new_thread->sched_inheritor_queue);
+ priority_queue_init(&new_thread->base_inheritor_queue);
+#if CONFIG_SCHED_CLUTCH
+ priority_queue_entry_init(&new_thread->th_clutch_runq_link);
+ priority_queue_entry_init(&new_thread->th_clutch_pri_link);
+#endif /* CONFIG_SCHED_CLUTCH */
+
+#if CONFIG_SCHED_EDGE
+ new_thread->th_bound_cluster_enqueued = false;
+#endif /* CONFIG_SCHED_EDGE */
+
+ /* Allocate I/O Statistics structure */
+ new_thread->thread_io_stats = kheap_alloc(KHEAP_DATA_BUFFERS,
+ sizeof(struct io_stat_info), Z_WAITOK | Z_ZERO);
+ assert(new_thread->thread_io_stats != NULL);
+
+#if KASAN
+ kasan_init_thread(&new_thread->kasan_data);
+#endif
+
+#if CONFIG_KSANCOV
+ new_thread->ksancov_data = NULL;
+#endif
+
+#if CONFIG_IOSCHED
+ /* Clear out the I/O Scheduling info for AppleFSCompression */
+ new_thread->decmp_upl = NULL;
+#endif /* CONFIG_IOSCHED */
+
+ new_thread->thread_region_page_shift = 0;
+
+#if DEVELOPMENT || DEBUG
+ task_lock(parent_task);
+ uint16_t thread_limit = parent_task->task_thread_limit;
+ if (exc_resource_threads_enabled &&
+ thread_limit > 0 &&
+ parent_task->thread_count >= thread_limit &&
+ !parent_task->task_has_crossed_thread_limit &&
+ !(parent_task->t_flags & TF_CORPSE)) {
+ int thread_count = parent_task->thread_count;
+ parent_task->task_has_crossed_thread_limit = TRUE;
+ task_unlock(parent_task);
+ SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(parent_task, thread_count);
+ } else {
+ task_unlock(parent_task);
+ }
+#endif
+
+ lck_mtx_lock(&tasks_threads_lock);
+ task_lock(parent_task);
+
+ /*
+ * Fail thread creation if parent task is being torn down or has too many threads
+ * If the caller asked for TH_OPTION_NOSUSP, also fail if the parent task is suspended
+ */
+ if (parent_task->active == 0 || parent_task->halting ||
+ (parent_task->suspend_count > 0 && (options & TH_OPTION_NOSUSP) != 0) ||
+ (parent_task->thread_count >= task_threadmax && parent_task != kernel_task)) {
+ task_unlock(parent_task);
+ lck_mtx_unlock(&tasks_threads_lock);
+
+#ifdef MACH_BSD
+ {
+ void *ut = new_thread->uthread;
+
+ new_thread->uthread = NULL;
+ uthread_cleanup(parent_task, ut, parent_task->bsd_info);
+ /* cred free may not be necessary */
+ uthread_cred_free(ut);
+ uthread_zone_free(ut);
+ }
+#endif /* MACH_BSD */
+ ipc_thread_disable(new_thread);
+ ipc_thread_terminate(new_thread);
+ kheap_free(KHEAP_DATA_BUFFERS, new_thread->thread_io_stats,
+ sizeof(struct io_stat_info));
+ lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
+ machine_thread_destroy(new_thread);
+ zfree(thread_zone, new_thread);
+ return KERN_FAILURE;
+ }
+
+ /* Protected by the tasks_threads_lock */
+ new_thread->thread_id = ++thread_unique_id;
+
+ /* New threads inherit any default state on the task */
+ machine_thread_inherit_taskwide(new_thread, parent_task);
+
+ task_reference_internal(parent_task);
+
+ if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
+ /*
+ * This task has a per-thread CPU limit; make sure this new thread
+ * gets its limit set too, before it gets out of the kernel.
+ */
+ act_set_astledger(new_thread);
+ }
+
+ /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
+ if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
+ LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
+ ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
+ }
+
+ new_thread->t_bankledger = LEDGER_NULL;
+ new_thread->t_deduct_bank_ledger_time = 0;
+ new_thread->t_deduct_bank_ledger_energy = 0;
+
+ new_thread->t_ledger = new_thread->task->ledger;
+ if (new_thread->t_ledger) {
+ ledger_reference(new_thread->t_ledger);
+ }
+
+#if defined(CONFIG_SCHED_MULTIQ)
+ /* Cache the task's sched_group */
+ new_thread->sched_group = parent_task->sched_group;
+#endif /* defined(CONFIG_SCHED_MULTIQ) */
+
+ /* Cache the task's map */
+ new_thread->map = parent_task->map;
+
+ timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
+ timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
+
+#if KPC
+ kpc_thread_create(new_thread);
+#endif
+
+ /* Set the thread's scheduling parameters */
+ new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
+ new_thread->max_priority = parent_task->max_priority;
+ new_thread->task_priority = parent_task->priority;
+
+#if CONFIG_THREAD_GROUPS
+ thread_group_init_thread(new_thread, parent_task);
+#endif /* CONFIG_THREAD_GROUPS */
+
+ int new_priority = (priority < 0) ? parent_task->priority: priority;
+ new_priority = (priority < 0)? parent_task->priority: priority;
+ if (new_priority > new_thread->max_priority) {
+ new_priority = new_thread->max_priority;
+ }
+#if !defined(XNU_TARGET_OS_OSX)
+ if (new_priority < MAXPRI_THROTTLE) {
+ new_priority = MAXPRI_THROTTLE;
+ }
+#endif /* !defined(XNU_TARGET_OS_OSX) */
+
+ new_thread->importance = new_priority - new_thread->task_priority;
+
+ sched_set_thread_base_priority(new_thread, new_priority);
+
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+ new_thread->sched_stamp = sched_tick;
+#if CONFIG_SCHED_CLUTCH
+ new_thread->pri_shift = sched_clutch_thread_pri_shift(new_thread, new_thread->th_sched_bucket);
+#else /* CONFIG_SCHED_CLUTCH */
+ new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];
+#endif /* CONFIG_SCHED_CLUTCH */
+#endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
+
+ if (parent_task->max_priority <= MAXPRI_THROTTLE) {
+ sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED);
+ }
+
+ thread_policy_create(new_thread);
+
+ /* Chain the thread onto the task's list */
+ queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
+ parent_task->thread_count++;
+
+ /* So terminating threads don't need to take the task lock to decrement */
+ os_atomic_inc(&parent_task->active_thread_count, relaxed);
+
+ queue_enter(&threads, new_thread, thread_t, threads);
+ threads_count++;
+
+ new_thread->active = TRUE;
+ if (task_is_a_corpse_fork(parent_task)) {
+ /* Set the inspection bit if the task is a corpse fork */
+ new_thread->inspection = TRUE;
+ } else {
+ new_thread->inspection = FALSE;
+ }
+ new_thread->corpse_dup = FALSE;
+ new_thread->turnstile = turnstile_alloc();
+
+
+ *out_thread = new_thread;
+
+ if (kdebug_enable) {
+ long args[4] = {};
+
+ kdbg_trace_data(parent_task->bsd_info, &args[1], &args[3]);
+
+ /*
+ * Starting with 26604425, exec'ing creates a new task/thread.
+ *
+ * NEWTHREAD in the current process has two possible meanings:
+ *
+ * 1) Create a new thread for this process.
+ * 2) Create a new thread for the future process this will become in an
+ * exec.
+ *
+ * To disambiguate these, arg3 will be set to TRUE for case #2.
+ *
+ * The value we need to find (TPF_EXEC_COPY) is stable in the case of a
+ * task exec'ing. The read of t_procflags does not take the proc_lock.
+ */
+ args[2] = task_is_exec_copy(parent_task) ? 1 : 0;
+
+ KDBG_RELEASE(TRACE_DATA_NEWTHREAD, (uintptr_t)thread_tid(new_thread),
+ args[1], args[2], args[3]);
+
+ kdbg_trace_string(parent_task->bsd_info, &args[0], &args[1],
+ &args[2], &args[3]);
+ KDBG_RELEASE(TRACE_STRING_NEWTHREAD, args[0], args[1], args[2],
+ args[3]);
+ }
+
+ DTRACE_PROC1(lwp__create, thread_t, *out_thread);
+
+ return KERN_SUCCESS;
+}
+
+static kern_return_t
+thread_create_with_options_internal(
+ task_t task,
+ thread_t *new_thread,
+ boolean_t from_user,
+ thread_create_internal_options_t options,
+ thread_continue_t continuation)
+{
+ kern_return_t result;
+ thread_t thread;
+
+ if (task == TASK_NULL || task == kernel_task) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+#if CONFIG_MACF
+ if (from_user && current_task() != task &&
+ mac_proc_check_remote_thread_create(task, -1, NULL, 0) != 0) {
+ return KERN_DENIED;
+ }
+#endif
+
+ result = thread_create_internal(task, -1, continuation, NULL, options, &thread);
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
+
+ thread->user_stop_count = 1;
+ thread_hold(thread);
+ if (task->suspend_count > 0) {
+ thread_hold(thread);
+ }
+
+ if (from_user) {
+ extmod_statistics_incr_thread_create(task);
+ }
+
+ task_unlock(task);
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ *new_thread = thread;
+
+ return KERN_SUCCESS;
+}
+
+/* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
+kern_return_t
+thread_create(
+ task_t task,
+ thread_t *new_thread);
+
+kern_return_t
+thread_create(
+ task_t task,
+ thread_t *new_thread)
+{
+ return thread_create_with_options_internal(task, new_thread, FALSE, TH_OPTION_NONE,
+ (thread_continue_t)thread_bootstrap_return);
+}
+
+/*
+ * Create a thread that has its itk_self pinned
+ * Deprecated, should be cleanup once rdar://70892168 lands
+ */
+kern_return_t
+thread_create_pinned(
+ task_t task,
+ thread_t *new_thread)
+{
+ return thread_create_with_options_internal(task, new_thread, FALSE,
+ TH_OPTION_PINNED | TH_OPTION_IMMOVABLE, (thread_continue_t)thread_bootstrap_return);
+}
+
+kern_return_t
+thread_create_immovable(
+ task_t task,
+ thread_t *new_thread)
+{
+ return thread_create_with_options_internal(task, new_thread, FALSE,
+ TH_OPTION_IMMOVABLE, (thread_continue_t)thread_bootstrap_return);
+}
+
+kern_return_t
+thread_create_from_user(
+ task_t task,
+ thread_t *new_thread)
+{
+ return thread_create_with_options_internal(task, new_thread, TRUE, TH_OPTION_NONE,
+ (thread_continue_t)thread_bootstrap_return);
+}
+
+kern_return_t
+thread_create_with_continuation(
+ task_t task,
+ thread_t *new_thread,
+ thread_continue_t continuation)
+{
+ return thread_create_with_options_internal(task, new_thread, FALSE, TH_OPTION_NONE, continuation);
+}
+
+/*
+ * Create a thread that is already started, but is waiting on an event
+ */
+static kern_return_t
+thread_create_waiting_internal(
+ task_t task,
+ thread_continue_t continuation,
+ event_t event,
+ block_hint_t block_hint,
+ int options,
+ thread_t *new_thread)
+{
+ kern_return_t result;
+ thread_t thread;
+
+ if (task == TASK_NULL || task == kernel_task) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ result = thread_create_internal(task, -1, continuation, NULL,
+ options, &thread);
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
+
+ /* note no user_stop_count or thread_hold here */
+
+ if (task->suspend_count > 0) {
+ thread_hold(thread);
+ }
+
+ thread_mtx_lock(thread);
+ thread_set_pending_block_hint(thread, block_hint);
+ if (options & TH_OPTION_WORKQ) {
+ thread->static_param = true;
+ event = workq_thread_init_and_wq_lock(task, thread);
+ }
+ thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE);
+ thread_mtx_unlock(thread);
+
+ task_unlock(task);
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ *new_thread = thread;
+
+ return KERN_SUCCESS;
+}
+
+kern_return_t
+thread_create_waiting(
+ task_t task,
+ thread_continue_t continuation,
+ event_t event,
+ th_create_waiting_options_t options,
+ thread_t *new_thread)
+{
+ thread_create_internal_options_t ci_options = TH_OPTION_NONE;
+
+ assert((options & ~TH_CREATE_WAITING_OPTION_MASK) == 0);
+ if (options & TH_CREATE_WAITING_OPTION_PINNED) {
+ ci_options |= TH_OPTION_PINNED;
+ }
+ if (options & TH_CREATE_WAITING_OPTION_IMMOVABLE) {
+ ci_options |= TH_OPTION_IMMOVABLE;
+ }
+
+ return thread_create_waiting_internal(task, continuation, event,
+ kThreadWaitNone, ci_options, new_thread);
+}
+
+
+static kern_return_t
+thread_create_running_internal2(
+ task_t task,
+ int flavor,
+ thread_state_t new_state,
+ mach_msg_type_number_t new_state_count,
+ thread_t *new_thread,
+ boolean_t from_user)
+{
+ kern_return_t result;
+ thread_t thread;
+
+ if (task == TASK_NULL || task == kernel_task) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+#if CONFIG_MACF
+ if (from_user && current_task() != task &&
+ mac_proc_check_remote_thread_create(task, flavor, new_state, new_state_count) != 0) {
+ return KERN_DENIED;
+ }
+#endif
+
+ result = thread_create_internal(task, -1,
+ (thread_continue_t)thread_bootstrap_return, NULL,
+ TH_OPTION_NONE, &thread);
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
+
+ if (task->suspend_count > 0) {
+ thread_hold(thread);
+ }
+
+ if (from_user) {
+ result = machine_thread_state_convert_from_user(thread, flavor,
+ new_state, new_state_count);
+ }
+ if (result == KERN_SUCCESS) {
+ result = machine_thread_set_state(thread, flavor, new_state,
+ new_state_count);
+ }
+ if (result != KERN_SUCCESS) {
+ task_unlock(task);
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ thread_terminate(thread);
+ thread_deallocate(thread);
+ return result;
+ }
+
+ thread_mtx_lock(thread);
+ thread_start(thread);
+ thread_mtx_unlock(thread);
+
+ if (from_user) {
+ extmod_statistics_incr_thread_create(task);
+ }
+
+ task_unlock(task);
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ *new_thread = thread;
+
+ return result;
+}
+
+/* Prototype, see justification above */
+kern_return_t
+thread_create_running(
+ task_t task,
+ int flavor,
+ thread_state_t new_state,
+ mach_msg_type_number_t new_state_count,
+ thread_t *new_thread);
+
+kern_return_t
+thread_create_running(
+ task_t task,
+ int flavor,
+ thread_state_t new_state,
+ mach_msg_type_number_t new_state_count,
+ thread_t *new_thread)
+{
+ return thread_create_running_internal2(
+ task, flavor, new_state, new_state_count,
+ new_thread, FALSE);
+}
+
+kern_return_t
+thread_create_running_from_user(
+ task_t task,
+ int flavor,
+ thread_state_t new_state,
+ mach_msg_type_number_t new_state_count,
+ thread_t *new_thread)
+{
+ return thread_create_running_internal2(
+ task, flavor, new_state, new_state_count,
+ new_thread, TRUE);
+}
+
+kern_return_t
+thread_create_workq_waiting(
+ task_t task,
+ thread_continue_t continuation,
+ thread_t *new_thread)
+{
+ /*
+ * Create thread, but don't pin control port just yet, in case someone calls
+ * task_threads() and deallocates pinned port before kernel copyout happens,
+ * which will result in pinned port guard exception. Instead, pin and make
+ * it immovable atomically at copyout during workq_setup_and_run().
+ */
+ int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ | TH_OPTION_IMMOVABLE;
+ return thread_create_waiting_internal(task, continuation, NULL,
+ kThreadWaitParkedWorkQueue, options, new_thread);
+}
+
+/*
+ * kernel_thread_create:
+ *
+ * Create a thread in the kernel task
+ * to execute in kernel context.
+ */
+kern_return_t
+kernel_thread_create(
+ thread_continue_t continuation,
+ void *parameter,
+ integer_t priority,
+ thread_t *new_thread)
+{
+ kern_return_t result;
+ thread_t thread;
+ task_t task = kernel_task;
+
+ result = thread_create_internal(task, priority, continuation, parameter,
+ TH_OPTION_NOCRED | TH_OPTION_NONE, &thread);
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
+
+ task_unlock(task);
+ lck_mtx_unlock(&tasks_threads_lock);
+
+ stack_alloc(thread);
+ assert(thread->kernel_stack != 0);
+#if !defined(XNU_TARGET_OS_OSX)
+ if (priority > BASEPRI_KERNEL)
+#endif
+ thread->reserved_stack = thread->kernel_stack;
+
+ if (debug_task & 1) {
+ kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
+ }
+ *new_thread = thread;
+
+ return result;
+}
+
+kern_return_t
+kernel_thread_start_priority(
+ thread_continue_t continuation,
+ void *parameter,
+ integer_t priority,
+ thread_t *new_thread)
+{
+ kern_return_t result;
+ thread_t thread;
+
+ result = kernel_thread_create(continuation, parameter, priority, &thread);
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
+
+ *new_thread = thread;
+
+ thread_mtx_lock(thread);
+ thread_start(thread);
+ thread_mtx_unlock(thread);
+
+ return result;
+}
+
+kern_return_t
+kernel_thread_start(
+ thread_continue_t continuation,
+ void *parameter,
+ thread_t *new_thread)
+{
+ return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
+}
+
+/* Separated into helper function so it can be used by THREAD_BASIC_INFO and THREAD_EXTENDED_INFO */
+/* it is assumed that the thread is locked by the caller */
+static void
+retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
+{
+ int state, flags;
+
+ /* fill in info */
+
+ thread_read_times(thread, &basic_info->user_time,
+ &basic_info->system_time, NULL);
+
+ /*
+ * Update lazy-evaluated scheduler info because someone wants it.
+ */
+ if (SCHED(can_update_priority)(thread)) {
+ SCHED(update_priority)(thread);
+ }
+
+ basic_info->sleep_time = 0;
+
+ /*
+ * To calculate cpu_usage, first correct for timer rate,
+ * then for 5/8 ageing. The correction factor [3/5] is
+ * (1/(5/8) - 1).
+ */
+ basic_info->cpu_usage = 0;
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+ if (sched_tick_interval) {
+ basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
+ * TH_USAGE_SCALE) / sched_tick_interval);
+ basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
+ }
+#endif
+
+ if (basic_info->cpu_usage > TH_USAGE_SCALE) {
+ basic_info->cpu_usage = TH_USAGE_SCALE;
+ }
+
+ basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
+ POLICY_TIMESHARE: POLICY_RR);
+
+ flags = 0;
+ if (thread->options & TH_OPT_IDLE_THREAD) {
+ flags |= TH_FLAGS_IDLE;
+ }
+
+ if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
+ flags |= TH_FLAGS_GLOBAL_FORCED_IDLE;
+ }
+
+ if (!thread->kernel_stack) {
+ flags |= TH_FLAGS_SWAPPED;
+ }
+
+ state = 0;
+ if (thread->state & TH_TERMINATE) {
+ state = TH_STATE_HALTED;
+ } else if (thread->state & TH_RUN) {
+ state = TH_STATE_RUNNING;
+ } else if (thread->state & TH_UNINT) {
+ state = TH_STATE_UNINTERRUPTIBLE;
+ } else if (thread->state & TH_SUSP) {
+ state = TH_STATE_STOPPED;
+ } else if (thread->state & TH_WAIT) {
+ state = TH_STATE_WAITING;
+ }
+
+ basic_info->run_state = state;
+ basic_info->flags = flags;
+
+ basic_info->suspend_count = thread->user_stop_count;
+
+ return;
+}
+
+kern_return_t
+thread_info_internal(
+ thread_t thread,
+ thread_flavor_t flavor,
+ thread_info_t thread_info_out, /* ptr to OUT array */
+ mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
+{
+ spl_t s;
+
+ if (thread == THREAD_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (flavor == THREAD_BASIC_INFO) {
+ if (*thread_info_count < THREAD_BASIC_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ s = splsched();
+ thread_lock(thread);
+
+ retrieve_thread_basic_info(thread, (thread_basic_info_t) thread_info_out);
+
+ thread_unlock(thread);
+ splx(s);
+
+ *thread_info_count = THREAD_BASIC_INFO_COUNT;
+
+ return KERN_SUCCESS;
+ } else if (flavor == THREAD_IDENTIFIER_INFO) {
+ thread_identifier_info_t identifier_info;
+
+ if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ identifier_info = __IGNORE_WCASTALIGN((thread_identifier_info_t)thread_info_out);
+
+ s = splsched();
+ thread_lock(thread);
+
+ identifier_info->thread_id = thread->thread_id;
+ identifier_info->thread_handle = thread->machine.cthread_self;
+ identifier_info->dispatch_qaddr = thread_dispatchqaddr(thread);
+
+ thread_unlock(thread);
+ splx(s);
+ return KERN_SUCCESS;
+ } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
+ policy_timeshare_info_t ts_info;
+
+ if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ ts_info = (policy_timeshare_info_t)thread_info_out;
+
+ s = splsched();
+ thread_lock(thread);
+
+ if (thread->sched_mode != TH_MODE_TIMESHARE) {
+ thread_unlock(thread);
+ splx(s);
+ return KERN_INVALID_POLICY;
+ }
+
+ ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
+ if (ts_info->depressed) {
+ ts_info->base_priority = DEPRESSPRI;
+ ts_info->depress_priority = thread->base_pri;
+ } else {
+ ts_info->base_priority = thread->base_pri;
+ ts_info->depress_priority = -1;
+ }
+
+ ts_info->cur_priority = thread->sched_pri;
+ ts_info->max_priority = thread->max_priority;
+
+ thread_unlock(thread);
+ splx(s);
+
+ *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
+
+ return KERN_SUCCESS;
+ } else if (flavor == THREAD_SCHED_FIFO_INFO) {
+ if (*thread_info_count < POLICY_FIFO_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ return KERN_INVALID_POLICY;
+ } else if (flavor == THREAD_SCHED_RR_INFO) {
+ policy_rr_info_t rr_info;
+ uint32_t quantum_time;
+ uint64_t quantum_ns;
+
+ if (*thread_info_count < POLICY_RR_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ rr_info = (policy_rr_info_t) thread_info_out;
+
+ s = splsched();
+ thread_lock(thread);
+
+ if (thread->sched_mode == TH_MODE_TIMESHARE) {
+ thread_unlock(thread);
+ splx(s);
+
+ return KERN_INVALID_POLICY;
+ }
+
+ rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
+ if (rr_info->depressed) {
+ rr_info->base_priority = DEPRESSPRI;
+ rr_info->depress_priority = thread->base_pri;
+ } else {
+ rr_info->base_priority = thread->base_pri;
+ rr_info->depress_priority = -1;
+ }
+
+ quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
+ absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
+
+ rr_info->max_priority = thread->max_priority;
+ rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
+
+ thread_unlock(thread);
+ splx(s);
+
+ *thread_info_count = POLICY_RR_INFO_COUNT;
+
+ return KERN_SUCCESS;
+ } else if (flavor == THREAD_EXTENDED_INFO) {
+ thread_basic_info_data_t basic_info;
+ thread_extended_info_t extended_info = __IGNORE_WCASTALIGN((thread_extended_info_t)thread_info_out);
+
+ if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ s = splsched();
+ thread_lock(thread);
+
+ /* NOTE: This mimics fill_taskthreadinfo(), which is the function used by proc_pidinfo() for
+ * the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
+ */
+ retrieve_thread_basic_info(thread, &basic_info);
+ extended_info->pth_user_time = (((uint64_t)basic_info.user_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.user_time.microseconds * NSEC_PER_USEC));
+ extended_info->pth_system_time = (((uint64_t)basic_info.system_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.system_time.microseconds * NSEC_PER_USEC));
+
+ extended_info->pth_cpu_usage = basic_info.cpu_usage;
+ extended_info->pth_policy = basic_info.policy;
+ extended_info->pth_run_state = basic_info.run_state;
+ extended_info->pth_flags = basic_info.flags;
+ extended_info->pth_sleep_time = basic_info.sleep_time;
+ extended_info->pth_curpri = thread->sched_pri;
+ extended_info->pth_priority = thread->base_pri;
+ extended_info->pth_maxpriority = thread->max_priority;
+
+ bsd_getthreadname(thread->uthread, extended_info->pth_name);
+
+ thread_unlock(thread);
+ splx(s);
+
+ *thread_info_count = THREAD_EXTENDED_INFO_COUNT;
+
+ return KERN_SUCCESS;
+ } else if (flavor == THREAD_DEBUG_INFO_INTERNAL) {
+#if DEVELOPMENT || DEBUG
+ thread_debug_info_internal_t dbg_info;
+ if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT) {
+ return KERN_NOT_SUPPORTED;
+ }
+
+ if (thread_info_out == NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ dbg_info = __IGNORE_WCASTALIGN((thread_debug_info_internal_t)thread_info_out);
+ dbg_info->page_creation_count = thread->t_page_creation_count;
+
+ *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;
+ return KERN_SUCCESS;
+#endif /* DEVELOPMENT || DEBUG */
+ return KERN_NOT_SUPPORTED;
+ }
+
+ return KERN_INVALID_ARGUMENT;
+}
projects / apple / xnu.git / blobdiff