X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/ecc0ceb4089d506a0b8d16686a95817b331af9cb..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/bsd/kern/kern_fork.c diff --git a/bsd/kern/kern_fork.c b/bsd/kern/kern_fork.c index 284752296..a42891ae7 100644 --- a/bsd/kern/kern_fork.c +++ b/bsd/kern/kern_fork.c @@ -87,6 +87,7 @@ #include #include #include +#include #include #include #include @@ -96,12 +97,11 @@ #if CONFIG_PERSONAS #include #endif +#include #if CONFIG_DTRACE /* Do not include dtrace.h, it redefines kmem_[alloc/free] */ -extern void dtrace_fasttrap_fork(proc_t, proc_t); -extern void (*dtrace_helpers_fork)(proc_t, proc_t); extern void (*dtrace_proc_waitfor_exec_ptr)(proc_t); -extern void dtrace_lazy_dofs_duplicate(proc_t, proc_t); +extern void dtrace_proc_fork(proc_t, proc_t, int); /* * Since dtrace_proc_waitfor_exec_ptr can be added/removed in dtrace_subr.c, @@ -150,12 +150,13 @@ void thread_set_parent(thread_t parent, int pid); extern void act_thread_catt(void *ctx); void thread_set_child(thread_t child, int pid); void *act_thread_csave(void); +extern boolean_t task_is_exec_copy(task_t); thread_t cloneproc(task_t, coalition_t *, proc_t, int, int); proc_t forkproc(proc_t); void forkproc_free(proc_t); -thread_t fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t child, int inherit_memory, int is64bit); +thread_t fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t child, int inherit_memory, int is64bit, int in_exec); void proc_vfork_begin(proc_t parent_proc); void proc_vfork_end(proc_t parent_proc); @@ -324,7 +325,7 @@ vfork(proc_t parent_proc, __unused struct vfork_args *uap, int32_t *retval) * Parameters: parent_proc parent process of the process being * child_threadp pointer to location to receive the * Mach thread_t of the child process - * breated + * created * kind kind of creation being requested * coalitions if spawn, the set of coalitions the * child process should join, or NULL to @@ -617,60 +618,8 @@ fork1(proc_t parent_proc, thread_t *child_threadp, int kind, coalition_t *coalit child_proc->p_acflag = AFORK; /* forked but not exec'ed */ -// dtrace code cleanup needed #if CONFIG_DTRACE - /* - * This code applies to new processes who are copying the task - * and thread state and address spaces of their parent process. - */ - if (!spawn) { -// call dtrace specific function here instead of all this... - /* - * APPLE NOTE: Solaris does a sprlock() and drops the - * proc_lock here. We're cheating a bit and only taking - * the p_dtrace_sprlock lock. A full sprlock would - * task_suspend the parent. - */ - lck_mtx_lock(&parent_proc->p_dtrace_sprlock); - - /* - * Remove all DTrace tracepoints from the child process. We - * need to do this _before_ duplicating USDT providers since - * any associated probes may be immediately enabled. - */ - if (parent_proc->p_dtrace_count > 0) { - dtrace_fasttrap_fork(parent_proc, child_proc); - } - - lck_mtx_unlock(&parent_proc->p_dtrace_sprlock); - - /* - * Duplicate any lazy dof(s). This must be done while NOT - * holding the parent sprlock! Lock ordering is - * dtrace_dof_mode_lock, then sprlock. It is imperative we - * always call dtrace_lazy_dofs_duplicate, rather than null - * check and call if !NULL. If we NULL test, during lazy dof - * faulting we can race with the faulting code and proceed - * from here to beyond the helpers copy. The lazy dof - * faulting will then fail to copy the helpers to the child - * process. - */ - dtrace_lazy_dofs_duplicate(parent_proc, child_proc); - - /* - * Duplicate any helper actions and providers. The SFORKING - * we set above informs the code to enable USDT probes that - * sprlock() may fail because the child is being forked. - */ - /* - * APPLE NOTE: As best I can tell, Apple's sprlock() equivalent - * never fails to find the child. We do not set SFORKING. - */ - if (parent_proc->p_dtrace_helpers != NULL && dtrace_helpers_fork) { - (*dtrace_helpers_fork)(parent_proc, child_proc); - } - - } + dtrace_proc_fork(parent_proc, child_proc, spawn); #endif /* CONFIG_DTRACE */ if (!spawn) { /* @@ -776,6 +725,8 @@ vfork_return(proc_t child_proc, int32_t *retval, int rval) * is64bit TRUE, if the child being created will * be associated with a 64 bit process * rather than a 32 bit process + * in_exec TRUE, if called from execve or posix spawn set exec + * FALSE, if called from fork or vfexec * * Note: This code is called in the fork() case, from the execve() call * graph, if implementing an execve() following a vfork(), from @@ -794,7 +745,7 @@ vfork_return(proc_t child_proc, int32_t *retval, int rval) * in this case, 'inherit_memory' MUST be FALSE. */ thread_t -fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t child_proc, int inherit_memory, int is64bit) +fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t child_proc, int inherit_memory, int is64bit, int in_exec) { thread_t child_thread = NULL; task_t child_task; @@ -805,6 +756,8 @@ fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t chi parent_coalitions, inherit_memory, is64bit, + TF_LRETURNWAIT | TF_LRETURNWAITER, /* All created threads will wait in task_wait_to_return */ + in_exec ? TPF_EXEC_COPY : TPF_NONE, /* Mark the task exec copy if in execve */ &child_task); if (result != KERN_SUCCESS) { printf("%s: task_create_internal failed. Code: %d\n", @@ -812,8 +765,13 @@ fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t chi goto bad; } - /* Set the child process task to the new task */ - child_proc->task = child_task; + if (!in_exec) { + /* + * Set the child process task to the new task if not in exec, + * will set the task for exec case in proc_exec_switch_task after image activation. + */ + child_proc->task = child_task; + } /* Set child task process to child proc */ set_bsdtask_info(child_task, child_proc); @@ -822,12 +780,6 @@ fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t chi if (timerisset(&child_proc->p_rlim_cpu)) task_vtimer_set(child_task, TASK_VTIMER_RLIM); - /* Set/clear 64 bit vm_map flag */ - if (is64bit) - vm_map_set_64bit(get_task_map(child_task)); - else - vm_map_set_32bit(get_task_map(child_task)); - /* * Set child process BSD visible scheduler priority if nice value * inherited from parent @@ -835,8 +787,15 @@ fork_create_child(task_t parent_task, coalition_t *parent_coalitions, proc_t chi if (child_proc->p_nice != 0) resetpriority(child_proc); - /* Create a new thread for the child process */ - result = thread_create_with_continuation(child_task, &child_thread, (thread_continue_t)proc_wait_to_return); + /* + * Create a new thread for the child process + * The new thread is waiting on the event triggered by 'task_clear_return_wait' + */ + result = thread_create_waiting(child_task, + (thread_continue_t)task_wait_to_return, + task_get_return_wait_event(child_task), + &child_thread); + if (result != KERN_SUCCESS) { printf("%s: thread_create failed. Code: %d\n", __func__, result); @@ -924,7 +883,7 @@ fork(proc_t parent_proc, __unused struct fork_args *uap, int32_t *retval) #endif /* "Return" to the child */ - proc_clear_return_wait(child_proc, child_thread); + task_clear_return_wait(get_threadtask(child_thread)); /* drop the extra references we got during the creation */ if ((child_task = (task_t)get_threadtask(child_thread)) != NULL) { @@ -990,7 +949,7 @@ cloneproc(task_t parent_task, coalition_t *parent_coalitions, proc_t parent_proc goto bad; } - child_thread = fork_create_child(parent_task, parent_coalitions, child_proc, inherit_memory, (parent_task == TASK_NULL) ? FALSE : (parent_proc->p_flag & P_LP64)); + child_thread = fork_create_child(parent_task, parent_coalitions, child_proc, inherit_memory, parent_proc->p_flag & P_LP64, FALSE); if (child_thread == NULL) { /* @@ -1055,8 +1014,6 @@ forkproc_free(proc_t p) pth_proc_hashdelete(p); #endif /* PSYNCH */ - workqueue_destroy_lock(p); - /* We held signal and a transition locks; drop them */ proc_signalend(p, 0); proc_transend(p, 0); @@ -1370,7 +1327,6 @@ retry: */ proc_signalstart(child_proc, 0); proc_transstart(child_proc, 0, 0); - proc_set_return_wait(child_proc); child_proc->p_pcaction = 0; @@ -1381,12 +1337,6 @@ retry: /* Inherit the parent flags for code sign */ child_proc->p_csflags = (parent_proc->p_csflags & ~CS_KILLED); - /* - * All processes have work queue locks; cleaned up by - * reap_child_locked() - */ - workqueue_init_lock(child_proc); - /* * Copy work queue information * @@ -1401,7 +1351,6 @@ retry: child_proc->p_wqthread = parent_proc->p_wqthread; child_proc->p_threadstart = parent_proc->p_threadstart; child_proc->p_pthsize = parent_proc->p_pthsize; - child_proc->p_targconc = parent_proc->p_targconc; if ((parent_proc->p_lflag & P_LREGISTER) != 0) { child_proc->p_lflag |= P_LREGISTER; } @@ -1424,11 +1373,16 @@ retry: #endif #if CONFIG_MEMORYSTATUS - /* Memorystatus + jetsam init */ + /* Memorystatus init */ child_proc->p_memstat_state = 0; child_proc->p_memstat_effectivepriority = JETSAM_PRIORITY_DEFAULT; child_proc->p_memstat_requestedpriority = JETSAM_PRIORITY_DEFAULT; - child_proc->p_memstat_userdata = 0; + child_proc->p_memstat_userdata = 0; + child_proc->p_memstat_idle_start = 0; + child_proc->p_memstat_idle_delta = 0; + child_proc->p_memstat_memlimit = 0; + child_proc->p_memstat_memlimit_active = 0; + child_proc->p_memstat_memlimit_inactive = 0; #if CONFIG_FREEZE child_proc->p_memstat_suspendedfootprint = 0; #endif @@ -1491,19 +1445,25 @@ proc_ucred_unlock(proc_t p) #include -struct zone *uthread_zone; -static int uthread_zone_inited = 0; +struct zone *uthread_zone = NULL; + +static lck_grp_t *rethrottle_lock_grp; +static lck_attr_t *rethrottle_lock_attr; +static lck_grp_attr_t *rethrottle_lock_grp_attr; static void uthread_zone_init(void) { - if (!uthread_zone_inited) { - uthread_zone = zinit(sizeof(struct uthread), - thread_max * sizeof(struct uthread), - THREAD_CHUNK * sizeof(struct uthread), - "uthreads"); - uthread_zone_inited = 1; - } + assert(uthread_zone == NULL); + + rethrottle_lock_grp_attr = lck_grp_attr_alloc_init(); + rethrottle_lock_grp = lck_grp_alloc_init("rethrottle", rethrottle_lock_grp_attr); + rethrottle_lock_attr = lck_attr_alloc_init(); + + uthread_zone = zinit(sizeof(struct uthread), + thread_max * sizeof(struct uthread), + THREAD_CHUNK * sizeof(struct uthread), + "uthreads"); } void * @@ -1514,7 +1474,7 @@ uthread_alloc(task_t task, thread_t thread, int noinherit) uthread_t uth_parent; void *ut; - if (!uthread_zone_inited) + if (uthread_zone == NULL) uthread_zone_init(); ut = (void *)zalloc(uthread_zone); @@ -1524,6 +1484,9 @@ uthread_alloc(task_t task, thread_t thread, int noinherit) uth = (uthread_t)ut; uth->uu_thread = thread; + lck_spin_init(&uth->uu_rethrottle_lock, rethrottle_lock_grp, + rethrottle_lock_attr); + /* * Thread inherits credential from the creating thread, if both * are in the same task. @@ -1568,11 +1531,17 @@ uthread_alloc(task_t task, thread_t thread, int noinherit) uth->uu_sigmask = uth_parent->uu_sigmask; } uth->uu_context.vc_thread = thread; - TAILQ_INSERT_TAIL(&p->p_uthlist, uth, uu_list); + /* + * Do not add the uthread to proc uthlist for exec copy task, + * since they do not hold a ref on proc. + */ + if (!task_is_exec_copy(task)) { + TAILQ_INSERT_TAIL(&p->p_uthlist, uth, uu_list); + } proc_unlock(p); #if CONFIG_DTRACE - if (p->p_dtrace_ptss_pages != NULL) { + if (p->p_dtrace_ptss_pages != NULL && !task_is_exec_copy(task)) { uth->t_dtrace_scratch = dtrace_ptss_claim_entry(p); } #endif @@ -1583,9 +1552,7 @@ uthread_alloc(task_t task, thread_t thread, int noinherit) /* * This routine frees the thread name field of the uthread_t structure. Split out of - * uthread_cleanup() so it can be called separately on the threads of a corpse after - * the corpse notification has been sent, and the handler has had a chance to extract - * the thread names. + * uthread_cleanup() so thread name does not get deallocated while generating a corpse fork. */ void uthread_cleanup_name(void *uthread) @@ -1614,7 +1581,7 @@ uthread_cleanup_name(void *uthread) * It does not free the uthread structure as well */ void -uthread_cleanup(task_t task, void *uthread, void * bsd_info, boolean_t is_corpse) +uthread_cleanup(task_t task, void *uthread, void * bsd_info) { struct _select *sel; uthread_t uth = (uthread_t)uthread; @@ -1647,6 +1614,15 @@ uthread_cleanup(task_t task, void *uthread, void * bsd_info, boolean_t is_corpse */ assert(uth->uu_ar == NULL); + if (uth->uu_kqueue_bound) { + kevent_qos_internal_unbind(p, + uth->uu_kqueue_bound, + uth->uu_thread, + uth->uu_kqueue_flags); + uth->uu_kqueue_flags = 0; + uth->uu_kqueue_bound = 0; + } + sel = &uth->uu_select; /* cleanup the select bit space */ if (sel->nbytes) { @@ -1668,13 +1644,7 @@ uthread_cleanup(task_t task, void *uthread, void * bsd_info, boolean_t is_corpse uth->uu_wqstate_sz = 0; } - /* - * defer the removal of the thread name on process corpses until the corpse has - * been autopsied. - */ - if (!is_corpse) { - uthread_cleanup_name(uth); - } + os_reason_free(uth->uu_exit_reason); if ((task != kernel_task) && p) { @@ -1684,9 +1654,13 @@ uthread_cleanup(task_t task, void *uthread, void * bsd_info, boolean_t is_corpse /* * Remove the thread from the process list and * transfer [appropriate] pending signals to the process. + * Do not remove the uthread from proc uthlist for exec + * copy task, since they does not have a ref on proc and + * would not have been added to the list. */ - if (get_bsdtask_info(task) == p) { + if (get_bsdtask_info(task) == p && !task_is_exec_copy(task)) { proc_lock(p); + TAILQ_REMOVE(&p->p_uthlist, uth, uu_list); p->p_siglist |= (uth->uu_siglist & execmask & (~p->p_sigignore | sigcantmask)); proc_unlock(p); @@ -1694,7 +1668,7 @@ uthread_cleanup(task_t task, void *uthread, void * bsd_info, boolean_t is_corpse #if CONFIG_DTRACE struct dtrace_ptss_page_entry *tmpptr = uth->t_dtrace_scratch; uth->t_dtrace_scratch = NULL; - if (tmpptr != NULL) { + if (tmpptr != NULL && !task_is_exec_copy(task)) { dtrace_ptss_release_entry(p, tmpptr); } #endif @@ -1726,6 +1700,9 @@ uthread_zone_free(void *uthread) uth->t_tombstone = NULL; } + lck_spin_destroy(&uth->uu_rethrottle_lock, rethrottle_lock_grp); + + uthread_cleanup_name(uthread); /* and free the uthread itself */ zfree(uthread_zone, uthread); }