X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..d9a64523371fa019c4575bb400cbbc3a50ac9903:/osfmk/ipc/ipc_importance.c diff --git a/osfmk/ipc/ipc_importance.c b/osfmk/ipc/ipc_importance.c index a8aa9c400..7ab4eb355 100644 --- a/osfmk/ipc/ipc_importance.c +++ b/osfmk/ipc/ipc_importance.c @@ -40,6 +40,7 @@ #include #include #include +#include #include @@ -83,11 +84,8 @@ static lck_spin_t ipc_importance_lock_data; /* single lock for now */ lck_spin_try_lock(&ipc_importance_lock_data) #define ipc_importance_unlock() \ lck_spin_unlock(&ipc_importance_lock_data) -#define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data, \ - LCK_SLEEP_DEFAULT, \ - (event_t)(elem), \ - THREAD_UNINT) -#define ipc_importance_wakeup(elem) thread_wakeup((event_t)(elem)) +#define ipc_importance_assert_held() \ + lck_spin_assert(&ipc_importance_lock_data, LCK_ASSERT_OWNED) #if IIE_REF_DEBUG #define incr_ref_counter(x) (hw_atomic_add(&(x), 1)) @@ -160,6 +158,14 @@ static zone_t ipc_importance_inherit_zone; static ipc_voucher_attr_control_t ipc_importance_control; +static boolean_t ipc_importance_task_check_transition(ipc_importance_task_t task_imp, + iit_update_type_t type, uint32_t delta); + +static void ipc_importance_task_propagate_assertion_locked(ipc_importance_task_t task_imp, + iit_update_type_t type, boolean_t update_task_imp); + +static ipc_importance_inherit_t ipc_importance_inherit_from_task(task_t from_task, task_t to_task); + /* * Routine: ipc_importance_kmsg_link * Purpose: @@ -236,18 +242,53 @@ ipc_importance_inherit_link( ipc_importance_inherit_t inherit, ipc_importance_elem_t elem) { - ipc_importance_elem_t link_elem; + ipc_importance_task_t link_task; assert(IIE_NULL == inherit->iii_from_elem); - link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : - elem; + link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? + ((ipc_importance_inherit_t)elem)->iii_to_task : + (ipc_importance_task_t)elem; - queue_enter(&link_elem->iie_inherits, inherit, + queue_enter(&link_task->iit_inherits, inherit, ipc_importance_inherit_t, iii_inheritance); inherit->iii_from_elem = elem; } +/* + * Routine: ipc_importance_inherit_find + * Purpose: + * Find an existing inherit that links the from element to the + * to_task at a given nesting depth. As inherits from other + * inherits are actually linked off the original inherit's donation + * receiving task, we have to conduct our search from there if + * the from element is an inherit. + * Returns: + * A pointer (not a reference) to the matching inherit. + * Conditions: + * Importance lock held. + */ +static ipc_importance_inherit_t +ipc_importance_inherit_find( + ipc_importance_elem_t from, + ipc_importance_task_t to_task, + unsigned int depth) +{ + ipc_importance_task_t link_task; + ipc_importance_inherit_t inherit; + + link_task = (IIE_TYPE_INHERIT == IIE_TYPE(from)) ? + ((ipc_importance_inherit_t)from)->iii_to_task : + (ipc_importance_task_t)from; + + queue_iterate(&link_task->iit_inherits, inherit, + ipc_importance_inherit_t, iii_inheritance) { + if (inherit->iii_to_task == to_task && inherit->iii_depth == depth) { + return inherit; + } + } + return III_NULL; +} + /* * Routine: ipc_importance_inherit_unlink * Purpose: @@ -268,13 +309,13 @@ ipc_importance_inherit_unlink( ipc_importance_elem_t elem = inherit->iii_from_elem; if (IIE_NULL != elem) { - ipc_importance_elem_t unlink_elem; + ipc_importance_task_t unlink_task; - unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : - elem; + unlink_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? + ((ipc_importance_inherit_t)elem)->iii_to_task : + (ipc_importance_task_t)elem; - queue_remove(&unlink_elem->iie_inherits, inherit, + queue_remove(&unlink_task->iit_inherits, inherit, ipc_importance_inherit_t, iii_inheritance); inherit->iii_from_elem = IIE_NULL; } @@ -308,40 +349,36 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) { assert(0 < IIE_REFS(elem)); - if (0 < ipc_importance_release_internal(elem)) { - -#if DEVELOPMENT || DEBUG - ipc_importance_inherit_t temp_inherit; - ipc_importance_task_t link_task; - ipc_kmsg_t temp_kmsg; - uint32_t expected = 0; - - if (0 < elem->iie_made) - expected++; - - link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - ((ipc_importance_inherit_t)elem)->iii_to_task : - (ipc_importance_task_t)elem; - - queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) - if (temp_kmsg->ikm_importance == elem) - expected++; - queue_iterate(&link_task->iit_inherits, temp_inherit, - ipc_importance_inherit_t, iii_inheritance) - if (temp_inherit->iii_from_elem == elem) - expected++; +#if IMPORTANCE_DEBUG + ipc_importance_inherit_t temp_inherit; + ipc_importance_task_t link_task; + ipc_kmsg_t temp_kmsg; + uint32_t expected = 0; + + if (0 < elem->iie_made) + expected++; + + link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? + ((ipc_importance_inherit_t)elem)->iii_to_task : + (ipc_importance_task_t)elem; + + queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) + if (temp_kmsg->ikm_importance == elem) + expected++; + queue_iterate(&link_task->iit_inherits, temp_inherit, + ipc_importance_inherit_t, iii_inheritance) + if (temp_inherit->iii_from_elem == elem) + expected++; + if (IIE_REFS(elem) < expected + 1) + panic("ipc_importance_release_locked (%p)", elem); +#endif /* IMPORTANCE_DEBUG */ - if (IIE_REFS(elem) < expected) - panic("ipc_importance_release_locked (%p)", elem); -#endif + if (0 < ipc_importance_release_internal(elem)) { ipc_importance_unlock(); return; } /* last ref */ - /* can't get to no refs if we contribute to something else's importance */ - assert(queue_empty(&elem->iie_kmsgs)); - assert(queue_empty(&elem->iie_inherits)); switch (IIE_TYPE(elem)) { @@ -351,6 +388,8 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) ipc_importance_task_t task_elem; task_elem = (ipc_importance_task_t)elem; + + /* the task can't still hold a reference on the task importance */ assert(TASK_NULL == task_elem->iit_task); #if DEVELOPMENT || DEBUG @@ -366,25 +405,63 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) /* dropping an inherit element */ case IIE_TYPE_INHERIT: { - ipc_importance_inherit_t inherit; + ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem; + ipc_importance_task_t to_task = inherit->iii_to_task; ipc_importance_elem_t from_elem; - ipc_importance_task_t to_task; - - inherit = (ipc_importance_inherit_t)elem; - to_task = inherit->iii_to_task; assert(IIT_NULL != to_task); - assert(!inherit->iii_donating); - - /* unlink and release the inherit */ assert(ipc_importance_task_is_any_receiver_type(to_task)); + + /* unlink the inherit from its source element */ from_elem = ipc_importance_inherit_unlink(inherit); assert(IIE_NULL != from_elem); + + /* + * The attribute might have pending external boosts if the attribute + * was given out during exec, drop them from the appropriate destination + * task. + * + * The attribute will not have any pending external boosts if the + * attribute was given out to voucher system since it would have been + * dropped by ipc_importance_release_value, but there is not way to + * detect that, thus if the attribute has a pending external boost, + * drop them from the appropriate destination task. + * + * The inherit attribute from exec and voucher system would not + * get deduped to each other, thus dropping the external boost + * from destination task at two different places will not have + * any unintended side effects. + */ + assert(inherit->iii_externcnt >= inherit->iii_externdrop); + if (inherit->iii_donating) { + uint32_t assertcnt = III_EXTERN(inherit); + + assert(ipc_importance_task_is_any_receiver_type(to_task)); + assert(to_task->iit_externcnt >= inherit->iii_externcnt); + assert(to_task->iit_externdrop >= inherit->iii_externdrop); + to_task->iit_externcnt -= inherit->iii_externcnt; + to_task->iit_externdrop -= inherit->iii_externdrop; + inherit->iii_externcnt = 0; + inherit->iii_externdrop = 0; + inherit->iii_donating = FALSE; + + /* adjust the internal assertions - and propagate as needed */ + if (ipc_importance_task_check_transition(to_task, IIT_UPDATE_DROP, assertcnt)) { + ipc_importance_task_propagate_assertion_locked(to_task, IIT_UPDATE_DROP, TRUE); + } + } else { + inherit->iii_externcnt = 0; + inherit->iii_externdrop = 0; + } + + /* release the reference on the source element */ ipc_importance_release_locked(from_elem); /* unlocked on return */ + /* release the reference on the destination task */ ipc_importance_task_release(to_task); + /* free the inherit */ zfree(ipc_importance_inherit_zone, inherit); break; } @@ -510,16 +587,19 @@ ipc_importance_task_check_transition( iit_update_type_t type, uint32_t delta) { - +#if IMPORTANCE_TRACE task_t target_task = task_imp->iit_task; +#endif boolean_t boost = (IIT_UPDATE_HOLD == type); boolean_t before_boosted, after_boosted; + ipc_importance_assert_held(); + if (!ipc_importance_task_is_any_receiver_type(task_imp)) return FALSE; -#if IMPORTANCE_DEBUG - int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1; +#if IMPORTANCE_TRACE + int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_START, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); @@ -531,34 +611,25 @@ ipc_importance_task_check_transition( /* Adjust the assertcnt appropriately */ if (boost) { task_imp->iit_assertcnt += delta; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid, task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt); #endif } else { // assert(delta <= task_imp->iit_assertcnt); - if (delta > task_imp->iit_assertcnt - IIT_EXTERN(task_imp)) { + if (task_imp->iit_assertcnt < delta + IIT_EXTERN(task_imp)) { /* TODO: Turn this back into a panic */ - if (target_task != TASK_NULL) { - printf("Over-release of kernel-internal importance assertions for pid %d (%s), " - "dropping %d assertion(s) but task only has %d remaining (%d external).\n", - audit_token_pid_from_task(target_task), - (target_task->bsd_info == NULL) ? "" : proc_name_address(target_task->bsd_info), - delta, - task_imp->iit_assertcnt, - IIT_EXTERN(task_imp)); - } task_imp->iit_assertcnt = IIT_EXTERN(task_imp); } else { task_imp->iit_assertcnt -= delta; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE // This convers both legacy and voucher-based importance. DTRACE_BOOST4(drop_boost, task_t, target_task, int, target_pid, int, delta, int, task_imp->iit_assertcnt); #endif } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); #endif @@ -653,7 +724,7 @@ ipc_importance_task_propagate_helper( assert(IP_VALID(port)); ip_lock(port); temp_task_imp = IIT_NULL; - if (!ipc_port_importance_delta_internal(port, &delta, &temp_task_imp)) { + if (!ipc_port_importance_delta_internal(port, IPID_OPTION_NORMAL, &delta, &temp_task_imp)) { ip_unlock(port); } @@ -666,7 +737,7 @@ ipc_importance_task_propagate_helper( /* Adjust the task assertions and determine if an edge was crossed */ if (ipc_importance_task_check_transition(temp_task_imp, type, 1)) { - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); + incr_ref_counter(temp_task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); /* reference donated */ } else { @@ -731,7 +802,7 @@ ipc_importance_task_propagate_helper( assert(ipc_importance_task_is_any_receiver_type(temp_task_imp)); if (ipc_importance_task_check_transition(temp_task_imp, type, assertcnt)) { ipc_importance_task_reference(temp_task_imp); - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); + incr_ref_counter(temp_task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); } } @@ -864,7 +935,7 @@ ipc_importance_task_process_updates( /* complete the policy update with the task unlocked */ ipc_importance_task_release(task_imp); task_unlock(target_task); - task_policy_update_complete_unlocked(target_task, THREAD_NULL, &pend_token); + task_policy_update_complete_unlocked(target_task, &pend_token); task_deallocate(target_task); ipc_importance_lock(); @@ -1007,12 +1078,16 @@ ipc_importance_task_propagate_assertion_locked( queue_init(&updates); queue_init(&propagate); + ipc_importance_assert_held(); + /* * If we're going to update the policy for the provided task, * enqueue it on the propagate queue itself. Otherwise, only * enqueue downstream things. */ if (update_task_imp) { + ipc_importance_task_reference(task_imp); + incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(&propagate, task_imp, ipc_importance_task_t, iit_props); } else { ipc_importance_task_propagate_helper(task_imp, type, &propagate); @@ -1026,6 +1101,8 @@ ipc_importance_task_propagate_assertion_locked( boolean_t need_update; queue_remove_first(&propagate, temp_task_imp, ipc_importance_task_t, iit_props); + /* hold a reference on temp_task_imp */ + assert(IIT_NULL != temp_task_imp); /* only propagate for receivers not already marked as a donor */ @@ -1039,22 +1116,36 @@ ipc_importance_task_propagate_assertion_locked( temp_task_imp->iit_updatepolicy = 0; if (need_update && TASK_NULL != temp_task_imp->iit_task) { if (NULL == temp_task_imp->iit_updateq) { - temp_task_imp->iit_updatetime = 0; - temp_task_imp->iit_updateq = &updates; - ipc_importance_task_reference_internal(temp_task_imp); - if (boost) { - queue_enter(&updates, temp_task_imp, - ipc_importance_task_t, iit_updates); + + /* + * If a downstream task that needs an update is subjects to AppNap, + * drop boosts according to the delay hysteresis. Otherwise, + * immediate update it. + */ + if (!boost && temp_task_imp != task_imp && + ipc_importance_delayed_drop_call != NULL && + ipc_importance_task_is_marked_denap_receiver(temp_task_imp)) { + ipc_importance_task_delayed_drop(temp_task_imp); } else { - queue_enter_first(&updates, temp_task_imp, - ipc_importance_task_t, iit_updates); + temp_task_imp->iit_updatetime = 0; + temp_task_imp->iit_updateq = &updates; + ipc_importance_task_reference_internal(temp_task_imp); + if (boost) { + queue_enter(&updates, temp_task_imp, + ipc_importance_task_t, iit_updates); + } else { + queue_enter_first(&updates, temp_task_imp, + ipc_importance_task_t, iit_updates); + } } } else { /* Must already be on the AppNap hysteresis queue */ - assert(&ipc_importance_delayed_drop_queue); + assert(ipc_importance_delayed_drop_call != NULL); assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp)); } } + + ipc_importance_task_release_internal(temp_task_imp); } /* apply updates to task (may drop importance lock) */ @@ -1241,8 +1332,8 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im ipc_importance_lock(); target_task = task_imp->iit_task; -#if IMPORTANCE_DEBUG - int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1; +#if IMPORTANCE_TRACE + int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); @@ -1267,7 +1358,7 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im } ipc_importance_unlock(); -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); // This covers the legacy case where a task takes an extra boost. @@ -1278,7 +1369,7 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im printf("BUG in process %s[%d]: " "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. " "(%d total, %d external, %d legacy-external)\n", - proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task), + proc_name_address(target_task->bsd_info), task_pid(target_task), target_assertcnt, target_externcnt, target_legacycnt); } @@ -1315,8 +1406,8 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im ipc_importance_lock(); target_task = task_imp->iit_task; -#if IMPORTANCE_DEBUG - int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1; +#if IMPORTANCE_TRACE + int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); @@ -1360,7 +1451,7 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im ret = KERN_SUCCESS; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); #endif @@ -1370,7 +1461,7 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im /* delayed printf for user-supplied data failures */ if (KERN_FAILURE == ret && TASK_NULL != target_task) { printf("BUG in process %s[%d]: over-released legacy external boost assertions (%d total, %d external, %d legacy-external)\n", - proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task), + proc_name_address(target_task->bsd_info), task_pid(target_task), target_assertcnt, target_externcnt, target_legacycnt); } @@ -1393,8 +1484,8 @@ ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, return KERN_FAILURE; } -#if IMPORTANCE_DEBUG - int target_pid = audit_token_pid_from_task(target_task); +#if IMPORTANCE_TRACE + int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_START, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); @@ -1407,12 +1498,12 @@ ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, task_imp->iit_externcnt += count; ipc_importance_unlock(); -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); // This is the legacy boosting path DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, sender_pid, int, count, int, IIT_LEGACY_EXTERN(task_imp)); -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ return(KERN_SUCCESS); } @@ -1455,10 +1546,10 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) before_donor = ipc_importance_task_is_marked_donor(task_imp); /* snapshot task live donor status - may change, but another call will accompany the change */ - task_live_donor = target_task->effective_policy.t_live_donor; + task_live_donor = target_task->effective_policy.tep_live_donor; -#if IMPORTANCE_DEBUG - int target_pid = audit_token_pid_from_task(target_task); +#if IMPORTANCE_TRACE + int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_START, @@ -1484,7 +1575,7 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) ipc_importance_task_propagate_assertion_locked(task_imp, type, FALSE); } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END, target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0); @@ -1520,7 +1611,7 @@ ipc_importance_task_mark_donor(ipc_importance_task_t task_imp, boolean_t donatin KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_INIT_DONOR_STATE)) | DBG_FUNC_NONE, - audit_token_pid_from_task(task_imp->iit_task), donating, + task_pid(task_imp->iit_task), donating, old_donor, task_imp->iit_donor, 0); ipc_importance_unlock(); @@ -1564,7 +1655,7 @@ ipc_importance_task_mark_live_donor(ipc_importance_task_t task_imp, boolean_t li } /* - * Routine: ipc_importance_task_marked_live_donor + * Routine: ipc_importance_task_is_marked_live_donor * Purpose: * Query the live donor and donor flags for the given task importance. * Conditions: @@ -1901,7 +1992,7 @@ void task_importance_update_owner_info(task_t task) { if (task != TASK_NULL && task->task_imp_base != IIT_NULL) { ipc_importance_task_t task_elem = task->task_imp_base; - task_elem->iit_bsd_pid = audit_token_pid_from_task(task); + task_elem->iit_bsd_pid = task_pid(task); if (task->bsd_info) { strncpy(&task_elem->iit_procname[0], proc_name_address(task->bsd_info), 16); task_elem->iit_procname[16] = '\0'; @@ -1940,11 +2031,10 @@ ipc_importance_reset_locked(ipc_importance_task_t task_imp, boolean_t donor) task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop; /* assert(IIT_LEGACY_EXTERN(task_imp) <= task_imp->iit_assertcnt); */ - if (IIT_LEGACY_EXTERN(task_imp) < task_imp->iit_assertcnt) { + if (IIT_EXTERN(task_imp) < task_imp->iit_assertcnt) { task_imp->iit_assertcnt -= IIT_LEGACY_EXTERN(task_imp); } else { - assert(IIT_LEGACY_EXTERN(task_imp) == task_imp->iit_assertcnt); - task_imp->iit_assertcnt = 0; + task_imp->iit_assertcnt = IIT_EXTERN(task_imp); } task_imp->iit_legacy_externcnt = 0; task_imp->iit_legacy_externdrop = 0; @@ -2033,6 +2123,363 @@ ipc_importance_disconnect_task(task_t task) task_deallocate(task); } +/* + * Routine: ipc_importance_exec_switch_task + * Purpose: + * Switch importance task base from old task to new task in exec. + * + * Create an ipc importance linkage from old task to new task, + * once the linkage is created, switch the importance task base + * from old task to new task. After the switch, the linkage will + * represent importance linkage from new task to old task with + * watch port importance inheritance linked to new task. + * Conditions: + * Nothing locked. + * Returns a reference on importance inherit. + */ +ipc_importance_inherit_t +ipc_importance_exec_switch_task( + task_t old_task, + task_t new_task) +{ + ipc_importance_inherit_t inherit = III_NULL; + ipc_importance_task_t old_task_imp = IIT_NULL; + ipc_importance_task_t new_task_imp = IIT_NULL; + + task_importance_reset(old_task); + + /* Create an importance linkage from old_task to new_task */ + inherit = ipc_importance_inherit_from_task(old_task, new_task); + + /* Switch task importance base from old task to new task */ + ipc_importance_lock(); + + old_task_imp = old_task->task_imp_base; + new_task_imp = new_task->task_imp_base; + + old_task_imp->iit_task = new_task; + new_task_imp->iit_task = old_task; + + old_task->task_imp_base = new_task_imp; + new_task->task_imp_base = old_task_imp; + +#if DEVELOPMENT || DEBUG + /* + * Update the pid an proc name for importance base if any + */ + task_importance_update_owner_info(new_task); +#endif + ipc_importance_unlock(); + + return inherit; +} + +/* + * Routine: ipc_importance_check_circularity + * Purpose: + * Check if queueing "port" in a message for "dest" + * would create a circular group of ports and messages. + * + * If no circularity (FALSE returned), then "port" + * is changed from "in limbo" to "in transit". + * + * That is, we want to set port->ip_destination == dest, + * but guaranteeing that this doesn't create a circle + * port->ip_destination->ip_destination->... == port + * + * Additionally, if port was successfully changed to "in transit", + * propagate boost assertions from the "in limbo" port to all + * the ports in the chain, and, if the destination task accepts + * boosts, to the destination task. + * + * Conditions: + * No ports locked. References held for "port" and "dest". + */ + +boolean_t +ipc_importance_check_circularity( + ipc_port_t port, + ipc_port_t dest) +{ + ipc_importance_task_t imp_task = IIT_NULL; + ipc_importance_task_t release_imp_task = IIT_NULL; + boolean_t imp_lock_held = FALSE; + int assertcnt = 0; + ipc_port_t base; + struct turnstile *send_turnstile = TURNSTILE_NULL; + + assert(port != IP_NULL); + assert(dest != IP_NULL); + + if (port == dest) + return TRUE; + base = dest; + + /* Check if destination needs a turnstile */ + ipc_port_send_turnstile_prepare(dest); + + /* port is in limbo, so donation status is safe to latch */ + if (port->ip_impdonation != 0) { + imp_lock_held = TRUE; + ipc_importance_lock(); + } + + /* + * First try a quick check that can run in parallel. + * No circularity if dest is not in transit. + */ + ip_lock(port); + + /* + * Even if port is just carrying assertions for others, + * we need the importance lock. + */ + if (port->ip_impcount > 0 && !imp_lock_held) { + if (!ipc_importance_lock_try()) { + ip_unlock(port); + ipc_importance_lock(); + ip_lock(port); + } + imp_lock_held = TRUE; + } + + if (ip_lock_try(dest)) { + if (!ip_active(dest) || + (dest->ip_receiver_name != MACH_PORT_NULL) || + (dest->ip_destination == IP_NULL)) + goto not_circular; + + /* dest is in transit; further checking necessary */ + + ip_unlock(dest); + } + ip_unlock(port); + + /* + * We're about to pay the cost to serialize, + * just go ahead and grab importance lock. + */ + if (!imp_lock_held) { + ipc_importance_lock(); + imp_lock_held = TRUE; + } + + ipc_port_multiple_lock(); /* massive serialization */ + + /* + * Search for the end of the chain (a port not in transit), + * acquiring locks along the way. + */ + + for (;;) { + ip_lock(base); + + if (!ip_active(base) || + (base->ip_receiver_name != MACH_PORT_NULL) || + (base->ip_destination == IP_NULL)) + break; + + base = base->ip_destination; + } + + /* all ports in chain from dest to base, inclusive, are locked */ + + if (port == base) { + /* circularity detected! */ + + ipc_port_multiple_unlock(); + + /* port (== base) is in limbo */ + + assert(ip_active(port)); + assert(port->ip_receiver_name == MACH_PORT_NULL); + assert(port->ip_destination == IP_NULL); + + base = dest; + while (base != IP_NULL) { + ipc_port_t next; + + /* base is in transit or in limbo */ + + assert(ip_active(base)); + assert(base->ip_receiver_name == MACH_PORT_NULL); + + next = base->ip_destination; + ip_unlock(base); + base = next; + } + + if (imp_lock_held) + ipc_importance_unlock(); + + ipc_port_send_turnstile_complete(dest); + return TRUE; + } + + /* + * The guarantee: lock port while the entire chain is locked. + * Once port is locked, we can take a reference to dest, + * add port to the chain, and unlock everything. + */ + + ip_lock(port); + ipc_port_multiple_unlock(); + +not_circular: + /* port is in limbo */ + imq_lock(&port->ip_messages); + + assert(ip_active(port)); + assert(port->ip_receiver_name == MACH_PORT_NULL); + assert(port->ip_destination == IP_NULL); + + ip_reference(dest); + port->ip_destination = dest; + + /* must have been in limbo or still bound to a task */ + assert(port->ip_tempowner != 0); + + /* + * We delayed dropping assertions from a specific task. + * Cache that info now (we'll drop assertions and the + * task reference below). + */ + release_imp_task = port->ip_imp_task; + if (IIT_NULL != release_imp_task) { + port->ip_imp_task = IIT_NULL; + } + assertcnt = port->ip_impcount; + + /* take the port out of limbo w.r.t. assertions */ + port->ip_tempowner = 0; + + /* + * Setup linkage for source port if it has a send turnstile i.e. it has + * a thread waiting in send or has a port enqueued in it or has sync ipc + * push from a special reply port. + */ + if (port_send_turnstile(port)) { + send_turnstile = turnstile_prepare((uintptr_t)port, + port_send_turnstile_address(port), + TURNSTILE_NULL, TURNSTILE_SYNC_IPC); + + turnstile_update_inheritor(send_turnstile, port_send_turnstile(dest), + (TURNSTILE_INHERITOR_TURNSTILE | TURNSTILE_IMMEDIATE_UPDATE)); + + /* update complete and turnstile complete called after dropping all locks */ + } + imq_unlock(&port->ip_messages); + + /* now unlock chain */ + + ip_unlock(port); + + for (;;) { + + ipc_port_t next; + /* every port along chain track assertions behind it */ + ipc_port_impcount_delta(dest, assertcnt, base); + + if (dest == base) + break; + + /* port is in transit */ + + assert(ip_active(dest)); + assert(dest->ip_receiver_name == MACH_PORT_NULL); + assert(dest->ip_destination != IP_NULL); + assert(dest->ip_tempowner == 0); + + next = dest->ip_destination; + ip_unlock(dest); + dest = next; + } + + /* base is not in transit */ + assert(!ip_active(base) || + (base->ip_receiver_name != MACH_PORT_NULL) || + (base->ip_destination == IP_NULL)); + + /* + * Find the task to boost (if any). + * We will boost "through" ports that don't know + * about inheritance to deliver receive rights that + * do. + */ + if (ip_active(base) && (assertcnt > 0)) { + assert(imp_lock_held); + if (base->ip_tempowner != 0) { + if (IIT_NULL != base->ip_imp_task) { + /* specified tempowner task */ + imp_task = base->ip_imp_task; + assert(ipc_importance_task_is_any_receiver_type(imp_task)); + } + /* otherwise don't boost current task */ + + } else if (base->ip_receiver_name != MACH_PORT_NULL) { + ipc_space_t space = base->ip_receiver; + + /* only spaces with boost-accepting tasks */ + if (space->is_task != TASK_NULL && + ipc_importance_task_is_any_receiver_type(space->is_task->task_imp_base)) + imp_task = space->is_task->task_imp_base; + } + + /* take reference before unlocking base */ + if (imp_task != IIT_NULL) { + ipc_importance_task_reference(imp_task); + } + } + + ip_unlock(base); + + /* + * Transfer assertions now that the ports are unlocked. + * Avoid extra overhead if transferring to/from the same task. + * + * NOTE: If a transfer is occurring, the new assertions will + * be added to imp_task BEFORE the importance lock is unlocked. + * This is critical - to avoid decrements coming from the kmsgs + * beating the increment to the task. + */ + boolean_t transfer_assertions = (imp_task != release_imp_task); + + if (imp_task != IIT_NULL) { + assert(imp_lock_held); + if (transfer_assertions) + ipc_importance_task_hold_internal_assertion_locked(imp_task, assertcnt); + } + + if (release_imp_task != IIT_NULL) { + assert(imp_lock_held); + if (transfer_assertions) + ipc_importance_task_drop_internal_assertion_locked(release_imp_task, assertcnt); + } + + if (imp_lock_held) + ipc_importance_unlock(); + + /* All locks dropped, call turnstile_update_inheritor_complete for source port's turnstile */ + if (send_turnstile) { + turnstile_update_inheritor_complete(send_turnstile, TURNSTILE_INTERLOCK_NOT_HELD); + + /* Take the mq lock to call turnstile complete */ + imq_lock(&port->ip_messages); + turnstile_complete((uintptr_t)port, port_send_turnstile_address(port), NULL); + send_turnstile = TURNSTILE_NULL; + imq_unlock(&port->ip_messages); + turnstile_cleanup(); + } + + if (imp_task != IIT_NULL) + ipc_importance_task_release(imp_task); + + if (release_imp_task != IIT_NULL) + ipc_importance_task_release(release_imp_task); + + return FALSE; +} + /* * Routine: ipc_importance_send * Purpose: @@ -2055,7 +2502,6 @@ ipc_importance_send( ipc_importance_task_t task_imp; kern_return_t kr; - assert(IP_VALID(port)); /* If no donation to be made, return quickly */ @@ -2068,7 +2514,12 @@ ipc_importance_send( /* If forced sending a static boost, go update the port */ if ((option & MACH_SEND_IMPORTANCE) != 0) { - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; + /* acquire the importance lock while trying to hang on to port lock */ + if (!ipc_importance_lock_try()) { + port_lock_dropped = TRUE; + ip_unlock(port); + ipc_importance_lock(); + } goto portupdate; } @@ -2151,43 +2602,74 @@ ipc_importance_send( return port_lock_dropped; } +portupdate: /* Mark the fact that we are (currently) donating through this message */ kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; /* * If we need to relock the port, do it with the importance still locked. * This assures we get to add the importance boost through the port to - * the task BEFORE anyone else can attempt to undo that operation because + * the task BEFORE anyone else can attempt to undo that operation if * the sender lost donor status. */ if (TRUE == port_lock_dropped) { ip_lock(port); } - ipc_importance_unlock(); - portupdate: - -#if IMPORTANCE_DEBUG + ipc_importance_assert_held(); + +#if IMPORTANCE_TRACE if (kdebug_enable) { mach_msg_max_trailer_t *dbgtrailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); unsigned int sender_pid = dbgtrailer->msgh_audit.val[5]; mach_msg_id_t imp_msgh_id = kmsg->ikm_header->msgh_id; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_START, - audit_token_pid_from_task(task), sender_pid, imp_msgh_id, 0, 0); + task_pid(task), sender_pid, imp_msgh_id, 0, 0); + } +#endif /* IMPORTANCE_TRACE */ + + mach_port_delta_t delta = 1; + boolean_t need_port_lock; + task_imp = IIT_NULL; + + /* adjust port boost count (with importance and port locked) */ + need_port_lock = ipc_port_importance_delta_internal(port, IPID_OPTION_NORMAL, &delta, &task_imp); + /* hold a reference on task_imp */ + + /* if we need to adjust a task importance as a result, apply that here */ + if (IIT_NULL != task_imp && delta != 0) { + assert(delta == 1); + + /* if this results in a change of state, propagate the transistion */ + if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_HOLD, delta)) { + + /* can't hold the port lock during task transition(s) */ + if (!need_port_lock) { + need_port_lock = TRUE; + ip_unlock(port); + } + ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_HOLD, TRUE); + } } -#endif /* IMPORTANCE_DEBUG */ - /* adjust port boost count (with port locked) */ - if (TRUE == ipc_port_importance_delta(port, 1)) { + if (task_imp) { + ipc_importance_task_release_locked(task_imp); + /* importance unlocked */ + } else { + ipc_importance_unlock(); + } + + if (need_port_lock) { port_lock_dropped = TRUE; ip_lock(port); } + return port_lock_dropped; } /* - * Routine: ipc_importance_inherit_from + * Routine: ipc_importance_inherit_from_kmsg * Purpose: * Create a "made" reference for an importance attribute representing * an inheritance between the sender of a message (if linked) and the @@ -2201,7 +2683,7 @@ ipc_importance_send( * Nothing locked on entry. May block. */ static ipc_importance_inherit_t -ipc_importance_inherit_from(ipc_kmsg_t kmsg) +ipc_importance_inherit_from_kmsg(ipc_kmsg_t kmsg) { ipc_importance_task_t task_imp = IIT_NULL; ipc_importance_elem_t from_elem = kmsg->ikm_importance; @@ -2211,7 +2693,6 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) ipc_port_t port = kmsg->ikm_header->msgh_remote_port; ipc_importance_inherit_t inherit = III_NULL; ipc_importance_inherit_t alloc = III_NULL; - ipc_importance_inherit_t temp_inherit; boolean_t cleared_self_donation = FALSE; boolean_t donating; uint32_t depth = 1; @@ -2309,14 +2790,7 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) * check to see if we already have an inherit for this pairing */ while (III_NULL == inherit) { - queue_iterate(&from_elem->iie_inherits, temp_inherit, - ipc_importance_inherit_t, iii_inheritance) { - if (temp_inherit->iii_to_task == task_imp && - temp_inherit->iii_depth == depth) { - inherit = temp_inherit; - break; - } - } + inherit = ipc_importance_inherit_find(from_elem, task_imp, depth); /* Do we have to allocate a new inherit */ if (III_NULL == inherit) { @@ -2361,9 +2835,6 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) /* add in a external reference for this use of the inherit */ inherit->iii_externcnt++; - if (donating) { - task_imp->iit_externcnt++; - } } else { /* initialize the previously allocated space */ inherit = alloc; @@ -2375,12 +2846,9 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) inherit->iii_to_task = task_imp; inherit->iii_from_elem = IIE_NULL; queue_init(&inherit->iii_kmsgs); - queue_init(&inherit->iii_inherits); - /* If donating, reflect that in the task externcnt */ if (donating) { inherit->iii_donating = TRUE; - task_imp->iit_externcnt++; } else { inherit->iii_donating = FALSE; } @@ -2409,6 +2877,14 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) elem = ipc_importance_kmsg_unlink(kmsg); assert(elem == from_elem); + /* If found inherit and donating, reflect that in the task externcnt */ + if (III_NULL != inherit && donating) { + task_imp->iit_externcnt++; + /* The owner of receive right might have changed, take the internal assertion */ + ipc_importance_task_hold_internal_assertion_locked(task_imp, 1); + /* may have dropped and retaken importance lock */ + } + /* If we didn't create a new inherit, we have some resources to release */ if (III_NULL == inherit || inherit != alloc) { if (IIE_NULL != from_elem) { @@ -2437,23 +2913,16 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) ipc_importance_unlock(); } - /* decrement port boost count */ - if (donating) { - ip_lock(port); - if (III_NULL != inherit) { - /* task assertions transferred to inherit, just adjust port count */ - ipc_port_impcount_delta(port, -1, IP_NULL); - ip_unlock(port); - } else { - /* drop importance from port and destination task */ - if (ipc_port_importance_delta(port, -1) == FALSE) { - ip_unlock(port); - } - } - } else if (cleared_self_donation) { + /* + * decrement port boost count + * This is OK to do without the importance lock as we atomically + * unlinked the kmsg and snapshot the donating state while holding + * the importance lock + */ + if (donating || cleared_self_donation) { ip_lock(port); - /* drop cleared donation from port and destination task */ - if (ipc_port_importance_delta(port, -1) == FALSE) { + /* drop importance from port and destination task */ + if (ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { ip_unlock(port); } } @@ -2469,6 +2938,181 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) return inherit; } +/* + * Routine: ipc_importance_inherit_from_task + * Purpose: + * Create a reference for an importance attribute representing + * an inheritance between the to_task and from_task. The iii + * created will be marked as III_FLAGS_FOR_OTHERS. + * + * It will not dedup any iii which are not marked as III_FLAGS_FOR_OTHERS. + * + * If the task is inactive, there isn't any need to return a new reference. + * Conditions: + * Nothing locked on entry. May block. + * It should not be called from voucher subsystem. + */ +static ipc_importance_inherit_t +ipc_importance_inherit_from_task( + task_t from_task, + task_t to_task) +{ + ipc_importance_task_t to_task_imp = IIT_NULL; + ipc_importance_task_t from_task_imp = IIT_NULL; + ipc_importance_elem_t from_elem = IIE_NULL; + + ipc_importance_inherit_t inherit = III_NULL; + ipc_importance_inherit_t alloc = III_NULL; + boolean_t donating; + uint32_t depth = 1; + + to_task_imp = ipc_importance_for_task(to_task, FALSE); + from_task_imp = ipc_importance_for_task(from_task, FALSE); + from_elem = (ipc_importance_elem_t)from_task_imp; + + ipc_importance_lock(); + + if (IIT_NULL == to_task_imp || IIT_NULL == from_task_imp) { + goto out_locked; + } + + /* + * No need to set up an inherit linkage if the to_task or from_task + * isn't a receiver of one type or the other. + */ + if (!ipc_importance_task_is_any_receiver_type(to_task_imp) || + !ipc_importance_task_is_any_receiver_type(from_task_imp)) { + goto out_locked; + } + + /* Do not allow to create a linkage to self */ + if (to_task_imp == from_task_imp) { + goto out_locked; + } + + incr_ref_counter(to_task_imp->iit_elem.iie_task_refs_added_inherit_from); + incr_ref_counter(from_elem->iie_kmsg_refs_added); + + /* + * Now that we have the from_elem figured out, + * check to see if we already have an inherit for this pairing + */ + while (III_NULL == inherit) { + inherit = ipc_importance_inherit_find(from_elem, to_task_imp, depth); + + /* Do we have to allocate a new inherit */ + if (III_NULL == inherit) { + if (III_NULL != alloc) { + break; + } + + /* allocate space */ + ipc_importance_unlock(); + alloc = (ipc_importance_inherit_t) + zalloc(ipc_importance_inherit_zone); + ipc_importance_lock(); + } + } + + /* snapshot the donating status while we have importance locked */ + donating = ipc_importance_task_is_donor(from_task_imp); + + if (III_NULL != inherit) { + /* We found one, piggyback on that */ + assert(0 < III_REFS(inherit)); + assert(0 < IIE_REFS(inherit->iii_from_elem)); + + /* Take a reference for inherit */ + assert(III_REFS_MAX > III_REFS(inherit)); + ipc_importance_inherit_reference_internal(inherit); + + /* Reflect the inherit's change of status into the task boosts */ + if (0 == III_EXTERN(inherit)) { + assert(!inherit->iii_donating); + inherit->iii_donating = donating; + if (donating) { + to_task_imp->iit_externcnt += inherit->iii_externcnt; + to_task_imp->iit_externdrop += inherit->iii_externdrop; + } + } else { + assert(donating == inherit->iii_donating); + } + + /* add in a external reference for this use of the inherit */ + inherit->iii_externcnt++; + } else { + /* initialize the previously allocated space */ + inherit = alloc; + inherit->iii_bits = IIE_TYPE_INHERIT | 1; + inherit->iii_made = 0; + inherit->iii_externcnt = 1; + inherit->iii_externdrop = 0; + inherit->iii_depth = depth; + inherit->iii_to_task = to_task_imp; + inherit->iii_from_elem = IIE_NULL; + queue_init(&inherit->iii_kmsgs); + + if (donating) { + inherit->iii_donating = TRUE; + } else { + inherit->iii_donating = FALSE; + } + + /* + * Chain our new inherit on the element it inherits from. + * The new inherit takes our reference on from_elem. + */ + ipc_importance_inherit_link(inherit, from_elem); + +#if IIE_REF_DEBUG + ipc_importance_counter_init(&inherit->iii_elem); + from_elem->iie_kmsg_refs_inherited++; + task_imp->iit_elem.iie_task_refs_inherited++; +#endif + } + +out_locked: + + /* If found inherit and donating, reflect that in the task externcnt */ + if (III_NULL != inherit && donating) { + to_task_imp->iit_externcnt++; + /* take the internal assertion */ + ipc_importance_task_hold_internal_assertion_locked(to_task_imp, 1); + /* may have dropped and retaken importance lock */ + } + + /* If we didn't create a new inherit, we have some resources to release */ + if (III_NULL == inherit || inherit != alloc) { + if (IIE_NULL != from_elem) { + if (III_NULL != inherit) { + incr_ref_counter(from_elem->iie_kmsg_refs_coalesced); + } else { + incr_ref_counter(from_elem->iie_kmsg_refs_dropped); + } + ipc_importance_release_locked(from_elem); + /* importance unlocked */ + } else { + ipc_importance_unlock(); + } + + if (IIT_NULL != to_task_imp) { + if (III_NULL != inherit) { + incr_ref_counter(to_task_imp->iit_elem.iie_task_refs_coalesced); + } + ipc_importance_task_release(to_task_imp); + } + + if (III_NULL != alloc) { + zfree(ipc_importance_inherit_zone, alloc); + } + } else { + /* from_elem and to_task_imp references transferred to new inherit */ + ipc_importance_unlock(); + } + + return inherit; +} + /* * Routine: ipc_importance_receive * Purpose: @@ -2521,34 +3165,41 @@ ipc_importance_receive( * transferring any boosts from the kmsg linkage through the * port directly to the new inheritance object. */ - inherit = ipc_importance_inherit_from(kmsg); + inherit = ipc_importance_inherit_from_kmsg(kmsg); handle = (mach_voucher_attr_value_handle_t)inherit; assert(IIE_NULL == kmsg->ikm_importance); - /* replace the importance attribute with the handle we created */ - /* our made reference on the inhert is donated to the voucher */ - recipe = (ipc_voucher_attr_recipe_t)&recipes[recipe_size]; - recipe->key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE; - recipe->command = MACH_VOUCHER_ATTR_SET_VALUE_HANDLE; - recipe->previous_voucher = IPC_VOUCHER_NULL; - recipe->content_size = sizeof(mach_voucher_attr_value_handle_t); - *(mach_voucher_attr_value_handle_t *)(void *)recipe->content = handle; - recipe_size += sizeof(*recipe) + sizeof(mach_voucher_attr_value_handle_t); - - kr = ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control, - recipes, - recipe_size, - &recv_voucher); - assert(KERN_SUCCESS == kr); - - /* swap the voucher port (and set voucher bits in case it didn't already exist) */ - kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16); - ipc_port_release_send(kmsg->ikm_voucher); - kmsg->ikm_voucher = convert_voucher_to_port(recv_voucher); - if (III_NULL != inherit) - impresult = 2; - + /* + * Only create a new voucher if we have an inherit object + * (from the ikm_importance field of the incoming message), OR + * we have a valid incoming voucher. If we have neither of + * these things then there is no need to create a new voucher. + */ + if (IP_VALID(kmsg->ikm_voucher) || inherit != III_NULL) { + /* replace the importance attribute with the handle we created */ + /* our made reference on the inherit is donated to the voucher */ + recipe = (ipc_voucher_attr_recipe_t)&recipes[recipe_size]; + recipe->key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE; + recipe->command = MACH_VOUCHER_ATTR_SET_VALUE_HANDLE; + recipe->previous_voucher = IPC_VOUCHER_NULL; + recipe->content_size = sizeof(mach_voucher_attr_value_handle_t); + *(mach_voucher_attr_value_handle_t *)(void *)recipe->content = handle; + recipe_size += sizeof(*recipe) + sizeof(mach_voucher_attr_value_handle_t); + + kr = ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control, + recipes, + recipe_size, + &recv_voucher); + assert(KERN_SUCCESS == kr); + + /* swap the voucher port (and set voucher bits in case it didn't already exist) */ + kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16); + ipc_port_release_send(kmsg->ikm_voucher); + kmsg->ikm_voucher = convert_voucher_to_port(recv_voucher); + if (III_NULL != inherit) + impresult = 2; + } } else { /* Don't want a voucher */ /* got linked importance? have to drop */ @@ -2569,26 +3220,28 @@ ipc_importance_receive( ipc_importance_task_t task_imp = task_self->task_imp_base; ipc_port_t port = kmsg->ikm_header->msgh_remote_port; - /* defensive deduction for release builds lacking the assert */ - ip_lock(port); - ipc_port_impcount_delta(port, -1, IP_NULL); - ip_unlock(port); - - /* will user accept legacy responsibility for the importance boost */ - if (KERN_SUCCESS == ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid)) { + /* The owner of receive right might have changed, take the internal assertion */ + if (KERN_SUCCESS == ipc_importance_task_hold_internal_assertion(task_imp, 1)) { + ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid); impresult = 1; } else { /* The importance boost never applied to task (clear the bit) */ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; impresult = 0; } + + /* Drop the boost on the port and the owner of the receive right */ + ip_lock(port); + if (ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { + ip_unlock(port); + } } } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE if (-1 < impresult) KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_DELV)) | DBG_FUNC_NONE, - sender_pid, audit_token_pid_from_task(task_self), + sender_pid, task_pid(task_self), kmsg->ikm_header->msgh_id, impresult, 0); if (impresult == 2){ /* @@ -2596,9 +3249,9 @@ ipc_importance_receive( * will trigger the probe in ipc_importance_task_externalize_assertion() * above and have impresult==1 here. */ - DTRACE_BOOST5(receive_boost, task_t, task_self, int, audit_token_pid_from_task(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt); + DTRACE_BOOST5(receive_boost, task_t, task_self, int, task_pid(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt); } -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ } /* @@ -2669,7 +3322,7 @@ ipc_importance_clean( ip_lock(port); /* inactive ports already had their importance boosts dropped */ if (!ip_active(port) || - ipc_port_importance_delta(port, -1) == FALSE) { + ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { ip_unlock(port); } } @@ -2704,6 +3357,7 @@ ipc_importance_get_value( mach_voucher_attr_content_t content, mach_voucher_attr_content_size_t content_size, mach_voucher_attr_value_handle_t *out_value, + mach_voucher_attr_value_flags_t *out_flags, ipc_voucher_t *out_value_voucher); static kern_return_t @@ -2738,6 +3392,7 @@ struct ipc_voucher_attr_manager ipc_importance_manager = { .ivam_extract_content = ipc_importance_extract_content, .ivam_command = ipc_importance_command, .ivam_release = ipc_importance_manager_release, + .ivam_flags = IVAM_FLAGS_NONE, }; #define IMPORTANCE_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_IMPORTANCE == (key)) @@ -2780,9 +3435,9 @@ ipc_importance_release_value( /* clear made */ elem->iie_made = 0; - /* - * If there are pending external boosts represented by this attribute, - * drop them from the apropriate task + /* + * If there are pending external boosts represented by this attribute, + * drop them from the apropriate task */ if (IIE_TYPE_INHERIT == IIE_TYPE(elem)) { ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem; @@ -2810,7 +3465,7 @@ ipc_importance_release_value( inherit->iii_externcnt = 0; inherit->iii_externdrop = 0; } - } + } /* drop the made reference on elem */ ipc_importance_release_locked(elem); @@ -2840,6 +3495,7 @@ ipc_importance_get_value( mach_voucher_attr_content_t __unused content, mach_voucher_attr_content_size_t content_size, mach_voucher_attr_value_handle_t *out_value, + mach_voucher_attr_value_flags_t *out_flags, ipc_voucher_t *out_value_voucher) { ipc_importance_elem_t elem; @@ -2851,6 +3507,7 @@ ipc_importance_get_value( if (0 != content_size) return KERN_INVALID_ARGUMENT; + *out_flags = MACH_VOUCHER_ATTR_VALUE_FLAGS_NONE; /* never an out voucher */ switch (command) { @@ -2939,22 +3596,22 @@ ipc_importance_extract_content( ipc_importance_inherit_t inherit = III_NULL; ipc_importance_task_t task_imp; task_t task; - int task_pid; + int t_pid; if (IIE_TYPE_TASK == IIE_TYPE(elem)) { task_imp = (ipc_importance_task_t)elem; task = task_imp->iit_task; - task_pid = (TASK_NULL != task) ? - audit_token_pid_from_task(task) : -1; - snprintf((char *)out_content + size, *in_out_content_size - size, "%d", task_pid); + t_pid = (TASK_NULL != task) ? + task_pid(task) : -1; + snprintf((char *)out_content + size, *in_out_content_size - size, "%d", t_pid); } else { inherit = (ipc_importance_inherit_t)elem; task_imp = inherit->iii_to_task; task = task_imp->iit_task; - task_pid = (TASK_NULL != task) ? - audit_token_pid_from_task(task) : -1; + t_pid = (TASK_NULL != task) ? + task_pid(task) : -1; snprintf((char *)out_content + size, *in_out_content_size - size, - "%d (%d of %d boosts) %s from pid ", task_pid, + "%d (%d of %d boosts) %s from pid ", t_pid, III_EXTERN(inherit), inherit->iii_externcnt, (inherit->iii_donating) ? "donated" : "linked"); } @@ -3037,15 +3694,21 @@ ipc_importance_command( return KERN_SUCCESS; } + to_task = inherit->iii_to_task; + assert(ipc_importance_task_is_any_receiver_type(to_task)); + + /* if not donating to a denap receiver, it was called incorrectly */ + if (!ipc_importance_task_is_marked_denap_receiver(to_task)) { + ipc_importance_unlock(); + return KERN_INVALID_TASK; /* keeps dispatch happy */ + } + /* Enough external references left to drop? */ if (III_EXTERN(inherit) < refs) { ipc_importance_unlock(); return KERN_FAILURE; } - to_task = inherit->iii_to_task; - assert(ipc_importance_task_is_any_receiver_type(to_task)); - /* re-base external and internal counters at the inherit and the to-task (if apropos) */ if (inherit->iii_donating) { assert(IIT_EXTERN(to_task) >= III_EXTERN(inherit)); @@ -3185,9 +3848,9 @@ ipc_importance_thread_call_init(void) * Will panic the system otherwise. */ extern int -task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int max_count) +task_importance_list_pids(task_t task, int flags, char *pid_list, unsigned int max_count) { - if (lck_spin_is_acquired(&ipc_importance_lock_data) || + if (kdp_lck_spin_is_acquired(&ipc_importance_lock_data) || max_count < 1 || task->task_imp_base == IIT_NULL || pid_list == NULL || @@ -3200,12 +3863,13 @@ task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int ma ipc_kmsg_t temp_kmsg; ipc_importance_inherit_t temp_inherit; ipc_importance_elem_t elem; - int target_pid; + int target_pid = 0, previous_pid; queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) { /* check space in buffer */ if (pidcount >= max_count) break; + previous_pid = target_pid; target_pid = -1; if (temp_inherit->iii_donating) { @@ -3215,20 +3879,24 @@ task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int ma #else temp_task = temp_inherit->iii_to_task->iit_task; if (temp_task != TASK_NULL) { - target_pid = audit_token_pid_from_task(temp_task); + target_pid = task_pid(temp_task); } #endif } - if (target_pid != -1) { - pid_list[pidcount++] = target_pid; + if (target_pid != -1 && previous_pid != target_pid) { + memcpy(pid_list, &target_pid, sizeof(target_pid)); + pid_list += sizeof(target_pid); + pidcount++; } } + target_pid = 0; queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) { if (pidcount >= max_count) break; + previous_pid = target_pid; target_pid = -1; elem = temp_kmsg->ikm_importance; temp_task = TASK_NULL; @@ -3243,7 +3911,7 @@ task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int ma if (IIE_TYPE_TASK == IIE_TYPE(elem) && (((ipc_importance_task_t)elem)->iit_task != TASK_NULL)) { - target_pid = audit_token_pid_from_task(((ipc_importance_task_t)elem)->iit_task); + target_pid = task_pid(((ipc_importance_task_t)elem)->iit_task); } else { temp_inherit = (ipc_importance_inherit_t)elem; #if DEVELOPMENT || DEBUG @@ -3251,13 +3919,15 @@ task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int ma #else temp_task = temp_inherit->iii_to_task->iit_task; if (temp_task != TASK_NULL) { - target_pid = audit_token_pid_from_task(temp_task); + target_pid = task_pid(temp_task); } #endif } - if (target_pid != -1) { - pid_list[pidcount++] = target_pid; + if (target_pid != -1 && previous_pid != target_pid) { + memcpy(pid_list, &target_pid, sizeof(target_pid)); + pid_list += sizeof(target_pid); + pidcount++; } }