X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4bd07ac2140668789aa3ee8ec4dde4a3e0a3bba5..bca245acd4c03fd752d1a45f011ad495e60fe53d:/osfmk/ipc/ipc_importance.c diff --git a/osfmk/ipc/ipc_importance.c b/osfmk/ipc/ipc_importance.c index bd48b1da7..c403b9f46 100644 --- a/osfmk/ipc/ipc_importance.c +++ b/osfmk/ipc/ipc_importance.c @@ -40,16 +40,17 @@ #include #include #include +#include #include #include #include -extern int proc_pid(void *); -extern int proc_selfpid(void); +extern int proc_pid(void *); +extern int proc_selfpid(void); extern uint64_t proc_uniqueid(void *p); -extern char *proc_name_address(void *p); +extern char *proc_name_address(void *p); /* * Globals for delayed boost drop processing. @@ -70,66 +71,67 @@ static boolean_t ipc_importance_delayed_drop_call_requested = FALSE; * Importance Voucher Attribute Manager */ -static lck_spin_t ipc_importance_lock_data; /* single lock for now */ +static lck_spin_t ipc_importance_lock_data; /* single lock for now */ #define ipc_importance_lock_init() \ lck_spin_init(&ipc_importance_lock_data, &ipc_lck_grp, &ipc_lck_attr) #define ipc_importance_lock_destroy() \ lck_spin_destroy(&ipc_importance_lock_data, &ipc_lck_grp) -#define ipc_importance_lock() \ - lck_spin_lock(&ipc_importance_lock_data) -#define ipc_importance_lock_try() \ - lck_spin_try_lock(&ipc_importance_lock_data) -#define ipc_importance_unlock() \ +#define ipc_importance_lock() \ + lck_spin_lock_grp(&ipc_importance_lock_data, &ipc_lck_grp) +#define ipc_importance_lock_try() \ + lck_spin_try_lock_grp(&ipc_importance_lock_data, &ipc_lck_grp) +#define ipc_importance_unlock() \ lck_spin_unlock(&ipc_importance_lock_data) -#define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data, \ - LCK_SLEEP_DEFAULT, \ - (event_t)(elem), \ - THREAD_UNINT) -#define ipc_importance_wakeup(elem) thread_wakeup((event_t)(elem)) +#define ipc_importance_assert_held() \ + lck_spin_assert(&ipc_importance_lock_data, LCK_ASSERT_OWNED) #if IIE_REF_DEBUG -#define incr_ref_counter(x) (hw_atomic_add(&(x), 1)) +#define incr_ref_counter(x) (os_atomic_inc(&(x), relaxed)) static inline -uint32_t ipc_importance_reference_internal(ipc_importance_elem_t elem) +uint32_t +ipc_importance_reference_internal(ipc_importance_elem_t elem) { incr_ref_counter(elem->iie_refs_added); - return (hw_atomic_add(&elem->iie_bits, 1) & IIE_REFS_MASK); + return os_atomic_inc(&elem->iie_bits, relaxed) & IIE_REFS_MASK; } static inline -uint32_t ipc_importance_release_internal(ipc_importance_elem_t elem) +uint32_t +ipc_importance_release_internal(ipc_importance_elem_t elem) { incr_ref_counter(elem->iie_refs_dropped); - return (hw_atomic_sub(&elem->iie_bits, 1) & IIE_REFS_MASK); + return os_atomic_dec(&elem->iie_bits, relaxed) & IIE_REFS_MASK; } -static inline -uint32_t ipc_importance_task_reference_internal(ipc_importance_task_t task_imp) +static inline +uint32_t +ipc_importance_task_reference_internal(ipc_importance_task_t task_imp) { - uint32_t out; - out = ipc_importance_reference_internal(&task_imp->iit_elem); - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added); - return out; + uint32_t out; + out = ipc_importance_reference_internal(&task_imp->iit_elem); + incr_ref_counter(task_imp->iit_elem.iie_task_refs_added); + return out; } static inline -uint32_t ipc_importance_task_release_internal(ipc_importance_task_t task_imp) +uint32_t +ipc_importance_task_release_internal(ipc_importance_task_t task_imp) { - uint32_t out; + uint32_t out; - assert(1 < IIT_REFS(task_imp)); - incr_ref_counter(task_imp->iit_elem.iie_task_refs_dropped); - out = ipc_importance_release_internal(&task_imp->iit_elem); - return out; + assert(1 < IIT_REFS(task_imp)); + incr_ref_counter(task_imp->iit_elem.iie_task_refs_dropped); + out = ipc_importance_release_internal(&task_imp->iit_elem); + return out; } static inline -void ipc_importance_counter_init(ipc_importance_elem_t elem) +void +ipc_importance_counter_init(ipc_importance_elem_t elem) { - elem->iie_refs_added = 0; elem->iie_refs_dropped = 0; elem->iie_kmsg_refs_added = 0; @@ -160,6 +162,14 @@ static zone_t ipc_importance_inherit_zone; static ipc_voucher_attr_control_t ipc_importance_control; +static boolean_t ipc_importance_task_check_transition(ipc_importance_task_t task_imp, + iit_update_type_t type, uint32_t delta); + +static void ipc_importance_task_propagate_assertion_locked(ipc_importance_task_t task_imp, + iit_update_type_t type, boolean_t update_task_imp); + +static ipc_importance_inherit_t ipc_importance_inherit_from_task(task_t from_task, task_t to_task); + /* * Routine: ipc_importance_kmsg_link * Purpose: @@ -173,16 +183,16 @@ static ipc_voucher_attr_control_t ipc_importance_control; */ static void ipc_importance_kmsg_link( - ipc_kmsg_t kmsg, - ipc_importance_elem_t elem) + ipc_kmsg_t kmsg, + ipc_importance_elem_t elem) { ipc_importance_elem_t link_elem; assert(IIE_NULL == kmsg->ikm_importance); link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : - elem; + (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : + elem; queue_enter(&link_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance); kmsg->ikm_importance = elem; @@ -203,7 +213,7 @@ ipc_importance_kmsg_link( */ static ipc_importance_elem_t ipc_importance_kmsg_unlink( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { ipc_importance_elem_t elem = kmsg->ikm_importance; @@ -211,8 +221,8 @@ ipc_importance_kmsg_unlink( ipc_importance_elem_t unlink_elem; unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : - elem; + (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : + elem; queue_remove(&unlink_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance); kmsg->ikm_importance = IIE_NULL; @@ -236,18 +246,53 @@ ipc_importance_inherit_link( ipc_importance_inherit_t inherit, ipc_importance_elem_t elem) { - ipc_importance_elem_t link_elem; + ipc_importance_task_t link_task; assert(IIE_NULL == inherit->iii_from_elem); - link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : - elem; + link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? + ((ipc_importance_inherit_t)elem)->iii_to_task : + (ipc_importance_task_t)elem; - queue_enter(&link_elem->iie_inherits, inherit, - ipc_importance_inherit_t, iii_inheritance); + queue_enter(&link_task->iit_inherits, inherit, + ipc_importance_inherit_t, iii_inheritance); inherit->iii_from_elem = elem; } +/* + * Routine: ipc_importance_inherit_find + * Purpose: + * Find an existing inherit that links the from element to the + * to_task at a given nesting depth. As inherits from other + * inherits are actually linked off the original inherit's donation + * receiving task, we have to conduct our search from there if + * the from element is an inherit. + * Returns: + * A pointer (not a reference) to the matching inherit. + * Conditions: + * Importance lock held. + */ +static ipc_importance_inherit_t +ipc_importance_inherit_find( + ipc_importance_elem_t from, + ipc_importance_task_t to_task, + unsigned int depth) +{ + ipc_importance_task_t link_task; + ipc_importance_inherit_t inherit; + + link_task = (IIE_TYPE_INHERIT == IIE_TYPE(from)) ? + ((ipc_importance_inherit_t)from)->iii_to_task : + (ipc_importance_task_t)from; + + queue_iterate(&link_task->iit_inherits, inherit, + ipc_importance_inherit_t, iii_inheritance) { + if (inherit->iii_to_task == to_task && inherit->iii_depth == depth) { + return inherit; + } + } + return III_NULL; +} + /* * Routine: ipc_importance_inherit_unlink * Purpose: @@ -268,14 +313,14 @@ ipc_importance_inherit_unlink( ipc_importance_elem_t elem = inherit->iii_from_elem; if (IIE_NULL != elem) { - ipc_importance_elem_t unlink_elem; + ipc_importance_task_t unlink_task; - unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : - elem; + unlink_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? + ((ipc_importance_inherit_t)elem)->iii_to_task : + (ipc_importance_task_t)elem; - queue_remove(&unlink_elem->iie_inherits, inherit, - ipc_importance_inherit_t, iii_inheritance); + queue_remove(&unlink_task->iit_inherits, inherit, + ipc_importance_inherit_t, iii_inheritance); inherit->iii_from_elem = IIE_NULL; } return elem; @@ -308,49 +353,50 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) { assert(0 < IIE_REFS(elem)); - if (0 < ipc_importance_release_internal(elem)) { +#if IMPORTANCE_DEBUG + ipc_importance_inherit_t temp_inherit; + ipc_importance_task_t link_task; + ipc_kmsg_t temp_kmsg; + uint32_t expected = 0; -#if DEVELOPMENT || DEBUG - ipc_importance_inherit_t temp_inherit; - ipc_importance_task_t link_task; - ipc_kmsg_t temp_kmsg; - uint32_t expected = 0; - - if (0 < elem->iie_made) - expected++; - - link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? - ((ipc_importance_inherit_t)elem)->iii_to_task : - (ipc_importance_task_t)elem; - - queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) - if (temp_kmsg->ikm_importance == elem) - expected++; - queue_iterate(&link_task->iit_inherits, temp_inherit, - ipc_importance_inherit_t, iii_inheritance) - if (temp_inherit->iii_from_elem == elem) - expected++; - - if (IIE_REFS(elem) < expected) - panic("ipc_importance_release_locked (%p)", elem); -#endif + if (0 < elem->iie_made) { + expected++; + } + + link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? + ((ipc_importance_inherit_t)elem)->iii_to_task : + (ipc_importance_task_t)elem; + + queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) + if (temp_kmsg->ikm_importance == elem) { + expected++; + } + queue_iterate(&link_task->iit_inherits, temp_inherit, + ipc_importance_inherit_t, iii_inheritance) + if (temp_inherit->iii_from_elem == elem) { + expected++; + } + if (IIE_REFS(elem) < expected + 1) { + panic("ipc_importance_release_locked (%p)", elem); + } +#endif /* IMPORTANCE_DEBUG */ + + if (0 < ipc_importance_release_internal(elem)) { ipc_importance_unlock(); return; } /* last ref */ - /* can't get to no refs if we contribute to something else's importance */ - assert(queue_empty(&elem->iie_kmsgs)); - assert(queue_empty(&elem->iie_inherits)); switch (IIE_TYPE(elem)) { - /* just a "from" task reference to drop */ case IIE_TYPE_TASK: { ipc_importance_task_t task_elem; task_elem = (ipc_importance_task_t)elem; + + /* the task can't still hold a reference on the task importance */ assert(TASK_NULL == task_elem->iit_task); #if DEVELOPMENT || DEBUG @@ -366,25 +412,63 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) /* dropping an inherit element */ case IIE_TYPE_INHERIT: { - ipc_importance_inherit_t inherit; + ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem; + ipc_importance_task_t to_task = inherit->iii_to_task; ipc_importance_elem_t from_elem; - ipc_importance_task_t to_task; - - inherit = (ipc_importance_inherit_t)elem; - to_task = inherit->iii_to_task; assert(IIT_NULL != to_task); - assert(!inherit->iii_donating); - - /* unlink and release the inherit */ assert(ipc_importance_task_is_any_receiver_type(to_task)); + + /* unlink the inherit from its source element */ from_elem = ipc_importance_inherit_unlink(inherit); assert(IIE_NULL != from_elem); + + /* + * The attribute might have pending external boosts if the attribute + * was given out during exec, drop them from the appropriate destination + * task. + * + * The attribute will not have any pending external boosts if the + * attribute was given out to voucher system since it would have been + * dropped by ipc_importance_release_value, but there is not way to + * detect that, thus if the attribute has a pending external boost, + * drop them from the appropriate destination task. + * + * The inherit attribute from exec and voucher system would not + * get deduped to each other, thus dropping the external boost + * from destination task at two different places will not have + * any unintended side effects. + */ + assert(inherit->iii_externcnt >= inherit->iii_externdrop); + if (inherit->iii_donating) { + uint32_t assertcnt = III_EXTERN(inherit); + + assert(ipc_importance_task_is_any_receiver_type(to_task)); + assert(to_task->iit_externcnt >= inherit->iii_externcnt); + assert(to_task->iit_externdrop >= inherit->iii_externdrop); + to_task->iit_externcnt -= inherit->iii_externcnt; + to_task->iit_externdrop -= inherit->iii_externdrop; + inherit->iii_externcnt = 0; + inherit->iii_externdrop = 0; + inherit->iii_donating = FALSE; + + /* adjust the internal assertions - and propagate as needed */ + if (ipc_importance_task_check_transition(to_task, IIT_UPDATE_DROP, assertcnt)) { + ipc_importance_task_propagate_assertion_locked(to_task, IIT_UPDATE_DROP, TRUE); + } + } else { + inherit->iii_externcnt = 0; + inherit->iii_externdrop = 0; + } + + /* release the reference on the source element */ ipc_importance_release_locked(from_elem); /* unlocked on return */ + /* release the reference on the destination task */ ipc_importance_task_release(to_task); + /* free the inherit */ zfree(ipc_importance_inherit_zone, inherit); break; } @@ -403,8 +487,9 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) void ipc_importance_release(ipc_importance_elem_t elem) { - if (IIE_NULL == elem) + if (IIE_NULL == elem) { return; + } ipc_importance_lock(); ipc_importance_release_locked(elem); @@ -413,8 +498,8 @@ ipc_importance_release(ipc_importance_elem_t elem) /* * Routine: ipc_importance_task_reference - - + * + * * Purpose: * Retain a reference on a task importance attribute value. * Conditions: @@ -424,8 +509,9 @@ ipc_importance_release(ipc_importance_elem_t elem) void ipc_importance_task_reference(ipc_importance_task_t task_elem) { - if (IIT_NULL == task_elem) + if (IIT_NULL == task_elem) { return; + } #if IIE_REF_DEBUG incr_ref_counter(task_elem->iit_elem.iie_task_refs_added); #endif @@ -444,8 +530,9 @@ ipc_importance_task_reference(ipc_importance_task_t task_elem) void ipc_importance_task_release(ipc_importance_task_t task_elem) { - if (IIT_NULL == task_elem) + if (IIT_NULL == task_elem) { return; + } ipc_importance_lock(); #if IIE_REF_DEBUG @@ -510,19 +597,23 @@ ipc_importance_task_check_transition( iit_update_type_t type, uint32_t delta) { - +#if IMPORTANCE_TRACE task_t target_task = task_imp->iit_task; +#endif boolean_t boost = (IIT_UPDATE_HOLD == type); boolean_t before_boosted, after_boosted; - if (!ipc_importance_task_is_any_receiver_type(task_imp)) + ipc_importance_assert_held(); + + if (!ipc_importance_task_is_any_receiver_type(task_imp)) { return FALSE; + } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_START, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); #endif /* snapshot the effective boosting status before making any changes */ @@ -531,46 +622,36 @@ ipc_importance_task_check_transition( /* Adjust the assertcnt appropriately */ if (boost) { task_imp->iit_assertcnt += delta; -#if IMPORTANCE_DEBUG - DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid, - task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt); +#if IMPORTANCE_TRACE + DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid, + task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt); #endif } else { - // assert(delta <= task_imp->iit_assertcnt); - if (delta > task_imp->iit_assertcnt - IIT_EXTERN(task_imp)) { + // assert(delta <= task_imp->iit_assertcnt); + if (task_imp->iit_assertcnt < delta + IIT_EXTERN(task_imp)) { /* TODO: Turn this back into a panic */ - if (target_task != TASK_NULL) { - printf("Over-release of kernel-internal importance assertions for pid %d (%s), " - "dropping %d assertion(s) but task only has %d remaining (%d external).\n", - task_pid(target_task), - (target_task->bsd_info == NULL) ? "" : proc_name_address(target_task->bsd_info), - delta, - task_imp->iit_assertcnt, - IIT_EXTERN(task_imp)); - } task_imp->iit_assertcnt = IIT_EXTERN(task_imp); } else { task_imp->iit_assertcnt -= delta; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE // This convers both legacy and voucher-based importance. DTRACE_BOOST4(drop_boost, task_t, target_task, int, target_pid, int, delta, int, task_imp->iit_assertcnt); #endif } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_END, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); #endif /* did the change result in an effective donor status change? */ after_boosted = (task_imp->iit_assertcnt > 0); if (after_boosted != before_boosted) { - - /* + /* * If the task importance is already on an update queue, we just reversed the need for a - * pending policy update. If the queue is any other than the delayed-drop-queue, pull it + * pending policy update. If the queue is any other than the delayed-drop-queue, pull it * off that queue and release the reference it got going onto the update queue. If it is * the delayed-drop-queue we leave it in place in case it comes back into the drop state * before its time delay is up. @@ -617,8 +698,8 @@ ipc_importance_task_propagate_helper( { ipc_importance_task_t temp_task_imp; - /* - * iterate the downstream kmsgs, adjust their boosts, + /* + * iterate the downstream kmsgs, adjust their boosts, * and capture the next task to adjust for each message */ @@ -649,7 +730,7 @@ ipc_importance_task_propagate_helper( } /* determine the task importance to adjust as result (if any) */ - port = (ipc_port_t) hdr->msgh_remote_port; + port = hdr->msgh_remote_port; assert(IP_VALID(port)); ip_lock(port); temp_task_imp = IIT_NULL; @@ -666,7 +747,7 @@ ipc_importance_task_propagate_helper( /* Adjust the task assertions and determine if an edge was crossed */ if (ipc_importance_task_check_transition(temp_task_imp, type, 1)) { - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); + incr_ref_counter(temp_task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); /* reference donated */ } else { @@ -681,7 +762,7 @@ ipc_importance_task_propagate_helper( ipc_importance_inherit_t temp_inherit; queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) { - uint32_t assertcnt = III_EXTERN(temp_inherit); + uint32_t assertcnt = III_EXTERN(temp_inherit); temp_task_imp = temp_inherit->iii_to_task; assert(IIT_NULL != temp_task_imp); @@ -702,7 +783,6 @@ ipc_importance_task_propagate_helper( temp_inherit->iii_donating = TRUE; temp_task_imp->iit_externcnt += temp_inherit->iii_externcnt; temp_task_imp->iit_externdrop += temp_inherit->iii_externdrop; - } else { /* if no contributing assertions, move on */ if (0 == assertcnt) { @@ -711,7 +791,7 @@ ipc_importance_task_propagate_helper( } /* nothing to do if the inherit is not donating */ - if (!temp_inherit->iii_donating) { + if (!temp_inherit->iii_donating) { continue; } @@ -724,16 +804,15 @@ ipc_importance_task_propagate_helper( assert(temp_task_imp->iit_externdrop >= temp_inherit->iii_externdrop); temp_task_imp->iit_externcnt -= temp_inherit->iii_externcnt; temp_task_imp->iit_externdrop -= temp_inherit->iii_externdrop; - } /* Adjust the task assertions and determine if an edge was crossed */ assert(ipc_importance_task_is_any_receiver_type(temp_task_imp)); if (ipc_importance_task_check_transition(temp_task_imp, type, assertcnt)) { ipc_importance_task_reference(temp_task_imp); - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); + incr_ref_counter(temp_task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); - } + } } } @@ -765,8 +844,8 @@ ipc_importance_task_process_updates( queue_init(&second_chance); /* process any resulting policy updates */ - retry: - while(!queue_empty(queue)) { +retry: + while (!queue_empty(queue)) { task_t target_task; struct task_pend_token pend_token = {}; @@ -804,14 +883,14 @@ ipc_importance_task_process_updates( continue; } - /* - * Can we get the task lock out-of-order? + /* + * Can we get the task lock out-of-order? * If not, stick this back on the second-chance queue. */ if (!task_lock_try(target_task)) { boolean_t should_wait_lock = (queue == &second_chance); task_imp->iit_updateq = &second_chance; - + /* * If we're already processing second-chances on * tasks, keep this task on the front of the queue. @@ -820,16 +899,16 @@ ipc_importance_task_process_updates( * chance of re-acquiring the lock if we come back * to it right away. */ - if (should_wait_lock){ + if (should_wait_lock) { task_reference(target_task); queue_enter_first(&second_chance, task_imp, - ipc_importance_task_t, iit_updates); + ipc_importance_task_t, iit_updates); } else { - queue_enter(&second_chance, task_imp, - ipc_importance_task_t, iit_updates); + queue_enter(&second_chance, task_imp, + ipc_importance_task_t, iit_updates); } ipc_importance_unlock(); - + if (should_wait_lock) { task_lock(target_task); task_unlock(target_task); @@ -853,8 +932,9 @@ ipc_importance_task_process_updates( task_reference(target_task); /* count the transition */ - if (boost) + if (boost) { task_imp->iit_transitions++; + } ipc_importance_unlock(); @@ -864,7 +944,7 @@ ipc_importance_task_process_updates( /* complete the policy update with the task unlocked */ ipc_importance_task_release(task_imp); task_unlock(target_task); - task_policy_update_complete_unlocked(target_task, THREAD_NULL, &pend_token); + task_policy_update_complete_unlocked(target_task, &pend_token); task_deallocate(target_task); ipc_importance_lock(); @@ -892,14 +972,14 @@ ipc_importance_task_process_updates( static void ipc_importance_task_delayed_drop_scan( __unused void *arg1, - __unused void *arg2) + __unused void *arg2) { ipc_importance_lock(); /* process all queued task drops with timestamps up to TARGET(first)+SKEW */ - ipc_importance_task_process_updates(&ipc_importance_delayed_drop_queue, - FALSE, - ipc_importance_delayed_drop_timestamp); + ipc_importance_task_process_updates(&ipc_importance_delayed_drop_queue, + FALSE, + ipc_importance_delayed_drop_timestamp); /* importance lock may have been temporarily dropped */ @@ -918,11 +998,11 @@ ipc_importance_task_delayed_drop_scan( nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway); thread_call_enter_delayed_with_leeway( - ipc_importance_delayed_drop_call, - NULL, - deadline, - leeway, - DENAP_DROP_FLAGS); + ipc_importance_delayed_drop_call, + NULL, + deadline, + leeway, + DENAP_DROP_FLAGS); } else { ipc_importance_delayed_drop_call_requested = FALSE; } @@ -950,8 +1030,8 @@ ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp) * a new reference for the delay drop update queue. */ if (NULL != task_imp->iit_updateq) { - queue_remove(task_imp->iit_updateq, task_imp, - ipc_importance_task_t, iit_updates); + queue_remove(task_imp->iit_updateq, task_imp, + ipc_importance_task_t, iit_updates); } else { ipc_importance_task_reference_internal(task_imp); } @@ -959,8 +1039,8 @@ ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp) task_imp->iit_updateq = &ipc_importance_delayed_drop_queue; task_imp->iit_updatetime = timestamp; - queue_enter(&ipc_importance_delayed_drop_queue, task_imp, - ipc_importance_task_t, iit_updates); + queue_enter(&ipc_importance_delayed_drop_queue, task_imp, + ipc_importance_task_t, iit_updates); /* request the delayed thread-call if not already requested */ if (!ipc_importance_delayed_drop_call_requested) { @@ -975,11 +1055,11 @@ ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp) ipc_importance_delayed_drop_call_requested = TRUE; thread_call_enter_delayed_with_leeway( - ipc_importance_delayed_drop_call, - NULL, - deadline, - leeway, - DENAP_DROP_FLAGS); + ipc_importance_delayed_drop_call, + NULL, + deadline, + leeway, + DENAP_DROP_FLAGS); } } @@ -1007,12 +1087,16 @@ ipc_importance_task_propagate_assertion_locked( queue_init(&updates); queue_init(&propagate); + ipc_importance_assert_held(); + /* * If we're going to update the policy for the provided task, * enqueue it on the propagate queue itself. Otherwise, only * enqueue downstream things. */ if (update_task_imp) { + ipc_importance_task_reference(task_imp); + incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(&propagate, task_imp, ipc_importance_task_t, iit_props); } else { ipc_importance_task_propagate_helper(task_imp, type, &propagate); @@ -1026,6 +1110,8 @@ ipc_importance_task_propagate_assertion_locked( boolean_t need_update; queue_remove_first(&propagate, temp_task_imp, ipc_importance_task_t, iit_props); + /* hold a reference on temp_task_imp */ + assert(IIT_NULL != temp_task_imp); /* only propagate for receivers not already marked as a donor */ @@ -1039,7 +1125,6 @@ ipc_importance_task_propagate_assertion_locked( temp_task_imp->iit_updatepolicy = 0; if (need_update && TASK_NULL != temp_task_imp->iit_task) { if (NULL == temp_task_imp->iit_updateq) { - /* * If a downstream task that needs an update is subjects to AppNap, * drop boosts according to the delay hysteresis. Otherwise, @@ -1055,18 +1140,20 @@ ipc_importance_task_propagate_assertion_locked( ipc_importance_task_reference_internal(temp_task_imp); if (boost) { queue_enter(&updates, temp_task_imp, - ipc_importance_task_t, iit_updates); + ipc_importance_task_t, iit_updates); } else { queue_enter_first(&updates, temp_task_imp, - ipc_importance_task_t, iit_updates); + ipc_importance_task_t, iit_updates); } } } else { /* Must already be on the AppNap hysteresis queue */ assert(ipc_importance_delayed_drop_call != NULL); assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp)); - } + } } + + ipc_importance_task_release_internal(temp_task_imp); } /* apply updates to task (may drop importance lock) */ @@ -1253,11 +1340,11 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im ipc_importance_lock(); target_task = task_imp->iit_task; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); #endif if (IIT_LEGACY_EXTERN(task_imp) == 0) { @@ -1279,22 +1366,22 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im } ipc_importance_unlock(); -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); - // This covers the legacy case where a task takes an extra boost. + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); + // This covers the legacy case where a task takes an extra boost. DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, proc_selfpid(), int, count, int, task_imp->iit_assertcnt); #endif if (KERN_FAILURE == ret && target_task != TASK_NULL) { printf("BUG in process %s[%d]: " - "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. " - "(%d total, %d external, %d legacy-external)\n", - proc_name_address(target_task->bsd_info), task_pid(target_task), - target_assertcnt, target_externcnt, target_legacycnt); + "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. " + "(%d total, %d external, %d legacy-external)\n", + proc_name_address(target_task->bsd_info), task_pid(target_task), + target_assertcnt, target_externcnt, target_legacycnt); } - return(ret); + return ret; } /* @@ -1323,15 +1410,15 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im if (count > 1) { return KERN_INVALID_ARGUMENT; } - + ipc_importance_lock(); target_task = task_imp->iit_task; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); #endif if (count > IIT_LEGACY_EXTERN(task_imp)) { @@ -1342,7 +1429,7 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im target_legacycnt = IIT_LEGACY_EXTERN(task_imp); ret = KERN_FAILURE; } else { - /* + /* * decrement legacy external count from the top level and reflect * into internal for this and all subsequent updates. */ @@ -1364,7 +1451,7 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im task_imp->iit_legacy_externcnt = 0; task_imp->iit_legacy_externdrop = 0; } - + /* reflect the drop to the internal assertion count (and effect any importance change) */ if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) { ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE); @@ -1372,9 +1459,9 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im ret = KERN_SUCCESS; } -#if IMPORTANCE_DEBUG - KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); +#if IMPORTANCE_TRACE + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); #endif ipc_importance_unlock(); @@ -1382,15 +1469,15 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im /* delayed printf for user-supplied data failures */ if (KERN_FAILURE == ret && TASK_NULL != target_task) { printf("BUG in process %s[%d]: over-released legacy external boost assertions (%d total, %d external, %d legacy-external)\n", - proc_name_address(target_task->bsd_info), task_pid(target_task), - target_assertcnt, target_externcnt, target_legacycnt); + proc_name_address(target_task->bsd_info), task_pid(target_task), + target_assertcnt, target_externcnt, target_legacycnt); } - return(ret); + return ret; } - +#if LEGACY_IMPORTANCE_DELIVERY /* Transfer an assertion to legacy userspace responsibility */ static kern_return_t ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, uint32_t count, __unused int sender_pid) @@ -1405,11 +1492,11 @@ ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, return KERN_FAILURE; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_START, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); #endif ipc_importance_lock(); @@ -1419,15 +1506,16 @@ ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, task_imp->iit_externcnt += count; ipc_importance_unlock(); -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_END, - proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); - // This is the legacy boosting path + proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); + // This is the legacy boosting path DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, sender_pid, int, count, int, IIT_LEGACY_EXTERN(task_imp)); -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ - return(KERN_SUCCESS); + return KERN_SUCCESS; } +#endif /* LEGACY_IMPORTANCE_DELIVERY */ /* * Routine: ipc_importance_task_update_live_donor @@ -1445,7 +1533,7 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) boolean_t before_donor; boolean_t after_donor; task_t target_task; - + assert(task_imp != NULL); /* @@ -1467,15 +1555,15 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) before_donor = ipc_importance_task_is_marked_donor(task_imp); /* snapshot task live donor status - may change, but another call will accompany the change */ - task_live_donor = target_task->effective_policy.t_live_donor; + task_live_donor = target_task->effective_policy.tep_live_donor; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_START, - target_pid, task_imp->iit_donor, task_live_donor, before_donor, 0); -#endif + (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_START, + target_pid, task_imp->iit_donor, task_live_donor, before_donor, 0); +#endif /* update the task importance live donor status based on the task's value */ task_imp->iit_donor = task_live_donor; @@ -1496,10 +1584,10 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) ipc_importance_task_propagate_assertion_locked(task_imp, type, FALSE); } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END, - target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0); + (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END, + target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0); #endif ipc_importance_unlock(); @@ -1527,14 +1615,15 @@ ipc_importance_task_mark_donor(ipc_importance_task_t task_imp, boolean_t donatin task_imp->iit_donor = (donating ? 1 : 0); - if (task_imp->iit_donor > 0 && old_donor == 0) + if (task_imp->iit_donor > 0 && old_donor == 0) { task_imp->iit_transitions++; + } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_INIT_DONOR_STATE)) | DBG_FUNC_NONE, - task_pid(task_imp->iit_task), donating, - old_donor, task_imp->iit_donor, 0); - + (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_INIT_DONOR_STATE)) | DBG_FUNC_NONE, + task_pid(task_imp->iit_task), donating, + old_donor, task_imp->iit_donor, 0); + ipc_importance_unlock(); } @@ -1551,9 +1640,9 @@ boolean_t ipc_importance_task_is_marked_donor(ipc_importance_task_t task_imp) { if (IIT_NULL == task_imp) { - return FALSE; + return FALSE; } - return (0 != task_imp->iit_donor); + return 0 != task_imp->iit_donor; } /* @@ -1576,7 +1665,7 @@ ipc_importance_task_mark_live_donor(ipc_importance_task_t task_imp, boolean_t li } /* - * Routine: ipc_importance_task_marked_live_donor + * Routine: ipc_importance_task_is_marked_live_donor * Purpose: * Query the live donor and donor flags for the given task importance. * Conditions: @@ -1588,9 +1677,9 @@ boolean_t ipc_importance_task_is_marked_live_donor(ipc_importance_task_t task_imp) { if (IIT_NULL == task_imp) { - return FALSE; + return FALSE; } - return (0 != task_imp->iit_live_donor); + return 0 != task_imp->iit_live_donor; } /* @@ -1606,11 +1695,11 @@ boolean_t ipc_importance_task_is_donor(ipc_importance_task_t task_imp) { if (IIT_NULL == task_imp) { - return FALSE; + return FALSE; } - return (ipc_importance_task_is_marked_donor(task_imp) || - (ipc_importance_task_is_marked_receiver(task_imp) && - task_imp->iit_assertcnt > 0)); + return ipc_importance_task_is_marked_donor(task_imp) || + (ipc_importance_task_is_marked_receiver(task_imp) && + task_imp->iit_assertcnt > 0); } /* @@ -1619,17 +1708,17 @@ ipc_importance_task_is_donor(ipc_importance_task_t task_imp) * Query if a given task can ever donate importance. * Conditions: * May be called without taking the importance lock. - * Condition is permanent for a give task. + * Condition is permanent for a give task. */ boolean_t ipc_importance_task_is_never_donor(ipc_importance_task_t task_imp) { if (IIT_NULL == task_imp) { - return FALSE; + return FALSE; } - return (!ipc_importance_task_is_marked_donor(task_imp) && - !ipc_importance_task_is_marked_live_donor(task_imp) && - !ipc_importance_task_is_marked_receiver(task_imp)); + return !ipc_importance_task_is_marked_donor(task_imp) && + !ipc_importance_task_is_marked_live_donor(task_imp) && + !ipc_importance_task_is_marked_receiver(task_imp); } /* @@ -1644,7 +1733,7 @@ ipc_importance_task_is_never_donor(ipc_importance_task_t task_imp) void ipc_importance_task_mark_receiver(ipc_importance_task_t task_imp, boolean_t receiving) { - assert(task_imp != NULL); + assert(task_imp != NULL); ipc_importance_lock(); if (receiving) { @@ -1675,7 +1764,7 @@ ipc_importance_task_mark_receiver(ipc_importance_task_t task_imp, boolean_t rece boolean_t ipc_importance_task_is_marked_receiver(ipc_importance_task_t task_imp) { - return (IIT_NULL != task_imp && 0 != task_imp->iit_receiver); + return IIT_NULL != task_imp && 0 != task_imp->iit_receiver; } @@ -1691,7 +1780,7 @@ ipc_importance_task_is_marked_receiver(ipc_importance_task_t task_imp) void ipc_importance_task_mark_denap_receiver(ipc_importance_task_t task_imp, boolean_t denap) { - assert(task_imp != NULL); + assert(task_imp != NULL); ipc_importance_lock(); if (denap) { @@ -1721,7 +1810,7 @@ ipc_importance_task_mark_denap_receiver(ipc_importance_task_t task_imp, boolean_ boolean_t ipc_importance_task_is_marked_denap_receiver(ipc_importance_task_t task_imp) { - return (IIT_NULL != task_imp && 0 != task_imp->iit_denap); + return IIT_NULL != task_imp && 0 != task_imp->iit_denap; } /* @@ -1736,7 +1825,7 @@ ipc_importance_task_is_marked_denap_receiver(ipc_importance_task_t task_imp) boolean_t ipc_importance_task_is_denap_receiver(ipc_importance_task_t task_imp) { - return (ipc_importance_task_is_marked_denap_receiver(task_imp)); + return ipc_importance_task_is_marked_denap_receiver(task_imp); } /* @@ -1752,8 +1841,8 @@ ipc_importance_task_is_denap_receiver(ipc_importance_task_t task_imp) boolean_t ipc_importance_task_is_any_receiver_type(ipc_importance_task_t task_imp) { - return (ipc_importance_task_is_marked_receiver(task_imp) || - ipc_importance_task_is_marked_denap_receiver(task_imp)); + return ipc_importance_task_is_marked_receiver(task_imp) || + ipc_importance_task_is_marked_denap_receiver(task_imp); } #if 0 /* currently unused */ @@ -1799,11 +1888,12 @@ ipc_importance_inherit_release_locked(ipc_importance_inherit_t inherit) void ipc_importance_inherit_release(ipc_importance_inherit_t inherit) { - if (III_NULL != inherit) + if (III_NULL != inherit) { ipc_importance_release(&inherit->iii_elem); + } } #endif /* 0 currently unused */ - + /* * Routine: ipc_importance_for_task * Purpose: @@ -1825,10 +1915,11 @@ ipc_importance_for_task(task_t task, boolean_t made) assert(TASK_NULL != task); - retry: +retry: /* No use returning anything for inactive task */ - if (!task->active) + if (!task->active) { return IIT_NULL; + } ipc_importance_lock(); task_elem = task->task_imp_base; @@ -1848,14 +1939,16 @@ ipc_importance_for_task(task_t task, boolean_t made) } ipc_importance_unlock(); - if (!first_pass) + if (!first_pass) { return IIT_NULL; + } first_pass = FALSE; /* Need to make one - may race with others (be prepared to drop) */ task_elem = (ipc_importance_task_t)zalloc(ipc_importance_task_zone); - if (IIT_NULL == task_elem) + if (IIT_NULL == task_elem) { goto retry; + } task_elem->iit_bits = IIE_TYPE_TASK | 2; /* one for task, one for return/made */ task_elem->iit_made = (made) ? 1 : 0; @@ -1908,8 +2001,9 @@ ipc_importance_for_task(task_t task, boolean_t made) } #if DEVELOPMENT || DEBUG -void task_importance_update_owner_info(task_t task) { - +void +task_importance_update_owner_info(task_t task) +{ if (task != TASK_NULL && task->task_imp_base != IIT_NULL) { ipc_importance_task_t task_elem = task->task_imp_base; @@ -1952,11 +2046,10 @@ ipc_importance_reset_locked(ipc_importance_task_t task_imp, boolean_t donor) task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop; /* assert(IIT_LEGACY_EXTERN(task_imp) <= task_imp->iit_assertcnt); */ - if (IIT_LEGACY_EXTERN(task_imp) < task_imp->iit_assertcnt) { + if (IIT_EXTERN(task_imp) < task_imp->iit_assertcnt) { task_imp->iit_assertcnt -= IIT_LEGACY_EXTERN(task_imp); } else { - assert(IIT_LEGACY_EXTERN(task_imp) == task_imp->iit_assertcnt); - task_imp->iit_assertcnt = 0; + task_imp->iit_assertcnt = IIT_EXTERN(task_imp); } task_imp->iit_legacy_externcnt = 0; task_imp->iit_legacy_externdrop = 0; @@ -1964,8 +2057,8 @@ ipc_importance_reset_locked(ipc_importance_task_t task_imp, boolean_t donor) #if DEVELOPMENT || DEBUG if (task_imp->iit_assertcnt > 0 && task_imp->iit_live_donor) { - printf("Live donor task %s[%d] still has %d importance assertions after reset\n", - task_imp->iit_procname, task_imp->iit_bsd_pid, task_imp->iit_assertcnt); + printf("Live donor task %s[%d] still has %d importance assertions after reset\n", + task_imp->iit_procname, task_imp->iit_bsd_pid, task_imp->iit_assertcnt); } #endif @@ -2034,7 +2127,7 @@ ipc_importance_disconnect_task(task_t task) task_imp->iit_task = TASK_NULL; task->task_imp_base = IIT_NULL; task_unlock(task); - + /* reset the effects the current task hold on the importance */ ipc_importance_reset_locked(task_imp, TRUE); @@ -2045,6 +2138,57 @@ ipc_importance_disconnect_task(task_t task) task_deallocate(task); } +/* + * Routine: ipc_importance_exec_switch_task + * Purpose: + * Switch importance task base from old task to new task in exec. + * + * Create an ipc importance linkage from old task to new task, + * once the linkage is created, switch the importance task base + * from old task to new task. After the switch, the linkage will + * represent importance linkage from new task to old task with + * watch port importance inheritance linked to new task. + * Conditions: + * Nothing locked. + * Returns a reference on importance inherit. + */ +ipc_importance_inherit_t +ipc_importance_exec_switch_task( + task_t old_task, + task_t new_task) +{ + ipc_importance_inherit_t inherit = III_NULL; + ipc_importance_task_t old_task_imp = IIT_NULL; + ipc_importance_task_t new_task_imp = IIT_NULL; + + task_importance_reset(old_task); + + /* Create an importance linkage from old_task to new_task */ + inherit = ipc_importance_inherit_from_task(old_task, new_task); + + /* Switch task importance base from old task to new task */ + ipc_importance_lock(); + + old_task_imp = old_task->task_imp_base; + new_task_imp = new_task->task_imp_base; + + old_task_imp->iit_task = new_task; + new_task_imp->iit_task = old_task; + + old_task->task_imp_base = new_task_imp; + new_task->task_imp_base = old_task_imp; + +#if DEVELOPMENT || DEBUG + /* + * Update the pid an proc name for importance base if any + */ + task_importance_update_owner_info(new_task); +#endif + ipc_importance_unlock(); + + return inherit; +} + /* * Routine: ipc_importance_check_circularity * Purpose: @@ -2069,22 +2213,28 @@ ipc_importance_disconnect_task(task_t task) boolean_t ipc_importance_check_circularity( - ipc_port_t port, - ipc_port_t dest) + ipc_port_t port, + ipc_port_t dest) { ipc_importance_task_t imp_task = IIT_NULL; ipc_importance_task_t release_imp_task = IIT_NULL; boolean_t imp_lock_held = FALSE; int assertcnt = 0; ipc_port_t base; + struct turnstile *send_turnstile = TURNSTILE_NULL; + struct task_watchport_elem *watchport_elem = NULL; assert(port != IP_NULL); assert(dest != IP_NULL); - if (port == dest) + if (port == dest) { return TRUE; + } base = dest; + /* Check if destination needs a turnstile */ + ipc_port_send_turnstile_prepare(dest); + /* port is in limbo, so donation status is safe to latch */ if (port->ip_impdonation != 0) { imp_lock_held = TRUE; @@ -2097,7 +2247,7 @@ ipc_importance_check_circularity( */ ip_lock(port); - /* + /* * Even if port is just carrying assertions for others, * we need the importance lock. */ @@ -2113,8 +2263,9 @@ ipc_importance_check_circularity( if (ip_lock_try(dest)) { if (!ip_active(dest) || (dest->ip_receiver_name != MACH_PORT_NULL) || - (dest->ip_destination == IP_NULL)) + (dest->ip_destination == IP_NULL)) { goto not_circular; + } /* dest is in transit; further checking necessary */ @@ -2122,7 +2273,7 @@ ipc_importance_check_circularity( } ip_unlock(port); - /* + /* * We're about to pay the cost to serialize, * just go ahead and grab importance lock. */ @@ -2143,8 +2294,9 @@ ipc_importance_check_circularity( if (!ip_active(base) || (base->ip_receiver_name != MACH_PORT_NULL) || - (base->ip_destination == IP_NULL)) + (base->ip_destination == IP_NULL)) { break; + } base = base->ip_destination; } @@ -2158,26 +2310,29 @@ ipc_importance_check_circularity( /* port (== base) is in limbo */ - assert(ip_active(port)); + require_ip_active(port); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination == IP_NULL); - while (dest != IP_NULL) { + base = dest; + while (base != IP_NULL) { ipc_port_t next; - /* dest is in transit or in limbo */ + /* base is in transit or in limbo */ - assert(ip_active(dest)); - assert(dest->ip_receiver_name == MACH_PORT_NULL); + require_ip_active(base); + assert(base->ip_receiver_name == MACH_PORT_NULL); - next = dest->ip_destination; - ip_unlock(dest); - dest = next; + next = base->ip_destination; + ip_unlock(base); + base = next; } - if (imp_lock_held) + if (imp_lock_held) { ipc_importance_unlock(); + } + ipc_port_send_turnstile_complete(dest); return TRUE; } @@ -2190,14 +2345,22 @@ ipc_importance_check_circularity( ip_lock(port); ipc_port_multiple_unlock(); - not_circular: - +not_circular: /* port is in limbo */ + imq_lock(&port->ip_messages); - assert(ip_active(port)); + require_ip_active(port); assert(port->ip_receiver_name == MACH_PORT_NULL); assert(port->ip_destination == IP_NULL); + /* Port is being enqueued in a kmsg, remove the watchport boost in order to push on destination port */ + watchport_elem = ipc_port_clear_watchport_elem_internal(port); + + /* Check if the port is being enqueued as a part of sync bootstrap checkin */ + if (dest->ip_specialreply && dest->ip_sync_bootstrap_checkin) { + port->ip_sync_bootstrap_checkin = 1; + } + ip_reference(dest); port->ip_destination = dest; @@ -2218,34 +2381,52 @@ ipc_importance_check_circularity( /* take the port out of limbo w.r.t. assertions */ port->ip_tempowner = 0; + /* + * Setup linkage for source port if it has a send turnstile i.e. it has + * a thread waiting in send or has a port enqueued in it or has sync ipc + * push from a special reply port. + */ + if (port_send_turnstile(port)) { + send_turnstile = turnstile_prepare((uintptr_t)port, + port_send_turnstile_address(port), + TURNSTILE_NULL, TURNSTILE_SYNC_IPC); + + turnstile_update_inheritor(send_turnstile, port_send_turnstile(dest), + (TURNSTILE_INHERITOR_TURNSTILE | TURNSTILE_IMMEDIATE_UPDATE)); + + /* update complete and turnstile complete called after dropping all locks */ + } + imq_unlock(&port->ip_messages); + /* now unlock chain */ ip_unlock(port); for (;;) { - + ipc_port_t next; /* every port along chain track assertions behind it */ ipc_port_impcount_delta(dest, assertcnt, base); - if (dest == base) + if (dest == base) { break; + } /* port is in transit */ - assert(ip_active(dest)); + require_ip_active(dest); assert(dest->ip_receiver_name == MACH_PORT_NULL); assert(dest->ip_destination != IP_NULL); assert(dest->ip_tempowner == 0); - port = dest->ip_destination; + next = dest->ip_destination; ip_unlock(dest); - dest = port; + dest = next; } /* base is not in transit */ assert(!ip_active(base) || - (base->ip_receiver_name != MACH_PORT_NULL) || - (base->ip_destination == IP_NULL)); + (base->ip_receiver_name != MACH_PORT_NULL) || + (base->ip_destination == IP_NULL)); /* * Find the task to boost (if any). @@ -2262,14 +2443,14 @@ ipc_importance_check_circularity( assert(ipc_importance_task_is_any_receiver_type(imp_task)); } /* otherwise don't boost current task */ - } else if (base->ip_receiver_name != MACH_PORT_NULL) { ipc_space_t space = base->ip_receiver; /* only spaces with boost-accepting tasks */ if (space->is_task != TASK_NULL && - ipc_importance_task_is_any_receiver_type(space->is_task->task_imp_base)) + ipc_importance_task_is_any_receiver_type(space->is_task->task_imp_base)) { imp_task = space->is_task->task_imp_base; + } } /* take reference before unlocking base */ @@ -2280,6 +2461,18 @@ ipc_importance_check_circularity( ip_unlock(base); + /* All locks dropped, call turnstile_update_inheritor_complete for source port's turnstile */ + if (send_turnstile) { + turnstile_update_inheritor_complete(send_turnstile, TURNSTILE_INTERLOCK_NOT_HELD); + + /* Take the mq lock to call turnstile complete */ + imq_lock(&port->ip_messages); + turnstile_complete((uintptr_t)port, port_send_turnstile_address(port), NULL, TURNSTILE_SYNC_IPC); + send_turnstile = TURNSTILE_NULL; + imq_unlock(&port->ip_messages); + turnstile_cleanup(); + } + /* * Transfer assertions now that the ports are unlocked. * Avoid extra overhead if transferring to/from the same task. @@ -2293,24 +2486,33 @@ ipc_importance_check_circularity( if (imp_task != IIT_NULL) { assert(imp_lock_held); - if (transfer_assertions) + if (transfer_assertions) { ipc_importance_task_hold_internal_assertion_locked(imp_task, assertcnt); + } } if (release_imp_task != IIT_NULL) { assert(imp_lock_held); - if (transfer_assertions) + if (transfer_assertions) { ipc_importance_task_drop_internal_assertion_locked(release_imp_task, assertcnt); + } } - if (imp_lock_held) + if (imp_lock_held) { ipc_importance_unlock(); + } - if (imp_task != IIT_NULL) + if (imp_task != IIT_NULL) { ipc_importance_task_release(imp_task); + } - if (release_imp_task != IIT_NULL) + if (release_imp_task != IIT_NULL) { ipc_importance_task_release(release_imp_task); + } + + if (watchport_elem) { + task_watchport_elem_deallocate(watchport_elem); + } return FALSE; } @@ -2327,10 +2529,10 @@ ipc_importance_check_circularity( */ boolean_t ipc_importance_send( - ipc_kmsg_t kmsg, - mach_msg_option_t option) + ipc_kmsg_t kmsg, + mach_msg_option_t option) { - ipc_port_t port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; + ipc_port_t port = kmsg->ikm_header->msgh_remote_port; boolean_t port_lock_dropped = FALSE; ipc_importance_elem_t elem; task_t task; @@ -2349,7 +2551,12 @@ ipc_importance_send( /* If forced sending a static boost, go update the port */ if ((option & MACH_SEND_IMPORTANCE) != 0) { - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; + /* acquire the importance lock while trying to hang on to port lock */ + if (!ipc_importance_lock_try()) { + port_lock_dropped = TRUE; + ip_unlock(port); + ipc_importance_lock(); + } goto portupdate; } @@ -2369,14 +2576,14 @@ ipc_importance_send( mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; mach_voucher_attr_value_handle_array_size_t val_count; ipc_voucher_t voucher; - + assert(ip_kotype(kmsg->ikm_voucher) == IKOT_VOUCHER); - voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject; + voucher = (ipc_voucher_t)ip_get_kobject(kmsg->ikm_voucher); /* check to see if the voucher has an importance attribute */ val_count = MACH_VOUCHER_ATTR_VALUE_MAX_NESTED; kr = mach_voucher_attr_control_get_values(ipc_importance_control, voucher, - vals, &val_count); + vals, &val_count); assert(KERN_SUCCESS == kr); /* @@ -2426,12 +2633,14 @@ ipc_importance_send( ipc_importance_unlock(); /* re-acquire port lock, if needed */ - if (TRUE == port_lock_dropped) + if (TRUE == port_lock_dropped) { ip_lock(port); + } return port_lock_dropped; } +portupdate: /* Mark the fact that we are (currently) donating through this message */ kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; @@ -2445,18 +2654,18 @@ ipc_importance_send( ip_lock(port); } - portupdate: - -#if IMPORTANCE_DEBUG + ipc_importance_assert_held(); + +#if IMPORTANCE_TRACE if (kdebug_enable) { mach_msg_max_trailer_t *dbgtrailer = (mach_msg_max_trailer_t *) - ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); + ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); unsigned int sender_pid = dbgtrailer->msgh_audit.val[5]; mach_msg_id_t imp_msgh_id = kmsg->ikm_header->msgh_id; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_START, - task_pid(task), sender_pid, imp_msgh_id, 0, 0); + task_pid(task), sender_pid, imp_msgh_id, 0, 0); } -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ mach_port_delta_t delta = 1; boolean_t need_port_lock; @@ -2464,6 +2673,7 @@ ipc_importance_send( /* adjust port boost count (with importance and port locked) */ need_port_lock = ipc_port_importance_delta_internal(port, IPID_OPTION_NORMAL, &delta, &task_imp); + /* hold a reference on task_imp */ /* if we need to adjust a task importance as a result, apply that here */ if (IIT_NULL != task_imp && delta != 0) { @@ -2471,7 +2681,6 @@ ipc_importance_send( /* if this results in a change of state, propagate the transistion */ if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_HOLD, delta)) { - /* can't hold the port lock during task transition(s) */ if (!need_port_lock) { need_port_lock = TRUE; @@ -2481,7 +2690,12 @@ ipc_importance_send( } } - ipc_importance_unlock(); + if (task_imp) { + ipc_importance_task_release_locked(task_imp); + /* importance unlocked */ + } else { + ipc_importance_unlock(); + } if (need_port_lock) { port_lock_dropped = TRUE; @@ -2490,9 +2704,9 @@ ipc_importance_send( return port_lock_dropped; } - + /* - * Routine: ipc_importance_inherit_from + * Routine: ipc_importance_inherit_from_kmsg * Purpose: * Create a "made" reference for an importance attribute representing * an inheritance between the sender of a message (if linked) and the @@ -2506,17 +2720,16 @@ ipc_importance_send( * Nothing locked on entry. May block. */ static ipc_importance_inherit_t -ipc_importance_inherit_from(ipc_kmsg_t kmsg) +ipc_importance_inherit_from_kmsg(ipc_kmsg_t kmsg) { - ipc_importance_task_t task_imp = IIT_NULL; - ipc_importance_elem_t from_elem = kmsg->ikm_importance; - ipc_importance_elem_t elem; - task_t task_self = current_task(); + ipc_importance_task_t task_imp = IIT_NULL; + ipc_importance_elem_t from_elem = kmsg->ikm_importance; + ipc_importance_elem_t elem; + task_t task_self = current_task(); ipc_port_t port = kmsg->ikm_header->msgh_remote_port; ipc_importance_inherit_t inherit = III_NULL; ipc_importance_inherit_t alloc = III_NULL; - ipc_importance_inherit_t temp_inherit; boolean_t cleared_self_donation = FALSE; boolean_t donating; uint32_t depth = 1; @@ -2527,7 +2740,7 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) return III_NULL; } - /* + /* * No need to set up an inherit linkage if the dest isn't a receiver * of one type or the other. */ @@ -2560,7 +2773,6 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) cleared_self_donation = TRUE; } inherit = from_inherit; - } else if (III_DEPTH_MAX == III_DEPTH(from_inherit)) { ipc_importance_task_t to_task; ipc_importance_elem_t unlinked_from; @@ -2568,7 +2780,7 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) /* * Chain too long. Switch to looking * directly at the from_inherit's to-task - * as our source of importance. + * as our source of importance. */ to_task = from_inherit->iii_to_task; ipc_importance_task_reference(to_task); @@ -2582,14 +2794,13 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) ipc_importance_inherit_release_locked(from_inherit); /* importance unlocked */ ipc_importance_lock(); - } else { /* inheriting from an inherit */ depth = from_inherit->iii_depth + 1; } - } + } - /* + /* * Don't allow a task to inherit from itself (would keep it permanently * boosted even if all other donors to the task went away). */ @@ -2598,7 +2809,7 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) goto out_locked; } - /* + /* * But if the message isn't associated with any linked source, it is * intended to be permanently boosting (static boost from kernel). * In that case DO let the process permanently boost itself. @@ -2608,20 +2819,13 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) ipc_importance_task_reference_internal(task_imp); from_elem = (ipc_importance_elem_t)task_imp; } - - /* + + /* * Now that we have the from_elem figured out, * check to see if we already have an inherit for this pairing */ while (III_NULL == inherit) { - queue_iterate(&from_elem->iie_inherits, temp_inherit, - ipc_importance_inherit_t, iii_inheritance) { - if (temp_inherit->iii_to_task == task_imp && - temp_inherit->iii_depth == depth) { - inherit = temp_inherit; - break; - } - } + inherit = ipc_importance_inherit_find(from_elem, task_imp, depth); /* Do we have to allocate a new inherit */ if (III_NULL == inherit) { @@ -2632,7 +2836,7 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) /* allocate space */ ipc_importance_unlock(); alloc = (ipc_importance_inherit_t) - zalloc(ipc_importance_inherit_zone); + zalloc(ipc_importance_inherit_zone); ipc_importance_lock(); } } @@ -2666,9 +2870,6 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) /* add in a external reference for this use of the inherit */ inherit->iii_externcnt++; - if (donating) { - task_imp->iit_externcnt++; - } } else { /* initialize the previously allocated space */ inherit = alloc; @@ -2680,12 +2881,9 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) inherit->iii_to_task = task_imp; inherit->iii_from_elem = IIE_NULL; queue_init(&inherit->iii_kmsgs); - queue_init(&inherit->iii_inherits); - /* If donating, reflect that in the task externcnt */ if (donating) { inherit->iii_donating = TRUE; - task_imp->iit_externcnt++; } else { inherit->iii_donating = FALSE; } @@ -2703,8 +2901,8 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) #endif } - out_locked: - /* +out_locked: + /* * for those paths that came straight here: snapshot the donating status * (this should match previous snapshot for other paths). */ @@ -2714,6 +2912,14 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) elem = ipc_importance_kmsg_unlink(kmsg); assert(elem == from_elem); + /* If found inherit and donating, reflect that in the task externcnt */ + if (III_NULL != inherit && donating) { + task_imp->iit_externcnt++; + /* The owner of receive right might have changed, take the internal assertion */ + ipc_importance_task_hold_internal_assertion_locked(task_imp, 1); + /* may have dropped and retaken importance lock */ + } + /* If we didn't create a new inherit, we have some resources to release */ if (III_NULL == inherit || inherit != alloc) { if (IIE_NULL != from_elem) { @@ -2735,8 +2941,9 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) ipc_importance_task_release(task_imp); } - if (III_NULL != alloc) + if (III_NULL != alloc) { zfree(ipc_importance_inherit_zone, alloc); + } } else { /* from_elem and task_imp references transferred to new inherit */ ipc_importance_unlock(); @@ -2747,22 +2954,10 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) * This is OK to do without the importance lock as we atomically * unlinked the kmsg and snapshot the donating state while holding * the importance lock - */ - if (donating) { - ip_lock(port); - if (III_NULL != inherit) { - /* task assertions transferred to inherit, just adjust port count */ - ipc_port_impcount_delta(port, -1, IP_NULL); - ip_unlock(port); - } else { - /* drop importance from port and destination task */ - if (ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { - ip_unlock(port); - } - } - } else if (cleared_self_donation) { + */ + if (donating || cleared_self_donation) { ip_lock(port); - /* drop cleared donation from port and destination task */ + /* drop importance from port and destination task */ if (ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { ip_unlock(port); } @@ -2779,6 +2974,181 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) return inherit; } +/* + * Routine: ipc_importance_inherit_from_task + * Purpose: + * Create a reference for an importance attribute representing + * an inheritance between the to_task and from_task. The iii + * created will be marked as III_FLAGS_FOR_OTHERS. + * + * It will not dedup any iii which are not marked as III_FLAGS_FOR_OTHERS. + * + * If the task is inactive, there isn't any need to return a new reference. + * Conditions: + * Nothing locked on entry. May block. + * It should not be called from voucher subsystem. + */ +static ipc_importance_inherit_t +ipc_importance_inherit_from_task( + task_t from_task, + task_t to_task) +{ + ipc_importance_task_t to_task_imp = IIT_NULL; + ipc_importance_task_t from_task_imp = IIT_NULL; + ipc_importance_elem_t from_elem = IIE_NULL; + + ipc_importance_inherit_t inherit = III_NULL; + ipc_importance_inherit_t alloc = III_NULL; + boolean_t donating; + uint32_t depth = 1; + + to_task_imp = ipc_importance_for_task(to_task, FALSE); + from_task_imp = ipc_importance_for_task(from_task, FALSE); + from_elem = (ipc_importance_elem_t)from_task_imp; + + ipc_importance_lock(); + + if (IIT_NULL == to_task_imp || IIT_NULL == from_task_imp) { + goto out_locked; + } + + /* + * No need to set up an inherit linkage if the to_task or from_task + * isn't a receiver of one type or the other. + */ + if (!ipc_importance_task_is_any_receiver_type(to_task_imp) || + !ipc_importance_task_is_any_receiver_type(from_task_imp)) { + goto out_locked; + } + + /* Do not allow to create a linkage to self */ + if (to_task_imp == from_task_imp) { + goto out_locked; + } + + incr_ref_counter(to_task_imp->iit_elem.iie_task_refs_added_inherit_from); + incr_ref_counter(from_elem->iie_kmsg_refs_added); + + /* + * Now that we have the from_elem figured out, + * check to see if we already have an inherit for this pairing + */ + while (III_NULL == inherit) { + inherit = ipc_importance_inherit_find(from_elem, to_task_imp, depth); + + /* Do we have to allocate a new inherit */ + if (III_NULL == inherit) { + if (III_NULL != alloc) { + break; + } + + /* allocate space */ + ipc_importance_unlock(); + alloc = (ipc_importance_inherit_t) + zalloc(ipc_importance_inherit_zone); + ipc_importance_lock(); + } + } + + /* snapshot the donating status while we have importance locked */ + donating = ipc_importance_task_is_donor(from_task_imp); + + if (III_NULL != inherit) { + /* We found one, piggyback on that */ + assert(0 < III_REFS(inherit)); + assert(0 < IIE_REFS(inherit->iii_from_elem)); + + /* Take a reference for inherit */ + assert(III_REFS_MAX > III_REFS(inherit)); + ipc_importance_inherit_reference_internal(inherit); + + /* Reflect the inherit's change of status into the task boosts */ + if (0 == III_EXTERN(inherit)) { + assert(!inherit->iii_donating); + inherit->iii_donating = donating; + if (donating) { + to_task_imp->iit_externcnt += inherit->iii_externcnt; + to_task_imp->iit_externdrop += inherit->iii_externdrop; + } + } else { + assert(donating == inherit->iii_donating); + } + + /* add in a external reference for this use of the inherit */ + inherit->iii_externcnt++; + } else { + /* initialize the previously allocated space */ + inherit = alloc; + inherit->iii_bits = IIE_TYPE_INHERIT | 1; + inherit->iii_made = 0; + inherit->iii_externcnt = 1; + inherit->iii_externdrop = 0; + inherit->iii_depth = depth; + inherit->iii_to_task = to_task_imp; + inherit->iii_from_elem = IIE_NULL; + queue_init(&inherit->iii_kmsgs); + + if (donating) { + inherit->iii_donating = TRUE; + } else { + inherit->iii_donating = FALSE; + } + + /* + * Chain our new inherit on the element it inherits from. + * The new inherit takes our reference on from_elem. + */ + ipc_importance_inherit_link(inherit, from_elem); + +#if IIE_REF_DEBUG + ipc_importance_counter_init(&inherit->iii_elem); + from_elem->iie_kmsg_refs_inherited++; + task_imp->iit_elem.iie_task_refs_inherited++; +#endif + } + +out_locked: + + /* If found inherit and donating, reflect that in the task externcnt */ + if (III_NULL != inherit && donating) { + to_task_imp->iit_externcnt++; + /* take the internal assertion */ + ipc_importance_task_hold_internal_assertion_locked(to_task_imp, 1); + /* may have dropped and retaken importance lock */ + } + + /* If we didn't create a new inherit, we have some resources to release */ + if (III_NULL == inherit || inherit != alloc) { + if (IIE_NULL != from_elem) { + if (III_NULL != inherit) { + incr_ref_counter(from_elem->iie_kmsg_refs_coalesced); + } else { + incr_ref_counter(from_elem->iie_kmsg_refs_dropped); + } + ipc_importance_release_locked(from_elem); + /* importance unlocked */ + } else { + ipc_importance_unlock(); + } + + if (IIT_NULL != to_task_imp) { + if (III_NULL != inherit) { + incr_ref_counter(to_task_imp->iit_elem.iie_task_refs_coalesced); + } + ipc_importance_task_release(to_task_imp); + } + + if (III_NULL != alloc) { + zfree(ipc_importance_inherit_zone, alloc); + } + } else { + /* from_elem and to_task_imp references transferred to new inherit */ + ipc_importance_unlock(); + } + + return inherit; +} + /* * Routine: ipc_importance_receive * Purpose: @@ -2795,29 +3165,32 @@ ipc_importance_inherit_from(ipc_kmsg_t kmsg) */ void ipc_importance_receive( - ipc_kmsg_t kmsg, - mach_msg_option_t option) + ipc_kmsg_t kmsg, + mach_msg_option_t option) { - unsigned int sender_pid = ((mach_msg_max_trailer_t *) - ((vm_offset_t)kmsg->ikm_header + - round_msg(kmsg->ikm_header->msgh_size)))->msgh_audit.val[5]; - task_t task_self = current_task(); int impresult = -1; - + +#if IMPORTANCE_TRACE || LEGACY_IMPORTANCE_DELIVERY + task_t task_self = current_task(); + unsigned int sender_pid = ((mach_msg_max_trailer_t *) + ((vm_offset_t)kmsg->ikm_header + + round_msg(kmsg->ikm_header->msgh_size)))->msgh_audit.val[5]; +#endif + /* convert to a voucher with an inherit importance attribute? */ if ((option & MACH_RCV_VOUCHER) != 0) { - uint8_t recipes[2 * sizeof(ipc_voucher_attr_recipe_data_t) + - sizeof(mach_voucher_attr_value_handle_t)]; + uint8_t recipes[2 * sizeof(ipc_voucher_attr_recipe_data_t) + + sizeof(mach_voucher_attr_value_handle_t)]; ipc_voucher_attr_raw_recipe_array_size_t recipe_size = 0; ipc_voucher_attr_recipe_t recipe = (ipc_voucher_attr_recipe_t)recipes; - ipc_voucher_t recv_voucher; + ipc_voucher_t recv_voucher; mach_voucher_attr_value_handle_t handle; ipc_importance_inherit_t inherit; kern_return_t kr; /* set up recipe to copy the old voucher */ if (IP_VALID(kmsg->ikm_voucher)) { - ipc_voucher_t sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject; + ipc_voucher_t sent_voucher = (ipc_voucher_t)ip_get_kobject(kmsg->ikm_voucher); recipe->key = MACH_VOUCHER_ATTR_KEY_ALL; recipe->command = MACH_VOUCHER_ATTR_COPY; @@ -2831,7 +3204,7 @@ ipc_importance_receive( * transferring any boosts from the kmsg linkage through the * port directly to the new inheritance object. */ - inherit = ipc_importance_inherit_from(kmsg); + inherit = ipc_importance_inherit_from_kmsg(kmsg); handle = (mach_voucher_attr_value_handle_t)inherit; assert(IIE_NULL == kmsg->ikm_importance); @@ -2854,20 +3227,20 @@ ipc_importance_receive( recipe_size += sizeof(*recipe) + sizeof(mach_voucher_attr_value_handle_t); kr = ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control, - recipes, - recipe_size, - &recv_voucher); + recipes, + recipe_size, + &recv_voucher); assert(KERN_SUCCESS == kr); /* swap the voucher port (and set voucher bits in case it didn't already exist) */ kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16); ipc_port_release_send(kmsg->ikm_voucher); kmsg->ikm_voucher = convert_voucher_to_port(recv_voucher); - if (III_NULL != inherit) + if (III_NULL != inherit) { impresult = 2; + } } } else { /* Don't want a voucher */ - /* got linked importance? have to drop */ if (IIE_NULL != kmsg->ikm_importance) { ipc_importance_elem_t elem; @@ -2883,38 +3256,45 @@ ipc_importance_receive( /* With kmsg unlinked, can safely examine message importance attribute. */ if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) { - ipc_importance_task_t task_imp = task_self->task_imp_base; ipc_port_t port = kmsg->ikm_header->msgh_remote_port; +#if LEGACY_IMPORTANCE_DELIVERY + ipc_importance_task_t task_imp = task_self->task_imp_base; - ip_lock(port); - ipc_port_impcount_delta(port, -1, IP_NULL); - ip_unlock(port); - - /* will user accept legacy responsibility for the importance boost */ - if (KERN_SUCCESS == ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid)) { + /* The owner of receive right might have changed, take the internal assertion */ + if (KERN_SUCCESS == ipc_importance_task_hold_internal_assertion(task_imp, 1)) { + ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid); impresult = 1; - } else { + } else +#endif + { /* The importance boost never applied to task (clear the bit) */ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; impresult = 0; } + + /* Drop the boost on the port and the owner of the receive right */ + ip_lock(port); + if (ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { + ip_unlock(port); + } } } -#if IMPORTANCE_DEBUG - if (-1 < impresult) +#if IMPORTANCE_TRACE + if (-1 < impresult) { KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_DELV)) | DBG_FUNC_NONE, - sender_pid, task_pid(task_self), - kmsg->ikm_header->msgh_id, impresult, 0); - if (impresult == 2){ + sender_pid, task_pid(task_self), + kmsg->ikm_header->msgh_id, impresult, 0); + } + if (impresult == 2) { /* * This probe only covers new voucher-based path. Legacy importance - * will trigger the probe in ipc_importance_task_externalize_assertion() + * will trigger the probe in ipc_importance_task_externalize_assertion() * above and have impresult==1 here. */ DTRACE_BOOST5(receive_boost, task_t, task_self, int, task_pid(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt); - } -#endif /* IMPORTANCE_DEBUG */ + } +#endif /* IMPORTANCE_TRACE */ } /* @@ -2927,8 +3307,8 @@ ipc_importance_receive( */ void ipc_importance_unreceive( - ipc_kmsg_t kmsg, - mach_msg_option_t __unused option) + ipc_kmsg_t kmsg, + mach_msg_option_t __unused option) { /* importance should already be in the voucher and out of the kmsg */ assert(IIE_NULL == kmsg->ikm_importance); @@ -2962,13 +3342,13 @@ ipc_importance_unreceive( */ void ipc_importance_clean( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { - ipc_port_t port; + ipc_port_t port; /* Is the kmsg still linked? If so, remove that first */ if (IIE_NULL != kmsg->ikm_importance) { - ipc_importance_elem_t elem; + ipc_importance_elem_t elem; ipc_importance_lock(); elem = ipc_importance_kmsg_unlink(kmsg); @@ -2984,7 +3364,7 @@ ipc_importance_clean( if (IP_VALID(port)) { ip_lock(port); /* inactive ports already had their importance boosts dropped */ - if (!ip_active(port) || + if (!ip_active(port) || ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == FALSE) { ip_unlock(port); } @@ -2995,8 +3375,8 @@ ipc_importance_clean( void ipc_importance_assert_clean(__assert_only ipc_kmsg_t kmsg) { - assert(IIE_NULL == kmsg->ikm_importance); - assert(!MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)); + assert(IIE_NULL == kmsg->ikm_importance); + assert(!MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)); } /* @@ -3005,55 +3385,57 @@ ipc_importance_assert_clean(__assert_only ipc_kmsg_t kmsg) static kern_return_t ipc_importance_release_value( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_t value, - mach_voucher_attr_value_reference_t sync); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_t value, + mach_voucher_attr_value_reference_t sync); static kern_return_t ipc_importance_get_value( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_recipe_command_t command, - mach_voucher_attr_value_handle_array_t prev_values, - mach_voucher_attr_value_handle_array_size_t prev_value_count, - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t content_size, - mach_voucher_attr_value_handle_t *out_value, - ipc_voucher_t *out_value_voucher); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_recipe_command_t command, + mach_voucher_attr_value_handle_array_t prev_values, + mach_voucher_attr_value_handle_array_size_t prev_value_count, + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t content_size, + mach_voucher_attr_value_handle_t *out_value, + mach_voucher_attr_value_flags_t *out_flags, + ipc_voucher_t *out_value_voucher); static kern_return_t ipc_importance_extract_content( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_array_t values, - mach_voucher_attr_value_handle_array_size_t value_count, - mach_voucher_attr_recipe_command_t *out_command, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *in_out_content_size); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_array_t values, + mach_voucher_attr_value_handle_array_size_t value_count, + mach_voucher_attr_recipe_command_t *out_command, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *in_out_content_size); static kern_return_t ipc_importance_command( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_array_t values, - mach_msg_type_number_t value_count, - mach_voucher_attr_command_t command, - mach_voucher_attr_content_t in_content, - mach_voucher_attr_content_size_t in_content_size, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *out_content_size); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_array_t values, + mach_msg_type_number_t value_count, + mach_voucher_attr_command_t command, + mach_voucher_attr_content_t in_content, + mach_voucher_attr_content_size_t in_content_size, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *out_content_size); static void ipc_importance_manager_release( - ipc_voucher_attr_manager_t manager); - -struct ipc_voucher_attr_manager ipc_importance_manager = { - .ivam_release_value = ipc_importance_release_value, - .ivam_get_value = ipc_importance_get_value, - .ivam_extract_content = ipc_importance_extract_content, - .ivam_command = ipc_importance_command, - .ivam_release = ipc_importance_manager_release, + ipc_voucher_attr_manager_t manager); + +const struct ipc_voucher_attr_manager ipc_importance_manager = { + .ivam_release_value = ipc_importance_release_value, + .ivam_get_value = ipc_importance_get_value, + .ivam_extract_content = ipc_importance_extract_content, + .ivam_command = ipc_importance_command, + .ivam_release = ipc_importance_manager_release, + .ivam_flags = IVAM_FLAGS_NONE, }; #define IMPORTANCE_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_IMPORTANCE == (key)) @@ -3071,10 +3453,10 @@ struct ipc_voucher_attr_manager ipc_importance_manager = { */ static kern_return_t ipc_importance_release_value( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_value_handle_t value, - mach_voucher_attr_value_reference_t sync) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_value_handle_t value, + mach_voucher_attr_value_reference_t sync) { ipc_importance_elem_t elem; @@ -3093,12 +3475,12 @@ ipc_importance_release_value( return KERN_FAILURE; } - /* clear made */ + /* clear made */ elem->iie_made = 0; - /* - * If there are pending external boosts represented by this attribute, - * drop them from the apropriate task + /* + * If there are pending external boosts represented by this attribute, + * drop them from the apropriate task */ if (IIE_TYPE_INHERIT == IIE_TYPE(elem)) { ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem; @@ -3126,7 +3508,7 @@ ipc_importance_release_value( inherit->iii_externcnt = 0; inherit->iii_externdrop = 0; } - } + } /* drop the made reference on elem */ ipc_importance_release_locked(elem); @@ -3148,15 +3530,16 @@ ipc_importance_release_value( */ static kern_return_t ipc_importance_get_value( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_recipe_command_t command, - mach_voucher_attr_value_handle_array_t prev_values, - mach_voucher_attr_value_handle_array_size_t prev_value_count, - mach_voucher_attr_content_t __unused content, - mach_voucher_attr_content_size_t content_size, - mach_voucher_attr_value_handle_t *out_value, - ipc_voucher_t *out_value_voucher) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_recipe_command_t command, + mach_voucher_attr_value_handle_array_t prev_values, + mach_voucher_attr_value_handle_array_size_t prev_value_count, + mach_voucher_attr_content_t __unused content, + mach_voucher_attr_content_size_t content_size, + mach_voucher_attr_value_handle_t *out_value, + mach_voucher_attr_value_flags_t *out_flags, + ipc_voucher_t *out_value_voucher) { ipc_importance_elem_t elem; task_t self; @@ -3164,13 +3547,14 @@ ipc_importance_get_value( IMPORTANCE_ASSERT_MANAGER(manager); IMPORTANCE_ASSERT_KEY(key); - if (0 != content_size) + if (0 != content_size) { return KERN_INVALID_ARGUMENT; + } + *out_flags = MACH_VOUCHER_ATTR_VALUE_FLAGS_NONE; /* never an out voucher */ switch (command) { - case MACH_VOUCHER_ATTR_REDEEM: /* redeem of previous values is the value */ @@ -3201,9 +3585,9 @@ ipc_importance_get_value( *out_value = (mach_voucher_attr_value_handle_t)elem; *out_value_voucher = IPC_VOUCHER_NULL; return KERN_SUCCESS; - + default: - /* + /* * every other command is unknown * * Specifically, there is no mechanism provided to construct an @@ -3227,13 +3611,13 @@ ipc_importance_get_value( */ static kern_return_t ipc_importance_extract_content( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_value_handle_array_t values, - mach_voucher_attr_value_handle_array_size_t value_count, - mach_voucher_attr_recipe_command_t *out_command, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *in_out_content_size) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_value_handle_array_t values, + mach_voucher_attr_value_handle_array_size_t value_count, + mach_voucher_attr_recipe_command_t *out_command, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *in_out_content_size) { mach_voucher_attr_content_size_t size = 0; ipc_importance_elem_t elem; @@ -3243,15 +3627,16 @@ ipc_importance_extract_content( IMPORTANCE_ASSERT_KEY(key); /* the first non-default value provides the data */ - for (i = 0; i < value_count ; i++) { + for (i = 0; i < value_count && *in_out_content_size > 0; i++) { elem = (ipc_importance_elem_t)values[i]; - if (IIE_NULL == elem) + if (IIE_NULL == elem) { continue; + } snprintf((char *)out_content, *in_out_content_size, "Importance for pid "); size = (mach_voucher_attr_content_size_t)strlen((char *)out_content); - for(;;) { + for (;;) { ipc_importance_inherit_t inherit = III_NULL; ipc_importance_task_t task_imp; task_t task; @@ -3261,28 +3646,29 @@ ipc_importance_extract_content( task_imp = (ipc_importance_task_t)elem; task = task_imp->iit_task; t_pid = (TASK_NULL != task) ? - task_pid(task) : -1; + task_pid(task) : -1; snprintf((char *)out_content + size, *in_out_content_size - size, "%d", t_pid); } else { inherit = (ipc_importance_inherit_t)elem; task_imp = inherit->iii_to_task; task = task_imp->iit_task; t_pid = (TASK_NULL != task) ? - task_pid(task) : -1; - snprintf((char *)out_content + size, *in_out_content_size - size, - "%d (%d of %d boosts) %s from pid ", t_pid, - III_EXTERN(inherit), inherit->iii_externcnt, - (inherit->iii_donating) ? "donated" : "linked"); + task_pid(task) : -1; + snprintf((char *)out_content + size, *in_out_content_size - size, + "%d (%d of %d boosts) %s from pid ", t_pid, + III_EXTERN(inherit), inherit->iii_externcnt, + (inherit->iii_donating) ? "donated" : "linked"); } - + size = (mach_voucher_attr_content_size_t)strlen((char *)out_content); - if (III_NULL == inherit) + if (III_NULL == inherit) { break; + } elem = inherit->iii_from_elem; - } - size++; /* account for NULL */ + } + size++; /* account for NULL */ } *out_command = MACH_VOUCHER_ATTR_NOOP; /* cannot be used to regenerate value */ *in_out_content_size = size; @@ -3299,15 +3685,15 @@ ipc_importance_extract_content( */ static kern_return_t ipc_importance_command( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_value_handle_array_t values, - mach_msg_type_number_t value_count, - mach_voucher_attr_command_t command, - mach_voucher_attr_content_t in_content, - mach_voucher_attr_content_size_t in_content_size, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *out_content_size) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_value_handle_array_t values, + mach_msg_type_number_t value_count, + mach_voucher_attr_command_t command, + mach_voucher_attr_content_t in_content, + mach_voucher_attr_content_size_t in_content_size, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *out_content_size) { ipc_importance_inherit_t inherit; ipc_importance_task_t to_task; @@ -3342,7 +3728,7 @@ ipc_importance_command( if (III_NULL == inherit) { return KERN_INVALID_ARGUMENT; } - + ipc_importance_lock(); if (0 == refs) { @@ -3359,7 +3745,7 @@ ipc_importance_command( /* if not donating to a denap receiver, it was called incorrectly */ if (!ipc_importance_task_is_marked_denap_receiver(to_task)) { ipc_importance_unlock(); - return KERN_INVALID_ARGUMENT; /* keeps dispatch happy */ + return KERN_INVALID_TASK; /* keeps dispatch happy */ } /* Enough external references left to drop? */ @@ -3386,7 +3772,7 @@ ipc_importance_command( if (ipc_importance_delayed_drop_call != NULL && ipc_importance_task_is_marked_denap_receiver(to_task)) { ipc_importance_task_delayed_drop(to_task); - } + } /* drop task assertions associated with the dropped boosts */ if (ipc_importance_task_check_transition(to_task, IIT_UPDATE_DROP, refs)) { @@ -3426,9 +3812,10 @@ ipc_importance_command( * reference granted back at registration time, and that reference is never * dropped, this should never be called. */ +__abortlike static void ipc_importance_manager_release( - ipc_voucher_attr_manager_t __assert_only manager) + ipc_voucher_attr_manager_t __assert_only manager) { IMPORTANCE_ASSERT_MANAGER(manager); panic("Voucher importance manager released"); @@ -3453,19 +3840,19 @@ ipc_importance_init(void) } ipc_importance_task_zone = zinit(sizeof(struct ipc_importance_task), - ipc_importance_max * sizeof(struct ipc_importance_task), - sizeof(struct ipc_importance_task), - "ipc task importance"); + ipc_importance_max * sizeof(struct ipc_importance_task), + sizeof(struct ipc_importance_task), + "ipc task importance"); zone_change(ipc_importance_task_zone, Z_NOENCRYPT, TRUE); ipc_importance_inherit_zone = zinit(sizeof(struct ipc_importance_inherit), - ipc_importance_max * sizeof(struct ipc_importance_inherit), - sizeof(struct ipc_importance_inherit), - "ipc importance inherit"); + ipc_importance_max * sizeof(struct ipc_importance_inherit), + sizeof(struct ipc_importance_inherit), + "ipc importance inherit"); zone_change(ipc_importance_inherit_zone, Z_NOENCRYPT, TRUE); -#if DEVELOPMENT || DEBUG +#if DEVELOPMENT || DEBUG queue_init(&global_iit_alloc_queue); #endif @@ -3473,11 +3860,12 @@ ipc_importance_init(void) ipc_importance_lock_init(); kr = ipc_register_well_known_mach_voucher_attr_manager(&ipc_importance_manager, - (mach_voucher_attr_value_handle_t)0, - MACH_VOUCHER_ATTR_KEY_IMPORTANCE, - &ipc_importance_control); - if (KERN_SUCCESS != kr) + (mach_voucher_attr_value_handle_t)0, + MACH_VOUCHER_ATTR_KEY_IMPORTANCE, + &ipc_importance_control); + if (KERN_SUCCESS != kr) { printf("Voucher importance manager register returned %d", kr); + } } /* @@ -3493,8 +3881,8 @@ ipc_importance_thread_call_init(void) { /* initialize delayed drop queue and thread-call */ queue_init(&ipc_importance_delayed_drop_queue); - ipc_importance_delayed_drop_call = - thread_call_allocate(ipc_importance_task_delayed_drop_scan, NULL); + ipc_importance_delayed_drop_call = + thread_call_allocate(ipc_importance_task_delayed_drop_scan, NULL); if (NULL == ipc_importance_delayed_drop_call) { panic("ipc_importance_init"); } @@ -3510,10 +3898,10 @@ extern int task_importance_list_pids(task_t task, int flags, char *pid_list, unsigned int max_count) { if (kdp_lck_spin_is_acquired(&ipc_importance_lock_data) || - max_count < 1 || - task->task_imp_base == IIT_NULL || - pid_list == NULL || - flags != TASK_IMP_LIST_DONATING_PIDS) { + max_count < 1 || + task->task_imp_base == IIT_NULL || + pid_list == NULL || + flags != TASK_IMP_LIST_DONATING_PIDS) { return 0; } unsigned int pidcount = 0; @@ -3526,13 +3914,13 @@ task_importance_list_pids(task_t task, int flags, char *pid_list, unsigned int m queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) { /* check space in buffer */ - if (pidcount >= max_count) + if (pidcount >= max_count) { break; + } previous_pid = target_pid; target_pid = -1; if (temp_inherit->iii_donating) { - #if DEVELOPMENT || DEBUG target_pid = temp_inherit->iii_to_task->iit_bsd_pid; #else @@ -3548,13 +3936,13 @@ task_importance_list_pids(task_t task, int flags, char *pid_list, unsigned int m pid_list += sizeof(target_pid); pidcount++; } - } target_pid = 0; queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) { - if (pidcount >= max_count) + if (pidcount >= max_count) { break; + } previous_pid = target_pid; target_pid = -1; elem = temp_kmsg->ikm_importance; @@ -3568,8 +3956,8 @@ task_importance_list_pids(task_t task, int flags, char *pid_list, unsigned int m continue; } - if (IIE_TYPE_TASK == IIE_TYPE(elem) && - (((ipc_importance_task_t)elem)->iit_task != TASK_NULL)) { + if (IIE_TYPE_TASK == IIE_TYPE(elem) && + (((ipc_importance_task_t)elem)->iit_task != TASK_NULL)) { target_pid = task_pid(((ipc_importance_task_t)elem)->iit_task); } else { temp_inherit = (ipc_importance_inherit_t)elem; @@ -3592,4 +3980,3 @@ task_importance_list_pids(task_t task, int flags, char *pid_list, unsigned int m return pidcount; } -