--- /dev/null
+/*
+ * Copyright (c) 2013 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mach/mach_types.h>
+#include <mach/notify.h>
+#include <ipc/ipc_types.h>
+#include <ipc/ipc_importance.h>
+#include <ipc/ipc_port.h>
+#include <ipc/ipc_voucher.h>
+#include <kern/ipc_kobject.h>
+#include <kern/ipc_tt.h>
+#include <kern/mach_param.h>
+#include <kern/misc_protos.h>
+#include <kern/kalloc.h>
+#include <kern/zalloc.h>
+#include <kern/queue.h>
+#include <kern/task.h>
+
+#include <sys/kdebug.h>
+
+#include <mach/mach_voucher_attr_control.h>
+#include <mach/machine/sdt.h>
+
+extern int proc_pid(void *);
+extern int proc_selfpid(void);
+extern uint64_t proc_uniqueid(void *p);
+extern char *proc_name_address(void *p);
+
+/*
+ * Globals for delayed boost drop processing.
+ */
+static queue_head_t ipc_importance_delayed_drop_queue;
+static thread_call_t ipc_importance_delayed_drop_call;
+static uint64_t ipc_importance_delayed_drop_timestamp;
+static boolean_t ipc_importance_delayed_drop_call_requested = FALSE;
+
+#define DENAP_DROP_TARGET (1000 * NSEC_PER_MSEC) /* optimum denap delay */
+#define DENAP_DROP_SKEW (100 * NSEC_PER_MSEC) /* request skew for wakeup */
+#define DENAP_DROP_LEEWAY (2 * DENAP_DROP_SKEW) /* specified wakeup leeway */
+
+#define DENAP_DROP_DELAY (DENAP_DROP_TARGET + DENAP_DROP_SKEW)
+#define DENAP_DROP_FLAGS (THREAD_CALL_DELAY_SYS_NORMAL | THREAD_CALL_DELAY_LEEWAY)
+
+/*
+ * Importance Voucher Attribute Manager
+ */
+
+static lck_spin_t ipc_importance_lock_data; /* single lock for now */
+
+
+#define ipc_importance_lock_init() \
+ lck_spin_init(&ipc_importance_lock_data, &ipc_lck_grp, &ipc_lck_attr)
+#define ipc_importance_lock_destroy() \
+ lck_spin_destroy(&ipc_importance_lock_data, &ipc_lck_grp)
+#define ipc_importance_lock() \
+ lck_spin_lock(&ipc_importance_lock_data)
+#define ipc_importance_lock_try() \
+ lck_spin_try_lock(&ipc_importance_lock_data)
+#define ipc_importance_unlock() \
+ lck_spin_unlock(&ipc_importance_lock_data)
+#define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data, \
+ LCK_SLEEP_DEFAULT, \
+ (event_t)(elem), \
+ THREAD_UNINT)
+#define ipc_importance_wakeup(elem) thread_wakeup((event_t)(elem))
+
+#if IIE_REF_DEBUG
+#define incr_ref_counter(x) (hw_atomic_add(&(x), 1))
+
+static inline
+uint32_t ipc_importance_reference_internal(ipc_importance_elem_t elem)
+{
+ incr_ref_counter(elem->iie_refs_added);
+ return (hw_atomic_add(&elem->iie_bits, 1) & IIE_REFS_MASK);
+}
+
+static inline
+uint32_t ipc_importance_release_internal(ipc_importance_elem_t elem)
+{
+ incr_ref_counter(elem->iie_refs_dropped);
+ return (hw_atomic_sub(&elem->iie_bits, 1) & IIE_REFS_MASK);
+}
+
+static inline
+uint32_t ipc_importance_task_reference_internal(ipc_importance_task_t task_imp)
+{
+ uint32_t out;
+ out = ipc_importance_reference_internal(&task_imp->iit_elem);
+ incr_ref_counter(task_imp->iit_elem.iie_task_refs_added);
+ return out;
+}
+
+static inline
+uint32_t ipc_importance_task_release_internal(ipc_importance_task_t task_imp)
+{
+ uint32_t out;
+
+ assert(1 < IIT_REFS(task_imp));
+ incr_ref_counter(task_imp->iit_elem.iie_task_refs_dropped);
+ out = ipc_importance_release_internal(&task_imp->iit_elem);
+ return out;
+}
+
+static inline
+void ipc_importance_counter_init(ipc_importance_elem_t elem)
+{
+
+ elem->iie_refs_added = 0;
+ elem->iie_refs_dropped = 0;
+ elem->iie_kmsg_refs_added = 0;
+ elem->iie_kmsg_refs_inherited = 0;
+ elem->iie_kmsg_refs_coalesced = 0;
+ elem->iie_kmsg_refs_dropped = 0;
+ elem->iie_task_refs_added = 0;
+ elem->iie_task_refs_added_inherit_from = 0;
+ elem->iie_task_refs_added_transition = 0;
+ elem->iie_task_refs_self_added = 0;
+ elem->iie_task_refs_inherited = 0;
+ elem->iie_task_refs_coalesced = 0;
+ elem->iie_task_refs_dropped = 0;
+}
+#else
+#define incr_ref_counter(x)
+#endif
+
+#if DEVELOPMENT || DEBUG
+static queue_head_t global_iit_alloc_queue;
+#endif
+
+/* TODO: remove this varibale when interactive daemon audit is complete */
+boolean_t ipc_importance_interactive_receiver = FALSE;
+
+static zone_t ipc_importance_task_zone;
+static zone_t ipc_importance_inherit_zone;
+
+static ipc_voucher_attr_control_t ipc_importance_control;
+
+/*
+ * Routine: ipc_importance_kmsg_link
+ * Purpose:
+ * Link the kmsg onto the appropriate propagation chain.
+ * If the element is a task importance, we link directly
+ * on its propagation chain. Otherwise, we link onto the
+ * destination task of the inherit.
+ * Conditions:
+ * Importance lock held.
+ * Caller is donating an importance elem reference to the kmsg.
+ */
+static void
+ipc_importance_kmsg_link(
+ ipc_kmsg_t kmsg,
+ ipc_importance_elem_t elem)
+{
+ ipc_importance_elem_t link_elem;
+
+ assert(IIE_NULL == kmsg->ikm_importance);
+
+ link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
+ (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task :
+ elem;
+
+ queue_enter(&link_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance);
+ kmsg->ikm_importance = elem;
+}
+
+/*
+ * Routine: ipc_importance_kmsg_unlink
+ * Purpose:
+ * Unlink the kmsg from its current propagation chain.
+ * If the element is a task importance, we unlink directly
+ * from its propagation chain. Otherwise, we unlink from the
+ * destination task of the inherit.
+ * Returns:
+ * The reference to the importance element it was linked on.
+ * Conditions:
+ * Importance lock held.
+ * Caller is responsible for dropping reference on returned elem.
+ */
+static ipc_importance_elem_t
+ipc_importance_kmsg_unlink(
+ ipc_kmsg_t kmsg)
+{
+ ipc_importance_elem_t elem = kmsg->ikm_importance;
+
+ if (IIE_NULL != elem) {
+ ipc_importance_elem_t unlink_elem;
+
+ unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
+ (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task :
+ elem;
+
+ queue_remove(&unlink_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance);
+ kmsg->ikm_importance = IIE_NULL;
+ }
+ return elem;
+}
+
+/*
+ * Routine: ipc_importance_inherit_link
+ * Purpose:
+ * Link the inherit onto the appropriate propagation chain.
+ * If the element is a task importance, we link directly
+ * on its propagation chain. Otherwise, we link onto the
+ * destination task of the inherit.
+ * Conditions:
+ * Importance lock held.
+ * Caller is donating an elem importance reference to the inherit.
+ */
+static void
+ipc_importance_inherit_link(
+ ipc_importance_inherit_t inherit,
+ ipc_importance_elem_t elem)
+{
+ ipc_importance_elem_t link_elem;
+
+ assert(IIE_NULL == inherit->iii_from_elem);
+ link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
+ (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task :
+ elem;
+
+ queue_enter(&link_elem->iie_inherits, inherit,
+ ipc_importance_inherit_t, iii_inheritance);
+ inherit->iii_from_elem = elem;
+}
+
+/*
+ * Routine: ipc_importance_inherit_unlink
+ * Purpose:
+ * Unlink the inherit from its current propagation chain.
+ * If the element is a task importance, we unlink directly
+ * from its propagation chain. Otherwise, we unlink from the
+ * destination task of the inherit.
+ * Returns:
+ * The reference to the importance element it was linked on.
+ * Conditions:
+ * Importance lock held.
+ * Caller is responsible for dropping reference on returned elem.
+ */
+static ipc_importance_elem_t
+ipc_importance_inherit_unlink(
+ ipc_importance_inherit_t inherit)
+{
+ ipc_importance_elem_t elem = inherit->iii_from_elem;
+
+ if (IIE_NULL != elem) {
+ ipc_importance_elem_t unlink_elem;
+
+ unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
+ (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task :
+ elem;
+
+ queue_remove(&unlink_elem->iie_inherits, inherit,
+ ipc_importance_inherit_t, iii_inheritance);
+ inherit->iii_from_elem = IIE_NULL;
+ }
+ return elem;
+}
+
+/*
+ * Routine: ipc_importance_reference
+ * Purpose:
+ * Add a reference to the importance element.
+ * Conditions:
+ * Caller must hold a reference on the element.
+ */
+void
+ipc_importance_reference(ipc_importance_elem_t elem)
+{
+ assert(0 < IIE_REFS(elem));
+ ipc_importance_reference_internal(elem);
+}
+
+/*
+ * Routine: ipc_importance_release_locked
+ * Purpose:
+ * Release a reference on an importance attribute value,
+ * unlinking and deallocating the attribute if the last reference.
+ * Conditions:
+ * Entered with importance lock held, leaves with it unlocked.
+ */
+static void
+ipc_importance_release_locked(ipc_importance_elem_t elem)
+{
+ assert(0 < IIE_REFS(elem));
+
+ if (0 < ipc_importance_release_internal(elem)) {
+
+#if DEVELOPMENT || DEBUG
+ ipc_importance_inherit_t temp_inherit;
+ ipc_importance_task_t link_task;
+ ipc_kmsg_t temp_kmsg;
+ uint32_t expected = 0;
+
+ if (0 < elem->iie_made)
+ expected++;
+
+ link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
+ ((ipc_importance_inherit_t)elem)->iii_to_task :
+ (ipc_importance_task_t)elem;
+
+ queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance)
+ if (temp_kmsg->ikm_importance == elem)
+ expected++;
+ queue_iterate(&link_task->iit_inherits, temp_inherit,
+ ipc_importance_inherit_t, iii_inheritance)
+ if (temp_inherit->iii_from_elem == elem)
+ expected++;
+
+ if (IIE_REFS(elem) < expected)
+ panic("ipc_importance_release_locked (%p)", elem);
+#endif
+ ipc_importance_unlock();
+ return;
+ }
+
+ /* last ref */
+ /* can't get to no refs if we contribute to something else's importance */
+ assert(queue_empty(&elem->iie_kmsgs));
+ assert(queue_empty(&elem->iie_inherits));
+
+ switch (IIE_TYPE(elem)) {
+
+ /* just a "from" task reference to drop */
+ case IIE_TYPE_TASK:
+ {
+ ipc_importance_task_t task_elem;
+
+ task_elem = (ipc_importance_task_t)elem;
+ assert(TASK_NULL == task_elem->iit_task);
+
+#if DEVELOPMENT || DEBUG
+ queue_remove(&global_iit_alloc_queue, task_elem, ipc_importance_task_t, iit_allocation);
+#endif
+
+ ipc_importance_unlock();
+
+ zfree(ipc_importance_task_zone, task_elem);
+ break;
+ }
+
+ /* dropping an inherit element */
+ case IIE_TYPE_INHERIT:
+ {
+ ipc_importance_inherit_t inherit;
+ ipc_importance_elem_t from_elem;
+ ipc_importance_task_t to_task;
+
+
+ inherit = (ipc_importance_inherit_t)elem;
+ to_task = inherit->iii_to_task;
+ assert(IIT_NULL != to_task);
+ assert(!inherit->iii_donating);
+
+ /* unlink and release the inherit */
+ assert(ipc_importance_task_is_any_receiver_type(to_task));
+ from_elem = ipc_importance_inherit_unlink(inherit);
+ assert(IIE_NULL != from_elem);
+ ipc_importance_release_locked(from_elem);
+ /* unlocked on return */
+
+ ipc_importance_task_release(to_task);
+
+ zfree(ipc_importance_inherit_zone, inherit);
+ break;
+ }
+ }
+}
+
+/*
+ * Routine: ipc_importance_release
+ * Purpose:
+ * Release a reference on an importance attribute value,
+ * unlinking and deallocating the attribute if the last reference.
+ * Conditions:
+ * nothing locked on entrance, nothing locked on exit.
+ * May block.
+ */
+void
+ipc_importance_release(ipc_importance_elem_t elem)
+{
+ if (IIE_NULL == elem)
+ return;
+
+ ipc_importance_lock();
+ ipc_importance_release_locked(elem);
+ /* unlocked */
+}
+
+/*
+ * Routine: ipc_importance_task_reference
+
+
+ * Purpose:
+ * Retain a reference on a task importance attribute value.
+ * Conditions:
+ * nothing locked on entrance, nothing locked on exit.
+ * caller holds a reference already.
+ */
+void
+ipc_importance_task_reference(ipc_importance_task_t task_elem)
+{
+ if (IIT_NULL == task_elem)
+ return;
+#if IIE_REF_DEBUG
+ incr_ref_counter(task_elem->iit_elem.iie_task_refs_added);
+#endif
+ ipc_importance_reference(&task_elem->iit_elem);
+}
+
+/*
+ * Routine: ipc_importance_task_release
+ * Purpose:
+ * Release a reference on a task importance attribute value,
+ * unlinking and deallocating the attribute if the last reference.
+ * Conditions:
+ * nothing locked on entrance, nothing locked on exit.
+ * May block.
+ */
+void
+ipc_importance_task_release(ipc_importance_task_t task_elem)
+{
+ if (IIT_NULL == task_elem)
+ return;
+
+ ipc_importance_lock();
+#if IIE_REF_DEBUG
+ incr_ref_counter(task_elem->iit_elem.iie_task_refs_dropped);
+#endif
+ ipc_importance_release_locked(&task_elem->iit_elem);
+ /* unlocked */
+}
+
+/*
+ * Routine: ipc_importance_task_release_locked
+ * Purpose:
+ * Release a reference on a task importance attribute value,
+ * unlinking and deallocating the attribute if the last reference.
+ * Conditions:
+ * importance lock held on entry, nothing locked on exit.
+ * May block.
+ */
+static void
+ipc_importance_task_release_locked(ipc_importance_task_t task_elem)
+{
+ if (IIT_NULL == task_elem) {
+ ipc_importance_unlock();
+ return;
+ }
+#if IIE_REF_DEBUG
+ incr_ref_counter(task_elem->iit_elem.iie_task_refs_dropped);
+#endif
+ ipc_importance_release_locked(&task_elem->iit_elem);
+ /* unlocked */
+}
+
+/*
+ * Routines for importance donation/inheritance/boosting
+ */
+
+
+/*
+ * External importance assertions are managed by the process in userspace
+ * Internal importance assertions are the responsibility of the kernel
+ * Assertions are changed from internal to external via task_importance_externalize_assertion
+ */
+
+/*
+ * Routine: ipc_importance_task_check_transition
+ * Purpose:
+ * Increase or decrement the internal task importance counter of the
+ * specified task and determine if propagation and a task policy
+ * update is required.
+ *
+ * If it is already enqueued for a policy update, steal it from that queue
+ * (as we are reversing that update before it happens).
+ *
+ * Conditions:
+ * Called with the importance lock held.
+ * It is the caller's responsibility to perform the propagation of the
+ * transition and/or policy changes by checking the return value.
+ */
+static boolean_t
+ipc_importance_task_check_transition(
+ ipc_importance_task_t task_imp,
+ iit_update_type_t type,
+ uint32_t delta)
+{
+
+ task_t target_task = task_imp->iit_task;
+ boolean_t boost = (IIT_UPDATE_HOLD == type);
+ boolean_t before_boosted, after_boosted;
+
+ if (!ipc_importance_task_is_any_receiver_type(task_imp))
+ return FALSE;
+
+#if IMPORTANCE_DEBUG
+ int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_START,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0);
+#endif
+
+ /* snapshot the effective boosting status before making any changes */
+ before_boosted = (task_imp->iit_assertcnt > 0);
+
+ /* Adjust the assertcnt appropriately */
+ if (boost) {
+ task_imp->iit_assertcnt += delta;
+#if IMPORTANCE_DEBUG
+ DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid,
+ task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt);
+#endif
+ } else {
+ // assert(delta <= task_imp->iit_assertcnt);
+ if (delta > task_imp->iit_assertcnt - IIT_EXTERN(task_imp)) {
+ /* TODO: Turn this back into a panic <rdar://problem/12592649> */
+ if (target_task != TASK_NULL) {
+ printf("Over-release of kernel-internal importance assertions for pid %d (%s), "
+ "dropping %d assertion(s) but task only has %d remaining (%d external).\n",
+ audit_token_pid_from_task(target_task),
+ (target_task->bsd_info == NULL) ? "" : proc_name_address(target_task->bsd_info),
+ delta,
+ task_imp->iit_assertcnt,
+ IIT_EXTERN(task_imp));
+ }
+ task_imp->iit_assertcnt = IIT_EXTERN(task_imp);
+ } else {
+ task_imp->iit_assertcnt -= delta;
+ }
+#if IMPORTANCE_DEBUG
+ // This convers both legacy and voucher-based importance.
+ DTRACE_BOOST4(drop_boost, task_t, target_task, int, target_pid, int, delta, int, task_imp->iit_assertcnt);
+#endif
+ }
+
+#if IMPORTANCE_DEBUG
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_END,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0);
+#endif
+
+ /* did the change result in an effective donor status change? */
+ after_boosted = (task_imp->iit_assertcnt > 0);
+
+ if (after_boosted != before_boosted) {
+
+ /*
+ * If the task importance is already on an update queue, we just reversed the need for a
+ * pending policy update. If the queue is any other than the delayed-drop-queue, pull it
+ * off that queue and release the reference it got going onto the update queue. If it is
+ * the delayed-drop-queue we leave it in place in case it comes back into the drop state
+ * before its time delay is up.
+ *
+ * We still need to propagate the change downstream to reverse the assertcnt effects,
+ * but we no longer need to update this task's boost policy state.
+ *
+ * Otherwise, mark it as needing a policy update.
+ */
+ assert(0 == task_imp->iit_updatepolicy);
+ if (NULL != task_imp->iit_updateq) {
+ if (&ipc_importance_delayed_drop_queue != task_imp->iit_updateq) {
+ queue_remove(task_imp->iit_updateq, task_imp, ipc_importance_task_t, iit_updates);
+ task_imp->iit_updateq = NULL;
+ ipc_importance_task_release_internal(task_imp); /* can't be last ref */
+ }
+ } else {
+ task_imp->iit_updatepolicy = 1;
+ }
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+
+/*
+ * Routine: ipc_importance_task_propagate_helper
+ * Purpose:
+ * Increase or decrement the internal task importance counter of all
+ * importance tasks inheriting from the specified one. If this causes
+ * that importance task to change state, add it to the list of tasks
+ * to do a policy update against.
+ * Conditions:
+ * Called with the importance lock held.
+ * It is the caller's responsibility to iterate down the generated list
+ * and propagate any subsequent assertion changes from there.
+ */
+static void
+ipc_importance_task_propagate_helper(
+ ipc_importance_task_t task_imp,
+ iit_update_type_t type,
+ queue_t propagation)
+{
+ ipc_importance_task_t temp_task_imp;
+
+ /*
+ * iterate the downstream kmsgs, adjust their boosts,
+ * and capture the next task to adjust for each message
+ */
+
+ ipc_kmsg_t temp_kmsg;
+
+ queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) {
+ mach_msg_header_t *hdr = temp_kmsg->ikm_header;
+ mach_port_delta_t delta;
+ ipc_port_t port;
+
+ /* toggle the kmsg importance bit as a barrier to parallel adjusts */
+ if (IIT_UPDATE_HOLD == type) {
+ if (MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr->msgh_bits)) {
+ continue;
+ }
+
+ /* mark the message as now carrying importance */
+ hdr->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
+ delta = 1;
+ } else {
+ if (!MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr->msgh_bits)) {
+ continue;
+ }
+
+ /* clear the message as now carrying importance */
+ hdr->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
+ delta = -1;
+ }
+
+ /* determine the task importance to adjust as result (if any) */
+ port = (ipc_port_t) hdr->msgh_remote_port;
+ assert(IP_VALID(port));
+ ip_lock(port);
+ temp_task_imp = IIT_NULL;
+ if (!ipc_port_importance_delta_internal(port, &delta, &temp_task_imp)) {
+ ip_unlock(port);
+ }
+
+ /* no task importance to adjust associated with the port? */
+ if (IIT_NULL == temp_task_imp) {
+ continue;
+ }
+
+ /* hold a reference on temp_task_imp */
+
+ /* Adjust the task assertions and determine if an edge was crossed */
+ if (ipc_importance_task_check_transition(temp_task_imp, type, 1)) {
+ incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition);
+ queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props);
+ /* reference donated */
+ } else {
+ ipc_importance_task_release_internal(temp_task_imp);
+ }
+ }
+
+ /*
+ * iterate the downstream importance inherits
+ * and capture the next task importance to boost for each
+ */
+ ipc_importance_inherit_t temp_inherit;
+
+ queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) {
+ uint32_t assertcnt = III_EXTERN(temp_inherit);
+
+ temp_task_imp = temp_inherit->iii_to_task;
+ assert(IIT_NULL != temp_task_imp);
+
+ if (IIT_UPDATE_HOLD == type) {
+ /* if no undropped externcnts in the inherit, nothing to do */
+ if (0 == assertcnt) {
+ assert(temp_inherit->iii_donating == FALSE);
+ continue;
+ }
+
+ /* nothing to do if the inherit is already donating (forced donation) */
+ if (temp_inherit->iii_donating) {
+ continue;
+ }
+
+ /* mark it donating and contribute to the task externcnts */
+ temp_inherit->iii_donating = TRUE;
+ temp_task_imp->iit_externcnt += temp_inherit->iii_externcnt;
+ temp_task_imp->iit_externdrop += temp_inherit->iii_externdrop;
+
+ } else {
+ /* if no contributing assertions, move on */
+ if (0 == assertcnt) {
+ assert(temp_inherit->iii_donating == FALSE);
+ continue;
+ }
+
+ /* nothing to do if the inherit is not donating */
+ if (!temp_inherit->iii_donating) {
+ continue;
+ }
+
+ /* mark it no longer donating */
+ temp_inherit->iii_donating = FALSE;
+
+ /* remove the contribution the inherit made to the to-task */
+ assert(IIT_EXTERN(temp_task_imp) >= III_EXTERN(temp_inherit));
+ assert(temp_task_imp->iit_externcnt >= temp_inherit->iii_externcnt);
+ assert(temp_task_imp->iit_externdrop >= temp_inherit->iii_externdrop);
+ temp_task_imp->iit_externcnt -= temp_inherit->iii_externcnt;
+ temp_task_imp->iit_externdrop -= temp_inherit->iii_externdrop;
+
+ }
+
+ /* Adjust the task assertions and determine if an edge was crossed */
+ assert(ipc_importance_task_is_any_receiver_type(temp_task_imp));
+ if (ipc_importance_task_check_transition(temp_task_imp, type, assertcnt)) {
+ ipc_importance_task_reference(temp_task_imp);
+ incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition);
+ queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props);
+ }
+ }
+}
+
+/*
+ * Routine: ipc_importance_task_process_updates
+ * Purpose:
+ * Process the queue of task importances and apply the policy
+ * update called for. Only process tasks in the queue with an
+ * update timestamp less than the supplied max.
+ * Conditions:
+ * Called and returns with importance locked.
+ * May drop importance lock and block temporarily.
+ */
+static void
+ipc_importance_task_process_updates(
+ queue_t supplied_queue,
+ boolean_t boost,
+ uint64_t max_timestamp)
+{
+ ipc_importance_task_t task_imp;
+ queue_head_t second_chance;
+ queue_t queue = supplied_queue;
+
+ /*
+ * This queue will hold the task's we couldn't trylock on first pass.
+ * By using a second (private) queue, we guarantee all tasks that get
+ * entered on this queue have a timestamp under the maximum.
+ */
+ queue_init(&second_chance);
+
+ /* process any resulting policy updates */
+ retry:
+ while(!queue_empty(queue)) {
+ task_t target_task;
+ struct task_pend_token pend_token = {};
+
+ task_imp = (ipc_importance_task_t)queue_first(queue);
+ assert(0 == task_imp->iit_updatepolicy);
+ assert(queue == task_imp->iit_updateq);
+
+ /* if timestamp is too big, we're done */
+ if (task_imp->iit_updatetime > max_timestamp) {
+ break;
+ }
+
+ /* we were given a reference on each task in the queue */
+
+ /* remove it from the supplied queue */
+ queue_remove(queue, task_imp, ipc_importance_task_t, iit_updates);
+ task_imp->iit_updateq = NULL;
+
+ target_task = task_imp->iit_task;
+
+ /* Is it well on the way to exiting? */
+ if (TASK_NULL == target_task) {
+ ipc_importance_task_release_locked(task_imp);
+ /* importance unlocked */
+ ipc_importance_lock();
+ continue;
+ }
+
+ /* Has the update been reversed on the hysteresis queue? */
+ if (0 < task_imp->iit_assertcnt &&
+ queue == &ipc_importance_delayed_drop_queue) {
+ ipc_importance_task_release_locked(task_imp);
+ /* importance unlocked */
+ ipc_importance_lock();
+ continue;
+ }
+
+ /*
+ * Can we get the task lock out-of-order?
+ * If not, stick this back on the second-chance queue.
+ */
+ if (!task_lock_try(target_task)) {
+ boolean_t should_wait_lock = (queue == &second_chance);
+ task_imp->iit_updateq = &second_chance;
+
+ /*
+ * If we're already processing second-chances on
+ * tasks, keep this task on the front of the queue.
+ * We will wait for the task lock before coming
+ * back and trying again, and we have a better
+ * chance of re-acquiring the lock if we come back
+ * to it right away.
+ */
+ if (should_wait_lock){
+ task_reference(target_task);
+ queue_enter_first(&second_chance, task_imp,
+ ipc_importance_task_t, iit_updates);
+ } else {
+ queue_enter(&second_chance, task_imp,
+ ipc_importance_task_t, iit_updates);
+ }
+ ipc_importance_unlock();
+
+ if (should_wait_lock) {
+ task_lock(target_task);
+ task_unlock(target_task);
+ task_deallocate(target_task);
+ }
+
+ ipc_importance_lock();
+ continue;
+ }
+
+ /* is it going away? */
+ if (!target_task->active) {
+ task_unlock(target_task);
+ ipc_importance_task_release_locked(task_imp);
+ /* importance unlocked */
+ ipc_importance_lock();
+ continue;
+ }
+
+ /* take a task reference for while we don't have the importance lock */
+ task_reference(target_task);
+
+ /* count the transition */
+ if (boost)
+ task_imp->iit_transitions++;
+
+ ipc_importance_unlock();
+
+ /* apply the policy adjust to the target task (while it is still locked) */
+ task_update_boost_locked(target_task, boost, &pend_token);
+
+ /* complete the policy update with the task unlocked */
+ ipc_importance_task_release(task_imp);
+ task_unlock(target_task);
+ task_policy_update_complete_unlocked(target_task, THREAD_NULL, &pend_token);
+ task_deallocate(target_task);
+
+ ipc_importance_lock();
+ }
+
+ /* If there are tasks we couldn't update the first time, try again */
+ if (!queue_empty(&second_chance)) {
+ queue = &second_chance;
+ goto retry;
+ }
+}
+
+
+/*
+ * Routine: ipc_importance_task_delayed_drop_scan
+ * Purpose:
+ * The thread call routine to scan the delayed drop queue,
+ * requesting all updates with a deadline up to the last target
+ * for the thread-call (which is DENAP_DROP_SKEW beyond the first
+ * thread's optimum delay).
+ * update to drop its boost.
+ * Conditions:
+ * Nothing locked
+ */
+static void
+ipc_importance_task_delayed_drop_scan(
+ __unused void *arg1,
+ __unused void *arg2)
+{
+ ipc_importance_lock();
+
+ /* process all queued task drops with timestamps up to TARGET(first)+SKEW */
+ ipc_importance_task_process_updates(&ipc_importance_delayed_drop_queue,
+ FALSE,
+ ipc_importance_delayed_drop_timestamp);
+
+ /* importance lock may have been temporarily dropped */
+
+ /* If there are any entries left in the queue, re-arm the call here */
+ if (!queue_empty(&ipc_importance_delayed_drop_queue)) {
+ ipc_importance_task_t task_imp;
+ uint64_t deadline;
+ uint64_t leeway;
+
+ task_imp = (ipc_importance_task_t)queue_first(&ipc_importance_delayed_drop_queue);
+
+ nanoseconds_to_absolutetime(DENAP_DROP_DELAY, &deadline);
+ deadline += task_imp->iit_updatetime;
+ ipc_importance_delayed_drop_timestamp = deadline;
+
+ nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway);
+
+ thread_call_enter_delayed_with_leeway(
+ ipc_importance_delayed_drop_call,
+ NULL,
+ deadline,
+ leeway,
+ DENAP_DROP_FLAGS);
+ } else {
+ ipc_importance_delayed_drop_call_requested = FALSE;
+ }
+ ipc_importance_unlock();
+}
+
+/*
+ * Routine: ipc_importance_task_delayed_drop
+ * Purpose:
+ * Queue the specified task importance for delayed policy
+ * update to drop its boost.
+ * Conditions:
+ * Called with the importance lock held.
+ */
+static void
+ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp)
+{
+ uint64_t timestamp = mach_absolute_time(); /* no mach_approximate_time() in kernel */
+
+ assert(ipc_importance_delayed_drop_call != NULL);
+
+ /*
+ * If still on an update queue from a previous change,
+ * remove it first (and use that reference). Otherwise, take
+ * a new reference for the delay drop update queue.
+ */
+ if (NULL != task_imp->iit_updateq) {
+ queue_remove(task_imp->iit_updateq, task_imp,
+ ipc_importance_task_t, iit_updates);
+ } else {
+ ipc_importance_task_reference_internal(task_imp);
+ }
+
+ task_imp->iit_updateq = &ipc_importance_delayed_drop_queue;
+ task_imp->iit_updatetime = timestamp;
+
+ queue_enter(&ipc_importance_delayed_drop_queue, task_imp,
+ ipc_importance_task_t, iit_updates);
+
+ /* request the delayed thread-call if not already requested */
+ if (!ipc_importance_delayed_drop_call_requested) {
+ uint64_t deadline;
+ uint64_t leeway;
+
+ nanoseconds_to_absolutetime(DENAP_DROP_DELAY, &deadline);
+ deadline += task_imp->iit_updatetime;
+ ipc_importance_delayed_drop_timestamp = deadline;
+
+ nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway);
+
+ ipc_importance_delayed_drop_call_requested = TRUE;
+ thread_call_enter_delayed_with_leeway(
+ ipc_importance_delayed_drop_call,
+ NULL,
+ deadline,
+ leeway,
+ DENAP_DROP_FLAGS);
+ }
+}
+
+
+/*
+ * Routine: ipc_importance_task_propagate_assertion_locked
+ * Purpose:
+ * Propagate the importance transition type to every item
+ * If this causes a boost to be applied, determine if that
+ * boost should propagate downstream.
+ * Conditions:
+ * Called with the importance lock held.
+ */
+static void
+ipc_importance_task_propagate_assertion_locked(
+ ipc_importance_task_t task_imp,
+ iit_update_type_t type,
+ boolean_t update_task_imp)
+{
+ boolean_t boost = (IIT_UPDATE_HOLD == type);
+ ipc_importance_task_t temp_task_imp;
+ queue_head_t propagate;
+ queue_head_t updates;
+
+ queue_init(&updates);
+ queue_init(&propagate);
+
+ /*
+ * If we're going to update the policy for the provided task,
+ * enqueue it on the propagate queue itself. Otherwise, only
+ * enqueue downstream things.
+ */
+ if (update_task_imp) {
+ queue_enter(&propagate, task_imp, ipc_importance_task_t, iit_props);
+ } else {
+ ipc_importance_task_propagate_helper(task_imp, type, &propagate);
+ }
+
+ /*
+ * for each item on the propagation list, propagate any change downstream,
+ * adding new tasks to propagate further if they transistioned as well.
+ */
+ while (!queue_empty(&propagate)) {
+ boolean_t need_update;
+
+ queue_remove_first(&propagate, temp_task_imp, ipc_importance_task_t, iit_props);
+ assert(IIT_NULL != temp_task_imp);
+
+ /* only propagate for receivers not already marked as a donor */
+ if (!ipc_importance_task_is_marked_donor(temp_task_imp) &&
+ ipc_importance_task_is_marked_receiver(temp_task_imp)) {
+ ipc_importance_task_propagate_helper(temp_task_imp, type, &propagate);
+ }
+
+ /* if we have a policy update to apply, enqueue a reference for later processing */
+ need_update = (0 != temp_task_imp->iit_updatepolicy);
+ temp_task_imp->iit_updatepolicy = 0;
+ if (need_update && TASK_NULL != temp_task_imp->iit_task) {
+ if (NULL == temp_task_imp->iit_updateq) {
+ temp_task_imp->iit_updatetime = 0;
+ temp_task_imp->iit_updateq = &updates;
+ ipc_importance_task_reference_internal(temp_task_imp);
+ if (boost) {
+ queue_enter(&updates, temp_task_imp,
+ ipc_importance_task_t, iit_updates);
+ } else {
+ queue_enter_first(&updates, temp_task_imp,
+ ipc_importance_task_t, iit_updates);
+ }
+ } else {
+ /* Must already be on the AppNap hysteresis queue */
+ assert(&ipc_importance_delayed_drop_queue);
+ assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp));
+ }
+ }
+ }
+
+ /* apply updates to task (may drop importance lock) */
+ if (!queue_empty(&updates)) {
+ ipc_importance_task_process_updates(&updates, boost, 0);
+ }
+}
+
+/*
+ * Routine: ipc_importance_task_hold_internal_assertion_locked
+ * Purpose:
+ * Increment the assertion count on the task importance.
+ * If this results in a boost state change in that task,
+ * prepare to update task policy for this task AND, if
+ * if not just waking out of App Nap, all down-stream
+ * tasks that have a similar transition through inheriting
+ * this update.
+ * Conditions:
+ * importance locked on entry and exit.
+ * May temporarily drop importance lock and block.
+ */
+static kern_return_t
+ipc_importance_task_hold_internal_assertion_locked(ipc_importance_task_t task_imp, uint32_t count)
+{
+ if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_HOLD, count)) {
+ ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_HOLD, TRUE);
+ }
+ return KERN_SUCCESS;
+}
+
+/*
+ * Routine: ipc_importance_task_drop_internal_assertion_locked
+ * Purpose:
+ * Decrement the assertion count on the task importance.
+ * If this results in a boost state change in that task,
+ * prepare to update task policy for this task AND, if
+ * if not just waking out of App Nap, all down-stream
+ * tasks that have a similar transition through inheriting
+ * this update.
+ * Conditions:
+ * importance locked on entry and exit.
+ * May temporarily drop importance lock and block.
+ */
+static kern_return_t
+ipc_importance_task_drop_internal_assertion_locked(ipc_importance_task_t task_imp, uint32_t count)
+{
+ if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) {
+ ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE);
+ }
+ return KERN_SUCCESS;
+}
+
+/*
+ * Routine: ipc_importance_task_hold_internal_assertion
+ * Purpose:
+ * Increment the assertion count on the task importance.
+ * If this results in a 0->1 change in that count,
+ * prepare to update task policy for this task AND
+ * (potentially) all down-stream tasks that have a
+ * similar transition through inheriting this update.
+ * Conditions:
+ * Nothing locked
+ * May block after dropping importance lock.
+ */
+int
+ipc_importance_task_hold_internal_assertion(ipc_importance_task_t task_imp, uint32_t count)
+{
+ int ret = KERN_SUCCESS;
+
+ if (ipc_importance_task_is_any_receiver_type(task_imp)) {
+ ipc_importance_lock();
+ ret = ipc_importance_task_hold_internal_assertion_locked(task_imp, count);
+ ipc_importance_unlock();
+ }
+ return ret;
+}
+
+/*
+ * Routine: ipc_importance_task_drop_internal_assertion
+ * Purpose:
+ * Decrement the assertion count on the task importance.
+ * If this results in a X->0 change in that count,
+ * prepare to update task policy for this task AND
+ * all down-stream tasks that have a similar transition
+ * through inheriting this drop update.
+ * Conditions:
+ * Nothing locked on entry.
+ * May block after dropping importance lock.
+ */
+kern_return_t
+ipc_importance_task_drop_internal_assertion(ipc_importance_task_t task_imp, uint32_t count)
+{
+ kern_return_t ret = KERN_SUCCESS;
+
+ if (ipc_importance_task_is_any_receiver_type(task_imp)) {
+ ipc_importance_lock();
+ ret = ipc_importance_task_drop_internal_assertion_locked(task_imp, count);
+ ipc_importance_unlock();
+ }
+ return ret;
+}
+
+/*
+ * Routine: ipc_importance_task_hold_file_lock_assertion
+ * Purpose:
+ * Increment the file lock assertion count on the task importance.
+ * If this results in a 0->1 change in that count,
+ * prepare to update task policy for this task AND
+ * (potentially) all down-stream tasks that have a
+ * similar transition through inheriting this update.
+ * Conditions:
+ * Nothing locked
+ * May block after dropping importance lock.
+ */
+kern_return_t
+ipc_importance_task_hold_file_lock_assertion(ipc_importance_task_t task_imp, uint32_t count)
+{
+ kern_return_t ret = KERN_SUCCESS;
+
+ if (ipc_importance_task_is_any_receiver_type(task_imp)) {
+ ipc_importance_lock();
+ ret = ipc_importance_task_hold_internal_assertion_locked(task_imp, count);
+ if (KERN_SUCCESS == ret) {
+ task_imp->iit_filelocks += count;
+ }
+ ipc_importance_unlock();
+ }
+ return ret;
+}
+
+/*
+ * Routine: ipc_importance_task_drop_file_lock_assertion
+ * Purpose:
+ * Decrement the assertion count on the task importance.
+ * If this results in a X->0 change in that count,
+ * prepare to update task policy for this task AND
+ * all down-stream tasks that have a similar transition
+ * through inheriting this drop update.
+ * Conditions:
+ * Nothing locked on entry.
+ * May block after dropping importance lock.
+ */
+kern_return_t
+ipc_importance_task_drop_file_lock_assertion(ipc_importance_task_t task_imp, uint32_t count)
+{
+ kern_return_t ret = KERN_SUCCESS;
+
+ if (ipc_importance_task_is_any_receiver_type(task_imp)) {
+ ipc_importance_lock();
+ if (count <= task_imp->iit_filelocks) {
+ task_imp->iit_filelocks -= count;
+ ret = ipc_importance_task_drop_internal_assertion_locked(task_imp, count);
+ } else {
+ ret = KERN_INVALID_ARGUMENT;
+ }
+ ipc_importance_unlock();
+ }
+ return ret;
+}
+
+/*
+ * Routine: ipc_importance_task_hold_legacy_external_assertion
+ * Purpose:
+ * Increment the external assertion count on the task importance.
+ * This cannot result in an 0->1 transition, as the caller must
+ * already hold an external boost.
+ * Conditions:
+ * Nothing locked on entry.
+ * May block after dropping importance lock.
+ * A queue of task importance structures is returned
+ * by ipc_importance_task_hold_assertion_locked(). Each
+ * needs to be updated (outside the importance lock hold).
+ */
+kern_return_t
+ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_imp, uint32_t count)
+{
+ task_t target_task;
+ uint32_t target_assertcnt;
+ uint32_t target_externcnt;
+ uint32_t target_legacycnt;
+
+ kern_return_t ret;
+
+ ipc_importance_lock();
+ target_task = task_imp->iit_task;
+
+#if IMPORTANCE_DEBUG
+ int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
+#endif
+
+ if (IIT_LEGACY_EXTERN(task_imp) == 0) {
+ /* Only allowed to take a new boost assertion when holding an external boost */
+ /* save data for diagnostic printf below */
+ target_assertcnt = task_imp->iit_assertcnt;
+ target_externcnt = IIT_EXTERN(task_imp);
+ target_legacycnt = IIT_LEGACY_EXTERN(task_imp);
+ ret = KERN_FAILURE;
+ count = 0;
+ } else {
+ assert(ipc_importance_task_is_any_receiver_type(task_imp));
+ assert(0 < task_imp->iit_assertcnt);
+ assert(0 < IIT_EXTERN(task_imp));
+ task_imp->iit_assertcnt += count;
+ task_imp->iit_externcnt += count;
+ task_imp->iit_legacy_externcnt += count;
+ ret = KERN_SUCCESS;
+ }
+ ipc_importance_unlock();
+
+#if IMPORTANCE_DEBUG
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
+ // This covers the legacy case where a task takes an extra boost.
+ DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, proc_selfpid(), int, count, int, task_imp->iit_assertcnt);
+#endif
+
+ if (KERN_FAILURE == ret && target_task != TASK_NULL) {
+ printf("BUG in process %s[%d]: "
+ "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. "
+ "(%d total, %d external, %d legacy-external)\n",
+ proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task),
+ target_assertcnt, target_externcnt, target_legacycnt);
+ }
+
+ return(ret);
+}
+
+/*
+ * Routine: ipc_importance_task_drop_legacy_external_assertion
+ * Purpose:
+ * Drop the legacy external assertion count on the task and
+ * reflect that change to total external assertion count and
+ * then onto the internal importance count.
+ *
+ * If this results in a X->0 change in the internal,
+ * count, prepare to update task policy for this task AND
+ * all down-stream tasks that have a similar transition
+ * through inheriting this update.
+ * Conditions:
+ * Nothing locked on entry.
+ */
+kern_return_t
+ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_imp, uint32_t count)
+{
+ int ret = KERN_SUCCESS;
+ task_t target_task;
+ uint32_t target_assertcnt;
+ uint32_t target_externcnt;
+ uint32_t target_legacycnt;
+
+ if (count > 1) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ ipc_importance_lock();
+ target_task = task_imp->iit_task;
+
+#if IMPORTANCE_DEBUG
+ int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
+#endif
+
+ if (count > IIT_LEGACY_EXTERN(task_imp)) {
+ /* Process over-released its boost count - save data for diagnostic printf */
+ /* TODO: If count > 1, we should clear out as many external assertions as there are left. */
+ target_assertcnt = task_imp->iit_assertcnt;
+ target_externcnt = IIT_EXTERN(task_imp);
+ target_legacycnt = IIT_LEGACY_EXTERN(task_imp);
+ ret = KERN_FAILURE;
+ } else {
+ /*
+ * decrement legacy external count from the top level and reflect
+ * into internal for this and all subsequent updates.
+ */
+ assert(ipc_importance_task_is_any_receiver_type(task_imp));
+ assert(IIT_EXTERN(task_imp) >= count);
+
+ task_imp->iit_legacy_externdrop += count;
+ task_imp->iit_externdrop += count;
+
+ /* reset extern counters (if appropriate) */
+ if (IIT_LEGACY_EXTERN(task_imp) == 0) {
+ if (IIT_EXTERN(task_imp) != 0) {
+ task_imp->iit_externcnt -= task_imp->iit_legacy_externcnt;
+ task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop;
+ } else {
+ task_imp->iit_externcnt = 0;
+ task_imp->iit_externdrop = 0;
+ }
+ task_imp->iit_legacy_externcnt = 0;
+ task_imp->iit_legacy_externdrop = 0;
+ }
+
+ /* reflect the drop to the internal assertion count (and effect any importance change) */
+ if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) {
+ ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE);
+ }
+ ret = KERN_SUCCESS;
+ }
+
+#if IMPORTANCE_DEBUG
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
+#endif
+
+ ipc_importance_unlock();
+
+ /* delayed printf for user-supplied data failures */
+ if (KERN_FAILURE == ret && TASK_NULL != target_task) {
+ printf("BUG in process %s[%d]: over-released legacy external boost assertions (%d total, %d external, %d legacy-external)\n",
+ proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task),
+ target_assertcnt, target_externcnt, target_legacycnt);
+ }
+
+ return(ret);
+}
+
+
+
+/* Transfer an assertion to legacy userspace responsibility */
+static kern_return_t
+ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, uint32_t count, __unused int sender_pid)
+{
+ task_t target_task;
+
+ assert(IIT_NULL != task_imp);
+ target_task = task_imp->iit_task;
+
+ if (TASK_NULL == target_task ||
+ !ipc_importance_task_is_any_receiver_type(task_imp)) {
+ return KERN_FAILURE;
+ }
+
+#if IMPORTANCE_DEBUG
+ int target_pid = audit_token_pid_from_task(target_task);
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_START,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0);
+#endif
+
+ ipc_importance_lock();
+ /* assert(task_imp->iit_assertcnt >= IIT_EXTERN(task_imp) + count); */
+ assert(IIT_EXTERN(task_imp) >= IIT_LEGACY_EXTERN(task_imp));
+ task_imp->iit_legacy_externcnt += count;
+ task_imp->iit_externcnt += count;
+ ipc_importance_unlock();
+
+#if IMPORTANCE_DEBUG
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_END,
+ proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
+ // This is the legacy boosting path
+ DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, sender_pid, int, count, int, IIT_LEGACY_EXTERN(task_imp));
+#endif /* IMPORTANCE_DEBUG */
+
+ return(KERN_SUCCESS);
+}
+
+/*
+ * Routine: ipc_importance_task_update_live_donor
+ * Purpose:
+ * Read the live donor status and update the live_donor bit/propagate the change in importance.
+ * Conditions:
+ * Nothing locked on entrance, nothing locked on exit.
+ *
+ * TODO: Need tracepoints around this function...
+ */
+void
+ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp)
+{
+ uint32_t task_live_donor;
+ boolean_t before_donor;
+ boolean_t after_donor;
+ task_t target_task;
+
+ assert(task_imp != NULL);
+
+ /*
+ * Nothing to do if the task is not marked as expecting
+ * live donor updates.
+ */
+ if (!ipc_importance_task_is_marked_live_donor(task_imp)) {
+ return;
+ }
+
+ ipc_importance_lock();
+
+ /* If the task got disconnected on the way here, no use (or ability) adjusting live donor status */
+ target_task = task_imp->iit_task;
+ if (TASK_NULL == target_task) {
+ ipc_importance_unlock();
+ return;
+ }
+ before_donor = ipc_importance_task_is_marked_donor(task_imp);
+
+ /* snapshot task live donor status - may change, but another call will accompany the change */
+ task_live_donor = target_task->effective_policy.t_live_donor;
+
+#if IMPORTANCE_DEBUG
+ int target_pid = audit_token_pid_from_task(target_task);
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_START,
+ target_pid, task_imp->iit_donor, task_live_donor, before_donor, 0);
+#endif
+
+ /* update the task importance live donor status based on the task's value */
+ task_imp->iit_donor = task_live_donor;
+
+ after_donor = ipc_importance_task_is_marked_donor(task_imp);
+
+ /* Has the effectiveness of being a donor changed as a result of this update? */
+ if (before_donor != after_donor) {
+ iit_update_type_t type;
+
+ /* propagate assertions without updating the current task policy (already handled) */
+ if (0 == before_donor) {
+ task_imp->iit_transitions++;
+ type = IIT_UPDATE_HOLD;
+ } else {
+ type = IIT_UPDATE_DROP;
+ }
+ ipc_importance_task_propagate_assertion_locked(task_imp, type, FALSE);
+ }
+
+#if IMPORTANCE_DEBUG
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END,
+ target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0);
+#endif
+
+ ipc_importance_unlock();
+}
+
+
+/*
+ * Routine: ipc_importance_task_mark_donor
+ * Purpose:
+ * Set the task importance donor flag.
+ * Conditions:
+ * Nothing locked on entrance, nothing locked on exit.
+ *
+ * This is only called while the task is being constructed,
+ * so no need to update task policy or propagate downstream.
+ */
+void
+ipc_importance_task_mark_donor(ipc_importance_task_t task_imp, boolean_t donating)
+{
+ assert(task_imp != NULL);
+
+ ipc_importance_lock();
+
+ int old_donor = task_imp->iit_donor;
+
+ task_imp->iit_donor = (donating ? 1 : 0);
+
+ if (task_imp->iit_donor > 0 && old_donor == 0)
+ task_imp->iit_transitions++;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_INIT_DONOR_STATE)) | DBG_FUNC_NONE,
+ audit_token_pid_from_task(task_imp->iit_task), donating,
+ old_donor, task_imp->iit_donor, 0);
+
+ ipc_importance_unlock();
+}
+
+/*
+ * Routine: ipc_importance_task_marked_donor
+ * Purpose:
+ * Query the donor flag for the given task importance.
+ * Conditions:
+ * May be called without taking the importance lock.
+ * In that case, donor status can change so you must
+ * check only once for each donation event.
+ */
+boolean_t
+ipc_importance_task_is_marked_donor(ipc_importance_task_t task_imp)
+{
+ if (IIT_NULL == task_imp) {
+ return FALSE;
+ }
+ return (0 != task_imp->iit_donor);
+}
+
+/*
+ * Routine: ipc_importance_task_mark_live_donor
+ * Purpose:
+ * Indicate that the task is eligible for live donor updates.
+ * Conditions:
+ * Nothing locked on entrance, nothing locked on exit.
+ *
+ * This is only called while the task is being constructed.
+ */
+void
+ipc_importance_task_mark_live_donor(ipc_importance_task_t task_imp, boolean_t live_donating)
+{
+ assert(task_imp != NULL);
+
+ ipc_importance_lock();
+ task_imp->iit_live_donor = (live_donating ? 1 : 0);
+ ipc_importance_unlock();
+}
+
+/*
+ * Routine: ipc_importance_task_marked_live_donor
+ * Purpose:
+ * Query the live donor and donor flags for the given task importance.
+ * Conditions:
+ * May be called without taking the importance lock.
+ * In that case, donor status can change so you must
+ * check only once for each donation event.
+ */
+boolean_t
+ipc_importance_task_is_marked_live_donor(ipc_importance_task_t task_imp)
+{
+ if (IIT_NULL == task_imp) {
+ return FALSE;
+ }
+ return (0 != task_imp->iit_live_donor);
+}
+
+/*
+ * Routine: ipc_importance_task_is_donor
+ * Purpose:
+ * Query the full donor status for the given task importance.
+ * Conditions:
+ * May be called without taking the importance lock.
+ * In that case, donor status can change so you must
+ * check only once for each donation event.
+ */
+boolean_t
+ipc_importance_task_is_donor(ipc_importance_task_t task_imp)
+{
+ if (IIT_NULL == task_imp) {
+ return FALSE;
+ }
+ return (ipc_importance_task_is_marked_donor(task_imp) ||
+ (ipc_importance_task_is_marked_receiver(task_imp) &&
+ task_imp->iit_assertcnt > 0));
+}
+
+/*
+ * Routine: ipc_importance_task_is_never_donor
+ * Purpose:
+ * Query if a given task can ever donate importance.
+ * Conditions:
+ * May be called without taking the importance lock.
+ * Condition is permanent for a give task.
+ */
+boolean_t
+ipc_importance_task_is_never_donor(ipc_importance_task_t task_imp)
+{
+ if (IIT_NULL == task_imp) {
+ return FALSE;
+ }
+ return (!ipc_importance_task_is_marked_donor(task_imp) &&
+ !ipc_importance_task_is_marked_live_donor(task_imp) &&
+ !ipc_importance_task_is_marked_receiver(task_imp));
+}
+
+/*
+ * Routine: ipc_importance_task_mark_receiver
+ * Purpose:
+ * Update the task importance receiver flag.
+ * Conditions:
+ * Nothing locked on entrance, nothing locked on exit.
+ * This can only be invoked before the task is discoverable,
+ * so no worries about atomicity(?)
+ */
+void
+ipc_importance_task_mark_receiver(ipc_importance_task_t task_imp, boolean_t receiving)
+{
+ assert(task_imp != NULL);
+
+ ipc_importance_lock();
+ if (receiving) {
+ assert(task_imp->iit_assertcnt == 0);
+ assert(task_imp->iit_externcnt == 0);
+ assert(task_imp->iit_externdrop == 0);
+ assert(task_imp->iit_denap == 0);
+ task_imp->iit_receiver = 1; /* task can receive importance boost */
+ } else if (task_imp->iit_receiver) {
+ assert(task_imp->iit_denap == 0);
+ if (task_imp->iit_assertcnt != 0 || IIT_EXTERN(task_imp) != 0) {
+ panic("disabling imp_receiver on task with pending importance boosts!");
+ }
+ task_imp->iit_receiver = 0;
+ }
+ ipc_importance_unlock();
+}
+
+
+/*
+ * Routine: ipc_importance_task_marked_receiver
+ * Purpose:
+ * Query the receiver flag for the given task importance.
+ * Conditions:
+ * May be called without taking the importance lock as
+ * the importance flag can never change after task init.
+ */
+boolean_t
+ipc_importance_task_is_marked_receiver(ipc_importance_task_t task_imp)
+{
+ return (IIT_NULL != task_imp && 0 != task_imp->iit_receiver);
+}
+
+
+/*
+ * Routine: ipc_importance_task_mark_denap_receiver
+ * Purpose:
+ * Update the task importance de-nap receiver flag.
+ * Conditions:
+ * Nothing locked on entrance, nothing locked on exit.
+ * This can only be invoked before the task is discoverable,
+ * so no worries about atomicity(?)
+ */
+void
+ipc_importance_task_mark_denap_receiver(ipc_importance_task_t task_imp, boolean_t denap)
+{
+ assert(task_imp != NULL);
+
+ ipc_importance_lock();
+ if (denap) {
+ assert(task_imp->iit_assertcnt == 0);
+ assert(task_imp->iit_externcnt == 0);
+ assert(task_imp->iit_receiver == 0);
+ task_imp->iit_denap = 1; /* task can receive de-nap boost */
+ } else if (task_imp->iit_denap) {
+ assert(task_imp->iit_receiver == 0);
+ if (0 < task_imp->iit_assertcnt || 0 < IIT_EXTERN(task_imp)) {
+ panic("disabling de-nap on task with pending de-nap boosts!");
+ }
+ task_imp->iit_denap = 0;
+ }
+ ipc_importance_unlock();
+}
+
+
+/*
+ * Routine: ipc_importance_task_marked_denap_receiver
+ * Purpose:
+ * Query the de-nap receiver flag for the given task importance.
+ * Conditions:
+ * May be called without taking the importance lock as
+ * the de-nap flag can never change after task init.
+ */
+boolean_t
+ipc_importance_task_is_marked_denap_receiver(ipc_importance_task_t task_imp)
+{
+ return (IIT_NULL != task_imp && 0 != task_imp->iit_denap);
+}
+
+/*
+ * Routine: ipc_importance_task_is_denap_receiver
+ * Purpose:
+ * Query the full de-nap receiver status for the given task importance.
+ * For now, that is simply whether the receiver flag is set.
+ * Conditions:
+ * May be called without taking the importance lock as
+ * the de-nap receiver flag can never change after task init.
+ */
+boolean_t
+ipc_importance_task_is_denap_receiver(ipc_importance_task_t task_imp)
+{
+ return (ipc_importance_task_is_marked_denap_receiver(task_imp));
+}
+
+/*
+ * Routine: ipc_importance_task_is_any_receiver_type
+ * Purpose:
+ * Query if the task is marked to receive boosts - either
+ * importance or denap.
+ * Conditions:
+ * May be called without taking the importance lock as both
+ * the importance and de-nap receiver flags can never change
+ * after task init.
+ */
+boolean_t
+ipc_importance_task_is_any_receiver_type(ipc_importance_task_t task_imp)
+{
+ return (ipc_importance_task_is_marked_receiver(task_imp) ||
+ ipc_importance_task_is_marked_denap_receiver(task_imp));
+}
+
+#if 0 /* currently unused */
+
+/*
+ * Routine: ipc_importance_inherit_reference
+ * Purpose:
+ * Add a reference to the inherit importance element.
+ * Conditions:
+ * Caller most hold a reference on the inherit element.
+ */
+static inline void
+ipc_importance_inherit_reference(ipc_importance_inherit_t inherit)
+{
+ ipc_importance_reference(&inherit->iii_elem);
+}
+#endif /* currently unused */
+
+/*
+ * Routine: ipc_importance_inherit_release_locked
+ * Purpose:
+ * Release a reference on an inherit importance attribute value,
+ * unlinking and deallocating the attribute if the last reference.
+ * Conditions:
+ * Entered with importance lock held, leaves with it unlocked.
+ */
+static inline void
+ipc_importance_inherit_release_locked(ipc_importance_inherit_t inherit)
+{
+ ipc_importance_release_locked(&inherit->iii_elem);
+}
+
+#if 0 /* currently unused */
+/*
+ * Routine: ipc_importance_inherit_release
+ * Purpose:
+ * Release a reference on an inherit importance attribute value,
+ * unlinking and deallocating the attribute if the last reference.
+ * Conditions:
+ * nothing locked on entrance, nothing locked on exit.
+ * May block.
+ */
+void
+ipc_importance_inherit_release(ipc_importance_inherit_t inherit)
+{
+ if (III_NULL != inherit)
+ ipc_importance_release(&inherit->iii_elem);
+}
+#endif /* 0 currently unused */
+
+/*
+ * Routine: ipc_importance_for_task
+ * Purpose:
+ * Create a reference for the specified task's base importance
+ * element. If the base importance element doesn't exist, make it and
+ * bind it to the active task. If the task is inactive, there isn't
+ * any need to return a new reference.
+ * Conditions:
+ * If made is true, a "made" reference is returned (for donating to
+ * the voucher system). Otherwise an internal reference is returned.
+ *
+ * Nothing locked on entry. May block.
+ */
+ipc_importance_task_t
+ipc_importance_for_task(task_t task, boolean_t made)
+{
+ ipc_importance_task_t task_elem;
+ boolean_t first_pass = TRUE;
+
+ assert(TASK_NULL != task);
+
+ retry:
+ /* No use returning anything for inactive task */
+ if (!task->active)
+ return IIT_NULL;
+
+ ipc_importance_lock();
+ task_elem = task->task_imp_base;
+ if (IIT_NULL != task_elem) {
+ /* Add a made reference (borrowing active task ref to do it) */
+ if (made) {
+ if (0 == task_elem->iit_made++) {
+ assert(IIT_REFS_MAX > IIT_REFS(task_elem));
+ ipc_importance_task_reference_internal(task_elem);
+ }
+ } else {
+ assert(IIT_REFS_MAX > IIT_REFS(task_elem));
+ ipc_importance_task_reference_internal(task_elem);
+ }
+ ipc_importance_unlock();
+ return task_elem;
+ }
+ ipc_importance_unlock();
+
+ if (!first_pass)
+ return IIT_NULL;
+ first_pass = FALSE;
+
+ /* Need to make one - may race with others (be prepared to drop) */
+ task_elem = (ipc_importance_task_t)zalloc(ipc_importance_task_zone);
+ if (IIT_NULL == task_elem)
+ goto retry;
+
+ task_elem->iit_bits = IIE_TYPE_TASK | 2; /* one for task, one for return/made */
+ task_elem->iit_made = (made) ? 1 : 0;
+ task_elem->iit_task = task; /* take actual ref when we're sure */
+ task_elem->iit_updateq = NULL;
+ task_elem->iit_receiver = 0;
+ task_elem->iit_denap = 0;
+ task_elem->iit_donor = 0;
+ task_elem->iit_live_donor = 0;
+ task_elem->iit_updatepolicy = 0;
+ task_elem->iit_reserved = 0;
+ task_elem->iit_filelocks = 0;
+ task_elem->iit_updatetime = 0;
+ task_elem->iit_transitions = 0;
+ task_elem->iit_assertcnt = 0;
+ task_elem->iit_externcnt = 0;
+ task_elem->iit_externdrop = 0;
+ task_elem->iit_legacy_externcnt = 0;
+ task_elem->iit_legacy_externdrop = 0;
+#if IIE_REF_DEBUG
+ ipc_importance_counter_init(&task_elem->iit_elem);
+#endif
+ queue_init(&task_elem->iit_kmsgs);
+ queue_init(&task_elem->iit_inherits);
+
+ ipc_importance_lock();
+ if (!task->active) {
+ ipc_importance_unlock();
+ zfree(ipc_importance_task_zone, task_elem);
+ return IIT_NULL;
+ }
+
+ /* did we lose the race? */
+ if (IIT_NULL != task->task_imp_base) {
+ ipc_importance_unlock();
+ zfree(ipc_importance_task_zone, task_elem);
+ goto retry;
+ }
+
+ /* we won the race */
+ task->task_imp_base = task_elem;
+ task_reference(task);
+#if DEVELOPMENT || DEBUG
+ queue_enter(&global_iit_alloc_queue, task_elem, ipc_importance_task_t, iit_allocation);
+ task_importance_update_owner_info(task);
+#endif
+ ipc_importance_unlock();
+
+ return task_elem;
+}
+
+#if DEVELOPMENT || DEBUG
+void task_importance_update_owner_info(task_t task) {
+
+ if (task != TASK_NULL && task->task_imp_base != IIT_NULL) {
+ ipc_importance_task_t task_elem = task->task_imp_base;
+
+ task_elem->iit_bsd_pid = audit_token_pid_from_task(task);
+ if (task->bsd_info) {
+ strncpy(&task_elem->iit_procname[0], proc_name_address(task->bsd_info), 16);
+ task_elem->iit_procname[16] = '\0';
+ } else {
+ strncpy(&task_elem->iit_procname[0], "unknown", 16);
+ }
+ }
+}
+#endif
+
+/*
+ * Routine: ipc_importance_reset_locked
+ * Purpose:
+ * Reset a task's IPC importance (the task is going away or exec'ing)
+ *
+ * Remove the donor bit and legacy externalized assertions from the
+ * current task importance and see if that wipes out downstream donations.
+ * Conditions:
+ * importance lock held.
+ */
+
+static void
+ipc_importance_reset_locked(ipc_importance_task_t task_imp, boolean_t donor)
+{
+ boolean_t before_donor, after_donor;
+
+ /* remove the donor bit, live-donor bit and externalized boosts */
+ before_donor = ipc_importance_task_is_donor(task_imp);
+ if (donor) {
+ task_imp->iit_donor = 0;
+ }
+ assert(IIT_LEGACY_EXTERN(task_imp) <= IIT_EXTERN(task_imp));
+ assert(task_imp->iit_legacy_externcnt <= task_imp->iit_externcnt);
+ assert(task_imp->iit_legacy_externdrop <= task_imp->iit_externdrop);
+ task_imp->iit_externcnt -= task_imp->iit_legacy_externcnt;
+ task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop;
+
+ /* assert(IIT_LEGACY_EXTERN(task_imp) <= task_imp->iit_assertcnt); */
+ if (IIT_LEGACY_EXTERN(task_imp) < task_imp->iit_assertcnt) {
+ task_imp->iit_assertcnt -= IIT_LEGACY_EXTERN(task_imp);
+ } else {
+ assert(IIT_LEGACY_EXTERN(task_imp) == task_imp->iit_assertcnt);
+ task_imp->iit_assertcnt = 0;
+ }
+ task_imp->iit_legacy_externcnt = 0;
+ task_imp->iit_legacy_externdrop = 0;
+ after_donor = ipc_importance_task_is_donor(task_imp);
+
+#if DEVELOPMENT || DEBUG
+ if (task_imp->iit_assertcnt > 0 && task_imp->iit_live_donor) {
+ printf("Live donor task %s[%d] still has %d importance assertions after reset\n",
+ task_imp->iit_procname, task_imp->iit_bsd_pid, task_imp->iit_assertcnt);
+ }
+#endif
+
+ /* propagate a downstream drop if there was a change in donor status */
+ if (after_donor != before_donor) {
+ ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, FALSE);
+ }
+}
+
+/*
+ * Routine: ipc_importance_reset
+ * Purpose:
+ * Reset a task's IPC importance
+ *
+ * The task is being reset, although staying around. Arrange to have the
+ * external state of the task reset from the importance.
+ * Conditions:
+ * importance lock not held.
+ */
+
+void
+ipc_importance_reset(ipc_importance_task_t task_imp, boolean_t donor)
+{
+ if (IIT_NULL == task_imp) {
+ return;
+ }
+ ipc_importance_lock();
+ ipc_importance_reset_locked(task_imp, donor);
+ ipc_importance_unlock();
+}
+
+/*
+ * Routine: ipc_importance_disconnect_task
+ * Purpose:
+ * Disconnect a task from its importance.
+ *
+ * Clear the task pointer from the importance and drop the
+ * reference the task held on the importance object. Before
+ * doing that, reset the effects the current task holds on
+ * the importance and see if that wipes out downstream donations.
+ *
+ * We allow the upstream boosts to continue to affect downstream
+ * even though the local task is being effectively pulled from
+ * the chain.
+ * Conditions:
+ * Nothing locked.
+ */
+void
+ipc_importance_disconnect_task(task_t task)
+{
+ ipc_importance_task_t task_imp;
+
+ task_lock(task);
+ ipc_importance_lock();
+ task_imp = task->task_imp_base;
+
+ /* did somebody beat us to it? */
+ if (IIT_NULL == task_imp) {
+ ipc_importance_unlock();
+ task_unlock(task);
+ return;
+ }
+
+ /* disconnect the task from this importance */
+ assert(task_imp->iit_task == task);
+ task_imp->iit_task = TASK_NULL;
+ task->task_imp_base = IIT_NULL;
+ task_unlock(task);
+
+ /* reset the effects the current task hold on the importance */
+ ipc_importance_reset_locked(task_imp, TRUE);
+
+ ipc_importance_task_release_locked(task_imp);
+ /* importance unlocked */
+
+ /* deallocate the task now that the importance is unlocked */
+ task_deallocate(task);
+}
+
+/*
+ * Routine: ipc_importance_send
+ * Purpose:
+ * Post the importance voucher attribute [if sent] or a static
+ * importance boost depending upon options and conditions.
+ * Conditions:
+ * Destination port locked on entry and exit, may be dropped during the call.
+ * Returns:
+ * A boolean identifying if the port lock was tempoarily dropped.
+ */
+boolean_t
+ipc_importance_send(
+ ipc_kmsg_t kmsg,
+ mach_msg_option_t option)
+{
+ ipc_port_t port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
+ boolean_t port_lock_dropped = FALSE;
+ ipc_importance_elem_t elem;
+ task_t task;
+ ipc_importance_task_t task_imp;
+ kern_return_t kr;
+
+
+ assert(IP_VALID(port));
+
+ /* If no donation to be made, return quickly */
+ if ((port->ip_impdonation == 0) ||
+ (option & MACH_SEND_NOIMPORTANCE) != 0) {
+ return port_lock_dropped;
+ }
+
+ task = current_task();
+
+ /* If forced sending a static boost, go update the port */
+ if ((option & MACH_SEND_IMPORTANCE) != 0) {
+ kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
+ goto portupdate;
+ }
+
+ task_imp = task->task_imp_base;
+ assert(IIT_NULL != task_imp);
+
+ /* If the sender can never donate importance, nothing to do */
+ if (ipc_importance_task_is_never_donor(task_imp)) {
+ return port_lock_dropped;
+ }
+
+ elem = IIE_NULL;
+
+ /* If importance receiver and passing a voucher, look for importance in there */
+ if (IP_VALID(kmsg->ikm_voucher) &&
+ ipc_importance_task_is_marked_receiver(task_imp)) {
+ mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED];
+ mach_voucher_attr_value_handle_array_size_t val_count;
+ ipc_voucher_t voucher;
+
+ assert(ip_kotype(kmsg->ikm_voucher) == IKOT_VOUCHER);
+ voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+
+ /* check to see if the voucher has an importance attribute */
+ val_count = MACH_VOUCHER_ATTR_VALUE_MAX_NESTED;
+ kr = mach_voucher_attr_control_get_values(ipc_importance_control, voucher,
+ vals, &val_count);
+ assert(KERN_SUCCESS == kr);
+
+ /*
+ * Only use importance associated with our task (either directly
+ * or through an inherit that donates to our task).
+ */
+ if (0 < val_count) {
+ ipc_importance_elem_t check_elem;
+
+ check_elem = (ipc_importance_elem_t)vals[0];
+ assert(IIE_NULL != check_elem);
+ if (IIE_TYPE_INHERIT == IIE_TYPE(check_elem)) {
+ ipc_importance_inherit_t inherit;
+ inherit = (ipc_importance_inherit_t) check_elem;
+ if (inherit->iii_to_task == task_imp) {
+ elem = check_elem;
+ }
+ } else if (check_elem == (ipc_importance_elem_t)task_imp) {
+ elem = check_elem;
+ }
+ }
+ }
+
+ /* If we haven't found an importance attribute to send yet, use the task's */
+ if (IIE_NULL == elem) {
+ elem = (ipc_importance_elem_t)task_imp;
+ }
+
+ /* take a reference for the message to hold */
+ ipc_importance_reference_internal(elem);
+
+ /* acquire the importance lock while trying to hang on to port lock */
+ if (!ipc_importance_lock_try()) {
+ port_lock_dropped = TRUE;
+ ip_unlock(port);
+ ipc_importance_lock();
+ }
+
+ /* link kmsg onto the donor element propagation chain */
+ ipc_importance_kmsg_link(kmsg, elem);
+ /* elem reference transfered to kmsg */
+
+ incr_ref_counter(elem->iie_kmsg_refs_added);
+
+ /* If the sender isn't currently a donor, no need to apply boost */
+ if (!ipc_importance_task_is_donor(task_imp)) {
+ ipc_importance_unlock();
+
+ /* re-acquire port lock, if needed */
+ if (TRUE == port_lock_dropped)
+ ip_lock(port);
+
+ return port_lock_dropped;
+ }
+
+ /* Mark the fact that we are (currently) donating through this message */
+ kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
+
+ /*
+ * If we need to relock the port, do it with the importance still locked.
+ * This assures we get to add the importance boost through the port to
+ * the task BEFORE anyone else can attempt to undo that operation because
+ * the sender lost donor status.
+ */
+ if (TRUE == port_lock_dropped) {
+ ip_lock(port);
+ }
+ ipc_importance_unlock();
+
+ portupdate:
+
+#if IMPORTANCE_DEBUG
+ if (kdebug_enable) {
+ mach_msg_max_trailer_t *dbgtrailer = (mach_msg_max_trailer_t *)
+ ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size));
+ unsigned int sender_pid = dbgtrailer->msgh_audit.val[5];
+ mach_msg_id_t imp_msgh_id = kmsg->ikm_header->msgh_id;
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_START,
+ audit_token_pid_from_task(task), sender_pid, imp_msgh_id, 0, 0);
+ }
+#endif /* IMPORTANCE_DEBUG */
+
+ /* adjust port boost count (with port locked) */
+ if (TRUE == ipc_port_importance_delta(port, 1)) {
+ port_lock_dropped = TRUE;
+ ip_lock(port);
+ }
+ return port_lock_dropped;
+}
+
+/*
+ * Routine: ipc_importance_inherit_from
+ * Purpose:
+ * Create a "made" reference for an importance attribute representing
+ * an inheritance between the sender of a message (if linked) and the
+ * current task importance. If the message is not linked, a static
+ * boost may be created, based on the boost state of the message.
+ *
+ * Any transfer from kmsg linkage to inherit linkage must be atomic.
+ *
+ * If the task is inactive, there isn't any need to return a new reference.
+ * Conditions:
+ * Nothing locked on entry. May block.
+ */
+static ipc_importance_inherit_t
+ipc_importance_inherit_from(ipc_kmsg_t kmsg)
+{
+ ipc_importance_task_t task_imp = IIT_NULL;
+ ipc_importance_elem_t from_elem = kmsg->ikm_importance;
+ ipc_importance_elem_t elem;
+ task_t task_self = current_task();
+
+ ipc_port_t port = kmsg->ikm_header->msgh_remote_port;
+ ipc_importance_inherit_t inherit = III_NULL;
+ ipc_importance_inherit_t alloc = III_NULL;
+ ipc_importance_inherit_t temp_inherit;
+ boolean_t cleared_self_donation = FALSE;
+ boolean_t donating;
+ uint32_t depth = 1;
+
+ /* The kmsg must have an importance donor or static boost to proceed */
+ if (IIE_NULL == kmsg->ikm_importance &&
+ !MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
+ return III_NULL;
+ }
+
+ /*
+ * No need to set up an inherit linkage if the dest isn't a receiver
+ * of one type or the other.
+ */
+ if (!ipc_importance_task_is_any_receiver_type(task_self->task_imp_base)) {
+ ipc_importance_lock();
+ goto out_locked;
+ }
+
+ /* Grab a reference on the importance of the destination */
+ task_imp = ipc_importance_for_task(task_self, FALSE);
+
+ ipc_importance_lock();
+
+ if (IIT_NULL == task_imp) {
+ goto out_locked;
+ }
+
+ incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_inherit_from);
+
+ /* If message is already associated with an inherit... */
+ if (IIE_TYPE_INHERIT == IIE_TYPE(from_elem)) {
+ ipc_importance_inherit_t from_inherit = (ipc_importance_inherit_t)from_elem;
+
+ /* already targeting our task? - just use it */
+ if (from_inherit->iii_to_task == task_imp) {
+ /* clear self-donation if not also present in inherit */
+ if (!from_inherit->iii_donating &&
+ MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
+ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
+ cleared_self_donation = TRUE;
+ }
+ inherit = from_inherit;
+
+ } else if (III_DEPTH_MAX == III_DEPTH(from_inherit)) {
+ ipc_importance_task_t to_task;
+ ipc_importance_elem_t unlinked_from;
+
+ /*
+ * Chain too long. Switch to looking
+ * directly at the from_inherit's to-task
+ * as our source of importance.
+ */
+ to_task = from_inherit->iii_to_task;
+ ipc_importance_task_reference(to_task);
+ from_elem = (ipc_importance_elem_t)to_task;
+ depth = III_DEPTH_RESET | 1;
+
+ /* Fixup the kmsg linkage to reflect change */
+ unlinked_from = ipc_importance_kmsg_unlink(kmsg);
+ assert(unlinked_from == (ipc_importance_elem_t)from_inherit);
+ ipc_importance_kmsg_link(kmsg, from_elem);
+ ipc_importance_inherit_release_locked(from_inherit);
+ /* importance unlocked */
+ ipc_importance_lock();
+
+ } else {
+ /* inheriting from an inherit */
+ depth = from_inherit->iii_depth + 1;
+ }
+ }
+
+ /*
+ * Don't allow a task to inherit from itself (would keep it permanently
+ * boosted even if all other donors to the task went away).
+ */
+
+ if (from_elem == (ipc_importance_elem_t)task_imp) {
+ goto out_locked;
+ }
+
+ /*
+ * But if the message isn't associated with any linked source, it is
+ * intended to be permanently boosting (static boost from kernel).
+ * In that case DO let the process permanently boost itself.
+ */
+ if (IIE_NULL == from_elem) {
+ assert(MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits));
+ ipc_importance_task_reference_internal(task_imp);
+ from_elem = (ipc_importance_elem_t)task_imp;
+ }
+
+ /*
+ * Now that we have the from_elem figured out,
+ * check to see if we already have an inherit for this pairing
+ */
+ while (III_NULL == inherit) {
+ queue_iterate(&from_elem->iie_inherits, temp_inherit,
+ ipc_importance_inherit_t, iii_inheritance) {
+ if (temp_inherit->iii_to_task == task_imp &&
+ temp_inherit->iii_depth == depth) {
+ inherit = temp_inherit;
+ break;
+ }
+ }
+
+ /* Do we have to allocate a new inherit */
+ if (III_NULL == inherit) {
+ if (III_NULL != alloc) {
+ break;
+ }
+
+ /* allocate space */
+ ipc_importance_unlock();
+ alloc = (ipc_importance_inherit_t)
+ zalloc(ipc_importance_inherit_zone);
+ ipc_importance_lock();
+ }
+ }
+
+ /* snapshot the donating status while we have importance locked */
+ donating = MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits);
+
+ if (III_NULL != inherit) {
+ /* We found one, piggyback on that */
+ assert(0 < III_REFS(inherit));
+ assert(0 < IIE_REFS(inherit->iii_from_elem));
+ assert(inherit->iii_externcnt >= inherit->iii_made);
+
+ /* add in a made reference */
+ if (0 == inherit->iii_made++) {
+ assert(III_REFS_MAX > III_REFS(inherit));
+ ipc_importance_inherit_reference_internal(inherit);
+ }
+
+ /* Reflect the inherit's change of status into the task boosts */
+ if (0 == III_EXTERN(inherit)) {
+ assert(!inherit->iii_donating);
+ inherit->iii_donating = donating;
+ if (donating) {
+ task_imp->iit_externcnt += inherit->iii_externcnt;
+ task_imp->iit_externdrop += inherit->iii_externdrop;
+ }
+ } else {
+ assert(donating == inherit->iii_donating);
+ }
+
+ /* add in a external reference for this use of the inherit */
+ inherit->iii_externcnt++;
+ if (donating) {
+ task_imp->iit_externcnt++;
+ }
+ } else {
+ /* initialize the previously allocated space */
+ inherit = alloc;
+ inherit->iii_bits = IIE_TYPE_INHERIT | 1;
+ inherit->iii_made = 1;
+ inherit->iii_externcnt = 1;
+ inherit->iii_externdrop = 0;
+ inherit->iii_depth = depth;
+ inherit->iii_to_task = task_imp;
+ inherit->iii_from_elem = IIE_NULL;
+ queue_init(&inherit->iii_kmsgs);
+ queue_init(&inherit->iii_inherits);
+
+ /* If donating, reflect that in the task externcnt */
+ if (donating) {
+ inherit->iii_donating = TRUE;
+ task_imp->iit_externcnt++;
+ } else {
+ inherit->iii_donating = FALSE;
+ }
+
+ /*
+ * Chain our new inherit on the element it inherits from.
+ * The new inherit takes our reference on from_elem.
+ */
+ ipc_importance_inherit_link(inherit, from_elem);
+
+#if IIE_REF_DEBUG
+ ipc_importance_counter_init(&inherit->iii_elem);
+ from_elem->iie_kmsg_refs_inherited++;
+ task_imp->iit_elem.iie_task_refs_inherited++;
+#endif
+ }
+
+ out_locked:
+ /*
+ * for those paths that came straight here: snapshot the donating status
+ * (this should match previous snapshot for other paths).
+ */
+ donating = MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits);
+
+ /* unlink the kmsg inheritance (if any) */
+ elem = ipc_importance_kmsg_unlink(kmsg);
+ assert(elem == from_elem);
+
+ /* If we didn't create a new inherit, we have some resources to release */
+ if (III_NULL == inherit || inherit != alloc) {
+ if (IIE_NULL != from_elem) {
+ if (III_NULL != inherit) {
+ incr_ref_counter(from_elem->iie_kmsg_refs_coalesced);
+ } else {
+ incr_ref_counter(from_elem->iie_kmsg_refs_dropped);
+ }
+ ipc_importance_release_locked(from_elem);
+ /* importance unlocked */
+ } else {
+ ipc_importance_unlock();
+ }
+
+ if (IIT_NULL != task_imp) {
+ if (III_NULL != inherit) {
+ incr_ref_counter(task_imp->iit_elem.iie_task_refs_coalesced);
+ }
+ ipc_importance_task_release(task_imp);
+ }
+
+ if (III_NULL != alloc)
+ zfree(ipc_importance_inherit_zone, alloc);
+ } else {
+ /* from_elem and task_imp references transferred to new inherit */
+ ipc_importance_unlock();
+ }
+
+ /* decrement port boost count */
+ if (donating) {
+ ip_lock(port);
+ if (III_NULL != inherit) {
+ /* task assertions transferred to inherit, just adjust port count */
+ ipc_port_impcount_delta(port, -1, IP_NULL);
+ ip_unlock(port);
+ } else {
+ /* drop importance from port and destination task */
+ if (ipc_port_importance_delta(port, -1) == FALSE) {
+ ip_unlock(port);
+ }
+ }
+ } else if (cleared_self_donation) {
+ ip_lock(port);
+ /* drop cleared donation from port and destination task */
+ if (ipc_port_importance_delta(port, -1) == FALSE) {
+ ip_unlock(port);
+ }
+ }
+
+ if (III_NULL != inherit) {
+ /* have an associated importance attr, even if currently not donating */
+ kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
+ } else {
+ /* we won't have an importance attribute associated with our message */
+ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
+ }
+
+ return inherit;
+}
+
+/*
+ * Routine: ipc_importance_receive
+ * Purpose:
+ * Process importance attributes in a received message.
+ *
+ * If an importance voucher attribute was sent, transform
+ * that into an attribute value reflecting the inheritance
+ * from the sender to the receiver.
+ *
+ * If a static boost is received (or the voucher isn't on
+ * a voucher-based boost), export a static boost.
+ * Conditions:
+ * Nothing locked.
+ */
+void
+ipc_importance_receive(
+ ipc_kmsg_t kmsg,
+ mach_msg_option_t option)
+{
+ unsigned int sender_pid = ((mach_msg_max_trailer_t *)
+ ((vm_offset_t)kmsg->ikm_header +
+ round_msg(kmsg->ikm_header->msgh_size)))->msgh_audit.val[5];
+ task_t task_self = current_task();
+ int impresult = -1;
+
+ /* convert to a voucher with an inherit importance attribute? */
+ if ((option & MACH_RCV_VOUCHER) != 0) {
+ uint8_t recipes[2 * sizeof(ipc_voucher_attr_recipe_data_t) +
+ sizeof(mach_voucher_attr_value_handle_t)];
+ ipc_voucher_attr_raw_recipe_array_size_t recipe_size = 0;
+ ipc_voucher_attr_recipe_t recipe = (ipc_voucher_attr_recipe_t)recipes;
+ ipc_voucher_t recv_voucher;
+ mach_voucher_attr_value_handle_t handle;
+ ipc_importance_inherit_t inherit;
+ kern_return_t kr;
+
+ /* set up recipe to copy the old voucher */
+ if (IP_VALID(kmsg->ikm_voucher)) {
+ ipc_voucher_t sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+
+ recipe->key = MACH_VOUCHER_ATTR_KEY_ALL;
+ recipe->command = MACH_VOUCHER_ATTR_COPY;
+ recipe->previous_voucher = sent_voucher;
+ recipe->content_size = 0;
+ recipe_size += sizeof(*recipe);
+ }
+
+ /*
+ * create an inheritance attribute from the kmsg (may be NULL)
+ * transferring any boosts from the kmsg linkage through the
+ * port directly to the new inheritance object.
+ */
+ inherit = ipc_importance_inherit_from(kmsg);
+ handle = (mach_voucher_attr_value_handle_t)inherit;
+
+ assert(IIE_NULL == kmsg->ikm_importance);
+
+ /* replace the importance attribute with the handle we created */
+ /* our made reference on the inhert is donated to the voucher */
+ recipe = (ipc_voucher_attr_recipe_t)&recipes[recipe_size];
+ recipe->key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE;
+ recipe->command = MACH_VOUCHER_ATTR_SET_VALUE_HANDLE;
+ recipe->previous_voucher = IPC_VOUCHER_NULL;
+ recipe->content_size = sizeof(mach_voucher_attr_value_handle_t);
+ *(mach_voucher_attr_value_handle_t *)(void *)recipe->content = handle;
+ recipe_size += sizeof(*recipe) + sizeof(mach_voucher_attr_value_handle_t);
+
+ kr = ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control,
+ recipes,
+ recipe_size,
+ &recv_voucher);
+ assert(KERN_SUCCESS == kr);
+
+ /* swap the voucher port (and set voucher bits in case it didn't already exist) */
+ kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16);
+ ipc_port_release_send(kmsg->ikm_voucher);
+ kmsg->ikm_voucher = convert_voucher_to_port(recv_voucher);
+ if (III_NULL != inherit)
+ impresult = 2;
+
+ } else { /* Don't want a voucher */
+
+ /* got linked importance? have to drop */
+ if (IIE_NULL != kmsg->ikm_importance) {
+ ipc_importance_elem_t elem;
+
+ ipc_importance_lock();
+ elem = ipc_importance_kmsg_unlink(kmsg);
+#if IIE_REF_DEBUG
+ elem->iie_kmsg_refs_dropped++;
+#endif
+ ipc_importance_release_locked(elem);
+ /* importance unlocked */
+ }
+
+ /* With kmsg unlinked, can safely examine message importance attribute. */
+ if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
+ ipc_importance_task_t task_imp = task_self->task_imp_base;
+ ipc_port_t port = kmsg->ikm_header->msgh_remote_port;
+
+ /* defensive deduction for release builds lacking the assert */
+ ip_lock(port);
+ ipc_port_impcount_delta(port, -1, IP_NULL);
+ ip_unlock(port);
+
+ /* will user accept legacy responsibility for the importance boost */
+ if (KERN_SUCCESS == ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid)) {
+ impresult = 1;
+ } else {
+ /* The importance boost never applied to task (clear the bit) */
+ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
+ impresult = 0;
+ }
+ }
+ }
+
+#if IMPORTANCE_DEBUG
+ if (-1 < impresult)
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_DELV)) | DBG_FUNC_NONE,
+ sender_pid, audit_token_pid_from_task(task_self),
+ kmsg->ikm_header->msgh_id, impresult, 0);
+ if (impresult == 2){
+ /*
+ * This probe only covers new voucher-based path. Legacy importance
+ * will trigger the probe in ipc_importance_task_externalize_assertion()
+ * above and have impresult==1 here.
+ */
+ DTRACE_BOOST5(receive_boost, task_t, task_self, int, audit_token_pid_from_task(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt);
+ }
+#endif /* IMPORTANCE_DEBUG */
+}
+
+/*
+ * Routine: ipc_importance_unreceive
+ * Purpose:
+ * Undo receive of importance attributes in a message.
+ *
+ * Conditions:
+ * Nothing locked.
+ */
+void
+ipc_importance_unreceive(
+ ipc_kmsg_t kmsg,
+ mach_msg_option_t __unused option)
+{
+ /* importance should already be in the voucher and out of the kmsg */
+ assert(IIE_NULL == kmsg->ikm_importance);
+
+ /* See if there is a legacy boost to be dropped from receiver */
+ if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
+ ipc_importance_task_t task_imp;
+
+ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
+ task_imp = current_task()->task_imp_base;
+ if (!IP_VALID(kmsg->ikm_voucher) && IIT_NULL != task_imp) {
+ ipc_importance_task_drop_legacy_external_assertion(task_imp, 1);
+ }
+ /*
+ * ipc_kmsg_copyout_dest() will consume the voucher
+ * and any contained importance.
+ */
+ }
+}
+
+/*
+ * Routine: ipc_importance_clean
+ * Purpose:
+ * Clean up importance state in a kmsg that is being cleaned.
+ * Unlink the importance chain if one was set up, and drop
+ * the reference this kmsg held on the donor. Then check to
+ * if importance was carried to the port, and remove that if
+ * needed.
+ * Conditions:
+ * Nothing locked.
+ */
+void
+ipc_importance_clean(
+ ipc_kmsg_t kmsg)
+{
+ ipc_port_t port;
+
+ /* Is the kmsg still linked? If so, remove that first */
+ if (IIE_NULL != kmsg->ikm_importance) {
+ ipc_importance_elem_t elem;
+
+ ipc_importance_lock();
+ elem = ipc_importance_kmsg_unlink(kmsg);
+ assert(IIE_NULL != elem);
+ ipc_importance_release_locked(elem);
+ /* importance unlocked */
+ }
+
+ /* See if there is a legacy importance boost to be dropped from port */
+ if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
+ kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
+ port = kmsg->ikm_header->msgh_remote_port;
+ if (IP_VALID(port)) {
+ ip_lock(port);
+ /* inactive ports already had their importance boosts dropped */
+ if (!ip_active(port) ||
+ ipc_port_importance_delta(port, -1) == FALSE) {
+ ip_unlock(port);
+ }
+ }
+ }
+}
+
+void
+ipc_importance_assert_clean(__assert_only ipc_kmsg_t kmsg)
+{
+ assert(IIE_NULL == kmsg->ikm_importance);
+ assert(!MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits));
+}
+
+/*
+ * IPC Importance Attribute Manager definition
+ */
+
+static kern_return_t
+ipc_importance_release_value(
+ ipc_voucher_attr_manager_t manager,
+ mach_voucher_attr_key_t key,
+ mach_voucher_attr_value_handle_t value,
+ mach_voucher_attr_value_reference_t sync);
+
+static kern_return_t
+ipc_importance_get_value(
+ ipc_voucher_attr_manager_t manager,
+ mach_voucher_attr_key_t key,
+ mach_voucher_attr_recipe_command_t command,
+ mach_voucher_attr_value_handle_array_t prev_values,
+ mach_voucher_attr_value_handle_array_size_t prev_value_count,
+ mach_voucher_attr_content_t content,
+ mach_voucher_attr_content_size_t content_size,
+ mach_voucher_attr_value_handle_t *out_value,
+ ipc_voucher_t *out_value_voucher);
+
+static kern_return_t
+ipc_importance_extract_content(
+ ipc_voucher_attr_manager_t manager,
+ mach_voucher_attr_key_t key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_voucher_attr_value_handle_array_size_t value_count,
+ mach_voucher_attr_recipe_command_t *out_command,
+ mach_voucher_attr_content_t out_content,
+ mach_voucher_attr_content_size_t *in_out_content_size);
+
+static kern_return_t
+ipc_importance_command(
+ ipc_voucher_attr_manager_t manager,
+ mach_voucher_attr_key_t key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_msg_type_number_t value_count,
+ mach_voucher_attr_command_t command,
+ mach_voucher_attr_content_t in_content,
+ mach_voucher_attr_content_size_t in_content_size,
+ mach_voucher_attr_content_t out_content,
+ mach_voucher_attr_content_size_t *out_content_size);
+
+static void
+ipc_importance_manager_release(
+ ipc_voucher_attr_manager_t manager);
+
+struct ipc_voucher_attr_manager ipc_importance_manager = {
+ .ivam_release_value = ipc_importance_release_value,
+ .ivam_get_value = ipc_importance_get_value,
+ .ivam_extract_content = ipc_importance_extract_content,
+ .ivam_command = ipc_importance_command,
+ .ivam_release = ipc_importance_manager_release,
+};
+
+#define IMPORTANCE_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_IMPORTANCE == (key))
+#define IMPORTANCE_ASSERT_MANAGER(manager) assert(&ipc_importance_manager == (manager))
+
+/*
+ * Routine: ipc_importance_release_value [Voucher Attribute Manager Interface]
+ * Purpose:
+ * Release what the voucher system believes is the last "made" reference
+ * on an importance attribute value handle. The sync parameter is used to
+ * avoid races with new made references concurrently being returned to the
+ * voucher system in other threads.
+ * Conditions:
+ * Nothing locked on entry. May block.
+ */
+static kern_return_t
+ipc_importance_release_value(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_t value,
+ mach_voucher_attr_value_reference_t sync)
+{
+ ipc_importance_elem_t elem;
+
+ IMPORTANCE_ASSERT_MANAGER(manager);
+ IMPORTANCE_ASSERT_KEY(key);
+ assert(0 < sync);
+
+ elem = (ipc_importance_elem_t)value;
+
+ ipc_importance_lock();
+
+ /* Any oustanding made refs? */
+ if (sync != elem->iie_made) {
+ assert(sync < elem->iie_made);
+ ipc_importance_unlock();
+ return KERN_FAILURE;
+ }
+
+ /* clear made */
+ elem->iie_made = 0;
+
+ /*
+ * If there are pending external boosts represented by this attribute,
+ * drop them from the apropriate task
+ */
+ if (IIE_TYPE_INHERIT == IIE_TYPE(elem)) {
+ ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem;
+
+ assert(inherit->iii_externcnt >= inherit->iii_externdrop);
+
+ if (inherit->iii_donating) {
+ ipc_importance_task_t imp_task = inherit->iii_to_task;
+ uint32_t assertcnt = III_EXTERN(inherit);
+
+ assert(ipc_importance_task_is_any_receiver_type(imp_task));
+ assert(imp_task->iit_externcnt >= inherit->iii_externcnt);
+ assert(imp_task->iit_externdrop >= inherit->iii_externdrop);
+ imp_task->iit_externcnt -= inherit->iii_externcnt;
+ imp_task->iit_externdrop -= inherit->iii_externdrop;
+ inherit->iii_externcnt = 0;
+ inherit->iii_externdrop = 0;
+ inherit->iii_donating = FALSE;
+
+ /* adjust the internal assertions - and propagate if needed */
+ if (ipc_importance_task_check_transition(imp_task, IIT_UPDATE_DROP, assertcnt)) {
+ ipc_importance_task_propagate_assertion_locked(imp_task, IIT_UPDATE_DROP, TRUE);
+ }
+ } else {
+ inherit->iii_externcnt = 0;
+ inherit->iii_externdrop = 0;
+ }
+ }
+
+ /* drop the made reference on elem */
+ ipc_importance_release_locked(elem);
+ /* returns unlocked */
+
+ return KERN_SUCCESS;
+}
+
+
+/*
+ * Routine: ipc_importance_get_value [Voucher Attribute Manager Interface]
+ * Purpose:
+ * Convert command and content data into a reference on a [potentially new]
+ * attribute value. The importance attribute manager will only allow the
+ * caller to get a value for the current task's importance, or to redeem
+ * an importance attribute from an existing voucher.
+ * Conditions:
+ * Nothing locked on entry. May block.
+ */
+static kern_return_t
+ipc_importance_get_value(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_recipe_command_t command,
+ mach_voucher_attr_value_handle_array_t prev_values,
+ mach_voucher_attr_value_handle_array_size_t prev_value_count,
+ mach_voucher_attr_content_t __unused content,
+ mach_voucher_attr_content_size_t content_size,
+ mach_voucher_attr_value_handle_t *out_value,
+ ipc_voucher_t *out_value_voucher)
+{
+ ipc_importance_elem_t elem;
+ task_t self;
+
+ IMPORTANCE_ASSERT_MANAGER(manager);
+ IMPORTANCE_ASSERT_KEY(key);
+
+ if (0 != content_size)
+ return KERN_INVALID_ARGUMENT;
+
+ /* never an out voucher */
+
+ switch (command) {
+
+ case MACH_VOUCHER_ATTR_REDEEM:
+
+ /* redeem of previous values is the value */
+ if (0 < prev_value_count) {
+ elem = (ipc_importance_elem_t)prev_values[0];
+ assert(IIE_NULL != elem);
+
+ ipc_importance_lock();
+ assert(0 < elem->iie_made);
+ elem->iie_made++;
+ ipc_importance_unlock();
+
+ *out_value = prev_values[0];
+ return KERN_SUCCESS;
+ }
+
+ /* redeem of default is default */
+ *out_value = 0;
+ *out_value_voucher = IPC_VOUCHER_NULL;
+ return KERN_SUCCESS;
+
+ case MACH_VOUCHER_ATTR_IMPORTANCE_SELF:
+ self = current_task();
+
+ elem = (ipc_importance_elem_t)ipc_importance_for_task(self, TRUE);
+ /* made reference added (or IIE_NULL which isn't referenced) */
+
+ *out_value = (mach_voucher_attr_value_handle_t)elem;
+ *out_value_voucher = IPC_VOUCHER_NULL;
+ return KERN_SUCCESS;
+
+ default:
+ /*
+ * every other command is unknown
+ *
+ * Specifically, there is no mechanism provided to construct an
+ * importance attribute for a task/process from just a pid or
+ * task port. It has to be copied (or redeemed) from a previous
+ * voucher that has it.
+ */
+ return KERN_INVALID_ARGUMENT;
+ }
+}
+
+/*
+ * Routine: ipc_importance_extract_content [Voucher Attribute Manager Interface]
+ * Purpose:
+ * Extract meaning from the attribute value present in a voucher. While
+ * the real goal is to provide commands and data that can reproduce the
+ * voucher's value "out of thin air", this isn't possible with importance
+ * attribute values. Instead, return debug info to help track down dependencies.
+ * Conditions:
+ * Nothing locked on entry. May block.
+ */
+static kern_return_t
+ipc_importance_extract_content(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_voucher_attr_value_handle_array_size_t value_count,
+ mach_voucher_attr_recipe_command_t *out_command,
+ mach_voucher_attr_content_t out_content,
+ mach_voucher_attr_content_size_t *in_out_content_size)
+{
+ mach_voucher_attr_content_size_t size = 0;
+ ipc_importance_elem_t elem;
+ unsigned int i;
+
+ IMPORTANCE_ASSERT_MANAGER(manager);
+ IMPORTANCE_ASSERT_KEY(key);
+
+ /* the first non-default value provides the data */
+ for (i = 0; i < value_count ; i++) {
+ elem = (ipc_importance_elem_t)values[i];
+ if (IIE_NULL == elem)
+ continue;
+
+ snprintf((char *)out_content, *in_out_content_size, "Importance for pid ");
+ size = (mach_voucher_attr_content_size_t)strlen((char *)out_content);
+
+ for(;;) {
+ ipc_importance_inherit_t inherit = III_NULL;
+ ipc_importance_task_t task_imp;
+ task_t task;
+ int task_pid;
+
+ if (IIE_TYPE_TASK == IIE_TYPE(elem)) {
+ task_imp = (ipc_importance_task_t)elem;
+ task = task_imp->iit_task;
+ task_pid = (TASK_NULL != task) ?
+ audit_token_pid_from_task(task) : -1;
+ snprintf((char *)out_content + size, *in_out_content_size - size, "%d", task_pid);
+ } else {
+ inherit = (ipc_importance_inherit_t)elem;
+ task_imp = inherit->iii_to_task;
+ task = task_imp->iit_task;
+ task_pid = (TASK_NULL != task) ?
+ audit_token_pid_from_task(task) : -1;
+ snprintf((char *)out_content + size, *in_out_content_size - size,
+ "%d (%d of %d boosts) %s from pid ", task_pid,
+ III_EXTERN(inherit), inherit->iii_externcnt,
+ (inherit->iii_donating) ? "donated" : "linked");
+ }
+
+ size = (mach_voucher_attr_content_size_t)strlen((char *)out_content);
+
+ if (III_NULL == inherit)
+ break;
+
+ elem = inherit->iii_from_elem;
+ }
+ size++; /* account for NULL */
+ }
+ *out_command = MACH_VOUCHER_ATTR_NOOP; /* cannot be used to regenerate value */
+ *in_out_content_size = size;
+ return KERN_SUCCESS;
+}
+
+/*
+ * Routine: ipc_importance_command [Voucher Attribute Manager Interface]
+ * Purpose:
+ * Run commands against the importance attribute value found in a voucher.
+ * No such commands are currently supported.
+ * Conditions:
+ * Nothing locked on entry. May block.
+ */
+static kern_return_t
+ipc_importance_command(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_msg_type_number_t value_count,
+ mach_voucher_attr_command_t command,
+ mach_voucher_attr_content_t in_content,
+ mach_voucher_attr_content_size_t in_content_size,
+ mach_voucher_attr_content_t out_content,
+ mach_voucher_attr_content_size_t *out_content_size)
+{
+ ipc_importance_inherit_t inherit;
+ ipc_importance_task_t to_task;
+ uint32_t refs, *outrefsp;
+ mach_msg_type_number_t i;
+ uint32_t externcnt;
+
+ IMPORTANCE_ASSERT_MANAGER(manager);
+ IMPORTANCE_ASSERT_KEY(key);
+
+ if (in_content_size != sizeof(refs) ||
+ (*out_content_size != 0 && *out_content_size != sizeof(refs))) {
+ return KERN_INVALID_ARGUMENT;
+ }
+ refs = *(uint32_t *)(void *)in_content;
+ outrefsp = (*out_content_size != 0) ? (uint32_t *)(void *)out_content : NULL;
+
+ if (MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL != command) {
+ return KERN_NOT_SUPPORTED;
+ }
+
+ /* the first non-default value of the apropos type provides the data */
+ inherit = III_NULL;
+ for (i = 0; i < value_count; i++) {
+ ipc_importance_elem_t elem = (ipc_importance_elem_t)values[i];
+
+ if (IIE_NULL != elem && IIE_TYPE_INHERIT == IIE_TYPE(elem)) {
+ inherit = (ipc_importance_inherit_t)elem;
+ break;
+ }
+ }
+ if (III_NULL == inherit) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ ipc_importance_lock();
+
+ if (0 == refs) {
+ if (NULL != outrefsp) {
+ *outrefsp = III_EXTERN(inherit);
+ }
+ ipc_importance_unlock();
+ return KERN_SUCCESS;
+ }
+
+ /* Enough external references left to drop? */
+ if (III_EXTERN(inherit) < refs) {
+ ipc_importance_unlock();
+ return KERN_FAILURE;
+ }
+
+ to_task = inherit->iii_to_task;
+ assert(ipc_importance_task_is_any_receiver_type(to_task));
+
+ /* re-base external and internal counters at the inherit and the to-task (if apropos) */
+ if (inherit->iii_donating) {
+ assert(IIT_EXTERN(to_task) >= III_EXTERN(inherit));
+ assert(to_task->iit_externcnt >= inherit->iii_externcnt);
+ assert(to_task->iit_externdrop >= inherit->iii_externdrop);
+ inherit->iii_externdrop += refs;
+ to_task->iit_externdrop += refs;
+ externcnt = III_EXTERN(inherit);
+ if (0 == externcnt) {
+ inherit->iii_donating = FALSE;
+ to_task->iit_externcnt -= inherit->iii_externcnt;
+ to_task->iit_externdrop -= inherit->iii_externdrop;
+
+
+ /* Start AppNap delay hysteresis - even if not the last boost for the task. */
+ if (ipc_importance_delayed_drop_call != NULL &&
+ ipc_importance_task_is_marked_denap_receiver(to_task)) {
+ ipc_importance_task_delayed_drop(to_task);
+ }
+
+ /* drop task assertions associated with the dropped boosts */
+ if (ipc_importance_task_check_transition(to_task, IIT_UPDATE_DROP, refs)) {
+ ipc_importance_task_propagate_assertion_locked(to_task, IIT_UPDATE_DROP, TRUE);
+ /* may have dropped and retaken importance lock */
+ }
+ } else {
+ /* assert(to_task->iit_assertcnt >= refs + externcnt); */
+ /* defensive deduction in case of assertcnt underflow */
+ if (to_task->iit_assertcnt > refs + externcnt) {
+ to_task->iit_assertcnt -= refs;
+ } else {
+ to_task->iit_assertcnt = externcnt;
+ }
+ }
+ } else {
+ inherit->iii_externdrop += refs;
+ externcnt = III_EXTERN(inherit);
+ }
+
+ /* capture result (if requested) */
+ if (NULL != outrefsp) {
+ *outrefsp = externcnt;
+ }
+
+ ipc_importance_unlock();
+ return KERN_SUCCESS;
+}
+
+/*
+ * Routine: ipc_importance_manager_release [Voucher Attribute Manager Interface]
+ * Purpose:
+ * Release the Voucher system's reference on the IPC importance attribute
+ * manager.
+ * Conditions:
+ * As this can only occur after the manager drops the Attribute control
+ * reference granted back at registration time, and that reference is never
+ * dropped, this should never be called.
+ */
+static void
+ipc_importance_manager_release(
+ ipc_voucher_attr_manager_t __assert_only manager)
+{
+ IMPORTANCE_ASSERT_MANAGER(manager);
+ panic("Voucher importance manager released");
+}
+
+/*
+ * Routine: ipc_importance_init
+ * Purpose:
+ * Initialize the IPC importance manager.
+ * Conditions:
+ * Zones and Vouchers are already initialized.
+ */
+void
+ipc_importance_init(void)
+{
+ natural_t ipc_importance_max = (task_max + thread_max) * 2;
+ char temp_buf[26];
+ kern_return_t kr;
+
+ if (PE_parse_boot_argn("imp_interactive_receiver", temp_buf, sizeof(temp_buf))) {
+ ipc_importance_interactive_receiver = TRUE;
+ }
+
+ ipc_importance_task_zone = zinit(sizeof(struct ipc_importance_task),
+ ipc_importance_max * sizeof(struct ipc_importance_task),
+ sizeof(struct ipc_importance_task),
+ "ipc task importance");
+ zone_change(ipc_importance_task_zone, Z_NOENCRYPT, TRUE);
+
+ ipc_importance_inherit_zone = zinit(sizeof(struct ipc_importance_inherit),
+ ipc_importance_max * sizeof(struct ipc_importance_inherit),
+ sizeof(struct ipc_importance_inherit),
+ "ipc importance inherit");
+ zone_change(ipc_importance_inherit_zone, Z_NOENCRYPT, TRUE);
+
+
+#if DEVELOPMENT || DEBUG
+ queue_init(&global_iit_alloc_queue);
+#endif
+
+ /* initialize global locking */
+ ipc_importance_lock_init();
+
+ kr = ipc_register_well_known_mach_voucher_attr_manager(&ipc_importance_manager,
+ (mach_voucher_attr_value_handle_t)0,
+ MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
+ &ipc_importance_control);
+ if (KERN_SUCCESS != kr)
+ printf("Voucher importance manager register returned %d", kr);
+}
+
+/*
+ * Routine: ipc_importance_thread_call_init
+ * Purpose:
+ * Initialize the IPC importance code dependent upon
+ * thread-call support being available.
+ * Conditions:
+ * Thread-call mechanism is already initialized.
+ */
+void
+ipc_importance_thread_call_init(void)
+{
+ /* initialize delayed drop queue and thread-call */
+ queue_init(&ipc_importance_delayed_drop_queue);
+ ipc_importance_delayed_drop_call =
+ thread_call_allocate(ipc_importance_task_delayed_drop_scan, NULL);
+ if (NULL == ipc_importance_delayed_drop_call) {
+ panic("ipc_importance_init");
+ }
+}
+
+/*
+ * Routing: task_importance_list_pids
+ * Purpose: list pids where task in donating importance.
+ * Conditions: To be called only from kdp stackshot code.
+ * Will panic the system otherwise.
+ */
+extern int
+task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int max_count)
+{
+ if (lck_spin_is_acquired(&ipc_importance_lock_data) ||
+ max_count < 1 ||
+ task->task_imp_base == IIT_NULL ||
+ pid_list == NULL ||
+ flags != TASK_IMP_LIST_DONATING_PIDS) {
+ return 0;
+ }
+ unsigned int pidcount = 0;
+ task_t temp_task;
+ ipc_importance_task_t task_imp = task->task_imp_base;
+ ipc_kmsg_t temp_kmsg;
+ ipc_importance_inherit_t temp_inherit;
+ ipc_importance_elem_t elem;
+ int target_pid;
+
+ queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) {
+ /* check space in buffer */
+ if (pidcount >= max_count)
+ break;
+ target_pid = -1;
+
+ if (temp_inherit->iii_donating) {
+
+#if DEVELOPMENT || DEBUG
+ target_pid = temp_inherit->iii_to_task->iit_bsd_pid;
+#else
+ temp_task = temp_inherit->iii_to_task->iit_task;
+ if (temp_task != TASK_NULL) {
+ target_pid = audit_token_pid_from_task(temp_task);
+ }
+#endif
+ }
+
+ if (target_pid != -1) {
+ pid_list[pidcount++] = target_pid;
+ }
+
+ }
+
+ queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) {
+ if (pidcount >= max_count)
+ break;
+ target_pid = -1;
+ elem = temp_kmsg->ikm_importance;
+ temp_task = TASK_NULL;
+
+ if (elem == IIE_NULL) {
+ continue;
+ }
+
+ if (!(temp_kmsg->ikm_header && MACH_MSGH_BITS_RAISED_IMPORTANCE(temp_kmsg->ikm_header->msgh_bits))) {
+ continue;
+ }
+
+ if (IIE_TYPE_TASK == IIE_TYPE(elem) &&
+ (((ipc_importance_task_t)elem)->iit_task != TASK_NULL)) {
+ target_pid = audit_token_pid_from_task(((ipc_importance_task_t)elem)->iit_task);
+ } else {
+ temp_inherit = (ipc_importance_inherit_t)elem;
+#if DEVELOPMENT || DEBUG
+ target_pid = temp_inherit->iii_to_task->iit_bsd_pid;
+#else
+ temp_task = temp_inherit->iii_to_task->iit_task;
+ if (temp_task != TASK_NULL) {
+ target_pid = audit_token_pid_from_task(temp_task);
+ }
+#endif
+ }
+
+ if (target_pid != -1) {
+ pid_list[pidcount++] = target_pid;
+ }
+ }
+
+ return pidcount;
+}
+
projects / apple / xnu.git / blobdiff