2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <mach/mach_types.h>
30 #include <mach/notify.h>
31 #include <ipc/ipc_types.h>
32 #include <ipc/ipc_importance.h>
33 #include <ipc/ipc_port.h>
34 #include <ipc/ipc_voucher.h>
35 #include <kern/ipc_kobject.h>
36 #include <kern/ipc_tt.h>
37 #include <kern/mach_param.h>
38 #include <kern/misc_protos.h>
39 #include <kern/kalloc.h>
40 #include <kern/zalloc.h>
41 #include <kern/queue.h>
42 #include <kern/task.h>
44 #include <sys/kdebug.h>
46 #include <mach/mach_voucher_attr_control.h>
47 #include <mach/machine/sdt.h>
49 extern int proc_pid(void *);
50 extern int proc_selfpid(void);
51 extern uint64_t proc_uniqueid(void *p
);
52 extern char *proc_name_address(void *p
);
55 * Globals for delayed boost drop processing.
57 static queue_head_t ipc_importance_delayed_drop_queue
;
58 static thread_call_t ipc_importance_delayed_drop_call
;
59 static uint64_t ipc_importance_delayed_drop_timestamp
;
60 static boolean_t ipc_importance_delayed_drop_call_requested
= FALSE
;
62 #define DENAP_DROP_TARGET (1000 * NSEC_PER_MSEC) /* optimum denap delay */
63 #define DENAP_DROP_SKEW (100 * NSEC_PER_MSEC) /* request skew for wakeup */
64 #define DENAP_DROP_LEEWAY (2 * DENAP_DROP_SKEW) /* specified wakeup leeway */
66 #define DENAP_DROP_DELAY (DENAP_DROP_TARGET + DENAP_DROP_SKEW)
67 #define DENAP_DROP_FLAGS (THREAD_CALL_DELAY_SYS_NORMAL | THREAD_CALL_DELAY_LEEWAY)
70 * Importance Voucher Attribute Manager
73 static lck_spin_t ipc_importance_lock_data
; /* single lock for now */
76 #define ipc_importance_lock_init() \
77 lck_spin_init(&ipc_importance_lock_data, &ipc_lck_grp, &ipc_lck_attr)
78 #define ipc_importance_lock_destroy() \
79 lck_spin_destroy(&ipc_importance_lock_data, &ipc_lck_grp)
80 #define ipc_importance_lock() \
81 lck_spin_lock(&ipc_importance_lock_data)
82 #define ipc_importance_lock_try() \
83 lck_spin_try_lock(&ipc_importance_lock_data)
84 #define ipc_importance_unlock() \
85 lck_spin_unlock(&ipc_importance_lock_data)
86 #define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data, \
90 #define ipc_importance_wakeup(elem) thread_wakeup((event_t)(elem))
93 #define incr_ref_counter(x) (hw_atomic_add(&(x), 1))
96 uint32_t ipc_importance_reference_internal(ipc_importance_elem_t elem
)
98 incr_ref_counter(elem
->iie_refs_added
);
99 return (hw_atomic_add(&elem
->iie_bits
, 1) & IIE_REFS_MASK
);
103 uint32_t ipc_importance_release_internal(ipc_importance_elem_t elem
)
105 incr_ref_counter(elem
->iie_refs_dropped
);
106 return (hw_atomic_sub(&elem
->iie_bits
, 1) & IIE_REFS_MASK
);
110 uint32_t ipc_importance_task_reference_internal(ipc_importance_task_t task_imp
)
113 out
= ipc_importance_reference_internal(&task_imp
->iit_elem
);
114 incr_ref_counter(task_imp
->iit_elem
.iie_task_refs_added
);
119 uint32_t ipc_importance_task_release_internal(ipc_importance_task_t task_imp
)
123 assert(1 < IIT_REFS(task_imp
));
124 incr_ref_counter(task_imp
->iit_elem
.iie_task_refs_dropped
);
125 out
= ipc_importance_release_internal(&task_imp
->iit_elem
);
130 void ipc_importance_counter_init(ipc_importance_elem_t elem
)
133 elem
->iie_refs_added
= 0;
134 elem
->iie_refs_dropped
= 0;
135 elem
->iie_kmsg_refs_added
= 0;
136 elem
->iie_kmsg_refs_inherited
= 0;
137 elem
->iie_kmsg_refs_coalesced
= 0;
138 elem
->iie_kmsg_refs_dropped
= 0;
139 elem
->iie_task_refs_added
= 0;
140 elem
->iie_task_refs_added_inherit_from
= 0;
141 elem
->iie_task_refs_added_transition
= 0;
142 elem
->iie_task_refs_self_added
= 0;
143 elem
->iie_task_refs_inherited
= 0;
144 elem
->iie_task_refs_coalesced
= 0;
145 elem
->iie_task_refs_dropped
= 0;
148 #define incr_ref_counter(x)
151 #if DEVELOPMENT || DEBUG
152 static queue_head_t global_iit_alloc_queue
;
155 /* TODO: remove this varibale when interactive daemon audit is complete */
156 boolean_t ipc_importance_interactive_receiver
= FALSE
;
158 static zone_t ipc_importance_task_zone
;
159 static zone_t ipc_importance_inherit_zone
;
161 static ipc_voucher_attr_control_t ipc_importance_control
;
164 * Routine: ipc_importance_kmsg_link
166 * Link the kmsg onto the appropriate propagation chain.
167 * If the element is a task importance, we link directly
168 * on its propagation chain. Otherwise, we link onto the
169 * destination task of the inherit.
171 * Importance lock held.
172 * Caller is donating an importance elem reference to the kmsg.
175 ipc_importance_kmsg_link(
177 ipc_importance_elem_t elem
)
179 ipc_importance_elem_t link_elem
;
181 assert(IIE_NULL
== kmsg
->ikm_importance
);
183 link_elem
= (IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) ?
184 (ipc_importance_elem_t
)((ipc_importance_inherit_t
)elem
)->iii_to_task
:
187 queue_enter(&link_elem
->iie_kmsgs
, kmsg
, ipc_kmsg_t
, ikm_inheritance
);
188 kmsg
->ikm_importance
= elem
;
192 * Routine: ipc_importance_kmsg_unlink
194 * Unlink the kmsg from its current propagation chain.
195 * If the element is a task importance, we unlink directly
196 * from its propagation chain. Otherwise, we unlink from the
197 * destination task of the inherit.
199 * The reference to the importance element it was linked on.
201 * Importance lock held.
202 * Caller is responsible for dropping reference on returned elem.
204 static ipc_importance_elem_t
205 ipc_importance_kmsg_unlink(
208 ipc_importance_elem_t elem
= kmsg
->ikm_importance
;
210 if (IIE_NULL
!= elem
) {
211 ipc_importance_elem_t unlink_elem
;
213 unlink_elem
= (IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) ?
214 (ipc_importance_elem_t
)((ipc_importance_inherit_t
)elem
)->iii_to_task
:
217 queue_remove(&unlink_elem
->iie_kmsgs
, kmsg
, ipc_kmsg_t
, ikm_inheritance
);
218 kmsg
->ikm_importance
= IIE_NULL
;
224 * Routine: ipc_importance_inherit_link
226 * Link the inherit onto the appropriate propagation chain.
227 * If the element is a task importance, we link directly
228 * on its propagation chain. Otherwise, we link onto the
229 * destination task of the inherit.
231 * Importance lock held.
232 * Caller is donating an elem importance reference to the inherit.
235 ipc_importance_inherit_link(
236 ipc_importance_inherit_t inherit
,
237 ipc_importance_elem_t elem
)
239 ipc_importance_elem_t link_elem
;
241 assert(IIE_NULL
== inherit
->iii_from_elem
);
242 link_elem
= (IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) ?
243 (ipc_importance_elem_t
)((ipc_importance_inherit_t
)elem
)->iii_to_task
:
246 queue_enter(&link_elem
->iie_inherits
, inherit
,
247 ipc_importance_inherit_t
, iii_inheritance
);
248 inherit
->iii_from_elem
= elem
;
252 * Routine: ipc_importance_inherit_unlink
254 * Unlink the inherit from its current propagation chain.
255 * If the element is a task importance, we unlink directly
256 * from its propagation chain. Otherwise, we unlink from the
257 * destination task of the inherit.
259 * The reference to the importance element it was linked on.
261 * Importance lock held.
262 * Caller is responsible for dropping reference on returned elem.
264 static ipc_importance_elem_t
265 ipc_importance_inherit_unlink(
266 ipc_importance_inherit_t inherit
)
268 ipc_importance_elem_t elem
= inherit
->iii_from_elem
;
270 if (IIE_NULL
!= elem
) {
271 ipc_importance_elem_t unlink_elem
;
273 unlink_elem
= (IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) ?
274 (ipc_importance_elem_t
)((ipc_importance_inherit_t
)elem
)->iii_to_task
:
277 queue_remove(&unlink_elem
->iie_inherits
, inherit
,
278 ipc_importance_inherit_t
, iii_inheritance
);
279 inherit
->iii_from_elem
= IIE_NULL
;
285 * Routine: ipc_importance_reference
287 * Add a reference to the importance element.
289 * Caller must hold a reference on the element.
292 ipc_importance_reference(ipc_importance_elem_t elem
)
294 assert(0 < IIE_REFS(elem
));
295 ipc_importance_reference_internal(elem
);
299 * Routine: ipc_importance_release_locked
301 * Release a reference on an importance attribute value,
302 * unlinking and deallocating the attribute if the last reference.
304 * Entered with importance lock held, leaves with it unlocked.
307 ipc_importance_release_locked(ipc_importance_elem_t elem
)
309 assert(0 < IIE_REFS(elem
));
311 if (0 < ipc_importance_release_internal(elem
)) {
313 #if DEVELOPMENT || DEBUG
314 ipc_importance_inherit_t temp_inherit
;
315 ipc_importance_task_t link_task
;
316 ipc_kmsg_t temp_kmsg
;
317 uint32_t expected
= 0;
319 if (0 < elem
->iie_made
)
322 link_task
= (IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) ?
323 ((ipc_importance_inherit_t
)elem
)->iii_to_task
:
324 (ipc_importance_task_t
)elem
;
326 queue_iterate(&link_task
->iit_kmsgs
, temp_kmsg
, ipc_kmsg_t
, ikm_inheritance
)
327 if (temp_kmsg
->ikm_importance
== elem
)
329 queue_iterate(&link_task
->iit_inherits
, temp_inherit
,
330 ipc_importance_inherit_t
, iii_inheritance
)
331 if (temp_inherit
->iii_from_elem
== elem
)
334 if (IIE_REFS(elem
) < expected
)
335 panic("ipc_importance_release_locked (%p)", elem
);
337 ipc_importance_unlock();
342 /* can't get to no refs if we contribute to something else's importance */
343 assert(queue_empty(&elem
->iie_kmsgs
));
344 assert(queue_empty(&elem
->iie_inherits
));
346 switch (IIE_TYPE(elem
)) {
348 /* just a "from" task reference to drop */
351 ipc_importance_task_t task_elem
;
353 task_elem
= (ipc_importance_task_t
)elem
;
354 assert(TASK_NULL
== task_elem
->iit_task
);
356 #if DEVELOPMENT || DEBUG
357 queue_remove(&global_iit_alloc_queue
, task_elem
, ipc_importance_task_t
, iit_allocation
);
360 ipc_importance_unlock();
362 zfree(ipc_importance_task_zone
, task_elem
);
366 /* dropping an inherit element */
367 case IIE_TYPE_INHERIT
:
369 ipc_importance_inherit_t inherit
;
370 ipc_importance_elem_t from_elem
;
371 ipc_importance_task_t to_task
;
374 inherit
= (ipc_importance_inherit_t
)elem
;
375 to_task
= inherit
->iii_to_task
;
376 assert(IIT_NULL
!= to_task
);
377 assert(!inherit
->iii_donating
);
379 /* unlink and release the inherit */
380 assert(ipc_importance_task_is_any_receiver_type(to_task
));
381 from_elem
= ipc_importance_inherit_unlink(inherit
);
382 assert(IIE_NULL
!= from_elem
);
383 ipc_importance_release_locked(from_elem
);
384 /* unlocked on return */
386 ipc_importance_task_release(to_task
);
388 zfree(ipc_importance_inherit_zone
, inherit
);
395 * Routine: ipc_importance_release
397 * Release a reference on an importance attribute value,
398 * unlinking and deallocating the attribute if the last reference.
400 * nothing locked on entrance, nothing locked on exit.
404 ipc_importance_release(ipc_importance_elem_t elem
)
406 if (IIE_NULL
== elem
)
409 ipc_importance_lock();
410 ipc_importance_release_locked(elem
);
415 * Routine: ipc_importance_task_reference
419 * Retain a reference on a task importance attribute value.
421 * nothing locked on entrance, nothing locked on exit.
422 * caller holds a reference already.
425 ipc_importance_task_reference(ipc_importance_task_t task_elem
)
427 if (IIT_NULL
== task_elem
)
430 incr_ref_counter(task_elem
->iit_elem
.iie_task_refs_added
);
432 ipc_importance_reference(&task_elem
->iit_elem
);
436 * Routine: ipc_importance_task_release
438 * Release a reference on a task importance attribute value,
439 * unlinking and deallocating the attribute if the last reference.
441 * nothing locked on entrance, nothing locked on exit.
445 ipc_importance_task_release(ipc_importance_task_t task_elem
)
447 if (IIT_NULL
== task_elem
)
450 ipc_importance_lock();
452 incr_ref_counter(task_elem
->iit_elem
.iie_task_refs_dropped
);
454 ipc_importance_release_locked(&task_elem
->iit_elem
);
459 * Routine: ipc_importance_task_release_locked
461 * Release a reference on a task importance attribute value,
462 * unlinking and deallocating the attribute if the last reference.
464 * importance lock held on entry, nothing locked on exit.
468 ipc_importance_task_release_locked(ipc_importance_task_t task_elem
)
470 if (IIT_NULL
== task_elem
) {
471 ipc_importance_unlock();
475 incr_ref_counter(task_elem
->iit_elem
.iie_task_refs_dropped
);
477 ipc_importance_release_locked(&task_elem
->iit_elem
);
482 * Routines for importance donation/inheritance/boosting
487 * External importance assertions are managed by the process in userspace
488 * Internal importance assertions are the responsibility of the kernel
489 * Assertions are changed from internal to external via task_importance_externalize_assertion
493 * Routine: ipc_importance_task_check_transition
495 * Increase or decrement the internal task importance counter of the
496 * specified task and determine if propagation and a task policy
497 * update is required.
499 * If it is already enqueued for a policy update, steal it from that queue
500 * (as we are reversing that update before it happens).
503 * Called with the importance lock held.
504 * It is the caller's responsibility to perform the propagation of the
505 * transition and/or policy changes by checking the return value.
508 ipc_importance_task_check_transition(
509 ipc_importance_task_t task_imp
,
510 iit_update_type_t type
,
514 task_t target_task
= task_imp
->iit_task
;
515 boolean_t boost
= (IIT_UPDATE_HOLD
== type
);
516 boolean_t before_boosted
, after_boosted
;
518 if (!ipc_importance_task_is_any_receiver_type(task_imp
))
522 int target_pid
= task_pid(target_task
);
524 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, (((boost
) ? IMP_HOLD
: IMP_DROP
) | TASK_POLICY_INTERNAL
))) | DBG_FUNC_START
,
525 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_EXTERN(task_imp
), 0);
528 /* snapshot the effective boosting status before making any changes */
529 before_boosted
= (task_imp
->iit_assertcnt
> 0);
531 /* Adjust the assertcnt appropriately */
533 task_imp
->iit_assertcnt
+= delta
;
535 DTRACE_BOOST6(send_boost
, task_t
, target_task
, int, target_pid
,
536 task_t
, current_task(), int, proc_selfpid(), int, delta
, int, task_imp
->iit_assertcnt
);
539 // assert(delta <= task_imp->iit_assertcnt);
540 if (delta
> task_imp
->iit_assertcnt
- IIT_EXTERN(task_imp
)) {
541 /* TODO: Turn this back into a panic <rdar://problem/12592649> */
542 if (target_task
!= TASK_NULL
) {
543 printf("Over-release of kernel-internal importance assertions for pid %d (%s), "
544 "dropping %d assertion(s) but task only has %d remaining (%d external).\n",
545 task_pid(target_task
),
546 (target_task
->bsd_info
== NULL
) ? "" : proc_name_address(target_task
->bsd_info
),
548 task_imp
->iit_assertcnt
,
549 IIT_EXTERN(task_imp
));
551 task_imp
->iit_assertcnt
= IIT_EXTERN(task_imp
);
553 task_imp
->iit_assertcnt
-= delta
;
556 // This convers both legacy and voucher-based importance.
557 DTRACE_BOOST4(drop_boost
, task_t
, target_task
, int, target_pid
, int, delta
, int, task_imp
->iit_assertcnt
);
562 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, (((boost
) ? IMP_HOLD
: IMP_DROP
) | TASK_POLICY_INTERNAL
))) | DBG_FUNC_END
,
563 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_EXTERN(task_imp
), 0);
566 /* did the change result in an effective donor status change? */
567 after_boosted
= (task_imp
->iit_assertcnt
> 0);
569 if (after_boosted
!= before_boosted
) {
572 * If the task importance is already on an update queue, we just reversed the need for a
573 * pending policy update. If the queue is any other than the delayed-drop-queue, pull it
574 * off that queue and release the reference it got going onto the update queue. If it is
575 * the delayed-drop-queue we leave it in place in case it comes back into the drop state
576 * before its time delay is up.
578 * We still need to propagate the change downstream to reverse the assertcnt effects,
579 * but we no longer need to update this task's boost policy state.
581 * Otherwise, mark it as needing a policy update.
583 assert(0 == task_imp
->iit_updatepolicy
);
584 if (NULL
!= task_imp
->iit_updateq
) {
585 if (&ipc_importance_delayed_drop_queue
!= task_imp
->iit_updateq
) {
586 queue_remove(task_imp
->iit_updateq
, task_imp
, ipc_importance_task_t
, iit_updates
);
587 task_imp
->iit_updateq
= NULL
;
588 ipc_importance_task_release_internal(task_imp
); /* can't be last ref */
591 task_imp
->iit_updatepolicy
= 1;
601 * Routine: ipc_importance_task_propagate_helper
603 * Increase or decrement the internal task importance counter of all
604 * importance tasks inheriting from the specified one. If this causes
605 * that importance task to change state, add it to the list of tasks
606 * to do a policy update against.
608 * Called with the importance lock held.
609 * It is the caller's responsibility to iterate down the generated list
610 * and propagate any subsequent assertion changes from there.
613 ipc_importance_task_propagate_helper(
614 ipc_importance_task_t task_imp
,
615 iit_update_type_t type
,
618 ipc_importance_task_t temp_task_imp
;
621 * iterate the downstream kmsgs, adjust their boosts,
622 * and capture the next task to adjust for each message
625 ipc_kmsg_t temp_kmsg
;
627 queue_iterate(&task_imp
->iit_kmsgs
, temp_kmsg
, ipc_kmsg_t
, ikm_inheritance
) {
628 mach_msg_header_t
*hdr
= temp_kmsg
->ikm_header
;
629 mach_port_delta_t delta
;
632 /* toggle the kmsg importance bit as a barrier to parallel adjusts */
633 if (IIT_UPDATE_HOLD
== type
) {
634 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr
->msgh_bits
)) {
638 /* mark the message as now carrying importance */
639 hdr
->msgh_bits
|= MACH_MSGH_BITS_RAISEIMP
;
642 if (!MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr
->msgh_bits
)) {
646 /* clear the message as now carrying importance */
647 hdr
->msgh_bits
&= ~MACH_MSGH_BITS_RAISEIMP
;
651 /* determine the task importance to adjust as result (if any) */
652 port
= (ipc_port_t
) hdr
->msgh_remote_port
;
653 assert(IP_VALID(port
));
655 temp_task_imp
= IIT_NULL
;
656 if (!ipc_port_importance_delta_internal(port
, &delta
, &temp_task_imp
)) {
660 /* no task importance to adjust associated with the port? */
661 if (IIT_NULL
== temp_task_imp
) {
665 /* hold a reference on temp_task_imp */
667 /* Adjust the task assertions and determine if an edge was crossed */
668 if (ipc_importance_task_check_transition(temp_task_imp
, type
, 1)) {
669 incr_ref_counter(task_imp
->iit_elem
.iie_task_refs_added_transition
);
670 queue_enter(propagation
, temp_task_imp
, ipc_importance_task_t
, iit_props
);
671 /* reference donated */
673 ipc_importance_task_release_internal(temp_task_imp
);
678 * iterate the downstream importance inherits
679 * and capture the next task importance to boost for each
681 ipc_importance_inherit_t temp_inherit
;
683 queue_iterate(&task_imp
->iit_inherits
, temp_inherit
, ipc_importance_inherit_t
, iii_inheritance
) {
684 uint32_t assertcnt
= III_EXTERN(temp_inherit
);
686 temp_task_imp
= temp_inherit
->iii_to_task
;
687 assert(IIT_NULL
!= temp_task_imp
);
689 if (IIT_UPDATE_HOLD
== type
) {
690 /* if no undropped externcnts in the inherit, nothing to do */
691 if (0 == assertcnt
) {
692 assert(temp_inherit
->iii_donating
== FALSE
);
696 /* nothing to do if the inherit is already donating (forced donation) */
697 if (temp_inherit
->iii_donating
) {
701 /* mark it donating and contribute to the task externcnts */
702 temp_inherit
->iii_donating
= TRUE
;
703 temp_task_imp
->iit_externcnt
+= temp_inherit
->iii_externcnt
;
704 temp_task_imp
->iit_externdrop
+= temp_inherit
->iii_externdrop
;
707 /* if no contributing assertions, move on */
708 if (0 == assertcnt
) {
709 assert(temp_inherit
->iii_donating
== FALSE
);
713 /* nothing to do if the inherit is not donating */
714 if (!temp_inherit
->iii_donating
) {
718 /* mark it no longer donating */
719 temp_inherit
->iii_donating
= FALSE
;
721 /* remove the contribution the inherit made to the to-task */
722 assert(IIT_EXTERN(temp_task_imp
) >= III_EXTERN(temp_inherit
));
723 assert(temp_task_imp
->iit_externcnt
>= temp_inherit
->iii_externcnt
);
724 assert(temp_task_imp
->iit_externdrop
>= temp_inherit
->iii_externdrop
);
725 temp_task_imp
->iit_externcnt
-= temp_inherit
->iii_externcnt
;
726 temp_task_imp
->iit_externdrop
-= temp_inherit
->iii_externdrop
;
730 /* Adjust the task assertions and determine if an edge was crossed */
731 assert(ipc_importance_task_is_any_receiver_type(temp_task_imp
));
732 if (ipc_importance_task_check_transition(temp_task_imp
, type
, assertcnt
)) {
733 ipc_importance_task_reference(temp_task_imp
);
734 incr_ref_counter(task_imp
->iit_elem
.iie_task_refs_added_transition
);
735 queue_enter(propagation
, temp_task_imp
, ipc_importance_task_t
, iit_props
);
741 * Routine: ipc_importance_task_process_updates
743 * Process the queue of task importances and apply the policy
744 * update called for. Only process tasks in the queue with an
745 * update timestamp less than the supplied max.
747 * Called and returns with importance locked.
748 * May drop importance lock and block temporarily.
751 ipc_importance_task_process_updates(
752 queue_t supplied_queue
,
754 uint64_t max_timestamp
)
756 ipc_importance_task_t task_imp
;
757 queue_head_t second_chance
;
758 queue_t queue
= supplied_queue
;
761 * This queue will hold the task's we couldn't trylock on first pass.
762 * By using a second (private) queue, we guarantee all tasks that get
763 * entered on this queue have a timestamp under the maximum.
765 queue_init(&second_chance
);
767 /* process any resulting policy updates */
769 while(!queue_empty(queue
)) {
771 struct task_pend_token pend_token
= {};
773 task_imp
= (ipc_importance_task_t
)queue_first(queue
);
774 assert(0 == task_imp
->iit_updatepolicy
);
775 assert(queue
== task_imp
->iit_updateq
);
777 /* if timestamp is too big, we're done */
778 if (task_imp
->iit_updatetime
> max_timestamp
) {
782 /* we were given a reference on each task in the queue */
784 /* remove it from the supplied queue */
785 queue_remove(queue
, task_imp
, ipc_importance_task_t
, iit_updates
);
786 task_imp
->iit_updateq
= NULL
;
788 target_task
= task_imp
->iit_task
;
790 /* Is it well on the way to exiting? */
791 if (TASK_NULL
== target_task
) {
792 ipc_importance_task_release_locked(task_imp
);
793 /* importance unlocked */
794 ipc_importance_lock();
798 /* Has the update been reversed on the hysteresis queue? */
799 if (0 < task_imp
->iit_assertcnt
&&
800 queue
== &ipc_importance_delayed_drop_queue
) {
801 ipc_importance_task_release_locked(task_imp
);
802 /* importance unlocked */
803 ipc_importance_lock();
808 * Can we get the task lock out-of-order?
809 * If not, stick this back on the second-chance queue.
811 if (!task_lock_try(target_task
)) {
812 boolean_t should_wait_lock
= (queue
== &second_chance
);
813 task_imp
->iit_updateq
= &second_chance
;
816 * If we're already processing second-chances on
817 * tasks, keep this task on the front of the queue.
818 * We will wait for the task lock before coming
819 * back and trying again, and we have a better
820 * chance of re-acquiring the lock if we come back
823 if (should_wait_lock
){
824 task_reference(target_task
);
825 queue_enter_first(&second_chance
, task_imp
,
826 ipc_importance_task_t
, iit_updates
);
828 queue_enter(&second_chance
, task_imp
,
829 ipc_importance_task_t
, iit_updates
);
831 ipc_importance_unlock();
833 if (should_wait_lock
) {
834 task_lock(target_task
);
835 task_unlock(target_task
);
836 task_deallocate(target_task
);
839 ipc_importance_lock();
843 /* is it going away? */
844 if (!target_task
->active
) {
845 task_unlock(target_task
);
846 ipc_importance_task_release_locked(task_imp
);
847 /* importance unlocked */
848 ipc_importance_lock();
852 /* take a task reference for while we don't have the importance lock */
853 task_reference(target_task
);
855 /* count the transition */
857 task_imp
->iit_transitions
++;
859 ipc_importance_unlock();
861 /* apply the policy adjust to the target task (while it is still locked) */
862 task_update_boost_locked(target_task
, boost
, &pend_token
);
864 /* complete the policy update with the task unlocked */
865 ipc_importance_task_release(task_imp
);
866 task_unlock(target_task
);
867 task_policy_update_complete_unlocked(target_task
, THREAD_NULL
, &pend_token
);
868 task_deallocate(target_task
);
870 ipc_importance_lock();
873 /* If there are tasks we couldn't update the first time, try again */
874 if (!queue_empty(&second_chance
)) {
875 queue
= &second_chance
;
882 * Routine: ipc_importance_task_delayed_drop_scan
884 * The thread call routine to scan the delayed drop queue,
885 * requesting all updates with a deadline up to the last target
886 * for the thread-call (which is DENAP_DROP_SKEW beyond the first
887 * thread's optimum delay).
888 * update to drop its boost.
893 ipc_importance_task_delayed_drop_scan(
897 ipc_importance_lock();
899 /* process all queued task drops with timestamps up to TARGET(first)+SKEW */
900 ipc_importance_task_process_updates(&ipc_importance_delayed_drop_queue
,
902 ipc_importance_delayed_drop_timestamp
);
904 /* importance lock may have been temporarily dropped */
906 /* If there are any entries left in the queue, re-arm the call here */
907 if (!queue_empty(&ipc_importance_delayed_drop_queue
)) {
908 ipc_importance_task_t task_imp
;
912 task_imp
= (ipc_importance_task_t
)queue_first(&ipc_importance_delayed_drop_queue
);
914 nanoseconds_to_absolutetime(DENAP_DROP_DELAY
, &deadline
);
915 deadline
+= task_imp
->iit_updatetime
;
916 ipc_importance_delayed_drop_timestamp
= deadline
;
918 nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY
, &leeway
);
920 thread_call_enter_delayed_with_leeway(
921 ipc_importance_delayed_drop_call
,
927 ipc_importance_delayed_drop_call_requested
= FALSE
;
929 ipc_importance_unlock();
933 * Routine: ipc_importance_task_delayed_drop
935 * Queue the specified task importance for delayed policy
936 * update to drop its boost.
938 * Called with the importance lock held.
941 ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp
)
943 uint64_t timestamp
= mach_absolute_time(); /* no mach_approximate_time() in kernel */
945 assert(ipc_importance_delayed_drop_call
!= NULL
);
948 * If still on an update queue from a previous change,
949 * remove it first (and use that reference). Otherwise, take
950 * a new reference for the delay drop update queue.
952 if (NULL
!= task_imp
->iit_updateq
) {
953 queue_remove(task_imp
->iit_updateq
, task_imp
,
954 ipc_importance_task_t
, iit_updates
);
956 ipc_importance_task_reference_internal(task_imp
);
959 task_imp
->iit_updateq
= &ipc_importance_delayed_drop_queue
;
960 task_imp
->iit_updatetime
= timestamp
;
962 queue_enter(&ipc_importance_delayed_drop_queue
, task_imp
,
963 ipc_importance_task_t
, iit_updates
);
965 /* request the delayed thread-call if not already requested */
966 if (!ipc_importance_delayed_drop_call_requested
) {
970 nanoseconds_to_absolutetime(DENAP_DROP_DELAY
, &deadline
);
971 deadline
+= task_imp
->iit_updatetime
;
972 ipc_importance_delayed_drop_timestamp
= deadline
;
974 nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY
, &leeway
);
976 ipc_importance_delayed_drop_call_requested
= TRUE
;
977 thread_call_enter_delayed_with_leeway(
978 ipc_importance_delayed_drop_call
,
988 * Routine: ipc_importance_task_propagate_assertion_locked
990 * Propagate the importance transition type to every item
991 * If this causes a boost to be applied, determine if that
992 * boost should propagate downstream.
994 * Called with the importance lock held.
997 ipc_importance_task_propagate_assertion_locked(
998 ipc_importance_task_t task_imp
,
999 iit_update_type_t type
,
1000 boolean_t update_task_imp
)
1002 boolean_t boost
= (IIT_UPDATE_HOLD
== type
);
1003 ipc_importance_task_t temp_task_imp
;
1004 queue_head_t propagate
;
1005 queue_head_t updates
;
1007 queue_init(&updates
);
1008 queue_init(&propagate
);
1011 * If we're going to update the policy for the provided task,
1012 * enqueue it on the propagate queue itself. Otherwise, only
1013 * enqueue downstream things.
1015 if (update_task_imp
) {
1016 queue_enter(&propagate
, task_imp
, ipc_importance_task_t
, iit_props
);
1018 ipc_importance_task_propagate_helper(task_imp
, type
, &propagate
);
1022 * for each item on the propagation list, propagate any change downstream,
1023 * adding new tasks to propagate further if they transistioned as well.
1025 while (!queue_empty(&propagate
)) {
1026 boolean_t need_update
;
1028 queue_remove_first(&propagate
, temp_task_imp
, ipc_importance_task_t
, iit_props
);
1029 assert(IIT_NULL
!= temp_task_imp
);
1031 /* only propagate for receivers not already marked as a donor */
1032 if (!ipc_importance_task_is_marked_donor(temp_task_imp
) &&
1033 ipc_importance_task_is_marked_receiver(temp_task_imp
)) {
1034 ipc_importance_task_propagate_helper(temp_task_imp
, type
, &propagate
);
1037 /* if we have a policy update to apply, enqueue a reference for later processing */
1038 need_update
= (0 != temp_task_imp
->iit_updatepolicy
);
1039 temp_task_imp
->iit_updatepolicy
= 0;
1040 if (need_update
&& TASK_NULL
!= temp_task_imp
->iit_task
) {
1041 if (NULL
== temp_task_imp
->iit_updateq
) {
1044 * If a downstream task that needs an update is subjects to AppNap,
1045 * drop boosts according to the delay hysteresis. Otherwise,
1046 * immediate update it.
1048 if (!boost
&& temp_task_imp
!= task_imp
&&
1049 ipc_importance_delayed_drop_call
!= NULL
&&
1050 ipc_importance_task_is_marked_denap_receiver(temp_task_imp
)) {
1051 ipc_importance_task_delayed_drop(temp_task_imp
);
1053 temp_task_imp
->iit_updatetime
= 0;
1054 temp_task_imp
->iit_updateq
= &updates
;
1055 ipc_importance_task_reference_internal(temp_task_imp
);
1057 queue_enter(&updates
, temp_task_imp
,
1058 ipc_importance_task_t
, iit_updates
);
1060 queue_enter_first(&updates
, temp_task_imp
,
1061 ipc_importance_task_t
, iit_updates
);
1065 /* Must already be on the AppNap hysteresis queue */
1066 assert(ipc_importance_delayed_drop_call
!= NULL
);
1067 assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp
));
1072 /* apply updates to task (may drop importance lock) */
1073 if (!queue_empty(&updates
)) {
1074 ipc_importance_task_process_updates(&updates
, boost
, 0);
1079 * Routine: ipc_importance_task_hold_internal_assertion_locked
1081 * Increment the assertion count on the task importance.
1082 * If this results in a boost state change in that task,
1083 * prepare to update task policy for this task AND, if
1084 * if not just waking out of App Nap, all down-stream
1085 * tasks that have a similar transition through inheriting
1088 * importance locked on entry and exit.
1089 * May temporarily drop importance lock and block.
1091 static kern_return_t
1092 ipc_importance_task_hold_internal_assertion_locked(ipc_importance_task_t task_imp
, uint32_t count
)
1094 if (ipc_importance_task_check_transition(task_imp
, IIT_UPDATE_HOLD
, count
)) {
1095 ipc_importance_task_propagate_assertion_locked(task_imp
, IIT_UPDATE_HOLD
, TRUE
);
1097 return KERN_SUCCESS
;
1101 * Routine: ipc_importance_task_drop_internal_assertion_locked
1103 * Decrement the assertion count on the task importance.
1104 * If this results in a boost state change in that task,
1105 * prepare to update task policy for this task AND, if
1106 * if not just waking out of App Nap, all down-stream
1107 * tasks that have a similar transition through inheriting
1110 * importance locked on entry and exit.
1111 * May temporarily drop importance lock and block.
1113 static kern_return_t
1114 ipc_importance_task_drop_internal_assertion_locked(ipc_importance_task_t task_imp
, uint32_t count
)
1116 if (ipc_importance_task_check_transition(task_imp
, IIT_UPDATE_DROP
, count
)) {
1117 ipc_importance_task_propagate_assertion_locked(task_imp
, IIT_UPDATE_DROP
, TRUE
);
1119 return KERN_SUCCESS
;
1123 * Routine: ipc_importance_task_hold_internal_assertion
1125 * Increment the assertion count on the task importance.
1126 * If this results in a 0->1 change in that count,
1127 * prepare to update task policy for this task AND
1128 * (potentially) all down-stream tasks that have a
1129 * similar transition through inheriting this update.
1132 * May block after dropping importance lock.
1135 ipc_importance_task_hold_internal_assertion(ipc_importance_task_t task_imp
, uint32_t count
)
1137 int ret
= KERN_SUCCESS
;
1139 if (ipc_importance_task_is_any_receiver_type(task_imp
)) {
1140 ipc_importance_lock();
1141 ret
= ipc_importance_task_hold_internal_assertion_locked(task_imp
, count
);
1142 ipc_importance_unlock();
1148 * Routine: ipc_importance_task_drop_internal_assertion
1150 * Decrement the assertion count on the task importance.
1151 * If this results in a X->0 change in that count,
1152 * prepare to update task policy for this task AND
1153 * all down-stream tasks that have a similar transition
1154 * through inheriting this drop update.
1156 * Nothing locked on entry.
1157 * May block after dropping importance lock.
1160 ipc_importance_task_drop_internal_assertion(ipc_importance_task_t task_imp
, uint32_t count
)
1162 kern_return_t ret
= KERN_SUCCESS
;
1164 if (ipc_importance_task_is_any_receiver_type(task_imp
)) {
1165 ipc_importance_lock();
1166 ret
= ipc_importance_task_drop_internal_assertion_locked(task_imp
, count
);
1167 ipc_importance_unlock();
1173 * Routine: ipc_importance_task_hold_file_lock_assertion
1175 * Increment the file lock assertion count on the task importance.
1176 * If this results in a 0->1 change in that count,
1177 * prepare to update task policy for this task AND
1178 * (potentially) all down-stream tasks that have a
1179 * similar transition through inheriting this update.
1182 * May block after dropping importance lock.
1185 ipc_importance_task_hold_file_lock_assertion(ipc_importance_task_t task_imp
, uint32_t count
)
1187 kern_return_t ret
= KERN_SUCCESS
;
1189 if (ipc_importance_task_is_any_receiver_type(task_imp
)) {
1190 ipc_importance_lock();
1191 ret
= ipc_importance_task_hold_internal_assertion_locked(task_imp
, count
);
1192 if (KERN_SUCCESS
== ret
) {
1193 task_imp
->iit_filelocks
+= count
;
1195 ipc_importance_unlock();
1201 * Routine: ipc_importance_task_drop_file_lock_assertion
1203 * Decrement the assertion count on the task importance.
1204 * If this results in a X->0 change in that count,
1205 * prepare to update task policy for this task AND
1206 * all down-stream tasks that have a similar transition
1207 * through inheriting this drop update.
1209 * Nothing locked on entry.
1210 * May block after dropping importance lock.
1213 ipc_importance_task_drop_file_lock_assertion(ipc_importance_task_t task_imp
, uint32_t count
)
1215 kern_return_t ret
= KERN_SUCCESS
;
1217 if (ipc_importance_task_is_any_receiver_type(task_imp
)) {
1218 ipc_importance_lock();
1219 if (count
<= task_imp
->iit_filelocks
) {
1220 task_imp
->iit_filelocks
-= count
;
1221 ret
= ipc_importance_task_drop_internal_assertion_locked(task_imp
, count
);
1223 ret
= KERN_INVALID_ARGUMENT
;
1225 ipc_importance_unlock();
1231 * Routine: ipc_importance_task_hold_legacy_external_assertion
1233 * Increment the external assertion count on the task importance.
1234 * This cannot result in an 0->1 transition, as the caller must
1235 * already hold an external boost.
1237 * Nothing locked on entry.
1238 * May block after dropping importance lock.
1239 * A queue of task importance structures is returned
1240 * by ipc_importance_task_hold_assertion_locked(). Each
1241 * needs to be updated (outside the importance lock hold).
1244 ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_imp
, uint32_t count
)
1247 uint32_t target_assertcnt
;
1248 uint32_t target_externcnt
;
1249 uint32_t target_legacycnt
;
1253 ipc_importance_lock();
1254 target_task
= task_imp
->iit_task
;
1256 #if IMPORTANCE_DEBUG
1257 int target_pid
= task_pid(target_task
);
1259 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, (IMP_HOLD
| TASK_POLICY_EXTERNAL
))) | DBG_FUNC_START
,
1260 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_LEGACY_EXTERN(task_imp
), 0);
1263 if (IIT_LEGACY_EXTERN(task_imp
) == 0) {
1264 /* Only allowed to take a new boost assertion when holding an external boost */
1265 /* save data for diagnostic printf below */
1266 target_assertcnt
= task_imp
->iit_assertcnt
;
1267 target_externcnt
= IIT_EXTERN(task_imp
);
1268 target_legacycnt
= IIT_LEGACY_EXTERN(task_imp
);
1272 assert(ipc_importance_task_is_any_receiver_type(task_imp
));
1273 assert(0 < task_imp
->iit_assertcnt
);
1274 assert(0 < IIT_EXTERN(task_imp
));
1275 task_imp
->iit_assertcnt
+= count
;
1276 task_imp
->iit_externcnt
+= count
;
1277 task_imp
->iit_legacy_externcnt
+= count
;
1280 ipc_importance_unlock();
1282 #if IMPORTANCE_DEBUG
1283 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, (IMP_HOLD
| TASK_POLICY_EXTERNAL
))) | DBG_FUNC_END
,
1284 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_LEGACY_EXTERN(task_imp
), 0);
1285 // This covers the legacy case where a task takes an extra boost.
1286 DTRACE_BOOST5(receive_boost
, task_t
, target_task
, int, target_pid
, int, proc_selfpid(), int, count
, int, task_imp
->iit_assertcnt
);
1289 if (KERN_FAILURE
== ret
&& target_task
!= TASK_NULL
) {
1290 printf("BUG in process %s[%d]: "
1291 "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. "
1292 "(%d total, %d external, %d legacy-external)\n",
1293 proc_name_address(target_task
->bsd_info
), task_pid(target_task
),
1294 target_assertcnt
, target_externcnt
, target_legacycnt
);
1301 * Routine: ipc_importance_task_drop_legacy_external_assertion
1303 * Drop the legacy external assertion count on the task and
1304 * reflect that change to total external assertion count and
1305 * then onto the internal importance count.
1307 * If this results in a X->0 change in the internal,
1308 * count, prepare to update task policy for this task AND
1309 * all down-stream tasks that have a similar transition
1310 * through inheriting this update.
1312 * Nothing locked on entry.
1315 ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_imp
, uint32_t count
)
1317 int ret
= KERN_SUCCESS
;
1319 uint32_t target_assertcnt
;
1320 uint32_t target_externcnt
;
1321 uint32_t target_legacycnt
;
1324 return KERN_INVALID_ARGUMENT
;
1327 ipc_importance_lock();
1328 target_task
= task_imp
->iit_task
;
1330 #if IMPORTANCE_DEBUG
1331 int target_pid
= task_pid(target_task
);
1333 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, (IMP_DROP
| TASK_POLICY_EXTERNAL
))) | DBG_FUNC_START
,
1334 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_LEGACY_EXTERN(task_imp
), 0);
1337 if (count
> IIT_LEGACY_EXTERN(task_imp
)) {
1338 /* Process over-released its boost count - save data for diagnostic printf */
1339 /* TODO: If count > 1, we should clear out as many external assertions as there are left. */
1340 target_assertcnt
= task_imp
->iit_assertcnt
;
1341 target_externcnt
= IIT_EXTERN(task_imp
);
1342 target_legacycnt
= IIT_LEGACY_EXTERN(task_imp
);
1346 * decrement legacy external count from the top level and reflect
1347 * into internal for this and all subsequent updates.
1349 assert(ipc_importance_task_is_any_receiver_type(task_imp
));
1350 assert(IIT_EXTERN(task_imp
) >= count
);
1352 task_imp
->iit_legacy_externdrop
+= count
;
1353 task_imp
->iit_externdrop
+= count
;
1355 /* reset extern counters (if appropriate) */
1356 if (IIT_LEGACY_EXTERN(task_imp
) == 0) {
1357 if (IIT_EXTERN(task_imp
) != 0) {
1358 task_imp
->iit_externcnt
-= task_imp
->iit_legacy_externcnt
;
1359 task_imp
->iit_externdrop
-= task_imp
->iit_legacy_externdrop
;
1361 task_imp
->iit_externcnt
= 0;
1362 task_imp
->iit_externdrop
= 0;
1364 task_imp
->iit_legacy_externcnt
= 0;
1365 task_imp
->iit_legacy_externdrop
= 0;
1368 /* reflect the drop to the internal assertion count (and effect any importance change) */
1369 if (ipc_importance_task_check_transition(task_imp
, IIT_UPDATE_DROP
, count
)) {
1370 ipc_importance_task_propagate_assertion_locked(task_imp
, IIT_UPDATE_DROP
, TRUE
);
1375 #if IMPORTANCE_DEBUG
1376 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, (IMP_DROP
| TASK_POLICY_EXTERNAL
))) | DBG_FUNC_END
,
1377 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_LEGACY_EXTERN(task_imp
), 0);
1380 ipc_importance_unlock();
1382 /* delayed printf for user-supplied data failures */
1383 if (KERN_FAILURE
== ret
&& TASK_NULL
!= target_task
) {
1384 printf("BUG in process %s[%d]: over-released legacy external boost assertions (%d total, %d external, %d legacy-external)\n",
1385 proc_name_address(target_task
->bsd_info
), task_pid(target_task
),
1386 target_assertcnt
, target_externcnt
, target_legacycnt
);
1394 /* Transfer an assertion to legacy userspace responsibility */
1395 static kern_return_t
1396 ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp
, uint32_t count
, __unused
int sender_pid
)
1400 assert(IIT_NULL
!= task_imp
);
1401 target_task
= task_imp
->iit_task
;
1403 if (TASK_NULL
== target_task
||
1404 !ipc_importance_task_is_any_receiver_type(task_imp
)) {
1405 return KERN_FAILURE
;
1408 #if IMPORTANCE_DEBUG
1409 int target_pid
= task_pid(target_task
);
1411 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, IMP_EXTERN
)) | DBG_FUNC_START
,
1412 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_EXTERN(task_imp
), 0);
1415 ipc_importance_lock();
1416 /* assert(task_imp->iit_assertcnt >= IIT_EXTERN(task_imp) + count); */
1417 assert(IIT_EXTERN(task_imp
) >= IIT_LEGACY_EXTERN(task_imp
));
1418 task_imp
->iit_legacy_externcnt
+= count
;
1419 task_imp
->iit_externcnt
+= count
;
1420 ipc_importance_unlock();
1422 #if IMPORTANCE_DEBUG
1423 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_ASSERTION
, IMP_EXTERN
)) | DBG_FUNC_END
,
1424 proc_selfpid(), target_pid
, task_imp
->iit_assertcnt
, IIT_LEGACY_EXTERN(task_imp
), 0);
1425 // This is the legacy boosting path
1426 DTRACE_BOOST5(receive_boost
, task_t
, target_task
, int, target_pid
, int, sender_pid
, int, count
, int, IIT_LEGACY_EXTERN(task_imp
));
1427 #endif /* IMPORTANCE_DEBUG */
1429 return(KERN_SUCCESS
);
1433 * Routine: ipc_importance_task_update_live_donor
1435 * Read the live donor status and update the live_donor bit/propagate the change in importance.
1437 * Nothing locked on entrance, nothing locked on exit.
1439 * TODO: Need tracepoints around this function...
1442 ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp
)
1444 uint32_t task_live_donor
;
1445 boolean_t before_donor
;
1446 boolean_t after_donor
;
1449 assert(task_imp
!= NULL
);
1452 * Nothing to do if the task is not marked as expecting
1453 * live donor updates.
1455 if (!ipc_importance_task_is_marked_live_donor(task_imp
)) {
1459 ipc_importance_lock();
1461 /* If the task got disconnected on the way here, no use (or ability) adjusting live donor status */
1462 target_task
= task_imp
->iit_task
;
1463 if (TASK_NULL
== target_task
) {
1464 ipc_importance_unlock();
1467 before_donor
= ipc_importance_task_is_marked_donor(task_imp
);
1469 /* snapshot task live donor status - may change, but another call will accompany the change */
1470 task_live_donor
= target_task
->effective_policy
.t_live_donor
;
1472 #if IMPORTANCE_DEBUG
1473 int target_pid
= task_pid(target_task
);
1475 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
,
1476 (IMPORTANCE_CODE(IMP_DONOR_CHANGE
, IMP_DONOR_UPDATE_LIVE_DONOR_STATE
)) | DBG_FUNC_START
,
1477 target_pid
, task_imp
->iit_donor
, task_live_donor
, before_donor
, 0);
1480 /* update the task importance live donor status based on the task's value */
1481 task_imp
->iit_donor
= task_live_donor
;
1483 after_donor
= ipc_importance_task_is_marked_donor(task_imp
);
1485 /* Has the effectiveness of being a donor changed as a result of this update? */
1486 if (before_donor
!= after_donor
) {
1487 iit_update_type_t type
;
1489 /* propagate assertions without updating the current task policy (already handled) */
1490 if (0 == before_donor
) {
1491 task_imp
->iit_transitions
++;
1492 type
= IIT_UPDATE_HOLD
;
1494 type
= IIT_UPDATE_DROP
;
1496 ipc_importance_task_propagate_assertion_locked(task_imp
, type
, FALSE
);
1499 #if IMPORTANCE_DEBUG
1500 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
,
1501 (IMPORTANCE_CODE(IMP_DONOR_CHANGE
, IMP_DONOR_UPDATE_LIVE_DONOR_STATE
)) | DBG_FUNC_END
,
1502 target_pid
, task_imp
->iit_donor
, task_live_donor
, after_donor
, 0);
1505 ipc_importance_unlock();
1510 * Routine: ipc_importance_task_mark_donor
1512 * Set the task importance donor flag.
1514 * Nothing locked on entrance, nothing locked on exit.
1516 * This is only called while the task is being constructed,
1517 * so no need to update task policy or propagate downstream.
1520 ipc_importance_task_mark_donor(ipc_importance_task_t task_imp
, boolean_t donating
)
1522 assert(task_imp
!= NULL
);
1524 ipc_importance_lock();
1526 int old_donor
= task_imp
->iit_donor
;
1528 task_imp
->iit_donor
= (donating
? 1 : 0);
1530 if (task_imp
->iit_donor
> 0 && old_donor
== 0)
1531 task_imp
->iit_transitions
++;
1533 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
,
1534 (IMPORTANCE_CODE(IMP_DONOR_CHANGE
, IMP_DONOR_INIT_DONOR_STATE
)) | DBG_FUNC_NONE
,
1535 task_pid(task_imp
->iit_task
), donating
,
1536 old_donor
, task_imp
->iit_donor
, 0);
1538 ipc_importance_unlock();
1542 * Routine: ipc_importance_task_marked_donor
1544 * Query the donor flag for the given task importance.
1546 * May be called without taking the importance lock.
1547 * In that case, donor status can change so you must
1548 * check only once for each donation event.
1551 ipc_importance_task_is_marked_donor(ipc_importance_task_t task_imp
)
1553 if (IIT_NULL
== task_imp
) {
1556 return (0 != task_imp
->iit_donor
);
1560 * Routine: ipc_importance_task_mark_live_donor
1562 * Indicate that the task is eligible for live donor updates.
1564 * Nothing locked on entrance, nothing locked on exit.
1566 * This is only called while the task is being constructed.
1569 ipc_importance_task_mark_live_donor(ipc_importance_task_t task_imp
, boolean_t live_donating
)
1571 assert(task_imp
!= NULL
);
1573 ipc_importance_lock();
1574 task_imp
->iit_live_donor
= (live_donating
? 1 : 0);
1575 ipc_importance_unlock();
1579 * Routine: ipc_importance_task_marked_live_donor
1581 * Query the live donor and donor flags for the given task importance.
1583 * May be called without taking the importance lock.
1584 * In that case, donor status can change so you must
1585 * check only once for each donation event.
1588 ipc_importance_task_is_marked_live_donor(ipc_importance_task_t task_imp
)
1590 if (IIT_NULL
== task_imp
) {
1593 return (0 != task_imp
->iit_live_donor
);
1597 * Routine: ipc_importance_task_is_donor
1599 * Query the full donor status for the given task importance.
1601 * May be called without taking the importance lock.
1602 * In that case, donor status can change so you must
1603 * check only once for each donation event.
1606 ipc_importance_task_is_donor(ipc_importance_task_t task_imp
)
1608 if (IIT_NULL
== task_imp
) {
1611 return (ipc_importance_task_is_marked_donor(task_imp
) ||
1612 (ipc_importance_task_is_marked_receiver(task_imp
) &&
1613 task_imp
->iit_assertcnt
> 0));
1617 * Routine: ipc_importance_task_is_never_donor
1619 * Query if a given task can ever donate importance.
1621 * May be called without taking the importance lock.
1622 * Condition is permanent for a give task.
1625 ipc_importance_task_is_never_donor(ipc_importance_task_t task_imp
)
1627 if (IIT_NULL
== task_imp
) {
1630 return (!ipc_importance_task_is_marked_donor(task_imp
) &&
1631 !ipc_importance_task_is_marked_live_donor(task_imp
) &&
1632 !ipc_importance_task_is_marked_receiver(task_imp
));
1636 * Routine: ipc_importance_task_mark_receiver
1638 * Update the task importance receiver flag.
1640 * Nothing locked on entrance, nothing locked on exit.
1641 * This can only be invoked before the task is discoverable,
1642 * so no worries about atomicity(?)
1645 ipc_importance_task_mark_receiver(ipc_importance_task_t task_imp
, boolean_t receiving
)
1647 assert(task_imp
!= NULL
);
1649 ipc_importance_lock();
1651 assert(task_imp
->iit_assertcnt
== 0);
1652 assert(task_imp
->iit_externcnt
== 0);
1653 assert(task_imp
->iit_externdrop
== 0);
1654 assert(task_imp
->iit_denap
== 0);
1655 task_imp
->iit_receiver
= 1; /* task can receive importance boost */
1656 } else if (task_imp
->iit_receiver
) {
1657 assert(task_imp
->iit_denap
== 0);
1658 if (task_imp
->iit_assertcnt
!= 0 || IIT_EXTERN(task_imp
) != 0) {
1659 panic("disabling imp_receiver on task with pending importance boosts!");
1661 task_imp
->iit_receiver
= 0;
1663 ipc_importance_unlock();
1668 * Routine: ipc_importance_task_marked_receiver
1670 * Query the receiver flag for the given task importance.
1672 * May be called without taking the importance lock as
1673 * the importance flag can never change after task init.
1676 ipc_importance_task_is_marked_receiver(ipc_importance_task_t task_imp
)
1678 return (IIT_NULL
!= task_imp
&& 0 != task_imp
->iit_receiver
);
1683 * Routine: ipc_importance_task_mark_denap_receiver
1685 * Update the task importance de-nap receiver flag.
1687 * Nothing locked on entrance, nothing locked on exit.
1688 * This can only be invoked before the task is discoverable,
1689 * so no worries about atomicity(?)
1692 ipc_importance_task_mark_denap_receiver(ipc_importance_task_t task_imp
, boolean_t denap
)
1694 assert(task_imp
!= NULL
);
1696 ipc_importance_lock();
1698 assert(task_imp
->iit_assertcnt
== 0);
1699 assert(task_imp
->iit_externcnt
== 0);
1700 assert(task_imp
->iit_receiver
== 0);
1701 task_imp
->iit_denap
= 1; /* task can receive de-nap boost */
1702 } else if (task_imp
->iit_denap
) {
1703 assert(task_imp
->iit_receiver
== 0);
1704 if (0 < task_imp
->iit_assertcnt
|| 0 < IIT_EXTERN(task_imp
)) {
1705 panic("disabling de-nap on task with pending de-nap boosts!");
1707 task_imp
->iit_denap
= 0;
1709 ipc_importance_unlock();
1714 * Routine: ipc_importance_task_marked_denap_receiver
1716 * Query the de-nap receiver flag for the given task importance.
1718 * May be called without taking the importance lock as
1719 * the de-nap flag can never change after task init.
1722 ipc_importance_task_is_marked_denap_receiver(ipc_importance_task_t task_imp
)
1724 return (IIT_NULL
!= task_imp
&& 0 != task_imp
->iit_denap
);
1728 * Routine: ipc_importance_task_is_denap_receiver
1730 * Query the full de-nap receiver status for the given task importance.
1731 * For now, that is simply whether the receiver flag is set.
1733 * May be called without taking the importance lock as
1734 * the de-nap receiver flag can never change after task init.
1737 ipc_importance_task_is_denap_receiver(ipc_importance_task_t task_imp
)
1739 return (ipc_importance_task_is_marked_denap_receiver(task_imp
));
1743 * Routine: ipc_importance_task_is_any_receiver_type
1745 * Query if the task is marked to receive boosts - either
1746 * importance or denap.
1748 * May be called without taking the importance lock as both
1749 * the importance and de-nap receiver flags can never change
1753 ipc_importance_task_is_any_receiver_type(ipc_importance_task_t task_imp
)
1755 return (ipc_importance_task_is_marked_receiver(task_imp
) ||
1756 ipc_importance_task_is_marked_denap_receiver(task_imp
));
1759 #if 0 /* currently unused */
1762 * Routine: ipc_importance_inherit_reference
1764 * Add a reference to the inherit importance element.
1766 * Caller most hold a reference on the inherit element.
1769 ipc_importance_inherit_reference(ipc_importance_inherit_t inherit
)
1771 ipc_importance_reference(&inherit
->iii_elem
);
1773 #endif /* currently unused */
1776 * Routine: ipc_importance_inherit_release_locked
1778 * Release a reference on an inherit importance attribute value,
1779 * unlinking and deallocating the attribute if the last reference.
1781 * Entered with importance lock held, leaves with it unlocked.
1784 ipc_importance_inherit_release_locked(ipc_importance_inherit_t inherit
)
1786 ipc_importance_release_locked(&inherit
->iii_elem
);
1789 #if 0 /* currently unused */
1791 * Routine: ipc_importance_inherit_release
1793 * Release a reference on an inherit importance attribute value,
1794 * unlinking and deallocating the attribute if the last reference.
1796 * nothing locked on entrance, nothing locked on exit.
1800 ipc_importance_inherit_release(ipc_importance_inherit_t inherit
)
1802 if (III_NULL
!= inherit
)
1803 ipc_importance_release(&inherit
->iii_elem
);
1805 #endif /* 0 currently unused */
1808 * Routine: ipc_importance_for_task
1810 * Create a reference for the specified task's base importance
1811 * element. If the base importance element doesn't exist, make it and
1812 * bind it to the active task. If the task is inactive, there isn't
1813 * any need to return a new reference.
1815 * If made is true, a "made" reference is returned (for donating to
1816 * the voucher system). Otherwise an internal reference is returned.
1818 * Nothing locked on entry. May block.
1820 ipc_importance_task_t
1821 ipc_importance_for_task(task_t task
, boolean_t made
)
1823 ipc_importance_task_t task_elem
;
1824 boolean_t first_pass
= TRUE
;
1826 assert(TASK_NULL
!= task
);
1829 /* No use returning anything for inactive task */
1833 ipc_importance_lock();
1834 task_elem
= task
->task_imp_base
;
1835 if (IIT_NULL
!= task_elem
) {
1836 /* Add a made reference (borrowing active task ref to do it) */
1838 if (0 == task_elem
->iit_made
++) {
1839 assert(IIT_REFS_MAX
> IIT_REFS(task_elem
));
1840 ipc_importance_task_reference_internal(task_elem
);
1843 assert(IIT_REFS_MAX
> IIT_REFS(task_elem
));
1844 ipc_importance_task_reference_internal(task_elem
);
1846 ipc_importance_unlock();
1849 ipc_importance_unlock();
1855 /* Need to make one - may race with others (be prepared to drop) */
1856 task_elem
= (ipc_importance_task_t
)zalloc(ipc_importance_task_zone
);
1857 if (IIT_NULL
== task_elem
)
1860 task_elem
->iit_bits
= IIE_TYPE_TASK
| 2; /* one for task, one for return/made */
1861 task_elem
->iit_made
= (made
) ? 1 : 0;
1862 task_elem
->iit_task
= task
; /* take actual ref when we're sure */
1863 task_elem
->iit_updateq
= NULL
;
1864 task_elem
->iit_receiver
= 0;
1865 task_elem
->iit_denap
= 0;
1866 task_elem
->iit_donor
= 0;
1867 task_elem
->iit_live_donor
= 0;
1868 task_elem
->iit_updatepolicy
= 0;
1869 task_elem
->iit_reserved
= 0;
1870 task_elem
->iit_filelocks
= 0;
1871 task_elem
->iit_updatetime
= 0;
1872 task_elem
->iit_transitions
= 0;
1873 task_elem
->iit_assertcnt
= 0;
1874 task_elem
->iit_externcnt
= 0;
1875 task_elem
->iit_externdrop
= 0;
1876 task_elem
->iit_legacy_externcnt
= 0;
1877 task_elem
->iit_legacy_externdrop
= 0;
1879 ipc_importance_counter_init(&task_elem
->iit_elem
);
1881 queue_init(&task_elem
->iit_kmsgs
);
1882 queue_init(&task_elem
->iit_inherits
);
1884 ipc_importance_lock();
1885 if (!task
->active
) {
1886 ipc_importance_unlock();
1887 zfree(ipc_importance_task_zone
, task_elem
);
1891 /* did we lose the race? */
1892 if (IIT_NULL
!= task
->task_imp_base
) {
1893 ipc_importance_unlock();
1894 zfree(ipc_importance_task_zone
, task_elem
);
1898 /* we won the race */
1899 task
->task_imp_base
= task_elem
;
1900 task_reference(task
);
1901 #if DEVELOPMENT || DEBUG
1902 queue_enter(&global_iit_alloc_queue
, task_elem
, ipc_importance_task_t
, iit_allocation
);
1903 task_importance_update_owner_info(task
);
1905 ipc_importance_unlock();
1910 #if DEVELOPMENT || DEBUG
1911 void task_importance_update_owner_info(task_t task
) {
1913 if (task
!= TASK_NULL
&& task
->task_imp_base
!= IIT_NULL
) {
1914 ipc_importance_task_t task_elem
= task
->task_imp_base
;
1916 task_elem
->iit_bsd_pid
= task_pid(task
);
1917 if (task
->bsd_info
) {
1918 strncpy(&task_elem
->iit_procname
[0], proc_name_address(task
->bsd_info
), 16);
1919 task_elem
->iit_procname
[16] = '\0';
1921 strncpy(&task_elem
->iit_procname
[0], "unknown", 16);
1928 * Routine: ipc_importance_reset_locked
1930 * Reset a task's IPC importance (the task is going away or exec'ing)
1932 * Remove the donor bit and legacy externalized assertions from the
1933 * current task importance and see if that wipes out downstream donations.
1935 * importance lock held.
1939 ipc_importance_reset_locked(ipc_importance_task_t task_imp
, boolean_t donor
)
1941 boolean_t before_donor
, after_donor
;
1943 /* remove the donor bit, live-donor bit and externalized boosts */
1944 before_donor
= ipc_importance_task_is_donor(task_imp
);
1946 task_imp
->iit_donor
= 0;
1948 assert(IIT_LEGACY_EXTERN(task_imp
) <= IIT_EXTERN(task_imp
));
1949 assert(task_imp
->iit_legacy_externcnt
<= task_imp
->iit_externcnt
);
1950 assert(task_imp
->iit_legacy_externdrop
<= task_imp
->iit_externdrop
);
1951 task_imp
->iit_externcnt
-= task_imp
->iit_legacy_externcnt
;
1952 task_imp
->iit_externdrop
-= task_imp
->iit_legacy_externdrop
;
1954 /* assert(IIT_LEGACY_EXTERN(task_imp) <= task_imp->iit_assertcnt); */
1955 if (IIT_LEGACY_EXTERN(task_imp
) < task_imp
->iit_assertcnt
) {
1956 task_imp
->iit_assertcnt
-= IIT_LEGACY_EXTERN(task_imp
);
1958 assert(IIT_LEGACY_EXTERN(task_imp
) == task_imp
->iit_assertcnt
);
1959 task_imp
->iit_assertcnt
= 0;
1961 task_imp
->iit_legacy_externcnt
= 0;
1962 task_imp
->iit_legacy_externdrop
= 0;
1963 after_donor
= ipc_importance_task_is_donor(task_imp
);
1965 #if DEVELOPMENT || DEBUG
1966 if (task_imp
->iit_assertcnt
> 0 && task_imp
->iit_live_donor
) {
1967 printf("Live donor task %s[%d] still has %d importance assertions after reset\n",
1968 task_imp
->iit_procname
, task_imp
->iit_bsd_pid
, task_imp
->iit_assertcnt
);
1972 /* propagate a downstream drop if there was a change in donor status */
1973 if (after_donor
!= before_donor
) {
1974 ipc_importance_task_propagate_assertion_locked(task_imp
, IIT_UPDATE_DROP
, FALSE
);
1979 * Routine: ipc_importance_reset
1981 * Reset a task's IPC importance
1983 * The task is being reset, although staying around. Arrange to have the
1984 * external state of the task reset from the importance.
1986 * importance lock not held.
1990 ipc_importance_reset(ipc_importance_task_t task_imp
, boolean_t donor
)
1992 if (IIT_NULL
== task_imp
) {
1995 ipc_importance_lock();
1996 ipc_importance_reset_locked(task_imp
, donor
);
1997 ipc_importance_unlock();
2001 * Routine: ipc_importance_disconnect_task
2003 * Disconnect a task from its importance.
2005 * Clear the task pointer from the importance and drop the
2006 * reference the task held on the importance object. Before
2007 * doing that, reset the effects the current task holds on
2008 * the importance and see if that wipes out downstream donations.
2010 * We allow the upstream boosts to continue to affect downstream
2011 * even though the local task is being effectively pulled from
2017 ipc_importance_disconnect_task(task_t task
)
2019 ipc_importance_task_t task_imp
;
2022 ipc_importance_lock();
2023 task_imp
= task
->task_imp_base
;
2025 /* did somebody beat us to it? */
2026 if (IIT_NULL
== task_imp
) {
2027 ipc_importance_unlock();
2032 /* disconnect the task from this importance */
2033 assert(task_imp
->iit_task
== task
);
2034 task_imp
->iit_task
= TASK_NULL
;
2035 task
->task_imp_base
= IIT_NULL
;
2038 /* reset the effects the current task hold on the importance */
2039 ipc_importance_reset_locked(task_imp
, TRUE
);
2041 ipc_importance_task_release_locked(task_imp
);
2042 /* importance unlocked */
2044 /* deallocate the task now that the importance is unlocked */
2045 task_deallocate(task
);
2049 * Routine: ipc_importance_send
2051 * Post the importance voucher attribute [if sent] or a static
2052 * importance boost depending upon options and conditions.
2054 * Destination port locked on entry and exit, may be dropped during the call.
2056 * A boolean identifying if the port lock was tempoarily dropped.
2059 ipc_importance_send(
2061 mach_msg_option_t option
)
2063 ipc_port_t port
= (ipc_port_t
) kmsg
->ikm_header
->msgh_remote_port
;
2064 boolean_t port_lock_dropped
= FALSE
;
2065 ipc_importance_elem_t elem
;
2067 ipc_importance_task_t task_imp
;
2071 assert(IP_VALID(port
));
2073 /* If no donation to be made, return quickly */
2074 if ((port
->ip_impdonation
== 0) ||
2075 (option
& MACH_SEND_NOIMPORTANCE
) != 0) {
2076 return port_lock_dropped
;
2079 task
= current_task();
2081 /* If forced sending a static boost, go update the port */
2082 if ((option
& MACH_SEND_IMPORTANCE
) != 0) {
2083 kmsg
->ikm_header
->msgh_bits
|= MACH_MSGH_BITS_RAISEIMP
;
2087 task_imp
= task
->task_imp_base
;
2088 assert(IIT_NULL
!= task_imp
);
2090 /* If the sender can never donate importance, nothing to do */
2091 if (ipc_importance_task_is_never_donor(task_imp
)) {
2092 return port_lock_dropped
;
2097 /* If importance receiver and passing a voucher, look for importance in there */
2098 if (IP_VALID(kmsg
->ikm_voucher
) &&
2099 ipc_importance_task_is_marked_receiver(task_imp
)) {
2100 mach_voucher_attr_value_handle_t vals
[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED
];
2101 mach_voucher_attr_value_handle_array_size_t val_count
;
2102 ipc_voucher_t voucher
;
2104 assert(ip_kotype(kmsg
->ikm_voucher
) == IKOT_VOUCHER
);
2105 voucher
= (ipc_voucher_t
)kmsg
->ikm_voucher
->ip_kobject
;
2107 /* check to see if the voucher has an importance attribute */
2108 val_count
= MACH_VOUCHER_ATTR_VALUE_MAX_NESTED
;
2109 kr
= mach_voucher_attr_control_get_values(ipc_importance_control
, voucher
,
2111 assert(KERN_SUCCESS
== kr
);
2114 * Only use importance associated with our task (either directly
2115 * or through an inherit that donates to our task).
2117 if (0 < val_count
) {
2118 ipc_importance_elem_t check_elem
;
2120 check_elem
= (ipc_importance_elem_t
)vals
[0];
2121 assert(IIE_NULL
!= check_elem
);
2122 if (IIE_TYPE_INHERIT
== IIE_TYPE(check_elem
)) {
2123 ipc_importance_inherit_t inherit
;
2124 inherit
= (ipc_importance_inherit_t
) check_elem
;
2125 if (inherit
->iii_to_task
== task_imp
) {
2128 } else if (check_elem
== (ipc_importance_elem_t
)task_imp
) {
2134 /* If we haven't found an importance attribute to send yet, use the task's */
2135 if (IIE_NULL
== elem
) {
2136 elem
= (ipc_importance_elem_t
)task_imp
;
2139 /* take a reference for the message to hold */
2140 ipc_importance_reference_internal(elem
);
2142 /* acquire the importance lock while trying to hang on to port lock */
2143 if (!ipc_importance_lock_try()) {
2144 port_lock_dropped
= TRUE
;
2146 ipc_importance_lock();
2149 /* link kmsg onto the donor element propagation chain */
2150 ipc_importance_kmsg_link(kmsg
, elem
);
2151 /* elem reference transfered to kmsg */
2153 incr_ref_counter(elem
->iie_kmsg_refs_added
);
2155 /* If the sender isn't currently a donor, no need to apply boost */
2156 if (!ipc_importance_task_is_donor(task_imp
)) {
2157 ipc_importance_unlock();
2159 /* re-acquire port lock, if needed */
2160 if (TRUE
== port_lock_dropped
)
2163 return port_lock_dropped
;
2166 /* Mark the fact that we are (currently) donating through this message */
2167 kmsg
->ikm_header
->msgh_bits
|= MACH_MSGH_BITS_RAISEIMP
;
2170 * If we need to relock the port, do it with the importance still locked.
2171 * This assures we get to add the importance boost through the port to
2172 * the task BEFORE anyone else can attempt to undo that operation because
2173 * the sender lost donor status.
2175 if (TRUE
== port_lock_dropped
) {
2178 ipc_importance_unlock();
2182 #if IMPORTANCE_DEBUG
2183 if (kdebug_enable
) {
2184 mach_msg_max_trailer_t
*dbgtrailer
= (mach_msg_max_trailer_t
*)
2185 ((vm_offset_t
)kmsg
->ikm_header
+ round_msg(kmsg
->ikm_header
->msgh_size
));
2186 unsigned int sender_pid
= dbgtrailer
->msgh_audit
.val
[5];
2187 mach_msg_id_t imp_msgh_id
= kmsg
->ikm_header
->msgh_id
;
2188 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_MSG
, IMP_MSG_SEND
)) | DBG_FUNC_START
,
2189 task_pid(task
), sender_pid
, imp_msgh_id
, 0, 0);
2191 #endif /* IMPORTANCE_DEBUG */
2193 /* adjust port boost count (with port locked) */
2194 if (TRUE
== ipc_port_importance_delta(port
, 1)) {
2195 port_lock_dropped
= TRUE
;
2198 return port_lock_dropped
;
2202 * Routine: ipc_importance_inherit_from
2204 * Create a "made" reference for an importance attribute representing
2205 * an inheritance between the sender of a message (if linked) and the
2206 * current task importance. If the message is not linked, a static
2207 * boost may be created, based on the boost state of the message.
2209 * Any transfer from kmsg linkage to inherit linkage must be atomic.
2211 * If the task is inactive, there isn't any need to return a new reference.
2213 * Nothing locked on entry. May block.
2215 static ipc_importance_inherit_t
2216 ipc_importance_inherit_from(ipc_kmsg_t kmsg
)
2218 ipc_importance_task_t task_imp
= IIT_NULL
;
2219 ipc_importance_elem_t from_elem
= kmsg
->ikm_importance
;
2220 ipc_importance_elem_t elem
;
2221 task_t task_self
= current_task();
2223 ipc_port_t port
= kmsg
->ikm_header
->msgh_remote_port
;
2224 ipc_importance_inherit_t inherit
= III_NULL
;
2225 ipc_importance_inherit_t alloc
= III_NULL
;
2226 ipc_importance_inherit_t temp_inherit
;
2227 boolean_t cleared_self_donation
= FALSE
;
2231 /* The kmsg must have an importance donor or static boost to proceed */
2232 if (IIE_NULL
== kmsg
->ikm_importance
&&
2233 !MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
)) {
2238 * No need to set up an inherit linkage if the dest isn't a receiver
2239 * of one type or the other.
2241 if (!ipc_importance_task_is_any_receiver_type(task_self
->task_imp_base
)) {
2242 ipc_importance_lock();
2246 /* Grab a reference on the importance of the destination */
2247 task_imp
= ipc_importance_for_task(task_self
, FALSE
);
2249 ipc_importance_lock();
2251 if (IIT_NULL
== task_imp
) {
2255 incr_ref_counter(task_imp
->iit_elem
.iie_task_refs_added_inherit_from
);
2257 /* If message is already associated with an inherit... */
2258 if (IIE_TYPE_INHERIT
== IIE_TYPE(from_elem
)) {
2259 ipc_importance_inherit_t from_inherit
= (ipc_importance_inherit_t
)from_elem
;
2261 /* already targeting our task? - just use it */
2262 if (from_inherit
->iii_to_task
== task_imp
) {
2263 /* clear self-donation if not also present in inherit */
2264 if (!from_inherit
->iii_donating
&&
2265 MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
)) {
2266 kmsg
->ikm_header
->msgh_bits
&= ~MACH_MSGH_BITS_RAISEIMP
;
2267 cleared_self_donation
= TRUE
;
2269 inherit
= from_inherit
;
2271 } else if (III_DEPTH_MAX
== III_DEPTH(from_inherit
)) {
2272 ipc_importance_task_t to_task
;
2273 ipc_importance_elem_t unlinked_from
;
2276 * Chain too long. Switch to looking
2277 * directly at the from_inherit's to-task
2278 * as our source of importance.
2280 to_task
= from_inherit
->iii_to_task
;
2281 ipc_importance_task_reference(to_task
);
2282 from_elem
= (ipc_importance_elem_t
)to_task
;
2283 depth
= III_DEPTH_RESET
| 1;
2285 /* Fixup the kmsg linkage to reflect change */
2286 unlinked_from
= ipc_importance_kmsg_unlink(kmsg
);
2287 assert(unlinked_from
== (ipc_importance_elem_t
)from_inherit
);
2288 ipc_importance_kmsg_link(kmsg
, from_elem
);
2289 ipc_importance_inherit_release_locked(from_inherit
);
2290 /* importance unlocked */
2291 ipc_importance_lock();
2294 /* inheriting from an inherit */
2295 depth
= from_inherit
->iii_depth
+ 1;
2300 * Don't allow a task to inherit from itself (would keep it permanently
2301 * boosted even if all other donors to the task went away).
2304 if (from_elem
== (ipc_importance_elem_t
)task_imp
) {
2309 * But if the message isn't associated with any linked source, it is
2310 * intended to be permanently boosting (static boost from kernel).
2311 * In that case DO let the process permanently boost itself.
2313 if (IIE_NULL
== from_elem
) {
2314 assert(MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
));
2315 ipc_importance_task_reference_internal(task_imp
);
2316 from_elem
= (ipc_importance_elem_t
)task_imp
;
2320 * Now that we have the from_elem figured out,
2321 * check to see if we already have an inherit for this pairing
2323 while (III_NULL
== inherit
) {
2324 queue_iterate(&from_elem
->iie_inherits
, temp_inherit
,
2325 ipc_importance_inherit_t
, iii_inheritance
) {
2326 if (temp_inherit
->iii_to_task
== task_imp
&&
2327 temp_inherit
->iii_depth
== depth
) {
2328 inherit
= temp_inherit
;
2333 /* Do we have to allocate a new inherit */
2334 if (III_NULL
== inherit
) {
2335 if (III_NULL
!= alloc
) {
2339 /* allocate space */
2340 ipc_importance_unlock();
2341 alloc
= (ipc_importance_inherit_t
)
2342 zalloc(ipc_importance_inherit_zone
);
2343 ipc_importance_lock();
2347 /* snapshot the donating status while we have importance locked */
2348 donating
= MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
);
2350 if (III_NULL
!= inherit
) {
2351 /* We found one, piggyback on that */
2352 assert(0 < III_REFS(inherit
));
2353 assert(0 < IIE_REFS(inherit
->iii_from_elem
));
2354 assert(inherit
->iii_externcnt
>= inherit
->iii_made
);
2356 /* add in a made reference */
2357 if (0 == inherit
->iii_made
++) {
2358 assert(III_REFS_MAX
> III_REFS(inherit
));
2359 ipc_importance_inherit_reference_internal(inherit
);
2362 /* Reflect the inherit's change of status into the task boosts */
2363 if (0 == III_EXTERN(inherit
)) {
2364 assert(!inherit
->iii_donating
);
2365 inherit
->iii_donating
= donating
;
2367 task_imp
->iit_externcnt
+= inherit
->iii_externcnt
;
2368 task_imp
->iit_externdrop
+= inherit
->iii_externdrop
;
2371 assert(donating
== inherit
->iii_donating
);
2374 /* add in a external reference for this use of the inherit */
2375 inherit
->iii_externcnt
++;
2377 task_imp
->iit_externcnt
++;
2380 /* initialize the previously allocated space */
2382 inherit
->iii_bits
= IIE_TYPE_INHERIT
| 1;
2383 inherit
->iii_made
= 1;
2384 inherit
->iii_externcnt
= 1;
2385 inherit
->iii_externdrop
= 0;
2386 inherit
->iii_depth
= depth
;
2387 inherit
->iii_to_task
= task_imp
;
2388 inherit
->iii_from_elem
= IIE_NULL
;
2389 queue_init(&inherit
->iii_kmsgs
);
2390 queue_init(&inherit
->iii_inherits
);
2392 /* If donating, reflect that in the task externcnt */
2394 inherit
->iii_donating
= TRUE
;
2395 task_imp
->iit_externcnt
++;
2397 inherit
->iii_donating
= FALSE
;
2401 * Chain our new inherit on the element it inherits from.
2402 * The new inherit takes our reference on from_elem.
2404 ipc_importance_inherit_link(inherit
, from_elem
);
2407 ipc_importance_counter_init(&inherit
->iii_elem
);
2408 from_elem
->iie_kmsg_refs_inherited
++;
2409 task_imp
->iit_elem
.iie_task_refs_inherited
++;
2415 * for those paths that came straight here: snapshot the donating status
2416 * (this should match previous snapshot for other paths).
2418 donating
= MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
);
2420 /* unlink the kmsg inheritance (if any) */
2421 elem
= ipc_importance_kmsg_unlink(kmsg
);
2422 assert(elem
== from_elem
);
2424 /* If we didn't create a new inherit, we have some resources to release */
2425 if (III_NULL
== inherit
|| inherit
!= alloc
) {
2426 if (IIE_NULL
!= from_elem
) {
2427 if (III_NULL
!= inherit
) {
2428 incr_ref_counter(from_elem
->iie_kmsg_refs_coalesced
);
2430 incr_ref_counter(from_elem
->iie_kmsg_refs_dropped
);
2432 ipc_importance_release_locked(from_elem
);
2433 /* importance unlocked */
2435 ipc_importance_unlock();
2438 if (IIT_NULL
!= task_imp
) {
2439 if (III_NULL
!= inherit
) {
2440 incr_ref_counter(task_imp
->iit_elem
.iie_task_refs_coalesced
);
2442 ipc_importance_task_release(task_imp
);
2445 if (III_NULL
!= alloc
)
2446 zfree(ipc_importance_inherit_zone
, alloc
);
2448 /* from_elem and task_imp references transferred to new inherit */
2449 ipc_importance_unlock();
2452 /* decrement port boost count */
2455 if (III_NULL
!= inherit
) {
2456 /* task assertions transferred to inherit, just adjust port count */
2457 ipc_port_impcount_delta(port
, -1, IP_NULL
);
2460 /* drop importance from port and destination task */
2461 if (ipc_port_importance_delta(port
, -1) == FALSE
) {
2465 } else if (cleared_self_donation
) {
2467 /* drop cleared donation from port and destination task */
2468 if (ipc_port_importance_delta(port
, -1) == FALSE
) {
2473 if (III_NULL
!= inherit
) {
2474 /* have an associated importance attr, even if currently not donating */
2475 kmsg
->ikm_header
->msgh_bits
|= MACH_MSGH_BITS_RAISEIMP
;
2477 /* we won't have an importance attribute associated with our message */
2478 kmsg
->ikm_header
->msgh_bits
&= ~MACH_MSGH_BITS_RAISEIMP
;
2485 * Routine: ipc_importance_receive
2487 * Process importance attributes in a received message.
2489 * If an importance voucher attribute was sent, transform
2490 * that into an attribute value reflecting the inheritance
2491 * from the sender to the receiver.
2493 * If a static boost is received (or the voucher isn't on
2494 * a voucher-based boost), export a static boost.
2499 ipc_importance_receive(
2501 mach_msg_option_t option
)
2503 unsigned int sender_pid
= ((mach_msg_max_trailer_t
*)
2504 ((vm_offset_t
)kmsg
->ikm_header
+
2505 round_msg(kmsg
->ikm_header
->msgh_size
)))->msgh_audit
.val
[5];
2506 task_t task_self
= current_task();
2509 /* convert to a voucher with an inherit importance attribute? */
2510 if ((option
& MACH_RCV_VOUCHER
) != 0) {
2511 uint8_t recipes
[2 * sizeof(ipc_voucher_attr_recipe_data_t
) +
2512 sizeof(mach_voucher_attr_value_handle_t
)];
2513 ipc_voucher_attr_raw_recipe_array_size_t recipe_size
= 0;
2514 ipc_voucher_attr_recipe_t recipe
= (ipc_voucher_attr_recipe_t
)recipes
;
2515 ipc_voucher_t recv_voucher
;
2516 mach_voucher_attr_value_handle_t handle
;
2517 ipc_importance_inherit_t inherit
;
2520 /* set up recipe to copy the old voucher */
2521 if (IP_VALID(kmsg
->ikm_voucher
)) {
2522 ipc_voucher_t sent_voucher
= (ipc_voucher_t
)kmsg
->ikm_voucher
->ip_kobject
;
2524 recipe
->key
= MACH_VOUCHER_ATTR_KEY_ALL
;
2525 recipe
->command
= MACH_VOUCHER_ATTR_COPY
;
2526 recipe
->previous_voucher
= sent_voucher
;
2527 recipe
->content_size
= 0;
2528 recipe_size
+= sizeof(*recipe
);
2532 * create an inheritance attribute from the kmsg (may be NULL)
2533 * transferring any boosts from the kmsg linkage through the
2534 * port directly to the new inheritance object.
2536 inherit
= ipc_importance_inherit_from(kmsg
);
2537 handle
= (mach_voucher_attr_value_handle_t
)inherit
;
2539 assert(IIE_NULL
== kmsg
->ikm_importance
);
2542 * Only create a new voucher if we have an inherit object
2543 * (from the ikm_importance field of the incoming message), OR
2544 * we have a valid incoming voucher. If we have neither of
2545 * these things then there is no need to create a new voucher.
2547 if (IP_VALID(kmsg
->ikm_voucher
) || inherit
!= III_NULL
) {
2548 /* replace the importance attribute with the handle we created */
2549 /* our made reference on the inherit is donated to the voucher */
2550 recipe
= (ipc_voucher_attr_recipe_t
)&recipes
[recipe_size
];
2551 recipe
->key
= MACH_VOUCHER_ATTR_KEY_IMPORTANCE
;
2552 recipe
->command
= MACH_VOUCHER_ATTR_SET_VALUE_HANDLE
;
2553 recipe
->previous_voucher
= IPC_VOUCHER_NULL
;
2554 recipe
->content_size
= sizeof(mach_voucher_attr_value_handle_t
);
2555 *(mach_voucher_attr_value_handle_t
*)(void *)recipe
->content
= handle
;
2556 recipe_size
+= sizeof(*recipe
) + sizeof(mach_voucher_attr_value_handle_t
);
2558 kr
= ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control
,
2562 assert(KERN_SUCCESS
== kr
);
2564 /* swap the voucher port (and set voucher bits in case it didn't already exist) */
2565 kmsg
->ikm_header
->msgh_bits
|= (MACH_MSG_TYPE_MOVE_SEND
<< 16);
2566 ipc_port_release_send(kmsg
->ikm_voucher
);
2567 kmsg
->ikm_voucher
= convert_voucher_to_port(recv_voucher
);
2568 if (III_NULL
!= inherit
)
2571 } else { /* Don't want a voucher */
2573 /* got linked importance? have to drop */
2574 if (IIE_NULL
!= kmsg
->ikm_importance
) {
2575 ipc_importance_elem_t elem
;
2577 ipc_importance_lock();
2578 elem
= ipc_importance_kmsg_unlink(kmsg
);
2580 elem
->iie_kmsg_refs_dropped
++;
2582 ipc_importance_release_locked(elem
);
2583 /* importance unlocked */
2586 /* With kmsg unlinked, can safely examine message importance attribute. */
2587 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
)) {
2588 ipc_importance_task_t task_imp
= task_self
->task_imp_base
;
2589 ipc_port_t port
= kmsg
->ikm_header
->msgh_remote_port
;
2591 /* defensive deduction for release builds lacking the assert */
2593 ipc_port_impcount_delta(port
, -1, IP_NULL
);
2596 /* will user accept legacy responsibility for the importance boost */
2597 if (KERN_SUCCESS
== ipc_importance_task_externalize_legacy_assertion(task_imp
, 1, sender_pid
)) {
2600 /* The importance boost never applied to task (clear the bit) */
2601 kmsg
->ikm_header
->msgh_bits
&= ~MACH_MSGH_BITS_RAISEIMP
;
2607 #if IMPORTANCE_DEBUG
2609 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE
, (IMPORTANCE_CODE(IMP_MSG
, IMP_MSG_DELV
)) | DBG_FUNC_NONE
,
2610 sender_pid
, task_pid(task_self
),
2611 kmsg
->ikm_header
->msgh_id
, impresult
, 0);
2612 if (impresult
== 2){
2614 * This probe only covers new voucher-based path. Legacy importance
2615 * will trigger the probe in ipc_importance_task_externalize_assertion()
2616 * above and have impresult==1 here.
2618 DTRACE_BOOST5(receive_boost
, task_t
, task_self
, int, task_pid(task_self
), int, sender_pid
, int, 1, int, task_self
->task_imp_base
->iit_assertcnt
);
2620 #endif /* IMPORTANCE_DEBUG */
2624 * Routine: ipc_importance_unreceive
2626 * Undo receive of importance attributes in a message.
2632 ipc_importance_unreceive(
2634 mach_msg_option_t __unused option
)
2636 /* importance should already be in the voucher and out of the kmsg */
2637 assert(IIE_NULL
== kmsg
->ikm_importance
);
2639 /* See if there is a legacy boost to be dropped from receiver */
2640 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
)) {
2641 ipc_importance_task_t task_imp
;
2643 kmsg
->ikm_header
->msgh_bits
&= ~MACH_MSGH_BITS_RAISEIMP
;
2644 task_imp
= current_task()->task_imp_base
;
2645 if (!IP_VALID(kmsg
->ikm_voucher
) && IIT_NULL
!= task_imp
) {
2646 ipc_importance_task_drop_legacy_external_assertion(task_imp
, 1);
2649 * ipc_kmsg_copyout_dest() will consume the voucher
2650 * and any contained importance.
2656 * Routine: ipc_importance_clean
2658 * Clean up importance state in a kmsg that is being cleaned.
2659 * Unlink the importance chain if one was set up, and drop
2660 * the reference this kmsg held on the donor. Then check to
2661 * if importance was carried to the port, and remove that if
2667 ipc_importance_clean(
2672 /* Is the kmsg still linked? If so, remove that first */
2673 if (IIE_NULL
!= kmsg
->ikm_importance
) {
2674 ipc_importance_elem_t elem
;
2676 ipc_importance_lock();
2677 elem
= ipc_importance_kmsg_unlink(kmsg
);
2678 assert(IIE_NULL
!= elem
);
2679 ipc_importance_release_locked(elem
);
2680 /* importance unlocked */
2683 /* See if there is a legacy importance boost to be dropped from port */
2684 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
)) {
2685 kmsg
->ikm_header
->msgh_bits
&= ~MACH_MSGH_BITS_RAISEIMP
;
2686 port
= kmsg
->ikm_header
->msgh_remote_port
;
2687 if (IP_VALID(port
)) {
2689 /* inactive ports already had their importance boosts dropped */
2690 if (!ip_active(port
) ||
2691 ipc_port_importance_delta(port
, -1) == FALSE
) {
2699 ipc_importance_assert_clean(__assert_only ipc_kmsg_t kmsg
)
2701 assert(IIE_NULL
== kmsg
->ikm_importance
);
2702 assert(!MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg
->ikm_header
->msgh_bits
));
2706 * IPC Importance Attribute Manager definition
2709 static kern_return_t
2710 ipc_importance_release_value(
2711 ipc_voucher_attr_manager_t manager
,
2712 mach_voucher_attr_key_t key
,
2713 mach_voucher_attr_value_handle_t value
,
2714 mach_voucher_attr_value_reference_t sync
);
2716 static kern_return_t
2717 ipc_importance_get_value(
2718 ipc_voucher_attr_manager_t manager
,
2719 mach_voucher_attr_key_t key
,
2720 mach_voucher_attr_recipe_command_t command
,
2721 mach_voucher_attr_value_handle_array_t prev_values
,
2722 mach_voucher_attr_value_handle_array_size_t prev_value_count
,
2723 mach_voucher_attr_content_t content
,
2724 mach_voucher_attr_content_size_t content_size
,
2725 mach_voucher_attr_value_handle_t
*out_value
,
2726 ipc_voucher_t
*out_value_voucher
);
2728 static kern_return_t
2729 ipc_importance_extract_content(
2730 ipc_voucher_attr_manager_t manager
,
2731 mach_voucher_attr_key_t key
,
2732 mach_voucher_attr_value_handle_array_t values
,
2733 mach_voucher_attr_value_handle_array_size_t value_count
,
2734 mach_voucher_attr_recipe_command_t
*out_command
,
2735 mach_voucher_attr_content_t out_content
,
2736 mach_voucher_attr_content_size_t
*in_out_content_size
);
2738 static kern_return_t
2739 ipc_importance_command(
2740 ipc_voucher_attr_manager_t manager
,
2741 mach_voucher_attr_key_t key
,
2742 mach_voucher_attr_value_handle_array_t values
,
2743 mach_msg_type_number_t value_count
,
2744 mach_voucher_attr_command_t command
,
2745 mach_voucher_attr_content_t in_content
,
2746 mach_voucher_attr_content_size_t in_content_size
,
2747 mach_voucher_attr_content_t out_content
,
2748 mach_voucher_attr_content_size_t
*out_content_size
);
2751 ipc_importance_manager_release(
2752 ipc_voucher_attr_manager_t manager
);
2754 struct ipc_voucher_attr_manager ipc_importance_manager
= {
2755 .ivam_release_value
= ipc_importance_release_value
,
2756 .ivam_get_value
= ipc_importance_get_value
,
2757 .ivam_extract_content
= ipc_importance_extract_content
,
2758 .ivam_command
= ipc_importance_command
,
2759 .ivam_release
= ipc_importance_manager_release
,
2762 #define IMPORTANCE_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_IMPORTANCE == (key))
2763 #define IMPORTANCE_ASSERT_MANAGER(manager) assert(&ipc_importance_manager == (manager))
2766 * Routine: ipc_importance_release_value [Voucher Attribute Manager Interface]
2768 * Release what the voucher system believes is the last "made" reference
2769 * on an importance attribute value handle. The sync parameter is used to
2770 * avoid races with new made references concurrently being returned to the
2771 * voucher system in other threads.
2773 * Nothing locked on entry. May block.
2775 static kern_return_t
2776 ipc_importance_release_value(
2777 ipc_voucher_attr_manager_t __assert_only manager
,
2778 mach_voucher_attr_key_t __assert_only key
,
2779 mach_voucher_attr_value_handle_t value
,
2780 mach_voucher_attr_value_reference_t sync
)
2782 ipc_importance_elem_t elem
;
2784 IMPORTANCE_ASSERT_MANAGER(manager
);
2785 IMPORTANCE_ASSERT_KEY(key
);
2788 elem
= (ipc_importance_elem_t
)value
;
2790 ipc_importance_lock();
2792 /* Any oustanding made refs? */
2793 if (sync
!= elem
->iie_made
) {
2794 assert(sync
< elem
->iie_made
);
2795 ipc_importance_unlock();
2796 return KERN_FAILURE
;
2803 * If there are pending external boosts represented by this attribute,
2804 * drop them from the apropriate task
2806 if (IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) {
2807 ipc_importance_inherit_t inherit
= (ipc_importance_inherit_t
)elem
;
2809 assert(inherit
->iii_externcnt
>= inherit
->iii_externdrop
);
2811 if (inherit
->iii_donating
) {
2812 ipc_importance_task_t imp_task
= inherit
->iii_to_task
;
2813 uint32_t assertcnt
= III_EXTERN(inherit
);
2815 assert(ipc_importance_task_is_any_receiver_type(imp_task
));
2816 assert(imp_task
->iit_externcnt
>= inherit
->iii_externcnt
);
2817 assert(imp_task
->iit_externdrop
>= inherit
->iii_externdrop
);
2818 imp_task
->iit_externcnt
-= inherit
->iii_externcnt
;
2819 imp_task
->iit_externdrop
-= inherit
->iii_externdrop
;
2820 inherit
->iii_externcnt
= 0;
2821 inherit
->iii_externdrop
= 0;
2822 inherit
->iii_donating
= FALSE
;
2824 /* adjust the internal assertions - and propagate if needed */
2825 if (ipc_importance_task_check_transition(imp_task
, IIT_UPDATE_DROP
, assertcnt
)) {
2826 ipc_importance_task_propagate_assertion_locked(imp_task
, IIT_UPDATE_DROP
, TRUE
);
2829 inherit
->iii_externcnt
= 0;
2830 inherit
->iii_externdrop
= 0;
2834 /* drop the made reference on elem */
2835 ipc_importance_release_locked(elem
);
2836 /* returns unlocked */
2838 return KERN_SUCCESS
;
2843 * Routine: ipc_importance_get_value [Voucher Attribute Manager Interface]
2845 * Convert command and content data into a reference on a [potentially new]
2846 * attribute value. The importance attribute manager will only allow the
2847 * caller to get a value for the current task's importance, or to redeem
2848 * an importance attribute from an existing voucher.
2850 * Nothing locked on entry. May block.
2852 static kern_return_t
2853 ipc_importance_get_value(
2854 ipc_voucher_attr_manager_t __assert_only manager
,
2855 mach_voucher_attr_key_t __assert_only key
,
2856 mach_voucher_attr_recipe_command_t command
,
2857 mach_voucher_attr_value_handle_array_t prev_values
,
2858 mach_voucher_attr_value_handle_array_size_t prev_value_count
,
2859 mach_voucher_attr_content_t __unused content
,
2860 mach_voucher_attr_content_size_t content_size
,
2861 mach_voucher_attr_value_handle_t
*out_value
,
2862 ipc_voucher_t
*out_value_voucher
)
2864 ipc_importance_elem_t elem
;
2867 IMPORTANCE_ASSERT_MANAGER(manager
);
2868 IMPORTANCE_ASSERT_KEY(key
);
2870 if (0 != content_size
)
2871 return KERN_INVALID_ARGUMENT
;
2873 /* never an out voucher */
2877 case MACH_VOUCHER_ATTR_REDEEM
:
2879 /* redeem of previous values is the value */
2880 if (0 < prev_value_count
) {
2881 elem
= (ipc_importance_elem_t
)prev_values
[0];
2882 assert(IIE_NULL
!= elem
);
2884 ipc_importance_lock();
2885 assert(0 < elem
->iie_made
);
2887 ipc_importance_unlock();
2889 *out_value
= prev_values
[0];
2890 return KERN_SUCCESS
;
2893 /* redeem of default is default */
2895 *out_value_voucher
= IPC_VOUCHER_NULL
;
2896 return KERN_SUCCESS
;
2898 case MACH_VOUCHER_ATTR_IMPORTANCE_SELF
:
2899 self
= current_task();
2901 elem
= (ipc_importance_elem_t
)ipc_importance_for_task(self
, TRUE
);
2902 /* made reference added (or IIE_NULL which isn't referenced) */
2904 *out_value
= (mach_voucher_attr_value_handle_t
)elem
;
2905 *out_value_voucher
= IPC_VOUCHER_NULL
;
2906 return KERN_SUCCESS
;
2910 * every other command is unknown
2912 * Specifically, there is no mechanism provided to construct an
2913 * importance attribute for a task/process from just a pid or
2914 * task port. It has to be copied (or redeemed) from a previous
2915 * voucher that has it.
2917 return KERN_INVALID_ARGUMENT
;
2922 * Routine: ipc_importance_extract_content [Voucher Attribute Manager Interface]
2924 * Extract meaning from the attribute value present in a voucher. While
2925 * the real goal is to provide commands and data that can reproduce the
2926 * voucher's value "out of thin air", this isn't possible with importance
2927 * attribute values. Instead, return debug info to help track down dependencies.
2929 * Nothing locked on entry. May block.
2931 static kern_return_t
2932 ipc_importance_extract_content(
2933 ipc_voucher_attr_manager_t __assert_only manager
,
2934 mach_voucher_attr_key_t __assert_only key
,
2935 mach_voucher_attr_value_handle_array_t values
,
2936 mach_voucher_attr_value_handle_array_size_t value_count
,
2937 mach_voucher_attr_recipe_command_t
*out_command
,
2938 mach_voucher_attr_content_t out_content
,
2939 mach_voucher_attr_content_size_t
*in_out_content_size
)
2941 mach_voucher_attr_content_size_t size
= 0;
2942 ipc_importance_elem_t elem
;
2945 IMPORTANCE_ASSERT_MANAGER(manager
);
2946 IMPORTANCE_ASSERT_KEY(key
);
2948 /* the first non-default value provides the data */
2949 for (i
= 0; i
< value_count
; i
++) {
2950 elem
= (ipc_importance_elem_t
)values
[i
];
2951 if (IIE_NULL
== elem
)
2954 snprintf((char *)out_content
, *in_out_content_size
, "Importance for pid ");
2955 size
= (mach_voucher_attr_content_size_t
)strlen((char *)out_content
);
2958 ipc_importance_inherit_t inherit
= III_NULL
;
2959 ipc_importance_task_t task_imp
;
2963 if (IIE_TYPE_TASK
== IIE_TYPE(elem
)) {
2964 task_imp
= (ipc_importance_task_t
)elem
;
2965 task
= task_imp
->iit_task
;
2966 t_pid
= (TASK_NULL
!= task
) ?
2967 task_pid(task
) : -1;
2968 snprintf((char *)out_content
+ size
, *in_out_content_size
- size
, "%d", t_pid
);
2970 inherit
= (ipc_importance_inherit_t
)elem
;
2971 task_imp
= inherit
->iii_to_task
;
2972 task
= task_imp
->iit_task
;
2973 t_pid
= (TASK_NULL
!= task
) ?
2974 task_pid(task
) : -1;
2975 snprintf((char *)out_content
+ size
, *in_out_content_size
- size
,
2976 "%d (%d of %d boosts) %s from pid ", t_pid
,
2977 III_EXTERN(inherit
), inherit
->iii_externcnt
,
2978 (inherit
->iii_donating
) ? "donated" : "linked");
2981 size
= (mach_voucher_attr_content_size_t
)strlen((char *)out_content
);
2983 if (III_NULL
== inherit
)
2986 elem
= inherit
->iii_from_elem
;
2988 size
++; /* account for NULL */
2990 *out_command
= MACH_VOUCHER_ATTR_NOOP
; /* cannot be used to regenerate value */
2991 *in_out_content_size
= size
;
2992 return KERN_SUCCESS
;
2996 * Routine: ipc_importance_command [Voucher Attribute Manager Interface]
2998 * Run commands against the importance attribute value found in a voucher.
2999 * No such commands are currently supported.
3001 * Nothing locked on entry. May block.
3003 static kern_return_t
3004 ipc_importance_command(
3005 ipc_voucher_attr_manager_t __assert_only manager
,
3006 mach_voucher_attr_key_t __assert_only key
,
3007 mach_voucher_attr_value_handle_array_t values
,
3008 mach_msg_type_number_t value_count
,
3009 mach_voucher_attr_command_t command
,
3010 mach_voucher_attr_content_t in_content
,
3011 mach_voucher_attr_content_size_t in_content_size
,
3012 mach_voucher_attr_content_t out_content
,
3013 mach_voucher_attr_content_size_t
*out_content_size
)
3015 ipc_importance_inherit_t inherit
;
3016 ipc_importance_task_t to_task
;
3017 uint32_t refs
, *outrefsp
;
3018 mach_msg_type_number_t i
;
3021 IMPORTANCE_ASSERT_MANAGER(manager
);
3022 IMPORTANCE_ASSERT_KEY(key
);
3024 if (in_content_size
!= sizeof(refs
) ||
3025 (*out_content_size
!= 0 && *out_content_size
!= sizeof(refs
))) {
3026 return KERN_INVALID_ARGUMENT
;
3028 refs
= *(uint32_t *)(void *)in_content
;
3029 outrefsp
= (*out_content_size
!= 0) ? (uint32_t *)(void *)out_content
: NULL
;
3031 if (MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL
!= command
) {
3032 return KERN_NOT_SUPPORTED
;
3035 /* the first non-default value of the apropos type provides the data */
3037 for (i
= 0; i
< value_count
; i
++) {
3038 ipc_importance_elem_t elem
= (ipc_importance_elem_t
)values
[i
];
3040 if (IIE_NULL
!= elem
&& IIE_TYPE_INHERIT
== IIE_TYPE(elem
)) {
3041 inherit
= (ipc_importance_inherit_t
)elem
;
3045 if (III_NULL
== inherit
) {
3046 return KERN_INVALID_ARGUMENT
;
3049 ipc_importance_lock();
3052 if (NULL
!= outrefsp
) {
3053 *outrefsp
= III_EXTERN(inherit
);
3055 ipc_importance_unlock();
3056 return KERN_SUCCESS
;
3059 to_task
= inherit
->iii_to_task
;
3060 assert(ipc_importance_task_is_any_receiver_type(to_task
));
3062 /* if not donating to a denap receiver, it was called incorrectly */
3063 if (!ipc_importance_task_is_marked_denap_receiver(to_task
)) {
3064 ipc_importance_unlock();
3065 return KERN_INVALID_ARGUMENT
; /* keeps dispatch happy */
3068 /* Enough external references left to drop? */
3069 if (III_EXTERN(inherit
) < refs
) {
3070 ipc_importance_unlock();
3071 return KERN_FAILURE
;
3074 /* re-base external and internal counters at the inherit and the to-task (if apropos) */
3075 if (inherit
->iii_donating
) {
3076 assert(IIT_EXTERN(to_task
) >= III_EXTERN(inherit
));
3077 assert(to_task
->iit_externcnt
>= inherit
->iii_externcnt
);
3078 assert(to_task
->iit_externdrop
>= inherit
->iii_externdrop
);
3079 inherit
->iii_externdrop
+= refs
;
3080 to_task
->iit_externdrop
+= refs
;
3081 externcnt
= III_EXTERN(inherit
);
3082 if (0 == externcnt
) {
3083 inherit
->iii_donating
= FALSE
;
3084 to_task
->iit_externcnt
-= inherit
->iii_externcnt
;
3085 to_task
->iit_externdrop
-= inherit
->iii_externdrop
;
3088 /* Start AppNap delay hysteresis - even if not the last boost for the task. */
3089 if (ipc_importance_delayed_drop_call
!= NULL
&&
3090 ipc_importance_task_is_marked_denap_receiver(to_task
)) {
3091 ipc_importance_task_delayed_drop(to_task
);
3094 /* drop task assertions associated with the dropped boosts */
3095 if (ipc_importance_task_check_transition(to_task
, IIT_UPDATE_DROP
, refs
)) {
3096 ipc_importance_task_propagate_assertion_locked(to_task
, IIT_UPDATE_DROP
, TRUE
);
3097 /* may have dropped and retaken importance lock */
3100 /* assert(to_task->iit_assertcnt >= refs + externcnt); */
3101 /* defensive deduction in case of assertcnt underflow */
3102 if (to_task
->iit_assertcnt
> refs
+ externcnt
) {
3103 to_task
->iit_assertcnt
-= refs
;
3105 to_task
->iit_assertcnt
= externcnt
;
3109 inherit
->iii_externdrop
+= refs
;
3110 externcnt
= III_EXTERN(inherit
);
3113 /* capture result (if requested) */
3114 if (NULL
!= outrefsp
) {
3115 *outrefsp
= externcnt
;
3118 ipc_importance_unlock();
3119 return KERN_SUCCESS
;
3123 * Routine: ipc_importance_manager_release [Voucher Attribute Manager Interface]
3125 * Release the Voucher system's reference on the IPC importance attribute
3128 * As this can only occur after the manager drops the Attribute control
3129 * reference granted back at registration time, and that reference is never
3130 * dropped, this should never be called.
3133 ipc_importance_manager_release(
3134 ipc_voucher_attr_manager_t __assert_only manager
)
3136 IMPORTANCE_ASSERT_MANAGER(manager
);
3137 panic("Voucher importance manager released");
3141 * Routine: ipc_importance_init
3143 * Initialize the IPC importance manager.
3145 * Zones and Vouchers are already initialized.
3148 ipc_importance_init(void)
3150 natural_t ipc_importance_max
= (task_max
+ thread_max
) * 2;
3154 if (PE_parse_boot_argn("imp_interactive_receiver", temp_buf
, sizeof(temp_buf
))) {
3155 ipc_importance_interactive_receiver
= TRUE
;
3158 ipc_importance_task_zone
= zinit(sizeof(struct ipc_importance_task
),
3159 ipc_importance_max
* sizeof(struct ipc_importance_task
),
3160 sizeof(struct ipc_importance_task
),
3161 "ipc task importance");
3162 zone_change(ipc_importance_task_zone
, Z_NOENCRYPT
, TRUE
);
3164 ipc_importance_inherit_zone
= zinit(sizeof(struct ipc_importance_inherit
),
3165 ipc_importance_max
* sizeof(struct ipc_importance_inherit
),
3166 sizeof(struct ipc_importance_inherit
),
3167 "ipc importance inherit");
3168 zone_change(ipc_importance_inherit_zone
, Z_NOENCRYPT
, TRUE
);
3171 #if DEVELOPMENT || DEBUG
3172 queue_init(&global_iit_alloc_queue
);
3175 /* initialize global locking */
3176 ipc_importance_lock_init();
3178 kr
= ipc_register_well_known_mach_voucher_attr_manager(&ipc_importance_manager
,
3179 (mach_voucher_attr_value_handle_t
)0,
3180 MACH_VOUCHER_ATTR_KEY_IMPORTANCE
,
3181 &ipc_importance_control
);
3182 if (KERN_SUCCESS
!= kr
)
3183 printf("Voucher importance manager register returned %d", kr
);
3187 * Routine: ipc_importance_thread_call_init
3189 * Initialize the IPC importance code dependent upon
3190 * thread-call support being available.
3192 * Thread-call mechanism is already initialized.
3195 ipc_importance_thread_call_init(void)
3197 /* initialize delayed drop queue and thread-call */
3198 queue_init(&ipc_importance_delayed_drop_queue
);
3199 ipc_importance_delayed_drop_call
=
3200 thread_call_allocate(ipc_importance_task_delayed_drop_scan
, NULL
);
3201 if (NULL
== ipc_importance_delayed_drop_call
) {
3202 panic("ipc_importance_init");
3207 * Routing: task_importance_list_pids
3208 * Purpose: list pids where task in donating importance.
3209 * Conditions: To be called only from kdp stackshot code.
3210 * Will panic the system otherwise.
3213 task_importance_list_pids(task_t task
, int flags
, char *pid_list
, unsigned int max_count
)
3215 if (kdp_lck_spin_is_acquired(&ipc_importance_lock_data
) ||
3217 task
->task_imp_base
== IIT_NULL
||
3219 flags
!= TASK_IMP_LIST_DONATING_PIDS
) {
3222 unsigned int pidcount
= 0;
3224 ipc_importance_task_t task_imp
= task
->task_imp_base
;
3225 ipc_kmsg_t temp_kmsg
;
3226 ipc_importance_inherit_t temp_inherit
;
3227 ipc_importance_elem_t elem
;
3228 int target_pid
= 0, previous_pid
;
3230 queue_iterate(&task_imp
->iit_inherits
, temp_inherit
, ipc_importance_inherit_t
, iii_inheritance
) {
3231 /* check space in buffer */
3232 if (pidcount
>= max_count
)
3234 previous_pid
= target_pid
;
3237 if (temp_inherit
->iii_donating
) {
3239 #if DEVELOPMENT || DEBUG
3240 target_pid
= temp_inherit
->iii_to_task
->iit_bsd_pid
;
3242 temp_task
= temp_inherit
->iii_to_task
->iit_task
;
3243 if (temp_task
!= TASK_NULL
) {
3244 target_pid
= task_pid(temp_task
);
3249 if (target_pid
!= -1 && previous_pid
!= target_pid
) {
3250 memcpy(pid_list
, &target_pid
, sizeof(target_pid
));
3251 pid_list
+= sizeof(target_pid
);
3258 queue_iterate(&task_imp
->iit_kmsgs
, temp_kmsg
, ipc_kmsg_t
, ikm_inheritance
) {
3259 if (pidcount
>= max_count
)
3261 previous_pid
= target_pid
;
3263 elem
= temp_kmsg
->ikm_importance
;
3264 temp_task
= TASK_NULL
;
3266 if (elem
== IIE_NULL
) {
3270 if (!(temp_kmsg
->ikm_header
&& MACH_MSGH_BITS_RAISED_IMPORTANCE(temp_kmsg
->ikm_header
->msgh_bits
))) {
3274 if (IIE_TYPE_TASK
== IIE_TYPE(elem
) &&
3275 (((ipc_importance_task_t
)elem
)->iit_task
!= TASK_NULL
)) {
3276 target_pid
= task_pid(((ipc_importance_task_t
)elem
)->iit_task
);
3278 temp_inherit
= (ipc_importance_inherit_t
)elem
;
3279 #if DEVELOPMENT || DEBUG
3280 target_pid
= temp_inherit
->iii_to_task
->iit_bsd_pid
;
3282 temp_task
= temp_inherit
->iii_to_task
->iit_task
;
3283 if (temp_task
!= TASK_NULL
) {
3284 target_pid
= task_pid(temp_task
);
3289 if (target_pid
!= -1 && previous_pid
!= target_pid
) {
3290 memcpy(pid_list
, &target_pid
, sizeof(target_pid
));
3291 pid_list
+= sizeof(target_pid
);