xnu-4903.231.4.tar.gz

[apple/xnu.git] / osfmk / kern / coalition.c

diff --git a/osfmk/kern/coalition.c b/osfmk/kern/coalition.c
index fd2afa85800b83596fbdbde5d1129377d36c1c67..26f9d33a49a89da93c939d485565ccafa8ed58ab 100644 (file)
--- a/osfmk/kern/coalition.c
+++ b/osfmk/kern/coalition.c
@@ -36,6 +36,7 @@
 #include <kern/ledger.h>
 #include <kern/mach_param.h> /* for TASK_CHUNK */
 #include <kern/task.h>
 #include <kern/ledger.h>
 #include <kern/mach_param.h> /* for TASK_CHUNK */
 #include <kern/task.h>

+#include <kern/thread_group.h>

 #include <kern/zalloc.h>
 
 #include <libkern/OSAtomic.h>
 #include <kern/zalloc.h>
 
 #include <libkern/OSAtomic.h>


@@ -68,6 +69,7 @@ extern ledger_template_t task_ledger_template;
 #define COALITION_CHUNK TASK_CHUNK
 
 int unrestrict_coalition_syscalls;
 #define COALITION_CHUNK TASK_CHUNK
 
 int unrestrict_coalition_syscalls;

+int merge_adaptive_coalitions;

 
 lck_attr_t coalitions_lck_attr;
 lck_grp_t coalitions_lck_grp;
 
 lck_attr_t coalitions_lck_attr;
 lck_grp_t coalitions_lck_grp;


@@ -80,6 +82,7 @@ static uint64_t coalition_next_id = 1;
 static queue_head_t coalitions_q;
 
 coalition_t init_coalition[COALITION_NUM_TYPES];
 static queue_head_t coalitions_q;
 
 coalition_t init_coalition[COALITION_NUM_TYPES];

+coalition_t corpse_coalition[COALITION_NUM_TYPES];

 
 zone_t coalition_zone;
 
 
 zone_t coalition_zone;
 


@@ -164,15 +167,25 @@ static int           i_coal_resource_get_taskrole(coalition_t coal, task_t task)
 static void          i_coal_resource_iterate_tasks(coalition_t coal, void *ctx,
                                                   void (*callback)(coalition_t, void *, task_t));
 
 static void          i_coal_resource_iterate_tasks(coalition_t coal, void *ctx,
                                                   void (*callback)(coalition_t, void *, task_t));
 

+/*
+ * Ensure COALITION_NUM_THREAD_QOS_TYPES defined in mach/coalition.h still
+ * matches THREAD_QOS_LAST defined in mach/thread_policy.h
+ */
+static_assert(COALITION_NUM_THREAD_QOS_TYPES == THREAD_QOS_LAST);
+

 struct i_resource_coalition {
        ledger_t ledger;
        uint64_t bytesread;
        uint64_t byteswritten;
 struct i_resource_coalition {
        ledger_t ledger;
        uint64_t bytesread;
        uint64_t byteswritten;

+       uint64_t energy;

        uint64_t gpu_time;
        uint64_t logical_immediate_writes;
        uint64_t logical_deferred_writes;
        uint64_t logical_invalidated_writes;
        uint64_t logical_metadata_writes;
        uint64_t gpu_time;
        uint64_t logical_immediate_writes;
        uint64_t logical_deferred_writes;
        uint64_t logical_invalidated_writes;
        uint64_t logical_metadata_writes;

+       uint64_t cpu_ptime;
+       uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES];      /* cpu time per effective QoS class */
+       uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES];      /* cpu time per requested QoS class */

 
        uint64_t task_count;      /* tasks that have started in this coalition */
        uint64_t dead_task_count; /* tasks that have exited in this coalition;
 
        uint64_t task_count;      /* tasks that have started in this coalition */
        uint64_t dead_task_count; /* tasks that have exited in this coalition;


@@ -207,6 +220,7 @@ struct i_jetsam_coalition {
        queue_head_t extensions;
        queue_head_t services;
        queue_head_t other;
        queue_head_t extensions;
        queue_head_t services;
        queue_head_t other;

+       struct thread_group *thread_group;

 };
 
 
 };
 
 


@@ -216,6 +230,7 @@ struct i_jetsam_coalition {
 struct coalition {
        uint64_t id;                /* monotonically increasing */
        uint32_t type;
 struct coalition {
        uint64_t id;                /* monotonically increasing */
        uint32_t type;

+       uint32_t role;              /* default task role (background, adaptive, interactive, etc) */

        uint32_t ref_count;         /* Number of references to the memory containing this struct */
        uint32_t active_count;      /* Number of members of (tasks in) the
                                       coalition, plus vouchers referring
        uint32_t ref_count;         /* Number of references to the memory containing this struct */
        uint32_t active_count;      /* Number of members of (tasks in) the
                                       coalition, plus vouchers referring


@@ -233,7 +248,8 @@ struct coalition {
        uint32_t terminated : 1;    /* coalition became empty and spawns are now forbidden */
        uint32_t reaped : 1;        /* reaped, invisible to userspace, but waiting for ref_count to go to zero */
        uint32_t notified : 1;      /* no-more-processes notification was sent via special port */
        uint32_t terminated : 1;    /* coalition became empty and spawns are now forbidden */
        uint32_t reaped : 1;        /* reaped, invisible to userspace, but waiting for ref_count to go to zero */
        uint32_t notified : 1;      /* no-more-processes notification was sent via special port */

-#if defined(DEVELOPMENT) || defined(DEBUG)
+       uint32_t efficient : 1;     /* launchd has marked the coalition as efficient */
+#if DEVELOPMENT || DEBUG

        uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */
 #endif
 
        uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */
 #endif
 


@@ -285,6 +301,20 @@ s_coalition_types[COALITION_NUM_TYPES] = {
 #define coalition_lock(c) do{ lck_mtx_lock(&c->lock); }while(0)
 #define coalition_unlock(c) do{ lck_mtx_unlock(&c->lock); }while(0)
 
 #define coalition_lock(c) do{ lck_mtx_lock(&c->lock); }while(0)
 #define coalition_unlock(c) do{ lck_mtx_unlock(&c->lock); }while(0)
 

+/*
+ * Define the coalition type to track focal tasks.
+ * On embedded, track them using jetsam coalitions since they have associated thread
+ * groups which reflect this property as a flag (and pass it down to CLPC).
+ * On non-embedded platforms, since not all coalitions have jetsam coalitions
+ * track focal counts on the resource coalition.
+ */
+#if CONFIG_EMBEDDED
+#define COALITION_FOCAL_TASKS_ACCOUNTING  COALITION_TYPE_JETSAM
+#else /* CONFIG_EMBEDDED */
+#define COALITION_FOCAL_TASKS_ACCOUNTING  COALITION_TYPE_RESOURCE
+#endif /* CONFIG_EMBEDDED */
+
+

 static void
 coalition_notify_user(uint64_t id, uint32_t flags)
 {
 static void
 coalition_notify_user(uint64_t id, uint32_t flags)
 {


@@ -297,6 +327,7 @@ coalition_notify_user(uint64_t id, uint32_t flags)
        }
 
        coalition_notification(user_port, id, flags);
        }
 
        coalition_notification(user_port, id, flags);

+       ipc_port_release_send(user_port);

 }
 
 /*
 }
 
 /*


@@ -386,14 +417,23 @@ i_coal_resource_remove_task(coalition_t coal, task_t task)
                cr->time_nonempty += last_time_nonempty;
        }
 
                cr->time_nonempty += last_time_nonempty;
        }
 

-       ledger_rollup(cr->ledger, task->ledger);
-       cr->bytesread += task->task_io_stats->disk_reads.size;
-       cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
-       cr->gpu_time += task_gpu_utilisation(task);
-       cr->logical_immediate_writes += task->task_immediate_writes;    
-       cr->logical_deferred_writes += task->task_deferred_writes;
-       cr->logical_invalidated_writes += task->task_invalidated_writes;
-       cr->logical_metadata_writes += task->task_metadata_writes;
+       /* Do not roll up for exec'd task or exec copy task */
+       if (!task_is_exec_copy(task) && !task_did_exec(task)) {
+               ledger_rollup(cr->ledger, task->ledger);
+               cr->bytesread += task->task_io_stats->disk_reads.size;
+               cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
+#if !CONFIG_EMBEDDED
+               cr->gpu_time += task_gpu_utilisation(task);
+#else
+               cr->energy += task_energy(task);
+#endif
+               cr->logical_immediate_writes += task->task_immediate_writes;
+               cr->logical_deferred_writes += task->task_deferred_writes;
+               cr->logical_invalidated_writes += task->task_invalidated_writes;
+               cr->logical_metadata_writes += task->task_metadata_writes;
+               cr->cpu_ptime += task_cpu_ptime(task);
+               task_update_cpu_time_qos_stats(task, cr->cpu_time_eqos, cr->cpu_time_rqos);
+       }

 
        /* remove the task from the coalition's list */
        remqueue(&task->task_coalition[COALITION_TYPE_RESOURCE]);
 
        /* remove the task from the coalition's list */
        remqueue(&task->task_coalition[COALITION_TYPE_RESOURCE]);


@@ -442,10 +482,18 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us
 {
        kern_return_t kr;
        ledger_amount_t credit, debit;
 {
        kern_return_t kr;
        ledger_amount_t credit, debit;

+       int i;

 
        if (coal->type != COALITION_TYPE_RESOURCE)
                return KERN_INVALID_ARGUMENT;
 
 
        if (coal->type != COALITION_TYPE_RESOURCE)
                return KERN_INVALID_ARGUMENT;
 

+       /* Return KERN_INVALID_ARGUMENT for Corpse coalition */
+       for (i = 0; i < COALITION_NUM_TYPES; i++) {
+               if (coal == corpse_coalition[i]) {
+                       return KERN_INVALID_ARGUMENT;
+               }
+       }
+

        ledger_t sum_ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
        if (sum_ledger == LEDGER_NULL)
                return KERN_RESOURCE_SHORTAGE;
        ledger_t sum_ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
        if (sum_ledger == LEDGER_NULL)
                return KERN_RESOURCE_SHORTAGE;


@@ -460,47 +508,68 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us
        uint64_t bytesread = coal->r.bytesread;
        uint64_t byteswritten = coal->r.byteswritten;
        uint64_t gpu_time = coal->r.gpu_time;
        uint64_t bytesread = coal->r.bytesread;
        uint64_t byteswritten = coal->r.byteswritten;
        uint64_t gpu_time = coal->r.gpu_time;

+       uint64_t energy = coal->r.energy;

        uint64_t logical_immediate_writes = coal->r.logical_immediate_writes;
        uint64_t logical_deferred_writes = coal->r.logical_deferred_writes;
        uint64_t logical_invalidated_writes = coal->r.logical_invalidated_writes;
        uint64_t logical_metadata_writes = coal->r.logical_metadata_writes;
        int64_t cpu_time_billed_to_me = 0;
        int64_t cpu_time_billed_to_others = 0;
        uint64_t logical_immediate_writes = coal->r.logical_immediate_writes;
        uint64_t logical_deferred_writes = coal->r.logical_deferred_writes;
        uint64_t logical_invalidated_writes = coal->r.logical_invalidated_writes;
        uint64_t logical_metadata_writes = coal->r.logical_metadata_writes;
        int64_t cpu_time_billed_to_me = 0;
        int64_t cpu_time_billed_to_others = 0;

-
-       kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me);
-       if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) {
-#if DEVELOPMENT || DEBUG
-               printf("ledger_get_balance failed or ledger negative in coalition_resource_usage_internal: %lld\n", cpu_time_billed_to_me);
-#endif /* DEVELOPMENT || DEBUG */
-               cpu_time_billed_to_me = 0;
-       }
-
-       kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others);
-       if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) {
-#if DEVELOPMENT || DEBUG
-               printf("ledger_get_balance failed or ledger negative in coalition_resource_usage_internal: %lld\n", cpu_time_billed_to_others);
-#endif /* DEVELOPMENT || DEBUG */
-               cpu_time_billed_to_others = 0;
-       }
-
+       int64_t energy_billed_to_me = 0;
+       int64_t energy_billed_to_others = 0;
+       uint64_t cpu_ptime = coal->r.cpu_ptime;
+       uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES];
+       memcpy(cpu_time_eqos, coal->r.cpu_time_eqos, sizeof(cpu_time_eqos));
+       uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES];
+       memcpy(cpu_time_rqos, coal->r.cpu_time_rqos, sizeof(cpu_time_rqos));

        /*
         * Add to that all the active tasks' ledgers. Tasks cannot deallocate
         * out from under us, since we hold the coalition lock.
        /*
         * Add to that all the active tasks' ledgers. Tasks cannot deallocate
         * out from under us, since we hold the coalition lock.

-        * Do not use the on-behalf of cpu time from ledger for live tasks, since
-        * it will not have cpu time for active linkages between tasks.

         */
        task_t task;
        qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
         */
        task_t task;
        qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {

+               /*
+                * Rolling up stats for exec copy task or exec'd task will lead to double accounting.
+                * Cannot take task lock after taking coaliton lock
+                */
+               if (task_is_exec_copy(task) || task_did_exec(task)) {
+                       continue;
+               }
+

                ledger_rollup(sum_ledger, task->ledger);
                bytesread += task->task_io_stats->disk_reads.size;
                byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
                ledger_rollup(sum_ledger, task->ledger);
                bytesread += task->task_io_stats->disk_reads.size;
                byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;

+#if !CONFIG_EMBEDDED

                gpu_time += task_gpu_utilisation(task);
                gpu_time += task_gpu_utilisation(task);

+#else
+               energy += task_energy(task);
+#endif

                logical_immediate_writes += task->task_immediate_writes;
                logical_deferred_writes += task->task_deferred_writes;
                logical_invalidated_writes += task->task_invalidated_writes;
                logical_metadata_writes += task->task_metadata_writes;
                logical_immediate_writes += task->task_immediate_writes;
                logical_deferred_writes += task->task_deferred_writes;
                logical_invalidated_writes += task->task_invalidated_writes;
                logical_metadata_writes += task->task_metadata_writes;

-               cpu_time_billed_to_me += (int64_t)bank_billed_time_safe(task);
-               cpu_time_billed_to_others += (int64_t)bank_serviced_time_safe(task);
+               cpu_ptime += task_cpu_ptime(task);
+               task_update_cpu_time_qos_stats(task, cpu_time_eqos, cpu_time_rqos);
+       }
+
+       kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me);
+       if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) {
+               cpu_time_billed_to_me = 0;
+       }
+
+       kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others);
+       if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) {
+               cpu_time_billed_to_others = 0;
+       }
+
+       kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_me, (int64_t *)&energy_billed_to_me);
+       if (kr != KERN_SUCCESS || energy_billed_to_me < 0) {
+               energy_billed_to_me = 0;
+       }
+
+       kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_others, (int64_t *)&energy_billed_to_others);
+       if (kr != KERN_SUCCESS || energy_billed_to_others < 0) {
+               energy_billed_to_others = 0;

        }
 
        /* collect information from the coalition itself */
        }
 
        /* collect information from the coalition itself */


@@ -521,6 +590,8 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us
        cru_out->cpu_time = credit;
        cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me;
        cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others;
        cru_out->cpu_time = credit;
        cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me;
        cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others;

+       cru_out->energy_billed_to_me = (uint64_t)energy_billed_to_me;
+       cru_out->energy_billed_to_others = (uint64_t)energy_billed_to_others;

 
        kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups,
                        &credit, &debit);
 
        kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups,
                        &credit, &debit);


@@ -539,11 +610,14 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us
        cru_out->bytesread = bytesread;
        cru_out->byteswritten = byteswritten;
        cru_out->gpu_time = gpu_time;
        cru_out->bytesread = bytesread;
        cru_out->byteswritten = byteswritten;
        cru_out->gpu_time = gpu_time;

+       cru_out->energy = energy;

        cru_out->logical_immediate_writes = logical_immediate_writes;
        cru_out->logical_deferred_writes = logical_deferred_writes;
        cru_out->logical_invalidated_writes = logical_invalidated_writes;
        cru_out->logical_metadata_writes = logical_metadata_writes;
        cru_out->logical_immediate_writes = logical_immediate_writes;
        cru_out->logical_deferred_writes = logical_deferred_writes;
        cru_out->logical_invalidated_writes = logical_invalidated_writes;
        cru_out->logical_metadata_writes = logical_metadata_writes;

-
+       cru_out->cpu_ptime = cpu_ptime;
+       cru_out->cpu_time_eqos_len = COALITION_NUM_THREAD_QOS_TYPES;
+       memcpy(cru_out->cpu_time_eqos, cpu_time_eqos, sizeof(cru_out->cpu_time_eqos));

        ledger_dereference(sum_ledger);
        sum_ledger = LEDGER_NULL;
 
        ledger_dereference(sum_ledger);
        sum_ledger = LEDGER_NULL;
 


@@ -584,6 +658,7 @@ i_coal_jetsam_dealloc(__unused coalition_t coal)
        assert(queue_empty(&coal->j.services));
        assert(queue_empty(&coal->j.other));
        assert(coal->j.leader == TASK_NULL);
        assert(queue_empty(&coal->j.services));
        assert(queue_empty(&coal->j.other));
        assert(coal->j.leader == TASK_NULL);

+

 }
 
 static kern_return_t
 }
 
 static kern_return_t


@@ -657,11 +732,6 @@ i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role)
                /* set the coalition leader */
                cj->leader = task;
                break;
                /* set the coalition leader */
                cj->leader = task;
                break;

-       case COALITION_TASKROLE_UNDEF:
-               coal_dbg("setting PID:%d as UNDEF in %lld",
-                        task_pid(task), coal->id);
-               q = (queue_t)&cj->other;
-               break;

        case COALITION_TASKROLE_XPC:
                coal_dbg("setting PID:%d as XPC in %lld",
                         task_pid(task), coal->id);
        case COALITION_TASKROLE_XPC:
                coal_dbg("setting PID:%d as XPC in %lld",
                         task_pid(task), coal->id);


@@ -672,6 +742,23 @@ i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role)
                         task_pid(task), coal->id);
                q = (queue_t)&cj->extensions;
                break;
                         task_pid(task), coal->id);
                q = (queue_t)&cj->extensions;
                break;

+       case COALITION_TASKROLE_NONE:
+               /*
+                * Tasks with a role of "none" should fall through to an
+                * undefined role so long as the task is currently a member
+                * of the coalition. This scenario can happen if a task is
+                * killed (usually via jetsam) during exec.
+                */
+               if (task->coalition[COALITION_TYPE_JETSAM] != coal) {
+                       panic("%s: task %p attempting to set role %d "
+                             "in coalition %p to which it does not belong!", __func__, task, role, coal);
+               }
+               /* fall through */
+       case COALITION_TASKROLE_UNDEF:
+               coal_dbg("setting PID:%d as UNDEF in %lld",
+                        task_pid(task), coal->id);
+               q = (queue_t)&cj->other;
+               break;

        default:
                panic("%s: invalid role(%d) for task", __func__, role);
                return KERN_INVALID_ARGUMENT;
        default:
                panic("%s: invalid role(%d) for task", __func__, role);
                return KERN_INVALID_ARGUMENT;


@@ -713,7 +800,7 @@ i_coal_jetsam_get_taskrole(coalition_t coal, task_t task)
        }
 
        /* task not in the coalition?! */
        }
 
        /* task not in the coalition?! */

-       return -1;
+       return COALITION_TASKROLE_NONE;

 }
 
 static void
 }
 
 static void


@@ -752,7 +839,7 @@ i_coal_jetsam_iterate_tasks(coalition_t coal, void *ctx, void (*callback)(coalit
  * Condition: coalitions_list_lock must be UNLOCKED.
  */
 kern_return_t
  * Condition: coalitions_list_lock must be UNLOCKED.
  */
 kern_return_t

-coalition_create_internal(int type, boolean_t privileged, coalition_t *out)
+coalition_create_internal(int type, int role, boolean_t privileged, coalition_t *out)

 {
        kern_return_t kr;
        struct coalition *new_coal;
 {
        kern_return_t kr;
        struct coalition *new_coal;


@@ -766,6 +853,7 @@ coalition_create_internal(int type, boolean_t privileged, coalition_t *out)
        bzero(new_coal, sizeof(*new_coal));
 
        new_coal->type = type;
        bzero(new_coal, sizeof(*new_coal));
 
        new_coal->type = type;

+       new_coal->role = role;

 
        /* initialize type-specific resources */
        kr = coal_call(new_coal, init, privileged);
 
        /* initialize type-specific resources */
        kr = coal_call(new_coal, init, privileged);


@@ -778,7 +866,7 @@ coalition_create_internal(int type, boolean_t privileged, coalition_t *out)
        new_coal->ref_count = 2;
 
        new_coal->privileged = privileged ? TRUE : FALSE;
        new_coal->ref_count = 2;
 
        new_coal->privileged = privileged ? TRUE : FALSE;

-#if defined(DEVELOPMENT) || defined(DEBUG)
+#if DEVELOPMENT || DEBUG

        new_coal->should_notify = 1;
 #endif
 
        new_coal->should_notify = 1;
 #endif
 


@@ -788,6 +876,9 @@ coalition_create_internal(int type, boolean_t privileged, coalition_t *out)
        new_coal->id = coalition_next_id++;
        coalition_count++;
        enqueue_tail(&coalitions_q, &new_coal->coalitions);
        new_coal->id = coalition_next_id++;
        coalition_count++;
        enqueue_tail(&coalitions_q, &new_coal->coalitions);

+
+       KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW),
+               new_coal->id, new_coal->type);

        lck_mtx_unlock(&coalitions_list_lock);
 
        coal_dbg("id:%llu, type:%s", new_coal->id, coal_type_str(new_coal->type));
        lck_mtx_unlock(&coalitions_list_lock);
 
        coal_dbg("id:%llu, type:%s", new_coal->id, coal_type_str(new_coal->type));


@@ -827,6 +918,8 @@ coalition_release(coalition_t coal)
        assert(coal->reaped);
        assert(coal->focal_task_count == 0);
        assert(coal->nonfocal_task_count == 0);
        assert(coal->reaped);
        assert(coal->focal_task_count == 0);
        assert(coal->nonfocal_task_count == 0);

+       KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE),
+               coal->id, coal->type);

 
        coal_call(coal, dealloc);
 
 
        coal_call(coal, dealloc);
 


@@ -992,7 +1085,7 @@ task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES])
                                             get_taskrole, task);
                        coalition_unlock(task->coalition[i]);
                } else {
                                             get_taskrole, task);
                        coalition_unlock(task->coalition[i]);
                } else {

-                       roles[i] = -1;
+                       roles[i] = COALITION_TASKROLE_NONE;

                }
        }
 }
                }
        }
 }


@@ -1004,6 +1097,24 @@ coalition_type(coalition_t coal)
        return coal->type;
 }
 
        return coal->type;
 }
 

+boolean_t
+coalition_term_requested(coalition_t coal)
+{
+       return coal->termrequested;
+}
+
+boolean_t
+coalition_is_terminated(coalition_t coal)
+{
+       return coal->terminated;
+}
+
+boolean_t
+coalition_is_reaped(coalition_t coal)
+{
+       return coal->reaped;
+}
+

 boolean_t
 coalition_is_privileged(coalition_t coal)
 {
 boolean_t
 coalition_is_privileged(coalition_t coal)
 {


@@ -1038,64 +1149,54 @@ void task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta)
        coalition_unlock(coal);
 }
 
        coalition_unlock(coal);
 }
 

-uint32_t task_coalition_adjust_focal_count(task_t task, int count)
+boolean_t task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count)

 {
 {

-       coalition_t coal;
-       uint32_t ret;
+       coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
+       if (coal == COALITION_NULL)
+           return FALSE;

 
 

-       /*
-        * For now: only use the resource coalition. Perhaps in the
-        * future we may combine all coalition types, or even make
-        * a special coalition type just for this.
-        */
-       coal = task->coalition[COALITION_TYPE_RESOURCE];
-       assert(coal != COALITION_NULL);
-
-       ret = hw_atomic_add(&coal->focal_task_count, count);
-
-       /* catch underflow */
-       assert(ret != UINT32_MAX);
-       return ret;
+       *new_count = hw_atomic_add(&coal->focal_task_count, count);
+       assert(*new_count != UINT32_MAX);
+       return TRUE;

 }
 
 uint32_t task_coalition_focal_count(task_t task)
 {
 }
 
 uint32_t task_coalition_focal_count(task_t task)
 {

-       coalition_t coal;
-       coal = task->coalition[COALITION_TYPE_RESOURCE];
-       assert(coal != COALITION_NULL);
+       coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
+       if (coal == COALITION_NULL)
+           return 0;

 
        return coal->focal_task_count;
 }
 
 
        return coal->focal_task_count;
 }
 

-uint32_t task_coalition_adjust_nonfocal_count(task_t task, int count)
+boolean_t task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count)

 {
 {

-       coalition_t coal;
-       uint32_t ret;
-
-       /*
-        * For now: only use the resource coalition. Perhaps in the
-        * future we may combine all coalition types, or even make
-        * a special coalition type just for this.
-        */
-       coal = task->coalition[COALITION_TYPE_RESOURCE];
-       assert(coal != COALITION_NULL);
-
-       ret = hw_atomic_add(&coal->nonfocal_task_count, count);
+       coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
+       if (coal == COALITION_NULL)
+           return FALSE;

 
 

-       /* catch underflow */
-       assert(ret != UINT32_MAX);
-       return ret;
+       *new_count = hw_atomic_add(&coal->nonfocal_task_count, count);
+       assert(*new_count != UINT32_MAX);
+       return TRUE;

 }
 
 uint32_t task_coalition_nonfocal_count(task_t task)
 {
 }
 
 uint32_t task_coalition_nonfocal_count(task_t task)
 {

-       coalition_t coal;
-       coal = task->coalition[COALITION_TYPE_RESOURCE];
-       assert(coal != COALITION_NULL);
+       coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
+       if (coal == COALITION_NULL)
+           return 0;

 
        return coal->nonfocal_task_count;
 }
 
 
        return coal->nonfocal_task_count;
 }
 

+void coalition_set_efficient(coalition_t coal)
+{
+    coalition_lock(coal);
+    coal->efficient = TRUE;
+    coalition_unlock(coal);
+}
+
+

 void coalition_for_each_task(coalition_t coal, void *ctx,
                             void (*callback)(coalition_t, void *, task_t))
 {
 void coalition_for_each_task(coalition_t coal, void *ctx,
                             void (*callback)(coalition_t, void *, task_t))
 {


@@ -1136,7 +1237,7 @@ coalition_remove_active(coalition_t coal)
                assert(!coal->notified);
 
                coal->notified = TRUE;
                assert(!coal->notified);
 
                coal->notified = TRUE;

-#if defined(DEVELOPMENT) || defined(DEBUG)
+#if DEVELOPMENT || DEBUG

                do_notify = coal->should_notify;
 #else
                do_notify = TRUE;
                do_notify = coal->should_notify;
 #else
                do_notify = TRUE;


@@ -1173,6 +1274,18 @@ coalitions_adopt_init_task(task_t task)
        return kr;
 }
 
        return kr;
 }
 

+/* Used for forked corpses. */
+kern_return_t
+coalitions_adopt_corpse_task(task_t task)
+{
+       kern_return_t kr;
+       kr = coalitions_adopt_task(corpse_coalition, task);
+       if (kr != KERN_SUCCESS) {
+               panic("failed to adopt task %p into corpse coalition: %d", task, kr);
+       }
+       return kr;
+}
+

 /*
  * coalition_adopt_task_internal
  * Condition: Coalition must be referenced and unlocked. Will fail if coalition
 /*
  * coalition_adopt_task_internal
  * Condition: Coalition must be referenced and unlocked. Will fail if coalition


@@ -1211,6 +1324,12 @@ out_unlock:
        uint32_t rc = coal->ref_count;
        uint32_t ct = coal->type;
 #endif
        uint32_t rc = coal->ref_count;
        uint32_t ct = coal->type;
 #endif

+       if (get_task_uniqueid(task) != UINT64_MAX) {
+               /* On 32-bit targets, uniqueid will get truncated to 32 bits */
+               KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_ADOPT),
+                       coal->id, get_task_uniqueid(task));
+       }
+

        coalition_unlock(coal);
 
        coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d",
        coalition_unlock(coal);
 
        coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d",


@@ -1240,6 +1359,8 @@ coalition_remove_task_internal(task_t task, int type)
        int      ac = coal->active_count;
        int      ct = coal->type;
 #endif
        int      ac = coal->active_count;
        int      ct = coal->type;
 #endif

+       KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_REMOVE),
+               coal->id, get_task_uniqueid(task));

        coalition_unlock(coal);
 
        coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d",
        coalition_unlock(coal);
 
        coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d",


@@ -1313,8 +1434,11 @@ task_release_coalitions(task_t task)
 {
        int i;
        for (i = 0; i < COALITION_NUM_TYPES; i++) {
 {
        int i;
        for (i = 0; i < COALITION_NUM_TYPES; i++) {

-               if (task->coalition[i])
+               if (task->coalition[i]) {

                        coalition_release(task->coalition[i]);
                        coalition_release(task->coalition[i]);

+               } else if (i == COALITION_TYPE_RESOURCE) {
+                       panic("deallocating task %p was not a member of a resource coalition", task);
+               }

        }
 }
 
        }
 }
 


@@ -1386,7 +1510,7 @@ coalition_request_terminate_internal(coalition_t coal)
                assert(!coal->notified);
 
                coal->notified = TRUE;
                assert(!coal->notified);
 
                coal->notified = TRUE;

-#if defined(DEVELOPMENT) || defined(DEBUG)
+#if DEVELOPMENT || DEBUG

                do_notify = coal->should_notify;
 #else
                do_notify = TRUE;
                do_notify = coal->should_notify;
 #else
                do_notify = TRUE;


@@ -1451,7 +1575,7 @@ coalition_reap_internal(coalition_t coal)
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

-#if defined(DEVELOPMENT) || defined(DEBUG)
+#if DEVELOPMENT || DEBUG

 int coalition_should_notify(coalition_t coal)
 {
        int should;
 int coalition_should_notify(coalition_t coal)
 {
        int should;


@@ -1494,6 +1618,11 @@ coalitions_init(void)
                unrestrict_coalition_syscalls = 0;
        }
 
                unrestrict_coalition_syscalls = 0;
        }
 

+       if (!PE_parse_boot_argn("tg_adaptive", &merge_adaptive_coalitions,
+               sizeof (merge_adaptive_coalitions))) {
+               merge_adaptive_coalitions = 0;
+       }
+

        lck_grp_attr_setdefault(&coalitions_lck_grp_attr);
        lck_grp_init(&coalitions_lck_grp, "coalition", &coalitions_lck_grp_attr);
        lck_attr_setdefault(&coalitions_lck_attr);
        lck_grp_attr_setdefault(&coalitions_lck_grp_attr);
        lck_grp_init(&coalitions_lck_grp, "coalition", &coalitions_lck_grp_attr);
        lck_attr_setdefault(&coalitions_lck_attr);


@@ -1513,10 +1642,14 @@ coalitions_init(void)
                }
                if (!ctype->has_default)
                        continue;
                }
                if (!ctype->has_default)
                        continue;

-               kr = coalition_create_internal(ctype->type, TRUE, &init_coalition[ctype->type]);
+               kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, TRUE, &init_coalition[ctype->type]);

                if (kr != KERN_SUCCESS)
                        panic("%s: could not create init %s coalition: kr:%d",
                              __func__, coal_type_str(i), kr);
                if (kr != KERN_SUCCESS)
                        panic("%s: could not create init %s coalition: kr:%d",
                              __func__, coal_type_str(i), kr);

+               kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, FALSE, &corpse_coalition[ctype->type]);
+               if (kr != KERN_SUCCESS)
+                       panic("%s: could not create corpse %s coalition: kr:%d",
+                             __func__, coal_type_str(i), kr);

        }
 
        /* "Leak" our reference to the global object */
        }
 
        /* "Leak" our reference to the global object */


@@ -1591,6 +1724,32 @@ boolean_t coalition_is_leader(task_t task, int coal_type, coalition_t *coal)
        return ret;
 }
 
        return ret;
 }
 

+kern_return_t coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type)
+{
+       coalition_t coal;
+       int i = 0;
+
+       qe_foreach_element(coal, &coalitions_q, coalitions) {
+               if (coal == NULL || !ml_validate_nofault((vm_offset_t)coal, sizeof(struct coalition)))
+                       return KERN_FAILURE;
+
+               if (coalition_type == coal->type)
+                       callout(arg, i++, coal);
+       }
+
+       return KERN_SUCCESS;
+}
+
+task_t kdp_coalition_get_leader(coalition_t coal)
+{
+       if (!coal)
+               return TASK_NULL;
+
+       if (coal->type == COALITION_TYPE_JETSAM) {
+               return coal->j.leader;
+       }
+       return TASK_NULL;
+}

 
 task_t coalition_get_leader(coalition_t coal)
 {
 
 task_t coalition_get_leader(coalition_t coal)
 {