X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/eb6b6ca394357805f2bdba989abae309f718b4d8..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/osfmk/kern/thread_group.c diff --git a/osfmk/kern/thread_group.c b/osfmk/kern/thread_group.c index 49f212298..147925485 100644 --- a/osfmk/kern/thread_group.c +++ b/osfmk/kern/thread_group.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -42,16 +41,901 @@ #include #include +#if CONFIG_THREAD_GROUPS + +#define CACHELINE_SIZE (1 << MMU_CLINE) + +struct thread_group { + uint64_t tg_id; + char tg_name[THREAD_GROUP_MAXNAME]; + struct os_refcnt tg_refcount; + uint32_t tg_flags; + cluster_type_t tg_recommendation; + queue_chain_t tg_queue_chain; +#if CONFIG_SCHED_CLUTCH + struct sched_clutch tg_sched_clutch; +#endif /* CONFIG_SCHED_CLUTCH */ + // 16 bytes of padding here + uint8_t tg_machine_data[] __attribute__((aligned(CACHELINE_SIZE))); +} __attribute__((aligned(8))); + +static SECURITY_READ_ONLY_LATE(zone_t) tg_zone; +static uint32_t tg_count; +static queue_head_t tg_queue; +static LCK_GRP_DECLARE(tg_lck_grp, "thread_group"); +static LCK_MTX_DECLARE(tg_lock, &tg_lck_grp); +static LCK_SPIN_DECLARE(tg_flags_update_lock, &tg_lck_grp); + +static uint64_t tg_next_id = 0; +static uint32_t tg_size; +static uint32_t tg_machine_data_size; +static struct thread_group *tg_system; +static struct thread_group *tg_background; +static struct thread_group *tg_adaptive; +static struct thread_group *tg_vm; +static struct thread_group *tg_io_storage; +static struct thread_group *tg_perf_controller; +int tg_set_by_bankvoucher; + +static bool thread_group_retain_try(struct thread_group *tg); + +/* + * Initialize thread groups at boot + */ +void +thread_group_init(void) +{ + // Get thread group structure extension from EDT or boot-args (which can override EDT) + if (!PE_parse_boot_argn("kern.thread_group_extra_bytes", &tg_machine_data_size, sizeof(tg_machine_data_size))) { + if (!PE_get_default("kern.thread_group_extra_bytes", &tg_machine_data_size, sizeof(tg_machine_data_size))) { + tg_machine_data_size = 8; + } + } + + // Check if thread group can be set by voucher adoption from EDT or boot-args (which can override EDT) + if (!PE_parse_boot_argn("kern.thread_group_set_by_bankvoucher", &tg_set_by_bankvoucher, sizeof(tg_set_by_bankvoucher))) { + if (!PE_get_default("kern.thread_group_set_by_bankvoucher", &tg_set_by_bankvoucher, sizeof(tg_set_by_bankvoucher))) { + tg_set_by_bankvoucher = 1; + } + } + + tg_size = sizeof(struct thread_group) + tg_machine_data_size; + if (tg_size % CACHELINE_SIZE) { + tg_size += CACHELINE_SIZE - (tg_size % CACHELINE_SIZE); + } + tg_machine_data_size = tg_size - sizeof(struct thread_group); + // printf("tg_size=%d(%lu+%d)\n", tg_size, sizeof(struct thread_group), tg_machine_data_size); + assert(offsetof(struct thread_group, tg_machine_data) % CACHELINE_SIZE == 0); + tg_zone = zone_create("thread_groups", tg_size, ZC_NOENCRYPT | ZC_ALIGNMENT_REQUIRED); + + queue_head_init(tg_queue); + tg_system = thread_group_create_and_retain(); + thread_group_set_name(tg_system, "system"); + tg_background = thread_group_create_and_retain(); + thread_group_set_name(tg_background, "background"); + tg_adaptive = thread_group_create_and_retain(); + thread_group_set_name(tg_adaptive, "adaptive"); + tg_vm = thread_group_create_and_retain(); + thread_group_set_name(tg_vm, "VM"); + tg_io_storage = thread_group_create_and_retain(); + thread_group_set_name(tg_io_storage, "io storage"); + tg_perf_controller = thread_group_create_and_retain(); + thread_group_set_name(tg_perf_controller, "perf_controller"); + + /* + * If CLPC is disabled, it would recommend SMP for all thread groups. + * In that mode, the scheduler would like to restrict the kernel thread + * groups to the E-cluster while all other thread groups are run on the + * P-cluster. To identify the kernel thread groups, mark them with a + * special flag THREAD_GROUP_FLAGS_SMP_RESTRICT which is looked at by + * recommended_pset_type(). + */ + tg_system->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT; + tg_vm->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT; + tg_io_storage->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT; + tg_perf_controller->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT; +} + +#if CONFIG_SCHED_CLUTCH +/* + * sched_clutch_for_thread + * + * The routine provides a back linkage from the thread to the + * sched_clutch it belongs to. This relationship is based on the + * thread group membership of the thread. Since that membership is + * changed from the thread context with the thread lock held, this + * linkage should be looked at only with the thread lock held or + * when the thread cannot be running (for eg. the thread is in the + * runq and being removed as part of thread_select(). + */ +sched_clutch_t +sched_clutch_for_thread(thread_t thread) +{ + assert(thread->thread_group != NULL); + return &(thread->thread_group->tg_sched_clutch); +} + +sched_clutch_t +sched_clutch_for_thread_group(struct thread_group *thread_group) +{ + return &(thread_group->tg_sched_clutch); +} + +/* + * Translate the TG flags to a priority boost for the sched_clutch. + * This priority boost will apply to the entire clutch represented + * by the thread group. + */ +static void +sched_clutch_update_tg_flags(sched_clutch_t clutch, uint8_t flags) +{ + sched_clutch_tg_priority_t sc_tg_pri = 0; + if (flags & THREAD_GROUP_FLAGS_UI_APP) { + sc_tg_pri = SCHED_CLUTCH_TG_PRI_HIGH; + } else if (flags & THREAD_GROUP_FLAGS_EFFICIENT) { + sc_tg_pri = SCHED_CLUTCH_TG_PRI_LOW; + } else { + sc_tg_pri = SCHED_CLUTCH_TG_PRI_MED; + } + os_atomic_store(&clutch->sc_tg_priority, sc_tg_pri, relaxed); +} + +#endif /* CONFIG_SCHED_CLUTCH */ + +/* + * Use a spinlock to protect all thread group flag updates. + * The lock should not have heavy contention since these flag updates should + * be infrequent. If this lock has contention issues, it should be changed to + * a per thread-group lock. + * + * The lock protects the flags field in the thread_group structure. It is also + * held while doing callouts to CLPC to reflect these flag changes. + */ + +void +thread_group_flags_update_lock(void) +{ + lck_spin_lock_grp(&tg_flags_update_lock, &tg_lck_grp); +} + +void +thread_group_flags_update_unlock(void) +{ + lck_spin_unlock(&tg_flags_update_lock); +} + +/* + * Inform platform code about already existing thread groups + * or ask it to free state for all thread groups + */ +void +thread_group_resync(boolean_t create) +{ + struct thread_group *tg; + + lck_mtx_lock(&tg_lock); + qe_foreach_element(tg, &tg_queue, tg_queue_chain) { + if (create) { + machine_thread_group_init(tg); + } else { + machine_thread_group_deinit(tg); + } + } + lck_mtx_unlock(&tg_lock); +} + +/* + * Create new thread group and add new reference to it. + */ +struct thread_group * +thread_group_create_and_retain(void) +{ + struct thread_group *tg; + + tg = (struct thread_group *)zalloc(tg_zone); + if (tg == NULL) { + panic("thread group zone over commit"); + } + assert((uintptr_t)tg % CACHELINE_SIZE == 0); + bzero(tg, sizeof(struct thread_group)); + +#if CONFIG_SCHED_CLUTCH + /* + * The clutch scheduler maintains a bunch of runqs per thread group. For + * each thread group it maintains a sched_clutch structure. The lifetime + * of that structure is tied directly to the lifetime of the thread group. + */ + sched_clutch_init_with_thread_group(&(tg->tg_sched_clutch), tg); + + /* + * Since the thread group flags are used to determine any priority promotions + * for the threads in the thread group, initialize them to 0. + */ + sched_clutch_update_tg_flags(&(tg->tg_sched_clutch), 0); + +#endif /* CONFIG_SCHED_CLUTCH */ + + lck_mtx_lock(&tg_lock); + tg->tg_id = tg_next_id++; + tg->tg_recommendation = CLUSTER_TYPE_SMP; // no recommendation yet + os_ref_init(&tg->tg_refcount, NULL); + tg_count++; + enqueue_tail(&tg_queue, &tg->tg_queue_chain); + lck_mtx_unlock(&tg_lock); + + // call machine layer init before this thread group becomes visible + machine_thread_group_init(tg); + + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NEW), tg->tg_id); + + return tg; +} + +/* + * Point newly created thread to its home thread group + */ +void +thread_group_init_thread(thread_t t, task_t task) +{ + struct thread_group *tg = task_coalition_get_thread_group(task); + t->thread_group = tg; + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_SET), + THREAD_GROUP_INVALID, tg->tg_id, (uintptr_t)thread_tid(t)); +} + +/* + * Set thread group name + */ +void +thread_group_set_name(__unused struct thread_group *tg, __unused const char *name) +{ + if (name == NULL) { + return; + } + if (!thread_group_retain_try(tg)) { + return; + } + if (tg->tg_name[0] == '\0') { + strncpy(&tg->tg_name[0], name, THREAD_GROUP_MAXNAME); +#if defined(__LP64__) + KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME), + tg->tg_id, + *(uint64_t*)(void*)&tg->tg_name[0], + *(uint64_t*)(void*)&tg->tg_name[sizeof(uint64_t)] + ); +#else /* defined(__LP64__) */ + KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME), + tg->tg_id, + *(uint32_t*)(void*)&tg->tg_name[0], + *(uint32_t*)(void*)&tg->tg_name[sizeof(uint32_t)] + ); +#endif /* defined(__LP64__) */ + } + thread_group_release(tg); +} + +void +thread_group_set_flags(struct thread_group *tg, uint64_t flags) +{ + thread_group_flags_update_lock(); + thread_group_set_flags_locked(tg, flags); + thread_group_flags_update_unlock(); +} + +void +thread_group_clear_flags(struct thread_group *tg, uint64_t flags) +{ + thread_group_flags_update_lock(); + thread_group_clear_flags_locked(tg, flags); + thread_group_flags_update_unlock(); +} + +/* + * Set thread group flags and perform related actions. + * The tg_flags_update_lock should be held. + * Currently supported flags are: + * - THREAD_GROUP_FLAGS_EFFICIENT + * - THREAD_GROUP_FLAGS_UI_APP + */ + +void +thread_group_set_flags_locked(struct thread_group *tg, uint64_t flags) +{ + if ((flags & THREAD_GROUP_FLAGS_VALID) != flags) { + panic("thread_group_set_flags: Invalid flags %llu", flags); + } + + if ((tg->tg_flags & flags) == flags) { + return; + } + + __kdebug_only uint64_t old_flags = tg->tg_flags; + tg->tg_flags |= flags; + machine_thread_group_flags_update(tg, tg->tg_flags); +#if CONFIG_SCHED_CLUTCH + sched_clutch_update_tg_flags(&(tg->tg_sched_clutch), tg->tg_flags); +#endif /* CONFIG_SCHED_CLUTCH */ + KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_FLAGS), + tg->tg_id, tg->tg_flags, old_flags); +} + +/* + * Clear thread group flags and perform related actions + * The tg_flags_update_lock should be held. + * Currently supported flags are: + * - THREAD_GROUP_FLAGS_EFFICIENT + * - THREAD_GROUP_FLAGS_UI_APP + */ + +void +thread_group_clear_flags_locked(struct thread_group *tg, uint64_t flags) +{ + if ((flags & THREAD_GROUP_FLAGS_VALID) != flags) { + panic("thread_group_clear_flags: Invalid flags %llu", flags); + } + + if ((tg->tg_flags & flags) == 0) { + return; + } + + __kdebug_only uint64_t old_flags = tg->tg_flags; + tg->tg_flags &= ~flags; +#if CONFIG_SCHED_CLUTCH + sched_clutch_update_tg_flags(&(tg->tg_sched_clutch), tg->tg_flags); +#endif /* CONFIG_SCHED_CLUTCH */ + machine_thread_group_flags_update(tg, tg->tg_flags); + KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_FLAGS), + tg->tg_id, tg->tg_flags, old_flags); +} + + + +/* + * Find thread group with specified name and put new reference to it. + */ +struct thread_group * +thread_group_find_by_name_and_retain(char *name) +{ + struct thread_group *result = NULL; + + if (name == NULL) { + return NULL; + } + + if (strncmp("system", name, THREAD_GROUP_MAXNAME) == 0) { + return thread_group_retain(tg_system); + } else if (strncmp("background", name, THREAD_GROUP_MAXNAME) == 0) { + return thread_group_retain(tg_background); + } else if (strncmp("adaptive", name, THREAD_GROUP_MAXNAME) == 0) { + return thread_group_retain(tg_adaptive); + } else if (strncmp("perf_controller", name, THREAD_GROUP_MAXNAME) == 0) { + return thread_group_retain(tg_perf_controller); + } + + struct thread_group *tg; + lck_mtx_lock(&tg_lock); + qe_foreach_element(tg, &tg_queue, tg_queue_chain) { + if (strncmp(tg->tg_name, name, THREAD_GROUP_MAXNAME) == 0 && + thread_group_retain_try(tg)) { + result = tg; + break; + } + } + lck_mtx_unlock(&tg_lock); + return result; +} + +/* + * Find thread group with specified ID and add new reference to it. + */ +struct thread_group * +thread_group_find_by_id_and_retain(uint64_t id) +{ + struct thread_group *tg = NULL; + struct thread_group *result = NULL; + + switch (id) { + case THREAD_GROUP_SYSTEM: + result = tg_system; + thread_group_retain(tg_system); + break; + case THREAD_GROUP_BACKGROUND: + result = tg_background; + thread_group_retain(tg_background); + break; + case THREAD_GROUP_ADAPTIVE: + result = tg_adaptive; + thread_group_retain(tg_adaptive); + break; + case THREAD_GROUP_VM: + result = tg_vm; + thread_group_retain(tg_vm); + break; + case THREAD_GROUP_IO_STORAGE: + result = tg_io_storage; + thread_group_retain(tg_io_storage); + break; + case THREAD_GROUP_PERF_CONTROLLER: + result = tg_perf_controller; + thread_group_retain(tg_perf_controller); + break; + default: + lck_mtx_lock(&tg_lock); + qe_foreach_element(tg, &tg_queue, tg_queue_chain) { + if (tg->tg_id == id && thread_group_retain_try(tg)) { + result = tg; + break; + } + } + lck_mtx_unlock(&tg_lock); + } + return result; +} + +/* + * Add new reference to specified thread group + */ +struct thread_group * +thread_group_retain(struct thread_group *tg) +{ + os_ref_retain(&tg->tg_refcount); + return tg; +} + +/* + * Similar to thread_group_retain, but fails for thread groups with a + * zero reference count. Returns true if retained successfully. + */ +static bool +thread_group_retain_try(struct thread_group *tg) +{ + return os_ref_retain_try(&tg->tg_refcount); +} + +/* + * Drop a reference to specified thread group + */ +void +thread_group_release(struct thread_group *tg) +{ + if (os_ref_release(&tg->tg_refcount) == 0) { + lck_mtx_lock(&tg_lock); + tg_count--; + remqueue(&tg->tg_queue_chain); + lck_mtx_unlock(&tg_lock); + static_assert(THREAD_GROUP_MAXNAME >= (sizeof(uint64_t) * 2), "thread group name is too short"); + static_assert(__alignof(struct thread_group) >= __alignof(uint64_t), "thread group name is not 8 bytes aligned"); +#if defined(__LP64__) + KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME_FREE), + tg->tg_id, + *(uint64_t*)(void*)&tg->tg_name[0], + *(uint64_t*)(void*)&tg->tg_name[sizeof(uint64_t)] + ); +#else /* defined(__LP64__) */ + KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME_FREE), + tg->tg_id, + *(uint32_t*)(void*)&tg->tg_name[0], + *(uint32_t*)(void*)&tg->tg_name[sizeof(uint32_t)] + ); +#endif /* defined(__LP64__) */ + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_FREE), tg->tg_id); +#if CONFIG_SCHED_CLUTCH + sched_clutch_destroy(&(tg->tg_sched_clutch)); +#endif /* CONFIG_SCHED_CLUTCH */ + machine_thread_group_deinit(tg); + zfree(tg_zone, tg); + } +} + +/* + * Get thread's current thread group + */ +inline struct thread_group * +thread_group_get(thread_t t) +{ + return t->thread_group; +} + +struct thread_group * +thread_group_get_home_group(thread_t t) +{ + return task_coalition_get_thread_group(t->task); +} + +#if CONFIG_SCHED_AUTO_JOIN + +/* + * thread_set_thread_group_auto_join() + * + * Sets the thread group of a thread based on auto-join rules. + * + * Preconditions: + * - Thread must not be part of a runq (freshly made runnable threads or terminating only) + * - Thread must be locked by the caller already + */ +static void +thread_set_thread_group_auto_join(thread_t t, struct thread_group *tg, __unused struct thread_group *old_tg) +{ + assert(t->runq == PROCESSOR_NULL); + t->thread_group = tg; + + /* + * If the thread group is being changed for the current thread, callout to + * CLPC to update the thread's information at that layer. This makes sure CLPC + * has consistent state when the current thread is going off-core. + */ + if (t == current_thread()) { + uint64_t ctime = mach_approximate_time(); + uint64_t arg1, arg2; + machine_thread_going_on_core(t, thread_get_urgency(t, &arg1, &arg2), 0, 0, ctime); + machine_switch_perfcontrol_state_update(THREAD_GROUP_UPDATE, ctime, PERFCONTROL_CALLOUT_WAKE_UNSAFE, t); + } +} + +#endif /* CONFIG_SCHED_AUTO_JOIN */ + +/* + * thread_set_thread_group_explicit() + * + * Sets the thread group of a thread based on default non auto-join rules. + * + * Preconditions: + * - Thread must be the current thread + * - Caller must not have the thread locked + * - Interrupts must be disabled + */ +static void +thread_set_thread_group_explicit(thread_t t, struct thread_group *tg, __unused struct thread_group *old_tg) +{ + assert(t == current_thread()); + /* + * In the clutch scheduler world, the runq membership of the thread + * is based on its thread group membership and its scheduling bucket. + * In order to synchronize with the priority (and therefore bucket) + * getting updated concurrently, it is important to perform the + * thread group change also under the thread lock. + */ + thread_lock(t); + t->thread_group = tg; + +#if CONFIG_SCHED_CLUTCH + sched_clutch_t old_clutch = (old_tg) ? &(old_tg->tg_sched_clutch) : NULL; + sched_clutch_t new_clutch = (tg) ? &(tg->tg_sched_clutch) : NULL; + if (SCHED_CLUTCH_THREAD_ELIGIBLE(t)) { + sched_clutch_thread_clutch_update(t, old_clutch, new_clutch); + } +#endif /* CONFIG_SCHED_CLUTCH */ + + thread_unlock(t); + + uint64_t ctime = mach_approximate_time(); + uint64_t arg1, arg2; + machine_thread_going_on_core(t, thread_get_urgency(t, &arg1, &arg2), 0, 0, ctime); + machine_switch_perfcontrol_state_update(THREAD_GROUP_UPDATE, ctime, 0, t); +} + +/* + * thread_set_thread_group() + * + * Overrides the current home thread group with an override group. However, + * an adopted work interval overrides the override. Does not take a reference + * on the group, so caller must guarantee group lifetime lasts as long as the + * group is set. + * + * The thread group is set according to a hierarchy: + * + * 1) work interval specified group (explicit API) + * 2) Auto-join thread group (wakeup tracking for special work intervals) + * 3) bank voucher carried group (implicitly set) + * 4) coalition default thread group (ambient) + */ +static void +thread_set_thread_group(thread_t t, struct thread_group *tg, bool auto_join) +{ + struct thread_group *home_tg = thread_group_get_home_group(t); + struct thread_group *old_tg = NULL; + + if (tg == NULL) { + /* when removing an override, revert to home group */ + tg = home_tg; + } + + spl_t s = splsched(); + + old_tg = t->thread_group; + + if (old_tg != tg) { + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_SET), + t->thread_group ? t->thread_group->tg_id : 0, + tg->tg_id, (uintptr_t)thread_tid(t), home_tg->tg_id); + + /* + * Based on whether this is a change due to auto-join, the join does + * different things and has different expectations. + */ + if (auto_join) { +#if CONFIG_SCHED_AUTO_JOIN + /* + * set thread group with auto-join rules. This has the + * implicit assumption that the thread lock is already held. + * Also this could happen to any thread (current or thread + * being context switched). + */ + thread_set_thread_group_auto_join(t, tg, old_tg); +#else /* CONFIG_SCHED_AUTO_JOIN */ + panic("Auto-Join unsupported on this platform"); +#endif /* CONFIG_SCHED_AUTO_JOIN */ + } else { + /* + * set thread group with the explicit join rules. This has + * the implicit assumption that the thread is not locked. Also + * this would be done only to the current thread. + */ + thread_set_thread_group_explicit(t, tg, old_tg); + } + } + + splx(s); +} + +void +thread_group_set_bank(thread_t t, struct thread_group *tg) +{ + /* work interval group overrides any bank override group */ + if (t->th_work_interval) { + return; + } + + /* boot arg disables groups in bank */ + if (tg_set_by_bankvoucher == FALSE) { + return; + } + + thread_set_thread_group(t, tg, false); +} + +/* + * thread_set_work_interval_thread_group() + * + * Sets the thread's group to the work interval thread group. + * If auto_join == true, thread group is being overriden through scheduler + * auto-join policies. + * + * Preconditions for auto-join case: + * - t is not current_thread and t should be locked. + * - t should not be running on a remote core; thread context switching is a valid state for this. + */ +void +thread_set_work_interval_thread_group(thread_t t, struct thread_group *tg, bool auto_join) +{ + if (tg == NULL) { + /* + * when removing a work interval override, fall back + * to the current voucher override. + * + * In the auto_join case, the thread is already locked by the caller so + * its unsafe to get the thread group from the current voucher (since + * that might require taking task lock and ivac lock). However, the + * auto-join policy does not allow threads to switch thread groups based + * on voucher overrides. + * + * For the normal case, lookup the thread group from the currently adopted + * voucher and use that as the fallback tg. + */ + + if (auto_join == false) { + tg = thread_get_current_voucher_thread_group(t); + } + } + + thread_set_thread_group(t, tg, auto_join); +} + +inline cluster_type_t +thread_group_recommendation(struct thread_group *tg) +{ + if (tg == NULL) { + return CLUSTER_TYPE_SMP; + } else { + return tg->tg_recommendation; + } +} + +inline uint64_t +thread_group_get_id(struct thread_group *tg) +{ + return tg->tg_id; +} + +uint32_t +thread_group_count(void) +{ + return tg_count; +} + +/* + * Can only be called while tg cannot be destroyed + */ +inline const char* +thread_group_get_name(struct thread_group *tg) +{ + return tg->tg_name; +} + +inline void * +thread_group_get_machine_data(struct thread_group *tg) +{ + return &tg->tg_machine_data; +} + +inline uint32_t +thread_group_machine_data_size(void) +{ + return tg_machine_data_size; +} + +kern_return_t +thread_group_iterate_stackshot(thread_group_iterate_fn_t callout, void *arg) +{ + struct thread_group *tg; + int i = 0; + qe_foreach_element(tg, &tg_queue, tg_queue_chain) { + if (tg == NULL || !ml_validate_nofault((vm_offset_t)tg, sizeof(struct thread_group))) { + return KERN_FAILURE; + } + callout(arg, i, tg); + i++; + } + return KERN_SUCCESS; +} -#if CONFIG_EMBEDDED void thread_group_join_io_storage(void) { + struct thread_group *tg = thread_group_find_by_id_and_retain(THREAD_GROUP_IO_STORAGE); + assert(tg != NULL); + thread_set_thread_group(current_thread(), tg, false); +} + +void +thread_group_join_perf_controller(void) +{ + struct thread_group *tg = thread_group_find_by_id_and_retain(THREAD_GROUP_PERF_CONTROLLER); + assert(tg != NULL); + thread_set_thread_group(current_thread(), tg, false); +} + +void +thread_group_vm_add(void) +{ + assert(tg_vm != NULL); + thread_set_thread_group(current_thread(), thread_group_find_by_id_and_retain(THREAD_GROUP_VM), false); +} + +uint64_t +kdp_thread_group_get_flags(struct thread_group *tg) +{ + return tg->tg_flags; +} + +/* + * Returns whether the thread group is restricted to the E-cluster when CLPC is + * turned off. + */ +boolean_t +thread_group_smp_restricted(struct thread_group *tg) +{ + if (tg->tg_flags & THREAD_GROUP_FLAGS_SMP_RESTRICT) { + return true; + } else { + return false; + } +} + +void +thread_group_update_recommendation(struct thread_group *tg, cluster_type_t new_recommendation) +{ + /* + * Since the tg->tg_recommendation field is read by CPUs trying to determine + * where a thread/thread group needs to be placed, it is important to use + * atomic operations to update the recommendation. + */ + os_atomic_store(&tg->tg_recommendation, new_recommendation, relaxed); } +#if CONFIG_SCHED_EDGE + +int sched_edge_restrict_ut = 1; +int sched_edge_restrict_bg = 1; + void -sched_perfcontrol_thread_group_recommend(void *machine_data __unused, cluster_type_t new_recommendation __unused) +sched_perfcontrol_thread_group_recommend(__unused void *machine_data, __unused cluster_type_t new_recommendation) { + struct thread_group *tg = (struct thread_group *)((uintptr_t)machine_data - offsetof(struct thread_group, tg_machine_data)); + /* + * CLUSTER_TYPE_SMP was used for some debugging support when CLPC dynamic control was turned off. + * In more recent implementations, CLPC simply recommends "P-spill" when dynamic control is turned off. So it should + * never be recommending CLUSTER_TYPE_SMP for thread groups. + */ + assert(new_recommendation != CLUSTER_TYPE_SMP); + /* + * The Edge scheduler expects preferred cluster recommendations for each QoS level within a TG. Until the new CLPC + * routine is being called, fake out the call from the old CLPC interface. + */ + uint32_t tg_bucket_preferred_cluster[TH_BUCKET_SCHED_MAX] = {0}; + /* + * For all buckets higher than UT, apply the recommendation to the thread group bucket + */ + for (sched_bucket_t bucket = TH_BUCKET_FIXPRI; bucket < TH_BUCKET_SHARE_UT; bucket++) { + tg_bucket_preferred_cluster[bucket] = (new_recommendation == pset_type_for_id(0)) ? 0 : 1; + } + /* For UT & BG QoS, set the recommendation only if they havent been restricted via sysctls */ + if (!sched_edge_restrict_ut) { + tg_bucket_preferred_cluster[TH_BUCKET_SHARE_UT] = (new_recommendation == pset_type_for_id(0)) ? 0 : 1; + } + if (!sched_edge_restrict_bg) { + tg_bucket_preferred_cluster[TH_BUCKET_SHARE_BG] = (new_recommendation == pset_type_for_id(0)) ? 0 : 1; + } + sched_perfcontrol_preferred_cluster_options_t options = 0; + if (new_recommendation == CLUSTER_TYPE_P) { + options |= SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING; + } + sched_edge_tg_preferred_cluster_change(tg, tg_bucket_preferred_cluster, options); } -#endif /* CONFIG_EMBEDDED */ +void +sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order) +{ + sched_edge_matrix_get(edge_matrix, edge_request_bitmap, flags, matrix_order); +} + +void +sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order) +{ + sched_edge_matrix_set(edge_matrix, edge_changes_bitmap, flags, matrix_order); +} + +void +sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster, + uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options) +{ + struct thread_group *tg = (struct thread_group *)((uintptr_t)machine_data - offsetof(struct thread_group, tg_machine_data)); + uint32_t tg_bucket_preferred_cluster[TH_BUCKET_SCHED_MAX] = { + [TH_BUCKET_FIXPRI] = (overrides[PERFCONTROL_CLASS_ABOVEUI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_ABOVEUI] : tg_preferred_cluster, + [TH_BUCKET_SHARE_FG] = (overrides[PERFCONTROL_CLASS_UI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_UI] : tg_preferred_cluster, + [TH_BUCKET_SHARE_IN] = (overrides[PERFCONTROL_CLASS_UI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_UI] : tg_preferred_cluster, + [TH_BUCKET_SHARE_DF] = (overrides[PERFCONTROL_CLASS_NONUI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_NONUI] : tg_preferred_cluster, + [TH_BUCKET_SHARE_UT] = (overrides[PERFCONTROL_CLASS_UTILITY] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_UTILITY] : tg_preferred_cluster, + [TH_BUCKET_SHARE_BG] = (overrides[PERFCONTROL_CLASS_BACKGROUND] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_BACKGROUND] : tg_preferred_cluster, + }; + sched_edge_tg_preferred_cluster_change(tg, tg_bucket_preferred_cluster, options); +} + +#else /* CONFIG_SCHED_EDGE */ + +void +sched_perfcontrol_thread_group_recommend(__unused void *machine_data, __unused cluster_type_t new_recommendation) +{ + struct thread_group *tg = (struct thread_group *)((uintptr_t)machine_data - offsetof(struct thread_group, tg_machine_data)); + SCHED(thread_group_recommendation_change)(tg, new_recommendation); +} + +void +sched_perfcontrol_edge_matrix_get(__unused sched_clutch_edge *edge_matrix, __unused bool *edge_request_bitmap, __unused uint64_t flags, __unused uint64_t matrix_order) +{ +} + +void +sched_perfcontrol_edge_matrix_set(__unused sched_clutch_edge *edge_matrix, __unused bool *edge_changes_bitmap, __unused uint64_t flags, __unused uint64_t matrix_order) +{ +} + +void +sched_perfcontrol_thread_group_preferred_clusters_set(__unused void *machine_data, __unused uint32_t tg_preferred_cluster, + __unused uint32_t overrides[PERFCONTROL_CLASS_MAX], __unused sched_perfcontrol_preferred_cluster_options_t options) +{ +} + +#endif /* CONFIG_SCHED_EDGE */ + +#endif /* CONFIG_THREAD_GROUPS */