+#include <sys/kdebug.h>
+#if CONFIG_DTRACE
+#include <mach/sdt.h>
+#endif
+#include <machine/machine_routines.h>
+
+static ZONE_DECLARE(thread_call_zone, "thread_call",
+ sizeof(thread_call_data_t), ZC_NOENCRYPT);
+
+static struct waitq daemon_waitq;
+
+typedef enum {
+ TCF_ABSOLUTE = 0,
+ TCF_CONTINUOUS = 1,
+ TCF_COUNT = 2,
+} thread_call_flavor_t;
+
+__options_decl(thread_call_group_flags_t, uint32_t, {
+ TCG_NONE = 0x0,
+ TCG_PARALLEL = 0x1,
+ TCG_DEALLOC_ACTIVE = 0x2,
+});
+
+static struct thread_call_group {
+ __attribute__((aligned(128))) lck_ticket_t tcg_lock;
+
+ const char * tcg_name;
+
+ queue_head_t pending_queue;
+ uint32_t pending_count;
+
+ queue_head_t delayed_queues[TCF_COUNT];
+ struct priority_queue_deadline_min delayed_pqueues[TCF_COUNT];
+ timer_call_data_t delayed_timers[TCF_COUNT];
+
+ timer_call_data_t dealloc_timer;
+
+ struct waitq idle_waitq;
+ uint64_t idle_timestamp;
+ uint32_t idle_count, active_count, blocked_count;
+
+ uint32_t tcg_thread_pri;
+ uint32_t target_thread_count;
+
+ thread_call_group_flags_t tcg_flags;
+} thread_call_groups[THREAD_CALL_INDEX_MAX] = {
+ [THREAD_CALL_INDEX_HIGH] = {
+ .tcg_name = "high",
+ .tcg_thread_pri = BASEPRI_PREEMPT_HIGH,
+ .target_thread_count = 4,
+ .tcg_flags = TCG_NONE,
+ },
+ [THREAD_CALL_INDEX_KERNEL] = {
+ .tcg_name = "kernel",
+ .tcg_thread_pri = BASEPRI_KERNEL,
+ .target_thread_count = 1,
+ .tcg_flags = TCG_PARALLEL,
+ },
+ [THREAD_CALL_INDEX_USER] = {
+ .tcg_name = "user",
+ .tcg_thread_pri = BASEPRI_DEFAULT,
+ .target_thread_count = 1,
+ .tcg_flags = TCG_PARALLEL,
+ },
+ [THREAD_CALL_INDEX_LOW] = {
+ .tcg_name = "low",
+ .tcg_thread_pri = MAXPRI_THROTTLE,
+ .target_thread_count = 1,
+ .tcg_flags = TCG_PARALLEL,
+ },
+ [THREAD_CALL_INDEX_KERNEL_HIGH] = {
+ .tcg_name = "kernel-high",
+ .tcg_thread_pri = BASEPRI_PREEMPT,
+ .target_thread_count = 2,
+ .tcg_flags = TCG_NONE,
+ },
+ [THREAD_CALL_INDEX_QOS_UI] = {
+ .tcg_name = "qos-ui",
+ .tcg_thread_pri = BASEPRI_FOREGROUND,
+ .target_thread_count = 1,
+ .tcg_flags = TCG_NONE,
+ },
+ [THREAD_CALL_INDEX_QOS_IN] = {
+ .tcg_name = "qos-in",
+ .tcg_thread_pri = BASEPRI_USER_INITIATED,
+ .target_thread_count = 1,
+ .tcg_flags = TCG_NONE,
+ },
+ [THREAD_CALL_INDEX_QOS_UT] = {
+ .tcg_name = "qos-ut",
+ .tcg_thread_pri = BASEPRI_UTILITY,
+ .target_thread_count = 1,
+ .tcg_flags = TCG_NONE,
+ },
+};
+
+typedef struct thread_call_group *thread_call_group_t;
+
+#define INTERNAL_CALL_COUNT 768
+#define THREAD_CALL_DEALLOC_INTERVAL_NS (5 * NSEC_PER_MSEC) /* 5 ms */
+#define THREAD_CALL_ADD_RATIO 4
+#define THREAD_CALL_MACH_FACTOR_CAP 3
+#define THREAD_CALL_GROUP_MAX_THREADS 500
+
+struct thread_call_thread_state {
+ struct thread_call_group * thc_group;
+ struct thread_call * thc_call; /* debug only, may be deallocated */
+ uint64_t thc_call_start;
+ uint64_t thc_call_soft_deadline;
+ uint64_t thc_call_hard_deadline;
+ uint64_t thc_call_pending_timestamp;
+ uint64_t thc_IOTES_invocation_timestamp;
+ thread_call_func_t thc_func;
+ thread_call_param_t thc_param0;
+ thread_call_param_t thc_param1;
+};
+
+static bool thread_call_daemon_awake = true;
+/*
+ * This special waitq exists because the daemon thread
+ * might need to be woken while already holding a global waitq locked.
+ */
+static struct waitq daemon_waitq;
+
+static thread_call_data_t internal_call_storage[INTERNAL_CALL_COUNT];
+static queue_head_t thread_call_internal_queue;
+int thread_call_internal_queue_count = 0;
+static uint64_t thread_call_dealloc_interval_abs;
+
+static void _internal_call_init(void);
+
+static thread_call_t _internal_call_allocate(thread_call_func_t func, thread_call_param_t param0);
+static bool _is_internal_call(thread_call_t call);
+static void _internal_call_release(thread_call_t call);
+static bool _pending_call_enqueue(thread_call_t call, thread_call_group_t group, uint64_t now);
+static bool _delayed_call_enqueue(thread_call_t call, thread_call_group_t group,
+ uint64_t deadline, thread_call_flavor_t flavor);
+static bool _call_dequeue(thread_call_t call, thread_call_group_t group);
+static void thread_call_wake(thread_call_group_t group);
+static void thread_call_daemon(void *arg);
+static void thread_call_thread(thread_call_group_t group, wait_result_t wres);
+static void thread_call_dealloc_timer(timer_call_param_t p0, timer_call_param_t p1);
+static void thread_call_group_setup(thread_call_group_t group);
+static void sched_call_thread(int type, thread_t thread);
+static void thread_call_start_deallocate_timer(thread_call_group_t group);
+static void thread_call_wait_locked(thread_call_t call, spl_t s);
+static bool thread_call_wait_once_locked(thread_call_t call, spl_t s);
+
+static boolean_t thread_call_enter_delayed_internal(thread_call_t call,
+ thread_call_func_t alt_func, thread_call_param_t alt_param0,
+ thread_call_param_t param1, uint64_t deadline,
+ uint64_t leeway, unsigned int flags);
+
+/* non-static so dtrace can find it rdar://problem/31156135&31379348 */
+extern void thread_call_delayed_timer(timer_call_param_t p0, timer_call_param_t p1);
+
+LCK_GRP_DECLARE(thread_call_lck_grp, "thread_call");
projects / apple / xnu.git / blobdiff