+/*
+ * kqfile - definition of a typical kqueue opened as a file descriptor
+ * via the kqueue() system call.
+ *
+ * Adds selinfo support to the base kqueue definition, as these
+ * fds can be fed into select().
+ */
+struct kqfile {
+ struct kqueue kqf_kqueue; /* common kqueue core */
+ struct kqtailq kqf_queue; /* queue of woken up knotes */
+ struct kqtailq kqf_suppressed; /* suppression queue */
+ struct selinfo kqf_sel; /* parent select/kqueue info */
+#define kqf_wqs kqf_kqueue.kq_wqs
+#define kqf_lock kqf_kqueue.kq_lock
+#define kqf_state kqf_kqueue.kq_state
+#define kqf_level kqf_kqueue.kq_level
+#define kqf_count kqf_kqueue.kq_count
+#define kqf_p kqf_kqueue.kq_p
+};
+
+#define QOS_INDEX_KQFILE 0 /* number of qos levels in a file kq */
+
+/*
+ * WorkQ kqueues need to request threads to service the triggered
+ * knotes in the queue. These threads are brought up on a
+ * effective-requested-QoS basis. Knotes are segregated based on
+ * that value - calculated by computing max(event-QoS, kevent-QoS).
+ * Only one servicing thread is requested at a time for all the
+ * knotes at a given effective-requested-QoS.
+ */
+
+#if !defined(KQWQ_QOS_MANAGER)
+#define KQWQ_QOS_MANAGER (THREAD_QOS_LAST)
+#endif
+
+#if !defined(KQWQ_NBUCKETS)
+#define KQWQ_NBUCKETS (KQWQ_QOS_MANAGER + 1)
+#endif
+
+/*
+ * kqworkq - definition of a private kqueue used to coordinate event
+ * handling for pthread work queues.
+ *
+ * These have per-qos processing queues and state to coordinate with
+ * the pthread kext to ask for threads at corresponding pthread priority
+ * values.
+ */
+struct kqworkq {
+ struct kqueue kqwq_kqueue;
+ struct kqtailq kqwq_queue[KQWQ_NBUCKETS]; /* array of queues */
+ struct kqtailq kqwq_suppressed[KQWQ_NBUCKETS]; /* Per-QoS suppression queues */
+ workq_threadreq_s kqwq_request[KQWQ_NBUCKETS]; /* per-QoS request states */
+};
+
+#define kqwq_wqs kqwq_kqueue.kq_wqs
+#define kqwq_lock kqwq_kqueue.kq_lock
+#define kqwq_state kqwq_kqueue.kq_state
+#define kqwq_waitq_hook kqwq_kqueue.kq_waitq_hook
+#define kqwq_count kqwq_kqueue.kq_count
+#define kqwq_p kqwq_kqueue.kq_p
+
+/*
+ * WorkLoop kqueues need to request a thread to service the triggered
+ * knotes in the queue. The thread is brought up on a
+ * effective-requested-QoS basis. Knotes are segregated based on
+ * that value. Once a request is made, it cannot be undone. If
+ * events with higher QoS arrive after, they are stored in their
+ * own queues and an override applied to the original request based
+ * on the delta between the two QoS values.
+ */
+
+/*
+ * "Stay-active" knotes are held in a separate bucket that indicates
+ * special handling required. They are kept separate because the
+ * wakeups issued to them don't have context to tell us where to go
+ * to find and process them. All processing of them happens at the
+ * highest QoS. Unlike WorkQ kqueues, there is no special singular
+ * "manager thread" for a process. We simply request a servicing
+ * thread at the higest known QoS when these are woken (or override
+ * an existing request to that).
+ */
+#define KQWL_BUCKET_STAYACTIVE (THREAD_QOS_LAST)
+
+#if !defined(KQWL_NBUCKETS)
+#define KQWL_NBUCKETS (KQWL_BUCKET_STAYACTIVE + 1)
+#endif
+
+/*
+ * kqworkloop - definition of a private kqueue used to coordinate event
+ * handling for pthread workloops.
+ *
+ * Workloops vary from workqs in that only a single thread is ever
+ * requested to service a workloop at a time. But unlike workqs,
+ * workloops may be "owned" by user-space threads that are
+ * synchronously draining an event off the workloop. In those cases,
+ * any overrides have to be applied to the owner until it relinqueshes
+ * ownership.
+ *
+ * NOTE: "lane" support is TBD.
+ */
+struct kqworkloop {
+ struct kqueue kqwl_kqueue; /* queue of events */
+ struct kqtailq kqwl_queue[KQWL_NBUCKETS]; /* array of queues */
+ struct kqtailq kqwl_suppressed; /* Per-QoS suppression queues */
+ workq_threadreq_s kqwl_request; /* thread request state */
+ lck_spin_t kqwl_statelock; /* state/debounce lock */
+ thread_t kqwl_owner; /* current [sync] owner thread */
+ uint32_t kqwl_retains; /* retain references */
+#define KQWL_STAYACTIVE_FIRED_BIT (1 << 0)
+ uint8_t kqwl_wakeup_indexes; /* QoS/override levels that woke */
+ kq_index_t kqwl_stayactive_qos; /* max QoS of statyactive knotes */
+ struct turnstile *kqwl_turnstile; /* turnstile for sync IPC/waiters */
+ kqueue_id_t kqwl_dynamicid; /* dynamic identity */
+ uint64_t kqwl_params; /* additional parameters */
+ LIST_ENTRY(kqworkloop) kqwl_hashlink; /* linkage for search list */
+#if CONFIG_WORKLOOP_DEBUG
+#define KQWL_HISTORY_COUNT 32
+#define KQWL_HISTORY_WRITE_ENTRY(kqwl, ...) ({ \
+ struct kqworkloop *__kqwl = (kqwl); \
+ unsigned int __index = os_atomic_inc_orig(&__kqwl->kqwl_index, relaxed); \
+ __kqwl->kqwl_history[__index % KQWL_HISTORY_COUNT] = \
+ (struct kqwl_history)__VA_ARGS__; \
+ })
+ struct kqwl_history {
+ thread_t updater; /* Note: updates can be reordered */
+ thread_t servicer;
+ thread_t old_owner;
+ thread_t new_owner;
+
+ uint64_t kev_ident;
+ int16_t error;
+ uint16_t kev_flags;
+ uint32_t kev_fflags;
+
+ uint64_t kev_mask;
+ uint64_t kev_value;
+ uint64_t in_value;
+ } kqwl_history[KQWL_HISTORY_COUNT];
+ unsigned int kqwl_index;
+#endif // CONFIG_WORKLOOP_DEBUG
+};
+LIST_HEAD(kqwllist, kqworkloop);
+
+typedef union {
+ struct kqueue *kq;
+ struct kqworkq *kqwq;
+ struct kqfile *kqf;
+ struct kqworkloop *kqwl;
+} __attribute__((transparent_union)) kqueue_t;
+
+
+#define kqwl_wqs kqwl_kqueue.kq_wqs
+#define kqwl_lock kqwl_kqueue.kq_lock
+#define kqwl_state kqwl_kqueue.kq_state
+#define kqwl_waitq_hook kqwl_kqueue.kq_waitq_hook
+#define kqwl_count kqwl_kqueue.kq_count
+#define kqwl_p kqwl_kqueue.kq_p
+
+#define KQ_WORKLOOP_RETAINS_MAX UINT32_MAX
+
+extern void kqueue_threadreq_unbind(struct proc *p, workq_threadreq_t);
+
+// called with the kq req held
+#define KQUEUE_THREADERQ_BIND_NO_INHERITOR_UPDATE 0x1
+extern void kqueue_threadreq_bind(struct proc *p, workq_threadreq_t req,
+ thread_t thread, unsigned int flags);
+
+struct turnstile *kqueue_threadreq_get_turnstile(workq_threadreq_t kqr);
+
+// called with the wq lock held
+extern void kqueue_threadreq_bind_prepost(struct proc *p, workq_threadreq_t req,
+ struct uthread *uth);
+
+// called with no lock held
+extern void kqueue_threadreq_bind_commit(struct proc *p, thread_t thread);
+
+extern void kqueue_threadreq_cancel(struct proc *p, workq_threadreq_t req);
+
+// lock not held as kqwl_params is immutable after creation
+extern workq_threadreq_param_t kqueue_threadreq_workloop_param(workq_threadreq_t req);
+
projects / apple / xnu.git / blobdiff