]> git.saurik.com Git - apple/xnu.git/blame_incremental - bsd/kern/kern_event.c
xnu-6153.81.5.tar.gz
[apple/xnu.git] / bsd / kern / kern_event.c
... / ...
CommitLineData
1/*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 *
28 */
29/*-
30 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
31 * All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
52 * SUCH DAMAGE.
53 */
54/*
55 * @(#)kern_event.c 1.0 (3/31/2000)
56 */
57#include <stdint.h>
58#include <machine/atomic.h>
59
60#include <sys/param.h>
61#include <sys/systm.h>
62#include <sys/filedesc.h>
63#include <sys/kernel.h>
64#include <sys/proc_internal.h>
65#include <sys/kauth.h>
66#include <sys/malloc.h>
67#include <sys/unistd.h>
68#include <sys/file_internal.h>
69#include <sys/fcntl.h>
70#include <sys/select.h>
71#include <sys/queue.h>
72#include <sys/event.h>
73#include <sys/eventvar.h>
74#include <sys/protosw.h>
75#include <sys/socket.h>
76#include <sys/socketvar.h>
77#include <sys/stat.h>
78#include <sys/syscall.h> // SYS_* constants
79#include <sys/sysctl.h>
80#include <sys/uio.h>
81#include <sys/sysproto.h>
82#include <sys/user.h>
83#include <sys/vnode_internal.h>
84#include <string.h>
85#include <sys/proc_info.h>
86#include <sys/codesign.h>
87#include <sys/pthread_shims.h>
88#include <sys/kdebug.h>
89#include <os/base.h>
90#include <pexpert/pexpert.h>
91
92#include <kern/locks.h>
93#include <kern/clock.h>
94#include <kern/cpu_data.h>
95#include <kern/policy_internal.h>
96#include <kern/thread_call.h>
97#include <kern/sched_prim.h>
98#include <kern/waitq.h>
99#include <kern/zalloc.h>
100#include <kern/kalloc.h>
101#include <kern/assert.h>
102#include <kern/ast.h>
103#include <kern/thread.h>
104#include <kern/kcdata.h>
105
106#include <pthread/priority_private.h>
107#include <pthread/workqueue_syscalls.h>
108#include <pthread/workqueue_internal.h>
109#include <libkern/libkern.h>
110
111#include "net/net_str_id.h"
112
113#include <mach/task.h>
114#include <libkern/section_keywords.h>
115
116#if CONFIG_MEMORYSTATUS
117#include <sys/kern_memorystatus.h>
118#endif
119
120extern mach_port_name_t ipc_entry_name_mask(mach_port_name_t name); /* osfmk/ipc/ipc_entry.h */
121
122#define KEV_EVTID(code) BSDDBG_CODE(DBG_BSD_KEVENT, (code))
123
124MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
125
126#define KQ_EVENT NO_EVENT64
127
128static int kqueue_select(struct fileproc *fp, int which, void *wq_link_id,
129 vfs_context_t ctx);
130static int kqueue_close(struct fileglob *fg, vfs_context_t ctx);
131static int kqueue_kqfilter(struct fileproc *fp, struct knote *kn,
132 struct kevent_qos_s *kev);
133static int kqueue_drain(struct fileproc *fp, vfs_context_t ctx);
134
135static const struct fileops kqueueops = {
136 .fo_type = DTYPE_KQUEUE,
137 .fo_read = fo_no_read,
138 .fo_write = fo_no_write,
139 .fo_ioctl = fo_no_ioctl,
140 .fo_select = kqueue_select,
141 .fo_close = kqueue_close,
142 .fo_drain = kqueue_drain,
143 .fo_kqfilter = kqueue_kqfilter,
144};
145
146static inline int kevent_modern_copyout(struct kevent_qos_s *, user_addr_t *);
147static int kevent_register_wait_prepare(struct knote *kn, struct kevent_qos_s *kev, int result);
148static void kevent_register_wait_block(struct turnstile *ts, thread_t handoff_thread,
149 thread_continue_t cont, struct _kevent_register *cont_args) __dead2;
150static void kevent_register_wait_return(struct _kevent_register *cont_args) __dead2;
151static void kevent_register_wait_cleanup(struct knote *kn);
152
153static struct kqtailq *kqueue_get_suppressed_queue(kqueue_t kq, struct knote *kn);
154static void kqueue_threadreq_initiate(struct kqueue *kq, workq_threadreq_t, kq_index_t qos, int flags);
155
156static void kqworkq_unbind(proc_t p, workq_threadreq_t);
157static thread_qos_t kqworkq_unbind_locked(struct kqworkq *kqwq, workq_threadreq_t, thread_t thread);
158static workq_threadreq_t kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index);
159
160static void kqworkloop_unbind(struct kqworkloop *kwql);
161
162enum kqwl_unbind_locked_mode {
163 KQWL_OVERRIDE_DROP_IMMEDIATELY,
164 KQWL_OVERRIDE_DROP_DELAYED,
165};
166static void kqworkloop_unbind_locked(struct kqworkloop *kwql, thread_t thread,
167 enum kqwl_unbind_locked_mode how);
168static void kqworkloop_unbind_delayed_override_drop(thread_t thread);
169static kq_index_t kqworkloop_override(struct kqworkloop *kqwl);
170static void kqworkloop_set_overcommit(struct kqworkloop *kqwl);
171enum {
172 KQWL_UTQ_NONE,
173 /*
174 * The wakeup qos is the qos of QUEUED knotes.
175 *
176 * This QoS is accounted for with the events override in the
177 * kqr_override_index field. It is raised each time a new knote is queued at
178 * a given QoS. The kqwl_wakeup_indexes field is a superset of the non empty
179 * knote buckets and is recomputed after each event delivery.
180 */
181 KQWL_UTQ_UPDATE_WAKEUP_QOS,
182 KQWL_UTQ_UPDATE_STAYACTIVE_QOS,
183 KQWL_UTQ_RECOMPUTE_WAKEUP_QOS,
184 KQWL_UTQ_UNBINDING, /* attempt to rebind */
185 KQWL_UTQ_PARKING,
186 /*
187 * The wakeup override is for suppressed knotes that have fired again at
188 * a higher QoS than the one for which they are suppressed already.
189 * This override is cleared when the knote suppressed list becomes empty.
190 */
191 KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE,
192 KQWL_UTQ_RESET_WAKEUP_OVERRIDE,
193 /*
194 * The QoS is the maximum QoS of an event enqueued on this workloop in
195 * userland. It is copied from the only EVFILT_WORKLOOP knote with
196 * a NOTE_WL_THREAD_REQUEST bit set allowed on this workloop. If there is no
197 * such knote, this QoS is 0.
198 */
199 KQWL_UTQ_SET_QOS_INDEX,
200 KQWL_UTQ_REDRIVE_EVENTS,
201};
202static void kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos);
203static int kqworkloop_end_processing(struct kqworkloop *kqwl, int flags, int kevent_flags);
204
205static struct knote *knote_alloc(void);
206static void knote_free(struct knote *kn);
207static int kq_add_knote(struct kqueue *kq, struct knote *kn,
208 struct knote_lock_ctx *knlc, struct proc *p);
209static struct knote *kq_find_knote_and_kq_lock(struct kqueue *kq,
210 struct kevent_qos_s *kev, bool is_fd, struct proc *p);
211
212static void knote_activate(kqueue_t kqu, struct knote *kn, int result);
213static void knote_dequeue(kqueue_t kqu, struct knote *kn);
214
215static void knote_apply_touch(kqueue_t kqu, struct knote *kn,
216 struct kevent_qos_s *kev, int result);
217static void knote_suppress(kqueue_t kqu, struct knote *kn);
218static void knote_unsuppress(kqueue_t kqu, struct knote *kn);
219static void knote_drop(kqueue_t kqu, struct knote *kn, struct knote_lock_ctx *knlc);
220
221// both these functions may dequeue the knote and it is up to the caller
222// to enqueue the knote back
223static void knote_adjust_qos(struct kqueue *kq, struct knote *kn, int result);
224static void knote_reset_priority(kqueue_t kqu, struct knote *kn, pthread_priority_t pp);
225
226static zone_t knote_zone;
227static zone_t kqfile_zone;
228static zone_t kqworkq_zone;
229static zone_t kqworkloop_zone;
230#if DEVELOPMENT || DEBUG
231#define KEVENT_PANIC_ON_WORKLOOP_OWNERSHIP_LEAK (1U << 0)
232#define KEVENT_PANIC_ON_NON_ENQUEUED_PROCESS (1U << 1)
233#define KEVENT_PANIC_BOOT_ARG_INITIALIZED (1U << 31)
234
235#define KEVENT_PANIC_DEFAULT_VALUE (0)
236static uint32_t
237kevent_debug_flags(void)
238{
239 static uint32_t flags = KEVENT_PANIC_DEFAULT_VALUE;
240
241 if ((flags & KEVENT_PANIC_BOOT_ARG_INITIALIZED) == 0) {
242 uint32_t value = 0;
243 if (!PE_parse_boot_argn("kevent_debug", &value, sizeof(value))) {
244 value = KEVENT_PANIC_DEFAULT_VALUE;
245 }
246 value |= KEVENT_PANIC_BOOT_ARG_INITIALIZED;
247 os_atomic_store(&flags, value, relaxed);
248 }
249 return flags;
250}
251#endif
252
253#define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
254
255static int filt_no_attach(struct knote *kn, struct kevent_qos_s *kev);
256static void filt_no_detach(struct knote *kn);
257static int filt_bad_event(struct knote *kn, long hint);
258static int filt_bad_touch(struct knote *kn, struct kevent_qos_s *kev);
259static int filt_bad_process(struct knote *kn, struct kevent_qos_s *kev);
260
261SECURITY_READ_ONLY_EARLY(static struct filterops) bad_filtops = {
262 .f_attach = filt_no_attach,
263 .f_detach = filt_no_detach,
264 .f_event = filt_bad_event,
265 .f_touch = filt_bad_touch,
266 .f_process = filt_bad_process,
267};
268
269#if CONFIG_MEMORYSTATUS
270extern const struct filterops memorystatus_filtops;
271#endif /* CONFIG_MEMORYSTATUS */
272extern const struct filterops fs_filtops;
273extern const struct filterops sig_filtops;
274extern const struct filterops machport_filtops;
275extern const struct filterops pipe_nfiltops;
276extern const struct filterops pipe_rfiltops;
277extern const struct filterops pipe_wfiltops;
278extern const struct filterops ptsd_kqops;
279extern const struct filterops ptmx_kqops;
280extern const struct filterops soread_filtops;
281extern const struct filterops sowrite_filtops;
282extern const struct filterops sock_filtops;
283extern const struct filterops soexcept_filtops;
284extern const struct filterops spec_filtops;
285extern const struct filterops bpfread_filtops;
286extern const struct filterops necp_fd_rfiltops;
287extern const struct filterops fsevent_filtops;
288extern const struct filterops vnode_filtops;
289extern const struct filterops tty_filtops;
290
291const static struct filterops file_filtops;
292const static struct filterops kqread_filtops;
293const static struct filterops proc_filtops;
294const static struct filterops timer_filtops;
295const static struct filterops user_filtops;
296const static struct filterops workloop_filtops;
297
298/*
299 *
300 * Rules for adding new filters to the system:
301 * Public filters:
302 * - Add a new "EVFILT_" option value to bsd/sys/event.h (typically a negative value)
303 * in the exported section of the header
304 * - Update the EVFILT_SYSCOUNT value to reflect the new addition
305 * - Add a filterops to the sysfilt_ops array. Public filters should be added at the end
306 * of the Public Filters section in the array.
307 * Private filters:
308 * - Add a new "EVFILT_" value to bsd/sys/event.h (typically a positive value)
309 * in the XNU_KERNEL_PRIVATE section of the header
310 * - Update the EVFILTID_MAX value to reflect the new addition
311 * - Add a filterops to the sysfilt_ops. Private filters should be added at the end of
312 * the Private filters section of the array.
313 */
314static_assert(EVFILTID_MAX < UINT8_MAX, "kn_filtid expects this to be true");
315static const struct filterops * const sysfilt_ops[EVFILTID_MAX] = {
316 /* Public Filters */
317 [~EVFILT_READ] = &file_filtops,
318 [~EVFILT_WRITE] = &file_filtops,
319 [~EVFILT_AIO] = &bad_filtops,
320 [~EVFILT_VNODE] = &file_filtops,
321 [~EVFILT_PROC] = &proc_filtops,
322 [~EVFILT_SIGNAL] = &sig_filtops,
323 [~EVFILT_TIMER] = &timer_filtops,
324 [~EVFILT_MACHPORT] = &machport_filtops,
325 [~EVFILT_FS] = &fs_filtops,
326 [~EVFILT_USER] = &user_filtops,
327 [~EVFILT_UNUSED_11] = &bad_filtops,
328 [~EVFILT_VM] = &bad_filtops,
329 [~EVFILT_SOCK] = &file_filtops,
330#if CONFIG_MEMORYSTATUS
331 [~EVFILT_MEMORYSTATUS] = &memorystatus_filtops,
332#else
333 [~EVFILT_MEMORYSTATUS] = &bad_filtops,
334#endif
335 [~EVFILT_EXCEPT] = &file_filtops,
336 [~EVFILT_WORKLOOP] = &workloop_filtops,
337
338 /* Private filters */
339 [EVFILTID_KQREAD] = &kqread_filtops,
340 [EVFILTID_PIPE_N] = &pipe_nfiltops,
341 [EVFILTID_PIPE_R] = &pipe_rfiltops,
342 [EVFILTID_PIPE_W] = &pipe_wfiltops,
343 [EVFILTID_PTSD] = &ptsd_kqops,
344 [EVFILTID_SOREAD] = &soread_filtops,
345 [EVFILTID_SOWRITE] = &sowrite_filtops,
346 [EVFILTID_SCK] = &sock_filtops,
347 [EVFILTID_SOEXCEPT] = &soexcept_filtops,
348 [EVFILTID_SPEC] = &spec_filtops,
349 [EVFILTID_BPFREAD] = &bpfread_filtops,
350 [EVFILTID_NECP_FD] = &necp_fd_rfiltops,
351 [EVFILTID_FSEVENT] = &fsevent_filtops,
352 [EVFILTID_VN] = &vnode_filtops,
353 [EVFILTID_TTY] = &tty_filtops,
354 [EVFILTID_PTMX] = &ptmx_kqops,
355
356 /* fake filter for detached knotes, keep last */
357 [EVFILTID_DETACHED] = &bad_filtops,
358};
359
360/* waitq prepost callback */
361void waitq_set__CALLING_PREPOST_HOOK__(waitq_set_prepost_hook_t *kq_hook);
362
363static inline bool
364kqr_thread_bound(workq_threadreq_t kqr)
365{
366 return kqr->tr_state == WORKQ_TR_STATE_BOUND;
367}
368
369static inline bool
370kqr_thread_requested_pending(workq_threadreq_t kqr)
371{
372 workq_tr_state_t tr_state = kqr->tr_state;
373 return tr_state > WORKQ_TR_STATE_IDLE && tr_state < WORKQ_TR_STATE_BOUND;
374}
375
376static inline bool
377kqr_thread_requested(workq_threadreq_t kqr)
378{
379 return kqr->tr_state != WORKQ_TR_STATE_IDLE;
380}
381
382static inline thread_t
383kqr_thread_fast(workq_threadreq_t kqr)
384{
385 assert(kqr_thread_bound(kqr));
386 return kqr->tr_thread;
387}
388
389static inline thread_t
390kqr_thread(workq_threadreq_t kqr)
391{
392 return kqr_thread_bound(kqr) ? kqr->tr_thread : THREAD_NULL;
393}
394
395static inline struct kqworkloop *
396kqr_kqworkloop(workq_threadreq_t kqr)
397{
398 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
399 return __container_of(kqr, struct kqworkloop, kqwl_request);
400 }
401 return NULL;
402}
403
404static inline kqueue_t
405kqr_kqueue(proc_t p, workq_threadreq_t kqr)
406{
407 kqueue_t kqu;
408 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
409 kqu.kqwl = kqr_kqworkloop(kqr);
410 } else {
411 kqu.kqwq = p->p_fd->fd_wqkqueue;
412 assert(kqr >= kqu.kqwq->kqwq_request &&
413 kqr < kqu.kqwq->kqwq_request + KQWQ_NBUCKETS);
414 }
415 return kqu;
416}
417
418/*
419 * kqueue/note lock implementations
420 *
421 * The kqueue lock guards the kq state, the state of its queues,
422 * and the kqueue-aware status and locks of individual knotes.
423 *
424 * The kqueue workq lock is used to protect state guarding the
425 * interaction of the kqueue with the workq. This state cannot
426 * be guarded by the kq lock - as it needs to be taken when we
427 * already have the waitq set lock held (during the waitq hook
428 * callback). It might be better to use the waitq lock itself
429 * for this, but the IRQ requirements make that difficult).
430 *
431 * Knote flags, filter flags, and associated data are protected
432 * by the underlying object lock - and are only ever looked at
433 * by calling the filter to get a [consistent] snapshot of that
434 * data.
435 */
436static lck_grp_attr_t *kq_lck_grp_attr;
437static lck_grp_t *kq_lck_grp;
438static lck_attr_t *kq_lck_attr;
439
440static inline void
441kqlock(kqueue_t kqu)
442{
443 lck_spin_lock(&kqu.kq->kq_lock);
444}
445
446static inline void
447kqlock_held(__assert_only kqueue_t kqu)
448{
449 LCK_SPIN_ASSERT(&kqu.kq->kq_lock, LCK_ASSERT_OWNED);
450}
451
452static inline void
453kqunlock(kqueue_t kqu)
454{
455 lck_spin_unlock(&kqu.kq->kq_lock);
456}
457
458static inline void
459knhash_lock(struct filedesc *fdp)
460{
461 lck_mtx_lock(&fdp->fd_knhashlock);
462}
463
464static inline void
465knhash_unlock(struct filedesc *fdp)
466{
467 lck_mtx_unlock(&fdp->fd_knhashlock);
468}
469
470/* wait event for knote locks */
471static inline event_t
472knote_lock_wev(struct knote *kn)
473{
474 return (event_t)(&kn->kn_hook);
475}
476
477/* wait event for kevent_register_wait_* */
478static inline event64_t
479knote_filt_wev64(struct knote *kn)
480{
481 /* kdp_workloop_sync_wait_find_owner knows about this */
482 return CAST_EVENT64_T(kn);
483}
484
485/* wait event for knote_post/knote_drop */
486static inline event64_t
487knote_post_wev64(struct knote *kn)
488{
489 return CAST_EVENT64_T(&kn->kn_kevent);
490}
491
492/*!
493 * @function knote_has_qos
494 *
495 * @brief
496 * Whether the knote has a regular QoS.
497 *
498 * @discussion
499 * kn_qos_override is:
500 * - 0 on kqfiles
501 * - THREAD_QOS_LAST for special buckets (stayactive, manager)
502 *
503 * Other values mean the knote participates to QoS propagation.
504 */
505static inline bool
506knote_has_qos(struct knote *kn)
507{
508 return kn->kn_qos_override > 0 && kn->kn_qos_override < THREAD_QOS_LAST;
509}
510
511#pragma mark knote locks
512
513/*
514 * Enum used by the knote_lock_* functions.
515 *
516 * KNOTE_KQ_LOCK_ALWAYS
517 * The function will always return with the kq lock held.
518 *
519 * KNOTE_KQ_LOCK_ON_SUCCESS
520 * The function will return with the kq lock held if it was successful
521 * (knote_lock() is the only function that can fail).
522 *
523 * KNOTE_KQ_LOCK_ON_FAILURE
524 * The function will return with the kq lock held if it was unsuccessful
525 * (knote_lock() is the only function that can fail).
526 *
527 * KNOTE_KQ_UNLOCK:
528 * The function returns with the kq unlocked.
529 */
530enum kqlocking {
531 KNOTE_KQ_LOCK_ALWAYS,
532 KNOTE_KQ_LOCK_ON_SUCCESS,
533 KNOTE_KQ_LOCK_ON_FAILURE,
534 KNOTE_KQ_UNLOCK,
535};
536
537static struct knote_lock_ctx *
538knote_lock_ctx_find(kqueue_t kqu, struct knote *kn)
539{
540 struct knote_lock_ctx *ctx;
541 LIST_FOREACH(ctx, &kqu.kq->kq_knlocks, knlc_link) {
542 if (ctx->knlc_knote == kn) {
543 return ctx;
544 }
545 }
546 panic("knote lock context not found: %p", kn);
547 __builtin_trap();
548}
549
550/* slowpath of knote_lock() */
551__attribute__((noinline))
552static bool __result_use_check
553knote_lock_slow(kqueue_t kqu, struct knote *kn,
554 struct knote_lock_ctx *knlc, int kqlocking)
555{
556 struct knote_lock_ctx *owner_lc;
557 struct uthread *uth = current_uthread();
558 wait_result_t wr;
559
560 kqlock_held(kqu);
561
562 owner_lc = knote_lock_ctx_find(kqu, kn);
563#if DEBUG || DEVELOPMENT
564 knlc->knlc_state = KNOTE_LOCK_CTX_WAITING;
565#endif
566 owner_lc->knlc_waiters++;
567
568 /*
569 * Make our lock context visible to knote_unlock()
570 */
571 uth->uu_knlock = knlc;
572
573 wr = lck_spin_sleep_with_inheritor(&kqu.kq->kq_lock, LCK_SLEEP_UNLOCK,
574 knote_lock_wev(kn), owner_lc->knlc_thread,
575 THREAD_UNINT | THREAD_WAIT_NOREPORT, TIMEOUT_WAIT_FOREVER);
576
577 if (wr == THREAD_RESTART) {
578 /*
579 * We haven't been woken up by knote_unlock() but knote_unlock_cancel.
580 * We need to cleanup the state since no one did.
581 */
582 uth->uu_knlock = NULL;
583#if DEBUG || DEVELOPMENT
584 assert(knlc->knlc_state == KNOTE_LOCK_CTX_WAITING);
585 knlc->knlc_state = KNOTE_LOCK_CTX_UNLOCKED;
586#endif
587
588 if (kqlocking == KNOTE_KQ_LOCK_ALWAYS ||
589 kqlocking == KNOTE_KQ_LOCK_ON_FAILURE) {
590 kqlock(kqu);
591 }
592 return false;
593 } else {
594 if (kqlocking == KNOTE_KQ_LOCK_ALWAYS ||
595 kqlocking == KNOTE_KQ_LOCK_ON_SUCCESS) {
596 kqlock(kqu);
597#if DEBUG || DEVELOPMENT
598 /*
599 * This state is set under the lock so we can't
600 * really assert this unless we hold the lock.
601 */
602 assert(knlc->knlc_state == KNOTE_LOCK_CTX_LOCKED);
603#endif
604 }
605 return true;
606 }
607}
608
609/*
610 * Attempts to take the "knote" lock.
611 *
612 * Called with the kqueue lock held.
613 *
614 * Returns true if the knote lock is acquired, false if it has been dropped
615 */
616static bool __result_use_check
617knote_lock(kqueue_t kqu, struct knote *kn, struct knote_lock_ctx *knlc,
618 enum kqlocking kqlocking)
619{
620 kqlock_held(kqu);
621
622#if DEBUG || DEVELOPMENT
623 assert(knlc->knlc_state == KNOTE_LOCK_CTX_UNLOCKED);
624#endif
625 knlc->knlc_knote = kn;
626 knlc->knlc_thread = current_thread();
627 knlc->knlc_waiters = 0;
628
629 if (__improbable(kn->kn_status & KN_LOCKED)) {
630 return knote_lock_slow(kqu, kn, knlc, kqlocking);
631 }
632
633 /*
634 * When the knote will be dropped, the knote lock is taken before
635 * KN_DROPPING is set, and then the knote will be removed from any
636 * hash table that references it before the lock is canceled.
637 */
638 assert((kn->kn_status & KN_DROPPING) == 0);
639 LIST_INSERT_HEAD(&kqu.kq->kq_knlocks, knlc, knlc_link);
640 kn->kn_status |= KN_LOCKED;
641#if DEBUG || DEVELOPMENT
642 knlc->knlc_state = KNOTE_LOCK_CTX_LOCKED;
643#endif
644
645 if (kqlocking == KNOTE_KQ_UNLOCK ||
646 kqlocking == KNOTE_KQ_LOCK_ON_FAILURE) {
647 kqunlock(kqu);
648 }
649 return true;
650}
651
652/*
653 * Unlocks a knote successfully locked with knote_lock().
654 *
655 * Called with the kqueue lock held.
656 *
657 * Returns with the kqueue lock held according to KNOTE_KQ_* mode.
658 */
659static void
660knote_unlock(kqueue_t kqu, struct knote *kn,
661 struct knote_lock_ctx *knlc, enum kqlocking kqlocking)
662{
663 kqlock_held(kqu);
664
665 assert(knlc->knlc_knote == kn);
666 assert(kn->kn_status & KN_LOCKED);
667#if DEBUG || DEVELOPMENT
668 assert(knlc->knlc_state == KNOTE_LOCK_CTX_LOCKED);
669#endif
670
671 LIST_REMOVE(knlc, knlc_link);
672
673 if (knlc->knlc_waiters) {
674 thread_t thread = THREAD_NULL;
675
676 wakeup_one_with_inheritor(knote_lock_wev(kn), THREAD_AWAKENED,
677 LCK_WAKE_DEFAULT, &thread);
678
679 /*
680 * knote_lock_slow() publishes the lock context of waiters
681 * in uthread::uu_knlock.
682 *
683 * Reach out and make this context the new owner.
684 */
685 struct uthread *ut = get_bsdthread_info(thread);
686 struct knote_lock_ctx *next_owner_lc = ut->uu_knlock;
687
688 assert(next_owner_lc->knlc_knote == kn);
689 next_owner_lc->knlc_waiters = knlc->knlc_waiters - 1;
690 LIST_INSERT_HEAD(&kqu.kq->kq_knlocks, next_owner_lc, knlc_link);
691#if DEBUG || DEVELOPMENT
692 next_owner_lc->knlc_state = KNOTE_LOCK_CTX_LOCKED;
693#endif
694 ut->uu_knlock = NULL;
695 thread_deallocate_safe(thread);
696 } else {
697 kn->kn_status &= ~KN_LOCKED;
698 }
699
700 if ((kn->kn_status & KN_MERGE_QOS) && !(kn->kn_status & KN_POSTING)) {
701 /*
702 * No f_event() in flight anymore, we can leave QoS "Merge" mode
703 *
704 * See knote_adjust_qos()
705 */
706 kn->kn_status &= ~KN_MERGE_QOS;
707 }
708 if (kqlocking == KNOTE_KQ_UNLOCK) {
709 kqunlock(kqu);
710 }
711#if DEBUG || DEVELOPMENT
712 knlc->knlc_state = KNOTE_LOCK_CTX_UNLOCKED;
713#endif
714}
715
716/*
717 * Aborts all waiters for a knote lock, and unlock the knote.
718 *
719 * Called with the kqueue lock held.
720 *
721 * Returns with the kqueue unlocked.
722 */
723static void
724knote_unlock_cancel(struct kqueue *kq, struct knote *kn,
725 struct knote_lock_ctx *knlc)
726{
727 kqlock_held(kq);
728
729 assert(knlc->knlc_knote == kn);
730 assert(kn->kn_status & KN_LOCKED);
731 assert(kn->kn_status & KN_DROPPING);
732
733 LIST_REMOVE(knlc, knlc_link);
734 kn->kn_status &= ~KN_LOCKED;
735 kqunlock(kq);
736
737 if (knlc->knlc_waiters) {
738 wakeup_all_with_inheritor(knote_lock_wev(kn), THREAD_RESTART);
739 }
740#if DEBUG || DEVELOPMENT
741 knlc->knlc_state = KNOTE_LOCK_CTX_UNLOCKED;
742#endif
743}
744
745/*
746 * Call the f_event hook of a given filter.
747 *
748 * Takes a use count to protect against concurrent drops.
749 */
750static void
751knote_post(struct knote *kn, long hint)
752{
753 struct kqueue *kq = knote_get_kq(kn);
754 int dropping, result;
755
756 kqlock(kq);
757
758 if (__improbable(kn->kn_status & (KN_DROPPING | KN_VANISHED))) {
759 return kqunlock(kq);
760 }
761
762 if (__improbable(kn->kn_status & KN_POSTING)) {
763 panic("KNOTE() called concurrently on knote %p", kn);
764 }
765
766 kn->kn_status |= KN_POSTING;
767
768 kqunlock(kq);
769 result = filter_call(knote_fops(kn), f_event(kn, hint));
770 kqlock(kq);
771
772 dropping = (kn->kn_status & KN_DROPPING);
773
774 if (!dropping && (result & FILTER_ACTIVE)) {
775 knote_activate(kq, kn, result);
776 }
777
778 if ((kn->kn_status & KN_LOCKED) == 0) {
779 /*
780 * There's no other f_* call in flight, we can leave QoS "Merge" mode.
781 *
782 * See knote_adjust_qos()
783 */
784 kn->kn_status &= ~(KN_POSTING | KN_MERGE_QOS);
785 } else {
786 kn->kn_status &= ~KN_POSTING;
787 }
788
789 if (__improbable(dropping)) {
790 waitq_wakeup64_all((struct waitq *)&kq->kq_wqs, knote_post_wev64(kn),
791 THREAD_AWAKENED, WAITQ_ALL_PRIORITIES);
792 }
793
794 kqunlock(kq);
795}
796
797/*
798 * Called by knote_drop() to wait for the last f_event() caller to be done.
799 *
800 * - kq locked at entry
801 * - kq unlocked at exit
802 */
803static void
804knote_wait_for_post(struct kqueue *kq, struct knote *kn)
805{
806 wait_result_t wr = THREAD_NOT_WAITING;
807
808 kqlock_held(kq);
809
810 assert(kn->kn_status & KN_DROPPING);
811
812 if (kn->kn_status & KN_POSTING) {
813 wr = waitq_assert_wait64((struct waitq *)&kq->kq_wqs,
814 knote_post_wev64(kn), THREAD_UNINT | THREAD_WAIT_NOREPORT,
815 TIMEOUT_WAIT_FOREVER);
816 }
817 kqunlock(kq);
818 if (wr == THREAD_WAITING) {
819 thread_block(THREAD_CONTINUE_NULL);
820 }
821}
822
823#pragma mark knote helpers for filters
824
825OS_ALWAYS_INLINE
826void
827knote_set_error(struct knote *kn, int error)
828{
829 kn->kn_flags |= EV_ERROR;
830 kn->kn_sdata = error;
831}
832
833OS_ALWAYS_INLINE
834int64_t
835knote_low_watermark(const struct knote *kn)
836{
837 return (kn->kn_sfflags & NOTE_LOWAT) ? kn->kn_sdata : 1;
838}
839
840/*!
841 * @function knote_fill_kevent_with_sdata
842 *
843 * @brief
844 * Fills in a kevent from the current content of a knote.
845 *
846 * @discussion
847 * This is meant to be called from filter's f_event hooks.
848 * The kevent data is filled with kn->kn_sdata.
849 *
850 * kn->kn_fflags is cleared if kn->kn_flags has EV_CLEAR set.
851 *
852 * Using knote_fill_kevent is typically preferred.
853 */
854OS_ALWAYS_INLINE
855void
856knote_fill_kevent_with_sdata(struct knote *kn, struct kevent_qos_s *kev)
857{
858#define knote_assert_aliases(name1, offs1, name2) \
859 static_assert(offsetof(struct kevent_qos_s, name1) + offs1 == \
860 offsetof(struct kevent_internal_s, name2), \
861 "kevent_qos_s::" #name1 " and kevent_internal_s::" #name2 "need to alias")
862 /*
863 * All the code makes assumptions on these aliasing,
864 * so make sure we fail the build if we ever ever ever break them.
865 */
866 knote_assert_aliases(ident, 0, kei_ident);
867#ifdef __LITTLE_ENDIAN__
868 knote_assert_aliases(filter, 0, kei_filter); // non trivial overlap
869 knote_assert_aliases(filter, 1, kei_filtid); // non trivial overlap
870#else
871 knote_assert_aliases(filter, 0, kei_filtid); // non trivial overlap
872 knote_assert_aliases(filter, 1, kei_filter); // non trivial overlap
873#endif
874 knote_assert_aliases(flags, 0, kei_flags);
875 knote_assert_aliases(qos, 0, kei_qos);
876 knote_assert_aliases(udata, 0, kei_udata);
877 knote_assert_aliases(fflags, 0, kei_fflags);
878 knote_assert_aliases(xflags, 0, kei_sfflags); // non trivial overlap
879 knote_assert_aliases(data, 0, kei_sdata); // non trivial overlap
880 knote_assert_aliases(ext, 0, kei_ext);
881#undef knote_assert_aliases
882
883 /*
884 * Fix the differences between kevent_qos_s and kevent_internal_s:
885 * - xflags is where kn_sfflags lives, we need to zero it
886 * - fixup the high bits of `filter` where kn_filtid lives
887 */
888 *kev = *(struct kevent_qos_s *)&kn->kn_kevent;
889 kev->xflags = 0;
890 kev->filter |= 0xff00;
891 if (kn->kn_flags & EV_CLEAR) {
892 kn->kn_fflags = 0;
893 }
894}
895
896/*!
897 * @function knote_fill_kevent
898 *
899 * @brief
900 * Fills in a kevent from the current content of a knote.
901 *
902 * @discussion
903 * This is meant to be called from filter's f_event hooks.
904 * The kevent data is filled with the passed in data.
905 *
906 * kn->kn_fflags is cleared if kn->kn_flags has EV_CLEAR set.
907 */
908OS_ALWAYS_INLINE
909void
910knote_fill_kevent(struct knote *kn, struct kevent_qos_s *kev, int64_t data)
911{
912 knote_fill_kevent_with_sdata(kn, kev);
913 kev->filter = kn->kn_filter;
914 kev->data = data;
915}
916
917
918#pragma mark file_filtops
919
920static int
921filt_fileattach(struct knote *kn, struct kevent_qos_s *kev)
922{
923 return fo_kqfilter(kn->kn_fp, kn, kev);
924}
925
926SECURITY_READ_ONLY_EARLY(static struct filterops) file_filtops = {
927 .f_isfd = 1,
928 .f_attach = filt_fileattach,
929};
930
931#pragma mark kqread_filtops
932
933#define f_flag f_fglob->fg_flag
934#define f_ops f_fglob->fg_ops
935#define f_data f_fglob->fg_data
936#define f_lflags f_fglob->fg_lflags
937
938static void
939filt_kqdetach(struct knote *kn)
940{
941 struct kqfile *kqf = (struct kqfile *)kn->kn_fp->f_data;
942 struct kqueue *kq = &kqf->kqf_kqueue;
943
944 kqlock(kq);
945 KNOTE_DETACH(&kqf->kqf_sel.si_note, kn);
946 kqunlock(kq);
947}
948
949static int
950filt_kqueue(struct knote *kn, __unused long hint)
951{
952 struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data;
953
954 return kq->kq_count > 0;
955}
956
957static int
958filt_kqtouch(struct knote *kn, struct kevent_qos_s *kev)
959{
960#pragma unused(kev)
961 struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data;
962 int res;
963
964 kqlock(kq);
965 res = (kq->kq_count > 0);
966 kqunlock(kq);
967
968 return res;
969}
970
971static int
972filt_kqprocess(struct knote *kn, struct kevent_qos_s *kev)
973{
974 struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data;
975 int res = 0;
976
977 kqlock(kq);
978 if (kq->kq_count) {
979 knote_fill_kevent(kn, kev, kq->kq_count);
980 res = 1;
981 }
982 kqunlock(kq);
983
984 return res;
985}
986
987SECURITY_READ_ONLY_EARLY(static struct filterops) kqread_filtops = {
988 .f_isfd = 1,
989 .f_detach = filt_kqdetach,
990 .f_event = filt_kqueue,
991 .f_touch = filt_kqtouch,
992 .f_process = filt_kqprocess,
993};
994
995#pragma mark proc_filtops
996
997static int
998filt_procattach(struct knote *kn, __unused struct kevent_qos_s *kev)
999{
1000 struct proc *p;
1001
1002 assert(PID_MAX < NOTE_PDATAMASK);
1003
1004 if ((kn->kn_sfflags & (NOTE_TRACK | NOTE_TRACKERR | NOTE_CHILD)) != 0) {
1005 knote_set_error(kn, ENOTSUP);
1006 return 0;
1007 }
1008
1009 p = proc_find(kn->kn_id);
1010 if (p == NULL) {
1011 knote_set_error(kn, ESRCH);
1012 return 0;
1013 }
1014
1015 const uint32_t NoteExitStatusBits = NOTE_EXIT | NOTE_EXITSTATUS;
1016
1017 if ((kn->kn_sfflags & NoteExitStatusBits) == NoteExitStatusBits) {
1018 do {
1019 pid_t selfpid = proc_selfpid();
1020
1021 if (p->p_ppid == selfpid) {
1022 break; /* parent => ok */
1023 }
1024 if ((p->p_lflag & P_LTRACED) != 0 &&
1025 (p->p_oppid == selfpid)) {
1026 break; /* parent-in-waiting => ok */
1027 }
1028 proc_rele(p);
1029 knote_set_error(kn, EACCES);
1030 return 0;
1031 } while (0);
1032 }
1033
1034 kn->kn_proc = p;
1035 kn->kn_flags |= EV_CLEAR; /* automatically set */
1036 kn->kn_sdata = 0; /* incoming data is ignored */
1037
1038 proc_klist_lock();
1039
1040 KNOTE_ATTACH(&p->p_klist, kn);
1041
1042 proc_klist_unlock();
1043
1044 proc_rele(p);
1045
1046 /*
1047 * only captures edge-triggered events after this point
1048 * so it can't already be fired.
1049 */
1050 return 0;
1051}
1052
1053
1054/*
1055 * The knote may be attached to a different process, which may exit,
1056 * leaving nothing for the knote to be attached to. In that case,
1057 * the pointer to the process will have already been nulled out.
1058 */
1059static void
1060filt_procdetach(struct knote *kn)
1061{
1062 struct proc *p;
1063
1064 proc_klist_lock();
1065
1066 p = kn->kn_proc;
1067 if (p != PROC_NULL) {
1068 kn->kn_proc = PROC_NULL;
1069 KNOTE_DETACH(&p->p_klist, kn);
1070 }
1071
1072 proc_klist_unlock();
1073}
1074
1075static int
1076filt_procevent(struct knote *kn, long hint)
1077{
1078 u_int event;
1079
1080 /* ALWAYS CALLED WITH proc_klist_lock */
1081
1082 /*
1083 * Note: a lot of bits in hint may be obtained from the knote
1084 * To free some of those bits, see <rdar://problem/12592988> Freeing up
1085 * bits in hint for filt_procevent
1086 *
1087 * mask off extra data
1088 */
1089 event = (u_int)hint & NOTE_PCTRLMASK;
1090
1091 /*
1092 * termination lifecycle events can happen while a debugger
1093 * has reparented a process, in which case notifications
1094 * should be quashed except to the tracing parent. When
1095 * the debugger reaps the child (either via wait4(2) or
1096 * process exit), the child will be reparented to the original
1097 * parent and these knotes re-fired.
1098 */
1099 if (event & NOTE_EXIT) {
1100 if ((kn->kn_proc->p_oppid != 0)
1101 && (knote_get_kq(kn)->kq_p->p_pid != kn->kn_proc->p_ppid)) {
1102 /*
1103 * This knote is not for the current ptrace(2) parent, ignore.
1104 */
1105 return 0;
1106 }
1107 }
1108
1109 /*
1110 * if the user is interested in this event, record it.
1111 */
1112 if (kn->kn_sfflags & event) {
1113 kn->kn_fflags |= event;
1114 }
1115
1116#pragma clang diagnostic push
1117#pragma clang diagnostic ignored "-Wdeprecated-declarations"
1118 if ((event == NOTE_REAP) || ((event == NOTE_EXIT) && !(kn->kn_sfflags & NOTE_REAP))) {
1119 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
1120 }
1121#pragma clang diagnostic pop
1122
1123
1124 /*
1125 * The kernel has a wrapper in place that returns the same data
1126 * as is collected here, in kn_hook32. Any changes to how
1127 * NOTE_EXITSTATUS and NOTE_EXIT_DETAIL are collected
1128 * should also be reflected in the proc_pidnoteexit() wrapper.
1129 */
1130 if (event == NOTE_EXIT) {
1131 kn->kn_hook32 = 0;
1132 if ((kn->kn_sfflags & NOTE_EXITSTATUS) != 0) {
1133 kn->kn_fflags |= NOTE_EXITSTATUS;
1134 kn->kn_hook32 |= (hint & NOTE_PDATAMASK);
1135 }
1136 if ((kn->kn_sfflags & NOTE_EXIT_DETAIL) != 0) {
1137 kn->kn_fflags |= NOTE_EXIT_DETAIL;
1138 if ((kn->kn_proc->p_lflag &
1139 P_LTERM_DECRYPTFAIL) != 0) {
1140 kn->kn_hook32 |= NOTE_EXIT_DECRYPTFAIL;
1141 }
1142 if ((kn->kn_proc->p_lflag &
1143 P_LTERM_JETSAM) != 0) {
1144 kn->kn_hook32 |= NOTE_EXIT_MEMORY;
1145 switch (kn->kn_proc->p_lflag & P_JETSAM_MASK) {
1146 case P_JETSAM_VMPAGESHORTAGE:
1147 kn->kn_hook32 |= NOTE_EXIT_MEMORY_VMPAGESHORTAGE;
1148 break;
1149 case P_JETSAM_VMTHRASHING:
1150 kn->kn_hook32 |= NOTE_EXIT_MEMORY_VMTHRASHING;
1151 break;
1152 case P_JETSAM_FCTHRASHING:
1153 kn->kn_hook32 |= NOTE_EXIT_MEMORY_FCTHRASHING;
1154 break;
1155 case P_JETSAM_VNODE:
1156 kn->kn_hook32 |= NOTE_EXIT_MEMORY_VNODE;
1157 break;
1158 case P_JETSAM_HIWAT:
1159 kn->kn_hook32 |= NOTE_EXIT_MEMORY_HIWAT;
1160 break;
1161 case P_JETSAM_PID:
1162 kn->kn_hook32 |= NOTE_EXIT_MEMORY_PID;
1163 break;
1164 case P_JETSAM_IDLEEXIT:
1165 kn->kn_hook32 |= NOTE_EXIT_MEMORY_IDLE;
1166 break;
1167 }
1168 }
1169 if ((kn->kn_proc->p_csflags &
1170 CS_KILLED) != 0) {
1171 kn->kn_hook32 |= NOTE_EXIT_CSERROR;
1172 }
1173 }
1174 }
1175
1176 /* if we have any matching state, activate the knote */
1177 return kn->kn_fflags != 0;
1178}
1179
1180static int
1181filt_proctouch(struct knote *kn, struct kevent_qos_s *kev)
1182{
1183 int res;
1184
1185 proc_klist_lock();
1186
1187 /* accept new filter flags and mask off output events no long interesting */
1188 kn->kn_sfflags = kev->fflags;
1189
1190 /* restrict the current results to the (smaller?) set of new interest */
1191 /*
1192 * For compatibility with previous implementations, we leave kn_fflags
1193 * as they were before.
1194 */
1195 //kn->kn_fflags &= kn->kn_sfflags;
1196
1197 res = (kn->kn_fflags != 0);
1198
1199 proc_klist_unlock();
1200
1201 return res;
1202}
1203
1204static int
1205filt_procprocess(struct knote *kn, struct kevent_qos_s *kev)
1206{
1207 int res = 0;
1208
1209 proc_klist_lock();
1210 if (kn->kn_fflags) {
1211 knote_fill_kevent(kn, kev, kn->kn_hook32);
1212 kn->kn_hook32 = 0;
1213 res = 1;
1214 }
1215 proc_klist_unlock();
1216 return res;
1217}
1218
1219SECURITY_READ_ONLY_EARLY(static struct filterops) proc_filtops = {
1220 .f_attach = filt_procattach,
1221 .f_detach = filt_procdetach,
1222 .f_event = filt_procevent,
1223 .f_touch = filt_proctouch,
1224 .f_process = filt_procprocess,
1225};
1226
1227#pragma mark timer_filtops
1228
1229struct filt_timer_params {
1230 uint64_t deadline; /* deadline in abs/cont time
1231 * (or 0 if NOTE_ABSOLUTE and deadline is in past) */
1232 uint64_t leeway; /* leeway in abstime, or 0 if none */
1233 uint64_t interval; /* interval in abstime or 0 if non-repeating timer */
1234};
1235
1236/*
1237 * Values stored in the knote at rest (using Mach absolute time units)
1238 *
1239 * kn->kn_thcall where the thread_call object is stored
1240 * kn->kn_ext[0] next deadline or 0 if immediate expiration
1241 * kn->kn_ext[1] leeway value
1242 * kn->kn_sdata interval timer: the interval
1243 * absolute/deadline timer: 0
1244 * kn->kn_hook32 timer state
1245 *
1246 * TIMER_IDLE:
1247 * The timer has either never been scheduled or been cancelled.
1248 * It is safe to schedule a new one in this state.
1249 *
1250 * TIMER_ARMED:
1251 * The timer has been scheduled
1252 *
1253 * TIMER_FIRED
1254 * The timer has fired and an event needs to be delivered.
1255 * When in this state, the callout may still be running.
1256 *
1257 * TIMER_IMMEDIATE
1258 * The timer has fired at registration time, and the callout was never
1259 * dispatched.
1260 */
1261#define TIMER_IDLE 0x0
1262#define TIMER_ARMED 0x1
1263#define TIMER_FIRED 0x2
1264#define TIMER_IMMEDIATE 0x3
1265
1266static void
1267filt_timer_set_params(struct knote *kn, struct filt_timer_params *params)
1268{
1269 kn->kn_ext[0] = params->deadline;
1270 kn->kn_ext[1] = params->leeway;
1271 kn->kn_sdata = params->interval;
1272}
1273
1274/*
1275 * filt_timervalidate - process data from user
1276 *
1277 * Sets up the deadline, interval, and leeway from the provided user data
1278 *
1279 * Input:
1280 * kn_sdata timer deadline or interval time
1281 * kn_sfflags style of timer, unit of measurement
1282 *
1283 * Output:
1284 * struct filter_timer_params to apply to the filter with
1285 * filt_timer_set_params when changes are ready to be commited.
1286 *
1287 * Returns:
1288 * EINVAL Invalid user data parameters
1289 * ERANGE Various overflows with the parameters
1290 *
1291 * Called with timer filter lock held.
1292 */
1293static int
1294filt_timervalidate(const struct kevent_qos_s *kev,
1295 struct filt_timer_params *params)
1296{
1297 /*
1298 * There are 5 knobs that need to be chosen for a timer registration:
1299 *
1300 * A) Units of time (what is the time duration of the specified number)
1301 * Absolute and interval take:
1302 * NOTE_SECONDS, NOTE_USECONDS, NOTE_NSECONDS, NOTE_MACHTIME
1303 * Defaults to milliseconds if not specified
1304 *
1305 * B) Clock epoch (what is the zero point of the specified number)
1306 * For interval, there is none
1307 * For absolute, defaults to the gettimeofday/calendar epoch
1308 * With NOTE_MACHTIME, uses mach_absolute_time()
1309 * With NOTE_MACHTIME and NOTE_MACH_CONTINUOUS_TIME, uses mach_continuous_time()
1310 *
1311 * C) The knote's behavior on delivery
1312 * Interval timer causes the knote to arm for the next interval unless one-shot is set
1313 * Absolute is a forced one-shot timer which deletes on delivery
1314 * TODO: Add a way for absolute to be not forced one-shot
1315 *
1316 * D) Whether the time duration is relative to now or absolute
1317 * Interval fires at now + duration when it is set up
1318 * Absolute fires at now + difference between now walltime and passed in walltime
1319 * With NOTE_MACHTIME it fires at an absolute MAT or MCT.
1320 *
1321 * E) Whether the timer continues to tick across sleep
1322 * By default all three do not.
1323 * For interval and absolute, NOTE_MACH_CONTINUOUS_TIME causes them to tick across sleep
1324 * With NOTE_ABSOLUTE | NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME:
1325 * expires when mach_continuous_time() is > the passed in value.
1326 */
1327
1328 uint64_t multiplier;
1329
1330 boolean_t use_abstime = FALSE;
1331
1332 switch (kev->fflags & (NOTE_SECONDS | NOTE_USECONDS | NOTE_NSECONDS | NOTE_MACHTIME)) {
1333 case NOTE_SECONDS:
1334 multiplier = NSEC_PER_SEC;
1335 break;
1336 case NOTE_USECONDS:
1337 multiplier = NSEC_PER_USEC;
1338 break;
1339 case NOTE_NSECONDS:
1340 multiplier = 1;
1341 break;
1342 case NOTE_MACHTIME:
1343 multiplier = 0;
1344 use_abstime = TRUE;
1345 break;
1346 case 0: /* milliseconds (default) */
1347 multiplier = NSEC_PER_SEC / 1000;
1348 break;
1349 default:
1350 return EINVAL;
1351 }
1352
1353 /* transform the leeway in kn_ext[1] to same time scale */
1354 if (kev->fflags & NOTE_LEEWAY) {
1355 uint64_t leeway_abs;
1356
1357 if (use_abstime) {
1358 leeway_abs = (uint64_t)kev->ext[1];
1359 } else {
1360 uint64_t leeway_ns;
1361 if (os_mul_overflow((uint64_t)kev->ext[1], multiplier, &leeway_ns)) {
1362 return ERANGE;
1363 }
1364
1365 nanoseconds_to_absolutetime(leeway_ns, &leeway_abs);
1366 }
1367
1368 params->leeway = leeway_abs;
1369 } else {
1370 params->leeway = 0;
1371 }
1372
1373 if (kev->fflags & NOTE_ABSOLUTE) {
1374 uint64_t deadline_abs;
1375
1376 if (use_abstime) {
1377 deadline_abs = (uint64_t)kev->data;
1378 } else {
1379 uint64_t calendar_deadline_ns;
1380
1381 if (os_mul_overflow((uint64_t)kev->data, multiplier, &calendar_deadline_ns)) {
1382 return ERANGE;
1383 }
1384
1385 /* calendar_deadline_ns is in nanoseconds since the epoch */
1386
1387 clock_sec_t seconds;
1388 clock_nsec_t nanoseconds;
1389
1390 /*
1391 * Note that the conversion through wall-time is only done once.
1392 *
1393 * If the relationship between MAT and gettimeofday changes,
1394 * the underlying timer does not update.
1395 *
1396 * TODO: build a wall-time denominated timer_call queue
1397 * and a flag to request DTRTing with wall-time timers
1398 */
1399 clock_get_calendar_nanotime(&seconds, &nanoseconds);
1400
1401 uint64_t calendar_now_ns = (uint64_t)seconds * NSEC_PER_SEC + nanoseconds;
1402
1403 /* if deadline is in the future */
1404 if (calendar_now_ns < calendar_deadline_ns) {
1405 uint64_t interval_ns = calendar_deadline_ns - calendar_now_ns;
1406 uint64_t interval_abs;
1407
1408 nanoseconds_to_absolutetime(interval_ns, &interval_abs);
1409
1410 /*
1411 * Note that the NOTE_MACH_CONTINUOUS_TIME flag here only
1412 * causes the timer to keep ticking across sleep, but
1413 * it does not change the calendar timebase.
1414 */
1415
1416 if (kev->fflags & NOTE_MACH_CONTINUOUS_TIME) {
1417 clock_continuoustime_interval_to_deadline(interval_abs,
1418 &deadline_abs);
1419 } else {
1420 clock_absolutetime_interval_to_deadline(interval_abs,
1421 &deadline_abs);
1422 }
1423 } else {
1424 deadline_abs = 0; /* cause immediate expiration */
1425 }
1426 }
1427
1428 params->deadline = deadline_abs;
1429 params->interval = 0; /* NOTE_ABSOLUTE is non-repeating */
1430 } else if (kev->data < 0) {
1431 /*
1432 * Negative interval timers fire immediately, once.
1433 *
1434 * Ideally a negative interval would be an error, but certain clients
1435 * pass negative values on accident, and expect an event back.
1436 *
1437 * In the old implementation the timer would repeat with no delay
1438 * N times until mach_absolute_time() + (N * interval) underflowed,
1439 * then it would wait ~forever by accidentally arming a timer for the far future.
1440 *
1441 * We now skip the power-wasting hot spin phase and go straight to the idle phase.
1442 */
1443
1444 params->deadline = 0; /* expire immediately */
1445 params->interval = 0; /* non-repeating */
1446 } else {
1447 uint64_t interval_abs = 0;
1448
1449 if (use_abstime) {
1450 interval_abs = (uint64_t)kev->data;
1451 } else {
1452 uint64_t interval_ns;
1453 if (os_mul_overflow((uint64_t)kev->data, multiplier, &interval_ns)) {
1454 return ERANGE;
1455 }
1456
1457 nanoseconds_to_absolutetime(interval_ns, &interval_abs);
1458 }
1459
1460 uint64_t deadline = 0;
1461
1462 if (kev->fflags & NOTE_MACH_CONTINUOUS_TIME) {
1463 clock_continuoustime_interval_to_deadline(interval_abs, &deadline);
1464 } else {
1465 clock_absolutetime_interval_to_deadline(interval_abs, &deadline);
1466 }
1467
1468 params->deadline = deadline;
1469 params->interval = interval_abs;
1470 }
1471
1472 return 0;
1473}
1474
1475/*
1476 * filt_timerexpire - the timer callout routine
1477 */
1478static void
1479filt_timerexpire(void *knx, __unused void *spare)
1480{
1481 struct knote *kn = knx;
1482 int v;
1483
1484 if (os_atomic_cmpxchgv(&kn->kn_hook32, TIMER_ARMED, TIMER_FIRED,
1485 &v, relaxed)) {
1486 // our f_event always would say FILTER_ACTIVE,
1487 // so be leaner and just do it.
1488 struct kqueue *kq = knote_get_kq(kn);
1489 kqlock(kq);
1490 knote_activate(kq, kn, FILTER_ACTIVE);
1491 kqunlock(kq);
1492 } else {
1493 /*
1494 * From TIMER_ARMED, the only allowed transition are:
1495 * - to TIMER_FIRED through the timer callout just above
1496 * - to TIMER_IDLE due to filt_timercancel() which will wait for the
1497 * timer callout (and any possible invocation of filt_timerexpire) to
1498 * have finished before the state is changed again.
1499 */
1500 assert(v == TIMER_IDLE);
1501 }
1502}
1503
1504static void
1505filt_timercancel(struct knote *kn)
1506{
1507 if (os_atomic_xchg(&kn->kn_hook32, TIMER_IDLE, relaxed) == TIMER_ARMED) {
1508 /* cancel the thread call and wait for any filt_timerexpire in flight */
1509 thread_call_cancel_wait(kn->kn_thcall);
1510 }
1511}
1512
1513/*
1514 * Does this deadline needs a timer armed for it, or has it expired?
1515 */
1516static bool
1517filt_timer_is_ready(struct knote *kn)
1518{
1519 uint64_t now, deadline = kn->kn_ext[0];
1520
1521 if (deadline == 0) {
1522 return true;
1523 }
1524
1525 if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) {
1526 now = mach_continuous_time();
1527 } else {
1528 now = mach_absolute_time();
1529 }
1530 return deadline <= now;
1531}
1532
1533/*
1534 * Arm a timer
1535 *
1536 * It is the responsibility of the caller to make sure the timer call
1537 * has completed or been cancelled properly prior to arming it.
1538 */
1539static void
1540filt_timerarm(struct knote *kn)
1541{
1542 uint64_t deadline = kn->kn_ext[0];
1543 uint64_t leeway = kn->kn_ext[1];
1544
1545 int filter_flags = kn->kn_sfflags;
1546 unsigned int timer_flags = 0;
1547
1548 assert(os_atomic_load(&kn->kn_hook32, relaxed) == TIMER_IDLE);
1549
1550 if (filter_flags & NOTE_CRITICAL) {
1551 timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL;
1552 } else if (filter_flags & NOTE_BACKGROUND) {
1553 timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND;
1554 } else {
1555 timer_flags |= THREAD_CALL_DELAY_USER_NORMAL;
1556 }
1557
1558 if (filter_flags & NOTE_LEEWAY) {
1559 timer_flags |= THREAD_CALL_DELAY_LEEWAY;
1560 }
1561
1562 if (filter_flags & NOTE_MACH_CONTINUOUS_TIME) {
1563 timer_flags |= THREAD_CALL_CONTINUOUS;
1564 }
1565
1566 os_atomic_store(&kn->kn_hook32, TIMER_ARMED, relaxed);
1567 thread_call_enter_delayed_with_leeway(kn->kn_thcall, NULL,
1568 deadline, leeway, timer_flags);
1569}
1570
1571/*
1572 * Allocate a thread call for the knote's lifetime, and kick off the timer.
1573 */
1574static int
1575filt_timerattach(struct knote *kn, struct kevent_qos_s *kev)
1576{
1577 thread_call_t callout;
1578 struct filt_timer_params params;
1579 int error;
1580
1581 if ((error = filt_timervalidate(kev, &params)) != 0) {
1582 knote_set_error(kn, error);
1583 return 0;
1584 }
1585
1586 callout = thread_call_allocate_with_options(filt_timerexpire,
1587 (thread_call_param_t)kn, THREAD_CALL_PRIORITY_HIGH,
1588 THREAD_CALL_OPTIONS_ONCE);
1589
1590 if (NULL == callout) {
1591 knote_set_error(kn, ENOMEM);
1592 return 0;
1593 }
1594
1595 filt_timer_set_params(kn, &params);
1596 kn->kn_thcall = callout;
1597 kn->kn_flags |= EV_CLEAR;
1598 os_atomic_store(&kn->kn_hook32, TIMER_IDLE, relaxed);
1599
1600 /* NOTE_ABSOLUTE implies EV_ONESHOT */
1601 if (kn->kn_sfflags & NOTE_ABSOLUTE) {
1602 kn->kn_flags |= EV_ONESHOT;
1603 }
1604
1605 if (filt_timer_is_ready(kn)) {
1606 os_atomic_store(&kn->kn_hook32, TIMER_IMMEDIATE, relaxed);
1607 return FILTER_ACTIVE;
1608 } else {
1609 filt_timerarm(kn);
1610 return 0;
1611 }
1612}
1613
1614/*
1615 * Shut down the timer if it's running, and free the callout.
1616 */
1617static void
1618filt_timerdetach(struct knote *kn)
1619{
1620 __assert_only boolean_t freed;
1621
1622 /*
1623 * Unconditionally cancel to make sure there can't be any filt_timerexpire()
1624 * running anymore.
1625 */
1626 thread_call_cancel_wait(kn->kn_thcall);
1627 freed = thread_call_free(kn->kn_thcall);
1628 assert(freed);
1629}
1630
1631/*
1632 * filt_timertouch - update timer knote with new user input
1633 *
1634 * Cancel and restart the timer based on new user data. When
1635 * the user picks up a knote, clear the count of how many timer
1636 * pops have gone off (in kn_data).
1637 */
1638static int
1639filt_timertouch(struct knote *kn, struct kevent_qos_s *kev)
1640{
1641 struct filt_timer_params params;
1642 uint32_t changed_flags = (kn->kn_sfflags ^ kev->fflags);
1643 int error;
1644
1645 if (changed_flags & NOTE_ABSOLUTE) {
1646 kev->flags |= EV_ERROR;
1647 kev->data = EINVAL;
1648 return 0;
1649 }
1650
1651 if ((error = filt_timervalidate(kev, &params)) != 0) {
1652 kev->flags |= EV_ERROR;
1653 kev->data = error;
1654 return 0;
1655 }
1656
1657 /* capture the new values used to compute deadline */
1658 filt_timercancel(kn);
1659 filt_timer_set_params(kn, &params);
1660 kn->kn_sfflags = kev->fflags;
1661
1662 if (filt_timer_is_ready(kn)) {
1663 os_atomic_store(&kn->kn_hook32, TIMER_IMMEDIATE, relaxed);
1664 return FILTER_ACTIVE | FILTER_UPDATE_REQ_QOS;
1665 } else {
1666 filt_timerarm(kn);
1667 return FILTER_UPDATE_REQ_QOS;
1668 }
1669}
1670
1671/*
1672 * filt_timerprocess - query state of knote and snapshot event data
1673 *
1674 * Determine if the timer has fired in the past, snapshot the state
1675 * of the kevent for returning to user-space, and clear pending event
1676 * counters for the next time.
1677 */
1678static int
1679filt_timerprocess(struct knote *kn, struct kevent_qos_s *kev)
1680{
1681 /*
1682 * filt_timerprocess is serialized with any filter routine except for
1683 * filt_timerexpire which atomically does a TIMER_ARMED -> TIMER_FIRED
1684 * transition, and on success, activates the knote.
1685 *
1686 * Hence, we don't need atomic modifications of the state, only to peek at
1687 * whether we see any of the "FIRED" state, and if we do, it is safe to
1688 * do simple state machine transitions.
1689 */
1690 switch (os_atomic_load(&kn->kn_hook32, relaxed)) {
1691 case TIMER_IDLE:
1692 case TIMER_ARMED:
1693 /*
1694 * This can happen if a touch resets a timer that had fired
1695 * without being processed
1696 */
1697 return 0;
1698 }
1699
1700 os_atomic_store(&kn->kn_hook32, TIMER_IDLE, relaxed);
1701
1702 /*
1703 * Copy out the interesting kevent state,
1704 * but don't leak out the raw time calculations.
1705 *
1706 * TODO: potential enhancements - tell the user about:
1707 * - deadline to which this timer thought it was expiring
1708 * - return kn_sfflags in the fflags field so the client can know
1709 * under what flags the timer fired
1710 */
1711 knote_fill_kevent(kn, kev, 1);
1712 kev->ext[0] = 0;
1713 /* kev->ext[1] = 0; JMM - shouldn't we hide this too? */
1714
1715 if (kn->kn_sdata != 0) {
1716 /*
1717 * This is a 'repeating' timer, so we have to emit
1718 * how many intervals expired between the arm
1719 * and the process.
1720 *
1721 * A very strange style of interface, because
1722 * this could easily be done in the client...
1723 */
1724
1725 uint64_t now;
1726
1727 if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) {
1728 now = mach_continuous_time();
1729 } else {
1730 now = mach_absolute_time();
1731 }
1732
1733 uint64_t first_deadline = kn->kn_ext[0];
1734 uint64_t interval_abs = kn->kn_sdata;
1735 uint64_t orig_arm_time = first_deadline - interval_abs;
1736
1737 assert(now > orig_arm_time);
1738 assert(now > first_deadline);
1739
1740 uint64_t elapsed = now - orig_arm_time;
1741
1742 uint64_t num_fired = elapsed / interval_abs;
1743
1744 /*
1745 * To reach this code, we must have seen the timer pop
1746 * and be in repeating mode, so therefore it must have been
1747 * more than 'interval' time since the attach or last
1748 * successful touch.
1749 */
1750 assert(num_fired > 0);
1751
1752 /* report how many intervals have elapsed to the user */
1753 kev->data = (int64_t)num_fired;
1754
1755 /* We only need to re-arm the timer if it's not about to be destroyed */
1756 if ((kn->kn_flags & EV_ONESHOT) == 0) {
1757 /* fire at the end of the next interval */
1758 uint64_t new_deadline = first_deadline + num_fired * interval_abs;
1759
1760 assert(new_deadline > now);
1761
1762 kn->kn_ext[0] = new_deadline;
1763
1764 /*
1765 * This can't shortcut setting up the thread call, because
1766 * knote_process deactivates EV_CLEAR knotes unconditionnally.
1767 */
1768 filt_timerarm(kn);
1769 }
1770 }
1771
1772 return FILTER_ACTIVE;
1773}
1774
1775SECURITY_READ_ONLY_EARLY(static struct filterops) timer_filtops = {
1776 .f_extended_codes = true,
1777 .f_attach = filt_timerattach,
1778 .f_detach = filt_timerdetach,
1779 .f_event = filt_bad_event,
1780 .f_touch = filt_timertouch,
1781 .f_process = filt_timerprocess,
1782};
1783
1784#pragma mark user_filtops
1785
1786static int
1787filt_userattach(struct knote *kn, __unused struct kevent_qos_s *kev)
1788{
1789 if (kn->kn_sfflags & NOTE_TRIGGER) {
1790 kn->kn_hook32 = FILTER_ACTIVE;
1791 } else {
1792 kn->kn_hook32 = 0;
1793 }
1794 return kn->kn_hook32;
1795}
1796
1797static int
1798filt_usertouch(struct knote *kn, struct kevent_qos_s *kev)
1799{
1800 uint32_t ffctrl;
1801 int fflags;
1802
1803 ffctrl = kev->fflags & NOTE_FFCTRLMASK;
1804 fflags = kev->fflags & NOTE_FFLAGSMASK;
1805 switch (ffctrl) {
1806 case NOTE_FFNOP:
1807 break;
1808 case NOTE_FFAND:
1809 kn->kn_sfflags &= fflags;
1810 break;
1811 case NOTE_FFOR:
1812 kn->kn_sfflags |= fflags;
1813 break;
1814 case NOTE_FFCOPY:
1815 kn->kn_sfflags = fflags;
1816 break;
1817 }
1818 kn->kn_sdata = kev->data;
1819
1820 if (kev->fflags & NOTE_TRIGGER) {
1821 kn->kn_hook32 = FILTER_ACTIVE;
1822 }
1823 return (int)kn->kn_hook32;
1824}
1825
1826static int
1827filt_userprocess(struct knote *kn, struct kevent_qos_s *kev)
1828{
1829 int result = (int)kn->kn_hook32;
1830
1831 if (result) {
1832 /* EVFILT_USER returns the data that was passed in */
1833 knote_fill_kevent_with_sdata(kn, kev);
1834 kev->fflags = kn->kn_sfflags;
1835 if (kn->kn_flags & EV_CLEAR) {
1836 /* knote_fill_kevent cleared kn_fflags */
1837 kn->kn_hook32 = 0;
1838 }
1839 }
1840
1841 return result;
1842}
1843
1844SECURITY_READ_ONLY_EARLY(static struct filterops) user_filtops = {
1845 .f_extended_codes = true,
1846 .f_attach = filt_userattach,
1847 .f_detach = filt_no_detach,
1848 .f_event = filt_bad_event,
1849 .f_touch = filt_usertouch,
1850 .f_process = filt_userprocess,
1851};
1852
1853#pragma mark workloop_filtops
1854
1855#define EPREEMPTDISABLED (-1)
1856
1857static inline void
1858filt_wllock(struct kqworkloop *kqwl)
1859{
1860 lck_spin_lock(&kqwl->kqwl_statelock);
1861}
1862
1863static inline void
1864filt_wlunlock(struct kqworkloop *kqwl)
1865{
1866 lck_spin_unlock(&kqwl->kqwl_statelock);
1867}
1868
1869/*
1870 * Returns true when the interlock for the turnstile is the workqueue lock
1871 *
1872 * When this is the case, all turnstiles operations are delegated
1873 * to the workqueue subsystem.
1874 *
1875 * This is required because kqueue_threadreq_bind_prepost only holds the
1876 * workqueue lock but needs to move the inheritor from the workloop turnstile
1877 * away from the creator thread, so that this now fulfilled request cannot be
1878 * picked anymore by other threads.
1879 */
1880static inline bool
1881filt_wlturnstile_interlock_is_workq(struct kqworkloop *kqwl)
1882{
1883 return kqr_thread_requested_pending(&kqwl->kqwl_request);
1884}
1885
1886static void
1887filt_wlupdate_inheritor(struct kqworkloop *kqwl, struct turnstile *ts,
1888 turnstile_update_flags_t flags)
1889{
1890 turnstile_inheritor_t inheritor = TURNSTILE_INHERITOR_NULL;
1891 workq_threadreq_t kqr = &kqwl->kqwl_request;
1892
1893 /*
1894 * binding to the workq should always happen through
1895 * workq_kern_threadreq_update_inheritor()
1896 */
1897 assert(!filt_wlturnstile_interlock_is_workq(kqwl));
1898
1899 if ((inheritor = kqwl->kqwl_owner)) {
1900 flags |= TURNSTILE_INHERITOR_THREAD;
1901 } else if ((inheritor = kqr_thread(kqr))) {
1902 flags |= TURNSTILE_INHERITOR_THREAD;
1903 }
1904
1905 turnstile_update_inheritor(ts, inheritor, flags);
1906}
1907
1908#define EVFILT_WORKLOOP_EFAULT_RETRY_COUNT 100
1909#define FILT_WLATTACH 0
1910#define FILT_WLTOUCH 1
1911#define FILT_WLDROP 2
1912
1913__result_use_check
1914static int
1915filt_wlupdate(struct kqworkloop *kqwl, struct knote *kn,
1916 struct kevent_qos_s *kev, kq_index_t qos_index, int op)
1917{
1918 user_addr_t uaddr = CAST_USER_ADDR_T(kev->ext[EV_EXTIDX_WL_ADDR]);
1919 workq_threadreq_t kqr = &kqwl->kqwl_request;
1920 thread_t cur_owner, new_owner, extra_thread_ref = THREAD_NULL;
1921 kq_index_t cur_override = THREAD_QOS_UNSPECIFIED;
1922 int efault_retry = EVFILT_WORKLOOP_EFAULT_RETRY_COUNT;
1923 int action = KQWL_UTQ_NONE, error = 0;
1924 bool wl_inheritor_updated = false, needs_wake = false;
1925 uint64_t kdata = kev->ext[EV_EXTIDX_WL_VALUE];
1926 uint64_t mask = kev->ext[EV_EXTIDX_WL_MASK];
1927 uint64_t udata = 0;
1928 struct turnstile *ts = TURNSTILE_NULL;
1929
1930 filt_wllock(kqwl);
1931
1932again:
1933 new_owner = cur_owner = kqwl->kqwl_owner;
1934
1935 /*
1936 * Phase 1:
1937 *
1938 * If asked, load the uint64 value at the user provided address and compare
1939 * it against the passed in mask and expected value.
1940 *
1941 * If NOTE_WL_DISCOVER_OWNER is specified, translate the loaded name as
1942 * a thread reference.
1943 *
1944 * If NOTE_WL_END_OWNERSHIP is specified and the currently known owner is
1945 * the current thread, then end ownership.
1946 *
1947 * Lastly decide whether we need to perform a QoS update.
1948 */
1949 if (uaddr) {
1950 /*
1951 * Until <rdar://problem/24999882> exists,
1952 * disabling preemption copyin forces any
1953 * vm_fault we encounter to fail.
1954 */
1955 error = copyin_atomic64(uaddr, &udata);
1956
1957 /*
1958 * If we get EFAULT, drop locks, and retry.
1959 * If we still get an error report it,
1960 * else assume the memory has been faulted
1961 * and attempt to copyin under lock again.
1962 */
1963 switch (error) {
1964 case 0:
1965 break;
1966 case EFAULT:
1967 if (efault_retry-- > 0) {
1968 filt_wlunlock(kqwl);
1969 error = copyin_atomic64(uaddr, &udata);
1970 filt_wllock(kqwl);
1971 if (error == 0) {
1972 goto again;
1973 }
1974 }
1975 /* FALLTHROUGH */
1976 default:
1977 goto out;
1978 }
1979
1980 /* Update state as copied in. */
1981 kev->ext[EV_EXTIDX_WL_VALUE] = udata;
1982
1983 if ((udata & mask) != (kdata & mask)) {
1984 error = ESTALE;
1985 } else if (kev->fflags & NOTE_WL_DISCOVER_OWNER) {
1986 /*
1987 * Decipher the owner port name, and translate accordingly.
1988 * The low 2 bits were borrowed for other flags, so mask them off.
1989 *
1990 * Then attempt translation to a thread reference or fail.
1991 */
1992 mach_port_name_t name = (mach_port_name_t)udata & ~0x3;
1993 if (name != MACH_PORT_NULL) {
1994 name = ipc_entry_name_mask(name);
1995 extra_thread_ref = port_name_to_thread(name,
1996 PORT_TO_THREAD_IN_CURRENT_TASK);
1997 if (extra_thread_ref == THREAD_NULL) {
1998 error = EOWNERDEAD;
1999 goto out;
2000 }
2001 new_owner = extra_thread_ref;
2002 }
2003 }
2004 }
2005
2006 if ((kev->fflags & NOTE_WL_END_OWNERSHIP) && new_owner == current_thread()) {
2007 new_owner = THREAD_NULL;
2008 }
2009
2010 if (error == 0) {
2011 if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && (kev->flags & EV_DELETE)) {
2012 action = KQWL_UTQ_SET_QOS_INDEX;
2013 } else if (qos_index && kqr->tr_kq_qos_index != qos_index) {
2014 action = KQWL_UTQ_SET_QOS_INDEX;
2015 }
2016
2017 if (op == FILT_WLTOUCH) {
2018 /*
2019 * Save off any additional fflags/data we just accepted
2020 * But only keep the last round of "update" bits we acted on which helps
2021 * debugging a lot.
2022 */
2023 kn->kn_sfflags &= ~NOTE_WL_UPDATES_MASK;
2024 kn->kn_sfflags |= kev->fflags;
2025 if (kev->fflags & NOTE_WL_SYNC_WAKE) {
2026 needs_wake = (kn->kn_thread != THREAD_NULL);
2027 }
2028 } else if (op == FILT_WLDROP) {
2029 if ((kn->kn_sfflags & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE)) ==
2030 NOTE_WL_SYNC_WAIT) {
2031 /*
2032 * When deleting a SYNC_WAIT knote that hasn't been woken up
2033 * explicitly, issue a wake up.
2034 */
2035 kn->kn_sfflags |= NOTE_WL_SYNC_WAKE;
2036 needs_wake = (kn->kn_thread != THREAD_NULL);
2037 }
2038 }
2039 }
2040
2041 /*
2042 * Phase 2:
2043 *
2044 * Commit ownership and QoS changes if any, possibly wake up waiters
2045 */
2046
2047 if (cur_owner == new_owner && action == KQWL_UTQ_NONE && !needs_wake) {
2048 goto out;
2049 }
2050
2051 kqlock(kqwl);
2052
2053 /* If already tracked as servicer, don't track as owner */
2054 if (new_owner == kqr_thread(kqr)) {
2055 new_owner = THREAD_NULL;
2056 }
2057
2058 if (cur_owner != new_owner) {
2059 kqwl->kqwl_owner = new_owner;
2060 if (new_owner == extra_thread_ref) {
2061 /* we just transfered this ref to kqwl_owner */
2062 extra_thread_ref = THREAD_NULL;
2063 }
2064 cur_override = kqworkloop_override(kqwl);
2065
2066 if (new_owner) {
2067 /* override it before we drop the old */
2068 if (cur_override != THREAD_QOS_UNSPECIFIED) {
2069 thread_add_kevent_override(new_owner, cur_override);
2070 }
2071 if (kqr_thread_requested_pending(kqr)) {
2072 if (action == KQWL_UTQ_NONE) {
2073 action = KQWL_UTQ_REDRIVE_EVENTS;
2074 }
2075 }
2076 } else {
2077 if (!kqr_thread_requested(kqr) && kqr->tr_kq_wakeup) {
2078 if (action == KQWL_UTQ_NONE) {
2079 action = KQWL_UTQ_REDRIVE_EVENTS;
2080 }
2081 }
2082 }
2083 }
2084
2085 if (action != KQWL_UTQ_NONE) {
2086 kqworkloop_update_threads_qos(kqwl, action, qos_index);
2087 }
2088
2089 ts = kqwl->kqwl_turnstile;
2090 if (cur_owner != new_owner && ts) {
2091 if (action == KQWL_UTQ_REDRIVE_EVENTS) {
2092 /*
2093 * Note that when action is KQWL_UTQ_REDRIVE_EVENTS,
2094 * the code went through workq_kern_threadreq_initiate()
2095 * and the workqueue has set the inheritor already
2096 */
2097 assert(filt_wlturnstile_interlock_is_workq(kqwl));
2098 } else if (filt_wlturnstile_interlock_is_workq(kqwl)) {
2099 workq_kern_threadreq_lock(kqwl->kqwl_p);
2100 workq_kern_threadreq_update_inheritor(kqwl->kqwl_p, kqr, new_owner,
2101 ts, TURNSTILE_IMMEDIATE_UPDATE);
2102 workq_kern_threadreq_unlock(kqwl->kqwl_p);
2103 if (!filt_wlturnstile_interlock_is_workq(kqwl)) {
2104 /*
2105 * If the workq is no longer the interlock, then
2106 * workq_kern_threadreq_update_inheritor() has finished a bind
2107 * and we need to fallback to the regular path.
2108 */
2109 filt_wlupdate_inheritor(kqwl, ts, TURNSTILE_IMMEDIATE_UPDATE);
2110 }
2111 wl_inheritor_updated = true;
2112 } else {
2113 filt_wlupdate_inheritor(kqwl, ts, TURNSTILE_IMMEDIATE_UPDATE);
2114 wl_inheritor_updated = true;
2115 }
2116
2117 /*
2118 * We need a turnstile reference because we are dropping the interlock
2119 * and the caller has not called turnstile_prepare.
2120 */
2121 if (wl_inheritor_updated) {
2122 turnstile_reference(ts);
2123 }
2124 }
2125
2126 if (needs_wake && ts) {
2127 waitq_wakeup64_thread(&ts->ts_waitq, knote_filt_wev64(kn),
2128 kn->kn_thread, THREAD_AWAKENED);
2129 if (op == FILT_WLATTACH || op == FILT_WLTOUCH) {
2130 disable_preemption();
2131 error = EPREEMPTDISABLED;
2132 }
2133 }
2134
2135 kqunlock(kqwl);
2136
2137out:
2138 /*
2139 * Phase 3:
2140 *
2141 * Unlock and cleanup various lingering references and things.
2142 */
2143 filt_wlunlock(kqwl);
2144
2145#if CONFIG_WORKLOOP_DEBUG
2146 KQWL_HISTORY_WRITE_ENTRY(kqwl, {
2147 .updater = current_thread(),
2148 .servicer = kqr_thread(kqr), /* Note: racy */
2149 .old_owner = cur_owner,
2150 .new_owner = new_owner,
2151
2152 .kev_ident = kev->ident,
2153 .error = (int16_t)error,
2154 .kev_flags = kev->flags,
2155 .kev_fflags = kev->fflags,
2156
2157 .kev_mask = mask,
2158 .kev_value = kdata,
2159 .in_value = udata,
2160 });
2161#endif // CONFIG_WORKLOOP_DEBUG
2162
2163 if (wl_inheritor_updated) {
2164 turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
2165 turnstile_deallocate_safe(ts);
2166 }
2167
2168 if (cur_owner && new_owner != cur_owner) {
2169 if (cur_override != THREAD_QOS_UNSPECIFIED) {
2170 thread_drop_kevent_override(cur_owner);
2171 }
2172 thread_deallocate_safe(cur_owner);
2173 }
2174 if (extra_thread_ref) {
2175 thread_deallocate_safe(extra_thread_ref);
2176 }
2177 return error;
2178}
2179
2180/*
2181 * Remembers the last updated that came in from userspace for debugging reasons.
2182 * - fflags is mirrored from the userspace kevent
2183 * - ext[i, i != VALUE] is mirrored from the userspace kevent
2184 * - ext[VALUE] is set to what the kernel loaded atomically
2185 * - data is set to the error if any
2186 */
2187static inline void
2188filt_wlremember_last_update(struct knote *kn, struct kevent_qos_s *kev,
2189 int error)
2190{
2191 kn->kn_fflags = kev->fflags;
2192 kn->kn_sdata = error;
2193 memcpy(kn->kn_ext, kev->ext, sizeof(kev->ext));
2194}
2195
2196static int
2197filt_wlupdate_sync_ipc(struct kqworkloop *kqwl, struct knote *kn,
2198 struct kevent_qos_s *kev, int op)
2199{
2200 uint64_t uaddr = kev->ext[EV_EXTIDX_WL_ADDR];
2201 uint64_t kdata = kev->ext[EV_EXTIDX_WL_VALUE];
2202 uint64_t mask = kev->ext[EV_EXTIDX_WL_MASK];
2203 uint64_t udata = 0;
2204 int efault_retry = EVFILT_WORKLOOP_EFAULT_RETRY_COUNT;
2205 int error = 0;
2206
2207 if (op == FILT_WLATTACH) {
2208 (void)kqueue_alloc_turnstile(&kqwl->kqwl_kqueue);
2209 } else if (uaddr == 0) {
2210 return 0;
2211 }
2212
2213 filt_wllock(kqwl);
2214
2215again:
2216
2217 /*
2218 * Do the debounce thing, the lock serializing the state is the knote lock.
2219 */
2220 if (uaddr) {
2221 /*
2222 * Until <rdar://problem/24999882> exists,
2223 * disabling preemption copyin forces any
2224 * vm_fault we encounter to fail.
2225 */
2226 error = copyin_atomic64(uaddr, &udata);
2227
2228 /*
2229 * If we get EFAULT, drop locks, and retry.
2230 * If we still get an error report it,
2231 * else assume the memory has been faulted
2232 * and attempt to copyin under lock again.
2233 */
2234 switch (error) {
2235 case 0:
2236 break;
2237 case EFAULT:
2238 if (efault_retry-- > 0) {
2239 filt_wlunlock(kqwl);
2240 error = copyin_atomic64(uaddr, &udata);
2241 filt_wllock(kqwl);
2242 if (error == 0) {
2243 goto again;
2244 }
2245 }
2246 /* FALLTHROUGH */
2247 default:
2248 goto out;
2249 }
2250
2251 kev->ext[EV_EXTIDX_WL_VALUE] = udata;
2252 kn->kn_ext[EV_EXTIDX_WL_VALUE] = udata;
2253
2254 if ((udata & mask) != (kdata & mask)) {
2255 error = ESTALE;
2256 goto out;
2257 }
2258 }
2259
2260 if (op == FILT_WLATTACH) {
2261 error = filt_wlattach_sync_ipc(kn);
2262 if (error == 0) {
2263 disable_preemption();
2264 error = EPREEMPTDISABLED;
2265 }
2266 }
2267
2268out:
2269 filt_wlunlock(kqwl);
2270 return error;
2271}
2272
2273static int
2274filt_wlattach(struct knote *kn, struct kevent_qos_s *kev)
2275{
2276 struct kqueue *kq = knote_get_kq(kn);
2277 struct kqworkloop *kqwl = (struct kqworkloop *)kq;
2278 int error = 0, result = 0;
2279 kq_index_t qos_index = 0;
2280
2281 if (__improbable((kq->kq_state & KQ_WORKLOOP) == 0)) {
2282 error = ENOTSUP;
2283 goto out;
2284 }
2285
2286 uint32_t command = (kn->kn_sfflags & NOTE_WL_COMMANDS_MASK);
2287 switch (command) {
2288 case NOTE_WL_THREAD_REQUEST:
2289 if (kn->kn_id != kqwl->kqwl_dynamicid) {
2290 error = EINVAL;
2291 goto out;
2292 }
2293 qos_index = _pthread_priority_thread_qos(kn->kn_qos);
2294 if (qos_index == THREAD_QOS_UNSPECIFIED) {
2295 error = ERANGE;
2296 goto out;
2297 }
2298 if (kqwl->kqwl_request.tr_kq_qos_index) {
2299 /*
2300 * There already is a thread request, and well, you're only allowed
2301 * one per workloop, so fail the attach.
2302 */
2303 error = EALREADY;
2304 goto out;
2305 }
2306 break;
2307 case NOTE_WL_SYNC_WAIT:
2308 case NOTE_WL_SYNC_WAKE:
2309 if (kn->kn_id == kqwl->kqwl_dynamicid) {
2310 error = EINVAL;
2311 goto out;
2312 }
2313 if ((kn->kn_flags & EV_DISABLE) == 0) {
2314 error = EINVAL;
2315 goto out;
2316 }
2317 if (kn->kn_sfflags & NOTE_WL_END_OWNERSHIP) {
2318 error = EINVAL;
2319 goto out;
2320 }
2321 break;
2322
2323 case NOTE_WL_SYNC_IPC:
2324 if ((kn->kn_flags & EV_DISABLE) == 0) {
2325 error = EINVAL;
2326 goto out;
2327 }
2328 if (kn->kn_sfflags & (NOTE_WL_UPDATE_QOS | NOTE_WL_DISCOVER_OWNER)) {
2329 error = EINVAL;
2330 goto out;
2331 }
2332 break;
2333 default:
2334 error = EINVAL;
2335 goto out;
2336 }
2337
2338 if (command == NOTE_WL_SYNC_IPC) {
2339 error = filt_wlupdate_sync_ipc(kqwl, kn, kev, FILT_WLATTACH);
2340 } else {
2341 error = filt_wlupdate(kqwl, kn, kev, qos_index, FILT_WLATTACH);
2342 }
2343
2344 if (error == EPREEMPTDISABLED) {
2345 error = 0;
2346 result = FILTER_THREADREQ_NODEFEER;
2347 }
2348out:
2349 if (error) {
2350 /* If userland wants ESTALE to be hidden, fail the attach anyway */
2351 if (error == ESTALE && (kn->kn_sfflags & NOTE_WL_IGNORE_ESTALE)) {
2352 error = 0;
2353 }
2354 knote_set_error(kn, error);
2355 return result;
2356 }
2357 if (command == NOTE_WL_SYNC_WAIT) {
2358 return kevent_register_wait_prepare(kn, kev, result);
2359 }
2360 /* Just attaching the thread request successfully will fire it */
2361 if (command == NOTE_WL_THREAD_REQUEST) {
2362 /*
2363 * Thread Request knotes need an explicit touch to be active again,
2364 * so delivering an event needs to also consume it.
2365 */
2366 kn->kn_flags |= EV_CLEAR;
2367 return result | FILTER_ACTIVE;
2368 }
2369 return result;
2370}
2371
2372static void __dead2
2373filt_wlwait_continue(void *parameter, wait_result_t wr)
2374{
2375 struct _kevent_register *cont_args = parameter;
2376 struct kqworkloop *kqwl = cont_args->kqwl;
2377
2378 kqlock(kqwl);
2379 if (filt_wlturnstile_interlock_is_workq(kqwl)) {
2380 workq_kern_threadreq_lock(kqwl->kqwl_p);
2381 turnstile_complete((uintptr_t)kqwl, &kqwl->kqwl_turnstile, NULL, TURNSTILE_WORKLOOPS);
2382 workq_kern_threadreq_unlock(kqwl->kqwl_p);
2383 } else {
2384 turnstile_complete((uintptr_t)kqwl, &kqwl->kqwl_turnstile, NULL, TURNSTILE_WORKLOOPS);
2385 }
2386 kqunlock(kqwl);
2387
2388 turnstile_cleanup();
2389
2390 if (wr == THREAD_INTERRUPTED) {
2391 cont_args->kev.flags |= EV_ERROR;
2392 cont_args->kev.data = EINTR;
2393 } else if (wr != THREAD_AWAKENED) {
2394 panic("Unexpected wait result: %d", wr);
2395 }
2396
2397 kevent_register_wait_return(cont_args);
2398}
2399
2400/*
2401 * Called with the workloop mutex held, most of the time never returns as it
2402 * calls filt_wlwait_continue through a continuation.
2403 */
2404static void __dead2
2405filt_wlpost_register_wait(struct uthread *uth, struct knote *kn,
2406 struct _kevent_register *cont_args)
2407{
2408 struct kqworkloop *kqwl = cont_args->kqwl;
2409 workq_threadreq_t kqr = &kqwl->kqwl_request;
2410 struct turnstile *ts;
2411 bool workq_locked = false;
2412
2413 kqlock_held(kqwl);
2414
2415 if (filt_wlturnstile_interlock_is_workq(kqwl)) {
2416 workq_kern_threadreq_lock(kqwl->kqwl_p);
2417 workq_locked = true;
2418 }
2419
2420 ts = turnstile_prepare((uintptr_t)kqwl, &kqwl->kqwl_turnstile,
2421 TURNSTILE_NULL, TURNSTILE_WORKLOOPS);
2422
2423 if (workq_locked) {
2424 workq_kern_threadreq_update_inheritor(kqwl->kqwl_p,
2425 &kqwl->kqwl_request, kqwl->kqwl_owner, ts,
2426 TURNSTILE_DELAYED_UPDATE);
2427 if (!filt_wlturnstile_interlock_is_workq(kqwl)) {
2428 /*
2429 * if the interlock is no longer the workqueue lock,
2430 * then we don't need to hold it anymore.
2431 */
2432 workq_kern_threadreq_unlock(kqwl->kqwl_p);
2433 workq_locked = false;
2434 }
2435 }
2436 if (!workq_locked) {
2437 /*
2438 * If the interlock is the workloop's, then it's our responsibility to
2439 * call update_inheritor, so just do it.
2440 */
2441 filt_wlupdate_inheritor(kqwl, ts, TURNSTILE_DELAYED_UPDATE);
2442 }
2443
2444 thread_set_pending_block_hint(uth->uu_thread, kThreadWaitWorkloopSyncWait);
2445 waitq_assert_wait64(&ts->ts_waitq, knote_filt_wev64(kn),
2446 THREAD_ABORTSAFE, TIMEOUT_WAIT_FOREVER);
2447
2448 if (workq_locked) {
2449 workq_kern_threadreq_unlock(kqwl->kqwl_p);
2450 }
2451
2452 thread_t thread = kqwl->kqwl_owner ?: kqr_thread(kqr);
2453 if (thread) {
2454 thread_reference(thread);
2455 }
2456
2457 kevent_register_wait_block(ts, thread, filt_wlwait_continue, cont_args);
2458}
2459
2460/* called in stackshot context to report the thread responsible for blocking this thread */
2461void
2462kdp_workloop_sync_wait_find_owner(__assert_only thread_t thread,
2463 event64_t event, thread_waitinfo_t *waitinfo)
2464{
2465 struct knote *kn = (struct knote *)event;
2466 assert(kdp_is_in_zone(kn, "knote zone"));
2467
2468 assert(kn->kn_thread == thread);
2469
2470 struct kqueue *kq = knote_get_kq(kn);
2471 assert(kdp_is_in_zone(kq, "kqueue workloop zone"));
2472 assert(kq->kq_state & KQ_WORKLOOP);
2473
2474 struct kqworkloop *kqwl = (struct kqworkloop *)kq;
2475 workq_threadreq_t kqr = &kqwl->kqwl_request;
2476
2477 thread_t kqwl_owner = kqwl->kqwl_owner;
2478
2479 if (kqwl_owner != THREAD_NULL) {
2480 assert(kdp_is_in_zone(kqwl_owner, "threads"));
2481
2482 waitinfo->owner = thread_tid(kqwl->kqwl_owner);
2483 } else if (kqr_thread_requested_pending(kqr)) {
2484 waitinfo->owner = STACKSHOT_WAITOWNER_THREQUESTED;
2485 } else if (kqr->tr_state >= WORKQ_TR_STATE_BINDING) {
2486 assert(kdp_is_in_zone(kqr->tr_thread, "threads"));
2487 waitinfo->owner = thread_tid(kqr->tr_thread);
2488 } else {
2489 waitinfo->owner = 0;
2490 }
2491
2492 waitinfo->context = kqwl->kqwl_dynamicid;
2493}
2494
2495static void
2496filt_wldetach(struct knote *kn)
2497{
2498 if (kn->kn_sfflags & NOTE_WL_SYNC_IPC) {
2499 filt_wldetach_sync_ipc(kn);
2500 } else if (kn->kn_thread) {
2501 kevent_register_wait_cleanup(kn);
2502 }
2503}
2504
2505static int
2506filt_wlvalidate_kev_flags(struct knote *kn, struct kevent_qos_s *kev,
2507 thread_qos_t *qos_index)
2508{
2509 uint32_t new_commands = kev->fflags & NOTE_WL_COMMANDS_MASK;
2510 uint32_t sav_commands = kn->kn_sfflags & NOTE_WL_COMMANDS_MASK;
2511
2512 if ((kev->fflags & NOTE_WL_DISCOVER_OWNER) && (kev->flags & EV_DELETE)) {
2513 return EINVAL;
2514 }
2515 if (kev->fflags & NOTE_WL_UPDATE_QOS) {
2516 if (kev->flags & EV_DELETE) {
2517 return EINVAL;
2518 }
2519 if (sav_commands != NOTE_WL_THREAD_REQUEST) {
2520 return EINVAL;
2521 }
2522 if (!(*qos_index = _pthread_priority_thread_qos(kev->qos))) {
2523 return ERANGE;
2524 }
2525 }
2526
2527 switch (new_commands) {
2528 case NOTE_WL_THREAD_REQUEST:
2529 /* thread requests can only update themselves */
2530 if (sav_commands != NOTE_WL_THREAD_REQUEST) {
2531 return EINVAL;
2532 }
2533 break;
2534
2535 case NOTE_WL_SYNC_WAIT:
2536 if (kev->fflags & NOTE_WL_END_OWNERSHIP) {
2537 return EINVAL;
2538 }
2539 goto sync_checks;
2540
2541 case NOTE_WL_SYNC_WAKE:
2542sync_checks:
2543 if (!(sav_commands & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE))) {
2544 return EINVAL;
2545 }
2546 if ((kev->flags & (EV_ENABLE | EV_DELETE)) == EV_ENABLE) {
2547 return EINVAL;
2548 }
2549 break;
2550
2551 case NOTE_WL_SYNC_IPC:
2552 if (sav_commands != NOTE_WL_SYNC_IPC) {
2553 return EINVAL;
2554 }
2555 if ((kev->flags & (EV_ENABLE | EV_DELETE)) == EV_ENABLE) {
2556 return EINVAL;
2557 }
2558 break;
2559
2560 default:
2561 return EINVAL;
2562 }
2563 return 0;
2564}
2565
2566static int
2567filt_wltouch(struct knote *kn, struct kevent_qos_s *kev)
2568{
2569 struct kqworkloop *kqwl = (struct kqworkloop *)knote_get_kq(kn);
2570 thread_qos_t qos_index = THREAD_QOS_UNSPECIFIED;
2571 int result = 0;
2572
2573 int error = filt_wlvalidate_kev_flags(kn, kev, &qos_index);
2574 if (error) {
2575 goto out;
2576 }
2577
2578 uint32_t command = kev->fflags & NOTE_WL_COMMANDS_MASK;
2579 if (command == NOTE_WL_SYNC_IPC) {
2580 error = filt_wlupdate_sync_ipc(kqwl, kn, kev, FILT_WLTOUCH);
2581 } else {
2582 error = filt_wlupdate(kqwl, kn, kev, qos_index, FILT_WLTOUCH);
2583 filt_wlremember_last_update(kn, kev, error);
2584 }
2585 if (error == EPREEMPTDISABLED) {
2586 error = 0;
2587 result = FILTER_THREADREQ_NODEFEER;
2588 }
2589
2590out:
2591 if (error) {
2592 if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) {
2593 /* If userland wants ESTALE to be hidden, do not activate */
2594 return result;
2595 }
2596 kev->flags |= EV_ERROR;
2597 kev->data = error;
2598 return result;
2599 }
2600 if (command == NOTE_WL_SYNC_WAIT && !(kn->kn_sfflags & NOTE_WL_SYNC_WAKE)) {
2601 return kevent_register_wait_prepare(kn, kev, result);
2602 }
2603 /* Just touching the thread request successfully will fire it */
2604 if (command == NOTE_WL_THREAD_REQUEST) {
2605 if (kev->fflags & NOTE_WL_UPDATE_QOS) {
2606 result |= FILTER_UPDATE_REQ_QOS;
2607 }
2608 result |= FILTER_ACTIVE;
2609 }
2610 return result;
2611}
2612
2613static bool
2614filt_wlallow_drop(struct knote *kn, struct kevent_qos_s *kev)
2615{
2616 struct kqworkloop *kqwl = (struct kqworkloop *)knote_get_kq(kn);
2617
2618 int error = filt_wlvalidate_kev_flags(kn, kev, NULL);
2619 if (error) {
2620 goto out;
2621 }
2622
2623 uint32_t command = (kev->fflags & NOTE_WL_COMMANDS_MASK);
2624 if (command == NOTE_WL_SYNC_IPC) {
2625 error = filt_wlupdate_sync_ipc(kqwl, kn, kev, FILT_WLDROP);
2626 } else {
2627 error = filt_wlupdate(kqwl, kn, kev, 0, FILT_WLDROP);
2628 filt_wlremember_last_update(kn, kev, error);
2629 }
2630 assert(error != EPREEMPTDISABLED);
2631
2632out:
2633 if (error) {
2634 if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) {
2635 return false;
2636 }
2637 kev->flags |= EV_ERROR;
2638 kev->data = error;
2639 return false;
2640 }
2641 return true;
2642}
2643
2644static int
2645filt_wlprocess(struct knote *kn, struct kevent_qos_s *kev)
2646{
2647 struct kqworkloop *kqwl = (struct kqworkloop *)knote_get_kq(kn);
2648 int rc = 0;
2649
2650 assert(kn->kn_sfflags & NOTE_WL_THREAD_REQUEST);
2651
2652 kqlock(kqwl);
2653
2654 if (kqwl->kqwl_owner) {
2655 /*
2656 * <rdar://problem/33584321> userspace sometimes due to events being
2657 * delivered but not triggering a drain session can cause a process
2658 * of the thread request knote.
2659 *
2660 * When that happens, the automatic deactivation due to process
2661 * would swallow the event, so we have to activate the knote again.
2662 */
2663 knote_activate(kqwl, kn, FILTER_ACTIVE);
2664 } else {
2665#if DEBUG || DEVELOPMENT
2666 if (kevent_debug_flags() & KEVENT_PANIC_ON_NON_ENQUEUED_PROCESS) {
2667 /*
2668 * see src/queue_internal.h in libdispatch
2669 */
2670#define DISPATCH_QUEUE_ENQUEUED 0x1ull
2671 user_addr_t addr = CAST_USER_ADDR_T(kn->kn_ext[EV_EXTIDX_WL_ADDR]);
2672 task_t t = current_task();
2673 uint64_t val;
2674 if (addr && task_is_active(t) && !task_is_halting(t) &&
2675 copyin_atomic64(addr, &val) == 0 &&
2676 val && (val & DISPATCH_QUEUE_ENQUEUED) == 0 &&
2677 (val >> 48) != 0xdead && (val >> 48) != 0 && (val >> 48) != 0xffff) {
2678 panic("kevent: workloop %#016llx is not enqueued "
2679 "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)",
2680 kn->kn_udata, kn, val, kn->kn_ext[EV_EXTIDX_WL_VALUE]);
2681 }
2682 }
2683#endif
2684 knote_fill_kevent(kn, kev, 0);
2685 kev->fflags = kn->kn_sfflags;
2686 rc |= FILTER_ACTIVE;
2687 }
2688
2689 kqunlock(kqwl);
2690
2691 if (rc & FILTER_ACTIVE) {
2692 workq_thread_set_max_qos(kqwl->kqwl_p, &kqwl->kqwl_request);
2693 }
2694 return rc;
2695}
2696
2697SECURITY_READ_ONLY_EARLY(static struct filterops) workloop_filtops = {
2698 .f_extended_codes = true,
2699 .f_attach = filt_wlattach,
2700 .f_detach = filt_wldetach,
2701 .f_event = filt_bad_event,
2702 .f_touch = filt_wltouch,
2703 .f_process = filt_wlprocess,
2704 .f_allow_drop = filt_wlallow_drop,
2705 .f_post_register_wait = filt_wlpost_register_wait,
2706};
2707
2708#pragma mark - kqueues allocation and deallocation
2709
2710/*!
2711 * @enum kqworkloop_dealloc_flags_t
2712 *
2713 * @brief
2714 * Flags that alter kqworkloop_dealloc() behavior.
2715 *
2716 * @const KQWL_DEALLOC_NONE
2717 * Convenient name for "no flags".
2718 *
2719 * @const KQWL_DEALLOC_SKIP_HASH_REMOVE
2720 * Do not remove the workloop fromt he hash table.
2721 * This is used for process tear-down codepaths as the workloops have been
2722 * removed by the caller already.
2723 */
2724OS_OPTIONS(kqworkloop_dealloc_flags, unsigned,
2725 KQWL_DEALLOC_NONE = 0x0000,
2726 KQWL_DEALLOC_SKIP_HASH_REMOVE = 0x0001,
2727 );
2728
2729static void
2730kqworkloop_dealloc(struct kqworkloop *, kqworkloop_dealloc_flags_t, uint32_t);
2731
2732OS_NOINLINE OS_COLD OS_NORETURN
2733static void
2734kqworkloop_retain_panic(struct kqworkloop *kqwl, uint32_t previous)
2735{
2736 if (previous == 0) {
2737 panic("kq(%p) resurrection", kqwl);
2738 } else {
2739 panic("kq(%p) retain overflow", kqwl);
2740 }
2741}
2742
2743OS_NOINLINE OS_COLD OS_NORETURN
2744static void
2745kqworkloop_release_panic(struct kqworkloop *kqwl)
2746{
2747 panic("kq(%p) over-release", kqwl);
2748}
2749
2750OS_ALWAYS_INLINE
2751static inline bool
2752kqworkloop_try_retain(struct kqworkloop *kqwl)
2753{
2754 uint32_t old_ref, new_ref;
2755 os_atomic_rmw_loop(&kqwl->kqwl_retains, old_ref, new_ref, relaxed, {
2756 if (__improbable(old_ref == 0)) {
2757 os_atomic_rmw_loop_give_up(return false);
2758 }
2759 if (__improbable(old_ref >= KQ_WORKLOOP_RETAINS_MAX)) {
2760 kqworkloop_retain_panic(kqwl, old_ref);
2761 }
2762 new_ref = old_ref + 1;
2763 });
2764 return true;
2765}
2766
2767OS_ALWAYS_INLINE
2768static inline void
2769kqworkloop_retain(struct kqworkloop *kqwl)
2770{
2771 uint32_t previous = os_atomic_inc_orig(&kqwl->kqwl_retains, relaxed);
2772 if (__improbable(previous == 0 || previous >= KQ_WORKLOOP_RETAINS_MAX)) {
2773 kqworkloop_retain_panic(kqwl, previous);
2774 }
2775}
2776
2777OS_ALWAYS_INLINE
2778static inline void
2779kqueue_retain(kqueue_t kqu)
2780{
2781 if (kqu.kq->kq_state & KQ_DYNAMIC) {
2782 kqworkloop_retain(kqu.kqwl);
2783 }
2784}
2785
2786OS_ALWAYS_INLINE
2787static inline void
2788kqworkloop_release_live(struct kqworkloop *kqwl)
2789{
2790 uint32_t refs = os_atomic_dec_orig(&kqwl->kqwl_retains, relaxed);
2791 if (__improbable(refs <= 1)) {
2792 kqworkloop_release_panic(kqwl);
2793 }
2794}
2795
2796OS_ALWAYS_INLINE
2797static inline void
2798kqueue_release_live(kqueue_t kqu)
2799{
2800 if (kqu.kq->kq_state & KQ_DYNAMIC) {
2801 kqworkloop_release_live(kqu.kqwl);
2802 }
2803}
2804
2805OS_ALWAYS_INLINE
2806static inline void
2807kqworkloop_release(struct kqworkloop *kqwl)
2808{
2809 uint32_t refs = os_atomic_dec_orig(&kqwl->kqwl_retains, relaxed);
2810
2811 if (__improbable(refs <= 1)) {
2812 kqworkloop_dealloc(kqwl, KQWL_DEALLOC_NONE, refs - 1);
2813 }
2814}
2815
2816OS_ALWAYS_INLINE
2817static inline void
2818kqueue_release(kqueue_t kqu)
2819{
2820 if (kqu.kq->kq_state & KQ_DYNAMIC) {
2821 kqworkloop_release(kqu.kqwl);
2822 }
2823}
2824
2825/*!
2826 * @function kqueue_destroy
2827 *
2828 * @brief
2829 * Common part to all kqueue dealloc functions.
2830 */
2831OS_NOINLINE
2832static void
2833kqueue_destroy(kqueue_t kqu, zone_t zone)
2834{
2835 /*
2836 * waitq_set_deinit() remove the KQ's waitq set from
2837 * any select sets to which it may belong.
2838 *
2839 * The order of these deinits matter: before waitq_set_deinit() returns,
2840 * waitq_set__CALLING_PREPOST_HOOK__ may be called and it will take the
2841 * kq_lock.
2842 */
2843 waitq_set_deinit(&kqu.kq->kq_wqs);
2844 lck_spin_destroy(&kqu.kq->kq_lock, kq_lck_grp);
2845
2846 zfree(zone, kqu.kq);
2847}
2848
2849/*!
2850 * @function kqueue_init
2851 *
2852 * @brief
2853 * Common part to all kqueue alloc functions.
2854 */
2855static kqueue_t
2856kqueue_init(kqueue_t kqu, waitq_set_prepost_hook_t *hook, int policy)
2857{
2858 waitq_set_init(&kqu.kq->kq_wqs, policy, NULL, hook);
2859 lck_spin_init(&kqu.kq->kq_lock, kq_lck_grp, kq_lck_attr);
2860 return kqu;
2861}
2862
2863#pragma mark kqfile allocation and deallocation
2864
2865/*!
2866 * @function kqueue_dealloc
2867 *
2868 * @brief
2869 * Detach all knotes from a kqfile and free it.
2870 *
2871 * @discussion
2872 * We walk each list looking for knotes referencing this
2873 * this kqueue. If we find one, we try to drop it. But
2874 * if we fail to get a drop reference, that will wait
2875 * until it is dropped. So, we can just restart again
2876 * safe in the assumption that the list will eventually
2877 * not contain any more references to this kqueue (either
2878 * we dropped them all, or someone else did).
2879 *
2880 * Assumes no new events are being added to the kqueue.
2881 * Nothing locked on entry or exit.
2882 */
2883void
2884kqueue_dealloc(struct kqueue *kq)
2885{
2886 KNOTE_LOCK_CTX(knlc);
2887 struct proc *p = kq->kq_p;
2888 struct filedesc *fdp = p->p_fd;
2889 struct knote *kn;
2890
2891 assert(kq && (kq->kq_state & (KQ_WORKLOOP | KQ_WORKQ)) == 0);
2892
2893 proc_fdlock(p);
2894 for (int i = 0; i < fdp->fd_knlistsize; i++) {
2895 kn = SLIST_FIRST(&fdp->fd_knlist[i]);
2896 while (kn != NULL) {
2897 if (kq == knote_get_kq(kn)) {
2898 kqlock(kq);
2899 proc_fdunlock(p);
2900 if (knote_lock(kq, kn, &knlc, KNOTE_KQ_LOCK_ON_SUCCESS)) {
2901 knote_drop(kq, kn, &knlc);
2902 }
2903 proc_fdlock(p);
2904 /* start over at beginning of list */
2905 kn = SLIST_FIRST(&fdp->fd_knlist[i]);
2906 continue;
2907 }
2908 kn = SLIST_NEXT(kn, kn_link);
2909 }
2910 }
2911
2912 knhash_lock(fdp);
2913 proc_fdunlock(p);
2914
2915 if (fdp->fd_knhashmask != 0) {
2916 for (int i = 0; i < (int)fdp->fd_knhashmask + 1; i++) {
2917 kn = SLIST_FIRST(&fdp->fd_knhash[i]);
2918 while (kn != NULL) {
2919 if (kq == knote_get_kq(kn)) {
2920 kqlock(kq);
2921 knhash_unlock(fdp);
2922 if (knote_lock(kq, kn, &knlc, KNOTE_KQ_LOCK_ON_SUCCESS)) {
2923 knote_drop(kq, kn, &knlc);
2924 }
2925 knhash_lock(fdp);
2926 /* start over at beginning of list */
2927 kn = SLIST_FIRST(&fdp->fd_knhash[i]);
2928 continue;
2929 }
2930 kn = SLIST_NEXT(kn, kn_link);
2931 }
2932 }
2933 }
2934 knhash_unlock(fdp);
2935
2936 kqueue_destroy(kq, kqfile_zone);
2937}
2938
2939/*!
2940 * @function kqueue_alloc
2941 *
2942 * @brief
2943 * Allocate a kqfile.
2944 */
2945struct kqueue *
2946kqueue_alloc(struct proc *p)
2947{
2948 struct kqfile *kqf;
2949
2950 kqf = (struct kqfile *)zalloc(kqfile_zone);
2951 if (__improbable(kqf == NULL)) {
2952 return NULL;
2953 }
2954 bzero(kqf, sizeof(struct kqfile));
2955
2956 /*
2957 * kqfiles are created with kqueue() so we need to wait for
2958 * the first kevent syscall to know which bit among
2959 * KQ_KEV_{32,64,QOS} will be set in kqf_state
2960 */
2961 kqf->kqf_p = p;
2962 TAILQ_INIT_AFTER_BZERO(&kqf->kqf_queue);
2963 TAILQ_INIT_AFTER_BZERO(&kqf->kqf_suppressed);
2964
2965 return kqueue_init(kqf, NULL, SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST).kq;
2966}
2967
2968/*!
2969 * @function kqueue_internal
2970 *
2971 * @brief
2972 * Core implementation for kqueue and guarded_kqueue_np()
2973 */
2974int
2975kqueue_internal(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval)
2976{
2977 struct kqueue *kq;
2978 struct fileproc *fp;
2979 int fd, error;
2980
2981 error = falloc_withalloc(p, &fp, &fd, vfs_context_current(), fp_zalloc, cra);
2982 if (error) {
2983 return error;
2984 }
2985
2986 kq = kqueue_alloc(p);
2987 if (kq == NULL) {
2988 fp_free(p, fd, fp);
2989 return ENOMEM;
2990 }
2991
2992 fp->f_flag = FREAD | FWRITE;
2993 fp->f_ops = &kqueueops;
2994 fp->f_data = kq;
2995 fp->f_lflags |= FG_CONFINED;
2996
2997 proc_fdlock(p);
2998 *fdflags(p, fd) |= UF_EXCLOSE | UF_FORKCLOSE;
2999 procfdtbl_releasefd(p, fd, NULL);
3000 fp_drop(p, fd, fp, 1);
3001 proc_fdunlock(p);
3002
3003 *retval = fd;
3004 return error;
3005}
3006
3007/*!
3008 * @function kqueue
3009 *
3010 * @brief
3011 * The kqueue syscall.
3012 */
3013int
3014kqueue(struct proc *p, __unused struct kqueue_args *uap, int32_t *retval)
3015{
3016 return kqueue_internal(p, fileproc_alloc_init, NULL, retval);
3017}
3018
3019#pragma mark kqworkq allocation and deallocation
3020
3021/*!
3022 * @function kqworkq_dealloc
3023 *
3024 * @brief
3025 * Deallocates a workqueue kqueue.
3026 *
3027 * @discussion
3028 * This only happens at process death, or for races with concurrent
3029 * kevent_get_kqwq calls, hence we don't have to care about knotes referencing
3030 * this kqueue, either there are none, or someone else took care of them.
3031 */
3032void
3033kqworkq_dealloc(struct kqworkq *kqwq)
3034{
3035 kqueue_destroy(kqwq, kqworkq_zone);
3036}
3037
3038/*!
3039 * @function kqworkq_alloc
3040 *
3041 * @brief
3042 * Allocates a workqueue kqueue.
3043 *
3044 * @discussion
3045 * This is the slow path of kevent_get_kqwq.
3046 * This takes care of making sure procs have a single workq kqueue.
3047 */
3048OS_NOINLINE
3049static struct kqworkq *
3050kqworkq_alloc(struct proc *p, unsigned int flags)
3051{
3052 struct kqworkq *kqwq, *tmp;
3053
3054 kqwq = (struct kqworkq *)zalloc(kqworkq_zone);
3055 if (__improbable(kqwq == NULL)) {
3056 return NULL;
3057 }
3058 bzero(kqwq, sizeof(struct kqworkq));
3059
3060 assert((flags & KEVENT_FLAG_LEGACY32) == 0);
3061 if (flags & KEVENT_FLAG_LEGACY64) {
3062 kqwq->kqwq_state = KQ_WORKQ | KQ_KEV64;
3063 } else {
3064 kqwq->kqwq_state = KQ_WORKQ | KQ_KEV_QOS;
3065 }
3066 kqwq->kqwq_p = p;
3067
3068 for (int i = 0; i < KQWQ_NBUCKETS; i++) {
3069 TAILQ_INIT_AFTER_BZERO(&kqwq->kqwq_queue[i]);
3070 TAILQ_INIT_AFTER_BZERO(&kqwq->kqwq_suppressed[i]);
3071 }
3072 for (int i = 0; i < KQWQ_NBUCKETS; i++) {
3073 /*
3074 * Because of how the bucketized system works, we mix overcommit
3075 * sources with not overcommit: each time we move a knote from
3076 * one bucket to the next due to overrides, we'd had to track
3077 * overcommitness, and it's really not worth it in the workloop
3078 * enabled world that track this faithfully.
3079 *
3080 * Incidentally, this behaves like the original manager-based
3081 * kqwq where event delivery always happened (hence is
3082 * "overcommit")
3083 */
3084 kqwq->kqwq_request[i].tr_state = WORKQ_TR_STATE_IDLE;
3085 kqwq->kqwq_request[i].tr_flags = WORKQ_TR_FLAG_KEVENT;
3086 if (i != KQWQ_QOS_MANAGER) {
3087 kqwq->kqwq_request[i].tr_flags |= WORKQ_TR_FLAG_OVERCOMMIT;
3088 }
3089 kqwq->kqwq_request[i].tr_kq_qos_index = i;
3090 }
3091
3092 kqueue_init(kqwq, &kqwq->kqwq_waitq_hook, SYNC_POLICY_FIFO);
3093
3094 if (!os_atomic_cmpxchgv(&p->p_fd->fd_wqkqueue, NULL, kqwq, &tmp, release)) {
3095 kqworkq_dealloc(kqwq);
3096 return tmp;
3097 }
3098
3099 return kqwq;
3100}
3101
3102#pragma mark kqworkloop allocation and deallocation
3103
3104#define KQ_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
3105#define CONFIG_KQ_HASHSIZE CONFIG_KN_HASHSIZE
3106
3107OS_ALWAYS_INLINE
3108static inline void
3109kqhash_lock(struct filedesc *fdp)
3110{
3111 lck_mtx_lock_spin_always(&fdp->fd_kqhashlock);
3112}
3113
3114OS_ALWAYS_INLINE
3115static inline void
3116kqhash_unlock(struct filedesc *fdp)
3117{
3118 lck_mtx_unlock(&fdp->fd_kqhashlock);
3119}
3120
3121OS_ALWAYS_INLINE
3122static inline void
3123kqworkloop_hash_insert_locked(struct filedesc *fdp, kqueue_id_t id,
3124 struct kqworkloop *kqwl)
3125{
3126 struct kqwllist *list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)];
3127 LIST_INSERT_HEAD(list, kqwl, kqwl_hashlink);
3128}
3129
3130OS_ALWAYS_INLINE
3131static inline struct kqworkloop *
3132kqworkloop_hash_lookup_locked(struct filedesc *fdp, kqueue_id_t id)
3133{
3134 struct kqwllist *list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)];
3135 struct kqworkloop *kqwl;
3136
3137 LIST_FOREACH(kqwl, list, kqwl_hashlink) {
3138 if (kqwl->kqwl_dynamicid == id) {
3139 return kqwl;
3140 }
3141 }
3142 return NULL;
3143}
3144
3145static struct kqworkloop *
3146kqworkloop_hash_lookup_and_retain(struct filedesc *fdp, kqueue_id_t kq_id)
3147{
3148 struct kqworkloop *kqwl = NULL;
3149
3150 kqhash_lock(fdp);
3151 if (__probable(fdp->fd_kqhash)) {
3152 kqwl = kqworkloop_hash_lookup_locked(fdp, kq_id);
3153 if (kqwl && !kqworkloop_try_retain(kqwl)) {
3154 kqwl = NULL;
3155 }
3156 }
3157 kqhash_unlock(fdp);
3158 return kqwl;
3159}
3160
3161OS_NOINLINE
3162static void
3163kqworkloop_hash_init(struct filedesc *fdp)
3164{
3165 struct kqwllist *alloc_hash;
3166 u_long alloc_mask;
3167
3168 kqhash_unlock(fdp);
3169 alloc_hash = hashinit(CONFIG_KQ_HASHSIZE, M_KQUEUE, &alloc_mask);
3170 kqhash_lock(fdp);
3171
3172 /* See if we won the race */
3173 if (__probable(fdp->fd_kqhashmask == 0)) {
3174 fdp->fd_kqhash = alloc_hash;
3175 fdp->fd_kqhashmask = alloc_mask;
3176 } else {
3177 kqhash_unlock(fdp);
3178 FREE(alloc_hash, M_KQUEUE);
3179 kqhash_lock(fdp);
3180 }
3181}
3182
3183/*!
3184 * @function kqworkloop_dealloc
3185 *
3186 * @brief
3187 * Deallocates a workloop kqueue.
3188 *
3189 * @discussion
3190 * Knotes hold references on the workloop, so we can't really reach this
3191 * function unless all of these are already gone.
3192 *
3193 * Nothing locked on entry or exit.
3194 *
3195 * @param flags
3196 * Unless KQWL_DEALLOC_SKIP_HASH_REMOVE is set, the workloop is removed
3197 * from its hash table.
3198 *
3199 * @param current_ref
3200 * This function is also called to undo a kqworkloop_alloc in case of
3201 * allocation races, expected_ref is the current refcount that is expected
3202 * on the workloop object, usually 0, and 1 when a dealloc race is resolved.
3203 */
3204static void
3205kqworkloop_dealloc(struct kqworkloop *kqwl, kqworkloop_dealloc_flags_t flags,
3206 uint32_t current_ref)
3207{
3208 thread_t cur_owner;
3209
3210 if (__improbable(current_ref > 1)) {
3211 kqworkloop_release_panic(kqwl);
3212 }
3213 assert(kqwl->kqwl_retains == current_ref);
3214
3215 /* pair with kqunlock() and other kq locks */
3216 os_atomic_thread_fence(acquire);
3217
3218 cur_owner = kqwl->kqwl_owner;
3219 if (cur_owner) {
3220 if (kqworkloop_override(kqwl) != THREAD_QOS_UNSPECIFIED) {
3221 thread_drop_kevent_override(cur_owner);
3222 }
3223 thread_deallocate(cur_owner);
3224 kqwl->kqwl_owner = THREAD_NULL;
3225 }
3226
3227 if (kqwl->kqwl_state & KQ_HAS_TURNSTILE) {
3228 struct turnstile *ts;
3229 turnstile_complete((uintptr_t)kqwl, &kqwl->kqwl_turnstile,
3230 &ts, TURNSTILE_WORKLOOPS);
3231 turnstile_cleanup();
3232 turnstile_deallocate(ts);
3233 }
3234
3235 if ((flags & KQWL_DEALLOC_SKIP_HASH_REMOVE) == 0) {
3236 struct filedesc *fdp = kqwl->kqwl_p->p_fd;
3237
3238 kqhash_lock(fdp);
3239 LIST_REMOVE(kqwl, kqwl_hashlink);
3240 kqhash_unlock(fdp);
3241 }
3242
3243 assert(TAILQ_EMPTY(&kqwl->kqwl_suppressed));
3244 assert(kqwl->kqwl_owner == THREAD_NULL);
3245 assert(kqwl->kqwl_turnstile == TURNSTILE_NULL);
3246
3247 lck_spin_destroy(&kqwl->kqwl_statelock, kq_lck_grp);
3248 kqueue_destroy(kqwl, kqworkloop_zone);
3249}
3250
3251/*!
3252 * @function kqworkloop_alloc
3253 *
3254 * @brief
3255 * Allocates a workloop kqueue.
3256 */
3257static void
3258kqworkloop_init(struct kqworkloop *kqwl, proc_t p,
3259 kqueue_id_t id, workq_threadreq_param_t *trp)
3260{
3261 bzero(kqwl, sizeof(struct kqworkloop));
3262
3263 kqwl->kqwl_state = KQ_WORKLOOP | KQ_DYNAMIC | KQ_KEV_QOS;
3264 kqwl->kqwl_retains = 1; /* donate a retain to creator */
3265 kqwl->kqwl_dynamicid = id;
3266 kqwl->kqwl_p = p;
3267 if (trp) {
3268 kqwl->kqwl_params = trp->trp_value;
3269 }
3270
3271 workq_tr_flags_t tr_flags = WORKQ_TR_FLAG_WORKLOOP;
3272 if (trp) {
3273 if (trp->trp_flags & TRP_PRIORITY) {
3274 tr_flags |= WORKQ_TR_FLAG_WL_OUTSIDE_QOS;
3275 }
3276 if (trp->trp_flags) {
3277 tr_flags |= WORKQ_TR_FLAG_WL_PARAMS;
3278 }
3279 }
3280 kqwl->kqwl_request.tr_state = WORKQ_TR_STATE_IDLE;
3281 kqwl->kqwl_request.tr_flags = tr_flags;
3282
3283 for (int i = 0; i < KQWL_NBUCKETS; i++) {
3284 TAILQ_INIT_AFTER_BZERO(&kqwl->kqwl_queue[i]);
3285 }
3286 TAILQ_INIT_AFTER_BZERO(&kqwl->kqwl_suppressed);
3287
3288 lck_spin_init(&kqwl->kqwl_statelock, kq_lck_grp, kq_lck_attr);
3289
3290 kqueue_init(kqwl, &kqwl->kqwl_waitq_hook, SYNC_POLICY_FIFO);
3291}
3292
3293/*!
3294 * @function kqworkloop_get_or_create
3295 *
3296 * @brief
3297 * Wrapper around kqworkloop_alloc that handles the uniquing of workloops.
3298 *
3299 * @returns
3300 * 0: success
3301 * EINVAL: invalid parameters
3302 * EEXIST: KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST is set and a collision exists.
3303 * ENOENT: KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST is set and the entry wasn't found.
3304 * ENOMEM: allocation failed
3305 */
3306static int
3307kqworkloop_get_or_create(struct proc *p, kqueue_id_t id,
3308 workq_threadreq_param_t *trp, unsigned int flags, struct kqworkloop **kqwlp)
3309{
3310 struct filedesc *fdp = p->p_fd;
3311 struct kqworkloop *alloc_kqwl = NULL;
3312 struct kqworkloop *kqwl = NULL;
3313 int error = 0;
3314
3315 assert(!trp || (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST));
3316
3317 if (id == 0 || id == (kqueue_id_t)-1) {
3318 return EINVAL;
3319 }
3320
3321 for (;;) {
3322 kqhash_lock(fdp);
3323 if (__improbable(fdp->fd_kqhash == NULL)) {
3324 kqworkloop_hash_init(fdp);
3325 }
3326
3327 kqwl = kqworkloop_hash_lookup_locked(fdp, id);
3328 if (kqwl) {
3329 if (__improbable(flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST)) {
3330 /*
3331 * If MUST_NOT_EXIST was passed, even if we would have failed
3332 * the try_retain, it could have gone the other way, and
3333 * userspace can't tell. Let'em fix their race.
3334 */
3335 error = EEXIST;
3336 break;
3337 }
3338
3339 if (__probable(kqworkloop_try_retain(kqwl))) {
3340 /*
3341 * This is a valid live workloop !
3342 */
3343 *kqwlp = kqwl;
3344 error = 0;
3345 break;
3346 }
3347 }
3348
3349 if (__improbable(flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST)) {
3350 error = ENOENT;
3351 break;
3352 }
3353
3354 /*
3355 * We didn't find what we were looking for.
3356 *
3357 * If this is the second time we reach this point (alloc_kqwl != NULL),
3358 * then we're done.
3359 *
3360 * If this is the first time we reach this point (alloc_kqwl == NULL),
3361 * then try to allocate one without blocking.
3362 */
3363 if (__probable(alloc_kqwl == NULL)) {
3364 alloc_kqwl = (struct kqworkloop *)zalloc_noblock(kqworkloop_zone);
3365 }
3366 if (__probable(alloc_kqwl)) {
3367 kqworkloop_init(alloc_kqwl, p, id, trp);
3368 kqworkloop_hash_insert_locked(fdp, id, alloc_kqwl);
3369 kqhash_unlock(fdp);
3370 *kqwlp = alloc_kqwl;
3371 return 0;
3372 }
3373
3374 /*
3375 * We have to block to allocate a workloop, drop the lock,
3376 * allocate one, but then we need to retry lookups as someone
3377 * else could race with us.
3378 */
3379 kqhash_unlock(fdp);
3380
3381 alloc_kqwl = (struct kqworkloop *)zalloc(kqworkloop_zone);
3382 if (__improbable(!alloc_kqwl)) {
3383 return ENOMEM;
3384 }
3385 }
3386
3387 kqhash_unlock(fdp);
3388
3389 if (__improbable(alloc_kqwl)) {
3390 zfree(kqworkloop_zone, alloc_kqwl);
3391 }
3392
3393 return error;
3394}
3395
3396#pragma mark - knotes
3397
3398static int
3399filt_no_attach(struct knote *kn, __unused struct kevent_qos_s *kev)
3400{
3401 knote_set_error(kn, ENOTSUP);
3402 return 0;
3403}
3404
3405static void
3406filt_no_detach(__unused struct knote *kn)
3407{
3408}
3409
3410static int __dead2
3411filt_bad_event(struct knote *kn, long hint)
3412{
3413 panic("%s[%d](%p, %ld)", __func__, kn->kn_filter, kn, hint);
3414}
3415
3416static int __dead2
3417filt_bad_touch(struct knote *kn, struct kevent_qos_s *kev)
3418{
3419 panic("%s[%d](%p, %p)", __func__, kn->kn_filter, kn, kev);
3420}
3421
3422static int __dead2
3423filt_bad_process(struct knote *kn, struct kevent_qos_s *kev)
3424{
3425 panic("%s[%d](%p, %p)", __func__, kn->kn_filter, kn, kev);
3426}
3427
3428/*
3429 * knotes_dealloc - detach all knotes for the process and drop them
3430 *
3431 * Called with proc_fdlock held.
3432 * Returns with it locked.
3433 * May drop it temporarily.
3434 * Process is in such a state that it will not try to allocate
3435 * any more knotes during this process (stopped for exit or exec).
3436 */
3437void
3438knotes_dealloc(proc_t p)
3439{
3440 struct filedesc *fdp = p->p_fd;
3441 struct kqueue *kq;
3442 struct knote *kn;
3443 struct klist *kn_hash = NULL;
3444 int i;
3445
3446 /* Close all the fd-indexed knotes up front */
3447 if (fdp->fd_knlistsize > 0) {
3448 for (i = 0; i < fdp->fd_knlistsize; i++) {
3449 while ((kn = SLIST_FIRST(&fdp->fd_knlist[i])) != NULL) {
3450 kq = knote_get_kq(kn);
3451 kqlock(kq);
3452 proc_fdunlock(p);
3453 knote_drop(kq, kn, NULL);
3454 proc_fdlock(p);
3455 }
3456 }
3457 /* free the table */
3458 FREE(fdp->fd_knlist, M_KQUEUE);
3459 fdp->fd_knlist = NULL;
3460 }
3461 fdp->fd_knlistsize = 0;
3462
3463 knhash_lock(fdp);
3464 proc_fdunlock(p);
3465
3466 /* Clean out all the hashed knotes as well */
3467 if (fdp->fd_knhashmask != 0) {
3468 for (i = 0; i <= (int)fdp->fd_knhashmask; i++) {
3469 while ((kn = SLIST_FIRST(&fdp->fd_knhash[i])) != NULL) {
3470 kq = knote_get_kq(kn);
3471 kqlock(kq);
3472 knhash_unlock(fdp);
3473 knote_drop(kq, kn, NULL);
3474 knhash_lock(fdp);
3475 }
3476 }
3477 kn_hash = fdp->fd_knhash;
3478 fdp->fd_knhashmask = 0;
3479 fdp->fd_knhash = NULL;
3480 }
3481
3482 knhash_unlock(fdp);
3483
3484 /* free the kn_hash table */
3485 if (kn_hash) {
3486 FREE(kn_hash, M_KQUEUE);
3487 }
3488
3489 proc_fdlock(p);
3490}
3491
3492/*
3493 * kqworkloops_dealloc - rebalance retains on kqworkloops created with
3494 * scheduling parameters
3495 *
3496 * Called with proc_fdlock held.
3497 * Returns with it locked.
3498 * Process is in such a state that it will not try to allocate
3499 * any more knotes during this process (stopped for exit or exec).
3500 */
3501void
3502kqworkloops_dealloc(proc_t p)
3503{
3504 struct filedesc *fdp = p->p_fd;
3505 struct kqworkloop *kqwl, *kqwln;
3506 struct kqwllist tofree;
3507
3508 if (!(fdp->fd_flags & FD_WORKLOOP)) {
3509 return;
3510 }
3511
3512 kqhash_lock(fdp);
3513
3514 if (fdp->fd_kqhashmask == 0) {
3515 kqhash_unlock(fdp);
3516 return;
3517 }
3518
3519 LIST_INIT(&tofree);
3520
3521 for (size_t i = 0; i <= fdp->fd_kqhashmask; i++) {
3522 LIST_FOREACH_SAFE(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink, kqwln) {
3523 /*
3524 * kqworkloops that have scheduling parameters have an
3525 * implicit retain from kqueue_workloop_ctl that needs
3526 * to be balanced on process exit.
3527 */
3528 assert(kqwl->kqwl_params);
3529 LIST_REMOVE(kqwl, kqwl_hashlink);
3530 LIST_INSERT_HEAD(&tofree, kqwl, kqwl_hashlink);
3531 }
3532 }
3533
3534 kqhash_unlock(fdp);
3535
3536 LIST_FOREACH_SAFE(kqwl, &tofree, kqwl_hashlink, kqwln) {
3537 kqworkloop_dealloc(kqwl, KQWL_DEALLOC_SKIP_HASH_REMOVE, 1);
3538 }
3539}
3540
3541static int
3542kevent_register_validate_priority(struct kqueue *kq, struct knote *kn,
3543 struct kevent_qos_s *kev)
3544{
3545 /* We don't care about the priority of a disabled or deleted knote */
3546 if (kev->flags & (EV_DISABLE | EV_DELETE)) {
3547 return 0;
3548 }
3549
3550 if (kq->kq_state & KQ_WORKLOOP) {
3551 /*
3552 * Workloops need valid priorities with a QOS (excluding manager) for
3553 * any enabled knote.
3554 *
3555 * When it is pre-existing, just make sure it has a valid QoS as
3556 * kevent_register() will not use the incoming priority (filters who do
3557 * have the responsibility to validate it again, see filt_wltouch).
3558 *
3559 * If the knote is being made, validate the incoming priority.
3560 */
3561 if (!_pthread_priority_thread_qos(kn ? kn->kn_qos : kev->qos)) {
3562 return ERANGE;
3563 }
3564 }
3565
3566 return 0;
3567}
3568
3569/*
3570 * Prepare a filter for waiting after register.
3571 *
3572 * The f_post_register_wait hook will be called later by kevent_register()
3573 * and should call kevent_register_wait_block()
3574 */
3575static int
3576kevent_register_wait_prepare(struct knote *kn, struct kevent_qos_s *kev, int rc)
3577{
3578 thread_t thread = current_thread();
3579
3580 assert(knote_fops(kn)->f_extended_codes);
3581
3582 if (kn->kn_thread == NULL) {
3583 thread_reference(thread);
3584 kn->kn_thread = thread;
3585 } else if (kn->kn_thread != thread) {
3586 /*
3587 * kn_thread may be set from a previous aborted wait
3588 * However, it has to be from the same thread.
3589 */
3590 kev->flags |= EV_ERROR;
3591 kev->data = EXDEV;
3592 return 0;
3593 }
3594
3595 return FILTER_REGISTER_WAIT | rc;
3596}
3597
3598/*
3599 * Cleanup a kevent_register_wait_prepare() effect for threads that have been
3600 * aborted instead of properly woken up with thread_wakeup_thread().
3601 */
3602static void
3603kevent_register_wait_cleanup(struct knote *kn)
3604{
3605 thread_t thread = kn->kn_thread;
3606 kn->kn_thread = NULL;
3607 thread_deallocate(thread);
3608}
3609
3610/*
3611 * Must be called at the end of a f_post_register_wait call from a filter.
3612 */
3613static void
3614kevent_register_wait_block(struct turnstile *ts, thread_t thread,
3615 thread_continue_t cont, struct _kevent_register *cont_args)
3616{
3617 turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
3618 kqunlock(cont_args->kqwl);
3619 cont_args->handoff_thread = thread;
3620 thread_handoff_parameter(thread, cont, cont_args);
3621}
3622
3623/*
3624 * Called by Filters using a f_post_register_wait to return from their wait.
3625 */
3626static void
3627kevent_register_wait_return(struct _kevent_register *cont_args)
3628{
3629 struct kqworkloop *kqwl = cont_args->kqwl;
3630 struct kevent_qos_s *kev = &cont_args->kev;
3631 int error = 0;
3632
3633 if (cont_args->handoff_thread) {
3634 thread_deallocate(cont_args->handoff_thread);
3635 }
3636
3637 if (kev->flags & (EV_ERROR | EV_RECEIPT)) {
3638 if ((kev->flags & EV_ERROR) == 0) {
3639 kev->flags |= EV_ERROR;
3640 kev->data = 0;
3641 }
3642 error = kevent_modern_copyout(kev, &cont_args->ueventlist);
3643 if (error == 0) {
3644 cont_args->eventout++;
3645 }
3646 }
3647
3648 kqworkloop_release(kqwl);
3649 if (error == 0) {
3650 *(int32_t *)&current_uthread()->uu_rval = cont_args->eventout;
3651 }
3652 unix_syscall_return(error);
3653}
3654
3655/*
3656 * kevent_register - add a new event to a kqueue
3657 *
3658 * Creates a mapping between the event source and
3659 * the kqueue via a knote data structure.
3660 *
3661 * Because many/most the event sources are file
3662 * descriptor related, the knote is linked off
3663 * the filedescriptor table for quick access.
3664 *
3665 * called with nothing locked
3666 * caller holds a reference on the kqueue
3667 */
3668
3669int
3670kevent_register(struct kqueue *kq, struct kevent_qos_s *kev,
3671 struct knote **kn_out)
3672{
3673 struct proc *p = kq->kq_p;
3674 const struct filterops *fops;
3675 struct knote *kn = NULL;
3676 int result = 0, error = 0;
3677 unsigned short kev_flags = kev->flags;
3678 KNOTE_LOCK_CTX(knlc);
3679
3680 if (__probable(kev->filter < 0 && kev->filter + EVFILT_SYSCOUNT >= 0)) {
3681 fops = sysfilt_ops[~kev->filter]; /* to 0-base index */
3682 } else {
3683 error = EINVAL;
3684 goto out;
3685 }
3686
3687 /* restrict EV_VANISHED to adding udata-specific dispatch kevents */
3688 if (__improbable((kev->flags & EV_VANISHED) &&
3689 (kev->flags & (EV_ADD | EV_DISPATCH2)) != (EV_ADD | EV_DISPATCH2))) {
3690 error = EINVAL;
3691 goto out;
3692 }
3693
3694 /* Simplify the flags - delete and disable overrule */
3695 if (kev->flags & EV_DELETE) {
3696 kev->flags &= ~EV_ADD;
3697 }
3698 if (kev->flags & EV_DISABLE) {
3699 kev->flags &= ~EV_ENABLE;
3700 }
3701
3702 if (kq->kq_state & KQ_WORKLOOP) {
3703 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_REGISTER),
3704 ((struct kqworkloop *)kq)->kqwl_dynamicid,
3705 kev->udata, kev->flags, kev->filter);
3706 } else if (kq->kq_state & KQ_WORKQ) {
3707 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_REGISTER),
3708 0, kev->udata, kev->flags, kev->filter);
3709 } else {
3710 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_REGISTER),
3711 VM_KERNEL_UNSLIDE_OR_PERM(kq),
3712 kev->udata, kev->flags, kev->filter);
3713 }
3714
3715restart:
3716 /* find the matching knote from the fd tables/hashes */
3717 kn = kq_find_knote_and_kq_lock(kq, kev, fops->f_isfd, p);
3718 error = kevent_register_validate_priority(kq, kn, kev);
3719 result = 0;
3720 if (error) {
3721 goto out;
3722 }
3723
3724 if (kn == NULL && (kev->flags & EV_ADD) == 0) {
3725 /*
3726 * No knote found, EV_ADD wasn't specified
3727 */
3728
3729 if ((kev_flags & EV_ADD) && (kev_flags & EV_DELETE) &&
3730 (kq->kq_state & KQ_WORKLOOP)) {
3731 /*
3732 * For workloops, understand EV_ADD|EV_DELETE as a "soft" delete
3733 * that doesn't care about ENOENT, so just pretend the deletion
3734 * happened.
3735 */
3736 } else {
3737 error = ENOENT;
3738 }
3739 goto out;
3740 } else if (kn == NULL) {
3741 /*
3742 * No knote found, need to attach a new one (attach)
3743 */
3744
3745 struct fileproc *knote_fp = NULL;
3746
3747 /* grab a file reference for the new knote */
3748 if (fops->f_isfd) {
3749 if ((error = fp_lookup(p, kev->ident, &knote_fp, 0)) != 0) {
3750 goto out;
3751 }
3752 }
3753
3754 kn = knote_alloc();
3755 if (kn == NULL) {
3756 error = ENOMEM;
3757 if (knote_fp != NULL) {
3758 fp_drop(p, kev->ident, knote_fp, 0);
3759 }
3760 goto out;
3761 }
3762
3763 kn->kn_fp = knote_fp;
3764 kn->kn_is_fd = fops->f_isfd;
3765 kn->kn_kq_packed = (intptr_t)(struct kqueue *)kq;
3766 kn->kn_status = 0;
3767
3768 /* was vanish support requested */
3769 if (kev->flags & EV_VANISHED) {
3770 kev->flags &= ~EV_VANISHED;
3771 kn->kn_status |= KN_REQVANISH;
3772 }
3773
3774 /* snapshot matching/dispatching protcol flags into knote */
3775 if (kev->flags & EV_DISABLE) {
3776 kn->kn_status |= KN_DISABLED;
3777 }
3778
3779 /*
3780 * copy the kevent state into knote
3781 * protocol is that fflags and data
3782 * are saved off, and cleared before
3783 * calling the attach routine.
3784 *
3785 * - kn->kn_sfflags aliases with kev->xflags
3786 * - kn->kn_sdata aliases with kev->data
3787 * - kn->kn_filter is the top 8 bits of kev->filter
3788 */
3789 kn->kn_kevent = *(struct kevent_internal_s *)kev;
3790 kn->kn_sfflags = kev->fflags;
3791 kn->kn_filtid = (uint8_t)~kev->filter;
3792 kn->kn_fflags = 0;
3793 knote_reset_priority(kq, kn, kev->qos);
3794
3795 /* Add the knote for lookup thru the fd table */
3796 error = kq_add_knote(kq, kn, &knlc, p);
3797 if (error) {
3798 knote_free(kn);
3799 if (knote_fp != NULL) {
3800 fp_drop(p, kev->ident, knote_fp, 0);
3801 }
3802
3803 if (error == ERESTART) {
3804 goto restart;
3805 }
3806 goto out;
3807 }
3808
3809 /* fp reference count now applies to knote */
3810
3811 /*
3812 * we can't use filter_call() because f_attach can change the filter ops
3813 * for a filter that supports f_extended_codes, so we need to reload
3814 * knote_fops() and not use `fops`.
3815 */
3816 result = fops->f_attach(kn, kev);
3817 if (result && !knote_fops(kn)->f_extended_codes) {
3818 result = FILTER_ACTIVE;
3819 }
3820
3821 kqlock(kq);
3822
3823 if (result & FILTER_THREADREQ_NODEFEER) {
3824 enable_preemption();
3825 }
3826
3827 if (kn->kn_flags & EV_ERROR) {
3828 /*
3829 * Failed to attach correctly, so drop.
3830 */
3831 kn->kn_filtid = EVFILTID_DETACHED;
3832 error = kn->kn_sdata;
3833 knote_drop(kq, kn, &knlc);
3834 result = 0;
3835 goto out;
3836 }
3837
3838 /*
3839 * end "attaching" phase - now just attached
3840 *
3841 * Mark the thread request overcommit, if appropos
3842 *
3843 * If the attach routine indicated that an
3844 * event is already fired, activate the knote.
3845 */
3846 if ((kn->kn_qos & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) &&
3847 (kq->kq_state & KQ_WORKLOOP)) {
3848 kqworkloop_set_overcommit((struct kqworkloop *)kq);
3849 }
3850 } else if (!knote_lock(kq, kn, &knlc, KNOTE_KQ_LOCK_ON_SUCCESS)) {
3851 /*
3852 * The knote was dropped while we were waiting for the lock,
3853 * we need to re-evaluate entirely
3854 */
3855
3856 goto restart;
3857 } else if (kev->flags & EV_DELETE) {
3858 /*
3859 * Deletion of a knote (drop)
3860 *
3861 * If the filter wants to filter drop events, let it do so.
3862 *
3863 * defer-delete: when trying to delete a disabled EV_DISPATCH2 knote,
3864 * we must wait for the knote to be re-enabled (unless it is being
3865 * re-enabled atomically here).
3866 */
3867
3868 if (knote_fops(kn)->f_allow_drop) {
3869 bool drop;
3870
3871 kqunlock(kq);
3872 drop = knote_fops(kn)->f_allow_drop(kn, kev);
3873 kqlock(kq);
3874
3875 if (!drop) {
3876 goto out_unlock;
3877 }
3878 }
3879
3880 if ((kev->flags & EV_ENABLE) == 0 &&
3881 (kn->kn_flags & EV_DISPATCH2) == EV_DISPATCH2 &&
3882 (kn->kn_status & KN_DISABLED) != 0) {
3883 kn->kn_status |= KN_DEFERDELETE;
3884 error = EINPROGRESS;
3885 goto out_unlock;
3886 }
3887
3888 knote_drop(kq, kn, &knlc);
3889 goto out;
3890 } else {
3891 /*
3892 * Regular update of a knote (touch)
3893 *
3894 * Call touch routine to notify filter of changes in filter values
3895 * (and to re-determine if any events are fired).
3896 *
3897 * If the knote is in defer-delete, avoid calling the filter touch
3898 * routine (it has delivered its last event already).
3899 *
3900 * If the touch routine had no failure,
3901 * apply the requested side effects to the knote.
3902 */
3903
3904 if (kn->kn_status & (KN_DEFERDELETE | KN_VANISHED)) {
3905 if (kev->flags & EV_ENABLE) {
3906 result = FILTER_ACTIVE;
3907 }
3908 } else {
3909 kqunlock(kq);
3910 result = filter_call(knote_fops(kn), f_touch(kn, kev));
3911 kqlock(kq);
3912 if (result & FILTER_THREADREQ_NODEFEER) {
3913 enable_preemption();
3914 }
3915 }
3916
3917 if (kev->flags & EV_ERROR) {
3918 result = 0;
3919 goto out_unlock;
3920 }
3921
3922 if ((kn->kn_flags & EV_UDATA_SPECIFIC) == 0 &&
3923 kn->kn_udata != kev->udata) {
3924 // this allows klist_copy_udata() not to take locks
3925 os_atomic_store_wide(&kn->kn_udata, kev->udata, relaxed);
3926 }
3927 if ((kev->flags & EV_DISABLE) && !(kn->kn_status & KN_DISABLED)) {
3928 kn->kn_status |= KN_DISABLED;
3929 knote_dequeue(kq, kn);
3930 }
3931 }
3932
3933 /* accept new kevent state */
3934 knote_apply_touch(kq, kn, kev, result);
3935
3936out_unlock:
3937 /*
3938 * When the filter asked for a post-register wait,
3939 * we leave the kqueue locked for kevent_register()
3940 * to call the filter's f_post_register_wait hook.
3941 */
3942 if (result & FILTER_REGISTER_WAIT) {
3943 knote_unlock(kq, kn, &knlc, KNOTE_KQ_LOCK_ALWAYS);
3944 *kn_out = kn;
3945 } else {
3946 knote_unlock(kq, kn, &knlc, KNOTE_KQ_UNLOCK);
3947 }
3948
3949out:
3950 /* output local errors through the kevent */
3951 if (error) {
3952 kev->flags |= EV_ERROR;
3953 kev->data = error;
3954 }
3955 return result;
3956}
3957
3958/*
3959 * knote_process - process a triggered event
3960 *
3961 * Validate that it is really still a triggered event
3962 * by calling the filter routines (if necessary). Hold
3963 * a use reference on the knote to avoid it being detached.
3964 *
3965 * If it is still considered triggered, we will have taken
3966 * a copy of the state under the filter lock. We use that
3967 * snapshot to dispatch the knote for future processing (or
3968 * not, if this was a lost event).
3969 *
3970 * Our caller assures us that nobody else can be processing
3971 * events from this knote during the whole operation. But
3972 * others can be touching or posting events to the knote
3973 * interspersed with our processing it.
3974 *
3975 * caller holds a reference on the kqueue.
3976 * kqueue locked on entry and exit - but may be dropped
3977 */
3978static int
3979knote_process(struct knote *kn, kevent_ctx_t kectx,
3980 kevent_callback_t callback)
3981{
3982 struct kevent_qos_s kev;
3983 struct kqueue *kq = knote_get_kq(kn);
3984 KNOTE_LOCK_CTX(knlc);
3985 int result = FILTER_ACTIVE;
3986 int error = 0;
3987 bool drop = false;
3988
3989 /*
3990 * Must be active or stayactive
3991 * Must be queued and not disabled/suppressed or dropping
3992 */
3993 assert(kn->kn_status & KN_QUEUED);
3994 assert(kn->kn_status & (KN_ACTIVE | KN_STAYACTIVE));
3995 assert(!(kn->kn_status & (KN_DISABLED | KN_SUPPRESSED | KN_DROPPING)));
3996
3997 if (kq->kq_state & KQ_WORKLOOP) {
3998 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS),
3999 ((struct kqworkloop *)kq)->kqwl_dynamicid,
4000 kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
4001 kn->kn_filtid);
4002 } else if (kq->kq_state & KQ_WORKQ) {
4003 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS),
4004 0, kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
4005 kn->kn_filtid);
4006 } else {
4007 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS),
4008 VM_KERNEL_UNSLIDE_OR_PERM(kq), kn->kn_udata,
4009 kn->kn_status | (kn->kn_id << 32), kn->kn_filtid);
4010 }
4011
4012 if (!knote_lock(kq, kn, &knlc, KNOTE_KQ_LOCK_ALWAYS)) {
4013 /*
4014 * When the knote is dropping or has dropped,
4015 * then there's nothing we want to process.
4016 */
4017 return EJUSTRETURN;
4018 }
4019
4020 /*
4021 * While waiting for the knote lock, we may have dropped the kq lock.
4022 * and a touch may have disabled and dequeued the knote.
4023 */
4024 if (!(kn->kn_status & KN_QUEUED)) {
4025 knote_unlock(kq, kn, &knlc, KNOTE_KQ_LOCK_ALWAYS);
4026 return EJUSTRETURN;
4027 }
4028
4029 /*
4030 * For deferred-drop or vanished events, we just create a fake
4031 * event to acknowledge end-of-life. Otherwise, we call the
4032 * filter's process routine to snapshot the kevent state under
4033 * the filter's locking protocol.
4034 *
4035 * suppress knotes to avoid returning the same event multiple times in
4036 * a single call.
4037 */
4038 knote_suppress(kq, kn);
4039
4040 if (kn->kn_status & (KN_DEFERDELETE | KN_VANISHED)) {
4041 int kev_flags = EV_DISPATCH2 | EV_ONESHOT;
4042 if (kn->kn_status & KN_DEFERDELETE) {
4043 kev_flags |= EV_DELETE;
4044 } else {
4045 kev_flags |= EV_VANISHED;
4046 }
4047
4048 /* create fake event */
4049 kev = (struct kevent_qos_s){
4050 .filter = kn->kn_filter,
4051 .ident = kn->kn_id,
4052 .flags = kev_flags,
4053 .udata = kn->kn_udata,
4054 };
4055 } else {
4056 kqunlock(kq);
4057 kev = (struct kevent_qos_s) { };
4058 result = filter_call(knote_fops(kn), f_process(kn, &kev));
4059 kqlock(kq);
4060 }
4061
4062 /*
4063 * Determine how to dispatch the knote for future event handling.
4064 * not-fired: just return (do not callout, leave deactivated).
4065 * One-shot: If dispatch2, enter deferred-delete mode (unless this is
4066 * is the deferred delete event delivery itself). Otherwise,
4067 * drop it.
4068 * Dispatch: don't clear state, just mark it disabled.
4069 * Cleared: just leave it deactivated.
4070 * Others: re-activate as there may be more events to handle.
4071 * This will not wake up more handlers right now, but
4072 * at the completion of handling events it may trigger
4073 * more handler threads (TODO: optimize based on more than
4074 * just this one event being detected by the filter).
4075 */
4076 if ((result & FILTER_ACTIVE) == 0) {
4077 if ((kn->kn_status & (KN_ACTIVE | KN_STAYACTIVE)) == 0) {
4078 /*
4079 * Stay active knotes should not be unsuppressed or we'd create an
4080 * infinite loop.
4081 *
4082 * Some knotes (like EVFILT_WORKLOOP) can be reactivated from
4083 * within f_process() but that doesn't necessarily make them
4084 * ready to process, so we should leave them be.
4085 *
4086 * For other knotes, since we will not return an event,
4087 * there's no point keeping the knote suppressed.
4088 */
4089 knote_unsuppress(kq, kn);
4090 }
4091 knote_unlock(kq, kn, &knlc, KNOTE_KQ_LOCK_ALWAYS);
4092 return EJUSTRETURN;
4093 }
4094
4095 if (result & FILTER_ADJUST_EVENT_QOS_BIT) {
4096 knote_adjust_qos(kq, kn, result);
4097 }
4098 kev.qos = _pthread_priority_combine(kn->kn_qos, kn->kn_qos_override);
4099
4100 if (kev.flags & EV_ONESHOT) {
4101 if ((kn->kn_flags & EV_DISPATCH2) == EV_DISPATCH2 &&
4102 (kn->kn_status & KN_DEFERDELETE) == 0) {
4103 /* defer dropping non-delete oneshot dispatch2 events */
4104 kn->kn_status |= KN_DEFERDELETE | KN_DISABLED;
4105 } else {
4106 drop = true;
4107 }
4108 } else if (kn->kn_flags & EV_DISPATCH) {
4109 /* disable all dispatch knotes */
4110 kn->kn_status |= KN_DISABLED;
4111 } else if ((kn->kn_flags & EV_CLEAR) == 0) {
4112 /* re-activate in case there are more events */
4113 knote_activate(kq, kn, FILTER_ACTIVE);
4114 }
4115
4116 /*
4117 * callback to handle each event as we find it.
4118 * If we have to detach and drop the knote, do
4119 * it while we have the kq unlocked.
4120 */
4121 if (drop) {
4122 knote_drop(kq, kn, &knlc);
4123 } else {
4124 knote_unlock(kq, kn, &knlc, KNOTE_KQ_UNLOCK);
4125 }
4126
4127 if (kev.flags & EV_VANISHED) {
4128 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KNOTE_VANISHED),
4129 kev.ident, kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
4130 kn->kn_filtid);
4131 }
4132
4133 error = (callback)(&kev, kectx);
4134 kqlock(kq);
4135 return error;
4136}
4137
4138/*
4139 * Returns -1 if the kqueue was unbound and processing should not happen
4140 */
4141#define KQWQAE_BEGIN_PROCESSING 1
4142#define KQWQAE_END_PROCESSING 2
4143#define KQWQAE_UNBIND 3
4144static int
4145kqworkq_acknowledge_events(struct kqworkq *kqwq, workq_threadreq_t kqr,
4146 int kevent_flags, int kqwqae_op)
4147{
4148 thread_qos_t old_override = THREAD_QOS_UNSPECIFIED;
4149 thread_t thread = kqr_thread_fast(kqr);
4150 struct knote *kn;
4151 int rc = 0;
4152 bool unbind;
4153 struct kqtailq *suppressq = &kqwq->kqwq_suppressed[kqr->tr_kq_qos_index];
4154
4155 kqlock_held(&kqwq->kqwq_kqueue);
4156
4157 if (!TAILQ_EMPTY(suppressq)) {
4158 /*
4159 * Return suppressed knotes to their original state.
4160 * For workq kqueues, suppressed ones that are still
4161 * truly active (not just forced into the queue) will
4162 * set flags we check below to see if anything got
4163 * woken up.
4164 */
4165 while ((kn = TAILQ_FIRST(suppressq)) != NULL) {
4166 assert(kn->kn_status & KN_SUPPRESSED);
4167 knote_unsuppress(kqwq, kn);
4168 }
4169 }
4170
4171#if DEBUG || DEVELOPMENT
4172 thread_t self = current_thread();
4173 struct uthread *ut = get_bsdthread_info(self);
4174
4175 assert(thread == self);
4176 assert(ut->uu_kqr_bound == kqr);
4177#endif // DEBUG || DEVELOPMENT
4178
4179 if (kqwqae_op == KQWQAE_UNBIND) {
4180 unbind = true;
4181 } else if ((kevent_flags & KEVENT_FLAG_PARKING) == 0) {
4182 unbind = false;
4183 } else {
4184 unbind = !kqr->tr_kq_wakeup;
4185 }
4186 if (unbind) {
4187 old_override = kqworkq_unbind_locked(kqwq, kqr, thread);
4188 rc = -1;
4189 /*
4190 * request a new thread if we didn't process the whole queue or real events
4191 * have happened (not just putting stay-active events back).
4192 */
4193 if (kqr->tr_kq_wakeup) {
4194 kqueue_threadreq_initiate(&kqwq->kqwq_kqueue, kqr,
4195 kqr->tr_kq_qos_index, 0);
4196 }
4197 }
4198
4199 if (rc == 0) {
4200 /*
4201 * Reset wakeup bit to notice events firing while we are processing,
4202 * as we cannot rely on the bucket queue emptiness because of stay
4203 * active knotes.
4204 */
4205 kqr->tr_kq_wakeup = false;
4206 }
4207
4208 if (old_override) {
4209 thread_drop_kevent_override(thread);
4210 }
4211
4212 return rc;
4213}
4214
4215/*
4216 * Return 0 to indicate that processing should proceed,
4217 * -1 if there is nothing to process.
4218 *
4219 * Called with kqueue locked and returns the same way,
4220 * but may drop lock temporarily.
4221 */
4222static int
4223kqworkq_begin_processing(struct kqworkq *kqwq, workq_threadreq_t kqr,
4224 int kevent_flags)
4225{
4226 int rc = 0;
4227
4228 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_START,
4229 0, kqr->tr_kq_qos_index);
4230
4231 rc = kqworkq_acknowledge_events(kqwq, kqr, kevent_flags,
4232 KQWQAE_BEGIN_PROCESSING);
4233
4234 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_END,
4235 thread_tid(kqr_thread(kqr)), kqr->tr_kq_wakeup);
4236
4237 return rc;
4238}
4239
4240static thread_qos_t
4241kqworkloop_acknowledge_events(struct kqworkloop *kqwl)
4242{
4243 kq_index_t qos = THREAD_QOS_UNSPECIFIED;
4244 struct knote *kn, *tmp;
4245
4246 kqlock_held(kqwl);
4247
4248 TAILQ_FOREACH_SAFE(kn, &kqwl->kqwl_suppressed, kn_tqe, tmp) {
4249 /*
4250 * If a knote that can adjust QoS is disabled because of the automatic
4251 * behavior of EV_DISPATCH, the knotes should stay suppressed so that
4252 * further overrides keep pushing.
4253 */
4254 if (knote_fops(kn)->f_adjusts_qos && (kn->kn_status & KN_DISABLED) &&
4255 (kn->kn_status & (KN_STAYACTIVE | KN_DROPPING)) == 0 &&
4256 (kn->kn_flags & (EV_DISPATCH | EV_DISABLE)) == EV_DISPATCH) {
4257 qos = MAX(qos, kn->kn_qos_override);
4258 continue;
4259 }
4260 knote_unsuppress(kqwl, kn);
4261 }
4262
4263 return qos;
4264}
4265
4266static int
4267kqworkloop_begin_processing(struct kqworkloop *kqwl, unsigned int kevent_flags)
4268{
4269 workq_threadreq_t kqr = &kqwl->kqwl_request;
4270 struct kqueue *kq = &kqwl->kqwl_kqueue;
4271 thread_qos_t qos_override;
4272 thread_t thread = kqr_thread_fast(kqr);
4273 int rc = 0, op = KQWL_UTQ_NONE;
4274
4275 kqlock_held(kq);
4276
4277 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_START,
4278 kqwl->kqwl_dynamicid, 0, 0);
4279
4280 /* nobody else should still be processing */
4281 assert((kq->kq_state & KQ_PROCESSING) == 0);
4282
4283 kq->kq_state |= KQ_PROCESSING;
4284
4285 if (!TAILQ_EMPTY(&kqwl->kqwl_suppressed)) {
4286 op = KQWL_UTQ_RESET_WAKEUP_OVERRIDE;
4287 }
4288
4289 if (kevent_flags & KEVENT_FLAG_PARKING) {
4290 /*
4291 * When "parking" we want to process events and if no events are found
4292 * unbind.
4293 *
4294 * However, non overcommit threads sometimes park even when they have
4295 * more work so that the pool can narrow. For these, we need to unbind
4296 * early, so that calling kqworkloop_update_threads_qos() can ask the
4297 * workqueue subsystem whether the thread should park despite having
4298 * pending events.
4299 */
4300 if (kqr->tr_flags & WORKQ_TR_FLAG_OVERCOMMIT) {
4301 op = KQWL_UTQ_PARKING;
4302 } else {
4303 op = KQWL_UTQ_UNBINDING;
4304 }
4305 }
4306 if (op == KQWL_UTQ_NONE) {
4307 goto done;
4308 }
4309
4310 qos_override = kqworkloop_acknowledge_events(kqwl);
4311
4312 if (op == KQWL_UTQ_UNBINDING) {
4313 kqworkloop_unbind_locked(kqwl, thread, KQWL_OVERRIDE_DROP_IMMEDIATELY);
4314 kqworkloop_release_live(kqwl);
4315 }
4316 kqworkloop_update_threads_qos(kqwl, op, qos_override);
4317 if (op == KQWL_UTQ_PARKING) {
4318 if (!TAILQ_EMPTY(&kqwl->kqwl_queue[KQWL_BUCKET_STAYACTIVE])) {
4319 /*
4320 * We cannot trust tr_kq_wakeup when looking at stay active knotes.
4321 * We need to process once, and kqworkloop_end_processing will
4322 * handle the unbind.
4323 */
4324 } else if (!kqr->tr_kq_wakeup || kqwl->kqwl_owner) {
4325 kqworkloop_unbind_locked(kqwl, thread, KQWL_OVERRIDE_DROP_DELAYED);
4326 kqworkloop_release_live(kqwl);
4327 rc = -1;
4328 }
4329 } else if (op == KQWL_UTQ_UNBINDING) {
4330 if (kqr_thread(kqr) == thread) {
4331 /*
4332 * The thread request fired again, passed the admission check and
4333 * got bound to the current thread again.
4334 */
4335 } else {
4336 rc = -1;
4337 }
4338 }
4339
4340 if (rc == 0) {
4341 /*
4342 * Reset wakeup bit to notice stay active events firing while we are
4343 * processing, as we cannot rely on the stayactive bucket emptiness.
4344 */
4345 kqwl->kqwl_wakeup_indexes &= ~KQWL_STAYACTIVE_FIRED_BIT;
4346 } else {
4347 kq->kq_state &= ~KQ_PROCESSING;
4348 }
4349
4350 if (rc == -1) {
4351 kqworkloop_unbind_delayed_override_drop(thread);
4352 }
4353
4354done:
4355 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_END,
4356 kqwl->kqwl_dynamicid, 0, 0);
4357
4358 return rc;
4359}
4360
4361/*
4362 * Return 0 to indicate that processing should proceed,
4363 * -1 if there is nothing to process.
4364 * EBADF if the kqueue is draining
4365 *
4366 * Called with kqueue locked and returns the same way,
4367 * but may drop lock temporarily.
4368 * May block.
4369 */
4370static int
4371kqfile_begin_processing(struct kqfile *kq)
4372{
4373 struct kqtailq *suppressq;
4374
4375 kqlock_held(kq);
4376
4377 assert((kq->kqf_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0);
4378 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_START,
4379 VM_KERNEL_UNSLIDE_OR_PERM(kq), 0);
4380
4381 /* wait to become the exclusive processing thread */
4382 for (;;) {
4383 if (kq->kqf_state & KQ_DRAIN) {
4384 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END,
4385 VM_KERNEL_UNSLIDE_OR_PERM(kq), 2);
4386 return EBADF;
4387 }
4388
4389 if ((kq->kqf_state & KQ_PROCESSING) == 0) {
4390 break;
4391 }
4392
4393 /* if someone else is processing the queue, wait */
4394 kq->kqf_state |= KQ_PROCWAIT;
4395 suppressq = &kq->kqf_suppressed;
4396 waitq_assert_wait64((struct waitq *)&kq->kqf_wqs,
4397 CAST_EVENT64_T(suppressq), THREAD_UNINT | THREAD_WAIT_NOREPORT,
4398 TIMEOUT_WAIT_FOREVER);
4399
4400 kqunlock(kq);
4401 thread_block(THREAD_CONTINUE_NULL);
4402 kqlock(kq);
4403 }
4404
4405 /* Nobody else processing */
4406
4407 /* clear pre-posts and KQ_WAKEUP now, in case we bail early */
4408 waitq_set_clear_preposts(&kq->kqf_wqs);
4409 kq->kqf_state &= ~KQ_WAKEUP;
4410
4411 /* anything left to process? */
4412 if (TAILQ_EMPTY(&kq->kqf_queue)) {
4413 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END,
4414 VM_KERNEL_UNSLIDE_OR_PERM(kq), 1);
4415 return -1;
4416 }
4417
4418 /* convert to processing mode */
4419 kq->kqf_state |= KQ_PROCESSING;
4420
4421 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END,
4422 VM_KERNEL_UNSLIDE_OR_PERM(kq));
4423
4424 return 0;
4425}
4426
4427/*
4428 * Try to end the processing, only called when a workq thread is attempting to
4429 * park (KEVENT_FLAG_PARKING is set).
4430 *
4431 * When returning -1, the kqworkq is setup again so that it is ready to be
4432 * processed.
4433 */
4434static int
4435kqworkq_end_processing(struct kqworkq *kqwq, workq_threadreq_t kqr,
4436 int kevent_flags)
4437{
4438 if (!TAILQ_EMPTY(&kqwq->kqwq_queue[kqr->tr_kq_qos_index])) {
4439 /* remember we didn't process everything */
4440 kqr->tr_kq_wakeup = true;
4441 }
4442
4443 if (kevent_flags & KEVENT_FLAG_PARKING) {
4444 /*
4445 * if acknowledge events "succeeds" it means there are events,
4446 * which is a failure condition for end_processing.
4447 */
4448 int rc = kqworkq_acknowledge_events(kqwq, kqr, kevent_flags,
4449 KQWQAE_END_PROCESSING);
4450 if (rc == 0) {
4451 return -1;
4452 }
4453 }
4454
4455 return 0;
4456}
4457
4458/*
4459 * Try to end the processing, only called when a workq thread is attempting to
4460 * park (KEVENT_FLAG_PARKING is set).
4461 *
4462 * When returning -1, the kqworkq is setup again so that it is ready to be
4463 * processed (as if kqworkloop_begin_processing had just been called).
4464 *
4465 * If successful and KEVENT_FLAG_PARKING was set in the kevent_flags,
4466 * the kqworkloop is unbound from its servicer as a side effect.
4467 */
4468static int
4469kqworkloop_end_processing(struct kqworkloop *kqwl, int flags, int kevent_flags)
4470{
4471 struct kqueue *kq = &kqwl->kqwl_kqueue;
4472 workq_threadreq_t kqr = &kqwl->kqwl_request;
4473 thread_qos_t qos_override;
4474 thread_t thread = kqr_thread_fast(kqr);
4475 int rc = 0;
4476
4477 kqlock_held(kq);
4478
4479 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_START,
4480 kqwl->kqwl_dynamicid, 0, 0);
4481
4482 if (flags & KQ_PROCESSING) {
4483 assert(kq->kq_state & KQ_PROCESSING);
4484
4485 /*
4486 * If we still have queued stayactive knotes, remember we didn't finish
4487 * processing all of them. This should be extremely rare and would
4488 * require to have a lot of them registered and fired.
4489 */
4490 if (!TAILQ_EMPTY(&kqwl->kqwl_queue[KQWL_BUCKET_STAYACTIVE])) {
4491 kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_WAKEUP_QOS,
4492 KQWL_BUCKET_STAYACTIVE);
4493 }
4494
4495 /*
4496 * When KEVENT_FLAG_PARKING is set, we need to attempt an unbind while
4497 * still under the lock.
4498 *
4499 * So we do everything kqworkloop_unbind() would do, but because we're
4500 * inside kqueue_process(), if the workloop actually received events
4501 * while our locks were dropped, we have the opportunity to fail the end
4502 * processing and loop again.
4503 *
4504 * This avoids going through the process-wide workqueue lock hence
4505 * scales better.
4506 */
4507 if (kevent_flags & KEVENT_FLAG_PARKING) {
4508 qos_override = kqworkloop_acknowledge_events(kqwl);
4509 }
4510 }
4511
4512 if (kevent_flags & KEVENT_FLAG_PARKING) {
4513 kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_PARKING, qos_override);
4514 if (kqr->tr_kq_wakeup && !kqwl->kqwl_owner) {
4515 /*
4516 * Reset wakeup bit to notice stay active events firing while we are
4517 * processing, as we cannot rely on the stayactive bucket emptiness.
4518 */
4519 kqwl->kqwl_wakeup_indexes &= ~KQWL_STAYACTIVE_FIRED_BIT;
4520 rc = -1;
4521 } else {
4522 kqworkloop_unbind_locked(kqwl, thread, KQWL_OVERRIDE_DROP_DELAYED);
4523 kqworkloop_release_live(kqwl);
4524 kq->kq_state &= ~flags;
4525 }
4526 } else {
4527 kq->kq_state &= ~flags;
4528 kq->kq_state |= KQ_R2K_ARMED;
4529 kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, 0);
4530 }
4531
4532 if ((kevent_flags & KEVENT_FLAG_PARKING) && rc == 0) {
4533 kqworkloop_unbind_delayed_override_drop(thread);
4534 }
4535
4536 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_END,
4537 kqwl->kqwl_dynamicid, 0, 0);
4538
4539 return rc;
4540}
4541
4542/*
4543 * Called with kqueue lock held.
4544 *
4545 * 0: no more events
4546 * -1: has more events
4547 * EBADF: kqueue is in draining mode
4548 */
4549static int
4550kqfile_end_processing(struct kqfile *kq)
4551{
4552 struct kqtailq *suppressq = &kq->kqf_suppressed;
4553 struct knote *kn;
4554 int procwait;
4555
4556 kqlock_held(kq);
4557
4558 assert((kq->kqf_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0);
4559
4560 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_END),
4561 VM_KERNEL_UNSLIDE_OR_PERM(kq), 0);
4562
4563 /*
4564 * Return suppressed knotes to their original state.
4565 */
4566 while ((kn = TAILQ_FIRST(suppressq)) != NULL) {
4567 assert(kn->kn_status & KN_SUPPRESSED);
4568 knote_unsuppress(kq, kn);
4569 }
4570
4571 procwait = (kq->kqf_state & KQ_PROCWAIT);
4572 kq->kqf_state &= ~(KQ_PROCESSING | KQ_PROCWAIT);
4573
4574 if (procwait) {
4575 /* first wake up any thread already waiting to process */
4576 waitq_wakeup64_all((struct waitq *)&kq->kqf_wqs,
4577 CAST_EVENT64_T(suppressq), THREAD_AWAKENED, WAITQ_ALL_PRIORITIES);
4578 }
4579
4580 if (kq->kqf_state & KQ_DRAIN) {
4581 return EBADF;
4582 }
4583 return (kq->kqf_state & KQ_WAKEUP) ? -1 : 0;
4584}
4585
4586static int
4587kqueue_workloop_ctl_internal(proc_t p, uintptr_t cmd, uint64_t __unused options,
4588 struct kqueue_workloop_params *params, int *retval)
4589{
4590 int error = 0;
4591 struct kqworkloop *kqwl;
4592 struct filedesc *fdp = p->p_fd;
4593 workq_threadreq_param_t trp = { };
4594
4595 switch (cmd) {
4596 case KQ_WORKLOOP_CREATE:
4597 if (!params->kqwlp_flags) {
4598 error = EINVAL;
4599 break;
4600 }
4601
4602 if ((params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_PRI) &&
4603 (params->kqwlp_sched_pri < 1 ||
4604 params->kqwlp_sched_pri > 63 /* MAXPRI_USER */)) {
4605 error = EINVAL;
4606 break;
4607 }
4608
4609 if ((params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_POL) &&
4610 invalid_policy(params->kqwlp_sched_pol)) {
4611 error = EINVAL;
4612 break;
4613 }
4614
4615 if ((params->kqwlp_flags & KQ_WORKLOOP_CREATE_CPU_PERCENT) &&
4616 (params->kqwlp_cpu_percent <= 0 ||
4617 params->kqwlp_cpu_percent > 100 ||
4618 params->kqwlp_cpu_refillms <= 0 ||
4619 params->kqwlp_cpu_refillms > 0x00ffffff)) {
4620 error = EINVAL;
4621 break;
4622 }
4623
4624 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_PRI) {
4625 trp.trp_flags |= TRP_PRIORITY;
4626 trp.trp_pri = params->kqwlp_sched_pri;
4627 }
4628 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_SCHED_POL) {
4629 trp.trp_flags |= TRP_POLICY;
4630 trp.trp_pol = params->kqwlp_sched_pol;
4631 }
4632 if (params->kqwlp_flags & KQ_WORKLOOP_CREATE_CPU_PERCENT) {
4633 trp.trp_flags |= TRP_CPUPERCENT;
4634 trp.trp_cpupercent = (uint8_t)params->kqwlp_cpu_percent;
4635 trp.trp_refillms = params->kqwlp_cpu_refillms;
4636 }
4637
4638 error = kqworkloop_get_or_create(p, params->kqwlp_id, &trp,
4639 KEVENT_FLAG_DYNAMIC_KQUEUE | KEVENT_FLAG_WORKLOOP |
4640 KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST, &kqwl);
4641 if (error) {
4642 break;
4643 }
4644
4645 if (!(fdp->fd_flags & FD_WORKLOOP)) {
4646 /* FD_WORKLOOP indicates we've ever created a workloop
4647 * via this syscall but its only ever added to a process, never
4648 * removed.
4649 */
4650 proc_fdlock(p);
4651 fdp->fd_flags |= FD_WORKLOOP;
4652 proc_fdunlock(p);
4653 }
4654 break;
4655 case KQ_WORKLOOP_DESTROY:
4656 error = kqworkloop_get_or_create(p, params->kqwlp_id, NULL,
4657 KEVENT_FLAG_DYNAMIC_KQUEUE | KEVENT_FLAG_WORKLOOP |
4658 KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST, &kqwl);
4659 if (error) {
4660 break;
4661 }
4662 kqlock(kqwl);
4663 trp.trp_value = kqwl->kqwl_params;
4664 if (trp.trp_flags && !(trp.trp_flags & TRP_RELEASED)) {
4665 trp.trp_flags |= TRP_RELEASED;
4666 kqwl->kqwl_params = trp.trp_value;
4667 kqworkloop_release_live(kqwl);
4668 } else {
4669 error = EINVAL;
4670 }
4671 kqunlock(kqwl);
4672 kqworkloop_release(kqwl);
4673 break;
4674 }
4675 *retval = 0;
4676 return error;
4677}
4678
4679int
4680kqueue_workloop_ctl(proc_t p, struct kqueue_workloop_ctl_args *uap, int *retval)
4681{
4682 struct kqueue_workloop_params params = {
4683 .kqwlp_id = 0,
4684 };
4685 if (uap->sz < sizeof(params.kqwlp_version)) {
4686 return EINVAL;
4687 }
4688
4689 size_t copyin_sz = MIN(sizeof(params), uap->sz);
4690 int rv = copyin(uap->addr, &params, copyin_sz);
4691 if (rv) {
4692 return rv;
4693 }
4694
4695 if (params.kqwlp_version != (int)uap->sz) {
4696 return EINVAL;
4697 }
4698
4699 return kqueue_workloop_ctl_internal(p, uap->cmd, uap->options, &params,
4700 retval);
4701}
4702
4703/*ARGSUSED*/
4704static int
4705kqueue_select(struct fileproc *fp, int which, void *wq_link_id,
4706 __unused vfs_context_t ctx)
4707{
4708 struct kqfile *kq = (struct kqfile *)fp->f_data;
4709 struct kqtailq *suppressq = &kq->kqf_suppressed;
4710 struct kqtailq *queue = &kq->kqf_queue;
4711 struct knote *kn;
4712 int retnum = 0;
4713
4714 if (which != FREAD) {
4715 return 0;
4716 }
4717
4718 kqlock(kq);
4719
4720 assert((kq->kqf_state & KQ_WORKQ) == 0);
4721
4722 /*
4723 * If this is the first pass, link the wait queue associated with the
4724 * the kqueue onto the wait queue set for the select(). Normally we
4725 * use selrecord() for this, but it uses the wait queue within the
4726 * selinfo structure and we need to use the main one for the kqueue to
4727 * catch events from KN_STAYQUEUED sources. So we do the linkage manually.
4728 * (The select() call will unlink them when it ends).
4729 */
4730 if (wq_link_id != NULL) {
4731 thread_t cur_act = current_thread();
4732 struct uthread * ut = get_bsdthread_info(cur_act);
4733
4734 kq->kqf_state |= KQ_SEL;
4735 waitq_link((struct waitq *)&kq->kqf_wqs, ut->uu_wqset,
4736 WAITQ_SHOULD_LOCK, (uint64_t *)wq_link_id);
4737
4738 /* always consume the reserved link object */
4739 waitq_link_release(*(uint64_t *)wq_link_id);
4740 *(uint64_t *)wq_link_id = 0;
4741
4742 /*
4743 * selprocess() is expecting that we send it back the waitq
4744 * that was just added to the thread's waitq set. In order
4745 * to not change the selrecord() API (which is exported to
4746 * kexts), we pass this value back through the
4747 * void *wq_link_id pointer we were passed. We need to use
4748 * memcpy here because the pointer may not be properly aligned
4749 * on 32-bit systems.
4750 */
4751 void *wqptr = &kq->kqf_wqs;
4752 memcpy(wq_link_id, (void *)&wqptr, sizeof(void *));
4753 }
4754
4755 if (kqfile_begin_processing(kq) == -1) {
4756 kqunlock(kq);
4757 return 0;
4758 }
4759
4760 if (!TAILQ_EMPTY(queue)) {
4761 /*
4762 * there is something queued - but it might be a
4763 * KN_STAYACTIVE knote, which may or may not have
4764 * any events pending. Otherwise, we have to walk
4765 * the list of knotes to see, and peek at the
4766 * (non-vanished) stay-active ones to be really sure.
4767 */
4768 while ((kn = (struct knote *)TAILQ_FIRST(queue)) != NULL) {
4769 if (kn->kn_status & KN_ACTIVE) {
4770 retnum = 1;
4771 goto out;
4772 }
4773 assert(kn->kn_status & KN_STAYACTIVE);
4774 knote_suppress(kq, kn);
4775 }
4776
4777 /*
4778 * There were no regular events on the queue, so take
4779 * a deeper look at the stay-queued ones we suppressed.
4780 */
4781 while ((kn = (struct knote *)TAILQ_FIRST(suppressq)) != NULL) {
4782 KNOTE_LOCK_CTX(knlc);
4783 int result = 0;
4784
4785 /* If didn't vanish while suppressed - peek at it */
4786 if ((kn->kn_status & KN_DROPPING) || !knote_lock(kq, kn, &knlc,
4787 KNOTE_KQ_LOCK_ON_FAILURE)) {
4788 continue;
4789 }
4790
4791 result = filter_call(knote_fops(kn), f_peek(kn));
4792
4793 kqlock(kq);
4794 knote_unlock(kq, kn, &knlc, KNOTE_KQ_LOCK_ALWAYS);
4795
4796 /* unsuppress it */
4797 knote_unsuppress(kq, kn);
4798
4799 /* has data or it has to report a vanish */
4800 if (result & FILTER_ACTIVE) {
4801 retnum = 1;
4802 goto out;
4803 }
4804 }
4805 }
4806
4807out:
4808 kqfile_end_processing(kq);
4809 kqunlock(kq);
4810 return retnum;
4811}
4812
4813/*
4814 * kqueue_close -
4815 */
4816/*ARGSUSED*/
4817static int
4818kqueue_close(struct fileglob *fg, __unused vfs_context_t ctx)
4819{
4820 struct kqfile *kqf = (struct kqfile *)fg->fg_data;
4821
4822 assert((kqf->kqf_state & KQ_WORKQ) == 0);
4823 kqueue_dealloc(&kqf->kqf_kqueue);
4824 fg->fg_data = NULL;
4825 return 0;
4826}
4827
4828/*
4829 * Max depth of the nested kq path that can be created.
4830 * Note that this has to be less than the size of kq_level
4831 * to avoid wrapping around and mislabeling the level.
4832 */
4833#define MAX_NESTED_KQ 1000
4834
4835/*ARGSUSED*/
4836/*
4837 * The callers has taken a use-count reference on this kqueue and will donate it
4838 * to the kqueue we are being added to. This keeps the kqueue from closing until
4839 * that relationship is torn down.
4840 */
4841static int
4842kqueue_kqfilter(struct fileproc *fp, struct knote *kn,
4843 __unused struct kevent_qos_s *kev)
4844{
4845 struct kqfile *kqf = (struct kqfile *)fp->f_data;
4846 struct kqueue *kq = &kqf->kqf_kqueue;
4847 struct kqueue *parentkq = knote_get_kq(kn);
4848
4849 assert((kqf->kqf_state & KQ_WORKQ) == 0);
4850
4851 if (parentkq == kq || kn->kn_filter != EVFILT_READ) {
4852 knote_set_error(kn, EINVAL);
4853 return 0;
4854 }
4855
4856 /*
4857 * We have to avoid creating a cycle when nesting kqueues
4858 * inside another. Rather than trying to walk the whole
4859 * potential DAG of nested kqueues, we just use a simple
4860 * ceiling protocol. When a kqueue is inserted into another,
4861 * we check that the (future) parent is not already nested
4862 * into another kqueue at a lower level than the potenial
4863 * child (because it could indicate a cycle). If that test
4864 * passes, we just mark the nesting levels accordingly.
4865 *
4866 * Only up to MAX_NESTED_KQ can be nested.
4867 *
4868 * Note: kqworkq and kqworkloop cannot be nested and have reused their
4869 * kq_level field, so ignore these as parent.
4870 */
4871
4872 kqlock(parentkq);
4873
4874 if ((parentkq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0) {
4875 if (parentkq->kq_level > 0 &&
4876 parentkq->kq_level < kq->kq_level) {
4877 kqunlock(parentkq);
4878 knote_set_error(kn, EINVAL);
4879 return 0;
4880 }
4881
4882 /* set parent level appropriately */
4883 uint16_t plevel = (parentkq->kq_level == 0)? 2: parentkq->kq_level;
4884 if (plevel < kq->kq_level + 1) {
4885 if (kq->kq_level + 1 > MAX_NESTED_KQ) {
4886 kqunlock(parentkq);
4887 knote_set_error(kn, EINVAL);
4888 return 0;
4889 }
4890 plevel = kq->kq_level + 1;
4891 }
4892
4893 parentkq->kq_level = plevel;
4894 }
4895
4896 kqunlock(parentkq);
4897
4898 kn->kn_filtid = EVFILTID_KQREAD;
4899 kqlock(kq);
4900 KNOTE_ATTACH(&kqf->kqf_sel.si_note, kn);
4901 /* indicate nesting in child, if needed */
4902 if (kq->kq_level == 0) {
4903 kq->kq_level = 1;
4904 }
4905
4906 int count = kq->kq_count;
4907 kqunlock(kq);
4908 return count > 0;
4909}
4910
4911/*
4912 * kqueue_drain - called when kq is closed
4913 */
4914/*ARGSUSED*/
4915static int
4916kqueue_drain(struct fileproc *fp, __unused vfs_context_t ctx)
4917{
4918 struct kqfile *kqf = (struct kqfile *)fp->f_fglob->fg_data;
4919
4920 assert((kqf->kqf_state & KQ_WORKQ) == 0);
4921
4922 kqlock(kqf);
4923 kqf->kqf_state |= KQ_DRAIN;
4924
4925 /* wakeup sleeping threads */
4926 if ((kqf->kqf_state & (KQ_SLEEP | KQ_SEL)) != 0) {
4927 kqf->kqf_state &= ~(KQ_SLEEP | KQ_SEL);
4928 (void)waitq_wakeup64_all((struct waitq *)&kqf->kqf_wqs,
4929 KQ_EVENT,
4930 THREAD_RESTART,
4931 WAITQ_ALL_PRIORITIES);
4932 }
4933
4934 /* wakeup threads waiting their turn to process */
4935 if (kqf->kqf_state & KQ_PROCWAIT) {
4936 assert(kqf->kqf_state & KQ_PROCESSING);
4937
4938 kqf->kqf_state &= ~KQ_PROCWAIT;
4939 (void)waitq_wakeup64_all((struct waitq *)&kqf->kqf_wqs,
4940 CAST_EVENT64_T(&kqf->kqf_suppressed),
4941 THREAD_RESTART, WAITQ_ALL_PRIORITIES);
4942 }
4943
4944 kqunlock(kqf);
4945 return 0;
4946}
4947
4948/*ARGSUSED*/
4949int
4950kqueue_stat(struct kqueue *kq, void *ub, int isstat64, proc_t p)
4951{
4952 assert((kq->kq_state & KQ_WORKQ) == 0);
4953
4954 kqlock(kq);
4955 if (isstat64 != 0) {
4956 struct stat64 *sb64 = (struct stat64 *)ub;
4957
4958 bzero((void *)sb64, sizeof(*sb64));
4959 sb64->st_size = kq->kq_count;
4960 if (kq->kq_state & KQ_KEV_QOS) {
4961 sb64->st_blksize = sizeof(struct kevent_qos_s);
4962 } else if (kq->kq_state & KQ_KEV64) {
4963 sb64->st_blksize = sizeof(struct kevent64_s);
4964 } else if (IS_64BIT_PROCESS(p)) {
4965 sb64->st_blksize = sizeof(struct user64_kevent);
4966 } else {
4967 sb64->st_blksize = sizeof(struct user32_kevent);
4968 }
4969 sb64->st_mode = S_IFIFO;
4970 } else {
4971 struct stat *sb = (struct stat *)ub;
4972
4973 bzero((void *)sb, sizeof(*sb));
4974 sb->st_size = kq->kq_count;
4975 if (kq->kq_state & KQ_KEV_QOS) {
4976 sb->st_blksize = sizeof(struct kevent_qos_s);
4977 } else if (kq->kq_state & KQ_KEV64) {
4978 sb->st_blksize = sizeof(struct kevent64_s);
4979 } else if (IS_64BIT_PROCESS(p)) {
4980 sb->st_blksize = sizeof(struct user64_kevent);
4981 } else {
4982 sb->st_blksize = sizeof(struct user32_kevent);
4983 }
4984 sb->st_mode = S_IFIFO;
4985 }
4986 kqunlock(kq);
4987 return 0;
4988}
4989
4990static inline bool
4991kqueue_threadreq_can_use_ast(struct kqueue *kq)
4992{
4993 if (current_proc() == kq->kq_p) {
4994 /*
4995 * Setting an AST from a non BSD syscall is unsafe: mach_msg_trap() can
4996 * do combined send/receive and in the case of self-IPC, the AST may bet
4997 * set on a thread that will not return to userspace and needs the
4998 * thread the AST would create to unblock itself.
4999 *
5000 * At this time, we really want to target:
5001 *
5002 * - kevent variants that can cause thread creations, and dispatch
5003 * really only uses kevent_qos and kevent_id,
5004 *
5005 * - workq_kernreturn (directly about thread creations)
5006 *
5007 * - bsdthread_ctl which is used for qos changes and has direct impact
5008 * on the creator thread scheduling decisions.
5009 */
5010 switch (current_uthread()->syscall_code) {
5011 case SYS_kevent_qos:
5012 case SYS_kevent_id:
5013 case SYS_workq_kernreturn:
5014 case SYS_bsdthread_ctl:
5015 return true;
5016 }
5017 }
5018 return false;
5019}
5020
5021/*
5022 * Interact with the pthread kext to request a servicing there at a specific QoS
5023 * level.
5024 *
5025 * - Caller holds the workq request lock
5026 *
5027 * - May be called with the kqueue's wait queue set locked,
5028 * so cannot do anything that could recurse on that.
5029 */
5030static void
5031kqueue_threadreq_initiate(struct kqueue *kq, workq_threadreq_t kqr,
5032 kq_index_t qos, int flags)
5033{
5034 assert(kqr->tr_kq_wakeup);
5035 assert(kqr_thread(kqr) == THREAD_NULL);
5036 assert(!kqr_thread_requested(kqr));
5037 struct turnstile *ts = TURNSTILE_NULL;
5038
5039 if (workq_is_exiting(kq->kq_p)) {
5040 return;
5041 }
5042
5043 kqlock_held(kq);
5044
5045 if (kq->kq_state & KQ_WORKLOOP) {
5046 __assert_only struct kqworkloop *kqwl = (struct kqworkloop *)kq;
5047
5048 assert(kqwl->kqwl_owner == THREAD_NULL);
5049 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_THREQUEST),
5050 kqwl->kqwl_dynamicid, 0, qos, kqr->tr_kq_wakeup);
5051 ts = kqwl->kqwl_turnstile;
5052 /* Add a thread request reference on the kqueue. */
5053 kqworkloop_retain(kqwl);
5054 } else {
5055 assert(kq->kq_state & KQ_WORKQ);
5056 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_THREQUEST),
5057 -1, 0, qos, kqr->tr_kq_wakeup);
5058 }
5059
5060 /*
5061 * New-style thread request supported.
5062 * Provide the pthread kext a pointer to a workq_threadreq_s structure for
5063 * its use until a corresponding kqueue_threadreq_bind callback.
5064 */
5065 if (kqueue_threadreq_can_use_ast(kq)) {
5066 flags |= WORKQ_THREADREQ_SET_AST_ON_FAILURE;
5067 }
5068 if (qos == KQWQ_QOS_MANAGER) {
5069 qos = WORKQ_THREAD_QOS_MANAGER;
5070 }
5071 if (!workq_kern_threadreq_initiate(kq->kq_p, kqr, ts, qos, flags)) {
5072 /*
5073 * Process is shutting down or exec'ing.
5074 * All the kqueues are going to be cleaned up
5075 * soon. Forget we even asked for a thread -
5076 * and make sure we don't ask for more.
5077 */
5078 kq->kq_state &= ~KQ_R2K_ARMED;
5079 kqueue_release_live(kq);
5080 }
5081}
5082
5083/*
5084 * kqueue_threadreq_bind_prepost - prepost the bind to kevent
5085 *
5086 * This is used when kqueue_threadreq_bind may cause a lock inversion.
5087 */
5088__attribute__((always_inline))
5089void
5090kqueue_threadreq_bind_prepost(struct proc *p __unused, workq_threadreq_t kqr,
5091 struct uthread *ut)
5092{
5093 ut->uu_kqr_bound = kqr;
5094 kqr->tr_thread = ut->uu_thread;
5095 kqr->tr_state = WORKQ_TR_STATE_BINDING;
5096}
5097
5098/*
5099 * kqueue_threadreq_bind_commit - commit a bind prepost
5100 *
5101 * The workq code has to commit any binding prepost before the thread has
5102 * a chance to come back to userspace (and do kevent syscalls) or be aborted.
5103 */
5104void
5105kqueue_threadreq_bind_commit(struct proc *p, thread_t thread)
5106{
5107 struct uthread *ut = get_bsdthread_info(thread);
5108 workq_threadreq_t kqr = ut->uu_kqr_bound;
5109 kqueue_t kqu = kqr_kqueue(p, kqr);
5110
5111 kqlock(kqu);
5112 if (kqr->tr_state == WORKQ_TR_STATE_BINDING) {
5113 kqueue_threadreq_bind(p, kqr, thread, 0);
5114 }
5115 kqunlock(kqu);
5116}
5117
5118static void
5119kqueue_threadreq_modify(kqueue_t kqu, workq_threadreq_t kqr, kq_index_t qos,
5120 workq_kern_threadreq_flags_t flags)
5121{
5122 assert(kqr_thread_requested_pending(kqr));
5123
5124 kqlock_held(kqu);
5125
5126 if (kqueue_threadreq_can_use_ast(kqu.kq)) {
5127 flags |= WORKQ_THREADREQ_SET_AST_ON_FAILURE;
5128 }
5129 workq_kern_threadreq_modify(kqu.kq->kq_p, kqr, qos, flags);
5130}
5131
5132/*
5133 * kqueue_threadreq_bind - bind thread to processing kqrequest
5134 *
5135 * The provided thread will be responsible for delivering events
5136 * associated with the given kqrequest. Bind it and get ready for
5137 * the thread to eventually arrive.
5138 */
5139void
5140kqueue_threadreq_bind(struct proc *p, workq_threadreq_t kqr, thread_t thread,
5141 unsigned int flags)
5142{
5143 kqueue_t kqu = kqr_kqueue(p, kqr);
5144 struct uthread *ut = get_bsdthread_info(thread);
5145
5146 kqlock_held(kqu);
5147
5148 assert(ut->uu_kqueue_override == 0);
5149
5150 if (kqr->tr_state == WORKQ_TR_STATE_BINDING) {
5151 assert(ut->uu_kqr_bound == kqr);
5152 assert(kqr->tr_thread == thread);
5153 } else {
5154 assert(kqr_thread_requested_pending(kqr));
5155 assert(kqr->tr_thread == THREAD_NULL);
5156 assert(ut->uu_kqr_bound == NULL);
5157 ut->uu_kqr_bound = kqr;
5158 kqr->tr_thread = thread;
5159 }
5160
5161 kqr->tr_state = WORKQ_TR_STATE_BOUND;
5162
5163 if (kqu.kq->kq_state & KQ_WORKLOOP) {
5164 struct turnstile *ts = kqu.kqwl->kqwl_turnstile;
5165
5166 if (__improbable(thread == kqu.kqwl->kqwl_owner)) {
5167 /*
5168 * <rdar://problem/38626999> shows that asserting here is not ok.
5169 *
5170 * This is not supposed to happen for correct use of the interface,
5171 * but it is sadly possible for userspace (with the help of memory
5172 * corruption, such as over-release of a dispatch queue) to make
5173 * the creator thread the "owner" of a workloop.
5174 *
5175 * Once that happens, and that creator thread picks up the same
5176 * workloop as a servicer, we trip this codepath. We need to fixup
5177 * the state to forget about this thread being the owner, as the
5178 * entire workloop state machine expects servicers to never be
5179 * owners and everything would basically go downhill from here.
5180 */
5181 kqu.kqwl->kqwl_owner = THREAD_NULL;
5182 if (kqworkloop_override(kqu.kqwl)) {
5183 thread_drop_kevent_override(thread);
5184 }
5185 }
5186
5187 if (ts && (flags & KQUEUE_THREADERQ_BIND_NO_INHERITOR_UPDATE) == 0) {
5188 /*
5189 * Past this point, the interlock is the kq req lock again,
5190 * so we can fix the inheritor for good.
5191 */
5192 filt_wlupdate_inheritor(kqu.kqwl, ts, TURNSTILE_IMMEDIATE_UPDATE);
5193 turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
5194 }
5195
5196 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_BIND), kqu.kqwl->kqwl_dynamicid,
5197 thread_tid(thread), kqr->tr_kq_qos_index,
5198 (kqr->tr_kq_override_index << 16) | kqr->tr_kq_wakeup);
5199
5200 ut->uu_kqueue_override = kqr->tr_kq_override_index;
5201 if (kqr->tr_kq_override_index) {
5202 thread_add_servicer_override(thread, kqr->tr_kq_override_index);
5203 }
5204 } else {
5205 assert(kqr->tr_kq_override_index == 0);
5206
5207 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_BIND), -1,
5208 thread_tid(thread), kqr->tr_kq_qos_index,
5209 (kqr->tr_kq_override_index << 16) | kqr->tr_kq_wakeup);
5210 }
5211}
5212
5213/*
5214 * kqueue_threadreq_cancel - abort a pending thread request
5215 *
5216 * Called when exiting/exec'ing. Forget our pending request.
5217 */
5218void
5219kqueue_threadreq_cancel(struct proc *p, workq_threadreq_t kqr)
5220{
5221 kqueue_release(kqr_kqueue(p, kqr));
5222}
5223
5224workq_threadreq_param_t
5225kqueue_threadreq_workloop_param(workq_threadreq_t kqr)
5226{
5227 struct kqworkloop *kqwl;
5228 workq_threadreq_param_t trp;
5229
5230 assert(kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP);
5231 kqwl = __container_of(kqr, struct kqworkloop, kqwl_request);
5232 trp.trp_value = kqwl->kqwl_params;
5233 return trp;
5234}
5235
5236/*
5237 * kqueue_threadreq_unbind - unbind thread from processing kqueue
5238 *
5239 * End processing the per-QoS bucket of events and allow other threads
5240 * to be requested for future servicing.
5241 *
5242 * caller holds a reference on the kqueue.
5243 */
5244void
5245kqueue_threadreq_unbind(struct proc *p, workq_threadreq_t kqr)
5246{
5247 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
5248 kqworkloop_unbind(kqr_kqworkloop(kqr));
5249 } else {
5250 kqworkq_unbind(p, kqr);
5251 }
5252}
5253
5254/*
5255 * If we aren't already busy processing events [for this QoS],
5256 * request workq thread support as appropriate.
5257 *
5258 * TBD - for now, we don't segregate out processing by QoS.
5259 *
5260 * - May be called with the kqueue's wait queue set locked,
5261 * so cannot do anything that could recurse on that.
5262 */
5263static void
5264kqworkq_wakeup(struct kqworkq *kqwq, kq_index_t qos_index)
5265{
5266 workq_threadreq_t kqr = kqworkq_get_request(kqwq, qos_index);
5267
5268 /* convert to thread qos value */
5269 assert(qos_index < KQWQ_NBUCKETS);
5270
5271 if (!kqr->tr_kq_wakeup) {
5272 kqr->tr_kq_wakeup = true;
5273 if (!kqr_thread_requested(kqr)) {
5274 kqueue_threadreq_initiate(&kqwq->kqwq_kqueue, kqr, qos_index, 0);
5275 }
5276 }
5277}
5278
5279/*
5280 * This represent the asynchronous QoS a given workloop contributes,
5281 * hence is the max of the current active knotes (override index)
5282 * and the workloop max qos (userspace async qos).
5283 */
5284static kq_index_t
5285kqworkloop_override(struct kqworkloop *kqwl)
5286{
5287 workq_threadreq_t kqr = &kqwl->kqwl_request;
5288 return MAX(kqr->tr_kq_qos_index, kqr->tr_kq_override_index);
5289}
5290
5291static inline void
5292kqworkloop_request_fire_r2k_notification(struct kqworkloop *kqwl)
5293{
5294 workq_threadreq_t kqr = &kqwl->kqwl_request;
5295
5296 kqlock_held(kqwl);
5297
5298 if (kqwl->kqwl_state & KQ_R2K_ARMED) {
5299 kqwl->kqwl_state &= ~KQ_R2K_ARMED;
5300 act_set_astkevent(kqr_thread_fast(kqr), AST_KEVENT_RETURN_TO_KERNEL);
5301 }
5302}
5303
5304static void
5305kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos)
5306{
5307 workq_threadreq_t kqr = &kqwl->kqwl_request;
5308 struct kqueue *kq = &kqwl->kqwl_kqueue;
5309 kq_index_t old_override = kqworkloop_override(kqwl);
5310 kq_index_t i;
5311
5312 kqlock_held(kqwl);
5313
5314 switch (op) {
5315 case KQWL_UTQ_UPDATE_WAKEUP_QOS:
5316 if (qos == KQWL_BUCKET_STAYACTIVE) {
5317 /*
5318 * the KQWL_BUCKET_STAYACTIVE is not a QoS bucket, we only remember
5319 * a high watermark (kqwl_stayactive_qos) of any stay active knote
5320 * that was ever registered with this workloop.
5321 *
5322 * When waitq_set__CALLING_PREPOST_HOOK__() wakes up any stay active
5323 * knote, we use this high-watermark as a wakeup-index, and also set
5324 * the magic KQWL_BUCKET_STAYACTIVE bit to make sure we remember
5325 * there is at least one stay active knote fired until the next full
5326 * processing of this bucket.
5327 */
5328 kqwl->kqwl_wakeup_indexes |= KQWL_STAYACTIVE_FIRED_BIT;
5329 qos = kqwl->kqwl_stayactive_qos;
5330 assert(qos);
5331 }
5332 if (kqwl->kqwl_wakeup_indexes & (1 << qos)) {
5333 assert(kqr->tr_kq_wakeup);
5334 break;
5335 }
5336
5337 kqwl->kqwl_wakeup_indexes |= (1 << qos);
5338 kqr->tr_kq_wakeup = true;
5339 kqworkloop_request_fire_r2k_notification(kqwl);
5340 goto recompute;
5341
5342 case KQWL_UTQ_UPDATE_STAYACTIVE_QOS:
5343 assert(qos);
5344 if (kqwl->kqwl_stayactive_qos < qos) {
5345 kqwl->kqwl_stayactive_qos = qos;
5346 if (kqwl->kqwl_wakeup_indexes & KQWL_STAYACTIVE_FIRED_BIT) {
5347 assert(kqr->tr_kq_wakeup);
5348 kqwl->kqwl_wakeup_indexes |= (1 << qos);
5349 goto recompute;
5350 }
5351 }
5352 break;
5353
5354 case KQWL_UTQ_PARKING:
5355 case KQWL_UTQ_UNBINDING:
5356 kqr->tr_kq_override_index = qos;
5357 /* FALLTHROUGH */
5358 case KQWL_UTQ_RECOMPUTE_WAKEUP_QOS:
5359 if (op == KQWL_UTQ_RECOMPUTE_WAKEUP_QOS) {
5360 assert(qos == THREAD_QOS_UNSPECIFIED);
5361 }
5362 i = KQWL_BUCKET_STAYACTIVE;
5363 if (TAILQ_EMPTY(&kqwl->kqwl_suppressed)) {
5364 kqr->tr_kq_override_index = THREAD_QOS_UNSPECIFIED;
5365 }
5366 if (!TAILQ_EMPTY(&kqwl->kqwl_queue[i]) &&
5367 (kqwl->kqwl_wakeup_indexes & KQWL_STAYACTIVE_FIRED_BIT)) {
5368 /*
5369 * If the KQWL_STAYACTIVE_FIRED_BIT is set, it means a stay active
5370 * knote may have fired, so we need to merge in kqwl_stayactive_qos.
5371 *
5372 * Unlike other buckets, this one is never empty but could be idle.
5373 */
5374 kqwl->kqwl_wakeup_indexes &= KQWL_STAYACTIVE_FIRED_BIT;
5375 kqwl->kqwl_wakeup_indexes |= (1 << kqwl->kqwl_stayactive_qos);
5376 } else {
5377 kqwl->kqwl_wakeup_indexes = 0;
5378 }
5379 for (i = THREAD_QOS_UNSPECIFIED + 1; i < KQWL_BUCKET_STAYACTIVE; i++) {
5380 if (!TAILQ_EMPTY(&kqwl->kqwl_queue[i])) {
5381 kqwl->kqwl_wakeup_indexes |= (1 << i);
5382 }
5383 }
5384 if (kqwl->kqwl_wakeup_indexes) {
5385 kqr->tr_kq_wakeup = true;
5386 kqworkloop_request_fire_r2k_notification(kqwl);
5387 } else {
5388 kqr->tr_kq_wakeup = false;
5389 }
5390 goto recompute;
5391
5392 case KQWL_UTQ_RESET_WAKEUP_OVERRIDE:
5393 kqr->tr_kq_override_index = qos;
5394 goto recompute;
5395
5396 case KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE:
5397recompute:
5398 /*
5399 * When modifying the wakeup QoS or the override QoS, we always need to
5400 * maintain our invariant that kqr_override_index is at least as large
5401 * as the highest QoS for which an event is fired.
5402 *
5403 * However this override index can be larger when there is an overriden
5404 * suppressed knote pushing on the kqueue.
5405 */
5406 if (kqwl->kqwl_wakeup_indexes > (1 << qos)) {
5407 qos = fls(kqwl->kqwl_wakeup_indexes) - 1; /* fls is 1-based */
5408 }
5409 if (kqr->tr_kq_override_index < qos) {
5410 kqr->tr_kq_override_index = qos;
5411 }
5412 break;
5413
5414 case KQWL_UTQ_REDRIVE_EVENTS:
5415 break;
5416
5417 case KQWL_UTQ_SET_QOS_INDEX:
5418 kqr->tr_kq_qos_index = qos;
5419 break;
5420
5421 default:
5422 panic("unknown kqwl thread qos update operation: %d", op);
5423 }
5424
5425 thread_t kqwl_owner = kqwl->kqwl_owner;
5426 thread_t servicer = kqr_thread(kqr);
5427 boolean_t qos_changed = FALSE;
5428 kq_index_t new_override = kqworkloop_override(kqwl);
5429
5430 /*
5431 * Apply the diffs to the owner if applicable
5432 */
5433 if (kqwl_owner) {
5434#if 0
5435 /* JMM - need new trace hooks for owner overrides */
5436 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST),
5437 kqwl->kqwl_dynamicid, thread_tid(kqwl_owner), kqr->tr_kq_qos_index,
5438 (kqr->tr_kq_override_index << 16) | kqr->tr_kq_wakeup);
5439#endif
5440 if (new_override == old_override) {
5441 // nothing to do
5442 } else if (old_override == THREAD_QOS_UNSPECIFIED) {
5443 thread_add_kevent_override(kqwl_owner, new_override);
5444 } else if (new_override == THREAD_QOS_UNSPECIFIED) {
5445 thread_drop_kevent_override(kqwl_owner);
5446 } else { /* old_override != new_override */
5447 thread_update_kevent_override(kqwl_owner, new_override);
5448 }
5449 }
5450
5451 /*
5452 * apply the diffs to the servicer
5453 */
5454 if (!kqr_thread_requested(kqr)) {
5455 /*
5456 * No servicer, nor thread-request
5457 *
5458 * Make a new thread request, unless there is an owner (or the workloop
5459 * is suspended in userland) or if there is no asynchronous work in the
5460 * first place.
5461 */
5462
5463 if (kqwl_owner == NULL && kqr->tr_kq_wakeup) {
5464 int initiate_flags = 0;
5465 if (op == KQWL_UTQ_UNBINDING) {
5466 initiate_flags = WORKQ_THREADREQ_ATTEMPT_REBIND;
5467 }
5468 kqueue_threadreq_initiate(kq, kqr, new_override, initiate_flags);
5469 }
5470 } else if (servicer) {
5471 /*
5472 * Servicer in flight
5473 *
5474 * Just apply the diff to the servicer
5475 */
5476 struct uthread *ut = get_bsdthread_info(servicer);
5477 if (ut->uu_kqueue_override != new_override) {
5478 if (ut->uu_kqueue_override == THREAD_QOS_UNSPECIFIED) {
5479 thread_add_servicer_override(servicer, new_override);
5480 } else if (new_override == THREAD_QOS_UNSPECIFIED) {
5481 thread_drop_servicer_override(servicer);
5482 } else { /* ut->uu_kqueue_override != new_override */
5483 thread_update_servicer_override(servicer, new_override);
5484 }
5485 ut->uu_kqueue_override = new_override;
5486 qos_changed = TRUE;
5487 }
5488 } else if (new_override == THREAD_QOS_UNSPECIFIED) {
5489 /*
5490 * No events to deliver anymore.
5491 *
5492 * However canceling with turnstiles is challenging, so the fact that
5493 * the request isn't useful will be discovered by the servicer himself
5494 * later on.
5495 */
5496 } else if (old_override != new_override) {
5497 /*
5498 * Request is in flight
5499 *
5500 * Apply the diff to the thread request
5501 */
5502 kqueue_threadreq_modify(kq, kqr, new_override, WORKQ_THREADREQ_NONE);
5503 qos_changed = TRUE;
5504 }
5505
5506 if (qos_changed) {
5507 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST), kqwl->kqwl_dynamicid,
5508 thread_tid(servicer), kqr->tr_kq_qos_index,
5509 (kqr->tr_kq_override_index << 16) | kqr->tr_kq_wakeup);
5510 }
5511}
5512
5513static void
5514kqworkloop_wakeup(struct kqworkloop *kqwl, kq_index_t qos)
5515{
5516 if ((kqwl->kqwl_state & KQ_PROCESSING) &&
5517 kqr_thread(&kqwl->kqwl_request) == current_thread()) {
5518 /*
5519 * kqworkloop_end_processing() will perform the required QoS
5520 * computations when it unsets the processing mode.
5521 */
5522 return;
5523 }
5524
5525 kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_WAKEUP_QOS, qos);
5526}
5527
5528static struct kqtailq *
5529kqueue_get_suppressed_queue(kqueue_t kq, struct knote *kn)
5530{
5531 if (kq.kq->kq_state & KQ_WORKLOOP) {
5532 return &kq.kqwl->kqwl_suppressed;
5533 } else if (kq.kq->kq_state & KQ_WORKQ) {
5534 return &kq.kqwq->kqwq_suppressed[kn->kn_qos_index];
5535 } else {
5536 return &kq.kqf->kqf_suppressed;
5537 }
5538}
5539
5540struct turnstile *
5541kqueue_alloc_turnstile(kqueue_t kqu)
5542{
5543 struct kqworkloop *kqwl = kqu.kqwl;
5544 kq_state_t kq_state;
5545
5546 kq_state = os_atomic_load(&kqu.kq->kq_state, dependency);
5547 if (kq_state & KQ_HAS_TURNSTILE) {
5548 /* force a dependency to pair with the atomic or with release below */
5549 return os_atomic_load_with_dependency_on(&kqwl->kqwl_turnstile,
5550 (uintptr_t)kq_state);
5551 }
5552
5553 if (!(kq_state & KQ_WORKLOOP)) {
5554 return TURNSTILE_NULL;
5555 }
5556
5557 struct turnstile *ts = turnstile_alloc(), *free_ts = TURNSTILE_NULL;
5558 bool workq_locked = false;
5559
5560 kqlock(kqu);
5561
5562 if (filt_wlturnstile_interlock_is_workq(kqwl)) {
5563 workq_locked = true;
5564 workq_kern_threadreq_lock(kqwl->kqwl_p);
5565 }
5566
5567 if (kqwl->kqwl_state & KQ_HAS_TURNSTILE) {
5568 free_ts = ts;
5569 ts = kqwl->kqwl_turnstile;
5570 } else {
5571 ts = turnstile_prepare((uintptr_t)kqwl, &kqwl->kqwl_turnstile,
5572 ts, TURNSTILE_WORKLOOPS);
5573
5574 /* release-barrier to pair with the unlocked load of kqwl_turnstile above */
5575 os_atomic_or(&kqwl->kqwl_state, KQ_HAS_TURNSTILE, release);
5576
5577 if (filt_wlturnstile_interlock_is_workq(kqwl)) {
5578 workq_kern_threadreq_update_inheritor(kqwl->kqwl_p,
5579 &kqwl->kqwl_request, kqwl->kqwl_owner,
5580 ts, TURNSTILE_IMMEDIATE_UPDATE);
5581 /*
5582 * The workq may no longer be the interlock after this.
5583 * In which case the inheritor wasn't updated.
5584 */
5585 }
5586 if (!filt_wlturnstile_interlock_is_workq(kqwl)) {
5587 filt_wlupdate_inheritor(kqwl, ts, TURNSTILE_IMMEDIATE_UPDATE);
5588 }
5589 }
5590
5591 if (workq_locked) {
5592 workq_kern_threadreq_unlock(kqwl->kqwl_p);
5593 }
5594
5595 kqunlock(kqu);
5596
5597 if (free_ts) {
5598 turnstile_deallocate(free_ts);
5599 } else {
5600 turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
5601 }
5602 return ts;
5603}
5604
5605__attribute__((always_inline))
5606struct turnstile *
5607kqueue_turnstile(kqueue_t kqu)
5608{
5609 kq_state_t kq_state = os_atomic_load(&kqu.kq->kq_state, relaxed);
5610 if (kq_state & KQ_WORKLOOP) {
5611 return os_atomic_load(&kqu.kqwl->kqwl_turnstile, relaxed);
5612 }
5613 return TURNSTILE_NULL;
5614}
5615
5616__attribute__((always_inline))
5617struct turnstile *
5618kqueue_threadreq_get_turnstile(workq_threadreq_t kqr)
5619{
5620 struct kqworkloop *kqwl = kqr_kqworkloop(kqr);
5621 if (kqwl) {
5622 return os_atomic_load(&kqwl->kqwl_turnstile, relaxed);
5623 }
5624 return TURNSTILE_NULL;
5625}
5626
5627static void
5628kqworkloop_set_overcommit(struct kqworkloop *kqwl)
5629{
5630 workq_threadreq_t kqr = &kqwl->kqwl_request;
5631
5632 /*
5633 * This test is racy, but since we never remove this bit,
5634 * it allows us to avoid taking a lock.
5635 */
5636 if (kqr->tr_flags & WORKQ_TR_FLAG_OVERCOMMIT) {
5637 return;
5638 }
5639
5640 kqlock_held(kqwl);
5641
5642 if (kqr_thread_requested_pending(kqr)) {
5643 kqueue_threadreq_modify(kqwl, kqr, kqr->tr_qos,
5644 WORKQ_THREADREQ_MAKE_OVERCOMMIT);
5645 } else {
5646 kqr->tr_flags |= WORKQ_TR_FLAG_OVERCOMMIT;
5647 }
5648}
5649
5650static void
5651kqworkq_update_override(struct kqworkq *kqwq, struct knote *kn,
5652 kq_index_t override_index)
5653{
5654 workq_threadreq_t kqr;
5655 kq_index_t old_override_index;
5656 kq_index_t queue_index = kn->kn_qos_index;
5657
5658 if (override_index <= queue_index) {
5659 return;
5660 }
5661
5662 kqr = kqworkq_get_request(kqwq, queue_index);
5663
5664 kqlock_held(kqwq);
5665
5666 old_override_index = kqr->tr_kq_override_index;
5667 if (override_index > MAX(kqr->tr_kq_qos_index, old_override_index)) {
5668 thread_t servicer = kqr_thread(kqr);
5669 kqr->tr_kq_override_index = override_index;
5670
5671 /* apply the override to [incoming?] servicing thread */
5672 if (servicer) {
5673 if (old_override_index) {
5674 thread_update_kevent_override(servicer, override_index);
5675 } else {
5676 thread_add_kevent_override(servicer, override_index);
5677 }
5678 }
5679 }
5680}
5681
5682static void
5683kqueue_update_override(kqueue_t kqu, struct knote *kn, thread_qos_t qos)
5684{
5685 if (kqu.kq->kq_state & KQ_WORKLOOP) {
5686 kqworkloop_update_threads_qos(kqu.kqwl, KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE,
5687 qos);
5688 } else {
5689 kqworkq_update_override(kqu.kqwq, kn, qos);
5690 }
5691}
5692
5693static void
5694kqworkloop_unbind_locked(struct kqworkloop *kqwl, thread_t thread,
5695 enum kqwl_unbind_locked_mode how)
5696{
5697 struct uthread *ut = get_bsdthread_info(thread);
5698 workq_threadreq_t kqr = &kqwl->kqwl_request;
5699
5700 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_UNBIND), kqwl->kqwl_dynamicid,
5701 thread_tid(thread), 0, 0);
5702
5703 kqlock_held(kqwl);
5704
5705 assert(ut->uu_kqr_bound == kqr);
5706 ut->uu_kqr_bound = NULL;
5707 if (how == KQWL_OVERRIDE_DROP_IMMEDIATELY &&
5708 ut->uu_kqueue_override != THREAD_QOS_UNSPECIFIED) {
5709 thread_drop_servicer_override(thread);
5710 ut->uu_kqueue_override = THREAD_QOS_UNSPECIFIED;
5711 }
5712
5713 if (kqwl->kqwl_owner == NULL && kqwl->kqwl_turnstile) {
5714 turnstile_update_inheritor(kqwl->kqwl_turnstile,
5715 TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE);
5716 turnstile_update_inheritor_complete(kqwl->kqwl_turnstile,
5717 TURNSTILE_INTERLOCK_HELD);
5718 }
5719
5720 kqr->tr_thread = THREAD_NULL;
5721 kqr->tr_state = WORKQ_TR_STATE_IDLE;
5722 kqwl->kqwl_state &= ~KQ_R2K_ARMED;
5723}
5724
5725static void
5726kqworkloop_unbind_delayed_override_drop(thread_t thread)
5727{
5728 struct uthread *ut = get_bsdthread_info(thread);
5729 assert(ut->uu_kqr_bound == NULL);
5730 if (ut->uu_kqueue_override != THREAD_QOS_UNSPECIFIED) {
5731 thread_drop_servicer_override(thread);
5732 ut->uu_kqueue_override = THREAD_QOS_UNSPECIFIED;
5733 }
5734}
5735
5736/*
5737 * kqworkloop_unbind - Unbind the servicer thread of a workloop kqueue
5738 *
5739 * It will acknowledge events, and possibly request a new thread if:
5740 * - there were active events left
5741 * - we pended waitq hook callouts during processing
5742 * - we pended wakeups while processing (or unsuppressing)
5743 *
5744 * Called with kqueue lock held.
5745 */
5746static void
5747kqworkloop_unbind(struct kqworkloop *kqwl)
5748{
5749 struct kqueue *kq = &kqwl->kqwl_kqueue;
5750 workq_threadreq_t kqr = &kqwl->kqwl_request;
5751 thread_t thread = kqr_thread_fast(kqr);
5752 int op = KQWL_UTQ_PARKING;
5753 kq_index_t qos_override = THREAD_QOS_UNSPECIFIED;
5754
5755 assert(thread == current_thread());
5756
5757 kqlock(kqwl);
5758
5759 /*
5760 * Forcing the KQ_PROCESSING flag allows for QoS updates because of
5761 * unsuppressing knotes not to be applied until the eventual call to
5762 * kqworkloop_update_threads_qos() below.
5763 */
5764 assert((kq->kq_state & KQ_PROCESSING) == 0);
5765 if (!TAILQ_EMPTY(&kqwl->kqwl_suppressed)) {
5766 kq->kq_state |= KQ_PROCESSING;
5767 qos_override = kqworkloop_acknowledge_events(kqwl);
5768 kq->kq_state &= ~KQ_PROCESSING;
5769 }
5770
5771 kqworkloop_unbind_locked(kqwl, thread, KQWL_OVERRIDE_DROP_DELAYED);
5772 kqworkloop_update_threads_qos(kqwl, op, qos_override);
5773
5774 kqunlock(kqwl);
5775
5776 /*
5777 * Drop the override on the current thread last, after the call to
5778 * kqworkloop_update_threads_qos above.
5779 */
5780 kqworkloop_unbind_delayed_override_drop(thread);
5781
5782 /* If last reference, dealloc the workloop kq */
5783 kqworkloop_release(kqwl);
5784}
5785
5786static thread_qos_t
5787kqworkq_unbind_locked(struct kqworkq *kqwq,
5788 workq_threadreq_t kqr, thread_t thread)
5789{
5790 struct uthread *ut = get_bsdthread_info(thread);
5791 kq_index_t old_override = kqr->tr_kq_override_index;
5792
5793 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_UNBIND), -1,
5794 thread_tid(kqr_thread(kqr)), kqr->tr_kq_qos_index, 0);
5795
5796 kqlock_held(kqwq);
5797
5798 assert(ut->uu_kqr_bound == kqr);
5799 ut->uu_kqr_bound = NULL;
5800 kqr->tr_thread = THREAD_NULL;
5801 kqr->tr_state = WORKQ_TR_STATE_IDLE;
5802 kqr->tr_kq_override_index = THREAD_QOS_UNSPECIFIED;
5803 kqwq->kqwq_state &= ~KQ_R2K_ARMED;
5804
5805 return old_override;
5806}
5807
5808/*
5809 * kqworkq_unbind - unbind of a workq kqueue from a thread
5810 *
5811 * We may have to request new threads.
5812 * This can happen there are no waiting processing threads and:
5813 * - there were active events we never got to (count > 0)
5814 * - we pended waitq hook callouts during processing
5815 * - we pended wakeups while processing (or unsuppressing)
5816 */
5817static void
5818kqworkq_unbind(proc_t p, workq_threadreq_t kqr)
5819{
5820 struct kqworkq *kqwq = (struct kqworkq *)p->p_fd->fd_wqkqueue;
5821 __assert_only int rc;
5822
5823 kqlock(kqwq);
5824 rc = kqworkq_acknowledge_events(kqwq, kqr, 0, KQWQAE_UNBIND);
5825 assert(rc == -1);
5826 kqunlock(kqwq);
5827}
5828
5829workq_threadreq_t
5830kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index)
5831{
5832 assert(qos_index < KQWQ_NBUCKETS);
5833 return &kqwq->kqwq_request[qos_index];
5834}
5835
5836static void
5837knote_reset_priority(kqueue_t kqu, struct knote *kn, pthread_priority_t pp)
5838{
5839 kq_index_t qos = _pthread_priority_thread_qos(pp);
5840
5841 if (kqu.kq->kq_state & KQ_WORKLOOP) {
5842 assert((pp & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG) == 0);
5843 pp = _pthread_priority_normalize(pp);
5844 } else if (kqu.kq->kq_state & KQ_WORKQ) {
5845 if (qos == THREAD_QOS_UNSPECIFIED) {
5846 /* On workqueues, outside of QoS means MANAGER */
5847 qos = KQWQ_QOS_MANAGER;
5848 pp = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
5849 } else {
5850 pp = _pthread_priority_normalize(pp);
5851 }
5852 } else {
5853 pp = _pthread_unspecified_priority();
5854 qos = THREAD_QOS_UNSPECIFIED;
5855 }
5856
5857 kn->kn_qos = pp;
5858
5859 if ((kn->kn_status & KN_MERGE_QOS) == 0 || qos > kn->kn_qos_override) {
5860 /* Never lower QoS when in "Merge" mode */
5861 kn->kn_qos_override = qos;
5862 }
5863
5864 /* only adjust in-use qos index when not suppressed */
5865 if (kn->kn_status & KN_SUPPRESSED) {
5866 kqueue_update_override(kqu, kn, qos);
5867 } else if (kn->kn_qos_index != qos) {
5868 knote_dequeue(kqu, kn);
5869 kn->kn_qos_index = qos;
5870 }
5871}
5872
5873static void
5874knote_adjust_qos(struct kqueue *kq, struct knote *kn, int result)
5875{
5876 thread_qos_t qos_index = (result >> FILTER_ADJUST_EVENT_QOS_SHIFT) & 7;
5877
5878 kqlock_held(kq);
5879
5880 assert(result & FILTER_ADJUST_EVENT_QOS_BIT);
5881 assert(qos_index < THREAD_QOS_LAST);
5882
5883 /*
5884 * Early exit for knotes that should not change QoS
5885 */
5886 if (__improbable(!knote_fops(kn)->f_adjusts_qos)) {
5887 panic("filter %d cannot change QoS", kn->kn_filtid);
5888 } else if (__improbable(!knote_has_qos(kn))) {
5889 return;
5890 }
5891
5892 /*
5893 * knotes with the FALLBACK flag will only use their registration QoS if the
5894 * incoming event has no QoS, else, the registration QoS acts as a floor.
5895 */
5896 thread_qos_t req_qos = _pthread_priority_thread_qos_fast(kn->kn_qos);
5897 if (kn->kn_qos & _PTHREAD_PRIORITY_FALLBACK_FLAG) {
5898 if (qos_index == THREAD_QOS_UNSPECIFIED) {
5899 qos_index = req_qos;
5900 }
5901 } else {
5902 if (qos_index < req_qos) {
5903 qos_index = req_qos;
5904 }
5905 }
5906 if ((kn->kn_status & KN_MERGE_QOS) && (qos_index < kn->kn_qos_override)) {
5907 /* Never lower QoS when in "Merge" mode */
5908 return;
5909 }
5910
5911 if ((kn->kn_status & KN_LOCKED) && (kn->kn_status & KN_POSTING)) {
5912 /*
5913 * When we're trying to update the QoS override and that both an
5914 * f_event() and other f_* calls are running concurrently, any of these
5915 * in flight calls may want to perform overrides that aren't properly
5916 * serialized with each other.
5917 *
5918 * The first update that observes this racy situation enters a "Merge"
5919 * mode which causes subsequent override requests to saturate the
5920 * override instead of replacing its value.
5921 *
5922 * This mode is left when knote_unlock() or knote_post()
5923 * observe that no other f_* routine is in flight.
5924 */
5925 kn->kn_status |= KN_MERGE_QOS;
5926 }
5927
5928 /*
5929 * Now apply the override if it changed.
5930 */
5931
5932 if (kn->kn_qos_override == qos_index) {
5933 return;
5934 }
5935
5936 kn->kn_qos_override = qos_index;
5937
5938 if (kn->kn_status & KN_SUPPRESSED) {
5939 /*
5940 * For suppressed events, the kn_qos_index field cannot be touched as it
5941 * allows us to know on which supress queue the knote is for a kqworkq.
5942 *
5943 * Also, there's no natural push applied on the kqueues when this field
5944 * changes anyway. We hence need to apply manual overrides in this case,
5945 * which will be cleared when the events are later acknowledged.
5946 */
5947 kqueue_update_override(kq, kn, qos_index);
5948 } else if (kn->kn_qos_index != qos_index) {
5949 knote_dequeue(kq, kn);
5950 kn->kn_qos_index = qos_index;
5951 }
5952}
5953
5954/*
5955 * Called back from waitq code when no threads waiting and the hook was set.
5956 *
5957 * Preemption is disabled - minimal work can be done in this context!!!
5958 */
5959void
5960waitq_set__CALLING_PREPOST_HOOK__(waitq_set_prepost_hook_t *kq_hook)
5961{
5962 kqueue_t kqu;
5963
5964 kqu.kq = __container_of(kq_hook, struct kqueue, kq_waitq_hook);
5965 assert(kqu.kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP));
5966
5967 kqlock(kqu);
5968
5969 if (kqu.kq->kq_count > 0) {
5970 if (kqu.kq->kq_state & KQ_WORKLOOP) {
5971 kqworkloop_wakeup(kqu.kqwl, KQWL_BUCKET_STAYACTIVE);
5972 } else {
5973 kqworkq_wakeup(kqu.kqwq, KQWQ_QOS_MANAGER);
5974 }
5975 }
5976
5977 kqunlock(kqu);
5978}
5979
5980void
5981klist_init(struct klist *list)
5982{
5983 SLIST_INIT(list);
5984}
5985
5986
5987/*
5988 * Query/Post each knote in the object's list
5989 *
5990 * The object lock protects the list. It is assumed
5991 * that the filter/event routine for the object can
5992 * determine that the object is already locked (via
5993 * the hint) and not deadlock itself.
5994 *
5995 * The object lock should also hold off pending
5996 * detach/drop operations.
5997 */
5998void
5999knote(struct klist *list, long hint)
6000{
6001 struct knote *kn;
6002
6003 SLIST_FOREACH(kn, list, kn_selnext) {
6004 knote_post(kn, hint);
6005 }
6006}
6007
6008/*
6009 * attach a knote to the specified list. Return true if this is the first entry.
6010 * The list is protected by whatever lock the object it is associated with uses.
6011 */
6012int
6013knote_attach(struct klist *list, struct knote *kn)
6014{
6015 int ret = SLIST_EMPTY(list);
6016 SLIST_INSERT_HEAD(list, kn, kn_selnext);
6017 return ret;
6018}
6019
6020/*
6021 * detach a knote from the specified list. Return true if that was the last entry.
6022 * The list is protected by whatever lock the object it is associated with uses.
6023 */
6024int
6025knote_detach(struct klist *list, struct knote *kn)
6026{
6027 SLIST_REMOVE(list, kn, knote, kn_selnext);
6028 return SLIST_EMPTY(list);
6029}
6030
6031/*
6032 * knote_vanish - Indicate that the source has vanished
6033 *
6034 * If the knote has requested EV_VANISHED delivery,
6035 * arrange for that. Otherwise, deliver a NOTE_REVOKE
6036 * event for backward compatibility.
6037 *
6038 * The knote is marked as having vanished, but is not
6039 * actually detached from the source in this instance.
6040 * The actual detach is deferred until the knote drop.
6041 *
6042 * Our caller already has the object lock held. Calling
6043 * the detach routine would try to take that lock
6044 * recursively - which likely is not supported.
6045 */
6046void
6047knote_vanish(struct klist *list, bool make_active)
6048{
6049 struct knote *kn;
6050 struct knote *kn_next;
6051
6052 SLIST_FOREACH_SAFE(kn, list, kn_selnext, kn_next) {
6053 struct kqueue *kq = knote_get_kq(kn);
6054
6055 kqlock(kq);
6056 if (__probable(kn->kn_status & KN_REQVANISH)) {
6057 /*
6058 * If EV_VANISH supported - prepare to deliver one
6059 */
6060 kn->kn_status |= KN_VANISHED;
6061 } else {
6062 /*
6063 * Handle the legacy way to indicate that the port/portset was
6064 * deallocated or left the current Mach portspace (modern technique
6065 * is with an EV_VANISHED protocol).
6066 *
6067 * Deliver an EV_EOF event for these changes (hopefully it will get
6068 * delivered before the port name recycles to the same generation
6069 * count and someone tries to re-register a kevent for it or the
6070 * events are udata-specific - avoiding a conflict).
6071 */
6072 kn->kn_flags |= EV_EOF | EV_ONESHOT;
6073 }
6074 if (make_active) {
6075 knote_activate(kq, kn, FILTER_ACTIVE);
6076 }
6077 kqunlock(kq);
6078 }
6079}
6080
6081/*
6082 * Force a lazy allocation of the waitqset link
6083 * of the kq_wqs associated with the kn
6084 * if it wasn't already allocated.
6085 *
6086 * This allows knote_link_waitq to never block
6087 * if reserved_link is not NULL.
6088 */
6089void
6090knote_link_waitqset_lazy_alloc(struct knote *kn)
6091{
6092 struct kqueue *kq = knote_get_kq(kn);
6093 waitq_set_lazy_init_link(&kq->kq_wqs);
6094}
6095
6096/*
6097 * Check if a lazy allocation for the waitqset link
6098 * of the kq_wqs is needed.
6099 */
6100boolean_t
6101knote_link_waitqset_should_lazy_alloc(struct knote *kn)
6102{
6103 struct kqueue *kq = knote_get_kq(kn);
6104 return waitq_set_should_lazy_init_link(&kq->kq_wqs);
6105}
6106
6107/*
6108 * For a given knote, link a provided wait queue directly with the kqueue.
6109 * Wakeups will happen via recursive wait queue support. But nothing will move
6110 * the knote to the active list at wakeup (nothing calls knote()). Instead,
6111 * we permanently enqueue them here.
6112 *
6113 * kqueue and knote references are held by caller.
6114 * waitq locked by caller.
6115 *
6116 * caller provides the wait queue link structure and insures that the kq->kq_wqs
6117 * is linked by previously calling knote_link_waitqset_lazy_alloc.
6118 */
6119int
6120knote_link_waitq(struct knote *kn, struct waitq *wq, uint64_t *reserved_link)
6121{
6122 struct kqueue *kq = knote_get_kq(kn);
6123 kern_return_t kr;
6124
6125 kr = waitq_link(wq, &kq->kq_wqs, WAITQ_ALREADY_LOCKED, reserved_link);
6126 if (kr == KERN_SUCCESS) {
6127 knote_markstayactive(kn);
6128 return 0;
6129 } else {
6130 return EINVAL;
6131 }
6132}
6133
6134/*
6135 * Unlink the provided wait queue from the kqueue associated with a knote.
6136 * Also remove it from the magic list of directly attached knotes.
6137 *
6138 * Note that the unlink may have already happened from the other side, so
6139 * ignore any failures to unlink and just remove it from the kqueue list.
6140 *
6141 * On success, caller is responsible for the link structure
6142 */
6143int
6144knote_unlink_waitq(struct knote *kn, struct waitq *wq)
6145{
6146 struct kqueue *kq = knote_get_kq(kn);
6147 kern_return_t kr;
6148
6149 kr = waitq_unlink(wq, &kq->kq_wqs);
6150 knote_clearstayactive(kn);
6151 return (kr != KERN_SUCCESS) ? EINVAL : 0;
6152}
6153
6154/*
6155 * remove all knotes referencing a specified fd
6156 *
6157 * Entered with the proc_fd lock already held.
6158 * It returns the same way, but may drop it temporarily.
6159 */
6160void
6161knote_fdclose(struct proc *p, int fd)
6162{
6163 struct klist *list;
6164 struct knote *kn;
6165 KNOTE_LOCK_CTX(knlc);
6166
6167restart:
6168 list = &p->p_fd->fd_knlist[fd];
6169 SLIST_FOREACH(kn, list, kn_link) {
6170 struct kqueue *kq = knote_get_kq(kn);
6171
6172 kqlock(kq);
6173
6174 if (kq->kq_p != p) {
6175 panic("%s: proc mismatch (kq->kq_p=%p != p=%p)",
6176 __func__, kq->kq_p, p);
6177 }
6178
6179 /*
6180 * If the knote supports EV_VANISHED delivery,
6181 * transition it to vanished mode (or skip over
6182 * it if already vanished).
6183 */
6184 if (kn->kn_status & KN_VANISHED) {
6185 kqunlock(kq);
6186 continue;
6187 }
6188
6189 proc_fdunlock(p);
6190 if (!knote_lock(kq, kn, &knlc, KNOTE_KQ_LOCK_ON_SUCCESS)) {
6191 /* the knote was dropped by someone, nothing to do */
6192 } else if (kn->kn_status & KN_REQVANISH) {
6193 kn->kn_status |= KN_VANISHED;
6194
6195 kqunlock(kq);
6196 knote_fops(kn)->f_detach(kn);
6197 if (kn->kn_is_fd) {
6198 fp_drop(p, kn->kn_id, kn->kn_fp, 0);
6199 }
6200 kn->kn_filtid = EVFILTID_DETACHED;
6201 kqlock(kq);
6202
6203 knote_activate(kq, kn, FILTER_ACTIVE);
6204 knote_unlock(kq, kn, &knlc, KNOTE_KQ_UNLOCK);
6205 } else {
6206 knote_drop(kq, kn, &knlc);
6207 }
6208
6209 proc_fdlock(p);
6210 goto restart;
6211 }
6212}
6213
6214/*
6215 * knote_fdfind - lookup a knote in the fd table for process
6216 *
6217 * If the filter is file-based, lookup based on fd index.
6218 * Otherwise use a hash based on the ident.
6219 *
6220 * Matching is based on kq, filter, and ident. Optionally,
6221 * it may also be based on the udata field in the kevent -
6222 * allowing multiple event registration for the file object
6223 * per kqueue.
6224 *
6225 * fd_knhashlock or fdlock held on entry (and exit)
6226 */
6227static struct knote *
6228knote_fdfind(struct kqueue *kq,
6229 const struct kevent_internal_s *kev,
6230 bool is_fd,
6231 struct proc *p)
6232{
6233 struct filedesc *fdp = p->p_fd;
6234 struct klist *list = NULL;
6235 struct knote *kn = NULL;
6236
6237 /*
6238 * determine where to look for the knote
6239 */
6240 if (is_fd) {
6241 /* fd-based knotes are linked off the fd table */
6242 if (kev->kei_ident < (u_int)fdp->fd_knlistsize) {
6243 list = &fdp->fd_knlist[kev->kei_ident];
6244 }
6245 } else if (fdp->fd_knhashmask != 0) {
6246 /* hash non-fd knotes here too */
6247 list = &fdp->fd_knhash[KN_HASH((u_long)kev->kei_ident, fdp->fd_knhashmask)];
6248 }
6249
6250 /*
6251 * scan the selected list looking for a match
6252 */
6253 if (list != NULL) {
6254 SLIST_FOREACH(kn, list, kn_link) {
6255 if (kq == knote_get_kq(kn) &&
6256 kev->kei_ident == kn->kn_id &&
6257 kev->kei_filter == kn->kn_filter) {
6258 if (kev->kei_flags & EV_UDATA_SPECIFIC) {
6259 if ((kn->kn_flags & EV_UDATA_SPECIFIC) &&
6260 kev->kei_udata == kn->kn_udata) {
6261 break; /* matching udata-specific knote */
6262 }
6263 } else if ((kn->kn_flags & EV_UDATA_SPECIFIC) == 0) {
6264 break; /* matching non-udata-specific knote */
6265 }
6266 }
6267 }
6268 }
6269 return kn;
6270}
6271
6272/*
6273 * kq_add_knote- Add knote to the fd table for process
6274 * while checking for duplicates.
6275 *
6276 * All file-based filters associate a list of knotes by file
6277 * descriptor index. All other filters hash the knote by ident.
6278 *
6279 * May have to grow the table of knote lists to cover the
6280 * file descriptor index presented.
6281 *
6282 * fd_knhashlock and fdlock unheld on entry (and exit).
6283 *
6284 * Takes a rwlock boost if inserting the knote is successful.
6285 */
6286static int
6287kq_add_knote(struct kqueue *kq, struct knote *kn, struct knote_lock_ctx *knlc,
6288 struct proc *p)
6289{
6290 struct filedesc *fdp = p->p_fd;
6291 struct klist *list = NULL;
6292 int ret = 0;
6293 bool is_fd = kn->kn_is_fd;
6294
6295 if (is_fd) {
6296 proc_fdlock(p);
6297 } else {
6298 knhash_lock(fdp);
6299 }
6300
6301 if (knote_fdfind(kq, &kn->kn_kevent, is_fd, p) != NULL) {
6302 /* found an existing knote: we can't add this one */
6303 ret = ERESTART;
6304 goto out_locked;
6305 }
6306
6307 /* knote was not found: add it now */
6308 if (!is_fd) {
6309 if (fdp->fd_knhashmask == 0) {
6310 u_long size = 0;
6311
6312 list = hashinit(CONFIG_KN_HASHSIZE, M_KQUEUE, &size);
6313 if (list == NULL) {
6314 ret = ENOMEM;
6315 goto out_locked;
6316 }
6317
6318 fdp->fd_knhash = list;
6319 fdp->fd_knhashmask = size;
6320 }
6321
6322 list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)];
6323 SLIST_INSERT_HEAD(list, kn, kn_link);
6324 ret = 0;
6325 goto out_locked;
6326 } else {
6327 /* knote is fd based */
6328
6329 if ((u_int)fdp->fd_knlistsize <= kn->kn_id) {
6330 u_int size = 0;
6331
6332 if (kn->kn_id >= (uint64_t)p->p_rlimit[RLIMIT_NOFILE].rlim_cur
6333 || kn->kn_id >= (uint64_t)maxfiles) {
6334 ret = EINVAL;
6335 goto out_locked;
6336 }
6337 /* have to grow the fd_knlist */
6338 size = fdp->fd_knlistsize;
6339 while (size <= kn->kn_id) {
6340 size += KQEXTENT;
6341 }
6342
6343 if (size >= (UINT_MAX / sizeof(struct klist *))) {
6344 ret = EINVAL;
6345 goto out_locked;
6346 }
6347
6348 MALLOC(list, struct klist *,
6349 size * sizeof(struct klist *), M_KQUEUE, M_WAITOK);
6350 if (list == NULL) {
6351 ret = ENOMEM;
6352 goto out_locked;
6353 }
6354
6355 bcopy((caddr_t)fdp->fd_knlist, (caddr_t)list,
6356 fdp->fd_knlistsize * sizeof(struct klist *));
6357 bzero((caddr_t)list +
6358 fdp->fd_knlistsize * sizeof(struct klist *),
6359 (size - fdp->fd_knlistsize) * sizeof(struct klist *));
6360 FREE(fdp->fd_knlist, M_KQUEUE);
6361 fdp->fd_knlist = list;
6362 fdp->fd_knlistsize = size;
6363 }
6364
6365 list = &fdp->fd_knlist[kn->kn_id];
6366 SLIST_INSERT_HEAD(list, kn, kn_link);
6367 ret = 0;
6368 goto out_locked;
6369 }
6370
6371out_locked:
6372 if (ret == 0) {
6373 kqlock(kq);
6374 assert((kn->kn_status & KN_LOCKED) == 0);
6375 (void)knote_lock(kq, kn, knlc, KNOTE_KQ_UNLOCK);
6376 kqueue_retain(kq); /* retain a kq ref */
6377 }
6378 if (is_fd) {
6379 proc_fdunlock(p);
6380 } else {
6381 knhash_unlock(fdp);
6382 }
6383
6384 return ret;
6385}
6386
6387/*
6388 * kq_remove_knote - remove a knote from the fd table for process
6389 *
6390 * If the filter is file-based, remove based on fd index.
6391 * Otherwise remove from the hash based on the ident.
6392 *
6393 * fd_knhashlock and fdlock unheld on entry (and exit).
6394 */
6395static void
6396kq_remove_knote(struct kqueue *kq, struct knote *kn, struct proc *p,
6397 struct knote_lock_ctx *knlc)
6398{
6399 struct filedesc *fdp = p->p_fd;
6400 struct klist *list = NULL;
6401 uint16_t kq_state;
6402 bool is_fd = kn->kn_is_fd;
6403
6404 if (is_fd) {
6405 proc_fdlock(p);
6406 } else {
6407 knhash_lock(fdp);
6408 }
6409
6410 if (is_fd) {
6411 assert((u_int)fdp->fd_knlistsize > kn->kn_id);
6412 list = &fdp->fd_knlist[kn->kn_id];
6413 } else {
6414 list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)];
6415 }
6416 SLIST_REMOVE(list, kn, knote, kn_link);
6417
6418 kqlock(kq);
6419 kq_state = kq->kq_state;
6420 if (knlc) {
6421 knote_unlock_cancel(kq, kn, knlc);
6422 } else {
6423 kqunlock(kq);
6424 }
6425 if (is_fd) {
6426 proc_fdunlock(p);
6427 } else {
6428 knhash_unlock(fdp);
6429 }
6430
6431 if (kq_state & KQ_DYNAMIC) {
6432 kqworkloop_release((struct kqworkloop *)kq);
6433 }
6434}
6435
6436/*
6437 * kq_find_knote_and_kq_lock - lookup a knote in the fd table for process
6438 * and, if the knote is found, acquires the kqlock while holding the fd table lock/spinlock.
6439 *
6440 * fd_knhashlock or fdlock unheld on entry (and exit)
6441 */
6442
6443static struct knote *
6444kq_find_knote_and_kq_lock(struct kqueue *kq, struct kevent_qos_s *kev,
6445 bool is_fd, struct proc *p)
6446{
6447 struct filedesc *fdp = p->p_fd;
6448 struct knote *kn;
6449
6450 if (is_fd) {
6451 proc_fdlock(p);
6452 } else {
6453 knhash_lock(fdp);
6454 }
6455
6456 /*
6457 * Temporary horrible hack:
6458 * this cast is gross and will go away in a future change.
6459 * It is OK to do because we don't look at xflags/s_fflags,
6460 * and that when we cast down the kev this way,
6461 * the truncated filter field works.
6462 */
6463 kn = knote_fdfind(kq, (struct kevent_internal_s *)kev, is_fd, p);
6464
6465 if (kn) {
6466 kqlock(kq);
6467 assert(knote_get_kq(kn) == kq);
6468 }
6469
6470 if (is_fd) {
6471 proc_fdunlock(p);
6472 } else {
6473 knhash_unlock(fdp);
6474 }
6475
6476 return kn;
6477}
6478
6479__attribute__((noinline))
6480static void
6481kqfile_wakeup(struct kqfile *kqf, __unused kq_index_t qos)
6482{
6483 /* flag wakeups during processing */
6484 if (kqf->kqf_state & KQ_PROCESSING) {
6485 kqf->kqf_state |= KQ_WAKEUP;
6486 }
6487
6488 /* wakeup a thread waiting on this queue */
6489 if (kqf->kqf_state & (KQ_SLEEP | KQ_SEL)) {
6490 kqf->kqf_state &= ~(KQ_SLEEP | KQ_SEL);
6491 waitq_wakeup64_all((struct waitq *)&kqf->kqf_wqs, KQ_EVENT,
6492 THREAD_AWAKENED, WAITQ_ALL_PRIORITIES);
6493 }
6494
6495 /* wakeup other kqueues/select sets we're inside */
6496 KNOTE(&kqf->kqf_sel.si_note, 0);
6497}
6498
6499static struct kqtailq *
6500knote_get_tailq(kqueue_t kqu, struct knote *kn)
6501{
6502 kq_index_t qos_index = kn->kn_qos_index;
6503
6504 if (kqu.kq->kq_state & KQ_WORKLOOP) {
6505 assert(qos_index < KQWL_NBUCKETS);
6506 } else if (kqu.kq->kq_state & KQ_WORKQ) {
6507 assert(qos_index < KQWQ_NBUCKETS);
6508 } else {
6509 assert(qos_index == QOS_INDEX_KQFILE);
6510 }
6511 static_assert(offsetof(struct kqueue, kq_queue) == sizeof(struct kqueue),
6512 "struct kqueue::kq_queue must be exactly at the end");
6513 return &kqu.kq->kq_queue[qos_index];
6514}
6515
6516static void
6517knote_enqueue(kqueue_t kqu, struct knote *kn, kn_status_t wakeup_mask)
6518{
6519 kqlock_held(kqu);
6520
6521 if ((kn->kn_status & (KN_ACTIVE | KN_STAYACTIVE)) == 0) {
6522 return;
6523 }
6524
6525 if (kn->kn_status & (KN_DISABLED | KN_SUPPRESSED | KN_DROPPING)) {
6526 return;
6527 }
6528
6529 if ((kn->kn_status & KN_QUEUED) == 0) {
6530 struct kqtailq *queue = knote_get_tailq(kqu, kn);
6531
6532 TAILQ_INSERT_TAIL(queue, kn, kn_tqe);
6533 kn->kn_status |= KN_QUEUED;
6534 kqu.kq->kq_count++;
6535 } else if ((kn->kn_status & KN_STAYACTIVE) == 0) {
6536 return;
6537 }
6538
6539 if (kn->kn_status & wakeup_mask) {
6540 if (kqu.kq->kq_state & KQ_WORKLOOP) {
6541 kqworkloop_wakeup(kqu.kqwl, kn->kn_qos_index);
6542 } else if (kqu.kq->kq_state & KQ_WORKQ) {
6543 kqworkq_wakeup(kqu.kqwq, kn->kn_qos_index);
6544 } else {
6545 kqfile_wakeup(kqu.kqf, kn->kn_qos_index);
6546 }
6547 }
6548}
6549
6550__attribute__((always_inline))
6551static inline void
6552knote_dequeue(kqueue_t kqu, struct knote *kn)
6553{
6554 if (kn->kn_status & KN_QUEUED) {
6555 struct kqtailq *queue = knote_get_tailq(kqu, kn);
6556
6557 // attaching the knote calls knote_reset_priority() without
6558 // the kqlock which is fine, so we can't call kqlock_held()
6559 // if we're not queued.
6560 kqlock_held(kqu);
6561
6562 TAILQ_REMOVE(queue, kn, kn_tqe);
6563 kn->kn_status &= ~KN_QUEUED;
6564 kqu.kq->kq_count--;
6565 }
6566}
6567
6568/* called with kqueue lock held */
6569static void
6570knote_suppress(kqueue_t kqu, struct knote *kn)
6571{
6572 struct kqtailq *suppressq;
6573
6574 kqlock_held(kqu);
6575
6576 assert((kn->kn_status & KN_SUPPRESSED) == 0);
6577 assert(kn->kn_status & KN_QUEUED);
6578
6579 knote_dequeue(kqu, kn);
6580 /* deactivate - so new activations indicate a wakeup */
6581 kn->kn_status &= ~KN_ACTIVE;
6582 kn->kn_status |= KN_SUPPRESSED;
6583 suppressq = kqueue_get_suppressed_queue(kqu, kn);
6584 TAILQ_INSERT_TAIL(suppressq, kn, kn_tqe);
6585}
6586
6587__attribute__((always_inline))
6588static inline void
6589knote_unsuppress_noqueue(kqueue_t kqu, struct knote *kn)
6590{
6591 struct kqtailq *suppressq;
6592
6593 kqlock_held(kqu);
6594
6595 assert(kn->kn_status & KN_SUPPRESSED);
6596
6597 kn->kn_status &= ~KN_SUPPRESSED;
6598 suppressq = kqueue_get_suppressed_queue(kqu, kn);
6599 TAILQ_REMOVE(suppressq, kn, kn_tqe);
6600
6601 /*
6602 * If the knote is no longer active, reset its push,
6603 * and resynchronize kn_qos_index with kn_qos_override
6604 * for knotes with a real qos.
6605 */
6606 if ((kn->kn_status & KN_ACTIVE) == 0 && knote_has_qos(kn)) {
6607 kn->kn_qos_override = _pthread_priority_thread_qos_fast(kn->kn_qos);
6608 }
6609 kn->kn_qos_index = kn->kn_qos_override;
6610}
6611
6612/* called with kqueue lock held */
6613static void
6614knote_unsuppress(kqueue_t kqu, struct knote *kn)
6615{
6616 if (kn->kn_status & KN_SUPPRESSED) {
6617 knote_unsuppress_noqueue(kqu, kn);
6618
6619 /* don't wakeup if unsuppressing just a stay-active knote */
6620 knote_enqueue(kqu, kn, KN_ACTIVE);
6621 }
6622}
6623
6624__attribute__((always_inline))
6625static inline void
6626knote_mark_active(struct knote *kn)
6627{
6628 if ((kn->kn_status & KN_ACTIVE) == 0) {
6629 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KNOTE_ACTIVATE),
6630 kn->kn_udata, kn->kn_status | (kn->kn_id << 32),
6631 kn->kn_filtid);
6632 }
6633
6634 kn->kn_status |= KN_ACTIVE;
6635}
6636
6637/* called with kqueue lock held */
6638static void
6639knote_activate(kqueue_t kqu, struct knote *kn, int result)
6640{
6641 assert(result & FILTER_ACTIVE);
6642 if (result & FILTER_ADJUST_EVENT_QOS_BIT) {
6643 // may dequeue the knote
6644 knote_adjust_qos(kqu.kq, kn, result);
6645 }
6646 knote_mark_active(kn);
6647 knote_enqueue(kqu, kn, KN_ACTIVE | KN_STAYACTIVE);
6648}
6649
6650/*
6651 * This function applies changes requested by f_attach or f_touch for
6652 * a given filter. It proceeds in a carefully chosen order to help
6653 * every single transition do the minimal amount of work possible.
6654 */
6655static void
6656knote_apply_touch(kqueue_t kqu, struct knote *kn, struct kevent_qos_s *kev,
6657 int result)
6658{
6659 kn_status_t wakeup_mask = KN_ACTIVE;
6660
6661 if ((kev->flags & EV_ENABLE) && (kn->kn_status & KN_DISABLED)) {
6662 /*
6663 * When a stayactive knote is reenabled, we may have missed wakeups
6664 * while it was disabled, so we need to poll it. To do so, ask
6665 * knote_enqueue() below to reenqueue it.
6666 */
6667 wakeup_mask |= KN_STAYACTIVE;
6668 kn->kn_status &= ~KN_DISABLED;
6669
6670 /*
6671 * it is possible for userland to have knotes registered for a given
6672 * workloop `wl_orig` but really handled on another workloop `wl_new`.
6673 *
6674 * In that case, rearming will happen from the servicer thread of
6675 * `wl_new` which if `wl_orig` is no longer being serviced, would cause
6676 * this knote to stay suppressed forever if we only relied on
6677 * kqworkloop_acknowledge_events to be called by `wl_orig`.
6678 *
6679 * However if we see the KQ_PROCESSING bit on `wl_orig` set, we can't
6680 * unsuppress because that would mess with the processing phase of
6681 * `wl_orig`, however it also means kqworkloop_acknowledge_events()
6682 * will be called.
6683 */
6684 if (__improbable(kn->kn_status & KN_SUPPRESSED)) {
6685 if ((kqu.kq->kq_state & KQ_PROCESSING) == 0) {
6686 knote_unsuppress_noqueue(kqu, kn);
6687 }
6688 }
6689 }
6690
6691 if ((result & FILTER_UPDATE_REQ_QOS) && kev->qos && kev->qos != kn->kn_qos) {
6692 // may dequeue the knote
6693 knote_reset_priority(kqu, kn, kev->qos);
6694 }
6695
6696 /*
6697 * When we unsuppress above, or because of knote_reset_priority(),
6698 * the knote may have been dequeued, we need to restore the invariant
6699 * that if the knote is active it needs to be queued now that
6700 * we're done applying changes.
6701 */
6702 if (result & FILTER_ACTIVE) {
6703 knote_activate(kqu, kn, result);
6704 } else {
6705 knote_enqueue(kqu, kn, wakeup_mask);
6706 }
6707
6708 if ((result & FILTER_THREADREQ_NODEFEER) &&
6709 act_clear_astkevent(current_thread(), AST_KEVENT_REDRIVE_THREADREQ)) {
6710 workq_kern_threadreq_redrive(kqu.kq->kq_p, WORKQ_THREADREQ_NONE);
6711 }
6712}
6713
6714/*
6715 * knote_drop - disconnect and drop the knote
6716 *
6717 * Called with the kqueue locked, returns with the kqueue unlocked.
6718 *
6719 * If a knote locking context is passed, it is canceled.
6720 *
6721 * The knote may have already been detached from
6722 * (or not yet attached to) its source object.
6723 */
6724static void
6725knote_drop(struct kqueue *kq, struct knote *kn, struct knote_lock_ctx *knlc)
6726{
6727 struct proc *p = kq->kq_p;
6728
6729 kqlock_held(kq);
6730
6731 assert((kn->kn_status & KN_DROPPING) == 0);
6732 if (knlc == NULL) {
6733 assert((kn->kn_status & KN_LOCKED) == 0);
6734 }
6735 kn->kn_status |= KN_DROPPING;
6736
6737 if (kn->kn_status & KN_SUPPRESSED) {
6738 knote_unsuppress_noqueue(kq, kn);
6739 } else {
6740 knote_dequeue(kq, kn);
6741 }
6742 knote_wait_for_post(kq, kn);
6743
6744 knote_fops(kn)->f_detach(kn);
6745
6746 /* kq may be freed when kq_remove_knote() returns */
6747 kq_remove_knote(kq, kn, p, knlc);
6748 if (kn->kn_is_fd && ((kn->kn_status & KN_VANISHED) == 0)) {
6749 fp_drop(p, kn->kn_id, kn->kn_fp, 0);
6750 }
6751
6752 knote_free(kn);
6753}
6754
6755void
6756knote_init(void)
6757{
6758 knote_zone = zinit(sizeof(struct knote), 8192 * sizeof(struct knote),
6759 8192, "knote zone");
6760 zone_change(knote_zone, Z_CACHING_ENABLED, TRUE);
6761
6762 kqfile_zone = zinit(sizeof(struct kqfile), 8192 * sizeof(struct kqfile),
6763 8192, "kqueue file zone");
6764
6765 kqworkq_zone = zinit(sizeof(struct kqworkq), 8192 * sizeof(struct kqworkq),
6766 8192, "kqueue workq zone");
6767
6768 kqworkloop_zone = zinit(sizeof(struct kqworkloop), 8192 * sizeof(struct kqworkloop),
6769 8192, "kqueue workloop zone");
6770 zone_change(kqworkloop_zone, Z_CACHING_ENABLED, TRUE);
6771
6772 /* allocate kq lock group attribute and group */
6773 kq_lck_grp_attr = lck_grp_attr_alloc_init();
6774
6775 kq_lck_grp = lck_grp_alloc_init("kqueue", kq_lck_grp_attr);
6776
6777 /* Allocate kq lock attribute */
6778 kq_lck_attr = lck_attr_alloc_init();
6779
6780#if CONFIG_MEMORYSTATUS
6781 /* Initialize the memorystatus list lock */
6782 memorystatus_kevent_init(kq_lck_grp, kq_lck_attr);
6783#endif
6784}
6785SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL);
6786
6787const struct filterops *
6788knote_fops(struct knote *kn)
6789{
6790 return sysfilt_ops[kn->kn_filtid];
6791}
6792
6793static struct knote *
6794knote_alloc(void)
6795{
6796 struct knote *kn = ((struct knote *)zalloc(knote_zone));
6797 bzero(kn, sizeof(struct knote));
6798 return kn;
6799}
6800
6801static void
6802knote_free(struct knote *kn)
6803{
6804 assert((kn->kn_status & (KN_LOCKED | KN_POSTING)) == 0);
6805 zfree(knote_zone, kn);
6806}
6807
6808#pragma mark - syscalls: kevent, kevent64, kevent_qos, kevent_id
6809
6810kevent_ctx_t
6811kevent_get_context(thread_t thread)
6812{
6813 uthread_t ut = get_bsdthread_info(thread);
6814 return &ut->uu_save.uus_kevent;
6815}
6816
6817static inline bool
6818kevent_args_requesting_events(unsigned int flags, int nevents)
6819{
6820 return !(flags & KEVENT_FLAG_ERROR_EVENTS) && nevents > 0;
6821}
6822
6823static inline int
6824kevent_adjust_flags_for_proc(proc_t p, int flags)
6825{
6826 __builtin_assume(p);
6827 return flags | (IS_64BIT_PROCESS(p) ? KEVENT_FLAG_PROC64 : 0);
6828}
6829
6830/*!
6831 * @function kevent_get_kqfile
6832 *
6833 * @brief
6834 * Lookup a kqfile by fd.
6835 *
6836 * @discussion
6837 * Callers: kevent, kevent64, kevent_qos
6838 *
6839 * This is not assumed to be a fastpath (kqfile interfaces are legacy)
6840 */
6841OS_NOINLINE
6842static int
6843kevent_get_kqfile(struct proc *p, int fd, int flags,
6844 struct fileproc **fp, struct kqueue **kqp)
6845{
6846 int error = 0;
6847 struct kqueue *kq;
6848
6849 error = fp_getfkq(p, fd, fp, &kq);
6850 if (__improbable(error)) {
6851 return error;
6852 }
6853
6854 uint16_t kq_state = os_atomic_load(&kq->kq_state, relaxed);
6855 if (__improbable((kq_state & (KQ_KEV32 | KQ_KEV64 | KQ_KEV_QOS)) == 0)) {
6856 kqlock(kq);
6857 kq_state = kq->kq_state;
6858 if (!(kq_state & (KQ_KEV32 | KQ_KEV64 | KQ_KEV_QOS))) {
6859 if (flags & KEVENT_FLAG_LEGACY32) {
6860 kq_state |= KQ_KEV32;
6861 } else if (flags & KEVENT_FLAG_LEGACY64) {
6862 kq_state |= KQ_KEV64;
6863 } else {
6864 kq_state |= KQ_KEV_QOS;
6865 }
6866 kq->kq_state = kq_state;
6867 }
6868 kqunlock(kq);
6869 }
6870
6871 /*
6872 * kqfiles can't be used through the legacy kevent()
6873 * and other interfaces at the same time.
6874 */
6875 if (__improbable((bool)(flags & KEVENT_FLAG_LEGACY32) !=
6876 (bool)(kq_state & KQ_KEV32))) {
6877 fp_drop(p, fd, *fp, 0);
6878 return EINVAL;
6879 }
6880
6881 *kqp = kq;
6882 return 0;
6883}
6884
6885/*!
6886 * @function kevent_get_kqwq
6887 *
6888 * @brief
6889 * Lookup or create the process kqwq (faspath).
6890 *
6891 * @discussion
6892 * Callers: kevent64, kevent_qos
6893 */
6894OS_ALWAYS_INLINE
6895static int
6896kevent_get_kqwq(proc_t p, int flags, int nevents, struct kqueue **kqp)
6897{
6898 struct kqworkq *kqwq = p->p_fd->fd_wqkqueue;
6899
6900 if (__improbable(kevent_args_requesting_events(flags, nevents))) {
6901 return EINVAL;
6902 }
6903 if (__improbable(kqwq == NULL)) {
6904 kqwq = kqworkq_alloc(p, flags);
6905 if (__improbable(kqwq == NULL)) {
6906 return ENOMEM;
6907 }
6908 }
6909
6910 *kqp = &kqwq->kqwq_kqueue;
6911 return 0;
6912}
6913
6914#pragma mark kevent copyio
6915
6916/*!
6917 * @function kevent_get_data_size
6918 *
6919 * @brief
6920 * Copies in the extra data size from user-space.
6921 */
6922static int
6923kevent_get_data_size(int flags, user_addr_t data_avail, user_addr_t data_out,
6924 kevent_ctx_t kectx)
6925{
6926 if (!data_avail || !data_out) {
6927 kectx->kec_data_size = 0;
6928 kectx->kec_data_resid = 0;
6929 } else if (flags & KEVENT_FLAG_PROC64) {
6930 user64_size_t usize = 0;
6931 int error = copyin((user_addr_t)data_avail, &usize, sizeof(usize));
6932 if (__improbable(error)) {
6933 return error;
6934 }
6935 kectx->kec_data_resid = kectx->kec_data_size = (user_size_t)usize;
6936 } else {
6937 user32_size_t usize = 0;
6938 int error = copyin((user_addr_t)data_avail, &usize, sizeof(usize));
6939 if (__improbable(error)) {
6940 return error;
6941 }
6942 kectx->kec_data_avail = data_avail;
6943 kectx->kec_data_resid = kectx->kec_data_size = (user_size_t)usize;
6944 }
6945 kectx->kec_data_out = data_out;
6946 kectx->kec_data_avail = data_avail;
6947 return 0;
6948}
6949
6950/*!
6951 * @function kevent_put_data_size
6952 *
6953 * @brief
6954 * Copies out the residual data size to user-space if any has been used.
6955 */
6956static int
6957kevent_put_data_size(unsigned int flags, kevent_ctx_t kectx)
6958{
6959 if (kectx->kec_data_resid == kectx->kec_data_size) {
6960 return 0;
6961 }
6962 if (flags & KEVENT_FLAG_KERNEL) {
6963 *(user_size_t *)(uintptr_t)kectx->kec_data_avail = kectx->kec_data_resid;
6964 return 0;
6965 }
6966 if (flags & KEVENT_FLAG_PROC64) {
6967 user64_size_t usize = (user64_size_t)kectx->kec_data_resid;
6968 return copyout(&usize, (user_addr_t)kectx->kec_data_avail, sizeof(usize));
6969 } else {
6970 user32_size_t usize = (user32_size_t)kectx->kec_data_resid;
6971 return copyout(&usize, (user_addr_t)kectx->kec_data_avail, sizeof(usize));
6972 }
6973}
6974
6975/*!
6976 * @function kevent_legacy_copyin
6977 *
6978 * @brief
6979 * Handles the copyin of a kevent/kevent64 event.
6980 */
6981static int
6982kevent_legacy_copyin(user_addr_t *addrp, struct kevent_qos_s *kevp, unsigned int flags)
6983{
6984 int error;
6985
6986 assert((flags & (KEVENT_FLAG_LEGACY32 | KEVENT_FLAG_LEGACY64)) != 0);
6987
6988 if (flags & KEVENT_FLAG_LEGACY64) {
6989 struct kevent64_s kev64;
6990
6991 error = copyin(*addrp, (caddr_t)&kev64, sizeof(kev64));
6992 if (__improbable(error)) {
6993 return error;
6994 }
6995 *addrp += sizeof(kev64);
6996 *kevp = (struct kevent_qos_s){
6997 .ident = kev64.ident,
6998 .filter = kev64.filter,
6999 /* Make sure user doesn't pass in any system flags */
7000 .flags = kev64.flags & ~EV_SYSFLAGS,
7001 .udata = kev64.udata,
7002 .fflags = kev64.fflags,
7003 .data = kev64.data,
7004 .ext[0] = kev64.ext[0],
7005 .ext[1] = kev64.ext[1],
7006 };
7007 } else if (flags & KEVENT_FLAG_PROC64) {
7008 struct user64_kevent kev64;
7009
7010 error = copyin(*addrp, (caddr_t)&kev64, sizeof(kev64));
7011 if (__improbable(error)) {
7012 return error;
7013 }
7014 *addrp += sizeof(kev64);
7015 *kevp = (struct kevent_qos_s){
7016 .ident = kev64.ident,
7017 .filter = kev64.filter,
7018 /* Make sure user doesn't pass in any system flags */
7019 .flags = kev64.flags & ~EV_SYSFLAGS,
7020 .udata = kev64.udata,
7021 .fflags = kev64.fflags,
7022 .data = kev64.data,
7023 };
7024 } else {
7025 struct user32_kevent kev32;
7026
7027 error = copyin(*addrp, (caddr_t)&kev32, sizeof(kev32));
7028 if (__improbable(error)) {
7029 return error;
7030 }
7031 *addrp += sizeof(kev32);
7032 *kevp = (struct kevent_qos_s){
7033 .ident = (uintptr_t)kev32.ident,
7034 .filter = kev32.filter,
7035 /* Make sure user doesn't pass in any system flags */
7036 .flags = kev32.flags & ~EV_SYSFLAGS,
7037 .udata = CAST_USER_ADDR_T(kev32.udata),
7038 .fflags = kev32.fflags,
7039 .data = (intptr_t)kev32.data,
7040 };
7041 }
7042
7043 return 0;
7044}
7045
7046/*!
7047 * @function kevent_modern_copyin
7048 *
7049 * @brief
7050 * Handles the copyin of a kevent_qos/kevent_id event.
7051 */
7052static int
7053kevent_modern_copyin(user_addr_t *addrp, struct kevent_qos_s *kevp)
7054{
7055 int error = copyin(*addrp, (caddr_t)kevp, sizeof(struct kevent_qos_s));
7056 if (__probable(!error)) {
7057 /* Make sure user doesn't pass in any system flags */
7058 *addrp += sizeof(struct kevent_qos_s);
7059 kevp->flags &= ~EV_SYSFLAGS;
7060 }
7061 return error;
7062}
7063
7064/*!
7065 * @function kevent_legacy_copyout
7066 *
7067 * @brief
7068 * Handles the copyout of a kevent/kevent64 event.
7069 */
7070static int
7071kevent_legacy_copyout(struct kevent_qos_s *kevp, user_addr_t *addrp, unsigned int flags)
7072{
7073 int advance;
7074 int error;
7075
7076 assert((flags & (KEVENT_FLAG_LEGACY32 | KEVENT_FLAG_LEGACY64)) != 0);
7077
7078 /*
7079 * fully initialize the differnt output event structure
7080 * types from the internal kevent (and some universal
7081 * defaults for fields not represented in the internal
7082 * form).
7083 *
7084 * Note: these structures have no padding hence the C99
7085 * initializers below do not leak kernel info.
7086 */
7087 if (flags & KEVENT_FLAG_LEGACY64) {
7088 struct kevent64_s kev64 = {
7089 .ident = kevp->ident,
7090 .filter = kevp->filter,
7091 .flags = kevp->flags,
7092 .fflags = kevp->fflags,
7093 .data = (int64_t)kevp->data,
7094 .udata = kevp->udata,
7095 .ext[0] = kevp->ext[0],
7096 .ext[1] = kevp->ext[1],
7097 };
7098 advance = sizeof(struct kevent64_s);
7099 error = copyout((caddr_t)&kev64, *addrp, advance);
7100 } else if (flags & KEVENT_FLAG_PROC64) {
7101 /*
7102 * deal with the special case of a user-supplied
7103 * value of (uintptr_t)-1.
7104 */
7105 uint64_t ident = (kevp->ident == (uintptr_t)-1) ?
7106 (uint64_t)-1LL : (uint64_t)kevp->ident;
7107 struct user64_kevent kev64 = {
7108 .ident = ident,
7109 .filter = kevp->filter,
7110 .flags = kevp->flags,
7111 .fflags = kevp->fflags,
7112 .data = (int64_t) kevp->data,
7113 .udata = kevp->udata,
7114 };
7115 advance = sizeof(kev64);
7116 error = copyout((caddr_t)&kev64, *addrp, advance);
7117 } else {
7118 struct user32_kevent kev32 = {
7119 .ident = (uint32_t)kevp->ident,
7120 .filter = kevp->filter,
7121 .flags = kevp->flags,
7122 .fflags = kevp->fflags,
7123 .data = (int32_t)kevp->data,
7124 .udata = kevp->udata,
7125 };
7126 advance = sizeof(kev32);
7127 error = copyout((caddr_t)&kev32, *addrp, advance);
7128 }
7129 if (__probable(!error)) {
7130 *addrp += advance;
7131 }
7132 return error;
7133}
7134
7135/*!
7136 * @function kevent_modern_copyout
7137 *
7138 * @brief
7139 * Handles the copyout of a kevent_qos/kevent_id event.
7140 */
7141OS_ALWAYS_INLINE
7142static inline int
7143kevent_modern_copyout(struct kevent_qos_s *kevp, user_addr_t *addrp)
7144{
7145 int error = copyout((caddr_t)kevp, *addrp, sizeof(struct kevent_qos_s));
7146 if (__probable(!error)) {
7147 *addrp += sizeof(struct kevent_qos_s);
7148 }
7149 return error;
7150}
7151
7152#pragma mark kevent core implementation
7153
7154/*!
7155 * @function kevent_callback_inline
7156 *
7157 * @brief
7158 * Callback for each individual event
7159 *
7160 * @discussion
7161 * This is meant to be inlined in kevent_modern_callback and
7162 * kevent_legacy_callback.
7163 */
7164OS_ALWAYS_INLINE
7165static inline int
7166kevent_callback_inline(struct kevent_qos_s *kevp, kevent_ctx_t kectx, bool legacy)
7167{
7168 int error;
7169
7170 assert(kectx->kec_process_noutputs < kectx->kec_process_nevents);
7171
7172 /*
7173 * Copy out the appropriate amount of event data for this user.
7174 */
7175 if (legacy) {
7176 error = kevent_legacy_copyout(kevp, &kectx->kec_process_eventlist,
7177 kectx->kec_process_flags);
7178 } else {
7179 error = kevent_modern_copyout(kevp, &kectx->kec_process_eventlist);
7180 }
7181
7182 /*
7183 * If there isn't space for additional events, return
7184 * a harmless error to stop the processing here
7185 */
7186 if (error == 0 && ++kectx->kec_process_noutputs == kectx->kec_process_nevents) {
7187 error = EWOULDBLOCK;
7188 }
7189 return error;
7190}
7191
7192/*!
7193 * @function kevent_modern_callback
7194 *
7195 * @brief
7196 * Callback for each individual modern event.
7197 *
7198 * @discussion
7199 * This callback handles kevent_qos/kevent_id events.
7200 */
7201static int
7202kevent_modern_callback(struct kevent_qos_s *kevp, kevent_ctx_t kectx)
7203{
7204 return kevent_callback_inline(kevp, kectx, /*legacy*/ false);
7205}
7206
7207/*!
7208 * @function kevent_legacy_callback
7209 *
7210 * @brief
7211 * Callback for each individual legacy event.
7212 *
7213 * @discussion
7214 * This callback handles kevent/kevent64 events.
7215 */
7216static int
7217kevent_legacy_callback(struct kevent_qos_s *kevp, kevent_ctx_t kectx)
7218{
7219 return kevent_callback_inline(kevp, kectx, /*legacy*/ true);
7220}
7221
7222/*!
7223 * @function kevent_cleanup
7224 *
7225 * @brief
7226 * Handles the cleanup returning from a kevent call.
7227 *
7228 * @discussion
7229 * kevent entry points will take a reference on workloops,
7230 * and a usecount on the fileglob of kqfiles.
7231 *
7232 * This function undoes this on the exit paths of kevents.
7233 *
7234 * @returns
7235 * The error to return to userspace.
7236 */
7237static int
7238kevent_cleanup(kqueue_t kqu, int flags, int error, kevent_ctx_t kectx)
7239{
7240 // poll should not call any codepath leading to this
7241 assert((flags & KEVENT_FLAG_POLL) == 0);
7242
7243 if (flags & KEVENT_FLAG_WORKLOOP) {
7244 kqworkloop_release(kqu.kqwl);
7245 } else if (flags & KEVENT_FLAG_WORKQ) {
7246 /* nothing held */
7247 } else {
7248 fp_drop(kqu.kqf->kqf_p, kectx->kec_fd, kectx->kec_fp, 0);
7249 }
7250
7251 /* don't restart after signals... */
7252 if (error == ERESTART) {
7253 error = EINTR;
7254 } else if (error == 0) {
7255 /* don't abandon other output just because of residual copyout failures */
7256 (void)kevent_put_data_size(flags, kectx);
7257 }
7258
7259 if (flags & KEVENT_FLAG_PARKING) {
7260 thread_t th = current_thread();
7261 struct uthread *uth = get_bsdthread_info(th);
7262 if (uth->uu_kqr_bound) {
7263 thread_unfreeze_base_pri(th);
7264 }
7265 }
7266 return error;
7267}
7268
7269/*!
7270 * @function kqueue_process
7271 *
7272 * @brief
7273 * Process the triggered events in a kqueue.
7274 *
7275 * @discussion
7276 * Walk the queued knotes and validate that they are really still triggered
7277 * events by calling the filter routines (if necessary).
7278 *
7279 * For each event that is still considered triggered, invoke the callback
7280 * routine provided.
7281 *
7282 * caller holds a reference on the kqueue.
7283 * kqueue locked on entry and exit - but may be dropped
7284 * kqueue list locked (held for duration of call)
7285 *
7286 * This is only called by kqueue_scan() so that the compiler can inline it.
7287 *
7288 * @returns
7289 * - 0: no event was returned, no other error occured
7290 * - EBADF: the kqueue is being destroyed (KQ_DRAIN is set)
7291 * - EWOULDBLOCK: (not an error) events have been found and we should return
7292 * - EFAULT: copyout failed
7293 * - filter specific errors
7294 */
7295static int
7296kqueue_process(kqueue_t kqu, int flags, kevent_ctx_t kectx,
7297 kevent_callback_t callback)
7298{
7299 workq_threadreq_t kqr = current_uthread()->uu_kqr_bound;
7300 struct knote *kn;
7301 int error = 0, rc = 0;
7302 struct kqtailq *base_queue, *queue;
7303#if DEBUG || DEVELOPMENT
7304 int retries = 64;
7305#endif
7306 uint16_t kq_type = (kqu.kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP));
7307
7308 if (kq_type & KQ_WORKQ) {
7309 rc = kqworkq_begin_processing(kqu.kqwq, kqr, flags);
7310 } else if (kq_type & KQ_WORKLOOP) {
7311 rc = kqworkloop_begin_processing(kqu.kqwl, flags);
7312 } else {
7313kqfile_retry:
7314 rc = kqfile_begin_processing(kqu.kqf);
7315 if (rc == EBADF) {
7316 return EBADF;
7317 }
7318 }
7319
7320 if (rc == -1) {
7321 /* Nothing to process */
7322 return 0;
7323 }
7324
7325 /*
7326 * loop through the enqueued knotes associated with this request,
7327 * processing each one. Each request may have several queues
7328 * of knotes to process (depending on the type of kqueue) so we
7329 * have to loop through all the queues as long as we have additional
7330 * space.
7331 */
7332
7333process_again:
7334 if (kq_type & KQ_WORKQ) {
7335 base_queue = queue = &kqu.kqwq->kqwq_queue[kqr->tr_kq_qos_index];
7336 } else if (kq_type & KQ_WORKLOOP) {
7337 base_queue = &kqu.kqwl->kqwl_queue[0];
7338 queue = &kqu.kqwl->kqwl_queue[KQWL_NBUCKETS - 1];
7339 } else {
7340 base_queue = queue = &kqu.kqf->kqf_queue;
7341 }
7342
7343 do {
7344 while ((kn = TAILQ_FIRST(queue)) != NULL) {
7345 error = knote_process(kn, kectx, callback);
7346 if (error == EJUSTRETURN) {
7347 error = 0;
7348 } else if (__improbable(error)) {
7349 /* error is EWOULDBLOCK when the out event array is full */
7350 goto stop_processing;
7351 }
7352 }
7353 } while (queue-- > base_queue);
7354
7355 if (kectx->kec_process_noutputs) {
7356 /* callers will transform this into no error */
7357 error = EWOULDBLOCK;
7358 }
7359
7360stop_processing:
7361 /*
7362 * If KEVENT_FLAG_PARKING is set, and no kevents have been returned,
7363 * we want to unbind the kqrequest from the thread.
7364 *
7365 * However, because the kq locks are dropped several times during process,
7366 * new knotes may have fired again, in which case, we want to fail the end
7367 * processing and process again, until it converges.
7368 *
7369 * If we have an error or returned events, end processing never fails.
7370 */
7371 if (error) {
7372 flags &= ~KEVENT_FLAG_PARKING;
7373 }
7374 if (kq_type & KQ_WORKQ) {
7375 rc = kqworkq_end_processing(kqu.kqwq, kqr, flags);
7376 } else if (kq_type & KQ_WORKLOOP) {
7377 rc = kqworkloop_end_processing(kqu.kqwl, KQ_PROCESSING, flags);
7378 } else {
7379 rc = kqfile_end_processing(kqu.kqf);
7380 }
7381
7382 if (__probable(error)) {
7383 return error;
7384 }
7385
7386 if (__probable(rc >= 0)) {
7387 assert(rc == 0 || rc == EBADF);
7388 return rc;
7389 }
7390
7391#if DEBUG || DEVELOPMENT
7392 if (retries-- == 0) {
7393 panic("kevent: way too many knote_process retries, kq: %p (0x%04x)",
7394 kqu.kq, kqu.kq->kq_state);
7395 }
7396#endif
7397 if (kq_type & (KQ_WORKQ | KQ_WORKLOOP)) {
7398 assert(flags & KEVENT_FLAG_PARKING);
7399 goto process_again;
7400 } else {
7401 goto kqfile_retry;
7402 }
7403}
7404
7405/*!
7406 * @function kqueue_scan_continue
7407 *
7408 * @brief
7409 * The continuation used by kqueue_scan for kevent entry points.
7410 *
7411 * @discussion
7412 * Assumes we inherit a use/ref count on the kq or its fileglob.
7413 *
7414 * This is called by kqueue_scan if neither KEVENT_FLAG_POLL nor
7415 * KEVENT_FLAG_KERNEL was set, and the caller had to wait.
7416 */
7417OS_NORETURN OS_NOINLINE
7418static void
7419kqueue_scan_continue(void *data, wait_result_t wait_result)
7420{
7421 uthread_t ut = current_uthread();
7422 kevent_ctx_t kectx = &ut->uu_save.uus_kevent;
7423 int error = 0, flags = kectx->kec_process_flags;
7424 struct kqueue *kq = data;
7425
7426 /*
7427 * only kevent variants call in here, so we know the callback is
7428 * kevent_legacy_callback or kevent_modern_callback.
7429 */
7430 assert((flags & (KEVENT_FLAG_POLL | KEVENT_FLAG_KERNEL)) == 0);
7431
7432 switch (wait_result) {
7433 case THREAD_AWAKENED:
7434 if (__improbable(flags & (KEVENT_FLAG_LEGACY32 | KEVENT_FLAG_LEGACY64))) {
7435 error = kqueue_scan(kq, flags, kectx, kevent_legacy_callback);
7436 } else {
7437 error = kqueue_scan(kq, flags, kectx, kevent_modern_callback);
7438 }
7439 break;
7440 case THREAD_TIMED_OUT:
7441 error = 0;
7442 break;
7443 case THREAD_INTERRUPTED:
7444 error = EINTR;
7445 break;
7446 case THREAD_RESTART:
7447 error = EBADF;
7448 break;
7449 default:
7450 panic("%s: - invalid wait_result (%d)", __func__, wait_result);
7451 }
7452
7453
7454 error = kevent_cleanup(kq, flags, error, kectx);
7455 *(int32_t *)&ut->uu_rval = kectx->kec_process_noutputs;
7456 unix_syscall_return(error);
7457}
7458
7459/*!
7460 * @function kqueue_scan
7461 *
7462 * @brief
7463 * Scan and wait for events in a kqueue (used by poll & kevent).
7464 *
7465 * @discussion
7466 * Process the triggered events in a kqueue.
7467 *
7468 * If there are no events triggered arrange to wait for them:
7469 * - unless KEVENT_FLAG_IMMEDIATE is set in kectx->kec_process_flags
7470 * - possibly until kectx->kec_deadline expires
7471 *
7472 * When it waits, and that neither KEVENT_FLAG_POLL nor KEVENT_FLAG_KERNEL
7473 * are set, then it will wait in the kqueue_scan_continue continuation.
7474 *
7475 * poll() will block in place, and KEVENT_FLAG_KERNEL calls
7476 * all pass KEVENT_FLAG_IMMEDIATE and will not wait.
7477 *
7478 * @param kq
7479 * The kqueue being scanned.
7480 *
7481 * @param flags
7482 * The KEVENT_FLAG_* flags for this call.
7483 *
7484 * @param kectx
7485 * The context used for this scan.
7486 * The uthread_t::uu_save.uus_kevent storage is used for this purpose.
7487 *
7488 * @param callback
7489 * The callback to be called on events sucessfully processed.
7490 * (Either kevent_legacy_callback, kevent_modern_callback or poll_callback)
7491 */
7492int
7493kqueue_scan(struct kqueue *kq, int flags, kevent_ctx_t kectx,
7494 kevent_callback_t callback)
7495{
7496 int error;
7497
7498 for (;;) {
7499 kqlock(kq);
7500 error = kqueue_process(kq, flags, kectx, callback);
7501
7502 /*
7503 * If we got an error, events returned (EWOULDBLOCK)
7504 * or blocking was disallowed (KEVENT_FLAG_IMMEDIATE),
7505 * just return.
7506 */
7507 if (__probable(error || (flags & KEVENT_FLAG_IMMEDIATE))) {
7508 kqunlock(kq);
7509 return error == EWOULDBLOCK ? 0 : error;
7510 }
7511
7512 waitq_assert_wait64_leeway((struct waitq *)&kq->kq_wqs,
7513 KQ_EVENT, THREAD_ABORTSAFE, TIMEOUT_URGENCY_USER_NORMAL,
7514 kectx->kec_deadline, TIMEOUT_NO_LEEWAY);
7515 kq->kq_state |= KQ_SLEEP;
7516
7517 kqunlock(kq);
7518
7519 if (__probable((flags & (KEVENT_FLAG_POLL | KEVENT_FLAG_KERNEL)) == 0)) {
7520 thread_block_parameter(kqueue_scan_continue, kq);
7521 __builtin_unreachable();
7522 }
7523
7524 wait_result_t wr = thread_block(THREAD_CONTINUE_NULL);
7525 switch (wr) {
7526 case THREAD_AWAKENED:
7527 break;
7528 case THREAD_TIMED_OUT:
7529 return 0;
7530 case THREAD_INTERRUPTED:
7531 return EINTR;
7532 case THREAD_RESTART:
7533 return EBADF;
7534 default:
7535 panic("%s: - bad wait_result (%d)", __func__, wr);
7536 }
7537 }
7538}
7539
7540/*!
7541 * @function kevent_internal
7542 *
7543 * @brief
7544 * Common kevent code.
7545 *
7546 * @discussion
7547 * Needs to be inlined to specialize for legacy or modern and
7548 * eliminate dead code.
7549 *
7550 * This is the core logic of kevent entry points, that will:
7551 * - register kevents
7552 * - optionally scan the kqueue for events
7553 *
7554 * The caller is giving kevent_internal a reference on the kqueue
7555 * or its fileproc that needs to be cleaned up by kevent_cleanup().
7556 */
7557OS_ALWAYS_INLINE
7558static inline int
7559kevent_internal(kqueue_t kqu,
7560 user_addr_t changelist, int nchanges,
7561 user_addr_t ueventlist, int nevents,
7562 int flags, kevent_ctx_t kectx, int32_t *retval,
7563 bool legacy)
7564{
7565 int error = 0, noutputs = 0, register_rc;
7566
7567 /* only bound threads can receive events on workloops */
7568 if (!legacy && (flags & KEVENT_FLAG_WORKLOOP)) {
7569#if CONFIG_WORKLOOP_DEBUG
7570 UU_KEVENT_HISTORY_WRITE_ENTRY(current_uthread(), {
7571 .uu_kqid = kqu.kqwl->kqwl_dynamicid,
7572 .uu_kq = error ? NULL : kqu.kq,
7573 .uu_error = error,
7574 .uu_nchanges = nchanges,
7575 .uu_nevents = nevents,
7576 .uu_flags = flags,
7577 });
7578#endif // CONFIG_WORKLOOP_DEBUG
7579
7580 if (flags & KEVENT_FLAG_KERNEL) {
7581 /* see kevent_workq_internal */
7582 error = copyout(&kqu.kqwl->kqwl_dynamicid,
7583 ueventlist - sizeof(kqueue_id_t), sizeof(kqueue_id_t));
7584 kectx->kec_data_resid -= sizeof(kqueue_id_t);
7585 if (__improbable(error)) {
7586 goto out;
7587 }
7588 }
7589
7590 if (kevent_args_requesting_events(flags, nevents)) {
7591 /*
7592 * Disable the R2K notification while doing a register, if the
7593 * caller wants events too, we don't want the AST to be set if we
7594 * will process these events soon.
7595 */
7596 kqlock(kqu);
7597 kqu.kq->kq_state &= ~KQ_R2K_ARMED;
7598 kqunlock(kqu);
7599 flags |= KEVENT_FLAG_NEEDS_END_PROCESSING;
7600 }
7601 }
7602
7603 /* register all the change requests the user provided... */
7604 while (nchanges > 0 && error == 0) {
7605 struct kevent_qos_s kev;
7606 struct knote *kn = NULL;
7607
7608 if (legacy) {
7609 error = kevent_legacy_copyin(&changelist, &kev, flags);
7610 } else {
7611 error = kevent_modern_copyin(&changelist, &kev);
7612 }
7613 if (error) {
7614 break;
7615 }
7616
7617 register_rc = kevent_register(kqu.kq, &kev, &kn);
7618 if (__improbable(!legacy && (register_rc & FILTER_REGISTER_WAIT))) {
7619 thread_t thread = current_thread();
7620
7621 kqlock_held(kqu);
7622
7623 if (act_clear_astkevent(thread, AST_KEVENT_REDRIVE_THREADREQ)) {
7624 workq_kern_threadreq_redrive(kqu.kq->kq_p, WORKQ_THREADREQ_NONE);
7625 }
7626
7627 // f_post_register_wait is meant to call a continuation and not to
7628 // return, which is why we don't support FILTER_REGISTER_WAIT if
7629 // KEVENT_FLAG_ERROR_EVENTS is not passed, or if the event that
7630 // waits isn't the last.
7631 //
7632 // It is implementable, but not used by any userspace code at the
7633 // moment, so for now return ENOTSUP if someone tries to do it.
7634 if (nchanges == 1 && noutputs < nevents &&
7635 (flags & KEVENT_FLAG_KERNEL) == 0 &&
7636 (flags & KEVENT_FLAG_PARKING) == 0 &&
7637 (flags & KEVENT_FLAG_ERROR_EVENTS) &&
7638 (flags & KEVENT_FLAG_WORKLOOP)) {
7639 uthread_t ut = get_bsdthread_info(thread);
7640
7641 /*
7642 * store the continuation/completion data in the uthread
7643 *
7644 * Note: the kectx aliases with this,
7645 * and is destroyed in the process.
7646 */
7647 ut->uu_save.uus_kevent_register = (struct _kevent_register){
7648 .kev = kev,
7649 .kqwl = kqu.kqwl,
7650 .eventout = noutputs,
7651 .ueventlist = ueventlist,
7652 };
7653 knote_fops(kn)->f_post_register_wait(ut, kn,
7654 &ut->uu_save.uus_kevent_register);
7655 __builtin_unreachable();
7656 }
7657 kqunlock(kqu);
7658
7659 kev.flags |= EV_ERROR;
7660 kev.data = ENOTSUP;
7661 } else {
7662 assert((register_rc & FILTER_REGISTER_WAIT) == 0);
7663 }
7664
7665 // keep in sync with kevent_register_wait_return()
7666 if (noutputs < nevents && (kev.flags & (EV_ERROR | EV_RECEIPT))) {
7667 if ((kev.flags & EV_ERROR) == 0) {
7668 kev.flags |= EV_ERROR;
7669 kev.data = 0;
7670 }
7671 if (legacy) {
7672 error = kevent_legacy_copyout(&kev, &ueventlist, flags);
7673 } else {
7674 error = kevent_modern_copyout(&kev, &ueventlist);
7675 }
7676 if (error == 0) {
7677 noutputs++;
7678 }
7679 } else if (kev.flags & EV_ERROR) {
7680 error = kev.data;
7681 }
7682 nchanges--;
7683 }
7684
7685 if ((flags & KEVENT_FLAG_ERROR_EVENTS) == 0 &&
7686 nevents > 0 && noutputs == 0 && error == 0) {
7687 kectx->kec_process_flags = flags;
7688 kectx->kec_process_nevents = nevents;
7689 kectx->kec_process_noutputs = 0;
7690 kectx->kec_process_eventlist = ueventlist;
7691
7692 if (legacy) {
7693 error = kqueue_scan(kqu.kq, flags, kectx, kevent_legacy_callback);
7694 } else {
7695 error = kqueue_scan(kqu.kq, flags, kectx, kevent_modern_callback);
7696 }
7697
7698 noutputs = kectx->kec_process_noutputs;
7699 } else if (!legacy && (flags & KEVENT_FLAG_NEEDS_END_PROCESSING)) {
7700 /*
7701 * If we didn't through kqworkloop_end_processing(),
7702 * we need to do it here.
7703 *
7704 * kqueue_scan will call kqworkloop_end_processing(),
7705 * so we only need to do it if we didn't scan.
7706 */
7707 kqlock(kqu);
7708 kqworkloop_end_processing(kqu.kqwl, 0, 0);
7709 kqunlock(kqu);
7710 }
7711
7712 *retval = noutputs;
7713out:
7714 return kevent_cleanup(kqu.kq, flags, error, kectx);
7715}
7716
7717#pragma mark modern syscalls: kevent_qos, kevent_id, kevent_workq_internal
7718
7719/*!
7720 * @function kevent_modern_internal
7721 *
7722 * @brief
7723 * The backend of the kevent_id and kevent_workq_internal entry points.
7724 *
7725 * @discussion
7726 * Needs to be inline due to the number of arguments.
7727 */
7728OS_NOINLINE
7729static int
7730kevent_modern_internal(kqueue_t kqu,
7731 user_addr_t changelist, int nchanges,
7732 user_addr_t ueventlist, int nevents,
7733 int flags, kevent_ctx_t kectx, int32_t *retval)
7734{
7735 return kevent_internal(kqu.kq, changelist, nchanges,
7736 ueventlist, nevents, flags, kectx, retval, /*legacy*/ false);
7737}
7738
7739/*!
7740 * @function kevent_id
7741 *
7742 * @brief
7743 * The kevent_id() syscall.
7744 */
7745int
7746kevent_id(struct proc *p, struct kevent_id_args *uap, int32_t *retval)
7747{
7748 int error, flags = uap->flags & KEVENT_FLAG_USER;
7749 uthread_t uth = current_uthread();
7750 workq_threadreq_t kqr = uth->uu_kqr_bound;
7751 kevent_ctx_t kectx = &uth->uu_save.uus_kevent;
7752 kqueue_t kqu;
7753
7754 flags = kevent_adjust_flags_for_proc(p, flags);
7755 flags |= KEVENT_FLAG_DYNAMIC_KQUEUE;
7756
7757 if (__improbable((flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_WORKLOOP)) !=
7758 KEVENT_FLAG_WORKLOOP)) {
7759 return EINVAL;
7760 }
7761
7762 error = kevent_get_data_size(flags, uap->data_available, uap->data_out, kectx);
7763 if (__improbable(error)) {
7764 return error;
7765 }
7766
7767 kectx->kec_deadline = 0;
7768 kectx->kec_fp = NULL;
7769 kectx->kec_fd = -1;
7770 /* the kec_process_* fields are filled if kqueue_scann is called only */
7771
7772 /*
7773 * Get the kq we are going to be working on
7774 * As a fastpath, look at the currently bound workloop.
7775 */
7776 kqu.kqwl = kqr ? kqr_kqworkloop(kqr) : NULL;
7777 if (kqu.kqwl && kqu.kqwl->kqwl_dynamicid == uap->id) {
7778 if (__improbable(flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST)) {
7779 return EEXIST;
7780 }
7781 kqworkloop_retain(kqu.kqwl);
7782 } else if (__improbable(kevent_args_requesting_events(flags, uap->nevents))) {
7783 return EXDEV;
7784 } else {
7785 error = kqworkloop_get_or_create(p, uap->id, NULL, flags, &kqu.kqwl);
7786 if (__improbable(error)) {
7787 return error;
7788 }
7789 }
7790
7791 return kevent_modern_internal(kqu, uap->changelist, uap->nchanges,
7792 uap->eventlist, uap->nevents, flags, kectx, retval);
7793}
7794
7795/**!
7796 * @function kevent_workq_internal
7797 *
7798 * @discussion
7799 * This function is exported for the sake of the workqueue subsystem.
7800 *
7801 * It is called in two ways:
7802 * - when a thread is about to go to userspace to ask for pending event
7803 * - when a thread is returning from userspace with events back
7804 *
7805 * the workqueue subsystem will only use the following flags:
7806 * - KEVENT_FLAG_STACK_DATA (always)
7807 * - KEVENT_FLAG_IMMEDIATE (always)
7808 * - KEVENT_FLAG_PARKING (depending on whether it is going to or returning from
7809 * userspace).
7810 *
7811 * It implicitly acts on the bound kqueue, and for the case of workloops
7812 * will copyout the kqueue ID before anything else.
7813 *
7814 *
7815 * Pthread will have setup the various arguments to fit this stack layout:
7816 *
7817 * +-------....----+--------------+-----------+--------------------+
7818 * | user stack | data avail | nevents | pthread_self() |
7819 * +-------....----+--------------+-----------+--------------------+
7820 * ^ ^
7821 * data_out eventlist
7822 *
7823 * When a workloop is used, the workloop ID is copied out right before
7824 * the eventlist and is taken from the data buffer.
7825 *
7826 * @warning
7827 * This function is carefuly tailored to not make any call except the final tail
7828 * call into kevent_modern_internal. (LTO inlines current_uthread()).
7829 *
7830 * This function is performance sensitive due to the workq subsystem.
7831 */
7832int
7833kevent_workq_internal(struct proc *p,
7834 user_addr_t changelist, int nchanges,
7835 user_addr_t eventlist, int nevents,
7836 user_addr_t data_out, user_size_t *data_available,
7837 unsigned int flags, int32_t *retval)
7838{
7839 uthread_t uth = current_uthread();
7840 workq_threadreq_t kqr = uth->uu_kqr_bound;
7841 kevent_ctx_t kectx = &uth->uu_save.uus_kevent;
7842 kqueue_t kqu;
7843
7844 assert(flags == (KEVENT_FLAG_STACK_DATA | KEVENT_FLAG_IMMEDIATE) ||
7845 flags == (KEVENT_FLAG_STACK_DATA | KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_PARKING));
7846
7847 kectx->kec_data_out = data_out;
7848 kectx->kec_data_avail = (uint64_t)data_available;
7849 kectx->kec_data_size = *data_available;
7850 kectx->kec_data_resid = *data_available;
7851 kectx->kec_deadline = 0;
7852 kectx->kec_fp = NULL;
7853 kectx->kec_fd = -1;
7854 /* the kec_process_* fields are filled if kqueue_scann is called only */
7855
7856 flags = kevent_adjust_flags_for_proc(p, flags);
7857
7858 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
7859 kqu.kqwl = __container_of(kqr, struct kqworkloop, kqwl_request);
7860 kqworkloop_retain(kqu.kqwl);
7861
7862 flags |= KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_DYNAMIC_KQUEUE |
7863 KEVENT_FLAG_KERNEL;
7864 } else {
7865 kqu.kqwq = p->p_fd->fd_wqkqueue;
7866
7867 flags |= KEVENT_FLAG_WORKQ | KEVENT_FLAG_KERNEL;
7868 }
7869
7870 return kevent_modern_internal(kqu, changelist, nchanges,
7871 eventlist, nevents, flags, kectx, retval);
7872}
7873
7874/*!
7875 * @function kevent_qos
7876 *
7877 * @brief
7878 * The kevent_qos() syscall.
7879 */
7880int
7881kevent_qos(struct proc *p, struct kevent_qos_args *uap, int32_t *retval)
7882{
7883 uthread_t uth = current_uthread();
7884 kevent_ctx_t kectx = &uth->uu_save.uus_kevent;
7885 int error, flags = uap->flags & KEVENT_FLAG_USER;
7886 struct kqueue *kq;
7887
7888 if (__improbable(flags & KEVENT_ID_FLAG_USER)) {
7889 return EINVAL;
7890 }
7891
7892 flags = kevent_adjust_flags_for_proc(p, flags);
7893
7894 error = kevent_get_data_size(flags, uap->data_available, uap->data_out, kectx);
7895 if (__improbable(error)) {
7896 return error;
7897 }
7898
7899 kectx->kec_deadline = 0;
7900 kectx->kec_fp = NULL;
7901 kectx->kec_fd = uap->fd;
7902 /* the kec_process_* fields are filled if kqueue_scann is called only */
7903
7904 /* get the kq we are going to be working on */
7905 if (__probable(flags & KEVENT_FLAG_WORKQ)) {
7906 error = kevent_get_kqwq(p, flags, uap->nevents, &kq);
7907 } else {
7908 error = kevent_get_kqfile(p, uap->fd, flags, &kectx->kec_fp, &kq);
7909 }
7910 if (__improbable(error)) {
7911 return error;
7912 }
7913
7914 return kevent_modern_internal(kq, uap->changelist, uap->nchanges,
7915 uap->eventlist, uap->nevents, flags, kectx, retval);
7916}
7917
7918#pragma mark legacy syscalls: kevent, kevent64
7919
7920/*!
7921 * @function kevent_legacy_get_deadline
7922 *
7923 * @brief
7924 * Compute the deadline for the legacy kevent syscalls.
7925 *
7926 * @discussion
7927 * This is not necessary if KEVENT_FLAG_IMMEDIATE is specified,
7928 * as this takes precedence over the deadline.
7929 *
7930 * This function will fail if utimeout is USER_ADDR_NULL
7931 * (the caller should check).
7932 */
7933static int
7934kevent_legacy_get_deadline(int flags, user_addr_t utimeout, uint64_t *deadline)
7935{
7936 struct timespec ts;
7937
7938 if (flags & KEVENT_FLAG_PROC64) {
7939 struct user64_timespec ts64;
7940 int error = copyin(utimeout, &ts64, sizeof(ts64));
7941 if (__improbable(error)) {
7942 return error;
7943 }
7944 ts.tv_sec = ts64.tv_sec;
7945 ts.tv_nsec = ts64.tv_nsec;
7946 } else {
7947 struct user32_timespec ts32;
7948 int error = copyin(utimeout, &ts32, sizeof(ts32));
7949 if (__improbable(error)) {
7950 return error;
7951 }
7952 ts.tv_sec = ts32.tv_sec;
7953 ts.tv_nsec = ts32.tv_nsec;
7954 }
7955 if (!timespec_is_valid(&ts)) {
7956 return EINVAL;
7957 }
7958
7959 clock_absolutetime_interval_to_deadline(tstoabstime(&ts), deadline);
7960 return 0;
7961}
7962
7963/*!
7964 * @function kevent_legacy_internal
7965 *
7966 * @brief
7967 * The core implementation for kevent and kevent64
7968 */
7969OS_NOINLINE
7970static int
7971kevent_legacy_internal(struct proc *p, struct kevent64_args *uap,
7972 int32_t *retval, int flags)
7973{
7974 uthread_t uth = current_uthread();
7975 kevent_ctx_t kectx = &uth->uu_save.uus_kevent;
7976 struct kqueue *kq;
7977 int error;
7978
7979 if (__improbable(uap->flags & KEVENT_ID_FLAG_USER)) {
7980 return EINVAL;
7981 }
7982
7983 flags = kevent_adjust_flags_for_proc(p, flags);
7984
7985 kectx->kec_data_out = 0;
7986 kectx->kec_data_avail = 0;
7987 kectx->kec_data_size = 0;
7988 kectx->kec_data_resid = 0;
7989 kectx->kec_deadline = 0;
7990 kectx->kec_fp = NULL;
7991 kectx->kec_fd = uap->fd;
7992 /* the kec_process_* fields are filled if kqueue_scann is called only */
7993
7994 /* convert timeout to absolute - if we have one (and not immediate) */
7995 if (__improbable(uap->timeout && !(flags & KEVENT_FLAG_IMMEDIATE))) {
7996 error = kevent_legacy_get_deadline(flags, uap->timeout,
7997 &kectx->kec_deadline);
7998 if (__improbable(error)) {
7999 return error;
8000 }
8001 }
8002
8003 /* get the kq we are going to be working on */
8004 if (flags & KEVENT_FLAG_WORKQ) {
8005 error = kevent_get_kqwq(p, flags, uap->nevents, &kq);
8006 } else {
8007 error = kevent_get_kqfile(p, uap->fd, flags, &kectx->kec_fp, &kq);
8008 }
8009 if (__improbable(error)) {
8010 return error;
8011 }
8012
8013 return kevent_internal(kq, uap->changelist, uap->nchanges,
8014 uap->eventlist, uap->nevents, flags, kectx, retval,
8015 /*legacy*/ true);
8016}
8017
8018/*!
8019 * @function kevent
8020 *
8021 * @brief
8022 * The legacy kevent() syscall.
8023 */
8024int
8025kevent(struct proc *p, struct kevent_args *uap, int32_t *retval)
8026{
8027 struct kevent64_args args = {
8028 .fd = uap->fd,
8029 .changelist = uap->changelist,
8030 .nchanges = uap->nchanges,
8031 .eventlist = uap->eventlist,
8032 .nevents = uap->nevents,
8033 .timeout = uap->timeout,
8034 };
8035
8036 return kevent_legacy_internal(p, &args, retval, KEVENT_FLAG_LEGACY32);
8037}
8038
8039/*!
8040 * @function kevent64
8041 *
8042 * @brief
8043 * The legacy kevent64() syscall.
8044 */
8045int
8046kevent64(struct proc *p, struct kevent64_args *uap, int32_t *retval)
8047{
8048 int flags = (uap->flags & KEVENT_FLAG_USER) | KEVENT_FLAG_LEGACY64;
8049 return kevent_legacy_internal(p, uap, retval, flags);
8050}
8051
8052#pragma mark - socket interface
8053
8054#if SOCKETS
8055#include <sys/param.h>
8056#include <sys/socket.h>
8057#include <sys/protosw.h>
8058#include <sys/domain.h>
8059#include <sys/mbuf.h>
8060#include <sys/kern_event.h>
8061#include <sys/malloc.h>
8062#include <sys/sys_domain.h>
8063#include <sys/syslog.h>
8064
8065#ifndef ROUNDUP64
8066#define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t))
8067#endif
8068
8069#ifndef ADVANCE64
8070#define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n))
8071#endif
8072
8073static lck_grp_attr_t *kev_lck_grp_attr;
8074static lck_attr_t *kev_lck_attr;
8075static lck_grp_t *kev_lck_grp;
8076static decl_lck_rw_data(, kev_lck_data);
8077static lck_rw_t *kev_rwlock = &kev_lck_data;
8078
8079static int kev_attach(struct socket *so, int proto, struct proc *p);
8080static int kev_detach(struct socket *so);
8081static int kev_control(struct socket *so, u_long cmd, caddr_t data,
8082 struct ifnet *ifp, struct proc *p);
8083static lck_mtx_t * event_getlock(struct socket *, int);
8084static int event_lock(struct socket *, int, void *);
8085static int event_unlock(struct socket *, int, void *);
8086
8087static int event_sofreelastref(struct socket *);
8088static void kev_delete(struct kern_event_pcb *);
8089
8090static struct pr_usrreqs event_usrreqs = {
8091 .pru_attach = kev_attach,
8092 .pru_control = kev_control,
8093 .pru_detach = kev_detach,
8094 .pru_soreceive = soreceive,
8095};
8096
8097static struct protosw eventsw[] = {
8098 {
8099 .pr_type = SOCK_RAW,
8100 .pr_protocol = SYSPROTO_EVENT,
8101 .pr_flags = PR_ATOMIC,
8102 .pr_usrreqs = &event_usrreqs,
8103 .pr_lock = event_lock,
8104 .pr_unlock = event_unlock,
8105 .pr_getlock = event_getlock,
8106 }
8107};
8108
8109__private_extern__ int kevt_getstat SYSCTL_HANDLER_ARGS;
8110__private_extern__ int kevt_pcblist SYSCTL_HANDLER_ARGS;
8111
8112SYSCTL_NODE(_net_systm, OID_AUTO, kevt,
8113 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "Kernel event family");
8114
8115struct kevtstat kevtstat;
8116SYSCTL_PROC(_net_systm_kevt, OID_AUTO, stats,
8117 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
8118 kevt_getstat, "S,kevtstat", "");
8119
8120SYSCTL_PROC(_net_systm_kevt, OID_AUTO, pcblist,
8121 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
8122 kevt_pcblist, "S,xkevtpcb", "");
8123
8124static lck_mtx_t *
8125event_getlock(struct socket *so, int flags)
8126{
8127#pragma unused(flags)
8128 struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb;
8129
8130 if (so->so_pcb != NULL) {
8131 if (so->so_usecount < 0) {
8132 panic("%s: so=%p usecount=%d lrh= %s\n", __func__,
8133 so, so->so_usecount, solockhistory_nr(so));
8134 }
8135 /* NOTREACHED */
8136 } else {
8137 panic("%s: so=%p NULL NO so_pcb %s\n", __func__,
8138 so, solockhistory_nr(so));
8139 /* NOTREACHED */
8140 }
8141 return &ev_pcb->evp_mtx;
8142}
8143
8144static int
8145event_lock(struct socket *so, int refcount, void *lr)
8146{
8147 void *lr_saved;
8148
8149 if (lr == NULL) {
8150 lr_saved = __builtin_return_address(0);
8151 } else {
8152 lr_saved = lr;
8153 }
8154
8155 if (so->so_pcb != NULL) {
8156 lck_mtx_lock(&((struct kern_event_pcb *)so->so_pcb)->evp_mtx);
8157 } else {
8158 panic("%s: so=%p NO PCB! lr=%p lrh= %s\n", __func__,
8159 so, lr_saved, solockhistory_nr(so));
8160 /* NOTREACHED */
8161 }
8162
8163 if (so->so_usecount < 0) {
8164 panic("%s: so=%p so_pcb=%p lr=%p ref=%d lrh= %s\n", __func__,
8165 so, so->so_pcb, lr_saved, so->so_usecount,
8166 solockhistory_nr(so));
8167 /* NOTREACHED */
8168 }
8169
8170 if (refcount) {
8171 so->so_usecount++;
8172 }
8173
8174 so->lock_lr[so->next_lock_lr] = lr_saved;
8175 so->next_lock_lr = (so->next_lock_lr + 1) % SO_LCKDBG_MAX;
8176 return 0;
8177}
8178
8179static int
8180event_unlock(struct socket *so, int refcount, void *lr)
8181{
8182 void *lr_saved;
8183 lck_mtx_t *mutex_held;
8184
8185 if (lr == NULL) {
8186 lr_saved = __builtin_return_address(0);
8187 } else {
8188 lr_saved = lr;
8189 }
8190
8191 if (refcount) {
8192 so->so_usecount--;
8193 }
8194 if (so->so_usecount < 0) {
8195 panic("%s: so=%p usecount=%d lrh= %s\n", __func__,
8196 so, so->so_usecount, solockhistory_nr(so));
8197 /* NOTREACHED */
8198 }
8199 if (so->so_pcb == NULL) {
8200 panic("%s: so=%p NO PCB usecount=%d lr=%p lrh= %s\n", __func__,
8201 so, so->so_usecount, (void *)lr_saved,
8202 solockhistory_nr(so));
8203 /* NOTREACHED */
8204 }
8205 mutex_held = (&((struct kern_event_pcb *)so->so_pcb)->evp_mtx);
8206
8207 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
8208 so->unlock_lr[so->next_unlock_lr] = lr_saved;
8209 so->next_unlock_lr = (so->next_unlock_lr + 1) % SO_LCKDBG_MAX;
8210
8211 if (so->so_usecount == 0) {
8212 VERIFY(so->so_flags & SOF_PCBCLEARING);
8213 event_sofreelastref(so);
8214 } else {
8215 lck_mtx_unlock(mutex_held);
8216 }
8217
8218 return 0;
8219}
8220
8221static int
8222event_sofreelastref(struct socket *so)
8223{
8224 struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb;
8225
8226 LCK_MTX_ASSERT(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_OWNED);
8227
8228 so->so_pcb = NULL;
8229
8230 /*
8231 * Disable upcall in the event another thread is in kev_post_msg()
8232 * appending record to the receive socket buffer, since sbwakeup()
8233 * may release the socket lock otherwise.
8234 */
8235 so->so_rcv.sb_flags &= ~SB_UPCALL;
8236 so->so_snd.sb_flags &= ~SB_UPCALL;
8237 so->so_event = sonullevent;
8238 lck_mtx_unlock(&(ev_pcb->evp_mtx));
8239
8240 LCK_MTX_ASSERT(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_NOTOWNED);
8241 lck_rw_lock_exclusive(kev_rwlock);
8242 LIST_REMOVE(ev_pcb, evp_link);
8243 kevtstat.kes_pcbcount--;
8244 kevtstat.kes_gencnt++;
8245 lck_rw_done(kev_rwlock);
8246 kev_delete(ev_pcb);
8247
8248 sofreelastref(so, 1);
8249 return 0;
8250}
8251
8252static int event_proto_count = (sizeof(eventsw) / sizeof(struct protosw));
8253
8254static
8255struct kern_event_head kern_event_head;
8256
8257static u_int32_t static_event_id = 0;
8258
8259#define EVPCB_ZONE_MAX 65536
8260#define EVPCB_ZONE_NAME "kerneventpcb"
8261static struct zone *ev_pcb_zone;
8262
8263/*
8264 * Install the protosw's for the NKE manager. Invoked at extension load time
8265 */
8266void
8267kern_event_init(struct domain *dp)
8268{
8269 struct protosw *pr;
8270 int i;
8271
8272 VERIFY(!(dp->dom_flags & DOM_INITIALIZED));
8273 VERIFY(dp == systemdomain);
8274
8275 kev_lck_grp_attr = lck_grp_attr_alloc_init();
8276 if (kev_lck_grp_attr == NULL) {
8277 panic("%s: lck_grp_attr_alloc_init failed\n", __func__);
8278 /* NOTREACHED */
8279 }
8280
8281 kev_lck_grp = lck_grp_alloc_init("Kernel Event Protocol",
8282 kev_lck_grp_attr);
8283 if (kev_lck_grp == NULL) {
8284 panic("%s: lck_grp_alloc_init failed\n", __func__);
8285 /* NOTREACHED */
8286 }
8287
8288 kev_lck_attr = lck_attr_alloc_init();
8289 if (kev_lck_attr == NULL) {
8290 panic("%s: lck_attr_alloc_init failed\n", __func__);
8291 /* NOTREACHED */
8292 }
8293
8294 lck_rw_init(kev_rwlock, kev_lck_grp, kev_lck_attr);
8295 if (kev_rwlock == NULL) {
8296 panic("%s: lck_mtx_alloc_init failed\n", __func__);
8297 /* NOTREACHED */
8298 }
8299
8300 for (i = 0, pr = &eventsw[0]; i < event_proto_count; i++, pr++) {
8301 net_add_proto(pr, dp, 1);
8302 }
8303
8304 ev_pcb_zone = zinit(sizeof(struct kern_event_pcb),
8305 EVPCB_ZONE_MAX * sizeof(struct kern_event_pcb), 0, EVPCB_ZONE_NAME);
8306 if (ev_pcb_zone == NULL) {
8307 panic("%s: failed allocating ev_pcb_zone", __func__);
8308 /* NOTREACHED */
8309 }
8310 zone_change(ev_pcb_zone, Z_EXPAND, TRUE);
8311 zone_change(ev_pcb_zone, Z_CALLERACCT, TRUE);
8312}
8313
8314static int
8315kev_attach(struct socket *so, __unused int proto, __unused struct proc *p)
8316{
8317 int error = 0;
8318 struct kern_event_pcb *ev_pcb;
8319
8320 error = soreserve(so, KEV_SNDSPACE, KEV_RECVSPACE);
8321 if (error != 0) {
8322 return error;
8323 }
8324
8325 if ((ev_pcb = (struct kern_event_pcb *)zalloc(ev_pcb_zone)) == NULL) {
8326 return ENOBUFS;
8327 }
8328 bzero(ev_pcb, sizeof(struct kern_event_pcb));
8329 lck_mtx_init(&ev_pcb->evp_mtx, kev_lck_grp, kev_lck_attr);
8330
8331 ev_pcb->evp_socket = so;
8332 ev_pcb->evp_vendor_code_filter = 0xffffffff;
8333
8334 so->so_pcb = (caddr_t) ev_pcb;
8335 lck_rw_lock_exclusive(kev_rwlock);
8336 LIST_INSERT_HEAD(&kern_event_head, ev_pcb, evp_link);
8337 kevtstat.kes_pcbcount++;
8338 kevtstat.kes_gencnt++;
8339 lck_rw_done(kev_rwlock);
8340
8341 return error;
8342}
8343
8344static void
8345kev_delete(struct kern_event_pcb *ev_pcb)
8346{
8347 VERIFY(ev_pcb != NULL);
8348 lck_mtx_destroy(&ev_pcb->evp_mtx, kev_lck_grp);
8349 zfree(ev_pcb_zone, ev_pcb);
8350}
8351
8352static int
8353kev_detach(struct socket *so)
8354{
8355 struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *) so->so_pcb;
8356
8357 if (ev_pcb != NULL) {
8358 soisdisconnected(so);
8359 so->so_flags |= SOF_PCBCLEARING;
8360 }
8361
8362 return 0;
8363}
8364
8365/*
8366 * For now, kev_vendor_code and mbuf_tags use the same
8367 * mechanism.
8368 */
8369errno_t
8370kev_vendor_code_find(
8371 const char *string,
8372 u_int32_t *out_vendor_code)
8373{
8374 if (strlen(string) >= KEV_VENDOR_CODE_MAX_STR_LEN) {
8375 return EINVAL;
8376 }
8377 return net_str_id_find_internal(string, out_vendor_code,
8378 NSI_VENDOR_CODE, 1);
8379}
8380
8381errno_t
8382kev_msg_post(struct kev_msg *event_msg)
8383{
8384 mbuf_tag_id_t min_vendor, max_vendor;
8385
8386 net_str_id_first_last(&min_vendor, &max_vendor, NSI_VENDOR_CODE);
8387
8388 if (event_msg == NULL) {
8389 return EINVAL;
8390 }
8391
8392 /*
8393 * Limit third parties to posting events for registered vendor codes
8394 * only
8395 */
8396 if (event_msg->vendor_code < min_vendor ||
8397 event_msg->vendor_code > max_vendor) {
8398 os_atomic_inc(&kevtstat.kes_badvendor, relaxed);
8399 return EINVAL;
8400 }
8401 return kev_post_msg(event_msg);
8402}
8403
8404int
8405kev_post_msg(struct kev_msg *event_msg)
8406{
8407 struct mbuf *m, *m2;
8408 struct kern_event_pcb *ev_pcb;
8409 struct kern_event_msg *ev;
8410 char *tmp;
8411 u_int32_t total_size;
8412 int i;
8413
8414 /* Verify the message is small enough to fit in one mbuf w/o cluster */
8415 total_size = KEV_MSG_HEADER_SIZE;
8416
8417 for (i = 0; i < 5; i++) {
8418 if (event_msg->dv[i].data_length == 0) {
8419 break;
8420 }
8421 total_size += event_msg->dv[i].data_length;
8422 }
8423
8424 if (total_size > MLEN) {
8425 os_atomic_inc(&kevtstat.kes_toobig, relaxed);
8426 return EMSGSIZE;
8427 }
8428
8429 m = m_get(M_WAIT, MT_DATA);
8430 if (m == 0) {
8431 os_atomic_inc(&kevtstat.kes_nomem, relaxed);
8432 return ENOMEM;
8433 }
8434 ev = mtod(m, struct kern_event_msg *);
8435 total_size = KEV_MSG_HEADER_SIZE;
8436
8437 tmp = (char *) &ev->event_data[0];
8438 for (i = 0; i < 5; i++) {
8439 if (event_msg->dv[i].data_length == 0) {
8440 break;
8441 }
8442
8443 total_size += event_msg->dv[i].data_length;
8444 bcopy(event_msg->dv[i].data_ptr, tmp,
8445 event_msg->dv[i].data_length);
8446 tmp += event_msg->dv[i].data_length;
8447 }
8448
8449 ev->id = ++static_event_id;
8450 ev->total_size = total_size;
8451 ev->vendor_code = event_msg->vendor_code;
8452 ev->kev_class = event_msg->kev_class;
8453 ev->kev_subclass = event_msg->kev_subclass;
8454 ev->event_code = event_msg->event_code;
8455
8456 m->m_len = total_size;
8457 lck_rw_lock_shared(kev_rwlock);
8458 for (ev_pcb = LIST_FIRST(&kern_event_head);
8459 ev_pcb;
8460 ev_pcb = LIST_NEXT(ev_pcb, evp_link)) {
8461 lck_mtx_lock(&ev_pcb->evp_mtx);
8462 if (ev_pcb->evp_socket->so_pcb == NULL) {
8463 lck_mtx_unlock(&ev_pcb->evp_mtx);
8464 continue;
8465 }
8466 if (ev_pcb->evp_vendor_code_filter != KEV_ANY_VENDOR) {
8467 if (ev_pcb->evp_vendor_code_filter != ev->vendor_code) {
8468 lck_mtx_unlock(&ev_pcb->evp_mtx);
8469 continue;
8470 }
8471
8472 if (ev_pcb->evp_class_filter != KEV_ANY_CLASS) {
8473 if (ev_pcb->evp_class_filter != ev->kev_class) {
8474 lck_mtx_unlock(&ev_pcb->evp_mtx);
8475 continue;
8476 }
8477
8478 if ((ev_pcb->evp_subclass_filter !=
8479 KEV_ANY_SUBCLASS) &&
8480 (ev_pcb->evp_subclass_filter !=
8481 ev->kev_subclass)) {
8482 lck_mtx_unlock(&ev_pcb->evp_mtx);
8483 continue;
8484 }
8485 }
8486 }
8487
8488 m2 = m_copym(m, 0, m->m_len, M_WAIT);
8489 if (m2 == 0) {
8490 os_atomic_inc(&kevtstat.kes_nomem, relaxed);
8491 m_free(m);
8492 lck_mtx_unlock(&ev_pcb->evp_mtx);
8493 lck_rw_done(kev_rwlock);
8494 return ENOMEM;
8495 }
8496 if (sbappendrecord(&ev_pcb->evp_socket->so_rcv, m2)) {
8497 /*
8498 * We use "m" for the socket stats as it would be
8499 * unsafe to use "m2"
8500 */
8501 so_inc_recv_data_stat(ev_pcb->evp_socket,
8502 1, m->m_len, MBUF_TC_BE);
8503
8504 sorwakeup(ev_pcb->evp_socket);
8505 os_atomic_inc(&kevtstat.kes_posted, relaxed);
8506 } else {
8507 os_atomic_inc(&kevtstat.kes_fullsock, relaxed);
8508 }
8509 lck_mtx_unlock(&ev_pcb->evp_mtx);
8510 }
8511 m_free(m);
8512 lck_rw_done(kev_rwlock);
8513
8514 return 0;
8515}
8516
8517static int
8518kev_control(struct socket *so,
8519 u_long cmd,
8520 caddr_t data,
8521 __unused struct ifnet *ifp,
8522 __unused struct proc *p)
8523{
8524 struct kev_request *kev_req = (struct kev_request *) data;
8525 struct kern_event_pcb *ev_pcb;
8526 struct kev_vendor_code *kev_vendor;
8527 u_int32_t *id_value = (u_int32_t *) data;
8528
8529 switch (cmd) {
8530 case SIOCGKEVID:
8531 *id_value = static_event_id;
8532 break;
8533 case SIOCSKEVFILT:
8534 ev_pcb = (struct kern_event_pcb *) so->so_pcb;
8535 ev_pcb->evp_vendor_code_filter = kev_req->vendor_code;
8536 ev_pcb->evp_class_filter = kev_req->kev_class;
8537 ev_pcb->evp_subclass_filter = kev_req->kev_subclass;
8538 break;
8539 case SIOCGKEVFILT:
8540 ev_pcb = (struct kern_event_pcb *) so->so_pcb;
8541 kev_req->vendor_code = ev_pcb->evp_vendor_code_filter;
8542 kev_req->kev_class = ev_pcb->evp_class_filter;
8543 kev_req->kev_subclass = ev_pcb->evp_subclass_filter;
8544 break;
8545 case SIOCGKEVVENDOR:
8546 kev_vendor = (struct kev_vendor_code *)data;
8547 /* Make sure string is NULL terminated */
8548 kev_vendor->vendor_string[KEV_VENDOR_CODE_MAX_STR_LEN - 1] = 0;
8549 return net_str_id_find_internal(kev_vendor->vendor_string,
8550 &kev_vendor->vendor_code, NSI_VENDOR_CODE, 0);
8551 default:
8552 return ENOTSUP;
8553 }
8554
8555 return 0;
8556}
8557
8558int
8559kevt_getstat SYSCTL_HANDLER_ARGS
8560{
8561#pragma unused(oidp, arg1, arg2)
8562 int error = 0;
8563
8564 lck_rw_lock_shared(kev_rwlock);
8565
8566 if (req->newptr != USER_ADDR_NULL) {
8567 error = EPERM;
8568 goto done;
8569 }
8570 if (req->oldptr == USER_ADDR_NULL) {
8571 req->oldidx = sizeof(struct kevtstat);
8572 goto done;
8573 }
8574
8575 error = SYSCTL_OUT(req, &kevtstat,
8576 MIN(sizeof(struct kevtstat), req->oldlen));
8577done:
8578 lck_rw_done(kev_rwlock);
8579
8580 return error;
8581}
8582
8583__private_extern__ int
8584kevt_pcblist SYSCTL_HANDLER_ARGS
8585{
8586#pragma unused(oidp, arg1, arg2)
8587 int error = 0;
8588 int n, i;
8589 struct xsystmgen xsg;
8590 void *buf = NULL;
8591 size_t item_size = ROUNDUP64(sizeof(struct xkevtpcb)) +
8592 ROUNDUP64(sizeof(struct xsocket_n)) +
8593 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) +
8594 ROUNDUP64(sizeof(struct xsockstat_n));
8595 struct kern_event_pcb *ev_pcb;
8596
8597 buf = _MALLOC(item_size, M_TEMP, M_WAITOK | M_ZERO);
8598 if (buf == NULL) {
8599 return ENOMEM;
8600 }
8601
8602 lck_rw_lock_shared(kev_rwlock);
8603
8604 n = kevtstat.kes_pcbcount;
8605
8606 if (req->oldptr == USER_ADDR_NULL) {
8607 req->oldidx = (n + n / 8) * item_size;
8608 goto done;
8609 }
8610 if (req->newptr != USER_ADDR_NULL) {
8611 error = EPERM;
8612 goto done;
8613 }
8614 bzero(&xsg, sizeof(xsg));
8615 xsg.xg_len = sizeof(xsg);
8616 xsg.xg_count = n;
8617 xsg.xg_gen = kevtstat.kes_gencnt;
8618 xsg.xg_sogen = so_gencnt;
8619 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
8620 if (error) {
8621 goto done;
8622 }
8623 /*
8624 * We are done if there is no pcb
8625 */
8626 if (n == 0) {
8627 goto done;
8628 }
8629
8630 i = 0;
8631 for (i = 0, ev_pcb = LIST_FIRST(&kern_event_head);
8632 i < n && ev_pcb != NULL;
8633 i++, ev_pcb = LIST_NEXT(ev_pcb, evp_link)) {
8634 struct xkevtpcb *xk = (struct xkevtpcb *)buf;
8635 struct xsocket_n *xso = (struct xsocket_n *)
8636 ADVANCE64(xk, sizeof(*xk));
8637 struct xsockbuf_n *xsbrcv = (struct xsockbuf_n *)
8638 ADVANCE64(xso, sizeof(*xso));
8639 struct xsockbuf_n *xsbsnd = (struct xsockbuf_n *)
8640 ADVANCE64(xsbrcv, sizeof(*xsbrcv));
8641 struct xsockstat_n *xsostats = (struct xsockstat_n *)
8642 ADVANCE64(xsbsnd, sizeof(*xsbsnd));
8643
8644 bzero(buf, item_size);
8645
8646 lck_mtx_lock(&ev_pcb->evp_mtx);
8647
8648 xk->kep_len = sizeof(struct xkevtpcb);
8649 xk->kep_kind = XSO_EVT;
8650 xk->kep_evtpcb = (uint64_t)VM_KERNEL_ADDRPERM(ev_pcb);
8651 xk->kep_vendor_code_filter = ev_pcb->evp_vendor_code_filter;
8652 xk->kep_class_filter = ev_pcb->evp_class_filter;
8653 xk->kep_subclass_filter = ev_pcb->evp_subclass_filter;
8654
8655 sotoxsocket_n(ev_pcb->evp_socket, xso);
8656 sbtoxsockbuf_n(ev_pcb->evp_socket ?
8657 &ev_pcb->evp_socket->so_rcv : NULL, xsbrcv);
8658 sbtoxsockbuf_n(ev_pcb->evp_socket ?
8659 &ev_pcb->evp_socket->so_snd : NULL, xsbsnd);
8660 sbtoxsockstat_n(ev_pcb->evp_socket, xsostats);
8661
8662 lck_mtx_unlock(&ev_pcb->evp_mtx);
8663
8664 error = SYSCTL_OUT(req, buf, item_size);
8665 }
8666
8667 if (error == 0) {
8668 /*
8669 * Give the user an updated idea of our state.
8670 * If the generation differs from what we told
8671 * her before, she knows that something happened
8672 * while we were processing this request, and it
8673 * might be necessary to retry.
8674 */
8675 bzero(&xsg, sizeof(xsg));
8676 xsg.xg_len = sizeof(xsg);
8677 xsg.xg_count = n;
8678 xsg.xg_gen = kevtstat.kes_gencnt;
8679 xsg.xg_sogen = so_gencnt;
8680 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
8681 if (error) {
8682 goto done;
8683 }
8684 }
8685
8686done:
8687 lck_rw_done(kev_rwlock);
8688
8689 return error;
8690}
8691
8692#endif /* SOCKETS */
8693
8694
8695int
8696fill_kqueueinfo(struct kqueue *kq, struct kqueue_info * kinfo)
8697{
8698 struct vinfo_stat * st;
8699
8700 st = &kinfo->kq_stat;
8701
8702 st->vst_size = kq->kq_count;
8703 if (kq->kq_state & KQ_KEV_QOS) {
8704 st->vst_blksize = sizeof(struct kevent_qos_s);
8705 } else if (kq->kq_state & KQ_KEV64) {
8706 st->vst_blksize = sizeof(struct kevent64_s);
8707 } else {
8708 st->vst_blksize = sizeof(struct kevent);
8709 }
8710 st->vst_mode = S_IFIFO;
8711 st->vst_ino = (kq->kq_state & KQ_DYNAMIC) ?
8712 ((struct kqworkloop *)kq)->kqwl_dynamicid : 0;
8713
8714 /* flags exported to libproc as PROC_KQUEUE_* (sys/proc_info.h) */
8715#define PROC_KQUEUE_MASK (KQ_SEL|KQ_SLEEP|KQ_KEV32|KQ_KEV64|KQ_KEV_QOS|KQ_WORKQ|KQ_WORKLOOP)
8716 kinfo->kq_state = kq->kq_state & PROC_KQUEUE_MASK;
8717
8718 return 0;
8719}
8720
8721static int
8722fill_kqueue_dyninfo(struct kqworkloop *kqwl, struct kqueue_dyninfo *kqdi)
8723{
8724 workq_threadreq_t kqr = &kqwl->kqwl_request;
8725 workq_threadreq_param_t trp = {};
8726 int err;
8727
8728 if ((kqwl->kqwl_state & KQ_WORKLOOP) == 0) {
8729 return EINVAL;
8730 }
8731
8732 if ((err = fill_kqueueinfo(&kqwl->kqwl_kqueue, &kqdi->kqdi_info))) {
8733 return err;
8734 }
8735
8736 kqlock(kqwl);
8737
8738 kqdi->kqdi_servicer = thread_tid(kqr_thread(kqr));
8739 kqdi->kqdi_owner = thread_tid(kqwl->kqwl_owner);
8740 kqdi->kqdi_request_state = kqr->tr_state;
8741 kqdi->kqdi_async_qos = kqr->tr_kq_qos_index;
8742 kqdi->kqdi_events_qos = kqr->tr_kq_override_index;
8743 kqdi->kqdi_sync_waiters = 0;
8744 kqdi->kqdi_sync_waiter_qos = 0;
8745
8746 trp.trp_value = kqwl->kqwl_params;
8747 if (trp.trp_flags & TRP_PRIORITY) {
8748 kqdi->kqdi_pri = trp.trp_pri;
8749 } else {
8750 kqdi->kqdi_pri = 0;
8751 }
8752
8753 if (trp.trp_flags & TRP_POLICY) {
8754 kqdi->kqdi_pol = trp.trp_pol;
8755 } else {
8756 kqdi->kqdi_pol = 0;
8757 }
8758
8759 if (trp.trp_flags & TRP_CPUPERCENT) {
8760 kqdi->kqdi_cpupercent = trp.trp_cpupercent;
8761 } else {
8762 kqdi->kqdi_cpupercent = 0;
8763 }
8764
8765 kqunlock(kqwl);
8766
8767 return 0;
8768}
8769
8770
8771void
8772knote_markstayactive(struct knote *kn)
8773{
8774 struct kqueue *kq = knote_get_kq(kn);
8775 kq_index_t qos;
8776
8777 kqlock(kq);
8778 kn->kn_status |= KN_STAYACTIVE;
8779
8780 /*
8781 * Making a knote stay active is a property of the knote that must be
8782 * established before it is fully attached.
8783 */
8784 assert((kn->kn_status & (KN_QUEUED | KN_SUPPRESSED)) == 0);
8785
8786 /* handle all stayactive knotes on the (appropriate) manager */
8787 if (kq->kq_state & KQ_WORKLOOP) {
8788 struct kqworkloop *kqwl = (struct kqworkloop *)kq;
8789
8790 qos = _pthread_priority_thread_qos(kn->kn_qos);
8791 assert(qos && qos < THREAD_QOS_LAST);
8792 kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_STAYACTIVE_QOS, qos);
8793 qos = KQWL_BUCKET_STAYACTIVE;
8794 } else if (kq->kq_state & KQ_WORKQ) {
8795 qos = KQWQ_QOS_MANAGER;
8796 } else {
8797 qos = THREAD_QOS_UNSPECIFIED;
8798 }
8799
8800 kn->kn_qos_override = qos;
8801 kn->kn_qos_index = qos;
8802
8803 knote_activate(kq, kn, FILTER_ACTIVE);
8804 kqunlock(kq);
8805}
8806
8807void
8808knote_clearstayactive(struct knote *kn)
8809{
8810 struct kqueue *kq = knote_get_kq(kn);
8811 kqlock(kq);
8812 kn->kn_status &= ~(KN_STAYACTIVE | KN_ACTIVE);
8813 knote_dequeue(kq, kn);
8814 kqunlock(kq);
8815}
8816
8817static unsigned long
8818kevent_extinfo_emit(struct kqueue *kq, struct knote *kn, struct kevent_extinfo *buf,
8819 unsigned long buflen, unsigned long nknotes)
8820{
8821 for (; kn; kn = SLIST_NEXT(kn, kn_link)) {
8822 if (kq == knote_get_kq(kn)) {
8823 if (nknotes < buflen) {
8824 struct kevent_extinfo *info = &buf[nknotes];
8825
8826 kqlock(kq);
8827
8828 info->kqext_kev = *(struct kevent_qos_s *)&kn->kn_kevent;
8829 if (knote_has_qos(kn)) {
8830 info->kqext_kev.qos =
8831 _pthread_priority_thread_qos_fast(kn->kn_qos);
8832 } else {
8833 info->kqext_kev.qos = kn->kn_qos_override;
8834 }
8835 info->kqext_kev.filter |= 0xff00; /* sign extend filter */
8836 info->kqext_kev.xflags = 0; /* this is where sfflags lives */
8837 info->kqext_kev.data = 0; /* this is where sdata lives */
8838 info->kqext_sdata = kn->kn_sdata;
8839 info->kqext_status = kn->kn_status;
8840 info->kqext_sfflags = kn->kn_sfflags;
8841
8842 kqunlock(kq);
8843 }
8844
8845 /* we return total number of knotes, which may be more than requested */
8846 nknotes++;
8847 }
8848 }
8849
8850 return nknotes;
8851}
8852
8853int
8854kevent_copyout_proc_dynkqids(void *proc, user_addr_t ubuf, uint32_t ubufsize,
8855 int32_t *nkqueues_out)
8856{
8857 proc_t p = (proc_t)proc;
8858 struct filedesc *fdp = p->p_fd;
8859 unsigned int nkqueues = 0;
8860 unsigned long ubuflen = ubufsize / sizeof(kqueue_id_t);
8861 size_t buflen, bufsize;
8862 kqueue_id_t *kq_ids = NULL;
8863 int err = 0;
8864
8865 assert(p != NULL);
8866
8867 if (ubuf == USER_ADDR_NULL && ubufsize != 0) {
8868 err = EINVAL;
8869 goto out;
8870 }
8871
8872 buflen = min(ubuflen, PROC_PIDDYNKQUEUES_MAX);
8873
8874 if (ubuflen != 0) {
8875 if (os_mul_overflow(sizeof(kqueue_id_t), buflen, &bufsize)) {
8876 err = ERANGE;
8877 goto out;
8878 }
8879 kq_ids = kalloc(bufsize);
8880 if (!kq_ids) {
8881 err = ENOMEM;
8882 goto out;
8883 }
8884 bzero(kq_ids, bufsize);
8885 }
8886
8887 kqhash_lock(fdp);
8888
8889 if (fdp->fd_kqhashmask > 0) {
8890 for (uint32_t i = 0; i < fdp->fd_kqhashmask + 1; i++) {
8891 struct kqworkloop *kqwl;
8892
8893 LIST_FOREACH(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink) {
8894 /* report the number of kqueues, even if they don't all fit */
8895 if (nkqueues < buflen) {
8896 kq_ids[nkqueues] = kqwl->kqwl_dynamicid;
8897 }
8898 nkqueues++;
8899 }
8900 }
8901 }
8902
8903 kqhash_unlock(fdp);
8904
8905 if (kq_ids) {
8906 size_t copysize;
8907 if (os_mul_overflow(sizeof(kqueue_id_t), min(buflen, nkqueues), &copysize)) {
8908 err = ERANGE;
8909 goto out;
8910 }
8911
8912 assert(ubufsize >= copysize);
8913 err = copyout(kq_ids, ubuf, copysize);
8914 }
8915
8916out:
8917 if (kq_ids) {
8918 kfree(kq_ids, bufsize);
8919 }
8920
8921 if (!err) {
8922 *nkqueues_out = (int)min(nkqueues, PROC_PIDDYNKQUEUES_MAX);
8923 }
8924 return err;
8925}
8926
8927int
8928kevent_copyout_dynkqinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf,
8929 uint32_t ubufsize, int32_t *size_out)
8930{
8931 proc_t p = (proc_t)proc;
8932 struct kqworkloop *kqwl;
8933 int err = 0;
8934 struct kqueue_dyninfo kqdi = { };
8935
8936 assert(p != NULL);
8937
8938 if (ubufsize < sizeof(struct kqueue_info)) {
8939 return ENOBUFS;
8940 }
8941
8942 kqwl = kqworkloop_hash_lookup_and_retain(p->p_fd, kq_id);
8943 if (!kqwl) {
8944 return ESRCH;
8945 }
8946
8947 /*
8948 * backward compatibility: allow the argument to this call to only be
8949 * a struct kqueue_info
8950 */
8951 if (ubufsize >= sizeof(struct kqueue_dyninfo)) {
8952 ubufsize = sizeof(struct kqueue_dyninfo);
8953 err = fill_kqueue_dyninfo(kqwl, &kqdi);
8954 } else {
8955 ubufsize = sizeof(struct kqueue_info);
8956 err = fill_kqueueinfo(&kqwl->kqwl_kqueue, &kqdi.kqdi_info);
8957 }
8958 if (err == 0 && (err = copyout(&kqdi, ubuf, ubufsize)) == 0) {
8959 *size_out = ubufsize;
8960 }
8961 kqworkloop_release(kqwl);
8962 return err;
8963}
8964
8965int
8966kevent_copyout_dynkqextinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf,
8967 uint32_t ubufsize, int32_t *nknotes_out)
8968{
8969 proc_t p = (proc_t)proc;
8970 struct kqworkloop *kqwl;
8971 int err;
8972
8973 kqwl = kqworkloop_hash_lookup_and_retain(p->p_fd, kq_id);
8974 if (!kqwl) {
8975 return ESRCH;
8976 }
8977
8978 err = pid_kqueue_extinfo(p, &kqwl->kqwl_kqueue, ubuf, ubufsize, nknotes_out);
8979 kqworkloop_release(kqwl);
8980 return err;
8981}
8982
8983int
8984pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf,
8985 uint32_t bufsize, int32_t *retval)
8986{
8987 struct knote *kn;
8988 int i;
8989 int err = 0;
8990 struct filedesc *fdp = p->p_fd;
8991 unsigned long nknotes = 0;
8992 unsigned long buflen = bufsize / sizeof(struct kevent_extinfo);
8993 struct kevent_extinfo *kqext = NULL;
8994
8995 /* arbitrary upper limit to cap kernel memory usage, copyout size, etc. */
8996 buflen = min(buflen, PROC_PIDFDKQUEUE_KNOTES_MAX);
8997
8998 kqext = kalloc(buflen * sizeof(struct kevent_extinfo));
8999 if (kqext == NULL) {
9000 err = ENOMEM;
9001 goto out;
9002 }
9003 bzero(kqext, buflen * sizeof(struct kevent_extinfo));
9004
9005 proc_fdlock(p);
9006 for (i = 0; i < fdp->fd_knlistsize; i++) {
9007 kn = SLIST_FIRST(&fdp->fd_knlist[i]);
9008 nknotes = kevent_extinfo_emit(kq, kn, kqext, buflen, nknotes);
9009 }
9010 proc_fdunlock(p);
9011
9012 if (fdp->fd_knhashmask != 0) {
9013 for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) {
9014 knhash_lock(fdp);
9015 kn = SLIST_FIRST(&fdp->fd_knhash[i]);
9016 nknotes = kevent_extinfo_emit(kq, kn, kqext, buflen, nknotes);
9017 knhash_unlock(fdp);
9018 }
9019 }
9020
9021 assert(bufsize >= sizeof(struct kevent_extinfo) * min(buflen, nknotes));
9022 err = copyout(kqext, ubuf, sizeof(struct kevent_extinfo) * min(buflen, nknotes));
9023
9024out:
9025 if (kqext) {
9026 kfree(kqext, buflen * sizeof(struct kevent_extinfo));
9027 kqext = NULL;
9028 }
9029
9030 if (!err) {
9031 *retval = min(nknotes, PROC_PIDFDKQUEUE_KNOTES_MAX);
9032 }
9033 return err;
9034}
9035
9036static unsigned int
9037klist_copy_udata(struct klist *list, uint64_t *buf,
9038 unsigned int buflen, unsigned int nknotes)
9039{
9040 struct knote *kn;
9041 SLIST_FOREACH(kn, list, kn_link) {
9042 if (nknotes < buflen) {
9043 /*
9044 * kevent_register will always set kn_udata atomically
9045 * so that we don't have to take any kqlock here.
9046 */
9047 buf[nknotes] = os_atomic_load_wide(&kn->kn_udata, relaxed);
9048 }
9049 /* we return total number of knotes, which may be more than requested */
9050 nknotes++;
9051 }
9052
9053 return nknotes;
9054}
9055
9056int
9057kevent_proc_copy_uptrs(void *proc, uint64_t *buf, int bufsize)
9058{
9059 proc_t p = (proc_t)proc;
9060 struct filedesc *fdp = p->p_fd;
9061 unsigned int nuptrs = 0;
9062 unsigned long buflen = bufsize / sizeof(uint64_t);
9063 struct kqworkloop *kqwl;
9064
9065 if (buflen > 0) {
9066 assert(buf != NULL);
9067 }
9068
9069 proc_fdlock(p);
9070 for (int i = 0; i < fdp->fd_knlistsize; i++) {
9071 nuptrs = klist_copy_udata(&fdp->fd_knlist[i], buf, buflen, nuptrs);
9072 }
9073 proc_fdunlock(p);
9074
9075 knhash_lock(fdp);
9076 if (fdp->fd_knhashmask != 0) {
9077 for (size_t i = 0; i < fdp->fd_knhashmask + 1; i++) {
9078 nuptrs = klist_copy_udata(&fdp->fd_knhash[i], buf, buflen, nuptrs);
9079 }
9080 }
9081 knhash_unlock(fdp);
9082
9083 kqhash_lock(fdp);
9084 if (fdp->fd_kqhashmask != 0) {
9085 for (size_t i = 0; i < fdp->fd_kqhashmask + 1; i++) {
9086 LIST_FOREACH(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink) {
9087 if (nuptrs < buflen) {
9088 buf[nuptrs] = kqwl->kqwl_dynamicid;
9089 }
9090 nuptrs++;
9091 }
9092 }
9093 }
9094 kqhash_unlock(fdp);
9095
9096 return (int)nuptrs;
9097}
9098
9099static void
9100kevent_set_return_to_kernel_user_tsd(proc_t p, thread_t thread)
9101{
9102 uint64_t ast_addr;
9103 bool proc_is_64bit = !!(p->p_flag & P_LP64);
9104 size_t user_addr_size = proc_is_64bit ? 8 : 4;
9105 uint32_t ast_flags32 = 0;
9106 uint64_t ast_flags64 = 0;
9107 struct uthread *ut = get_bsdthread_info(thread);
9108
9109 if (ut->uu_kqr_bound != NULL) {
9110 ast_flags64 |= R2K_WORKLOOP_PENDING_EVENTS;
9111 }
9112
9113 if (ast_flags64 == 0) {
9114 return;
9115 }
9116
9117 if (!(p->p_flag & P_LP64)) {
9118 ast_flags32 = (uint32_t)ast_flags64;
9119 assert(ast_flags64 < 0x100000000ull);
9120 }
9121
9122 ast_addr = thread_rettokern_addr(thread);
9123 if (ast_addr == 0) {
9124 return;
9125 }
9126
9127 if (copyout((proc_is_64bit ? (void *)&ast_flags64 : (void *)&ast_flags32),
9128 (user_addr_t)ast_addr,
9129 user_addr_size) != 0) {
9130 printf("pid %d (tid:%llu): copyout of return_to_kernel ast flags failed with "
9131 "ast_addr = %llu\n", p->p_pid, thread_tid(current_thread()), ast_addr);
9132 }
9133}
9134
9135void
9136kevent_ast(thread_t thread, uint16_t bits)
9137{
9138 proc_t p = current_proc();
9139
9140 if (bits & AST_KEVENT_REDRIVE_THREADREQ) {
9141 workq_kern_threadreq_redrive(p, WORKQ_THREADREQ_CAN_CREATE_THREADS);
9142 }
9143 if (bits & AST_KEVENT_RETURN_TO_KERNEL) {
9144 kevent_set_return_to_kernel_user_tsd(p, thread);
9145 }
9146}
9147
9148#if DEVELOPMENT || DEBUG
9149
9150#define KEVENT_SYSCTL_BOUND_ID 1
9151
9152static int
9153kevent_sysctl SYSCTL_HANDLER_ARGS
9154{
9155#pragma unused(oidp, arg2)
9156 uintptr_t type = (uintptr_t)arg1;
9157 uint64_t bound_id = 0;
9158
9159 if (type != KEVENT_SYSCTL_BOUND_ID) {
9160 return EINVAL;
9161 }
9162
9163 if (req->newptr) {
9164 return EINVAL;
9165 }
9166
9167 struct uthread *ut = get_bsdthread_info(current_thread());
9168 if (!ut) {
9169 return EFAULT;
9170 }
9171
9172 workq_threadreq_t kqr = ut->uu_kqr_bound;
9173 if (kqr) {
9174 if (kqr->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {
9175 bound_id = kqr_kqworkloop(kqr)->kqwl_dynamicid;
9176 } else {
9177 bound_id = -1;
9178 }
9179 }
9180
9181 return sysctl_io_number(req, bound_id, sizeof(bound_id), NULL, NULL);
9182}
9183
9184SYSCTL_NODE(_kern, OID_AUTO, kevent, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
9185 "kevent information");
9186
9187SYSCTL_PROC(_kern_kevent, OID_AUTO, bound_id,
9188 CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_MASKED,
9189 (void *)KEVENT_SYSCTL_BOUND_ID,
9190 sizeof(kqueue_id_t), kevent_sysctl, "Q",
9191 "get the ID of the bound kqueue");
9192
9193#endif /* DEVELOPMENT || DEBUG */